{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,13]],"date-time":"2026-07-13T15:07:52Z","timestamp":1783955272787,"version":"3.55.0"},"reference-count":53,"publisher":"Springer Science and Business Media LLC","issue":"6","license":[{"start":{"date-parts":[[2024,8,31]],"date-time":"2024-08-31T00:00:00Z","timestamp":1725062400000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2024,8,31]],"date-time":"2024-08-31T00:00:00Z","timestamp":1725062400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"funder":[{"name":"Beijing Natural Science Foundation","award":["4244098"],"award-info":[{"award-number":["4244098"]}]}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Cogn Comput"],"published-print":{"date-parts":[[2024,11]]},"DOI":"10.1007\/s12559-024-10350-9","type":"journal-article","created":{"date-parts":[[2024,8,31]],"date-time":"2024-08-31T10:02:22Z","timestamp":1725098542000},"page":"3457-3467","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":6,"title":["PDD: Pruning Neural Networks During Knowledge Distillation"],"prefix":"10.1007","volume":"16","author":[{"given":"Xi","family":"Dan","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Wenjie","family":"Yang","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Fuyan","family":"Zhang","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Yihang","family":"Zhou","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Zhuojun","family":"Yu","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Zhen","family":"Qiu","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Boyuan","family":"Zhao","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Zeyu","family":"Dong","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Libo","family":"Huang","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Chuanguang","family":"Yang","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2024,8,31]]},"reference":[{"issue":"3","key":"10350_CR1","first-page":"1","volume":"2","author":"R Adriana","year":"2015","unstructured":"Adriana R, Nicolas B, Ebrahimi KS, Antoine C, Carlo G, Yoshua B. Fitnets: hints for thin deep nets. Proc ICLR. 2015;2(3):1.","journal-title":"Proc ICLR."},{"key":"10350_CR2","doi-asserted-by":"crossref","unstructured":"Belagiannis V, Farshad A, Galasso F. Adversarial network compression. In: Proceedings of the European Conference on Computer Vision (ECCV) Workshops. 2018;0\u20130","DOI":"10.1007\/978-3-030-11018-5_37"},{"key":"10350_CR3","doi-asserted-by":"crossref","unstructured":"Cai L, An Z, Yang C, Xu Y. Softer pruning, incremental regularization. In: 2020 25th international conference on pattern recognition (ICPR). 2021;224\u2013230. IEEE","DOI":"10.1109\/ICPR48806.2021.9412993"},{"key":"10350_CR4","doi-asserted-by":"crossref","unstructured":"Cai L, An Z, Yang C, Yan Y, Xu Y. Prior gradient mask guided pruning-aware fine-tuning. In: Proceedings of the AAAI Conference on Artificial Intelligence. 2022;36:140\u2013148","DOI":"10.1609\/aaai.v36i1.19888"},{"key":"10350_CR5","doi-asserted-by":"crossref","unstructured":"Covington P, Adams J, Sargin E. Deep neural networks for youtube recommendations. In: Proceedings of the 10th ACM conference on recommender systems. 2016;191\u2013198","DOI":"10.1145\/2959100.2959190"},{"key":"10350_CR6","doi-asserted-by":"crossref","unstructured":"Deng J, Dong W, Socher R, Li LJ, Li K, Fei-Fei L. Imagenet: a large-scale hierarchical image database. In: 2009 IEEE conference on computer vision and pattern recognition. 2009;248\u2013255. Ieee","DOI":"10.1109\/CVPR.2009.5206848"},{"key":"10350_CR7","unstructured":"Devlin J, Chang MW, Lee K, Toutanova K. Bert: pre-training of deep bidirectional transformers for language understanding. 2018 arXiv:1810.04805"},{"key":"10350_CR8","doi-asserted-by":"crossref","unstructured":"Dong X, Huang J, Yang Y, Yan S. More is less: a more complicated network with less inference complexity. In: Proceedings of the IEEE conference on computer vision and pattern recognition. 2017;5840\u20135848","DOI":"10.1109\/CVPR.2017.205"},{"key":"10350_CR9","doi-asserted-by":"crossref","unstructured":"Fang G, Ma X, Song M, Mi MB, Wang X. Depgraph: towards any structural pruning. 2023 arXiv:2301.12900","DOI":"10.1109\/CVPR52729.2023.01544"},{"key":"10350_CR10","unstructured":"Han S, Mao H, Dally WJ. Deep compression: compressing deep neural networks with pruning, trained quantization and huffman coding. 2015 arXiv:1510.00149"},{"key":"10350_CR11","doi-asserted-by":"crossref","unstructured":"Hassibi B, Stork DG, Wolff GJ. Optimal brain surgeon and general network pruning. In: IEEE international conference on neural networks. 1993;293\u2013299. IEEE","DOI":"10.1109\/ICNN.1993.298572"},{"key":"10350_CR12","doi-asserted-by":"crossref","unstructured":"He Y, Kang G, Dong X, Fu Y, Yang Y. Soft filter pruning for accelerating deep convolutional neural networks. In: Proceedings of the 27th International Joint Conference on Artificial Intelligence. 2018;2234\u20132240","DOI":"10.24963\/ijcai.2018\/309"},{"key":"10350_CR13","doi-asserted-by":"crossref","unstructured":"He Y, Liu P, Wang Z, Hu Z, Yang Y. Filter pruning via geometric median for deep convolutional neural networks acceleration. In: Proceedings of the IEEE\/CVF conference on computer vision and pattern recognition. 2019;4340\u20134349","DOI":"10.1109\/CVPR.2019.00447"},{"key":"10350_CR14","doi-asserted-by":"crossref","unstructured":"He Y, Zhang X, Sun J. Channel pruning for accelerating very deep neural networks. In: Proceedings of the IEEE international conference on computer vision. 2017;1389\u20131397","DOI":"10.1109\/ICCV.2017.155"},{"key":"10350_CR15","unstructured":"Hinton G, Vinyals O, Dean J. Distilling the knowledge in a neural network. 2015arXiv:1503.02531"},{"key":"10350_CR16","doi-asserted-by":"crossref","unstructured":"Hu H, Bai S, Li A, Cui J, Wang L. Dense relation distillation with context-aware aggregation for few-shot object detection. In: Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition (CVPR). June 2021;10185\u201310194","DOI":"10.1109\/CVPR46437.2021.01005"},{"key":"10350_CR17","doi-asserted-by":"crossref","unstructured":"Huang Z, Wang N. Data-driven sparse structure selection for deep neural networks. In: ECCV. 2018;304\u2013320","DOI":"10.1007\/978-3-030-01270-0_19"},{"key":"10350_CR18","doi-asserted-by":"crossref","unstructured":"Jung S, Lee D, Park T, Moon T. Fair feature distillation for visual recognition. In: Proceedings of the IEEE\/CVF conference on computer vision and pattern recognition. 2021;12115\u201312124","DOI":"10.1109\/CVPR46437.2021.01194"},{"key":"10350_CR19","first-page":"31","volume-title":"Kwak N","author":"J Kim","year":"2018","unstructured":"Kim J, Park S. Kwak N. Paraphrasing complex network: Network compression via factor transfer. NeurIPS; 2018. p. 31."},{"issue":"8","key":"10350_CR20","doi-asserted-by":"publisher","first-page":"30","DOI":"10.1109\/MC.2009.263","volume":"42","author":"Y Koren","year":"2009","unstructured":"Koren Y, Bell R, Volinsky C. Matrix factorization techniques for recommender systems. Computer. 2009;42(8):30\u20137.","journal-title":"Computer."},{"key":"10350_CR21","doi-asserted-by":"crossref","unstructured":"Kovaleva O, Romanov A, Rogers A, Rumshisky A. Revealing the dark secrets of Bert. 2019 arXiv:1908.08593","DOI":"10.18653\/v1\/D19-1445"},{"key":"10350_CR22","volume-title":"Learning multiple layers of features from tiny images","author":"A Krizhevsky","year":"2009","unstructured":"Krizhevsky A. Learning multiple layers of features from tiny images. Citeseer: Tech. rep; 2009."},{"issue":"6","key":"10350_CR23","doi-asserted-by":"publisher","first-page":"84","DOI":"10.1145\/3065386","volume":"60","author":"A Krizhevsky","year":"2017","unstructured":"Krizhevsky A, Sutskever I, Hinton GE. Imagenet classification with deep convolutional neural networks. Commun ACM. 2017;60(6):84\u201390.","journal-title":"Commun ACM."},{"key":"10350_CR24","doi-asserted-by":"crossref","unstructured":"Lai KH, Zha D, Li Y, Hu X. Dual policy distillation. 2020 arXiv:2006.04061","DOI":"10.24963\/ijcai.2020\/435"},{"key":"10350_CR25","unstructured":"LeCun Y, Denker J, Solla S. Optimal brain damage. NeurIPS 1989:2"},{"issue":"6755","key":"10350_CR26","doi-asserted-by":"publisher","first-page":"788","DOI":"10.1038\/44565","volume":"401","author":"DD Lee","year":"1999","unstructured":"Lee DD, Seung HS. Learning the parts of objects by non-negative matrix factorization. Nature. 1999;401(6755):788\u201391.","journal-title":"Nature."},{"key":"10350_CR27","unstructured":"Li H, Kadav A, Durdanovic I, Samet H, Graf HP. Pruning filters for efficient convnets. 2016arXiv:1608.08710"},{"key":"10350_CR28","doi-asserted-by":"crossref","unstructured":"Li L, Gan Z, Cheng Y, Liu J. Relation-aware graph attention network for visual question answering. In: Proceedings of the IEEE\/CVF International Conference on Computer Vision (ICCV) October 2019","DOI":"10.1109\/ICCV.2019.01041"},{"key":"10350_CR29","doi-asserted-by":"crossref","unstructured":"Lin M, Ji R, Wang Y, Zhang Y, Zhang B, Tian Y, Shao L. Hrank: filter pruning using high-rank feature map. In: CVPR. 2020;1529\u20131538","DOI":"10.1109\/CVPR42600.2020.00160"},{"key":"10350_CR30","doi-asserted-by":"crossref","unstructured":"Lin S, Ji R, Yan C, Zhang B, Cao L, Ye Q, Huang F, Doermann D. Towards optimal structured CNN pruning via generative adversarial learning. In: Proceedings of the IEEE\/CVF conference on computer vision and pattern recognition. 2019;2790\u20132799","DOI":"10.1109\/CVPR.2019.00290"},{"key":"10350_CR31","doi-asserted-by":"crossref","unstructured":"Liu J, Tang J, Wu G. Residual feature distillation network for lightweight image super-resolution. In: Computer Vision\u2013ECCV 2020 Workshops: Glasgow, UK, August 23\u201328, 2020, Proceedings, Part III 16. 2020;41\u201355. Springer","DOI":"10.1007\/978-3-030-67070-2_2"},{"issue":"8","key":"10350_CR32","first-page":"4035","volume":"44","author":"J Liu","year":"2021","unstructured":"Liu J, Zhuang B, Zhuang Z, Guo Y, Huang J, Zhu J, Tan M. Discrimination-aware network pruning for deep model compression. IEEE Trans Pattern Anal Mach Intell. 2021;44(8):4035\u201351.","journal-title":"IEEE Trans Pattern Anal Mach Intell."},{"key":"10350_CR33","doi-asserted-by":"crossref","unstructured":"Lu Y, Yang W, Zhang Y, Chen Z, Chen J, Xuan Q, Wang Z, Yang, X. Understanding the dynamics of DNNs using graph modularity. In: Computer Vision\u2013ECCV 2022: 17th European Conference, Tel Aviv, Israel, October 23\u201327, 2022, Proceedings, Part XII. 2022;225\u2013242. Springer","DOI":"10.1007\/978-3-031-19775-8_14"},{"key":"10350_CR34","doi-asserted-by":"crossref","unstructured":"Niu W, Ma X, Lin S, Wang S, Qian X, Lin X, Wang Y, Ren B. PatDNN: achieving real-time DNN execution on mobile devices with pattern-based weight pruning. In: Proceedings of the Twenty-Fifth International Conference on Architectural Support for Programming Languages and Operating Systems. 2020;907\u2013922","DOI":"10.1145\/3373376.3378534"},{"key":"10350_CR35","doi-asserted-by":"crossref","unstructured":"Papernot N, McDaniel P, Wu X, Jha S, Swami A. Distillation as a defense to adversarial perturbations against deep neural networks. In: 2016 IEEE symposium on security and privacy (SP). 2016;582\u2013597. IEEE","DOI":"10.1109\/SP.2016.41"},{"key":"10350_CR36","doi-asserted-by":"crossref","unstructured":"Park E, Ahn J, Yoo S. Weighted-entropy-based quantization for deep neural networks. In: CVPR. 2017;5456\u20135464","DOI":"10.1109\/CVPR.2017.761"},{"key":"10350_CR37","unstructured":"Sanh V, Debut L, Chaumond J, Wolf T. Distilbert, a distilled version of Bert: smaller, faster, cheaper and lighter. 2019 arXiv:1910.01108"},{"key":"10350_CR38","doi-asserted-by":"crossref","unstructured":"Shan Y, Hoens TR, Jiao J, Wang H, Yu D, Mao J. Deep crossing: web-scale modeling without manually crafted combinatorial features. In: Proceedings of the 22nd ACM SIGKDD international conference on knowledge discovery and data mining. 2016;255\u2013262","DOI":"10.1145\/2939672.2939704"},{"key":"10350_CR39","doi-asserted-by":"crossref","unstructured":"Silver D, Huang A, Maddison CJ, Guez A, Sifre L, Van Den\u00a0Driessche G, Schrittwieser J, Antonoglou I, Panneershelvam V, Lanctot M, et\u00a0al. Mastering the game of go with deep neural networks and tree search. nature 2016;529(7587):484\u2013489","DOI":"10.1038\/nature16961"},{"key":"10350_CR40","doi-asserted-by":"crossref","unstructured":"Tang H, Lu Y, Xuan Q. Sr-init: an interpretable layer pruning method. In: ICASSP 2023-2023 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP). 2023;1\u20135. IEEE","DOI":"10.1109\/ICASSP49357.2023.10095306"},{"key":"10350_CR41","doi-asserted-by":"crossref","unstructured":"Wu S, Rupprecht C, Vedaldi A. Unsupervised learning of probably symmetric deformable 3d objects from images in the wild. In: Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition (CVPR) June 2020","DOI":"10.1109\/CVPR42600.2020.00008"},{"key":"10350_CR42","doi-asserted-by":"crossref","unstructured":"Yang C, An Z, Cai L, Xu Y. Hierarchical self-supervised augmented knowledge distillation. In: Proceedings of the Thirtieth International Joint Conference on Artificial Intelligence (IJCAI). 2021;1217\u20131223","DOI":"10.24963\/ijcai.2021\/168"},{"key":"10350_CR43","unstructured":"Yang C, An Z, Cai L, Xu Y. Knowledge distillation using hierarchical self-supervision augmented distribution. IEEE Transactions on Neural Networks and Learning Systems 2022;1\u201315"},{"key":"10350_CR44","doi-asserted-by":"crossref","unstructured":"Yang C, An Z, Cai L, Xu Y. Mutual contrastive learning for visual representation learning. In: Proceedings of the AAAI Conference on Artificial Intelligence. 2022;36:3045\u20133053","DOI":"10.1609\/aaai.v36i3.20211"},{"key":"10350_CR45","doi-asserted-by":"crossref","unstructured":"Yang C, An Z, Li C, Diao B, Xu Y. Multi-objective pruning for CNNs using genetic algorithm. In: International Conference on Artificial Neural Networks. 2019;299\u2013305. Springer","DOI":"10.1007\/978-3-030-30484-3_25"},{"key":"10350_CR46","doi-asserted-by":"crossref","unstructured":"Yang C, An Z, Xu Y. Multi-view contrastive learning for online knowledge distillation. In: ICASSP 2021-2021 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP). 2021;3750\u20133754. IEEE","DOI":"10.1109\/ICASSP39728.2021.9414664"},{"key":"10350_CR47","doi-asserted-by":"crossref","unstructured":"Yang C, An Z, Zhou H, Cai L, Zhi X, Wu J, Xu Y, Zhang Q. Mixskd: self-knowledge distillation from mixup for image recognition. In: European Conference on Computer Vision. 2022;534\u2013551. Springer","DOI":"10.1007\/978-3-031-20053-3_31"},{"issue":"8","key":"10350_CR48","doi-asserted-by":"publisher","first-page":"10212","DOI":"10.1109\/TPAMI.2023.3257878","volume":"45","author":"C Yang","year":"2023","unstructured":"Yang C, An Z, Zhou H, Zhuang F, Xu Y, Zhang Q. Online knowledge distillation via mutual contrastive learning for visual recognition. IEEE Trans Pattern Anal Mach Intell. 2023;45(8):10212\u201327.","journal-title":"IEEE Trans Pattern Anal Mach Intell."},{"key":"10350_CR49","doi-asserted-by":"crossref","unstructured":"Yang C, An Z, Zhu H, Hu X, Zhang K, Xu K, Li C, Xu Y. Gated convolutional networks with hybrid connectivity for image classification. In: Proceedings of the AAAI Conference on Artificial Intelligence. 2020;34 :12581\u201312588","DOI":"10.1609\/aaai.v34i07.6948"},{"key":"10350_CR50","doi-asserted-by":"crossref","unstructured":"Yang C, Yu X, An Z, Xu Y. Categories of response-based, feature-based, and relation-based knowledge distillation. In: Advancements in Knowledge Distillation: Towards New Horizons of Intelligent Systems, 2023;1\u201332. Springer","DOI":"10.1007\/978-3-031-32095-8_1"},{"key":"10350_CR51","doi-asserted-by":"crossref","unstructured":"Yang C, Zhou H, An Z, Jiang X, Xu Y, Zhang Q. Cross-image relational knowledge distillation for semantic segmentation. In: Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition. 2022;12319\u201312328","DOI":"10.1109\/CVPR52688.2022.01200"},{"key":"10350_CR52","doi-asserted-by":"crossref","unstructured":"Yu R, Li A, Chen CF, Lai JH, Morariu VI, Han X, Gao M, Lin CY, Davis LS. NISP: pruning networks using neuron importance score propagation. In: Proceedings of the IEEE conference on computer vision and pattern recognition. 2018;9194\u20139203","DOI":"10.1109\/CVPR.2018.00958"},{"issue":"5","key":"10350_CR53","doi-asserted-by":"publisher","first-page":"3165","DOI":"10.1109\/TCYB.2021.3124284","volume":"53","author":"X Zhang","year":"2021","unstructured":"Zhang X, Xie W, Li Y, Lei J, Du Q. Filter pruning via learned representation median in the frequency domain. IEEE Transactions on Cybernetics. 2021;53(5):3165\u201375.","journal-title":"IEEE Transactions on Cybernetics."}],"container-title":["Cognitive Computation"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s12559-024-10350-9.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s12559-024-10350-9\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s12559-024-10350-9.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,11,7]],"date-time":"2024-11-07T09:47:30Z","timestamp":1730972850000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s12559-024-10350-9"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,8,31]]},"references-count":53,"journal-issue":{"issue":"6","published-print":{"date-parts":[[2024,11]]}},"alternative-id":["10350"],"URL":"https:\/\/doi.org\/10.1007\/s12559-024-10350-9","relation":{},"ISSN":["1866-9956","1866-9964"],"issn-type":[{"value":"1866-9956","type":"print"},{"value":"1866-9964","type":"electronic"}],"subject":[],"published":{"date-parts":[[2024,8,31]]},"assertion":[{"value":"12 March 2024","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"12 August 2024","order":2,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"31 August 2024","order":3,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Declarations"}},{"value":"The authors declare no competing interests.","order":2,"name":"Ethics","group":{"name":"EthicsHeading","label":"Competing interests"}},{"value":"The authors declare that there is no conflict of interest regarding the publication of this paper. No financial or personal relationship with other people or organizations has influenced the work reported in this manuscript, titled \u201cPDD: Pruning Neural Networks During Knowledge Distillation,\u201d Submission ID dccb382d-1a4b-4ee3-a28e-dd6137112c47. This research is conducted purely from an academic perspective, and any affiliations or financial supports are disclosed in the manuscript. Our study has been carried out with a commitment to transparency and honesty in all our research findings and discussions related to this work.","order":3,"name":"Ethics","group":{"name":"EthicsHeading","label":"Conflict of Interest"}}]}}