{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,17]],"date-time":"2026-07-17T04:25:14Z","timestamp":1784262314434,"version":"3.55.0"},"reference-count":41,"publisher":"Springer Science and Business Media LLC","issue":"5","license":[{"start":{"date-parts":[[2023,1,3]],"date-time":"2023-01-03T00:00:00Z","timestamp":1672704000000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2023,1,3]],"date-time":"2023-01-03T00:00:00Z","timestamp":1672704000000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"funder":[{"DOI":"10.13039\/501100012166","name":"National Key R &D Program of China","doi-asserted-by":"crossref","award":["2021YFB2501800"],"award-info":[{"award-number":["2021YFB2501800"]}],"id":[{"id":"10.13039\/501100012166","id-type":"DOI","asserted-by":"crossref"}]},{"name":"Tianjin Technology Innovation Guide Special","award":["21YDTPJC00130"],"award-info":[{"award-number":["21YDTPJC00130"]}]}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Neural Process Lett"],"published-print":{"date-parts":[[2023,10]]},"DOI":"10.1007\/s11063-022-11132-w","type":"journal-article","created":{"date-parts":[[2023,1,3]],"date-time":"2023-01-03T03:02:54Z","timestamp":1672714974000},"page":"6165-6180","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":19,"title":["Knowledge Fusion Distillation: Improving Distillation with Multi-scale Attention Mechanisms"],"prefix":"10.1007","volume":"55","author":[{"given":"Linfeng","family":"Li","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Weixing","family":"Su","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Fang","family":"Liu","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Maowei","family":"He","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Xiaodan","family":"Liang","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2023,1,3]]},"reference":[{"key":"11132_CR1","doi-asserted-by":"publisher","unstructured":"Bao L, Ma B, Chang H, et\u00a0al. (2019) Preserving structural relationships for person re-identification. In: IEEE International conference on multimedia and expo workshops, ICME workshops 2019, Shanghai, China, July 8-12, 2019. IEEE, pp 120\u2013125, https:\/\/doi.org\/10.1109\/ICMEW.2019.00028","DOI":"10.1109\/ICMEW.2019.00028"},{"key":"11132_CR2","doi-asserted-by":"publisher","unstructured":"Bhosale YH, Patnaik KS (2022) Iot deployable lightweight deep learning application for covid-19 detection with lung diseases using raspberrypi. In: 2022 International conference on IoT and blockchain technology (ICIBT), pp 1\u20136, https:\/\/doi.org\/10.1109\/ICIBT52874.2022.9807725","DOI":"10.1109\/ICIBT52874.2022.9807725"},{"issue":"4","key":"11132_CR3","doi-asserted-by":"publisher","first-page":"6997","DOI":"10.1109\/JIOT.2019.2913176","volume":"6","author":"M Chen","year":"2019","unstructured":"Chen M, Zeng G, Lu K et al (2019) A two-layer nonlinear combination method for short-term wind speed prediction based on ELM, ENN, and LSTM. IEEE Internet Things J 6(4):6997\u20137010. https:\/\/doi.org\/10.1109\/JIOT.2019.2913176","journal-title":"IEEE Internet Things J"},{"key":"11132_CR4","doi-asserted-by":"publisher","unstructured":"Cho JH, Hariharan B (2019) On the efficacy of knowledge distillation. In: 2019 IEEE\/CVF International conference on computer vision, ICCV 2019, Seoul, Korea (South), October 27 - November 2, 2019. IEEE, pp 4793\u20134801, https:\/\/doi.org\/10.1109\/ICCV.2019.00489","DOI":"10.1109\/ICCV.2019.00489"},{"key":"11132_CR5","doi-asserted-by":"publisher","DOI":"10.1109\/TNNLS.2021.3133262","author":"G Du","year":"2021","unstructured":"Du G, Zhang J, Jiang M et al (2021) Graph-based class-imbalance learning with label enhancement. Trans Neural Netw Learn Syst Early Access. https:\/\/doi.org\/10.1109\/TNNLS.2021.3133262","journal-title":"Trans Neural Netw Learn Syst Early Access"},{"key":"11132_CR6","unstructured":"Glorot X, Bordes A, Bengio Y (2011) Deep sparse rectifier neural networks. In: Gordon GJ, Dunson DB, Dud\u00edk M (eds) Proceedings of the fourteenth international conference on artificial intelligence and statistics, AISTATS 2011, Fort Lauderdale, JMLR Proceedings, vol\u00a015. JMLR.org, pp 315\u2013323, URL http:\/\/proceedings.mlr.press\/v15\/glorot11a\/glorot11a.pdf"},{"key":"11132_CR7","doi-asserted-by":"publisher","unstructured":"He K, Zhang X, Ren S, et\u00a0al. (2016) Deep residual learning for image recognition. In: 2016 IEEE Conference on computer vision and pattern recognition, CVPR 2016, Las Vegas. IEEE Computer Society. https:\/\/doi.org\/10.1109\/CVPR.2016.90","DOI":"10.1109\/CVPR.2016.90"},{"key":"11132_CR8","unstructured":"Hinton GE, Vinyals O, Dean J (2015) Distilling the knowledge in a neural network. CoRRarXiv: 1503.02531"},{"key":"11132_CR9","doi-asserted-by":"crossref","unstructured":"Hou Q, Zhou D, Feng J (2021) Coordinate attention for efficient mobile network design. In: IEEE Conference on computer vision and pattern recognition, CVPR 2021. Computer vision foundation \/ IEEE, pp 13,713\u201313,722, URL https:\/\/openaccess.thecvf.com\/content\/CVPR2021\/html\/Hou_Coordinate_Attention_for_Efficient_Mobile_Network_Design_CVPR_2021_paper.html","DOI":"10.1109\/CVPR46437.2021.01350"},{"key":"11132_CR10","doi-asserted-by":"publisher","unstructured":"Howard A, Pang R, Adam H, et\u00a0al. (2019) Searching for mobilenetv3. In: 2019 IEEE\/CVF International conference on computer vision. ICCV 2019, Seoul, Korea. IEEE, pp 1314\u20131324, https:\/\/doi.org\/10.1109\/ICCV.2019.00140","DOI":"10.1109\/ICCV.2019.00140"},{"key":"11132_CR11","unstructured":"Howard AG, Zhu M, Chen B, et\u00a0al. (2017) Mobilenets: efficient convolutional neural networks for mobile vision applications. CoRR arXiv:1704.04861"},{"key":"11132_CR12","doi-asserted-by":"publisher","unstructured":"Hu J, Shen L, Sun G (2018) Squeeze-and-excitation networks. In: 2018 IEEE Conference on computer vision and pattern recognition. CVPR 2018, Salt Lake City. Computer vision foundation \/ IEEE computer society, pp 7132\u20137141, https:\/\/doi.org\/10.1109\/CVPR.2018.00745","DOI":"10.1109\/CVPR.2018.00745"},{"issue":"3","key":"11132_CR13","doi-asserted-by":"publisher","first-page":"1010","DOI":"10.1109\/TITS.2018.2838132","volume":"20","author":"X Hu","year":"2019","unstructured":"Hu X, Xu X, Xiao Y et al (2019) Sinet: a scale-insensitive convolutional neural network for fast vehicle detection. IEEE Trans Intell Transp Syst 20(3):1010\u20131019. https:\/\/doi.org\/10.1109\/TITS.2018.2838132","journal-title":"IEEE Trans Intell Transp Syst"},{"key":"11132_CR14","unstructured":"Ioffe S, Szegedy C (2015) Batch normalization: accelerating deep network training by reducing internal covariate shift. In: Bach FR, Blei DM (eds) Proceedings of the 32nd international conference on machine learning. ICML 2015, Lille, JMLR workshop and conference proceedings, vol\u00a037. JMLR.org, pp 448\u2013456, URL http:\/\/proceedings.mlr.press\/v37\/ioffe15.html"},{"key":"11132_CR15","doi-asserted-by":"publisher","unstructured":"Ji M, Shin S, Hwang S, et\u00a0al. (2021) Refine myself by teaching myself: feature refinement via self-knowledge distillation. In: IEEE Conference on computer vision and pattern recognition, CVPR 2021. Computer vision foundation \/ IEEE, pp 10,664\u201310,673, https:\/\/doi.org\/10.1109\/CVPR46437.2021.01052, URL https:\/\/openaccess.thecvf.com\/content\/CVPR2021\/html\/Ji_Refine_Myself_by_Teaching_Myself_Feature_Refinement_via_Self-Knowledge_Distillation_CVPR_2021_paper.html","DOI":"10.1109\/CVPR46437.2021.01052"},{"key":"11132_CR16","unstructured":"Krizhevsky A, Hinton G, et\u00a0al. (2009) Learning multiple layers of features from tiny images"},{"key":"11132_CR17","doi-asserted-by":"publisher","unstructured":"Li X, Wang W, Hu X, et\u00a0al. (2019) Selective kernel networks. In: IEEE Conference on computer vision and pattern recognition, CVPR 2019, Long Beach. Computer vision foundation\/IEEE, pp 510\u2013519, https:\/\/doi.org\/10.1109\/CVPR.2019.00060","DOI":"10.1109\/CVPR.2019.00060"},{"key":"11132_CR18","doi-asserted-by":"crossref","unstructured":"Lin T, Doll\u00e1r P, Girshick RB, et\u00a0al. (2016) Feature pyramid networks for object detection. CoRR arXiv:1612.03144","DOI":"10.1109\/CVPR.2017.106"},{"issue":"11","key":"11132_CR19","doi-asserted-by":"publisher","first-page":"7618","DOI":"10.1109\/TII.2021.3053304","volume":"17","author":"K Lu","year":"2021","unstructured":"Lu K, Zeng G, Luo X et al (2021) Evolutionary deep belief network for cyber-attack detection in industrial automation and control system. IEEE Trans Ind Inform 17(11):7618\u20137627. https:\/\/doi.org\/10.1109\/TII.2021.3053304","journal-title":"IEEE Trans Ind Inform"},{"issue":"5","key":"11132_CR20","doi-asserted-by":"publisher","first-page":"3545","DOI":"10.1007\/s11063-021-10560-4","volume":"53","author":"L Mao","year":"2021","unstructured":"Mao L, Li X, Yang D et al (2021) Convolutional feature frequency adaptive fusion object detection network. Neural Process Lett 53(5):3545\u20133560. https:\/\/doi.org\/10.1007\/s11063-021-10560-4","journal-title":"Neural Process Lett"},{"key":"11132_CR21","doi-asserted-by":"crossref","unstructured":"Mirzadeh S, Farajtabar M, Li A, et\u00a0al. (2020) Improved knowledge distillation via teacher assistant. In: The thirty-fourth AAAI conference on artificial intelligence, AAAI 2020. The thirty-second innovative applications of artificial intelligence conference, IAAI 2020. The tenth AAAI symposium on educational advances in artificial intelligence, EAAI 2020, New York. AAAI Press, pp 5191\u20135198, URL https:\/\/aaai.org\/ojs\/index.php\/AAAI\/article\/view\/5963","DOI":"10.1609\/aaai.v34i04.5963"},{"key":"11132_CR22","unstructured":"Park J, Woo S, Lee J, et\u00a0al. (2018) BAM: bottleneck attention module. In: British Machine Vision Conference 2018, BMVC 2018, Newcastle. BMVA Press, p 147, URL http:\/\/bmvc2018.org\/contents\/papers\/0092.pdf"},{"key":"11132_CR23","unstructured":"Romero A, Ballas N, Kahou SE, et\u00a0al. (2015) Fitnets: Hints for thin deep nets. In: Bengio Y, LeCun Y (eds) 3rd International conference on learning representations, ICLR 2015, San Diego. Conference track proceedings, arxiv:1412.6550"},{"key":"11132_CR24","unstructured":"Simonyan K, Zisserman A (2015) Very deep convolutional networks for large-scale image recognition. In: Bengio Y, LeCun Y (eds) 3rd International conference on learning representations, ICLR 2015, San Diego. Conference track proceedings. arxiv:1409.1556"},{"issue":"106","key":"11132_CR25","doi-asserted-by":"publisher","first-page":"837","DOI":"10.1016\/j.knosys.2021.106837","volume":"218","author":"C Tan","year":"2021","unstructured":"Tan C, Liu J, Zhang X (2021) Improving knowledge distillation via an expressive teacher. Knowl Based Syst 218(106):837. https:\/\/doi.org\/10.1016\/j.knosys.2021.106837","journal-title":"Knowl Based Syst"},{"key":"11132_CR26","doi-asserted-by":"publisher","unstructured":"Tan M, Pang R, Le QV (2020) Efficientdet: Scalable and efficient object detection. In: 2020 IEEE\/CVF Conference on computer vision and pattern recognition, CVPR 2020, Seattle. Computer vision foundation \/ IEEE, pp 10,778\u201310,787, https:\/\/doi.org\/10.1109\/CVPR42600.2020.01079, URL https:\/\/openaccess.thecvf.com\/content_CVPR_2020\/html\/Tan_EfficientDet_Scalable_and_Efficient_Object_Detection_CVPR_2020_paper.html","DOI":"10.1109\/CVPR42600.2020.01079"},{"key":"11132_CR27","doi-asserted-by":"publisher","unstructured":"Wang F, Jiang M, Qian C, et\u00a0al. (2017) Residual attention network for image classification. In: 2017 IEEE Conference on computer vision and pattern recognition, CVPR 2017, Honolulu. IEEE Computer society, pp 6450\u20136458, https:\/\/doi.org\/10.1109\/CVPR.2017.683","DOI":"10.1109\/CVPR.2017.683"},{"key":"11132_CR28","doi-asserted-by":"publisher","unstructured":"Woo S, Park J, Lee J, et\u00a0al. (2018) CBAM: convolutional block attention module. In: Ferrari V, Hebert M, Sminchisescu C, et\u00a0al (eds) Computer vision - ECCV 2018 - 15th European conference, Munich. Proceedings, part VII, Lecture notes in computer science, vol 11211. Springer, pp 3\u201319, https:\/\/doi.org\/10.1007\/978-3-030-01234-2_1","DOI":"10.1007\/978-3-030-01234-2_1"},{"key":"11132_CR29","doi-asserted-by":"publisher","unstructured":"Xie S, Girshick RB, Doll\u00e1r P, et\u00a0al. (2017) Aggregated residual transformations for deep neural networks. In: 2017 IEEE Conference on computer vision and pattern recognition, CVPR 2017, Honolulu. IEEE Computer Society, pp 5987\u20135995, https:\/\/doi.org\/10.1109\/CVPR.2017.634","DOI":"10.1109\/CVPR.2017.634"},{"issue":"3","key":"11132_CR30","doi-asserted-by":"publisher","first-page":"1921","DOI":"10.1007\/s11063-021-10493-y","volume":"53","author":"Z Yan","year":"2021","unstructured":"Yan Z, Zheng H, Li Y et al (2021) Detection-oriented backbone trained from near scratch and local feature refinement for small object detection. Neural Process Lett 53(3):1921\u20131943. https:\/\/doi.org\/10.1007\/s11063-021-10493-y","journal-title":"Neural Process Lett"},{"key":"11132_CR31","unstructured":"Yang J, Mart\u00ednez B, Bulat A, et\u00a0al. (2020) Knowledge distillation via adaptive instance normalization. CoRR arXiv:2003.04289"},{"key":"11132_CR32","doi-asserted-by":"publisher","unstructured":"Yim J, Joo D, Bae J, et\u00a0al. (2017) A gift from knowledge distillation: fast optimization, network minimization and transfer learning. In: 2017 IEEE Conference on computer vision and pattern recognition, CVPR 2017, Honolulu. IEEE Computer Society, pp 7130\u20137138, https:\/\/doi.org\/10.1109\/CVPR.2017.754","DOI":"10.1109\/CVPR.2017.754"},{"key":"11132_CR33","doi-asserted-by":"crossref","unstructured":"Zagoruyko S, Komodakis N (2016) Wide residual networks. In: Wilson RC, Hancock ER, Smith WAP (eds) Proceedings of the British machine vision conference 2016, BMVC 2016, York. BMVA Press. URL http:\/\/www.bmva.org\/bmvc\/2016\/papers\/paper087\/index.html","DOI":"10.5244\/C.30.87"},{"key":"11132_CR34","unstructured":"Zagoruyko S, Komodakis N (2017) Paying more attention to attention: improving the performance of convolutional neural networks via attention transfer. In: 5th International conference on learning representations, ICLR 2017, Toulon. Conference track proceedings. OpenReview.net, URL https:\/\/openreview.net\/forum?id=Sks9_ajex"},{"key":"11132_CR35","doi-asserted-by":"publisher","unstructured":"Zhang L, Song J, Gao A, et\u00a0al. (2019a) Be your own teacher: improve the performance of convolutional neural networks via self distillation. In: 2019 IEEE\/CVF International conference on computer vision, ICCV 2019, Seoul. IEEE, pp 3712\u20133721, https:\/\/doi.org\/10.1109\/ICCV.2019.00381","DOI":"10.1109\/ICCV.2019.00381"},{"key":"11132_CR36","unstructured":"Zhang L, Tan Z, Song J, et\u00a0al. (2019b) SCAN: a scalable neural networks framework towards compact and efficient models. In: Wallach HM, Larochelle H, Beygelzimer A, et\u00a0al (eds) Advances in neural information processing systems 32: annual conference on neural information processing systems 2019. NeurIPS 2019, Vancouver, pp 4029\u20134038. URL https:\/\/proceedings.neurips.cc\/paper\/2019\/hash\/934b535800b1cba8f96a5d72f72f1611-Abstract.html"},{"key":"11132_CR37","doi-asserted-by":"publisher","DOI":"10.1109\/TPAMI.2021.3067100","author":"L Zhang","year":"2021","unstructured":"Zhang L, Bao C, Ma K (2021) Self-distillation: towards efficient and compact neural networks. Trans Pattern Anal Mach Intell. https:\/\/doi.org\/10.1109\/TPAMI.2021.3067100","journal-title":"Trans Pattern Anal Mach Intell"},{"key":"11132_CR38","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1109\/LGRS.2021.3122875","volume":"19","author":"R Zhang","year":"2022","unstructured":"Zhang R, Jiang X, An J et al (2022) Data-free low-bit quantization for remote sensing object detection. IEEE Geosci Remote Sens Lett 19:1\u20135. https:\/\/doi.org\/10.1109\/LGRS.2021.3122875","journal-title":"IEEE Geosci Remote Sens Lett"},{"key":"11132_CR39","doi-asserted-by":"crossref","unstructured":"Zhao B, Cui Q, Song R, et\u00a0al. (2022) Decoupled knowledge distillation. CoRR arXiv:2203.08679","DOI":"10.1109\/CVPR52688.2022.01165"},{"issue":"107","key":"11132_CR40","doi-asserted-by":"publisher","first-page":"519","DOI":"10.1016\/j.knosys.2021.107519","volume":"233","author":"H Zhao","year":"2021","unstructured":"Zhao H, Sun X, Dong J et al (2021) Knowledge distillation via instance-level sequence learning. Knowl Based Syst 233(107):519. https:\/\/doi.org\/10.1016\/j.knosys.2021.107519","journal-title":"Knowl Based Syst"},{"key":"11132_CR41","doi-asserted-by":"publisher","DOI":"10.1109\/TNNLS.2022.3165123","author":"YJ Zheng","year":"2022","unstructured":"Zheng YJ, Chen SB, Ding CH et al (2022) Model compression based on differentiable network channel pruning. Trans Neural Netw Learn Syst. https:\/\/doi.org\/10.1109\/TNNLS.2022.3165123","journal-title":"Trans Neural Netw Learn Syst"}],"container-title":["Neural Processing Letters"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s11063-022-11132-w.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s11063-022-11132-w\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s11063-022-11132-w.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,4,9]],"date-time":"2025-04-09T14:30:11Z","timestamp":1744209011000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s11063-022-11132-w"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023,1,3]]},"references-count":41,"journal-issue":{"issue":"5","published-print":{"date-parts":[[2023,10]]}},"alternative-id":["11132"],"URL":"https:\/\/doi.org\/10.1007\/s11063-022-11132-w","relation":{},"ISSN":["1370-4621","1573-773X"],"issn-type":[{"value":"1370-4621","type":"print"},{"value":"1573-773X","type":"electronic"}],"subject":[],"published":{"date-parts":[[2023,1,3]]},"assertion":[{"value":"12 December 2022","order":1,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"3 January 2023","order":2,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Declarations"}},{"value":"No potential conflict of interest was reported by the authors.","order":2,"name":"Ethics","group":{"name":"EthicsHeading","label":"Conflict of interest"}}]}}