{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,3,19]],"date-time":"2026-03-19T14:30:23Z","timestamp":1773930623494,"version":"3.50.1"},"reference-count":85,"publisher":"Institute of Electrical and Electronics Engineers (IEEE)","issue":"4","license":[{"start":{"date-parts":[[2022,4,1]],"date-time":"2022-04-01T00:00:00Z","timestamp":1648771200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/ieeexplore.ieee.org\/Xplorehelp\/downloads\/license-information\/IEEE.html"},{"start":{"date-parts":[[2022,4,1]],"date-time":"2022-04-01T00:00:00Z","timestamp":1648771200000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2022,4,1]],"date-time":"2022-04-01T00:00:00Z","timestamp":1648771200000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"}],"funder":[{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["U1706218"],"award-info":[{"award-number":["U1706218"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["61971388"],"award-info":[{"award-number":["61971388"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100007129","name":"Major Program of Natural Science Foundation of Shandong Province","doi-asserted-by":"publisher","award":["ZR2018ZB0852"],"award-info":[{"award-number":["ZR2018ZB0852"]}],"id":[{"id":"10.13039\/501100007129","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["IEEE Trans. Cybern."],"published-print":{"date-parts":[[2022,4]]},"DOI":"10.1109\/tcyb.2020.3007506","type":"journal-article","created":{"date-parts":[[2020,7,28]],"date-time":"2020-07-28T22:17:49Z","timestamp":1595974669000},"page":"2070-2081","source":"Crossref","is-referenced-by-count":63,"title":["Highlight Every Step: Knowledge Distillation via Collaborative Teaching"],"prefix":"10.1109","volume":"52","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-2842-3526","authenticated-orcid":false,"given":"Haoran","family":"Zhao","sequence":"first","affiliation":[{"name":"Department of Computer Science and Technology, Ocean University of China, Qingdao, China"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-1870-9037","authenticated-orcid":false,"given":"Xin","family":"Sun","sequence":"additional","affiliation":[{"name":"Department of Computer Science and Technology, Ocean University of China, Qingdao, China"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-7012-2087","authenticated-orcid":false,"given":"Junyu","family":"Dong","sequence":"additional","affiliation":[{"name":"Department of Computer Science and Technology, Ocean University of China, Qingdao, China"}]},{"given":"Changrui","family":"Chen","sequence":"additional","affiliation":[{"name":"Department of Computer Science and Technology, Ocean University of China, Qingdao, China"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-1717-1204","authenticated-orcid":false,"given":"Zihe","family":"Dong","sequence":"additional","affiliation":[{"name":"Department of Computer Science and Technology, Ocean University of China, Qingdao, China"}]}],"member":"263","reference":[{"key":"ref1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2016.90"},{"key":"ref2","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2015.7298594"},{"key":"ref3","doi-asserted-by":"publisher","DOI":"10.1109\/TCSVT.2019.2919139"},{"key":"ref4","doi-asserted-by":"publisher","DOI":"10.1109\/TCYB.2013.2279002"},{"key":"ref5","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2015.279"},{"key":"ref6","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2016.11"},{"issue":"4","key":"ref7","first-page":"3","article-title":"Deep compression: Compressing deep neural networks with pruning, trained quantization and huffman coding","volume":"56","author":"Song","year":"2015","journal-title":"Fiber"},{"key":"ref8","first-page":"164","article-title":"Second order derivatives for network pruning: Optimal brain surgeon","volume-title":"Advances in Neural Information Processing Systems","volume":"5","author":"Hassibi","year":"1993"},{"key":"ref9","doi-asserted-by":"publisher","DOI":"10.5244\/c.28.88"},{"key":"ref10","first-page":"598","article-title":"Optimal brain damage","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Cun"},{"key":"ref11","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2019.00290"},{"key":"ref12","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v31i1.11231"},{"key":"ref13","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2019.00140"},{"key":"ref14","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2018.00716"},{"key":"ref15","volume-title":"Squeezenet: AlexNet-level accuracy with 50$\\times$\n fewer parameters and < 0.5 MB model size","author":"Iandola","year":"2016"},{"key":"ref16","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2018.00291"},{"key":"ref17","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v33i01.33011190"},{"key":"ref18","first-page":"1135","article-title":"Learning both weights and connections for efficient neural networks","volume-title":"Advances in Neural Information Processing Systems","author":"Han","year":"2015"},{"issue":"7","key":"ref19","first-page":"38","article-title":"Distilling the knowledge in a neural network","volume":"14","author":"Hinton","year":"2015","journal-title":"Comput. Sci."},{"key":"ref20","first-page":"1","article-title":"FitNets: Hints for thin deep nets","volume-title":"Proc. 3rd Int. Conf. Learn. Represent. (ICLR)","author":"Romero"},{"key":"ref21","article-title":"Paying more attention to attention: Improving the performance of convolutional neural networks via attention transfer","volume-title":"Proc. 5th Int. Conf. Learn. Represent. (ICLR)","author":"Zagoruyko"},{"key":"ref22","article-title":"Apprentice: Using knowledge distillation techniques to improve low-precision network accuracy","volume-title":"Proc. 6th Int. Conf. Learn. Represent. (ICLR)","author":"Mishra"},{"key":"ref23","doi-asserted-by":"publisher","DOI":"10.1109\/TIP.2019.2910667"},{"key":"ref24","doi-asserted-by":"publisher","DOI":"10.1109\/TNNLS.2015.2477537"},{"key":"ref25","doi-asserted-by":"publisher","DOI":"10.1093\/icesjms\/fsz171"},{"key":"ref26","doi-asserted-by":"publisher","DOI":"10.1109\/TSMCB.2012.2214210"},{"key":"ref27","doi-asserted-by":"publisher","DOI":"10.1109\/TCYB.2017.2675910"},{"key":"ref28","doi-asserted-by":"publisher","DOI":"10.1109\/TSMC.2016.2629509"},{"key":"ref29","doi-asserted-by":"publisher","DOI":"10.1016\/j.knosys.2020.105824"},{"key":"ref30","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-15561-1_11"},{"key":"ref31","doi-asserted-by":"publisher","DOI":"10.1007\/s11263-015-0816-y"},{"key":"ref32","doi-asserted-by":"publisher","DOI":"10.1109\/TCYB.2015.2484324"},{"key":"ref33","doi-asserted-by":"publisher","DOI":"10.1109\/TCYB.2014.2330853"},{"key":"ref34","article-title":"Very deep convolutional networks for large-scale image recognition","volume-title":"Proc. 3rd Int. Conf. Learn. Represent. (ICLR)","author":"Simonyan"},{"key":"ref35","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2017.243"},{"key":"ref36","doi-asserted-by":"publisher","DOI":"10.1109\/TIE.2020.2977553"},{"key":"ref37","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v34i07.6622"},{"key":"ref38","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2017.154"},{"key":"ref39","article-title":"Neural architecture search with reinforcement learning","volume-title":"Proc. 5th Int. Conf. Learn. Represent. (ICLR)","author":"Zoph"},{"key":"ref40","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-01246-5_2"},{"key":"ref41","doi-asserted-by":"publisher","DOI":"10.5555\/2999134.2999257"},{"key":"ref42","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v32i1.11667"},{"key":"ref43","doi-asserted-by":"publisher","DOI":"10.1109\/TSMC.2016.2531700"},{"key":"ref44","doi-asserted-by":"publisher","DOI":"10.1109\/TCYB.2015.2497711"},{"key":"ref45","first-page":"436","article-title":"Do deep convolutional nets really need to be deep (or even convolutional)?","volume":"521","author":"Urban","year":"2016","journal-title":"Nature"},{"key":"ref46","doi-asserted-by":"publisher","DOI":"10.1109\/TMM.2017.2788205"},{"key":"ref47","doi-asserted-by":"publisher","DOI":"10.24963\/ijcai.2017\/332"},{"key":"ref48","doi-asserted-by":"publisher","DOI":"10.1145\/1553374.1553516"},{"key":"ref49","first-page":"2148","article-title":"Predicting parameters in deep learning","volume-title":"Proc. 26th Int. Conf. Neural Inf. Process. Syst.","author":"Denil"},{"key":"ref50","first-page":"2285","article-title":"Compressing neural networks with the hashing trick","volume-title":"Proc. 32nd Int. Conf. Mach. Learn. (ICML)","author":"Chen"},{"key":"ref51","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2015.7298809"},{"key":"ref52","article-title":"Pruning filters for efficient ConvNets","volume-title":"Proc. 5th Int. Conf. Learn. Represent. (ICLR)","author":"Hao"},{"key":"ref53","doi-asserted-by":"publisher","DOI":"10.1109\/TNNLS.2019.2906563"},{"key":"ref54","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2017.541"},{"issue":"2","key":"ref55","first-page":"576","article-title":"Compression of deep convolutional neural networks for fast and low power mobile applications","volume":"71","author":"Kim","year":"2015","journal-title":"Comput. Sci."},{"key":"ref56","first-page":"295","article-title":"Deep non-blind deconvolution via generalized low-rank approximation","volume-title":"Proc. Adv. Neural Inf. Process. Syst. Annu. Conf. Neural Inf. Process. Syst.","author":"Ren"},{"key":"ref57","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2017.298"},{"key":"ref58","doi-asserted-by":"publisher","DOI":"10.1109\/TCYB.2017.2771463"},{"key":"ref59","first-page":"459","article-title":"Neural optimizer search with reinforcement learning","volume-title":"Proc. 34th Int. Conf. Mach. Learn.","author":"Bello"},{"key":"ref60","article-title":"Designing neural network architectures using reinforcement learning","volume-title":"Proc. 5th Int. Conf. Learn. Represent. (ICLR)","author":"Baker"},{"key":"ref61","doi-asserted-by":"publisher","DOI":"10.1145\/1150402.1150464"},{"key":"ref62","first-page":"2654","article-title":"Do deep nets really need to be deep?","volume-title":"Advances in Neural Information Processing Systems","author":"Lei","year":"2013"},{"key":"ref63","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2017.754"},{"key":"ref64","article-title":"Model compression via distillation and quantization","volume-title":"Proc. 6th Int. Conf. Learn. Represent. (ICLR)","author":"Polino"},{"key":"ref65","article-title":"Training shallow and thin networks for acceleration via knowledge distillation with conditional adversarial networks","volume-title":"Proc. 6th Int. Conf. Learn. Represent. (ICLR)","author":"Xu"},{"key":"ref66","article-title":"Data-free knowledge distillation for deep neural networks","volume-title":"Proc. Adv. Neural Inf. Process. Syst. Annu. Conf. Neural Inf. Process. Syst.","author":"Lopes"},{"key":"ref67","first-page":"1602","article-title":"Born again neural networks","volume-title":"Proc. 35th Int. Conf. Mach. Learn. (ICML)","author":"Furlanello"},{"key":"ref68","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2017.776"},{"key":"ref69","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v32i1.11783"},{"key":"ref70","first-page":"205","article-title":"Improving fast segmentation with teacher-student learning","volume-title":"Proc. Brit. Mach. Vis. Conf. (BMVC)","author":"Xie"},{"key":"ref71","first-page":"1894","article-title":"On the theory of learnining with privileged information","volume-title":"Advances in Neural Information Processing Systems","author":"Pechyony","year":"2010"},{"issue":"61","key":"ref72","first-page":"2023","article-title":"Learning using privileged information: Similarity control and knowledge transfer","volume":"16","author":"Vapnik","year":"2015","journal-title":"J. Mach. Learn. Res."},{"key":"ref73","doi-asserted-by":"publisher","DOI":"10.1016\/j.neunet.2009.06.042"},{"key":"ref74","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2018.00454"},{"key":"ref75","doi-asserted-by":"publisher","DOI":"10.1145\/3097983.3098135"},{"key":"ref76","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v33i01.33013068"},{"key":"ref77","first-page":"4580","article-title":"Rocket launching: A unified and effecient framework for training well-behaved light net","volume-title":"Proc. 32nd AAAI Conf. Artif. Intell.","author":"Zhou"},{"key":"ref78","doi-asserted-by":"publisher","DOI":"10.5244\/C.30.87"},{"key":"ref79","article-title":"Training shallow and thin networks for acceleration via knowledge distillation with conditional adversarial networks","volume-title":"Proc. Int. Conf. Learn. Represent. Workshop","author":"Xu"},{"key":"ref80","article-title":"Adam: A method for stochastic optimization","volume-title":"Proc. Int. Conf. Learn. Represent. (ICLR)","author":"Kingma"},{"key":"ref81","doi-asserted-by":"publisher","DOI":"10.1007\/s13398-014-0173-7.2"},{"key":"ref82","article-title":"Learning multiple layers of features from tiny images","author":"Krizhevsky","year":"2009"},{"key":"ref83","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2017.74"},{"key":"ref84","first-page":"5","article-title":"Reading digits in natural images with unsupervised feature learning","volume-title":"Proc. NIPS Workshop Deep Learn. Unsupervised Feature Learn.","author":"Netzer"},{"key":"ref85","volume-title":"Tiny ImageNet Visual Recognition Challenge","author":"Le","year":"2015"}],"container-title":["IEEE Transactions on Cybernetics"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx7\/6221036\/9749993\/09151346.pdf?arnumber=9151346","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,1,10]],"date-time":"2024-01-10T00:15:30Z","timestamp":1704845730000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/9151346\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2022,4]]},"references-count":85,"journal-issue":{"issue":"4"},"URL":"https:\/\/doi.org\/10.1109\/tcyb.2020.3007506","relation":{},"ISSN":["2168-2267","2168-2275"],"issn-type":[{"value":"2168-2267","type":"print"},{"value":"2168-2275","type":"electronic"}],"subject":[],"published":{"date-parts":[[2022,4]]}}}