{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2024,12,16]],"date-time":"2024-12-16T09:40:23Z","timestamp":1734342023518,"version":"3.30.2"},"reference-count":60,"publisher":"Springer Science and Business Media LLC","issue":"6","license":[{"start":{"date-parts":[[2024,10,15]],"date-time":"2024-10-15T00:00:00Z","timestamp":1728950400000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2024,10,15]],"date-time":"2024-10-15T00:00:00Z","timestamp":1728950400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"funder":[{"name":"Yunnan Fundamental Research Projects","award":["202401AU070052"],"award-info":[{"award-number":["202401AU070052"]}]}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Multimedia Systems"],"published-print":{"date-parts":[[2024,12]]},"DOI":"10.1007\/s00530-024-01507-5","type":"journal-article","created":{"date-parts":[[2024,10,15]],"date-time":"2024-10-15T15:07:46Z","timestamp":1729004866000},"update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":0,"title":["DS-SRD: a unified framework for structured representation distillation"],"prefix":"10.1007","volume":"30","author":[{"given":"Yuelong","family":"Xia","sequence":"first","affiliation":[]},{"given":"Jing","family":"Yang","sequence":"additional","affiliation":[]},{"given":"Xiaodi","family":"Sun","sequence":"additional","affiliation":[]},{"given":"Yungang","family":"Zhang","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2024,10,15]]},"reference":[{"key":"1507_CR1","unstructured":"Hinton, G.: Distilling the Knowledge in a Neural Network. arXiv preprint arXiv:1503.02531 (2015)"},{"key":"1507_CR2","doi-asserted-by":"publisher","first-page":"108316","DOI":"10.1016\/j.patcog.2021.108316","volume":"122","author":"C Shi","year":"2022","unstructured":"Shi, C., Fang, L., Lv, Z., Zhao, M.: Explainable scale distillation for hyperspectral image classification. Pattern Recogn. 122, 108316 (2022)","journal-title":"Pattern Recogn."},{"key":"1507_CR3","doi-asserted-by":"crossref","unstructured":"Yang, C., Zhou, H., An, Z., Jiang, X., Xu, Y., Zhang, Q.: Cross-image relational knowledge distillation for semantic segmentation. In: Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition, pp. 12319\u201312328 (2022)","DOI":"10.1109\/CVPR52688.2022.01200"},{"key":"1507_CR4","doi-asserted-by":"crossref","unstructured":"Yoon, J.W., Woo, B.J., Ahn, S., Lee, H., Kim, N.S.: Inter-kd: intermediate knowledge distillation for ctc-based automatic speech recognition. In: 2022 IEEE Spoken Language Technology Workshop (SLT), pp. 280\u2013286 (2023)","DOI":"10.1109\/SLT54892.2023.10022581"},{"key":"1507_CR5","unstructured":"Tian, Y., Krishnan, D., Isola, P.: Contrastive Representation Distillation. arXiv preprint arXiv:1910.10699 (2019)"},{"key":"1507_CR6","doi-asserted-by":"crossref","unstructured":"Jung, S., Lee, D., Park, T., Moon, T.: Fair feature distillation for visual recognition. In: Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition, pp. 12115\u201312124 (2021)","DOI":"10.1109\/CVPR46437.2021.01194"},{"key":"1507_CR7","unstructured":"Lee, K.: Prototypical contrastive predictive coding. In: International Conference on Learning Representations (2022)"},{"key":"1507_CR8","unstructured":"Zhang, H., Cisse, M., Dauphin, Y.N., Lopez-Paz, D.: mixup: Beyond Empirical Risk Minimization. arXiv preprint arXiv:1710.09412 (2017)"},{"key":"1507_CR9","doi-asserted-by":"crossref","unstructured":"Yun, S., Han, D., Oh, S.J., Chun, S., Choe, J., Yoo, Y.: Cutmix: regularization strategy to train strong classifiers with localizable features. In: Proceedings of the IEEE\/CVF International Conference on Computer Vision, pp. 6023\u20136032 (2019)","DOI":"10.1109\/ICCV.2019.00612"},{"key":"1507_CR10","doi-asserted-by":"crossref","unstructured":"Zhang, L., Song, J., Gao, A., Chen, J., Bao, C., Ma, K.: Be your own teacher: improve the performance of convolutional neural networks via self distillation. In: Proceedings of the IEEE\/CVF International Conference on Computer Vision, pp. 3713\u20133722 (2019)","DOI":"10.1109\/ICCV.2019.00381"},{"key":"1507_CR11","unstructured":"Oord, A.V.d., Li, Y., Vinyals, O.: Representation Learning With Contrastive Predictive Coding. arXiv preprint arXiv:1807.03748 (2018)"},{"key":"1507_CR12","doi-asserted-by":"crossref","unstructured":"Chen, L., Wang, D., Gan, Z., Liu, J., Henao, R., Carin, L.: Wasserstein contrastive representation distillation. In: Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition, pp. 16296\u201316305 (2021)","DOI":"10.1109\/CVPR46437.2021.01603"},{"issue":"11","key":"1507_CR13","doi-asserted-by":"publisher","first-page":"4037","DOI":"10.1109\/TPAMI.2020.2992393","volume":"43","author":"L Jing","year":"2020","unstructured":"Jing, L., Tian, Y.: Self-supervised visual feature learning with deep neural networks: a survey. IEEE Trans. Pattern Anal. Mach. Intell. 43(11), 4037\u20134058 (2020)","journal-title":"IEEE Trans. Pattern Anal. Mach. Intell."},{"key":"1507_CR14","doi-asserted-by":"crossref","unstructured":"He, K., Fan, H., Wu, Y., Xie, S., Girshick, R.: Momentum contrast for unsupervised visual representation learning. In: Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition, pp. 9729\u20139738 (2020)","DOI":"10.1109\/CVPR42600.2020.00975"},{"key":"1507_CR15","unstructured":"Alexey, D.: An Image is Worth 16x16 Words: Transformers for Image Recognition at Scale. arXiv preprint arXiv: 2010.11929 (2020)"},{"key":"1507_CR16","doi-asserted-by":"crossref","unstructured":"Caron, M., Touvron, H., Misra, I., J\u00e9gou, H., Mairal, J., Bojanowski, P., Joulin, A.: Emerging properties in self-supervised vision transformers. In: Proceedings of the IEEE\/CVF International Conference on Computer Vision, pp. 9650\u20139660 (2021)","DOI":"10.1109\/ICCV48922.2021.00951"},{"key":"1507_CR17","unstructured":"Fang, Z., Wang, J., Wang, L., Zhang, L., Yang, Y., Liu, Z.: Seed: Self-Supervised Distillation for Visual Representation. arXiv preprint arXiv:2101.04731 (2021)"},{"key":"1507_CR18","unstructured":"Chen, X., Fan, H., Girshick, R., He, K.: Improved Baselines with Momentum Contrastive Learning. arXiv preprint arXiv:2003.04297 (2020)"},{"key":"1507_CR19","unstructured":"Chen, T., Kornblith, S., Norouzi, M., Hinton, G.: A simple framework for contrastive learning of visual representations. In: International Conference on Machine Learning, pp. 1597\u20131607. PMLR (2020)"},{"key":"1507_CR20","first-page":"9912","volume":"33","author":"M Caron","year":"2020","unstructured":"Caron, M., Misra, I., Mairal, J., Goyal, P., Bojanowski, P., Joulin, A.: Unsupervised learning of visual features by contrasting cluster assignments. Adv. Neural Inf. Process. Syst. 33, 9912\u20139924 (2020)","journal-title":"Adv. Neural Inf. Process. Syst."},{"key":"1507_CR21","unstructured":"Nielsen, F.: A Family of Statistical Symmetric Divergences Based on Jensen\u2019s Inequality. arXiv preprint arXiv:1009.4004 (2010)"},{"key":"1507_CR22","unstructured":"Lodzimierz Bryc, W.: Normal Distribution Characterizations With Applications. Lecture Notes in Statistics, vol. 100, p. 17 (1995)"},{"key":"1507_CR23","first-page":"21271","volume":"33","author":"J-B Grill","year":"2020","unstructured":"Grill, J.-B., Strub, F., Altch\u00e9, F., Tallec, C., Richemond, P., Buchatskaya, E., Doersch, C., Avila Pires, B., Guo, Z., Gheshlaghi Azar, M.: Bootstrap your own latent\u2014a new approach to self-supervised learning. Adv. Neural Inf. Process. Syst. 33, 21271\u201321284 (2020)","journal-title":"Adv. Neural Inf. Process. Syst."},{"key":"1507_CR24","unstructured":"Lee, K., Zhu, Y., Sohn, K., Li, C.-L., Shin, J., Lee, H.: i-mix: A Domain-Agnostic Strategy for Contrastive Representation Learning. arXiv preprint arXiv:2010.08887 (2020)"},{"key":"1507_CR25","unstructured":"Verma, V., Luong, T., Kawaguchi, K., Pham, H., Le, Q.: Towards domain-agnostic contrastive learning. In: International Conference on Machine Learning, pp. 10530\u201310541 (2021)"},{"issue":"10s","key":"1507_CR26","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1145\/3505244","volume":"54","author":"S Khan","year":"2022","unstructured":"Khan, S., Naseer, M., Hayat, M., Zamir, S.W., Khan, F.S., Shah, M.: Transformers in vision: a survey. ACM Comput. Surv. 54(10s), 1\u201341 (2022)","journal-title":"ACM Comput. Surv."},{"key":"1507_CR27","doi-asserted-by":"crossref","unstructured":"Li, C., Tang, T., Wang, G., Peng, J., Wang, B., Liang, X., Chang, X.: Bossnas: exploring hybrid CNN-transformers with block-wisely self-supervised neural architecture search. In: Proceedings of the IEEE\/CVF International Conference on Computer Vision, pp. 12281\u201312291 (2021)","DOI":"10.1109\/ICCV48922.2021.01206"},{"key":"1507_CR28","unstructured":"Romero, A., Ballas, N., Kahou, S.E., Chassang, A., Gatta, C., Bengio, Y.: Fitnets: Hints for Thin Deep Nets. arXiv preprint arXiv:1412.6550 (2014)"},{"key":"1507_CR29","unstructured":"Zagoruyko, S., Komodakis, N.: Paying More Attention to Attention: Improving the Performance of Convolutional Neural Networks Via Attention Transfer. arXiv preprint arXiv:1612.03928 (2016)"},{"key":"1507_CR30","doi-asserted-by":"crossref","unstructured":"Tung, F., Mori, G.: Similarity-preserving knowledge distillation. In: Proceedings of the IEEE\/CVF International Conference on Computer Vision, pp. 1365\u20131374 (2019)","DOI":"10.1109\/ICCV.2019.00145"},{"key":"1507_CR31","doi-asserted-by":"crossref","unstructured":"Peng, B., Jin, X., Liu, J., Li, D., Wu, Y., Liu, Y., Zhou, S., Zhang, Z.: Correlation congruence for knowledge distillation. In: Proceedings of the IEEE\/CVF International Conference on Computer Vision, pp. 5007\u20135016 (2019)","DOI":"10.1109\/ICCV.2019.00511"},{"key":"1507_CR32","doi-asserted-by":"crossref","unstructured":"Ahn, S., Hu, S.X., Damianou, A., Lawrence, N.D., Dai, Z.: Variational information distillation for knowledge transfer. In: Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition, pp. 9163\u20139171 (2019)","DOI":"10.1109\/CVPR.2019.00938"},{"key":"1507_CR33","doi-asserted-by":"crossref","unstructured":"Park, W., Kim, D., Lu, Y., Cho, M.: Relational knowledge distillation. In: Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition, pp. 3967\u20133976 (2019)","DOI":"10.1109\/CVPR.2019.00409"},{"key":"1507_CR34","doi-asserted-by":"crossref","unstructured":"Passalis, N., Tefas, A.: Learning deep representations with probabilistic knowledge transfer. In: Proceedings of the European Conference on Computer Vision (ECCV), pp. 268\u2013284 (2018)","DOI":"10.1007\/978-3-030-01252-6_17"},{"key":"1507_CR35","doi-asserted-by":"crossref","unstructured":"Heo, B., Lee, M., Yun, S., Choi, J.Y.: Knowledge transfer via distillation of activation boundaries formed by hidden neurons. In: Proceedings of the AAAI Conference on Artificial Intelligence, vol. 33, pp. 3779\u20133787 (2019)","DOI":"10.1609\/aaai.v33i01.33013779"},{"key":"1507_CR36","unstructured":"Kim, J., Park, S., Kwak, N.: Paraphrasing complex network: network compression via factor transfer. Adv. Neural Inf. Process. syst. 31 (2018)"},{"key":"1507_CR37","doi-asserted-by":"crossref","unstructured":"Yim, J., Joo, D., Bae, J., Kim, J.: A gift from knowledge distillation: fast optimization, network minimization and transfer learning. In: Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition, pp. 4133\u20134141 (2017)","DOI":"10.1109\/CVPR.2017.754"},{"key":"1507_CR38","unstructured":"Huang, Z., Wang, N.: Like What You Like: Knowledge Distill Via Neuron Selectivity Transfer. arXiv preprint arXiv:1707.01219 (2017)"},{"key":"1507_CR39","doi-asserted-by":"crossref","unstructured":"Sandler, M., Howard, A., Zhu, M., Zhmoginov, A., Chen, L.-C.: Mobilenetv2: Inverted residuals and linear bottlenecks. In: Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition, pp. 4510\u20134520 (2018)","DOI":"10.1109\/CVPR.2018.00474"},{"key":"1507_CR40","doi-asserted-by":"crossref","unstructured":"Zhang, X., Zhou, X., Lin, M., Sun, J.: Shufflenet: an extremely efficient convolutional neural network for mobile devices. In: Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition, pp. 6848\u20136856 (2018)","DOI":"10.1109\/CVPR.2018.00716"},{"key":"1507_CR41","doi-asserted-by":"crossref","unstructured":"Tan, M., Chen, B., Pang, R., Vasudevan, V., Sandler, M., Howard, A., Le, Q.V.: Mnasnet: platform-aware neural architecture search for mobile. In: Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition, pp. 2820\u20132828 (2019)","DOI":"10.1109\/CVPR.2019.00293"},{"key":"1507_CR42","doi-asserted-by":"crossref","unstructured":"Deng, J., Dong, W., Socher, R., Li, L.-J., Li, K., Fei-Fei, L.: Imagenet: a large-scale hierarchical image database. In: 2009 IEEE Conference on Computer Vision and Pattern Recognition, pp. 248\u2013255 (2009). Ieee","DOI":"10.1109\/CVPR.2009.5206848"},{"key":"1507_CR43","unstructured":"Warden, P.: Speech Commands: A Dataset for Limited-Vocabulary Speech Recognition. arXiv preprint arXiv:1804.03209 (2018)"},{"key":"1507_CR44","unstructured":"Guo, C., Pleiss, G., Sun, Y., Weinberger, K.Q.: On calibration of modern neural networks. In: International Conference on Machine Learning, pp. 1321\u20131330 (2017)"},{"key":"1507_CR45","unstructured":"Snell, J., Swersky, K., Zemel, R.: Prototypical networks for few-shot learning. Adv. Neural Inf. Process. Syst. 30 (2017)"},{"key":"1507_CR46","doi-asserted-by":"crossref","unstructured":"Wu, Z., Xiong, Y., Yu, S.X., Lin, D.: Unsupervised feature learning via non-parametric instance discrimination. In: Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition, pp. 3733\u20133742 (2018)","DOI":"10.1109\/CVPR.2018.00393"},{"key":"1507_CR47","unstructured":"Li, J., Zhou, P., Xiong, C., Hoi, S.C.: Prototypical Contrastive Learning of Unsupervised Representations. arXiv preprint arXiv:2005.04966 (2020)"},{"key":"1507_CR48","unstructured":"Asano, Y.M., Rupprecht, C., Vedaldi, A.: Self-Labelling Via Simultaneous Clustering and Representation Learning. arXiv preprint arXiv:1911.05371 (2019)"},{"key":"1507_CR49","first-page":"6827","volume":"33","author":"Y Tian","year":"2020","unstructured":"Tian, Y., Sun, C., Poole, B., Krishnan, D., Schmid, C., Isola, P.: What makes for good views for contrastive learning? Adv. Neural Inf. Process. Syst. 33, 6827\u20136839 (2020)","journal-title":"Adv. Neural Inf. Process. Syst."},{"key":"1507_CR50","doi-asserted-by":"crossref","unstructured":"Caron, M., Bojanowski, P., Joulin, A., Douze, M.: Deep clustering for unsupervised learning of visual features. In: Proceedings of the European Conference on Computer Vision (ECCV), pp. 132\u2013149 (2018)","DOI":"10.1007\/978-3-030-01264-9_9"},{"key":"1507_CR51","doi-asserted-by":"crossref","unstructured":"Silberman, N., Hoiem, D., Kohli, P., Fergus, R.: Indoor segmentation and support inference from rgbd images. In: European Conference on Computer Vision, pp. 746\u2013760. Springer, Berlin (2012)","DOI":"10.1007\/978-3-642-33715-4_54"},{"issue":"3","key":"1507_CR52","doi-asserted-by":"publisher","first-page":"302","DOI":"10.1007\/s11263-018-1140-0","volume":"127","author":"B Zhou","year":"2019","unstructured":"Zhou, B., Zhao, H., Puig, X., Xiao, T., Fidler, S., Barriuso, A., Torralba, A.: Semantic understanding of scenes through the ade20k dataset. Int. J. Comput. Vis. 127(3), 302\u2013321 (2019)","journal-title":"Int. J. Comput. Vis."},{"key":"1507_CR53","doi-asserted-by":"crossref","unstructured":"Xiao, T., Liu, Y., Zhou, B., Jiang, Y., Sun, J.: Unified perceptual parsing for scene understanding. In: Proceedings of the European Conference on Computer Vision (ECCV), pp. 418\u2013434 (2018)","DOI":"10.1007\/978-3-030-01228-1_26"},{"key":"1507_CR54","doi-asserted-by":"crossref","unstructured":"Zhao, H., Shi, J., Qi, X., Wang, X., Jia, J.: Pyramid scene parsing network. In: Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition, pp. 2881\u20132890 (2017)","DOI":"10.1109\/CVPR.2017.660"},{"key":"1507_CR55","doi-asserted-by":"crossref","unstructured":"Ericsson, L., Gouk, H., Hospedales, T.M.: How well do self-supervised models transfer? In: Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition, pp. 5414\u20135423 (2021)","DOI":"10.1109\/CVPR46437.2021.00537"},{"key":"1507_CR56","doi-asserted-by":"crossref","unstructured":"Zeisl, B., Pollefeys, M., : Discriminatively trained dense surface normal estimation. In: European Conference on Computer Vision, pp. 468\u2013484. Springer, Berlin (2014)","DOI":"10.1007\/978-3-319-10602-1_31"},{"key":"1507_CR57","doi-asserted-by":"crossref","unstructured":"Cubuk, E.D., Zoph, B., Mane, D., Vasudevan, V., Le, Q.V.: Autoaugment: Learning Augmentation Policies From Data. arXiv preprint arXiv:1805.09501 (2018)","DOI":"10.1109\/CVPR.2019.00020"},{"key":"1507_CR58","doi-asserted-by":"crossref","unstructured":"Beyer, L., Zhai, X., Royer, A., Markeeva, L., Anil, R., Kolesnikov, A.: Knowledge distillation: A good teacher is patient and consistent. In: Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition, pp. 10925\u201310934 (2022)","DOI":"10.1109\/CVPR52688.2022.01065"},{"key":"1507_CR59","first-page":"13456","volume":"35","author":"H Wang","year":"2022","unstructured":"Wang, H., Lohit, S., Jones, M.N., Fu, Y.: What makes a \u201cgood\u2019\u2019 data augmentation in knowledge distillation\u2014a statistical perspective. Adv. Neural Inf. Process. Syst. 35, 13456\u201313469 (2022)","journal-title":"Adv. Neural Inf. Process. Syst."},{"key":"1507_CR60","unstructured":"Menon, A.K., Rawat, A.S., Reddi, S., Kim, S., Kumar, S.: A statistical perspective on distillation. In: International Conference on Machine Learning, pp. 7632\u20137642 (2021)"}],"container-title":["Multimedia Systems"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s00530-024-01507-5.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s00530-024-01507-5\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s00530-024-01507-5.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,12,16]],"date-time":"2024-12-16T09:07:22Z","timestamp":1734340042000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s00530-024-01507-5"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,10,15]]},"references-count":60,"journal-issue":{"issue":"6","published-print":{"date-parts":[[2024,12]]}},"alternative-id":["1507"],"URL":"https:\/\/doi.org\/10.1007\/s00530-024-01507-5","relation":{},"ISSN":["0942-4962","1432-1882"],"issn-type":[{"type":"print","value":"0942-4962"},{"type":"electronic","value":"1432-1882"}],"subject":[],"published":{"date-parts":[[2024,10,15]]},"assertion":[{"value":"17 April 2024","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"17 September 2024","order":2,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"15 October 2024","order":3,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Declarations"}},{"value":"The authors declare that there are no Conflict of interest, and manuscript is approved by all authors for publication.","order":2,"name":"Ethics","group":{"name":"EthicsHeading","label":"Conflict of interest"}}],"article-number":"319"}}