{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,3,28]],"date-time":"2025-03-28T07:58:23Z","timestamp":1743148703025,"version":"3.40.3"},"publisher-location":"Cham","reference-count":38,"publisher":"Springer International Publishing","isbn-type":[{"type":"print","value":"9783031024436"},{"type":"electronic","value":"9783031024443"}],"license":[{"start":{"date-parts":[[2022,1,1]],"date-time":"2022-01-01T00:00:00Z","timestamp":1640995200000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"},{"start":{"date-parts":[[2022,1,1]],"date-time":"2022-01-01T00:00:00Z","timestamp":1640995200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2022]]},"DOI":"10.1007\/978-3-031-02444-3_35","type":"book-chapter","created":{"date-parts":[[2022,5,9]],"date-time":"2022-05-09T12:02:50Z","timestamp":1652097770000},"page":"462-475","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":0,"title":["Visualizing the\u00a0Embedding Space to\u00a0Explain the\u00a0Effect of\u00a0Knowledge Distillation"],"prefix":"10.1007","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-4088-3632","authenticated-orcid":false,"given":"Hyun Seung","family":"Lee","sequence":"first","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0002-2604-9115","authenticated-orcid":false,"given":"Christian","family":"Wallraven","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2022,5,10]]},"reference":[{"key":"35_CR1","unstructured":"Aljalbout, E., Golkov, V., Siddiqui, Y., Cremers, D.: Clustering with deep learning: taxonomy and new methods. arXiv preprint arXiv:1801.07648 (2018)"},{"key":"35_CR2","doi-asserted-by":"crossref","unstructured":"Arazo, E., Ortego, D., Albert, P., O\u2019Connor, N.E., McGuinness, K.: Pseudo-labeling and confirmation bias in deep semi-supervised learning. In: 2020 International Joint Conference on Neural Networks (IJCNN), pp. 1\u20138. IEEE (2020)","DOI":"10.1109\/IJCNN48605.2020.9207304"},{"key":"35_CR3","unstructured":"Bello, I., et al.: Revisiting ResNets: improved training and scaling strategies. arXiv preprint arXiv:2103.07579 (2021)"},{"key":"35_CR4","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"437","DOI":"10.1007\/978-3-642-35289-8_26","volume-title":"Neural Networks: Tricks of the Trade","author":"Y Bengio","year":"2012","unstructured":"Bengio, Y.: Practical recommendations for gradient-based training of deep architectures. In: Montavon, G., Orr, G.B., M\u00fcller, K.-R. (eds.) Neural Networks: Tricks of the Trade. LNCS, vol. 7700, pp. 437\u2013478. Springer, Heidelberg (2012). https:\/\/doi.org\/10.1007\/978-3-642-35289-8_26"},{"key":"35_CR5","unstructured":"Bernhardsson, E.: Approximate Nearest Neighbors in C++\/Python optimized for memory usage and loading\/saving to disk (2018). https:\/\/github.com\/spotify\/annoy"},{"key":"35_CR6","unstructured":"Berthelot, D., Carlini, N., Goodfellow, I., Papernot, N., Oliver, A., Raffel, C.: MixMatch: a holistic approach to semi-supervised learning. arXiv preprint arXiv:1905.02249 (2019)"},{"key":"35_CR7","unstructured":"Canziani, A., Paszke, A., Culurciello, E.: An analysis of deep neural network models for practical applications. arXiv preprint arXiv:1605.07678 (2016)"},{"key":"35_CR8","doi-asserted-by":"crossref","unstructured":"Chan, D.M., Rao, R., Huang, F., Canny, J.F.: T-SNE-CUDA: GPU-accelerated t-SNE and its applications to modern data. In: 2018 30th International Symposium on Computer Architecture and High Performance Computing (SBAC-PAD), pp. 330\u2013338. IEEE (2018)","DOI":"10.1109\/CAHPC.2018.8645912"},{"key":"35_CR9","unstructured":"Chen, T., Kornblith, S., Swersky, K., Norouzi, M., Hinton, G.: Big self-supervised models are strong semi-supervised learners. arXiv preprint arXiv:2006.10029 (2020)"},{"key":"35_CR10","doi-asserted-by":"crossref","unstructured":"Cheng, X., Rao, Z., Chen, Y., Zhang, Q.: Explaining knowledge distillation by quantifying the knowledge. In: Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition, pp. 12925\u201312935 (2020)","DOI":"10.1109\/CVPR42600.2020.01294"},{"issue":"6","key":"35_CR11","doi-asserted-by":"publisher","first-page":"1789","DOI":"10.1007\/s11263-021-01453-z","volume":"129","author":"J Gou","year":"2021","unstructured":"Gou, J., Yu, B., Maybank, S.J., Tao, D.: Knowledge distillation: a survey. Int. J. Comput. Vis. 129(6), 1789\u20131819 (2021)","journal-title":"Int. J. Comput. Vis."},{"key":"35_CR12","unstructured":"Han, S., Mao, H., Dally, W.J.: Deep compression: compressing deep neural networks with pruning, trained quantization and Huffman coding. arXiv preprint arXiv:1510.00149 (2015)"},{"key":"35_CR13","doi-asserted-by":"crossref","unstructured":"He, K., Zhang, X., Ren, S., Sun, J.: Deep residual learning for image recognition. In: Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition, pp. 770\u2013778 (2016)","DOI":"10.1109\/CVPR.2016.90"},{"key":"35_CR14","unstructured":"Hinton, G., Vinyals, O., Dean, J.: Distilling the knowledge in a neural network (2015)"},{"key":"35_CR15","unstructured":"Huh, M., Mobahi, H., Zhang, R., Cheung, B., Agrawal, P., Isola, P.: The low-rank simplicity bias in deep networks. arXiv preprint arXiv:2103.10427 (2021)"},{"key":"35_CR16","unstructured":"Komodakis, N., Zagoruyko, S.: Paying more attention to attention: improving the performance of convolutional neural networks via attention transfer. In: ICLR (2017)"},{"key":"35_CR17","unstructured":"Krizhevsky, A., Hinton, G., et al.: Learning multiple layers of features from tiny images. University of Toronto (2009)"},{"key":"35_CR18","unstructured":"Lin, Z.Q., Wong, A.: Progressive label distillation: learning input-efficient deep neural networks. arXiv preprint arXiv:1901.09135 (2019)"},{"key":"35_CR19","doi-asserted-by":"publisher","first-page":"106","DOI":"10.1016\/j.neucom.2020.07.048","volume":"415","author":"Y Liu","year":"2020","unstructured":"Liu, Y., Zhang, W., Wang, J.: Adaptive multi-teacher multi-level knowledge distillation. Neurocomputing 415, 106\u2013113 (2020)","journal-title":"Neurocomputing"},{"issue":"11","key":"35_CR20","first-page":"2579","volume":"9","author":"L Van der Maaten","year":"2008","unstructured":"Van der Maaten, L., Hinton, G.: Visualizing data using t-SNE. J. Mach. Learn. Res. 9(11), 2579\u20132605 (2008)","journal-title":"J. Mach. Learn. Res."},{"key":"35_CR21","unstructured":"M\u00fcller, R., Kornblith, S., Hinton, G.: When does label smoothing help? arXiv preprint arXiv:1906.02629 (2019)"},{"key":"35_CR22","unstructured":"Phuong, M., Lampert, C.: Towards understanding knowledge distillation. In: International Conference on Machine Learning, pp. 5142\u20135151. PMLR (2019)"},{"key":"35_CR23","unstructured":"Polino, A., Pascanu, R., Alistarh, D.: Model compression via distillation and quantization. arXiv preprint arXiv:1802.05668 (2018)"},{"key":"35_CR24","unstructured":"Rasmus, A., Valpola, H., Honkala, M., Berglund, M., Raiko, T.: Semi-supervised learning with ladder networks. arXiv preprint arXiv:1507.02672 (2015)"},{"issue":"3","key":"35_CR25","doi-asserted-by":"publisher","first-page":"211","DOI":"10.1007\/s11263-015-0816-y","volume":"115","author":"O Russakovsky","year":"2015","unstructured":"Russakovsky, O.: ImageNet large scale visual recognition challenge. Int. J. Comput. Vis. 115(3), 211\u2013252 (2015)","journal-title":"Int. J. Comput. Vis."},{"key":"35_CR26","doi-asserted-by":"crossref","unstructured":"Schroff, F., Kalenichenko, D., Philbin, J.: FaceNet: a unified embedding for face recognition and clustering. In: Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition, pp. 815\u2013823 (2015)","DOI":"10.1109\/CVPR.2015.7298682"},{"key":"35_CR27","series-title":"Studies in Big Data","doi-asserted-by":"publisher","first-page":"123","DOI":"10.1007\/978-3-319-54024-5_6","volume-title":"Transparent Data Mining for Big and Small Data","author":"C Seifert","year":"2017","unstructured":"Seifert, C.: Visualizations of deep neural networks in computer vision: a survey. In: Cerquitelli, T., Quercia, D., Pasquale, F. (eds.) Transparent Data Mining for Big and Small Data. SBD, vol. 11, pp. 123\u2013144. Springer, Cham (2017). https:\/\/doi.org\/10.1007\/978-3-319-54024-5_6"},{"key":"35_CR28","unstructured":"Simonyan, K., Zisserman, A.: Very deep convolutional networks for large-scale image recognition. arXiv preprint arXiv:1409.1556 (2014)"},{"key":"35_CR29","unstructured":"Sohn, K., et al.: FixMatch: simplifying semi-supervised learning with consistency and confidence. arXiv preprint arXiv:2001.07685 (2020)"},{"issue":"1","key":"35_CR30","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1038\/s41598-019-45301-0","volume":"9","author":"B Szubert","year":"2019","unstructured":"Szubert, B., Cole, J.E., Monaco, C., Drozdov, I.: Structure-preserving visualisation of high dimensional single-cell datasets. Sci. Rep. 9(1), 1\u201310 (2019)","journal-title":"Sci. Rep."},{"key":"35_CR31","unstructured":"Tarvainen, A., Valpola, H.: Mean teachers are better role models: weight-averaged consistency targets improve semi-supervised deep learning results. arXiv preprint arXiv:1703.01780 (2017)"},{"key":"35_CR32","doi-asserted-by":"crossref","unstructured":"Thiagarajan, J.J., Kashyap, S., Karargyris, A.: Distill-to-label: weakly supervised instance labeling using knowledge distillation. In: 2019 18th IEEE International Conference on Machine Learning And Applications (ICMLA), pp. 902\u2013907. IEEE (2019)","DOI":"10.1109\/ICMLA.2019.00156"},{"key":"35_CR33","doi-asserted-by":"crossref","unstructured":"Wang, L., Yoon, K.J.: Knowledge distillation and student-teacher learning for visual intelligence: a review and new outlooks. IEEE Trans. Pattern Anal. Mach. Intell. (2021)","DOI":"10.1109\/TPAMI.2021.3055564"},{"key":"35_CR34","doi-asserted-by":"crossref","unstructured":"Wattenberg, M., Vi\u00e9gas, F., Johnson, I.: How to use t-SNE effectively. Distill 1(10), e2 (2016)","DOI":"10.23915\/distill.00002"},{"key":"35_CR35","unstructured":"Yu, W., Yang, K., Bai, Y., Yao, H., Rui, Y.: Visualizing and comparing convolutional neural networks. arXiv preprint arXiv:1412.6631 (2014)"},{"key":"35_CR36","doi-asserted-by":"crossref","unstructured":"Zagoruyko, S., Komodakis, N.: Wide residual networks. arXiv preprint arXiv:1605.07146 (2016)","DOI":"10.5244\/C.30.87"},{"key":"35_CR37","doi-asserted-by":"crossref","unstructured":"Zhu, L., Xu, Z., Yang, Y., Hauptmann, A.G.: Uncovering the temporal context for video question answering. Int. J. Comput. Vis. 124(3), 409\u2013421 (2017)","DOI":"10.1007\/s11263-017-1033-7"},{"key":"35_CR38","doi-asserted-by":"crossref","unstructured":"Zhuang, F.: A comprehensive survey on transfer learning. Proc. IEEE 109(1), 43\u201376 (2020)","DOI":"10.1109\/JPROC.2020.3004555"}],"container-title":["Lecture Notes in Computer Science","Pattern Recognition"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-031-02444-3_35","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2022,5,9]],"date-time":"2022-05-09T12:09:00Z","timestamp":1652098140000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-3-031-02444-3_35"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2022]]},"ISBN":["9783031024436","9783031024443"],"references-count":38,"URL":"https:\/\/doi.org\/10.1007\/978-3-031-02444-3_35","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"type":"print","value":"0302-9743"},{"type":"electronic","value":"1611-3349"}],"subject":[],"published":{"date-parts":[[2022]]},"assertion":[{"value":"10 May 2022","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"ACPR","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Asian Conference on Pattern Recognition","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Jeju Island","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Korea (Republic of)","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2021","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"9 November 2021","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"12 November 2021","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"6","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"acpr2021","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"http:\/\/www.acpr2021.org","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Single-blind","order":1,"name":"type","label":"Type","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"EasyChair","order":2,"name":"conference_management_system","label":"Conference Management System","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"154","order":3,"name":"number_of_submissions_sent_for_review","label":"Number of Submissions Sent for Review","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"85","order":4,"name":"number_of_full_papers_accepted","label":"Number of Full Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"0","order":5,"name":"number_of_short_papers_accepted","label":"Number of Short Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"55% - The value is computed by the equation \"Number of Full Papers Accepted \/ Number of Submissions Sent for Review * 100\" and then rounded to a whole number.","order":6,"name":"acceptance_rate_of_full_papers","label":"Acceptance Rate of Full Papers","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"3","order":7,"name":"average_number_of_reviews_per_paper","label":"Average Number of Reviews per Paper","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"4","order":8,"name":"average_number_of_papers_per_reviewer","label":"Average Number of Papers per Reviewer","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"No","order":9,"name":"external_reviewers_involved","label":"External Reviewers Involved","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}}]}}