{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,3,25]],"date-time":"2025-03-25T23:26:25Z","timestamp":1742945185402,"version":"3.40.3"},"publisher-location":"Cham","reference-count":26,"publisher":"Springer International Publishing","isbn-type":[{"type":"print","value":"9783031109850"},{"type":"electronic","value":"9783031109867"}],"license":[{"start":{"date-parts":[[2022,1,1]],"date-time":"2022-01-01T00:00:00Z","timestamp":1640995200000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"},{"start":{"date-parts":[[2022,1,1]],"date-time":"2022-01-01T00:00:00Z","timestamp":1640995200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2022]]},"DOI":"10.1007\/978-3-031-10986-7_11","type":"book-chapter","created":{"date-parts":[[2022,7,18]],"date-time":"2022-07-18T22:30:36Z","timestamp":1658183436000},"page":"128-139","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":0,"title":["Deep-to-Bottom Weights Decay: A Systemic Knowledge Review Learning Technique for\u00a0Transformer Layers in\u00a0Knowledge Distillation"],"prefix":"10.1007","author":[{"given":"Ankun","family":"Wang","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Feng","family":"Liu","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Zhen","family":"Huang","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Minghao","family":"Hu","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Dongsheng","family":"Li","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Yifan","family":"Chen","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Xinjia","family":"Xie","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2022,7,19]]},"reference":[{"key":"11_CR1","doi-asserted-by":"crossref","unstructured":"Cer, D.M., Diab, M.T., Agirre, E., Lopez-Gazpio, I., Specia, L.: SemEval-2017 task 1: semantic textual similarity - multilingual and cross-lingual focused evaluation. CoRR abs\/1708.00055 (2017)","DOI":"10.18653\/v1\/S17-2001"},{"key":"11_CR2","doi-asserted-by":"crossref","unstructured":"Chen, D., et al.: AdaBERT: task-adaptive BERT compression with differentiable neural architecture search. In: Proceedings of the 29th International Joint Conference on Artificial Intelligence, IJCAI 2020, pp. 2463\u20132469 (2020)","DOI":"10.24963\/ijcai.2020\/341"},{"key":"11_CR3","doi-asserted-by":"crossref","unstructured":"Chen, P., Liu, S., Zhao, H., Jia, J.: Distilling knowledge via knowledge review. In: IEEE Conference on Computer Vision and Pattern Recognition, CVPR 2021, virtual, 19\u201325 June 2021, pp. 5008\u20135017. Computer Vision Foundation\/IEEE (2021)","DOI":"10.1109\/CVPR46437.2021.00497"},{"key":"11_CR4","unstructured":"Devlin, J., Chang, M., Lee, K., Toutanova, K.: BERT: pre-training of deep bidirectional transformers for language understanding. In: Proceedings of the 2019 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, NAACL-HLT 2019, Minneapolis, MN, USA, 2\u20137 June 2019, Volume 1 (Long and Short Papers), pp. 4171\u20134186. Association for Computational Linguistics (2019)"},{"key":"11_CR5","unstructured":"Dolan, W.B., Brockett, C.: Automatically constructing a corpus of sentential paraphrases. In: Proceedings of the 3rd International Workshop on Paraphrasing, IWP@IJCNLP 2005, Jeju Island, Korea, October 2005 (2005)"},{"key":"11_CR6","unstructured":"Fedus, W., Zoph, B., Shazeer, N.: Switch transformers: scaling to trillion parameter models with simple and efficient sparsity. arXiv arXiv:2101.03961 (2021)"},{"key":"11_CR7","unstructured":"Hinton, G.E., Vinyals, O., Dean, J.: Distilling the knowledge in a neural network (2015)"},{"key":"11_CR8","doi-asserted-by":"crossref","unstructured":"Jawahar, G., Sagot, B., Seddah, D.: What does BERT learn about the structure of language? In: Proceedings of the 57th Conference of the Association for Computational Linguistics, ACL 2019, Florence, Italy, 28 July\u20132 August 2019, Volume 1: Long Papers, pp. 3651\u20133657. Association for Computational Linguistics (2019)","DOI":"10.18653\/v1\/P19-1356"},{"key":"11_CR9","doi-asserted-by":"crossref","unstructured":"Jiao, X., et al.: TinyBERT: distilling BERT for natural language understanding. In: Findings of the Association for Computational Linguistics, EMNLP 2020, Online Event, 16\u201320 November 2020. Findings of ACL, vol. EMNLP 2020, pp. 4163\u20134174. Association for Computational Linguistics (2020)","DOI":"10.18653\/v1\/2020.findings-emnlp.372"},{"key":"11_CR10","unstructured":"Lan, Z., Chen, M., Goodman, S., Gimpel, K., Sharma, P., Soricut, R.: ALBERT: a lite BERT for self-supervised learning of language representations. In: 8th International Conference on Learning Representations, ICLR 2020, Addis Ababa, Ethiopia, 26\u201330 April 2020 (2020)"},{"key":"11_CR11","unstructured":"Levesque, H., Davis, E., Morgenstern, L.: The Winograd schema challenge. In: 13th International Conference on the Principles of Knowledge Representation and Reasoning (2012)"},{"key":"11_CR12","doi-asserted-by":"crossref","unstructured":"Li, J., Liu, X., Zhao, H., Xu, R., Yang, M., Jin, Y.: BERT-EMD: many-to-many layer mapping for BERT compression with earth mover\u2019s distance. In: Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing, EMNLP 2020, Online, 16\u201320 November 2020, pp. 3009\u20133018. Association for Computational Linguistics (2020)","DOI":"10.18653\/v1\/2020.emnlp-main.242"},{"key":"11_CR13","unstructured":"Michel, P., Levy, O., Neubig, G.: Are sixteen heads really better than one? In: Advances in Neural Information Processing Systems 32: Annual Conference on Neural Information Processing Systems 2019, NeurIPS 2019, 8\u201314 December 2019, Vancouver, BC, Canada, pp. 14014\u201314024 (2019)"},{"key":"11_CR14","doi-asserted-by":"crossref","unstructured":"Passban, P., Wu, Y., Rezagholizadeh, M., Liu, Q.: ALP-KD: attention-based layer projection for knowledge distillation. In: 35th AAAI Conference on Artificial Intelligence, AAAI 2021, 33rd Conference on Innovative Applications of Artificial Intelligence, IAAI 2021, The 11th Symposium on Educational Advances in Artificial Intelligence, EAAI 2021, Virtual Event, 2\u20139 February 2021. pp. 13657\u201313665. AAAI Press (2021)","DOI":"10.1609\/aaai.v35i15.17610"},{"key":"11_CR15","unstructured":"Radford, A., Narasimhan, K., Salimans, T., Sutskever, I.: Improving language understanding with unsupervised learning (2018)"},{"key":"11_CR16","doi-asserted-by":"crossref","unstructured":"Rajpurkar, P., Zhang, J., Lopyrev, K., Liang, P.: SQuAD: 100,000+ questions for machine comprehension of text. In: Su, J., Carreras, X., Duh, K. (eds.) Proceedings of the 2016 Conference on Empirical Methods in Natural Language Processing, EMNLP 2016, Austin, Texas, USA, 1\u20134 November 2016, pp. 2383\u20132392 (2016)","DOI":"10.18653\/v1\/D16-1264"},{"key":"11_CR17","unstructured":"Sanh, V., Debut, L., Chaumond, J., Wolf, T.: DistilBERT, a distilled version of BERT: smaller, faster, cheaper and lighter. arXiv arXiv:1910.01108 (2019)"},{"key":"11_CR18","doi-asserted-by":"crossref","unstructured":"Shen, S., et al.: Q-BERT: Hessian based ultra low precision quantization of BERT. In: The 34th AAAI Conference on Artificial Intelligence, AAAI 2020, The 32nd Innovative Applications of Artificial Intelligence Conference, IAAI 2020, The 10th AAAI Symposium on Educational Advances in Artificial Intelligence, EAAI 2020, New York, NY, USA, 7\u201312 February 2020, pp. 8815\u20138821 (2020)","DOI":"10.1609\/aaai.v34i05.6409"},{"key":"11_CR19","unstructured":"Socher, R., et al.: Recursive deep models for semantic compositionality over a sentiment treebank. In: Proceedings of the 2013 Conference on Empirical Methods in Natural Language Processing, EMNLP 2013, 18\u201321 October 2013, Grand Hyatt Seattle, Seattle, Washington, USA, A Meeting of SIGDAT, A Special Interest Group of the ACL, pp. 1631\u20131642 (2013)"},{"key":"11_CR20","doi-asserted-by":"crossref","unstructured":"Sun, S., Cheng, Y., Gan, Z., Liu, J.: Patient knowledge distillation for BERT model compression. In: Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing and the 9th International Joint Conference on Natural Language Processing, EMNLP-IJCNLP 2019, Hong Kong, China, 3\u20137 November 2019, pp. 4322\u20134331. Association for Computational Linguistics (2019)","DOI":"10.18653\/v1\/D19-1441"},{"key":"11_CR21","unstructured":"Wang, A., Singh, A., Michael, J., Hill, F., Levy, O., Bowman, S.R.: GLUE: a multi-task benchmark and analysis platform for natural language understanding. In: 7th International Conference on Learning Representations, ICLR 2019, New Orleans, LA, USA, 6\u20139 May 2019 (2019)"},{"key":"11_CR22","doi-asserted-by":"publisher","first-page":"625","DOI":"10.1162\/tacl_a_00290","volume":"7","author":"A Warstadt","year":"2019","unstructured":"Warstadt, A., Singh, A., Bowman, S.R.: Neural network acceptability judgments. Trans. Assoc. Comput. Linguistics 7, 625\u2013641 (2019)","journal-title":"Trans. Assoc. Comput. Linguistics"},{"key":"11_CR23","doi-asserted-by":"crossref","unstructured":"Wasserblat, M., Pereg, O., Izsak, P.: Exploring the boundaries of low-resource BERT distillation. In: Proceedings of SustaiNLP: Workshop on Simple and Efficient Natural Language Processing, pp. 35\u201340 (2020)","DOI":"10.18653\/v1\/2020.sustainlp-1.5"},{"key":"11_CR24","doi-asserted-by":"crossref","unstructured":"Williams, A., Nangia, N., Bowman, S.R.: A broad-coverage challenge corpus for sentence understanding through inference. In: Proceedings of the 2018 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, NAACL-HLT 2018, New Orleans, Louisiana, USA, 1\u20136 June 2018, Volume 1 (Long Papers), pp. 1112\u20131122 (2018)","DOI":"10.18653\/v1\/N18-1101"},{"key":"11_CR25","doi-asserted-by":"publisher","unstructured":"Yang, Z., et al.: TextBrewer: an open-source knowledge distillation toolkit for natural language processing. In: Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics: System Demonstrations, ACL 2020, Online, 5\u201310 July 2020, pp. 9\u201316. Association for Computational Linguistics (2020). https:\/\/doi.org\/10.18653\/v1\/2020.acl-demos.2","DOI":"10.18653\/v1\/2020.acl-demos.2"},{"key":"11_CR26","doi-asserted-by":"crossref","unstructured":"Zafrir, O., Boudoukh, G., Izsak, P., Wasserblat, M.: Q8BERT: quantized 8bit BERT. In: 5th Workshop on Energy Efficient Machine Learning and Cognitive Computing - NeurIPS Edition, EMC2@NeurIPS 2019, Vancouver, Canada, 13 December 2019, pp. 36\u201339. IEEE (2019)","DOI":"10.1109\/EMC2-NIPS53020.2019.00016"}],"container-title":["Lecture Notes in Computer Science","Knowledge Science, Engineering and Management"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-031-10986-7_11","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2023,2,11]],"date-time":"2023-02-11T22:37:09Z","timestamp":1676155029000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-3-031-10986-7_11"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2022]]},"ISBN":["9783031109850","9783031109867"],"references-count":26,"URL":"https:\/\/doi.org\/10.1007\/978-3-031-10986-7_11","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"type":"print","value":"0302-9743"},{"type":"electronic","value":"1611-3349"}],"subject":[],"published":{"date-parts":[[2022]]},"assertion":[{"value":"19 July 2022","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"KSEM","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"International Conference on Knowledge Science, Engineering and Management","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Singapore","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Singapore","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2022","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"6 August 2022","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"8 August 2022","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"15","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"ksem2022","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"https:\/\/ksem22.smart-conf.net\/index.html","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Single-blind","order":1,"name":"type","label":"Type","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"EasyChair","order":2,"name":"conference_management_system","label":"Conference Management System","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"498","order":3,"name":"number_of_submissions_sent_for_review","label":"Number of Submissions Sent for Review","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"169","order":4,"name":"number_of_full_papers_accepted","label":"Number of Full Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"0","order":5,"name":"number_of_short_papers_accepted","label":"Number of Short Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"34% - The value is computed by the equation \"Number of Full Papers Accepted \/ Number of Submissions Sent for Review * 100\" and then rounded to a whole number.","order":6,"name":"acceptance_rate_of_full_papers","label":"Acceptance Rate of Full Papers","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"3","order":7,"name":"average_number_of_reviews_per_paper","label":"Average Number of Reviews per Paper","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"10","order":8,"name":"average_number_of_papers_per_reviewer","label":"Average Number of Papers per Reviewer","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"Yes","order":9,"name":"external_reviewers_involved","label":"External Reviewers Involved","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}}]}}