{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,3,26]],"date-time":"2025-03-26T19:06:02Z","timestamp":1743015962572,"version":"3.40.3"},"publisher-location":"Cham","reference-count":41,"publisher":"Springer International Publishing","isbn-type":[{"type":"print","value":"9783030835262"},{"type":"electronic","value":"9783030835279"}],"license":[{"start":{"date-parts":[[2021,1,1]],"date-time":"2021-01-01T00:00:00Z","timestamp":1609459200000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"},{"start":{"date-parts":[[2021,1,1]],"date-time":"2021-01-01T00:00:00Z","timestamp":1609459200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2021]]},"DOI":"10.1007\/978-3-030-83527-9_35","type":"book-chapter","created":{"date-parts":[[2021,8,29]],"date-time":"2021-08-29T23:04:59Z","timestamp":1630278299000},"page":"413-424","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":1,"title":["Exploiting Large-Scale Teacher-Student Training for On-Device Acoustic Models"],"prefix":"10.1007","author":[{"given":"Jing","family":"Liu","sequence":"first","affiliation":[]},{"given":"Rupak Vignesh","family":"Swaminathan","sequence":"additional","affiliation":[]},{"given":"Sree Hari Krishnan","family":"Parthasarathi","sequence":"additional","affiliation":[]},{"given":"Chunchuan","family":"Lyu","sequence":"additional","affiliation":[]},{"given":"Athanasios","family":"Mouchtaris","sequence":"additional","affiliation":[]},{"given":"Siegfried","family":"Kunzmann","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2021,8,30]]},"reference":[{"key":"35_CR1","unstructured":"Amodei, D., Ananthanarayanan, S., et al.: Deep speech 2: end-to-end speech recognition in English and Mandarin. In: Procedings of ICML (2016)"},{"key":"35_CR2","unstructured":"Ba, J., Caruana, R.: Do deep nets really need to be deep? In: Advances in Neural Information Processing Systems, pp. 2654\u20132662 (2014)"},{"key":"35_CR3","unstructured":"B\u00e9gin, L., Germain, P., Laviolette, F., Roy, J.F.: PAC-Bayesian theory for transductive learning. In: Proceedings of of AISTATS (2014)"},{"key":"35_CR4","doi-asserted-by":"crossref","unstructured":"Chen, K., Huo, Q.: Scalable training of deep learning machines by incremental block training with intra-block parallel optimization and blockwise model-update filtering. In: Proceedings of ICASSP (2016)","DOI":"10.1109\/ICASSP.2016.7472805"},{"key":"35_CR5","doi-asserted-by":"publisher","unstructured":"Chen, L., Leutnant, V.: Acoustic model bootstrapping using semi-supervised learning. In: Proceedings Interspeech 2019, pp. 3198\u20133202 (2019). https:\/\/doi.org\/10.21437\/Interspeech.2019-2818. http:\/\/dx.doi.org\/10.21437\/Interspeech.2019-2818","DOI":"10.21437\/Interspeech.2019-2818"},{"key":"35_CR6","doi-asserted-by":"crossref","unstructured":"Chen, Y., Wang, W., Wang, C.: Semi-supervised ASR by end-to-end self-training (2020)","DOI":"10.21437\/Interspeech.2020-1280"},{"key":"35_CR7","doi-asserted-by":"crossref","unstructured":"Garimella, S., Mandal, A., Strom, N., Hoffmeister, B., Matsoukas, S., Parthasarathi, S.H.K.: Robust I-vector based adaptation of DNN acoustic model for speech recognition. In: Proceedings of Interspeech (2015)","DOI":"10.21437\/Interspeech.2015-605"},{"key":"35_CR8","unstructured":"Guo, C., Pleiss, G., Sun, Y., Weinberger, K.Q.: On calibration of modern neural networks. arXiv preprint arXiv:1706.04599 (2017)"},{"key":"35_CR9","unstructured":"Hinton, G., Vinyals, O., Dean, J.: Distilling the knowledge in a neural network. arXiv preprint arXiv:1503.02531 (2015)"},{"issue":"8","key":"35_CR10","doi-asserted-by":"publisher","first-page":"1735","DOI":"10.1162\/neco.1997.9.8.1735","volume":"9","author":"S Hochreiter","year":"1997","unstructured":"Hochreiter, S., Schmidhuber, J.: Long short-term memory. Neural Comput. 9(8), 1735\u20131780 (1997)","journal-title":"Neural Comput."},{"key":"35_CR11","doi-asserted-by":"crossref","unstructured":"Huang, Y., Wang, Y., Gong, Y.: Semi-supervised training in deep learning acoustic model. In: Proceedings of Interspeech (2016)","DOI":"10.21437\/Interspeech.2016-1596"},{"key":"35_CR12","doi-asserted-by":"crossref","unstructured":"Huang, Y., Yu, D., Gong, Y., Liu, C.: Semi-supervised GMM and DNN acoustic model training with multi-system combination and confidence re-calibration. In: Proceedings of Interspeech (2013)","DOI":"10.21437\/Interspeech.2013-551"},{"key":"35_CR13","doi-asserted-by":"publisher","unstructured":"Kahn, J., Lee, A., Hannun, A.: Self-training for end-to-end speech recognition. In: ICASSP 2020\u20132020 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP), May 2020. https:\/\/doi.org\/10.1109\/icassp40776.2020.9054295. http:\/\/dx.doi.org\/10.1109\/ICASSP40776.2020.9054295","DOI":"10.1109\/icassp40776.2020.9054295"},{"key":"35_CR14","doi-asserted-by":"crossref","unstructured":"Kemp, T., Waibel, A.: Unsupervised training of a speech recognizer: recent experiments. In: Proceedings of Eurospeech (1999)","DOI":"10.21437\/Eurospeech.1999-599"},{"key":"35_CR15","doi-asserted-by":"publisher","unstructured":"Kingsbury, B.: Lattice-based optimization of sequence classification criteria for neural-network acoustic modeling. In: 2009 IEEE International Conference on Acoustics, Speech and Signal Processing, pp. 3761\u20133764 (2009). https:\/\/doi.org\/10.1109\/ICASSP.2009.4960445","DOI":"10.1109\/ICASSP.2009.4960445"},{"key":"35_CR16","doi-asserted-by":"crossref","unstructured":"Kurata, G., Audhkhasi, K.: Guiding CTC posterior spike timings for improved posterior fusion and knowledge distillation. CoRR abs\/1904.08311 (2019). http:\/\/arxiv.org\/abs\/1904.08311","DOI":"10.21437\/Interspeech.2019-1952"},{"key":"35_CR17","doi-asserted-by":"publisher","first-page":"115","DOI":"10.1006\/csla.2001.0186","volume":"16","author":"L Lamel","year":"2002","unstructured":"Lamel, L., Gauvain, J.L., Adda, G.: Lightly supervised and unsupervised acoustic model training. Comput. Speech Lang. 16, 115\u2013129 (2002)","journal-title":"Comput. Speech Lang."},{"key":"35_CR18","doi-asserted-by":"publisher","unstructured":"Li, J., Mohamed, A., Zweig, G., Gong, Y.: LSTM time and frequency recurrence for automatic speech recognition. In: 2015 IEEE Workshop on Automatic Speech Recognition and Understanding (ASRU), pp. 187\u2013191 (2015). https:\/\/doi.org\/10.1109\/ASRU.2015.7404793","DOI":"10.1109\/ASRU.2015.7404793"},{"key":"35_CR19","doi-asserted-by":"crossref","unstructured":"Li, J., Zhao, R., Huang, J.T., Gong, Y.: Learning small-size DNN with output-distribution-based criteria. In: Proceedings of Interspeech (2014)","DOI":"10.21437\/Interspeech.2014-432"},{"key":"35_CR20","doi-asserted-by":"publisher","unstructured":"Li, J., et al.: High-accuracy and low-latency speech recognition with two-head contextual layer trajectory LSTM model. In: ICASSP 2020\u20132020 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP), pp. 7699\u20137703 (2020). https:\/\/doi.org\/10.1109\/ICASSP40776.2020.9054387","DOI":"10.1109\/ICASSP40776.2020.9054387"},{"key":"35_CR21","doi-asserted-by":"crossref","unstructured":"Ma, J., Matsoukas, S., Kimball, O., Schwartz, R.: Unsupervised training on large amounts of broadcast news data. In: Proceedings of ICASSP (2006)","DOI":"10.1109\/ICASSP.2007.366244"},{"key":"35_CR22","doi-asserted-by":"publisher","unstructured":"Manohar, V., Ghahremani, P., Povey, D., Khudanpur, S.: A teacher-student learning approach for unsupervised domain adaptation of sequence-trained ASR models. In: 2018 IEEE Spoken Language Technology Workshop (SLT), pp. 250\u2013257 (2018). https:\/\/doi.org\/10.1109\/SLT.2018.8639635","DOI":"10.1109\/SLT.2018.8639635"},{"key":"35_CR23","doi-asserted-by":"crossref","unstructured":"Mirzadeh, S.I., Farajtabar, M., Li, A., Ghasemzadeh, H.: Improved knowledge distillation via teacher assistant: bridging the gap between student and teacher. arXiv preprint arXiv:1902.03393 (2019)","DOI":"10.1609\/aaai.v34i04.5963"},{"key":"35_CR24","doi-asserted-by":"publisher","unstructured":"Moriya, T., et al.: Efficient building strategy with knowledge distillation for small-footprint acoustic models. In: 2018 IEEE Spoken Language Technology Workshop (SLT), pp. 21\u201328 (2018). https:\/\/doi.org\/10.1109\/SLT.2018.8639545","DOI":"10.1109\/SLT.2018.8639545"},{"key":"35_CR25","doi-asserted-by":"crossref","unstructured":"Munim, R.M., Inoue, N., Shinoda, K.: Sequence-level knowledge distillation for model compression of attention-based sequence-to-sequence speech recognition. In: ICASSP 2019\u20132019 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP), pp. 6151\u20136155. IEEE (2019)","DOI":"10.1109\/ICASSP.2019.8683171"},{"key":"35_CR26","doi-asserted-by":"crossref","unstructured":"Parthasarathi, S.H.K., Hoffmeister, B., Matsoukas, S., Mandal, A., Strom, N., Garimella, S.: fMLLR based feature-space speaker adaptation of DNN acoustic models. In: Proceedings of Interspeech (2015)","DOI":"10.21437\/Interspeech.2015-720"},{"key":"35_CR27","doi-asserted-by":"crossref","unstructured":"Parthasarathi, S.H.K., Sivakrishnan, N., Ladkat, P., Strom, N.: Realizing petabyte scale acoustic modeling. IEEE J. Emerg. Sel. Top. Circuits Syst. 9, 422-432 (2019)","DOI":"10.1109\/JETCAS.2019.2912353"},{"key":"35_CR28","unstructured":"Parthasarathi, S.H.K., Strom, N.: Lessons from building acoustic models from a million hours of speech. In: Proceedings of ICASSP (2019)"},{"key":"35_CR29","doi-asserted-by":"crossref","unstructured":"Pundak, G., Sainath, T.: Lower frame rate neural network acoustic models. In: Proceedings of Interspeech (2016)","DOI":"10.21437\/Interspeech.2016-275"},{"key":"35_CR30","doi-asserted-by":"crossref","unstructured":"Sak, H., Senior, A., Rao, K., Beaufays, F.: Fast and accurate recurrent neural network acoustic models for speech recognition. In: INTERSPEECH (2015)","DOI":"10.21437\/Interspeech.2015-350"},{"key":"35_CR31","doi-asserted-by":"publisher","first-page":"637","DOI":"10.1109\/JIOT.2016.2579198","volume":"3","author":"W Shi","year":"2016","unstructured":"Shi, W., Cao, J., Zhang, Q., Li, Y., Xu, L.: Edge computing: vision and challenges. IEEE Internet Things J. 3, 637\u2013646 (2016)","journal-title":"IEEE Internet Things J."},{"key":"35_CR32","doi-asserted-by":"crossref","unstructured":"Siu, M.H., Gish, H., Richardson, F.: Improved estimation, evaluation and applications of confidence measures for speech recognition. In: Proceedings of European Conference on Speech Communication and Technology (1997)","DOI":"10.21437\/Eurospeech.1997-282"},{"key":"35_CR33","doi-asserted-by":"crossref","unstructured":"Strom, N.: Scalable distributed DNN training using commodity GPU cloud computing. In: Proceedings of Interspeech (2015)","DOI":"10.21437\/Interspeech.2015-354"},{"key":"35_CR34","doi-asserted-by":"publisher","unstructured":"Watanabe, S., Hori, T., Le Roux, J., Hershey, J.R.: Student-teacher network learning with enhanced features. In: 2017 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP), pp. 5275\u20135279 (2017). https:\/\/doi.org\/10.1109\/ICASSP.2017.7953163","DOI":"10.1109\/ICASSP.2017.7953163"},{"key":"35_CR35","doi-asserted-by":"crossref","unstructured":"Waters, A., Chebotar, Y.: Distilling knowledge from ensembles of neural networks for speech recognition. In: Interspeech (2016)","DOI":"10.21437\/Interspeech.2016-1190"},{"key":"35_CR36","doi-asserted-by":"crossref","unstructured":"Weninger, F., Mana, F., Gemello, R., Andres-Ferrer, J., Zhan, P.: Semi-supervised learning with data augmentation for end-to-end ASR (2020)","DOI":"10.21437\/Interspeech.2020-1337"},{"key":"35_CR37","doi-asserted-by":"crossref","unstructured":"Wong, J.H.M., Gales, M.: Sequence student-teacher training of deep neural networks. In: INTERSPEECH (2016)","DOI":"10.21437\/Interspeech.2016-911"},{"key":"35_CR38","unstructured":"Xie, Q., Dai, Z., Hovy, E., Luong, M.T., Le, Q.V.: Unsupervised data augmentation for consistency training (2020)"},{"key":"35_CR39","unstructured":"Yanzhang, H., et al.: Streaming end-to-end speech recognition for mobile devices. arXiv preprint arXiv:1811.06621 (2018)"},{"key":"35_CR40","doi-asserted-by":"publisher","first-page":"6900","DOI":"10.1109\/ACCESS.2017.2778504","volume":"6","author":"W Yu","year":"2017","unstructured":"Yu, W., Liang, F., He, X., Hatcher, W.G., Lu, C., Lin, J., Yang, X.: A survey on the edge computing for the Internet of Things. IEEE Access 6, 6900\u20136919 (2017)","journal-title":"IEEE Access"},{"key":"35_CR41","unstructured":"Zhang, Y., et al.: Pushing the limits of semi-supervised learning for automatic speech recognition (2020)"}],"container-title":["Lecture Notes in Computer Science","Text, Speech, and Dialogue"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-030-83527-9_35","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2023,1,8]],"date-time":"2023-01-08T07:42:17Z","timestamp":1673163737000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-3-030-83527-9_35"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2021]]},"ISBN":["9783030835262","9783030835279"],"references-count":41,"URL":"https:\/\/doi.org\/10.1007\/978-3-030-83527-9_35","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"type":"print","value":"0302-9743"},{"type":"electronic","value":"1611-3349"}],"subject":[],"published":{"date-parts":[[2021]]},"assertion":[{"value":"30 August 2021","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"TSD","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"International Conference on Text, Speech, and Dialogue","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Olomouc","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Czech Republic","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2021","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"6 September 2021","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"9 September 2021","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"24","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"tsd2021","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"http:\/\/www.kiv.zcu.cz\/tsd2021\/","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Double-blind","order":1,"name":"type","label":"Type","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"TSDEngine 3.2","order":2,"name":"conference_management_system","label":"Conference Management System","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"101","order":3,"name":"number_of_submissions_sent_for_review","label":"Number of Submissions Sent for Review","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"29","order":4,"name":"number_of_full_papers_accepted","label":"Number of Full Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"17","order":5,"name":"number_of_short_papers_accepted","label":"Number of Short Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"29% - The value is computed by the equation \"Number of Full Papers Accepted \/ Number of Submissions Sent for Review * 100\" and then rounded to a whole number.","order":6,"name":"acceptance_rate_of_full_papers","label":"Acceptance Rate of Full Papers","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"3","order":7,"name":"average_number_of_reviews_per_paper","label":"Average Number of Reviews per Paper","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"2,93","order":8,"name":"average_number_of_papers_per_reviewer","label":"Average Number of Papers per Reviewer","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"Yes","order":9,"name":"external_reviewers_involved","label":"External Reviewers Involved","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}}]}}