{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,11,10]],"date-time":"2025-11-10T21:11:14Z","timestamp":1762809074307,"version":"3.28.0"},"reference-count":43,"publisher":"IEEE","license":[{"start":{"date-parts":[[2021,6,6]],"date-time":"2021-06-06T00:00:00Z","timestamp":1622937600000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2021,6,6]],"date-time":"2021-06-06T00:00:00Z","timestamp":1622937600000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2021,6,6]]},"DOI":"10.1109\/icassp39728.2021.9414539","type":"proceedings-article","created":{"date-parts":[[2021,5,13]],"date-time":"2021-05-13T19:53:45Z","timestamp":1620935625000},"page":"6538-6542","source":"Crossref","is-referenced-by-count":17,"title":["A Further Study of Unsupervised Pretraining for Transformer Based Speech Recognition"],"prefix":"10.1109","author":[{"given":"Dongwei","family":"Jiang","sequence":"first","affiliation":[{"name":"Didi Chuxing,AI Labs,Beijing,China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Wubo","family":"Li","sequence":"additional","affiliation":[{"name":"Didi Chuxing,AI Labs,Beijing,China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Ruixiong","family":"Zhang","sequence":"additional","affiliation":[{"name":"Didi Chuxing,AI Labs,Beijing,China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Miao","family":"Cao","sequence":"additional","affiliation":[{"name":"Didi Chuxing,AI Labs,Beijing,China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Ne","family":"Luo","sequence":"additional","affiliation":[{"name":"Didi Chuxing,AI Labs,Beijing,China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Yang","family":"Han","sequence":"additional","affiliation":[{"name":"Didi Chuxing,AI Labs,Beijing,China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Wei","family":"Zou","sequence":"additional","affiliation":[{"name":"Didi Chuxing,AI Labs,Beijing,China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Kun","family":"Han","sequence":"additional","affiliation":[{"name":"Didi Chuxing,AI Labs,Beijing,China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Xiangang","family":"Li","sequence":"additional","affiliation":[{"name":"Didi Chuxing,AI Labs,Beijing,China"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"263","reference":[{"doi-asserted-by":"publisher","key":"ref39","DOI":"10.1109\/ICASSP.2019.8682539"},{"doi-asserted-by":"publisher","key":"ref38","DOI":"10.1109\/ASRU46091.2019.9003750"},{"key":"ref33","article-title":"Fine-grained analysis of sentence embeddings using auxiliary prediction tasks","author":"adi","year":"2017","journal-title":"ICLRE"},{"doi-asserted-by":"publisher","key":"ref32","DOI":"10.18653\/v1\/D16-1159"},{"doi-asserted-by":"publisher","key":"ref31","DOI":"10.18653\/v1\/N19-1112"},{"key":"ref30","article-title":"How transferable are features in deep neural networks?","author":"yosinski","year":"2014","journal-title":"NIPS"},{"doi-asserted-by":"publisher","key":"ref37","DOI":"10.21437\/Interspeech.2018-2057"},{"doi-asserted-by":"publisher","key":"ref36","DOI":"10.1109\/ISCSLP.2018.8706661"},{"key":"ref35","article-title":"The fisher corpus: a resource for the next generations of speech-to-text","author":"cieri","year":"2004","journal-title":"LREC"},{"doi-asserted-by":"publisher","key":"ref34","DOI":"10.1109\/ICASSP.2015.7178964"},{"key":"ref10","article-title":"An unsupervised autoregressive model for speech representation learning","author":"yu-an","year":"2019","journal-title":"InterSpeech"},{"key":"ref40","article-title":"Transformer-transducer: End-to-end speech recognition with self-attention","volume":"abs 1910 12977","author":"yeh","year":"2019","journal-title":"CoRR"},{"doi-asserted-by":"publisher","key":"ref11","DOI":"10.1109\/ICASSP40776.2020.9054438"},{"doi-asserted-by":"publisher","key":"ref12","DOI":"10.1109\/ICASSP40776.2020.9054458"},{"doi-asserted-by":"publisher","key":"ref13","DOI":"10.1109\/ICASSP40776.2020.9053541"},{"key":"ref14","article-title":"Speech-xlnet: Unsupervised acoustic model pretraining for self-attention networks","volume":"abs 1910 10387","author":"song","year":"2019","journal-title":"CoRR"},{"key":"ref15","article-title":"Improving transformer-based speech recognition using unsupervised pre-training","volume":"abs 1910 9932","author":"jiang","year":"2019","journal-title":"CoRR"},{"key":"ref16","article-title":"Audio ALBERT: A lite BERT for self-supervised learning of audio representation","volume":"abs 2005 8575","author":"chi","year":"2020","journal-title":"CoRR"},{"key":"ref17","article-title":"TERA: self-supervised learning of transformer encoder representation for speech","volume":"abs 2007 6028","author":"liu","year":"2020","journal-title":"CoRR"},{"key":"ref18","article-title":"Automatic speech recognition and speech variability: A review","volume":"49","author":"mohamed","year":"2007","journal-title":"Speech Communication"},{"key":"ref19","article-title":"Effect of speaking style on lvcsr performance","author":"mitch","year":"1996","journal-title":"ICSLP"},{"key":"ref28","article-title":"An empirical investigation of catastrophic forgeting in gradient-based neural networks","author":"goodfellow","year":"2014","journal-title":"ICLRE"},{"key":"ref4","article-title":"Learning speaker representations with mutual information","author":"mirco","year":"2019","journal-title":"InterSpeech"},{"key":"ref27","article-title":"Unified language model pre-training for natural language understanding and generation","author":"dong","year":"2019","journal-title":"NeurIPS"},{"key":"ref3","article-title":"Effectiveness of self-supervised pre-training for speech recognition","volume":"abs 1911 3912","author":"baevski","year":"2019","journal-title":"CoRR"},{"key":"ref6","article-title":"Learning problem-agnostic speech representations from multiple self-supervised tasks","author":"santiago","year":"2019","journal-title":"InterSpeech"},{"doi-asserted-by":"publisher","key":"ref29","DOI":"10.1073\/pnas.1611835114"},{"key":"ref5","article-title":"wav2vec: Unsupervised pre-training for speech recognition","author":"steffen","year":"2019","journal-title":"InterSpeech"},{"doi-asserted-by":"publisher","key":"ref8","DOI":"10.1109\/ICASSP40776.2020.9054548"},{"key":"ref7","article-title":"Learning robust and multilingual speech representations","volume":"abs 2001 11128","author":"kawakami","year":"2020","journal-title":"CoRR"},{"key":"ref2","article-title":"vq-wav2vec: Self-supervised learning of discrete speech representations","author":"baevski","year":"2020","journal-title":"ICLRE"},{"doi-asserted-by":"publisher","key":"ref9","DOI":"10.21437\/Interspeech.2019-1582"},{"key":"ref1","article-title":"Representation learning with contrastive predictive coding","volume":"abs 1807 3748","author":"van den oord","year":"2018","journal-title":"CoRR"},{"doi-asserted-by":"publisher","key":"ref20","DOI":"10.18653\/v1\/P18-1031"},{"doi-asserted-by":"publisher","key":"ref22","DOI":"10.1007\/978-3-030-32381-3_16"},{"doi-asserted-by":"publisher","key":"ref21","DOI":"10.18653\/v1\/N19-1213"},{"doi-asserted-by":"publisher","key":"ref42","DOI":"10.21437\/Interspeech.2019-2680"},{"doi-asserted-by":"publisher","key":"ref24","DOI":"10.1109\/ICASSP.2016.7472621"},{"key":"ref41","article-title":"Attention is all you need","author":"vaswani","year":"2017","journal-title":"NIPS"},{"key":"ref23","article-title":"Roberta: A robustly optimized BERT pretraining approach","volume":"abs 1907 11692","author":"liu","year":"2019","journal-title":"CoRR"},{"doi-asserted-by":"publisher","key":"ref26","DOI":"10.1109\/ICASSP40776.2020.9053165"},{"key":"ref43","article-title":"Joint ctc-attention based end-to-end speech recognition using multi-task learning","author":"suyoun","year":"2017","journal-title":"ICASSP"},{"doi-asserted-by":"publisher","key":"ref25","DOI":"10.1109\/ICASSP.2018.8462366"}],"event":{"name":"ICASSP 2021 - 2021 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)","start":{"date-parts":[[2021,6,6]]},"location":"Toronto, ON, Canada","end":{"date-parts":[[2021,6,11]]}},"container-title":["ICASSP 2021 - 2021 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx7\/9413349\/9413350\/09414539.pdf?arnumber=9414539","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2022,8,3]],"date-time":"2022-08-03T00:19:36Z","timestamp":1659485976000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/9414539\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2021,6,6]]},"references-count":43,"URL":"https:\/\/doi.org\/10.1109\/icassp39728.2021.9414539","relation":{},"subject":[],"published":{"date-parts":[[2021,6,6]]}}}