{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,8]],"date-time":"2026-04-08T16:57:14Z","timestamp":1775667434722,"version":"3.50.1"},"reference-count":42,"publisher":"IEEE","license":[{"start":{"date-parts":[[2021,12,13]],"date-time":"2021-12-13T00:00:00Z","timestamp":1639353600000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2021,12,13]],"date-time":"2021-12-13T00:00:00Z","timestamp":1639353600000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2021,12,13]]},"DOI":"10.1109\/asru51503.2021.9688253","type":"proceedings-article","created":{"date-parts":[[2022,2,3]],"date-time":"2022-02-03T20:31:00Z","timestamp":1643920260000},"page":"244-250","source":"Crossref","is-referenced-by-count":224,"title":["w2v-BERT: Combining Contrastive Learning and Masked Language Modeling for Self-Supervised Speech Pre-Training"],"prefix":"10.1109","author":[{"given":"Yu-An","family":"Chung","sequence":"first","affiliation":[{"name":"MIT Computer Science and Artificial Intelligence Laboratory"}]},{"given":"Yu","family":"Zhang","sequence":"additional","affiliation":[{"name":"Google Brain"}]},{"given":"Wei","family":"Han","sequence":"additional","affiliation":[{"name":"Google Brain"}]},{"given":"Chung-Cheng","family":"Chiu","sequence":"additional","affiliation":[{"name":"Google Brain"}]},{"given":"James","family":"Qin","sequence":"additional","affiliation":[{"name":"Google Brain"}]},{"given":"Ruoming","family":"Pang","sequence":"additional","affiliation":[{"name":"Google Brain"}]},{"given":"Yonghui","family":"Wu","sequence":"additional","affiliation":[{"name":"Google Brain"}]}],"member":"263","reference":[{"key":"ref39","doi-asserted-by":"publisher","DOI":"10.21437\/Interspeech.2019-2680"},{"key":"ref38","article-title":"Adafactor: Adaptive learning rates with sublinear memory cost","author":"shazeer","year":"0","journal-title":"ICML"},{"key":"ref33","article-title":"Batch normalization: Accelerating deep network training by reducing internal covariate shift","author":"ioffe","year":"0","journal-title":"ICML"},{"key":"ref32","article-title":"Searching for activation functions","author":"ramachandran","year":"2017","journal-title":"ArXiv Preprint"},{"key":"ref31","doi-asserted-by":"publisher","DOI":"10.1162\/neco.1997.9.8.1735"},{"key":"ref30","article-title":"Sequence transduction with recurrent neural networks","author":"graves","year":"2012","journal-title":"ArXiv Preprint"},{"key":"ref37","article-title":"Adam: A method for stochastic optimization","author":"kingma","year":"0","journal-title":"ICLRE"},{"key":"ref36","doi-asserted-by":"publisher","DOI":"10.1109\/ICASSP.2012.6289079"},{"key":"ref35","doi-asserted-by":"publisher","DOI":"10.1109\/ICASSP.2015.7178964"},{"key":"ref34","article-title":"Libri-light: A benchmark for ASR with limited or no super-vision","author":"kahn","year":"0","journal-title":"ICASSP"},{"key":"ref10","article-title":"Representation learning with contrastive predictive coding","author":"van den oord","year":"2018","journal-title":"ArXiv Preprint"},{"key":"ref40","doi-asserted-by":"publisher","DOI":"10.1109\/ICASSP40776.2020.9053205"},{"key":"ref11","doi-asserted-by":"publisher","DOI":"10.21437\/Interspeech.2019-1473"},{"key":"ref12","doi-asserted-by":"publisher","DOI":"10.21437\/Interspeech.2019-1873"},{"key":"ref13","doi-asserted-by":"publisher","DOI":"10.1109\/ICASSP40776.2020.9053176"},{"key":"ref14","doi-asserted-by":"publisher","DOI":"10.1109\/ICASSP40776.2020.9054438"},{"key":"ref15","doi-asserted-by":"publisher","DOI":"10.1109\/ICASSP40776.2020.9054458"},{"key":"ref16","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.acl-main.213"},{"key":"ref17","doi-asserted-by":"publisher","DOI":"10.1109\/TASLP.2021.3095662"},{"key":"ref18","doi-asserted-by":"publisher","DOI":"10.1109\/ICASSP40776.2020.9053541"},{"key":"ref19","article-title":"DeCoAR 2.0: Deep contextualized acoustic representations with vector quantization","author":"ling","year":"2020","journal-title":"ArXiv Preprint"},{"key":"ref28","article-title":"Attention is all you need","author":"vaswani","year":"0","journal-title":"NIPS"},{"key":"ref4","doi-asserted-by":"publisher","DOI":"10.1109\/ICASSP40776.2020.9054295"},{"key":"ref27","article-title":"Effectiveness of self-supervised pre-training for speech recognition","author":"baevski","year":"2019","journal-title":"ArXiv Preprint"},{"key":"ref3","doi-asserted-by":"publisher","DOI":"10.1109\/TIT.1965.1053799"},{"key":"ref6","doi-asserted-by":"publisher","DOI":"10.1109\/ICASSP.2019.8682172"},{"key":"ref29","doi-asserted-by":"publisher","DOI":"10.21437\/Interspeech.2020-3015"},{"key":"ref5","article-title":"End-to-end ASR: from supervised to semi-supervised learning with modern architectures","author":"synnaeve","year":"0","journal-title":"ICML SAS Workshop"},{"key":"ref8","doi-asserted-by":"publisher","DOI":"10.21437\/Interspeech.2009-86"},{"key":"ref7","article-title":"Lessons from building acoustic models with a million hours of speech","author":"hari","year":"0","journal-title":"ICASSP"},{"key":"ref2","doi-asserted-by":"publisher","DOI":"10.3115\/981658.981684"},{"key":"ref9","article-title":"Utilizing un-transcribed training data to improve performance","author":"zavaliagkos","year":"0","journal-title":"DARPA Broadcast News Transcription and Understanding Workshop"},{"key":"ref1","doi-asserted-by":"publisher","DOI":"10.3115\/1119355.1119369"},{"key":"ref20","article-title":"Representation learning for sequence data with deep autoencoding predictive components","author":"bai","year":"0","journal-title":"ICLRE"},{"key":"ref22","doi-asserted-by":"publisher","DOI":"10.1109\/ICASSP39728.2021.9414641"},{"key":"ref21","article-title":"Pushing the limits of semi-supervised learning for automatic speech recognition","author":"yu","year":"2020","journal-title":"ArXiv Preprint"},{"key":"ref42","article-title":"Scaling end-to-end models for large-scale multilingual ASR","author":"bo","year":"2021","journal-title":"ArXiv Preprint"},{"key":"ref24","article-title":"BERT: Pre-training of deep bidirectional transformers for language understanding","author":"devlin","year":"0","journal-title":"NAACL"},{"key":"ref41","doi-asserted-by":"publisher","DOI":"10.21437\/Interspeech.2020-1470"},{"key":"ref23","article-title":"wav2vec 2.0: A framework for self-supervised learning of speech representations","author":"baevski","year":"0","journal-title":"NeurIPS"},{"key":"ref26","article-title":"vq-wav2vec: Self-supervised learning of discrete speech representations","author":"baevski","year":"0","journal-title":"ICLRE"},{"key":"ref25","doi-asserted-by":"publisher","DOI":"10.1109\/TASLP.2021.3122291"}],"event":{"name":"2021 IEEE Automatic Speech Recognition and Understanding Workshop (ASRU)","location":"Cartagena, Colombia","start":{"date-parts":[[2021,12,13]]},"end":{"date-parts":[[2021,12,17]]}},"container-title":["2021 IEEE Automatic Speech Recognition and Understanding Workshop (ASRU)"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx7\/9687821\/9687855\/09688253.pdf?arnumber=9688253","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2022,5,16]],"date-time":"2022-05-16T20:42:19Z","timestamp":1652733739000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/9688253\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2021,12,13]]},"references-count":42,"URL":"https:\/\/doi.org\/10.1109\/asru51503.2021.9688253","relation":{},"subject":[],"published":{"date-parts":[[2021,12,13]]}}}