{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,6,3]],"date-time":"2026-06-03T15:33:38Z","timestamp":1780500818308,"version":"3.54.1"},"reference-count":93,"publisher":"Institute of Electrical and Electronics Engineers (IEEE)","issue":"2","license":[{"start":{"date-parts":[[2024,2,1]],"date-time":"2024-02-01T00:00:00Z","timestamp":1706745600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/legalcode"}],"funder":[{"DOI":"10.13039\/501100004543","name":"China Scholarship Council","doi-asserted-by":"publisher","id":[{"id":"10.13039\/501100004543","id-type":"DOI","asserted-by":"publisher"}]},{"name":"German Research Foundation DFG under project CML TRR 169"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["IEEE Trans. Neural Netw. Learning Syst."],"published-print":{"date-parts":[[2024,2]]},"DOI":"10.1109\/tnnls.2022.3191677","type":"journal-article","created":{"date-parts":[[2022,7,22]],"date-time":"2022-07-22T16:34:25Z","timestamp":1658507665000},"page":"2772-2782","source":"Crossref","is-referenced-by-count":33,"title":["LipSound2: Self-Supervised Pre-Training for Lip-to-Speech Reconstruction and Lip Reading"],"prefix":"10.1109","volume":"35","author":[{"ORCID":"https:\/\/orcid.org\/0000-0001-6694-5355","authenticated-orcid":false,"given":"Leyuan","family":"Qu","sequence":"first","affiliation":[{"name":"Department of Informatics, Knowledge Technology Institute, University of Hamburg, Hamburg, Germany"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-5163-938X","authenticated-orcid":false,"given":"Cornelius","family":"Weber","sequence":"additional","affiliation":[{"name":"Department of Informatics, Knowledge Technology Institute, University of Hamburg, Hamburg, Germany"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-1343-4775","authenticated-orcid":false,"given":"Stefan","family":"Wermter","sequence":"additional","affiliation":[{"name":"Department of Informatics, Knowledge Technology Institute, University of Hamburg, Hamburg, Germany"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"263","reference":[{"key":"ref1","doi-asserted-by":"publisher","DOI":"10.1111\/j.1460-9568.2004.03670.x"},{"key":"ref2","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2017.367"},{"key":"ref3","doi-asserted-by":"publisher","DOI":"10.1109\/tpami.2018.2889052"},{"key":"ref4","doi-asserted-by":"publisher","DOI":"10.21437\/Interspeech.2020-1697"},{"key":"ref5","doi-asserted-by":"publisher","DOI":"10.21437\/Interspeech.2020-1065"},{"key":"ref6","doi-asserted-by":"publisher","DOI":"10.21437\/Interspeech.2016-412"},{"key":"ref7","doi-asserted-by":"publisher","DOI":"10.1109\/ICASSP.2017.7953112"},{"key":"ref8","doi-asserted-by":"publisher","DOI":"10.3758\/BF03206096"},{"key":"ref9","doi-asserted-by":"publisher","DOI":"10.1109\/89.536928"},{"key":"ref10","doi-asserted-by":"publisher","DOI":"10.1016\/j.specom.2009.08.002"},{"key":"ref11","doi-asserted-by":"publisher","DOI":"10.1109\/TBME.2010.2053369"},{"key":"ref12","doi-asserted-by":"publisher","DOI":"10.1109\/TMM.2006.886263"},{"key":"ref13","doi-asserted-by":"publisher","DOI":"10.1109\/ICASSP.2018.8462425"},{"key":"ref14","doi-asserted-by":"publisher","DOI":"10.1145\/3136755.3136787"},{"key":"ref15","volume-title":"Articulatory Phonetics","author":"Gick","year":"2012"},{"key":"ref16","doi-asserted-by":"publisher","DOI":"10.1007\/978-94-009-2037-8_6"},{"key":"ref17","doi-asserted-by":"publisher","DOI":"10.21437\/Interspeech.2020-2136"},{"key":"ref18","doi-asserted-by":"publisher","DOI":"10.1109\/ICASSP.2017.7953127"},{"key":"ref19","doi-asserted-by":"publisher","DOI":"10.1109\/ICASSP.2018.8461856"},{"key":"ref20","doi-asserted-by":"publisher","DOI":"10.21437\/Interspeech.2019-1393"},{"key":"ref21","doi-asserted-by":"publisher","DOI":"10.1016\/S0167-6393(98)00085-5"},{"key":"ref22","doi-asserted-by":"publisher","DOI":"10.1587\/transinf.2015EDP7457"},{"key":"ref23","first-page":"7763","article-title":"Cooperative learning of audio and video models from self-supervised synchronization","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","author":"Korbar"},{"key":"ref24","first-page":"1","article-title":"Learning representations from audio-visual spatial alignment","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","volume":"33","author":"Morgado"},{"key":"ref25","doi-asserted-by":"publisher","DOI":"10.1109\/TASSP.1984.1164317"},{"key":"ref26","doi-asserted-by":"publisher","DOI":"10.1109\/ICASSP.2019.8683143"},{"key":"ref27","doi-asserted-by":"publisher","DOI":"10.21437\/interspeech.2019-1819"},{"key":"ref28","doi-asserted-by":"publisher","DOI":"10.21437\/Interspeech.2015-139"},{"key":"ref29","doi-asserted-by":"publisher","DOI":"10.1109\/TASLP.2017.2716178"},{"key":"ref30","doi-asserted-by":"publisher","DOI":"10.1109\/ICCVW.2017.61"},{"key":"ref31","doi-asserted-by":"publisher","DOI":"10.1109\/TMM.2015.2407694"},{"key":"ref32","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v33i01.33012588"},{"key":"ref33","doi-asserted-by":"publisher","DOI":"10.21437\/Interspeech.2019-1445"},{"key":"ref34","doi-asserted-by":"publisher","DOI":"10.1109\/ICASSP.2018.8461368"},{"key":"ref35","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR42600.2020.01381"},{"key":"ref36","doi-asserted-by":"publisher","DOI":"10.21437\/Interspeech.2020-1026"},{"key":"ref37","doi-asserted-by":"publisher","DOI":"10.1145\/1143844.1143891"},{"key":"ref38","doi-asserted-by":"publisher","DOI":"10.1109\/TASLP.2017.2757263"},{"key":"ref39","doi-asserted-by":"publisher","DOI":"10.1038\/s41598-018-37359-z"},{"key":"ref40","doi-asserted-by":"publisher","DOI":"10.21437\/ICSLP.2002-434"},{"key":"ref41","doi-asserted-by":"publisher","DOI":"10.1109\/ICIP.1998.999008"},{"key":"ref42","article-title":"Towards lipreading sentences with active appearance models","author":"Sterpu","year":"2018","journal-title":"arXiv:1805.11688"},{"key":"ref43","article-title":"Lip reading using convolutional auto encoders as feature extractor","author":"Parekh","year":"2018","journal-title":"arXiv:1805.12371"},{"key":"ref44","first-page":"1","article-title":"LipNet: End-to-end sentence-level lipreading","volume-title":"Proc. GPU Technol. Conf.","author":"Assael"},{"key":"ref45","doi-asserted-by":"publisher","DOI":"10.1109\/ICASSP.2016.7472852"},{"key":"ref46","doi-asserted-by":"publisher","DOI":"10.21437\/Interspeech.2017-85"},{"key":"ref47","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-54184-6_6"},{"key":"ref48","doi-asserted-by":"publisher","DOI":"10.1109\/FG.2019.8756582"},{"key":"ref49","article-title":"Learning spatio-temporal features with two-stream deep 3D CNNs for lipreading","author":"Weng","year":"2019","journal-title":"arXiv:1905.02540"},{"key":"ref50","doi-asserted-by":"publisher","DOI":"10.1109\/ICASSP40776.2020.9053841"},{"key":"ref51","doi-asserted-by":"publisher","DOI":"10.21437\/Interspeech.2017-106"},{"key":"ref52","doi-asserted-by":"publisher","DOI":"10.21437\/Interspeech.2017-421"},{"key":"ref53","doi-asserted-by":"publisher","DOI":"10.21437\/Interspeech.2019-1669"},{"key":"ref54","doi-asserted-by":"publisher","DOI":"10.21437\/Interspeech.2018-1943"},{"key":"ref55","first-page":"1","article-title":"Unsupervised representation learning by predicting image rotations","volume-title":"Proc. ICLR","author":"Gidaris"},{"key":"ref56","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2015.167"},{"key":"ref57","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-46466-4_5"},{"key":"ref58","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-46487-9_40"},{"key":"ref59","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2015.320"},{"key":"ref60","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-46448-0_32"},{"key":"ref61","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-01261-8_24"},{"key":"ref62","article-title":"Efficient estimation of word representations in vector space","author":"Mikolov","year":"2013","journal-title":"arXiv:1301.3781"},{"key":"ref63","article-title":"Improving language understanding by generative pre-training","author":"Radford","year":"2018"},{"key":"ref64","article-title":"BERT: Pre-training of deep bidirectional transformers for language understanding","author":"Devlin","year":"2018","journal-title":"arXiv:1810.04805"},{"key":"ref65","article-title":"ALBERT: A lite BERT for self-supervised learning of language representations","author":"Lan","year":"2019","journal-title":"arXiv:1909.11942"},{"key":"ref66","article-title":"BART: Denoising sequence-to-sequence pre-training for natural language generation, translation, and comprehension","author":"Lewis","year":"2019","journal-title":"arXiv:1910.13461"},{"key":"ref67","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2017.73"},{"key":"ref68","doi-asserted-by":"publisher","DOI":"10.1109\/ICASSP.2019.8682524"},{"key":"ref69","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-01246-5_27"},{"key":"ref70","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV48922.2021.00791"},{"key":"ref71","doi-asserted-by":"publisher","DOI":"10.1109\/LSP.2016.2603342"},{"key":"ref72","first-page":"577","article-title":"Attention-based models for speech recognition","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","volume":"28","author":"Chorowski"},{"key":"ref73","first-page":"10215","article-title":"Glow: Generative flow with invertible 1 \u00d7 1 convolutions","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","author":"Kingma"},{"key":"ref74","first-page":"125","article-title":"WaveNet: A generative model for raw audio","volume-title":"Proc. 9th ISCA Speech Synth. Workshop (SSW)","author":"van den Oord"},{"key":"ref75","volume-title":"The LJ Speech Dataset","author":"Ito","year":"2017"},{"key":"ref76","first-page":"187","article-title":"KenLM: Faster and smaller language model queries","volume-title":"Proc. 6th Workshop Stat. Mach. Transl.","author":"Heafield"},{"key":"ref77","doi-asserted-by":"publisher","DOI":"10.1613\/jair.282"},{"key":"ref78","doi-asserted-by":"publisher","DOI":"10.21437\/Interspeech.2018-1929"},{"key":"ref79","doi-asserted-by":"publisher","DOI":"10.1121\/1.2229005"},{"key":"ref80","doi-asserted-by":"publisher","DOI":"10.1145\/3338533.3366579"},{"key":"ref81","doi-asserted-by":"publisher","DOI":"10.1109\/ICASSP.2015.7178964"},{"key":"ref82","article-title":"AISHELL-2: Transforming Mandarin ASR research into industrial scale","author":"Du","year":"2018","journal-title":"arXiv:1808.10583"},{"key":"ref83","doi-asserted-by":"publisher","DOI":"10.1109\/ICASSP.2001.941023"},{"key":"ref84","doi-asserted-by":"publisher","DOI":"10.1109\/TASLP.2016.2585878"},{"key":"ref85","first-page":"1171","article-title":"Scheduled sampling for sequence prediction with recurrent neural networks","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","author":"Bengio"},{"key":"ref86","first-page":"249","article-title":"Understanding the difficulty of training deep feedforward neural networks","volume-title":"Proc. 13th Int. Conf. Artif. Intell. Statist.","author":"Glorot"},{"key":"ref87","article-title":"Comparing phonemes and visemes with DNN-based lipreading","author":"Thangthai","year":"2018","journal-title":"arXiv:1805.02924"},{"key":"ref88","doi-asserted-by":"publisher","DOI":"10.1109\/FG47880.2020.00010"},{"key":"ref89","doi-asserted-by":"publisher","DOI":"10.1109\/ICIP40778.2020.9190780"},{"key":"ref90","doi-asserted-by":"publisher","DOI":"10.1109\/FG.2018.00088"},{"key":"ref91","doi-asserted-by":"publisher","DOI":"10.1145\/3394171.3413623"},{"key":"ref92","doi-asserted-by":"publisher","DOI":"10.21437\/Interspeech.2019-2618"},{"key":"ref93","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v34i04.6174"}],"container-title":["IEEE Transactions on Neural Networks and Learning Systems"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx7\/5962385\/10422842\/09837888.pdf?arnumber=9837888","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,2,16]],"date-time":"2024-02-16T19:11:54Z","timestamp":1708110714000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/9837888\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,2]]},"references-count":93,"journal-issue":{"issue":"2"},"URL":"https:\/\/doi.org\/10.1109\/tnnls.2022.3191677","relation":{},"ISSN":["2162-237X","2162-2388"],"issn-type":[{"value":"2162-237X","type":"print"},{"value":"2162-2388","type":"electronic"}],"subject":[],"published":{"date-parts":[[2024,2]]}}}