{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,22]],"date-time":"2026-04-22T18:38:39Z","timestamp":1776883119690,"version":"3.51.2"},"reference-count":103,"publisher":"Institute of Electrical and Electronics Engineers (IEEE)","license":[{"start":{"date-parts":[[2024,1,1]],"date-time":"2024-01-01T00:00:00Z","timestamp":1704067200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/ieeexplore.ieee.org\/Xplorehelp\/downloads\/license-information\/IEEE.html"},{"start":{"date-parts":[[2024,1,1]],"date-time":"2024-01-01T00:00:00Z","timestamp":1704067200000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2024,1,1]],"date-time":"2024-01-01T00:00:00Z","timestamp":1704067200000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"}],"funder":[{"name":"China NSFC Projects","award":["62122050"],"award-info":[{"award-number":["62122050"]}]},{"name":"China NSFC Projects","award":["62071288"],"award-info":[{"award-number":["62071288"]}]},{"DOI":"10.13039\/501100003399","name":"Science and Technology Commission of Shanghai Municipality","doi-asserted-by":"publisher","award":["2021SHZDZX0102"],"award-info":[{"award-number":["2021SHZDZX0102"]}],"id":[{"id":"10.13039\/501100003399","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["IEEE\/ACM Trans. Audio Speech Lang. Process."],"published-print":{"date-parts":[[2024]]},"DOI":"10.1109\/taslp.2024.3375641","type":"journal-article","created":{"date-parts":[[2024,3,18]],"date-time":"2024-03-18T20:17:37Z","timestamp":1710793057000},"page":"1941-1953","source":"Crossref","is-referenced-by-count":16,"title":["Unified Cross-Modal Attention: Robust Audio-Visual Speech Recognition and Beyond"],"prefix":"10.1109","volume":"32","author":[{"ORCID":"https:\/\/orcid.org\/0009-0002-0639-1519","authenticated-orcid":false,"given":"Jiahong","family":"Li","sequence":"first","affiliation":[{"name":"Department of Computer Science and Engineering, Auditory Cognition and Computational Acoustics Lab, MoE Key Laboratory of Artificial Intelligence, AI Institute, Shanghai Jiao Tong University, Shanghai, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-0299-9914","authenticated-orcid":false,"given":"Chenda","family":"Li","sequence":"additional","affiliation":[{"name":"Department of Computer Science and Engineering, Auditory Cognition and Computational Acoustics Lab, MoE Key Laboratory of Artificial Intelligence, AI Institute, Shanghai Jiao Tong University, Shanghai, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0009-0002-8201-6438","authenticated-orcid":false,"given":"Yifei","family":"Wu","sequence":"additional","affiliation":[{"name":"Department of Computer Science and Engineering, Auditory Cognition and Computational Acoustics Lab, MoE Key Laboratory of Artificial Intelligence, AI Institute, Shanghai Jiao Tong University, Shanghai, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-0314-3790","authenticated-orcid":false,"given":"Yanmin","family":"Qian","sequence":"additional","affiliation":[{"name":"Department of Computer Science and Engineering, Auditory Cognition and Computational Acoustics Lab, MoE Key Laboratory of Artificial Intelligence, AI Institute, Shanghai Jiao Tong University, Shanghai, China"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"263","reference":[{"key":"ref1","doi-asserted-by":"publisher","DOI":"10.1109\/ICASSP.2013.6638947"},{"key":"ref2","first-page":"1764","article-title":"Towards end-to-end speech recognition with recurrent neural networks","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Graves","year":"2014"},{"key":"ref3","first-page":"577","article-title":"Attention-based models for speech recognition","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Chorowski","year":"2015"},{"key":"ref4","doi-asserted-by":"publisher","DOI":"10.1109\/ICASSP.2016.7472621"},{"key":"ref5","doi-asserted-by":"publisher","DOI":"10.1109\/ICASSP.2017.7953075"},{"key":"ref6","doi-asserted-by":"publisher","DOI":"10.1109\/ICASSP.2018.8462506"},{"key":"ref7","doi-asserted-by":"publisher","DOI":"10.1109\/ICASSP40776.2020.9053954"},{"key":"ref8","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-22482-4_11"},{"key":"ref9","doi-asserted-by":"publisher","DOI":"10.1109\/TASLP.2020.2998279"},{"key":"ref10","doi-asserted-by":"publisher","DOI":"10.1109\/ICASSP.2013.6639140"},{"key":"ref11","doi-asserted-by":"publisher","DOI":"10.1109\/TCYB.2013.2250954"},{"key":"ref12","doi-asserted-by":"publisher","DOI":"10.1109\/ICASSP.2015.7178347"},{"key":"ref13","doi-asserted-by":"publisher","DOI":"10.1109\/APSIPA.2015.7415335"},{"key":"ref14","doi-asserted-by":"publisher","DOI":"10.1007\/s10489-014-0629-7"},{"key":"ref15","doi-asserted-by":"publisher","DOI":"10.1109\/ICASSP.2018.8461326"},{"key":"ref16","doi-asserted-by":"publisher","DOI":"10.1145\/3242969.3243014"},{"key":"ref17","doi-asserted-by":"publisher","DOI":"10.1109\/SLT.2018.8639643"},{"key":"ref18","doi-asserted-by":"publisher","DOI":"10.1109\/TPAMI.2018.2889052"},{"key":"ref19","doi-asserted-by":"publisher","DOI":"10.1109\/ASRU46091.2019.9004036"},{"key":"ref20","doi-asserted-by":"publisher","DOI":"10.1109\/ICASSP.2019.8683733"},{"key":"ref21","doi-asserted-by":"publisher","DOI":"10.1109\/TASLP.2020.2980436"},{"key":"ref22","first-page":"638","article-title":"Attentive fusion enhanced audio-visual encoding for transformer based robust speech recognition","volume-title":"Proc. Signal Inf. Process. Assoc. Annu. Summit Conf.","author":"Wei","year":"2020"},{"key":"ref23","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR42600.2020.01444"},{"key":"ref24","doi-asserted-by":"publisher","DOI":"10.1109\/ICASSP39728.2021.9414567"},{"key":"ref25","doi-asserted-by":"publisher","DOI":"10.1109\/asru51503.2021.9688191"},{"key":"ref26","doi-asserted-by":"publisher","DOI":"10.21437\/Interspeech.2022-99"},{"key":"ref27","doi-asserted-by":"publisher","DOI":"10.1109\/ICIP46576.2022.9897235"},{"key":"ref28","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2022.acl-long.308"},{"key":"ref29","doi-asserted-by":"publisher","DOI":"10.21437\/Interspeech.2022-11311"},{"key":"ref30","doi-asserted-by":"publisher","DOI":"10.21437\/Interspeech.2022-10920"},{"key":"ref31","article-title":"Learning audio-visual speech representation by masked multimodal cluster prediction","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Shi","year":"2022"},{"key":"ref32","doi-asserted-by":"publisher","DOI":"10.1109\/ICASSP49357.2023.10096889"},{"key":"ref33","doi-asserted-by":"publisher","DOI":"10.21437\/Interspeech.2016-412"},{"key":"ref34","doi-asserted-by":"publisher","DOI":"10.21437\/Interspeech.2022-776"},{"key":"ref35","doi-asserted-by":"publisher","DOI":"10.21437\/Interspeech.2020-1065"},{"key":"ref36","doi-asserted-by":"publisher","DOI":"10.1109\/ICASSP40776.2020.9054127"},{"key":"ref37","doi-asserted-by":"publisher","DOI":"10.21437\/Interspeech.2021-2128"},{"key":"ref38","doi-asserted-by":"publisher","DOI":"10.1109\/ICASSP43922.2022.9747204"},{"key":"ref39","article-title":"LipNet: End-to-end sentence-level lipreading","author":"Assael","year":"2016"},{"key":"ref40","doi-asserted-by":"publisher","DOI":"10.21437\/Interspeech.2017-85"},{"key":"ref41","doi-asserted-by":"publisher","DOI":"10.21437\/Interspeech.2019-1669"},{"key":"ref42","doi-asserted-by":"publisher","DOI":"10.1109\/ICASSP40776.2020.9053841"},{"key":"ref43","doi-asserted-by":"publisher","DOI":"10.1109\/TMM.2021.3115626"},{"key":"ref44","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52688.2022.00510"},{"key":"ref45","article-title":"Jointly learning visual and auditory speech representations from raw data","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Haliassos","year":"2023"},{"key":"ref46","doi-asserted-by":"publisher","DOI":"10.21437\/Interspeech.2023-462"},{"key":"ref47","doi-asserted-by":"publisher","DOI":"10.1109\/ICASSP49357.2023.10096893"},{"key":"ref48","doi-asserted-by":"publisher","DOI":"10.1109\/TASL.2011.2109382"},{"key":"ref49","doi-asserted-by":"publisher","DOI":"10.1109\/MSP.2012.2205597"},{"key":"ref50","doi-asserted-by":"publisher","DOI":"10.1109\/TASL.2011.2134090"},{"key":"ref51","doi-asserted-by":"publisher","DOI":"10.1109\/ICASSP.2012.6288864"},{"key":"ref52","doi-asserted-by":"publisher","DOI":"10.1109\/TASLP.2014.2339736"},{"key":"ref53","first-page":"338","article-title":"Long short-term memory based recurrent neural network architectures for large vocabulary speech recognition","volume-title":"Proc. ISCA Interspeech","author":"Sak","year":"2014"},{"key":"ref54","first-page":"1","article-title":"Sequence transduction with recurrent neural networks","volume-title":"Proc. ICML Workshop Repres. Learn.","author":"Graves","year":"2012"},{"key":"ref55","doi-asserted-by":"publisher","DOI":"10.1145\/1143844.1143891"},{"key":"ref56","doi-asserted-by":"publisher","DOI":"10.48550\/ARXIV.1706.03762"},{"key":"ref57","doi-asserted-by":"publisher","DOI":"10.21437\/Interspeech.2020-3015"},{"key":"ref58","doi-asserted-by":"publisher","DOI":"10.21437\/Interspeech.2019-1873"},{"key":"ref59","first-page":"12449","article-title":"wav2vec 2.0: A framework for self-supervised learning of speech representations","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Baevski","year":"2020"},{"key":"ref60","doi-asserted-by":"publisher","DOI":"10.1109\/TASLP.2021.3122291"},{"key":"ref61","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2017.367"},{"key":"ref62","doi-asserted-by":"publisher","DOI":"10.21437\/Interspeech.2023-487"},{"key":"ref63","doi-asserted-by":"publisher","DOI":"10.1109\/ICASSP43922.2022.9746449"},{"key":"ref64","doi-asserted-by":"publisher","DOI":"10.1109\/ICASSP40776.2020.9054180"},{"key":"ref65","doi-asserted-by":"publisher","DOI":"10.21437\/Interspeech.2020-2028"},{"key":"ref66","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR46437.2021.01524"},{"key":"ref67","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52688.2022.01024"},{"key":"ref68","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-69544-6_17"},{"key":"ref69","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-58580-8_26"},{"key":"ref70","article-title":"Parameter efficient multimodal transformers for video representation learning","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Lee","year":"2021"},{"key":"ref71","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-58565-5_18"},{"key":"ref72","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR46437.2021.01659"},{"key":"ref73","doi-asserted-by":"publisher","DOI":"10.1109\/TASLP.2021.3129331"},{"key":"ref74","doi-asserted-by":"publisher","DOI":"10.1109\/TPAMI.2023.3325770"},{"key":"ref75","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v33i01.33019062"},{"key":"ref76","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v33i01.33019159"},{"key":"ref77","doi-asserted-by":"publisher","DOI":"10.1109\/tpami.2023.3339628"},{"key":"ref78","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2019.00756"},{"key":"ref79","article-title":"Labelling unlabelled videos from scratch with multi-modal self-supervision","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Asano","year":"2020"},{"key":"ref80","first-page":"8748","article-title":"Learning transferable visual models from natural language supervision","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Radford","year":"2021"},{"key":"ref81","first-page":"12888","article-title":"BLIP: Bootstrapping language-image pre-training for unified vision-language understanding and generation","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Li","year":"2022"},{"key":"ref82","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2015.279"},{"key":"ref83","doi-asserted-by":"publisher","DOI":"10.1109\/ICCVW60793.2023.00301"},{"key":"ref84","doi-asserted-by":"publisher","DOI":"10.1109\/ICASSP43922.2022.9746866"},{"key":"ref85","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2016.90"},{"key":"ref86","article-title":"Understanding and improving transformer from a multi-particle dynamic system point of view","volume-title":"Proc. ICLR Workshop Integ. Deep Neural Models Diff. Equ.","author":"Lu","year":"2020"},{"key":"ref87","doi-asserted-by":"publisher","DOI":"10.1109\/TPAMI.2015.2461544"},{"key":"ref88","doi-asserted-by":"publisher","DOI":"10.1145\/3382507.3418840"},{"key":"ref89","doi-asserted-by":"publisher","DOI":"10.21437\/Interspeech.2021-1739"},{"key":"ref90","doi-asserted-by":"publisher","DOI":"10.1109\/icassp48485.2024.10446421"},{"key":"ref91","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52688.2022.01957"},{"key":"ref92","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2022.naacl-main.50"},{"key":"ref93","article-title":"LRS3-TED: A large-scale dataset for visual speech recognition","author":"Afouras","year":"2018"},{"key":"ref94","doi-asserted-by":"publisher","DOI":"10.21437\/Interspeech.2019-2821"},{"key":"ref95","doi-asserted-by":"publisher","DOI":"10.1109\/ICASSP.2015.7178964"},{"key":"ref96","doi-asserted-by":"publisher","DOI":"10.21437\/Interspeech.2019-2225"},{"key":"ref97","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-54184-6_6"},{"key":"ref98","article-title":"Adam: A method for stochastic optimization","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Kingma","year":"2015"},{"key":"ref99","doi-asserted-by":"publisher","DOI":"10.21437\/Interspeech.2019-2680"},{"key":"ref100","doi-asserted-by":"publisher","DOI":"10.21437\/Interspeech.2018-1929"},{"key":"ref101","doi-asserted-by":"publisher","DOI":"10.1109\/CVPRW56347.2022.00517"},{"key":"ref102","doi-asserted-by":"publisher","DOI":"10.1016\/j.csl.2019.101027"},{"key":"ref103","doi-asserted-by":"publisher","DOI":"10.1109\/ICASSP49357.2023.10096626"}],"container-title":["IEEE\/ACM Transactions on Audio, Speech, and Language Processing"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx7\/6570655\/10304349\/10472123.pdf?arnumber=10472123","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,3,26]],"date-time":"2024-03-26T21:03:24Z","timestamp":1711487004000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/10472123\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024]]},"references-count":103,"URL":"https:\/\/doi.org\/10.1109\/taslp.2024.3375641","relation":{},"ISSN":["2329-9290","2329-9304"],"issn-type":[{"value":"2329-9290","type":"print"},{"value":"2329-9304","type":"electronic"}],"subject":[],"published":{"date-parts":[[2024]]}}}