{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,28]],"date-time":"2026-04-28T15:29:30Z","timestamp":1777390170743,"version":"3.51.4"},"publisher-location":"New York, NY, USA","reference-count":54,"publisher":"ACM","license":[{"start":{"date-parts":[[2024,10,28]],"date-time":"2024-10-28T00:00:00Z","timestamp":1730073600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2024,10,28]]},"DOI":"10.1145\/3664647.3681347","type":"proceedings-article","created":{"date-parts":[[2024,10,26]],"date-time":"2024-10-26T06:59:49Z","timestamp":1729925989000},"page":"3838-3847","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":6,"title":["Boosting Speech Recognition Robustness to Modality-Distortion with Contrast-Augmented Prompts"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0009-0000-7682-7678","authenticated-orcid":false,"given":"Dongjie","family":"Fu","sequence":"first","affiliation":[{"name":"Zhejiang University, Hangzhou, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-9708-3225","authenticated-orcid":false,"given":"Xize","family":"Cheng","sequence":"additional","affiliation":[{"name":"Zhejiang University, Hangzhou, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0009-0002-7297-4536","authenticated-orcid":false,"given":"Xiaoda","family":"Yang","sequence":"additional","affiliation":[{"name":"Zhejiang University, Hangzhou, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0009-0003-4032-9257","authenticated-orcid":false,"given":"Wang","family":"Hanting","sequence":"additional","affiliation":[{"name":"Zhejiang University, Hangzhou, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-6121-0384","authenticated-orcid":false,"given":"Zhou","family":"Zhao","sequence":"additional","affiliation":[{"name":"Zhejiang University, Hangzhou, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-3564-1628","authenticated-orcid":false,"given":"Tao","family":"Jin","sequence":"additional","affiliation":[{"name":"Zhejiang University, Hangzhou, China"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2024,10,28]]},"reference":[{"key":"e_1_3_2_1_1_1","volume-title":"Andrew Senior, Oriol Vinyals, and Andrew Zisserman.","author":"Afouras Triantafyllos","year":"2018","unstructured":"Triantafyllos Afouras, Joon Son Chung, Andrew Senior, Oriol Vinyals, and Andrew Zisserman. 2018. Deep audio-visual speech recognition. IEEE transactions on pattern analysis and machine intelligence 44, 12 (2018), 8717--8727."},{"key":"e_1_3_2_1_2_1","volume-title":"Juan Miguel Pino, and Changhan Wang","author":"Anwar Mohamed Sami","year":"2023","unstructured":"Mohamed Sami Anwar, Bowen Shi, Vedanuj Goswami, Wei-Ning Hsu, Juan Miguel Pino, and Changhan Wang. 2023. MuAViC: A Multilingual Audio-Visual Corpus for Robust Speech Recognition and Robust Speech-to-Text Translation. ArXiv abs\/2303.00628 (2023). https:\/\/api.semanticscholar.org\/CorpusID: 257255284"},{"key":"e_1_3_2_1_3_1","unstructured":"Oscar Chang Otavio Braga Hank Liao Dmitriy Serdyuk and Olivier Siohan. 2023. On Robustness to Missing Video for Audiovisual Speech Recognition. arXiv:2312.10088 [eess.AS]"},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"crossref","unstructured":"Xize Cheng Rongjie Huang Linjun Li Tao Jin ZehanWang Aoxiong Yin Minglei Li Xinyu Duan Zhou Zhao et al. 2023. TransFace: Unit-Based Audio-Visual Speech Synthesizer for Talking Head Translation. arXiv preprint arXiv:2312.15197 (2023).","DOI":"10.18653\/v1\/2024.findings-acl.593"},{"key":"e_1_3_2_1_5_1","volume-title":"Opensr: Open-modality speech recognition via maintaining multi-modality alignment. arXiv preprint arXiv:2306.06410","author":"Cheng Xize","year":"2023","unstructured":"Xize Cheng, Tao Jin, Linjun Li, Wang Lin, Xinyu Duan, and Zhou Zhao. 2023. Opensr: Open-modality speech recognition via maintaining multi-modality alignment. arXiv preprint arXiv:2306.06410 (2023)."},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"crossref","unstructured":"Xize Cheng Linjun Li Tao Jin Rongjie Huang Wang Lin Zehan Wang Huangdai Liu Ye Wang Aoxiong Yin and Zhou Zhao. 2023. MixSpeech: Cross-Modality Self-Learning with Audio-Visual Stream Mixup for Visual Speech Translation and Recognition. arXiv:2303.05309 [cs.CV]","DOI":"10.1109\/ICCV51070.2023.01442"},{"key":"e_1_3_2_1_7_1","unstructured":"Yusheng Dai Hang Chen Jun Du Ruoyu Wang Shihao Chen Jiefeng Ma Haotian Wang and Chin-Hui Lee. 2024. A Study of Dropout-Induced Modality Bias on Robustness to Missing Video Frames for Audio-Visual Speech Recognition. arXiv:2403.04245 [cs.SD]"},{"key":"e_1_3_2_1_8_1","volume-title":"SparseVSR: Lightweight and Noise Robust Visual Speech Recognition. ArXiv abs\/2307.04552","author":"Fernandez-Lopez Adriana","year":"2023","unstructured":"Adriana Fernandez-Lopez, Honglie Chen, Pingchuan Ma, Alexandros Haliassos, Stavros Petridis, and Maja Pantic. 2023. SparseVSR: Lightweight and Noise Robust Visual Speech Recognition. ArXiv abs\/2307.04552 (2023). https:\/\/api. semanticscholar.org\/CorpusID:259501735"},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1"},{"key":"e_1_3_2_1_10_1","volume-title":"Conformer: Convolution-augmented transformer for speech recognition. arXiv preprint arXiv:2005.08100","author":"Gulati Anmol","year":"2020","unstructured":"Anmol Gulati, James Qin, Chung-Cheng Chiu, Niki Parmar, Yu Zhang, Jiahui Yu, Wei Han, Shibo Wang, Zhengdong Zhang, Yonghui Wu, et al. 2020. Conformer: Convolution-augmented transformer for speech recognition. arXiv preprint arXiv:2005.08100 (2020)."},{"key":"e_1_3_2_1_11_1","volume-title":"PTR: Prompt Tuning with Rules for Text Classification. arXiv:2105.11259 [cs.CL]","author":"Han Xu","year":"2021","unstructured":"Xu Han, Weilin Zhao, Ning Ding, Zhiyuan Liu, and Maosong Sun. 2021. PTR: Prompt Tuning with Rules for Text Classification. arXiv:2105.11259 [cs.CL]"},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"crossref","unstructured":"Joanna Hong Minsu Kim Jeongsoo Choi and Yong Man Ro. 2023.Watch or Listen: Robust Audio-Visual Speech Recognition with Visual Corruption Modeling and Reliability Scoring. arXiv:2303.08536 [cs.MM]","DOI":"10.1109\/CVPR52729.2023.01801"},{"key":"e_1_3_2_1_13_1","volume-title":"Qiu shi Zhu, and Eng Siong Chng","author":"Hu Yuchen","year":"2023","unstructured":"Yuchen Hu, Ruizhe Li, Cheng Chen, Chengwei Qin, Qiu shi Zhu, and Eng Siong Chng. 2023. Hearing Lips in Noise: Universal Viseme-Phoneme Mapping and Transfer for Robust Audio-Visual Speech Recognition. In Annual Meeting of the Association for Computational Linguistics. https:\/\/api.semanticscholar.org\/ CorpusID:259202515"},{"key":"e_1_3_2_1_14_1","unstructured":"Rongjie Huang Jiawei Huang Dongchao Yang Yi Ren Luping Liu Mingze Li Zhenhui Ye Jinglin Liu Xiang Yin and Zhou Zhao. 2023. Make-An-Audio: Text-To-Audio Generation with Prompt-Enhanced Diffusion Models. arXiv:2301.12661 [cs.SD]"},{"key":"e_1_3_2_1_15_1","volume-title":"Av-transpeech: Audio-visual robust speech-to-speech translation. arXiv preprint arXiv:2305.15403","author":"Huang Rongjie","year":"2023","unstructured":"Rongjie Huang, Huadai Liu, Xize Cheng, Yi Ren, Linjun Li, Zhenhui Ye, Jinzheng He, Lichao Zhang, Jinglin Liu, Xiang Yin, et al. 2023. Av-transpeech: Audio-visual robust speech-to-speech translation. arXiv preprint arXiv:2305.15403 (2023)."},{"key":"e_1_3_2_1_16_1","unstructured":"Menglin Jia Luming Tang Bor-Chun Chen Claire Cardie Serge Belongie Bharath Hariharan and Ser-Nam Lim. 2022. Visual Prompt Tuning. arXiv:2203.12119 [cs.CV]"},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.1145\/3581783.3612291"},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1145\/3503161.3548069"},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"crossref","unstructured":"Muhammad Uzair Khattak Hanoona Rasheed Muhammad Maaz Salman Khan and Fahad Shahbaz Khan. 2023. MaPLe: Multi-modal Prompt Learning. arXiv:2210.03117 [cs.CV]","DOI":"10.1109\/CVPR52729.2023.01832"},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"crossref","unstructured":"Brian Lester Rami Al-Rfou and Noah Constant. 2021. The Power of Scale for Parameter-Efficient Prompt Tuning. arXiv:2104.08691 [cs.CL]","DOI":"10.18653\/v1\/2021.emnlp-main.243"},{"key":"e_1_3_2_1_21_1","unstructured":"Xiang Lisa Li and Percy Liang. 2021. Prefix-Tuning: Optimizing Continuous Prompts for Generation. arXiv:2101.00190 [cs.CL]"},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.21437\/Interspeech.2023-793"},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICPR48806.2021.9412817"},{"key":"e_1_3_2_1_24_1","unstructured":"Pengfei Liu Weizhe Yuan Jinlan Fu Zhengbao Jiang Hiroaki Hayashi and Graham Neubig. 2021. Pre-train Prompt and Predict: A Systematic Survey of Prompting Methods in Natural Language Processing. arXiv:2107.13586 [cs.CL]"},{"key":"e_1_3_2_1_25_1","volume-title":"Auto-AVSR: Audio-Visual Speech Recognition with Automatic Labels. ICASSP 2023 - 2023 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)","author":"Ma Pingchuan","year":"2023","unstructured":"Pingchuan Ma, Alexandros Haliassos, Adriana Fernandez-Lopez, Honglie Chen, Stavros Petridis, and Maja Pantic. 2023. Auto-AVSR: Audio-Visual Speech Recognition with Automatic Labels. ICASSP 2023 - 2023 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP) (2023), 1--5. https:\/\/api.semanticscholar.org\/CorpusID:257767381"},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICASSP39728.2021.9414567"},{"key":"e_1_3_2_1_27_1","doi-asserted-by":"crossref","unstructured":"Takaki Makino Hank Liao Yannis Assael Brendan Shillingford Basilio Garcia Otavio Braga and Olivier Siohan. 2019. Recurrent Neural Network Transducer for Audio-Visual Speech Recognition. arXiv:1911.04890 [eess.AS]","DOI":"10.1109\/ASRU46091.2019.9004036"},{"key":"e_1_3_2_1_28_1","unstructured":"Xichen Pan Peiyu Chen Yichen Gong Helong Zhou Xinbing Wang and Zhouhan Lin. 2022. Leveraging Unimodal Self-Supervised Learning for Multimodal Audio-Visual Speech Recognition. arXiv:2203.07996 [cs.SD]"},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"crossref","unstructured":"Stavros Petridis Themos Stafylakis Pingchuan Ma Georgios Tzimiropoulos and Maja Pantic. 2018. Audio-Visual Speech Recognition With A Hybrid CTC\/Attention Architecture. arXiv:1810.00108 [cs.CV]","DOI":"10.1109\/SLT.2018.8639643"},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV51070.2023.00276"},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"crossref","unstructured":"Kuniaki Saito Kihyuk Sohn Xiang Zhang Chun-Liang Li Chen-Yu Lee Kate Saenko and Tomas Pfister. 2023. Prefix Conditioning Unifies Language and Label Supervision. arXiv:2206.01125 [cs.CV]","DOI":"10.1109\/CVPR52729.2023.00280"},{"key":"e_1_3_2_1_32_1","doi-asserted-by":"crossref","unstructured":"Dmitriy Serdyuk Otavio Braga and Olivier Siohan. 2022. Transformer-Based Video Front-Ends for Audio-Visual Speech Recognition for Single and Multi-Person Video. arXiv:2201.10439 [cs.CV]","DOI":"10.21437\/Interspeech.2022-10920"},{"key":"e_1_3_2_1_33_1","unstructured":"Bowen Shi Wei-Ning Hsu Kushal Lakhotia and Abdelrahman Mohamed. 2022. Learning Audio-Visual Speech Representation by Masked Multimodal Cluster Prediction. arXiv:2201.02184 [eess.AS]"},{"key":"e_1_3_2_1_34_1","volume-title":"Robust selfsupervised audio-visual speech recognition. arXiv preprint arXiv:2201.01763","author":"Shi Bowen","year":"2022","unstructured":"Bowen Shi, Wei-Ning Hsu, and Abdelrahman Mohamed. 2022. Robust selfsupervised audio-visual speech recognition. arXiv preprint arXiv:2201.01763 (2022)."},{"key":"e_1_3_2_1_35_1","first-page":"1","article-title":"Diversifying Sequential Recommendation with Retrospective and Prospective Transformers","volume":"42","author":"Shi Chaoyu","year":"2024","unstructured":"Chaoyu Shi, Pengjie Ren, Dongjie Fu, Xin Xin, Shansong Yang, Fei Cai, Zhaochun Ren, and Zhumin Chen. 2024. Diversifying Sequential Recommendation with Retrospective and Prospective Transformers. ACM Transactions on Information Systems 42, 5 (2024), 1--37.","journal-title":"ACM Transactions on Information Systems"},{"key":"e_1_3_2_1_36_1","volume-title":"MUSAN: A Music, Speech, and Noise Corpus. arXiv:1510.08484 arXiv:1510.08484v1.","author":"Snyder David","year":"2015","unstructured":"David Snyder, Guoguo Chen, and Daniel Povey. 2015. MUSAN: A Music, Speech, and Noise Corpus. arXiv:1510.08484 arXiv:1510.08484v1."},{"key":"e_1_3_2_1_37_1","unstructured":"Aaron van den Oord Yazhe Li and Oriol Vinyals. 2019. Representation Learning with Contrastive Predictive Coding. arXiv:1807.03748 [cs.LG]"},{"key":"e_1_3_2_1_38_1","doi-asserted-by":"crossref","unstructured":"Tu Vu Brian Lester Noah Constant Rami Al-Rfou and Daniel Cer. 2022. SPoT: Better Frozen Model Adaptation through Soft Prompt Transfer. arXiv:2110.07904 [cs.CL]","DOI":"10.18653\/v1\/2022.acl-long.346"},{"key":"e_1_3_2_1_39_1","volume-title":"Audiobox: Unified Audio Generation with Natural Language Prompts. arXiv:2312.15821 [cs.SD]","author":"Vyas Apoorv","year":"2023","unstructured":"Apoorv Vyas, Bowen Shi, Matthew Le, Andros Tjandra, Yi-Chiao Wu, Baishan Guo, Jiemin Zhang, Xinyue Zhang, Robert Adkins, William Ngan, JeffWang, Ivan Cruz, Bapi Akula, Akinniyi Akinyemi, Brian Ellis, Rashel Moritz, Yael Yungster, Alice Rakotoarison, Liang Tan, Chris Summers, Carleigh Wood, Joshua Lane, Mary Williamson, and Wei-Ning Hsu. 2023. Audiobox: Unified Audio Generation with Natural Language Prompts. arXiv:2312.15821 [cs.SD]"},{"key":"e_1_3_2_1_40_1","volume-title":"3drp-net: 3d relative position-aware network for 3d visual grounding. arXiv preprint arXiv:2307.13363","author":"Wang Zehan","year":"2023","unstructured":"Zehan Wang, Haifeng Huang, Yang Zhao, Linjun Li, Xize Cheng, Yichen Zhu, Aoxiong Yin, and Zhou Zhao. 2023. 3drp-net: 3d relative position-aware network for 3d visual grounding. arXiv preprint arXiv:2307.13363 (2023)."},{"key":"e_1_3_2_1_41_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV51070.2023.00251"},{"key":"e_1_3_2_1_42_1","unstructured":"Zehan Wang Ziang Zhang Xize Cheng Rongjie Huang Luping Liu Zhenhui Ye Haifeng Huang Yang Zhao Tao Jin Peng Gao et al. 2024. Molecule-Space: Free Lunch in Unified Multimodal Space via Knowledge Fusion. arXiv preprint arXiv:2405.04883 (2024)."},{"key":"e_1_3_2_1_43_1","doi-asserted-by":"crossref","unstructured":"Zifeng Wang Zizhao Zhang Chen-Yu Lee Han Zhang Ruoxi Sun Xiaoqi Ren Guolong Su Vincent Perot Jennifer Dy and Tomas Pfister. 2022. Learning to Prompt for Continual Learning. arXiv:2112.08654 [cs.LG]","DOI":"10.1109\/CVPR52688.2022.00024"},{"key":"e_1_3_2_1_44_1","doi-asserted-by":"crossref","unstructured":"Zehan Wang Ziang Zhang Hang Zhang Luping Liu Rongjie Huang Xize Cheng Hengshuang Zhao and Zhou Zhao. 2024. OmniBind: Large-scale Omni Multimodal Representation via Binding Spaces. arXiv:2407.11895 [cs.CV] https:\/\/arxiv.org\/abs\/2407.11895","DOI":"10.1109\/TVT.2024.3374516"},{"key":"e_1_3_2_1_45_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-22482-4_11"},{"key":"e_1_3_2_1_46_1","doi-asserted-by":"crossref","unstructured":"Yi Xin Junlong Du Qiang Wang Ke Yan and Shouhong Ding. 2023. MmAP: Multi-modal Alignment Prompt for Cross-domain Multi-task Learning. arXiv:2312.08636 [cs.CV]","DOI":"10.1609\/aaai.v38i14.29540"},{"key":"e_1_3_2_1_47_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52733.2024.01016"},{"key":"e_1_3_2_1_48_1","doi-asserted-by":"publisher","DOI":"10.1145\/3664647.3681264"},{"key":"e_1_3_2_1_49_1","volume-title":"An Improved End-to-End Audio-Visual Speech Recognition Model. INTERSPEECH 2023","author":"Yang Sheng","year":"2023","unstructured":"Sheng Yang, Zheng Gong, and Jiacang Kang. 2023. An Improved End-to-End Audio-Visual Speech Recognition Model. INTERSPEECH 2023 (2023). https: \/\/api.semanticscholar.org\/CorpusID:260908510"},{"key":"e_1_3_2_1_50_1","doi-asserted-by":"publisher","DOI":"10.1145\/3664647.3681386"},{"key":"e_1_3_2_1_51_1","doi-asserted-by":"crossref","unstructured":"Yuan Yao Bowen Dong Ao Zhang Zhengyan Zhang Ruobing Xie Zhiyuan Liu Leyu Lin Maosong Sun and Jianyong Wang. 2022. Prompt Tuning for Discriminative Pre-trained Language Models. arXiv:2205.11166 [cs.CL]","DOI":"10.18653\/v1\/2022.findings-acl.273"},{"key":"e_1_3_2_1_52_1","volume-title":"Chen Change Loy, and Ziwei Liu","author":"Zhou Kaiyang","year":"2022","unstructured":"Kaiyang Zhou, Jingkang Yang, Chen Change Loy, and Ziwei Liu. 2022. Conditional Prompt Learning for Vision-Language Models. arXiv:2203.05557 [cs.CV]"},{"key":"e_1_3_2_1_53_1","doi-asserted-by":"publisher","DOI":"10.1007\/s11263-022-01653-1"},{"key":"e_1_3_2_1_54_1","doi-asserted-by":"crossref","unstructured":"Jiawen Zhu Simiao Lai Xin Chen Dong Wang and Huchuan Lu. 2023. Visual Prompt Multi-Modal Tracking. arXiv:2303.10826 [cs.CV]","DOI":"10.1109\/CVPR52729.2023.00918"}],"event":{"name":"MM '24: The 32nd ACM International Conference on Multimedia","location":"Melbourne VIC Australia","acronym":"MM '24","sponsor":["SIGMM ACM Special Interest Group on Multimedia"]},"container-title":["Proceedings of the 32nd ACM International Conference on Multimedia"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3664647.3681347","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3664647.3681347","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,19]],"date-time":"2025-06-19T01:17:44Z","timestamp":1750295864000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3664647.3681347"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,10,28]]},"references-count":54,"alternative-id":["10.1145\/3664647.3681347","10.1145\/3664647"],"URL":"https:\/\/doi.org\/10.1145\/3664647.3681347","relation":{},"subject":[],"published":{"date-parts":[[2024,10,28]]},"assertion":[{"value":"2024-10-28","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}