{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,20]],"date-time":"2026-10-20T16:53:13Z","timestamp":1792515193937,"version":"4.5.0"},"publisher-location":"New York, NY, USA","reference-count":55,"publisher":"ACM","license":[{"start":{"date-parts":[[2022,11,29]],"date-time":"2022-11-29T00:00:00Z","timestamp":1669680000000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2022,11,29]]},"DOI":"10.1145\/3550469.3555393","type":"proceedings-article","created":{"date-parts":[[2022,11,30]],"date-time":"2022-11-30T06:07:54Z","timestamp":1669788474000},"page":"1-9","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":27,"title":["Masked Lip-Sync Prediction by Audio-Visual Contextual Exploitation in Transformers"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-0589-4424","authenticated-orcid":false,"given":"Yasheng","family":"Sun","sequence":"first","affiliation":[{"name":"Tokyo Institute of Technology, Japan"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-2616-923X","authenticated-orcid":false,"given":"Hang","family":"Zhou","sequence":"additional","affiliation":[{"name":"Baidu Inc., China and The Chinese University of Hong Kong (CUHK), China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-2120-8383","authenticated-orcid":false,"given":"Kaisiyuan","family":"Wang","sequence":"additional","affiliation":[{"name":"University of Sydney, Australia"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-8764-6178","authenticated-orcid":false,"given":"Qianyi","family":"Wu","sequence":"additional","affiliation":[{"name":"Monash University, Australia"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-0207-4751","authenticated-orcid":false,"given":"Zhibin","family":"Hong","sequence":"additional","affiliation":[{"name":"Baidu, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-0566-0780","authenticated-orcid":false,"given":"Jingtuo","family":"Liu","sequence":"additional","affiliation":[{"name":"Baidu Inc., China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-1867-5378","authenticated-orcid":false,"given":"Errui","family":"Ding","sequence":"additional","affiliation":[{"name":"Baidu Inc., China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-4888-4445","authenticated-orcid":false,"given":"Jingdong","family":"Wang","sequence":"additional","affiliation":[{"name":"Baidu Inc., China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-4220-5958","authenticated-orcid":false,"given":"Ziwei","family":"Liu","sequence":"additional","affiliation":[{"name":"Nanyang Technological University, Singapore"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-8989-6434","authenticated-orcid":false,"given":"Koike","family":"Hideki","sequence":"additional","affiliation":[{"name":"Tokyo Institute of Technology, Japan"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2022,11,30]]},"reference":[{"key":"e_1_3_2_3_1_1","volume-title":"wav2vec 2.0: A framework for self-supervised learning of speech representations. Advances in Neural Information Processing Systems","author":"Baevski Alexei","year":"2020","unstructured":"Alexei Baevski, Yuhao Zhou, Abdelrahman Mohamed, and Michael Auli. 2020. wav2vec 2.0: A framework for self-supervised learning of speech representations. Advances in Neural Information Processing Systems (2020)."},{"key":"e_1_3_2_3_2_1","volume-title":"Beit: Bert pre-training of image transformers. arXiv preprint arXiv:2106.08254(2021).","author":"Bao Hangbo","year":"2021","unstructured":"Hangbo Bao, Li Dong, and Furu Wei. 2021. Beit: Bert pre-training of image transformers. arXiv preprint arXiv:2106.08254(2021)."},{"key":"e_1_3_2_3_3_1","doi-asserted-by":"publisher","DOI":"10.1145\/311535.311556"},{"key":"e_1_3_2_3_4_1","doi-asserted-by":"crossref","unstructured":"Huiwen Chang Han Zhang Lu Jiang Ce Liu and William\u00a0T Freeman. 2022. MaskGIT: Masked Generative Image Transformer. arXiv preprint arXiv:2202.04200(2022).","DOI":"10.1109\/CVPR52688.2022.01103"},{"key":"e_1_3_2_3_5_1","volume-title":"Talking-head Generation with Rhythmic Head Motion. In European Conference on Computer Vision. Springer, 35\u201351","author":"Chen Lele","year":"2020","unstructured":"Lele Chen, Guofeng Cui, Celong Liu, Zhong Li, Ziyi Kou, Yi Xu, and Chenliang Xu. 2020. Talking-head Generation with Rhythmic Head Motion. In European Conference on Computer Vision. Springer, 35\u201351."},{"key":"e_1_3_2_3_6_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-01234-2_32"},{"key":"e_1_3_2_3_7_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2019.00802"},{"key":"e_1_3_2_3_8_1","volume-title":"Wavlm: Large-scale self-supervised pre-training for full stack speech processing","author":"Chen Sanyuan","year":"2022","unstructured":"Sanyuan Chen, Chengyi Wang, Zhengyang Chen, Yu Wu, Shujie Liu, Zhuo Chen, Jinyu Li, Naoyuki Kanda, Takuya Yoshioka, Xiong Xiao, 2022b. Wavlm: Large-scale self-supervised pre-training for full stack speech processing. IEEE Journal of Selected Topics in Signal Processing (2022)."},{"key":"e_1_3_2_3_9_1","unstructured":"Xiaokang Chen Mingyu Ding Xiaodi Wang Ying Xin Shentong Mo Yunhao Wang Shumin Han Ping Luo Gang Zeng and Jingdong Wang. 2022a. Context autoencoder for self-supervised representation learning. arXiv preprint arXiv:2202.03026(2022)."},{"key":"e_1_3_2_3_10_1","doi-asserted-by":"crossref","unstructured":"J.\u00a0S. Chung A. Nagrani and A. Zisserman. 2018. VoxCeleb2: Deep Speaker Recognition. In INTERSPEECH.","DOI":"10.21437\/Interspeech.2018-1929"},{"key":"e_1_3_2_3_11_1","volume-title":"Asian conference on computer vision. Springer, 87\u2013103","author":"Chung Joon\u00a0Son","year":"2016","unstructured":"Joon\u00a0Son Chung and Andrew Zisserman. 2016a. Lip reading in the wild. In Asian conference on computer vision. Springer, 87\u2013103."},{"key":"e_1_3_2_3_12_1","doi-asserted-by":"crossref","unstructured":"Joon\u00a0Son Chung and Andrew Zisserman. 2016b. Out of time: automated lip sync in the wild. In ACCV.","DOI":"10.1007\/978-3-319-54427-4_19"},{"key":"e_1_3_2_3_13_1","volume-title":"An image is worth 16x16 words: Transformers for image recognition at scale. ICLR","author":"Dosovitskiy Alexey","year":"2021","unstructured":"Alexey Dosovitskiy, Lucas Beyer, Alexander Kolesnikov, Dirk Weissenborn, Xiaohua Zhai, Thomas Unterthiner, Mostafa Dehghani, Matthias Minderer, Georg Heigold, Sylvain Gelly, 2021. An image is worth 16x16 words: Transformers for image recognition at scale. ICLR (2021)."},{"key":"e_1_3_2_3_14_1","volume-title":"Imagebart: Bidirectional context with multinomial diffusion for autoregressive image synthesis. Advances in Neural Information Processing Systems 34","author":"Esser Patrick","year":"2021","unstructured":"Patrick Esser, Robin Rombach, Andreas Blattmann, and Bjorn Ommer. 2021b. Imagebart: Bidirectional context with multinomial diffusion for autoregressive image synthesis. Advances in Neural Information Processing Systems 34 (2021)."},{"key":"e_1_3_2_3_15_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR46437.2021.01268"},{"key":"e_1_3_2_3_16_1","volume-title":"AD-NeRF: Audio Driven Neural Radiance Fields for Talking Head Synthesis. In IEEE\/CVF International Conference on Computer Vision (ICCV).","author":"Guo Yudong","year":"2021","unstructured":"Yudong Guo, Keyu Chen, Sen Liang, Yongjin Liu, Hujun Bao, and Juyong Zhang. 2021. AD-NeRF: Audio Driven Neural Radiance Fields for Talking Head Synthesis. In IEEE\/CVF International Conference on Computer Vision (ICCV)."},{"key":"e_1_3_2_3_17_1","unstructured":"Kaiming He Xinlei Chen Saining Xie Yanghao Li Piotr Doll\u00e1r and Ross Girshick. 2021. Masked autoencoders are scalable vision learners. arXiv preprint arXiv:2111.06377(2021)."},{"key":"e_1_3_2_3_18_1","volume-title":"Image-to-Image Translation with Conditional Adversarial Networks. CVPR","author":"Isola Phillip","year":"2017","unstructured":"Phillip Isola, Jun-Yan Zhu, Tinghui Zhou, and Alexei\u00a0A Efros. 2017. Image-to-Image Translation with Conditional Adversarial Networks. CVPR (2017)."},{"key":"e_1_3_2_3_19_1","doi-asserted-by":"publisher","DOI":"10.1007\/s11263-019-01150-y"},{"key":"e_1_3_2_3_20_1","volume-title":"EAMM: One-Shot Emotional Talking Face via Audio-Based Emotion-Aware Motion Model. SIGGRAPH","author":"Ji Xinya","year":"2022","unstructured":"Xinya Ji, Hang Zhou, Kaisiyuan Wang, Qianyi Wu, Wayne Wu, Feng Xu, and Xun Cao. 2022. EAMM: One-Shot Emotional Talking Face via Audio-Based Emotion-Aware Motion Model. SIGGRAPH (2022)."},{"key":"e_1_3_2_3_21_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR46437.2021.01386"},{"key":"e_1_3_2_3_22_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR46437.2021.00278"},{"key":"e_1_3_2_3_23_1","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v35i3.16286"},{"key":"e_1_3_2_3_24_1","volume-title":"MAT: Mask-Aware Transformer for Large Hole Image Inpainting. CVPR","author":"Li Wenbo","year":"2022","unstructured":"Wenbo Li, Zhe Lin, Kun Zhou, Lu Qi, Yi Wang, and Jiaya Jia. 2022. MAT: Mask-Aware Transformer for Large Hole Image Inpainting. CVPR (2022)."},{"key":"e_1_3_2_3_25_1","volume-title":"Semantic-Aware Implicit Neural Audio-Driven Video Portrait Generation. ECCV","author":"Liu Xian","year":"2022","unstructured":"Xian Liu, Yinghao Xu, Qianyi Wu, Hang Zhou, Wayne Wu, and Bolei Zhou. 2022. Semantic-Aware Implicit Neural Audio-Driven Video Portrait Generation. ECCV (2022)."},{"key":"e_1_3_2_3_26_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV48922.2021.00986"},{"key":"e_1_3_2_3_27_1","doi-asserted-by":"crossref","unstructured":"A. Nagrani J.\u00a0S. Chung and A. Zisserman. 2017. VoxCeleb: a large-scale speaker identification dataset. In INTERSPEECH.","DOI":"10.21437\/Interspeech.2017-950"},{"key":"e_1_3_2_3_28_1","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v36i2.20102"},{"key":"e_1_3_2_3_29_1","doi-asserted-by":"publisher","DOI":"10.1145\/3394171.3413532"},{"key":"e_1_3_2_3_30_1","volume-title":"Do vision transformers see like convolutional neural networks?Advances in Neural Information Processing Systems 34","author":"Raghu Maithra","year":"2021","unstructured":"Maithra Raghu, Thomas Unterthiner, Simon Kornblith, Chiyuan Zhang, and Alexey Dosovitskiy. 2021. Do vision transformers see like convolutional neural networks?Advances in Neural Information Processing Systems 34 (2021)."},{"key":"e_1_3_2_3_31_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-24574-4_28"},{"key":"e_1_3_2_3_32_1","volume-title":"Everybody\u2019s Talkin\u2019: Let Me Talk as You Want. arXiv preprint arXiv:2001.05201(2020).","author":"Song Linsen","year":"2020","unstructured":"Linsen Song, Wayne Wu, Chen Qian, Ran He, and Chen\u00a0Change Loy. 2020. Everybody\u2019s Talkin\u2019: Let Me Talk as You Want. arXiv preprint arXiv:2001.05201(2020)."},{"key":"e_1_3_2_3_33_1","doi-asserted-by":"crossref","unstructured":"Yang Song Jingwen Zhu Dawei Li Xiaolong Wang and Hairong Qi. 2018. Talking face generation by conditional recurrent adversarial network. arXiv preprint arXiv:1804.04786(2018).","DOI":"10.24963\/ijcai.2019\/129"},{"key":"e_1_3_2_3_34_1","unstructured":"Yasheng Sun Hang Zhou Ziwei Liu and Hideki Koike. 2021. Speech2Talking-Face: Inferring and Driving a Face with Synchronized Audio-Visual Representation.. In IJCAI."},{"key":"e_1_3_2_3_35_1","doi-asserted-by":"publisher","DOI":"10.1145\/3072959.3073640"},{"key":"e_1_3_2_3_36_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-58517-4_42"},{"key":"e_1_3_2_3_37_1","volume-title":"Neural discrete representation learning. Advances in neural information processing systems 30","author":"Den\u00a0Oord Aaron Van","year":"2017","unstructured":"Aaron Van Den\u00a0Oord, Oriol Vinyals, 2017. Neural discrete representation learning. Advances in neural information processing systems 30 (2017)."},{"key":"e_1_3_2_3_38_1","volume-title":"Attention is all you need. Advances in neural information processing systems 30","author":"Vaswani Ashish","year":"2017","unstructured":"Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, Llion Jones, Aidan\u00a0N Gomez, \u0141ukasz Kaiser, and Illia Polosukhin. 2017. Attention is all you need. Advances in neural information processing systems 30 (2017)."},{"key":"e_1_3_2_3_39_1","doi-asserted-by":"crossref","unstructured":"Konstantinos Vougioukas Stavros Petridis and Maja Pantic. 2019. Realistic speech-driven facial animation with gans. International Journal of Computer Vision(2019).","DOI":"10.1007\/s11263-019-01251-8"},{"key":"e_1_3_2_3_40_1","doi-asserted-by":"crossref","unstructured":"Suzhen Wang Lincheng Li Yu Ding Changjie Fan and Xin Yu. 2021. Audio2Head: Audio-driven One-shot Talking-head Generation with Natural Head Motion. arXiv preprint arXiv:2107.09293(2021).","DOI":"10.24963\/ijcai.2021\/152"},{"key":"e_1_3_2_3_41_1","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v36i3.20154"},{"key":"e_1_3_2_3_42_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2018.00917"},{"key":"e_1_3_2_3_43_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2018.00813"},{"key":"e_1_3_2_3_44_1","volume-title":"Image quality assessment: from error visibility to structural similarity","author":"Wang Zhou","year":"2004","unstructured":"Zhou Wang, Alan\u00a0C Bovik, Hamid\u00a0R Sheikh, and Eero\u00a0P Simoncelli. 2004. Image quality assessment: from error visibility to structural similarity. IEEE transactions on image processing 13, 4 (2004), 600\u2013612."},{"key":"e_1_3_2_3_45_1","doi-asserted-by":"publisher","DOI":"10.1145\/3474085.3475280"},{"key":"e_1_3_2_3_46_1","first-page":"30392","article-title":"Early convolutions help transformers see better","volume":"34","author":"Xiao Tete","year":"2021","unstructured":"Tete Xiao, Mannat Singh, Eric Mintun, Trevor Darrell, Piotr Doll\u00e1r, and Ross Girshick. 2021. Early convolutions help transformers see better. Advances in Neural Information Processing Systems 34 (2021), 30392\u201330400.","journal-title":"Advances in Neural Information Processing Systems"},{"key":"e_1_3_2_3_47_1","doi-asserted-by":"crossref","unstructured":"Fuzhi Yang Huan Yang Jianlong Fu Hongtao Lu and Baining Guo. 2020. Learning Texture Transformer Network for Image Super-Resolution. In CVPR.","DOI":"10.1109\/CVPR42600.2020.00583"},{"key":"e_1_3_2_3_48_1","volume-title":"Implicit Transformer Network for Screen Content Image Continuous Super-Resolution. Advances in Neural Information Processing Systems 34","author":"Yang Jingyu","year":"2021","unstructured":"Jingyu Yang, Sheng Shen, Huanjing Yue, and Kun Li. 2021. Implicit Transformer Network for Screen Content Image Continuous Super-Resolution. Advances in Neural Information Processing Systems 34 (2021)."},{"key":"e_1_3_2_3_49_1","volume-title":"Audio-driven talking face video generation with learning-based personalized head pose. arXiv e-prints","author":"Yi Ran","year":"2020","unstructured":"Ran Yi, Zipeng Ye, Juyong Zhang, Hujun Bao, and Yong-Jin Liu. 2020. Audio-driven talking face video generation with learning-based personalized head pose. arXiv e-prints (2020), arXiv\u20132002."},{"key":"e_1_3_2_3_50_1","doi-asserted-by":"publisher","DOI":"10.1145\/3474085.3475436"},{"key":"e_1_3_2_3_51_1","doi-asserted-by":"crossref","unstructured":"Bowen Zhang Shuyang Gu Bo Zhang Jianmin Bao Dong Chen Fang Wen Yong Wang and Baining Guo. 2021. StyleSwin: Transformer-based GAN for High-resolution Image Generation. arXiv preprint arXiv:2112.10762(2021).","DOI":"10.1109\/CVPR52688.2022.01102"},{"key":"e_1_3_2_3_52_1","volume-title":"Tfill: Image completion via a transformer-based architecture. arXiv preprint arXiv:2104.00845(2021).","author":"Zheng Chuanxia","year":"2021","unstructured":"Chuanxia Zheng, Tat-Jen Cham, and Jianfei Cai. 2021. Tfill: Image completion via a transformer-based architecture. arXiv preprint arXiv:2104.00845(2021)."},{"key":"e_1_3_2_3_53_1","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v33i01.33019299"},{"key":"e_1_3_2_3_54_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR46437.2021.00416"},{"key":"e_1_3_2_3_55_1","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1145\/3414685.3417774","article-title":"MakeltTalk: speaker-aware talking-head animation","volume":"39","author":"Zhou Yang","year":"2020","unstructured":"Yang Zhou, Xintong Han, Eli Shechtman, Jose Echevarria, Evangelos Kalogerakis, and Dingzeyu Li. 2020. MakeltTalk: speaker-aware talking-head animation. ACM Transactions on Graphics (TOG) 39, 6 (2020), 1\u201315.","journal-title":"ACM Transactions on Graphics (TOG)"}],"event":{"name":"SA '22: SIGGRAPH Asia 2022","location":"Daegu Republic of Korea","acronym":"SA '22","sponsor":["SIGGRAPH ACM Special Interest Group on Computer Graphics and Interactive Techniques"]},"container-title":["SIGGRAPH Asia 2022 Conference Papers"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3550469.3555393","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3550469.3555393","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T13:51:44Z","timestamp":1750168304000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3550469.3555393"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2022,11,29]]},"references-count":55,"alternative-id":["10.1145\/3550469.3555393","10.1145\/3550469"],"URL":"https:\/\/doi.org\/10.1145\/3550469.3555393","relation":{},"subject":[],"published":{"date-parts":[[2022,11,29]]},"assertion":[{"value":"2022-11-30","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}