{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,12,4]],"date-time":"2025-12-04T07:51:00Z","timestamp":1764834660342,"version":"3.46.0"},"reference-count":93,"publisher":"Institute of Electrical and Electronics Engineers (IEEE)","issue":"1","license":[{"start":{"date-parts":[[2026,1,1]],"date-time":"2026-01-01T00:00:00Z","timestamp":1767225600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/legalcode"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["IEEE Trans. Pattern Anal. Mach. Intell."],"published-print":{"date-parts":[[2026,1]]},"DOI":"10.1109\/tpami.2025.3603653","type":"journal-article","created":{"date-parts":[[2025,8,28]],"date-time":"2025-08-28T18:09:25Z","timestamp":1756404565000},"page":"92-108","source":"Crossref","is-referenced-by-count":0,"title":["Creating Multimodal Interactive Digital Twin Characters From Videos: A Dataset and Baseline"],"prefix":"10.1109","volume":"48","author":[{"ORCID":"https:\/\/orcid.org\/0009-0000-1581-8743","authenticated-orcid":false,"given":"Meidai","family":"Xuanyuan","sequence":"first","affiliation":[{"name":"Tsinghua University, Beijing, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-6880-959X","authenticated-orcid":false,"given":"Yuwang","family":"Wang","sequence":"additional","affiliation":[{"name":"Tsinghua University, Beijing, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-1485-1987","authenticated-orcid":false,"given":"Honglei","family":"Guo","sequence":"additional","affiliation":[{"name":"Tsinghua University, Beijing, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0009-0000-1968-8691","authenticated-orcid":false,"given":"Hanshi","family":"Qu","sequence":"additional","affiliation":[{"name":"University of Chinese Academy of Sciences, Beijing, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Kun","family":"Zhang","sequence":"additional","affiliation":[{"name":"Tsinghua University, Beijing, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Zhongming","family":"Li","sequence":"additional","affiliation":[{"name":"University of Edinburgh, Edinburgh, U.K."}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0009-0008-7748-1806","authenticated-orcid":false,"given":"Danping","family":"Yan","sequence":"additional","affiliation":[{"name":"University of North Carolina at Chapel Hill, Chapel Hill, NC, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Tao","family":"Yu","sequence":"additional","affiliation":[{"name":"Tsinghua University, Beijing, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-9344-6428","authenticated-orcid":false,"given":"Jianhua","family":"Tao","sequence":"additional","affiliation":[{"name":"Tsinghua University, Beijing, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Qionghai","family":"Dai","sequence":"additional","affiliation":[{"name":"Tsinghua University, Beijing, China"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"263","reference":[{"key":"ref1","first-page":"3171","article-title":"FastSpeech: Fast, robust and controllable text to speech","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","author":"Ren"},{"key":"ref2","first-page":"195","article-title":"Deep voice: Real-time neural text-to-speech","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Ar\u0131k"},{"key":"ref3","first-page":"8599","article-title":"Grad-tts: A diffusion probabilistic model for text-to-speech","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Popov"},{"key":"ref4","first-page":"8821","article-title":"Zero-shot text-to-image generation","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Ramesh"},{"key":"ref5","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52688.2022.01042"},{"article-title":"Photorealistic text-to-image diffusion models with deep language understanding","year":"2022","author":"Saharia","key":"ref6"},{"article-title":"Hierarchical text-conditional image generation with clip latents","year":"2022","author":"Ramesh","key":"ref7"},{"article-title":"Dreamfusion: Text-to-3D using 2D diffusion","year":"2022","author":"Poole","key":"ref8"},{"article-title":"Make-a-video: Text-to-video generation without text-video data","year":"2022","author":"Singer","key":"ref9"},{"key":"ref10","doi-asserted-by":"publisher","DOI":"10.1145\/3474085.3475280"},{"key":"ref11","doi-asserted-by":"publisher","DOI":"10.1145\/3130800.3130813"},{"key":"ref12","doi-asserted-by":"publisher","DOI":"10.1007\/s11263-019-01251-8"},{"key":"ref13","first-page":"1","article-title":"First order motion model for image animation","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","author":"Siarohin"},{"key":"ref14","doi-asserted-by":"publisher","DOI":"10.1038\/s42256-021-00417-9"},{"key":"ref15","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-58517-4_42"},{"key":"ref16","first-page":"994","article-title":"A persona-based neural conversation model","volume-title":"Proc. 54th Annu. Meeting Assoc. Comput. Linguistics (ACL)","author":"Li"},{"key":"ref17","doi-asserted-by":"publisher","DOI":"10.1007\/s10579-008-9076-6"},{"key":"ref18","first-page":"527","article-title":"MELD: A multimodal multi-party dataset for emotion recognition in conversations","volume-title":"Proc. 57th Annu. Meeting Assoc. Comput. Linguistics","author":"Poria"},{"key":"ref19","first-page":"4441","article-title":"MEISD: A multimodal multi-label emotion, intensity and sentiment dialogue dataset for emotion recognition and sentiment analysis in conversations","volume-title":"Proc. 28th Int. Conf. Comput. Linguistics","author":"Firdaus"},{"key":"ref20","doi-asserted-by":"publisher","DOI":"10.1109\/TAFFC.2020.3015491"},{"key":"ref21","first-page":"4619","article-title":"Towards multimodal sarcasm detection (an_obviously_ perfect paper)","volume-title":"Proc. 57th Annu. Meeting Assoc. Comput. Linguistics","author":"Castro"},{"key":"ref22","first-page":"189","article-title":"Automatic evaluation of neural personality-based chatbots","volume-title":"Proc. 11th Int. Conf. Natural Lang. Gener.","author":"Xing"},{"key":"ref23","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D18-1298"},{"key":"ref24","first-page":"6556","article-title":"Towards persona-based empathetic conversational models","volume-title":"Proc. Conf. Empirical Methods Natural Lang. Process.","author":"Zhong"},{"key":"ref25","first-page":"1956","article-title":"Personalized response generation via generative split memory network","volume-title":"Proc. Conf. North Amer. Chapter Assoc. Comput. Linguistics: Hum. Lang. Technol.","author":"Wu"},{"key":"ref26","doi-asserted-by":"publisher","DOI":"10.1145\/3534678.3539135"},{"key":"ref27","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2023.findings-emnlp.570"},{"key":"ref28","first-page":"14743","article-title":"RoleLLM: Benchmarking, eliciting, and enhancing role-playing abilities of large language models","volume-title":"Proc. Findings Assoc. Comput. Linguistics: ACL","author":"Chen"},{"article-title":"Chatharuhi: Reviving anime character in reality via large language model","year":"2023","author":"Li","key":"ref29"},{"article-title":"Characterglm:customizing chinese conversational ai characters with large language models","year":"2023","author":"Zhou","key":"ref30"},{"key":"ref31","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2023.emnlp-main.814"},{"key":"ref32","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2024.acl-long.423"},{"article-title":"DFA-NeRF: Personalized talking head generation via disentangled face attributes neural rendering","year":"2022","author":"Yao","key":"ref33"},{"key":"ref34","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV51070.2023.02108"},{"key":"ref35","doi-asserted-by":"publisher","DOI":"10.1109\/cvpr52729.2023.02116"},{"key":"ref36","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52729.2023.00938"},{"key":"ref37","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52729.2023.00151"},{"article-title":"Styletalker: One-shot style-based audio-driven talking head video generation","year":"2024","author":"Min","key":"ref38"},{"key":"ref39","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2019.00453"},{"key":"ref40","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV51070.2023.01885"},{"key":"ref41","doi-asserted-by":"publisher","DOI":"10.1109\/TMM.2025.3535287"},{"article-title":"Diffusiontalker: Personalization and acceleration for speech-driven 3D face diffuser","year":"2023","author":"Chen","key":"ref42"},{"key":"ref43","doi-asserted-by":"publisher","DOI":"10.1109\/MIS.2016.94"},{"key":"ref44","first-page":"2236","article-title":"Multimodal language analysis in the wild: CMU-MOSEI dataset and interpretable dynamic fusion graph","volume-title":"Proc. 56th Annu. Meeting Assoc. Comput. Linguistics","author":"Bagher Zadeh"},{"key":"ref45","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2022.acl-long.186"},{"key":"ref46","first-page":"2046","article-title":"UR-FUNNY: A multimodal language dataset for understanding humor","volume-title":"Proc. Conf. Empirical Methods Natural Lang. Process. 9th Int. Joint Conf. Natural Lang. Process.","author":"Hasan"},{"key":"ref47","first-page":"3718","article-title":"CH-SIMS: A Chinese multimodal sentiment analysis dataset with fine-grained annotation of modality","volume-title":"Proc. 58th Annu. Meeting Assoc. Comput. Linguistics","author":"Yu"},{"article-title":"CPED: A large-scale chinese personalized and emotional dialogue dataset for conversational AI","year":"2022","author":"Chen","key":"ref48"},{"key":"ref49","article-title":"Voxceleb: Large-scale speaker verification in the wild","volume-title":"Comput. Speech Lang.","volume":"60","author":"Nagrani","year":"2020"},{"key":"ref50","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-58589-1_42"},{"key":"ref51","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR46437.2021.00366"},{"article-title":"Cstr vctk corpus: English multi-speaker corpus for CSTR voice cloning toolkit (version 0.92)","year":"2019","author":"Yamagishi","key":"ref52"},{"key":"ref53","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-54184-6_6"},{"key":"ref54","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52733.2024.00101"},{"article-title":"The LJ speech dataset","year":"2017","author":"Ito","key":"ref55"},{"key":"ref56","doi-asserted-by":"publisher","DOI":"10.1109\/ICASSP49357.2023.10095796"},{"article-title":"LRS3-TED: A large-scale dataset for visual speech recognition","year":"2018","author":"Afouras","key":"ref57"},{"key":"ref58","doi-asserted-by":"publisher","DOI":"10.1145\/3338533.3366579"},{"key":"ref59","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52729.2023.00053"},{"key":"ref60","first-page":"612","article-title":"BEAT: A large-scale semantic and emotional multi-modal dataset for conversational gestures synthesis","volume-title":"Proc. Eur. Conf. Comput. Vis.","author":"Haiyang"},{"key":"ref61","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52734.2025.01482"},{"author":"Schr\u00f6ter","key":"ref62","article-title":"DeepFilterNet: Perceptually motivated real-time speech enhancement"},{"key":"ref63","first-page":"28492","article-title":"Robust speech recognition via large-scale weak supervision","volume-title":"Proc. 40th Int. Conf. Mach. Learn.","author":"Radford"},{"key":"ref64","doi-asserted-by":"publisher","DOI":"10.1109\/ICASSP49357.2023.10096626"},{"key":"ref65","doi-asserted-by":"publisher","DOI":"10.l007\/978-3-319-46448-0_2"},{"key":"ref66","doi-asserted-by":"publisher","DOI":"10.5244\/C.29.41"},{"year":"2023","key":"ref67","article-title":"GPT-4 technical report"},{"key":"ref68","doi-asserted-by":"crossref","first-page":"4040","DOI":"10.18653\/v1\/2020.acl-main.372","article-title":"GoEmotions: A dataset of fine-grained emotions","volume-title":"Proc. 58th Annu. Meeting Assoc. Comput. Linguistics","author":"Demszky","year":"2020"},{"article-title":"Mediapipe: A framework for building perception pipelines","year":"2019","author":"Lugaresi","key":"ref69"},{"year":"2020","key":"ref70","article-title":"Openmmlab pose estimation toolbox and benchmark"},{"article-title":"Evaluating quality of answers for retrieval-augmented generation: A strong LLM is all you need","year":"2024","author":"Wang","key":"ref71"},{"key":"ref72","doi-asserted-by":"publisher","DOI":"10.1136\/thx.2005.043539"},{"key":"ref73","doi-asserted-by":"publisher","DOI":"10.1177\/001316447303300309"},{"key":"ref74","first-page":"1877","article-title":"Language models are few-shot learners","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","author":"Brown"},{"key":"ref75","doi-asserted-by":"publisher","DOI":"10.1038\/s41586-023-06647-8"},{"issue":"2","key":"ref76","first-page":"3","article-title":"LoRA: Low-rank adaptation of large language models","volume-title":"Proc. Int. Conf. Learn. Representations","volume":"1.1","author":"Hu"},{"key":"ref77","first-page":"5530","article-title":"Conditional variational autoencoder with adversarial learning for end-to-end text-to-speech","volume-title":"Proc. 38th Int. Conf. Mach. Learn.","author":"Kim"},{"article-title":"Aniportrait: Audio-driven synthesis of photorealistic portrait animations","year":"2024","author":"Wei","key":"ref78"},{"key":"ref79","first-page":"12449","article-title":"Wav2Vec 2.0: A framework for self-supervised learning of speech representations","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","author":"Baevski"},{"issue":"2000","key":"ref80","first-page":"39","article-title":"Computing euler angles from a rotation matrix","volume":"6","author":"Slabaugh","year":"1999","journal-title":"Retrieved"},{"key":"ref81","first-page":"6840","article-title":"Denoising diffusion probabilistic models","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","author":"Ho"},{"article-title":"Score-based generative modeling through stochastic differential equations","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Song","key":"ref82"},{"article-title":"Geneface: Generalized and high-fidelity audio-driven 3D talking face synthesis","year":"2023","author":"Ye","key":"ref83"},{"key":"ref84","doi-asserted-by":"crossref","DOI":"10.18653\/v1\/2023.emnlp-main.153","article-title":"G-eval: NLG evaluation using GPT-4 with better human alignment","author":"Liu","year":"2023"},{"article-title":"Vicuna: An open-source chatbot impressing GPT-4 with 90%* chatgpt quality","year":"2023","author":"Chiang","key":"ref85"},{"article-title":"Llama 2: Open foundation and fine-tuned chat models","year":"2023","author":"Touvron","key":"ref86"},{"article-title":"LLM pruning and distillation in practice: The minitron approach","year":"2024","author":"Sreenivas","key":"ref87"},{"key":"ref88","doi-asserted-by":"publisher","DOI":"10.1109\/ICASSP.2011.5946971"},{"key":"ref89","first-page":"4485","article-title":"Transfer learning from speaker verification to multispeaker text-to-speech synthesis","volume-title":"Proc. 32nd Int. Conf. Neural Inf. Process. Syst.","author":"Jia"},{"key":"ref90","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV48922.2021.00121"},{"key":"ref91","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52729.2023.01229"},{"key":"ref92","doi-asserted-by":"publisher","DOI":"10.1109\/cvpr52729.2023.00836"},{"key":"ref93","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV51070.2023.01891"}],"container-title":["IEEE Transactions on Pattern Analysis and Machine Intelligence"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx8\/34\/11275622\/11143865.pdf?arnumber=11143865","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,12,4]],"date-time":"2025-12-04T07:47:33Z","timestamp":1764834453000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/11143865\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026,1]]},"references-count":93,"journal-issue":{"issue":"1"},"URL":"https:\/\/doi.org\/10.1109\/tpami.2025.3603653","relation":{},"ISSN":["0162-8828","2160-9292","1939-3539"],"issn-type":[{"type":"print","value":"0162-8828"},{"type":"electronic","value":"2160-9292"},{"type":"electronic","value":"1939-3539"}],"subject":[],"published":{"date-parts":[[2026,1]]}}}