{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,27]],"date-time":"2026-10-27T17:58:23Z","timestamp":1793123903143,"version":"4.5.0"},"reference-count":322,"publisher":"Institute of Electrical and Electronics Engineers (IEEE)","issue":"12","license":[{"start":{"date-parts":[[2024,12,1]],"date-time":"2024-12-01T00:00:00Z","timestamp":1733011200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/ieeexplore.ieee.org\/Xplorehelp\/downloads\/license-information\/IEEE.html"},{"start":{"date-parts":[[2024,12,1]],"date-time":"2024-12-01T00:00:00Z","timestamp":1733011200000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2024,12,1]],"date-time":"2024-12-01T00:00:00Z","timestamp":1733011200000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"}],"funder":[{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["62372341"],"award-info":[{"award-number":["62372341"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]},{"name":"MUR PNRR","award":["PE00000013"],"award-info":[{"award-number":["PE00000013"]}]},{"DOI":"10.13039\/100031478","name":"NextGenerationEU","doi-asserted-by":"publisher","id":[{"id":"10.13039\/100031478","id-type":"DOI","asserted-by":"publisher"}]},{"name":"EU Horizon project ELIAS","award":["101120237"],"award-info":[{"award-number":["101120237"]}]}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["IEEE Trans. Pattern Anal. Mach. Intell."],"published-print":{"date-parts":[[2024,12]]},"DOI":"10.1109\/tpami.2024.3420239","type":"journal-article","created":{"date-parts":[[2024,6,28]],"date-time":"2024-06-28T14:36:50Z","timestamp":1719585410000},"page":"9102-9122","source":"Crossref","is-referenced-by-count":41,"title":["Vision + X: A Survey on Multimodal Learning in the Light of Data"],"prefix":"10.1109","volume":"46","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-0890-9476","authenticated-orcid":false,"given":"Ye","family":"Zhu","sequence":"first","affiliation":[{"name":"Department of Computer Science, Princeton University, Princeton, NJ, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-1680-8253","authenticated-orcid":false,"given":"Yu","family":"Wu","sequence":"additional","affiliation":[{"name":"School of Computer Science, Wuhan University, Wuhan, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-6597-7248","authenticated-orcid":false,"given":"Nicu","family":"Sebe","sequence":"additional","affiliation":[{"name":"Department of Information Engineering and Computer Science, University of Trento, Trento, Italy"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-7618-119X","authenticated-orcid":false,"given":"Yan","family":"Yan","sequence":"additional","affiliation":[{"name":"Department of Computer Science, Illinois Institute of Technology, Chicago, IL, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"263","reference":[{"key":"ref1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR42600.2020.00971"},{"key":"ref2","article-title":"Dance2Music: Automatic dance-driven music generation","author":"Aggarwal","year":"2021"},{"key":"ref3","article-title":"A causal masked multimodal model of the internet","author":"Aghajanyan","year":"2022"},{"key":"ref4","first-page":"24206","article-title":"VATT: Transformers for multimodal self-supervised learning from raw video, audio and text","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Akbari"},{"key":"ref5","doi-asserted-by":"publisher","DOI":"10.1073\/pnas.211209098"},{"key":"ref6","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2019.00774"},{"key":"ref7","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-46454-1_24"},{"key":"ref8","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2017.618"},{"key":"ref9","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2015.279"},{"key":"ref10","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2017.73"},{"key":"ref11","first-page":"214","article-title":"Wasserstein generative adversarial networks","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Arjovsky"},{"key":"ref12","first-page":"17981","article-title":"Structured denoising diffusion models in discrete state-spaces","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Austin"},{"key":"ref13","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52688.2022.01767"},{"key":"ref14","doi-asserted-by":"publisher","DOI":"10.21437\/Interspeech.2019-3181"},{"key":"ref15","doi-asserted-by":"publisher","DOI":"10.1109\/TPAMI.2018.2798607"},{"key":"ref16","first-page":"65","article-title":"METEOR: An automatic metric for MT evaluation with improved correlation with human judgments","volume-title":"Proc. ACL Workshop Intrinsic Extrinsic Eval. Measures Mach. Transl. Summarization","author":"Banerjee"},{"key":"ref17","doi-asserted-by":"publisher","DOI":"10.1007\/11744023_32"},{"key":"ref18","doi-asserted-by":"publisher","DOI":"10.1007\/s00371-021-02166-7"},{"key":"ref19","article-title":"On the opportunities and risks of foundation models","author":"Bommasani","year":"2021"},{"key":"ref20","doi-asserted-by":"publisher","DOI":"10.1016\/j.tics.2014.12.008"},{"key":"ref21","doi-asserted-by":"publisher","DOI":"10.1145\/3308561.3353774"},{"key":"ref22","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-70163-9"},{"key":"ref23","article-title":"Large scale GAN training for high fidelity natural image synthesis","author":"Brock","year":"2018"},{"key":"ref24","article-title":"Video generation models as world simulators"},{"key":"ref25","first-page":"1877","article-title":"Language models are few-shot learners","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Brown"},{"key":"ref26","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2015.7298698"},{"key":"ref27","doi-asserted-by":"publisher","DOI":"10.1109\/TAFFC.2014.2336244"},{"key":"ref28","doi-asserted-by":"publisher","DOI":"10.1109\/TPAMI.2019.2929257"},{"key":"ref29","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2017.143"},{"key":"ref30","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2017.502"},{"key":"ref31","article-title":"ShapeNet: An information-rich 3D model repository","author":"Chang","year":"2015"},{"key":"ref32","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR46437.2021.00356"},{"key":"ref33","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-58583-9_42"},{"key":"ref34","first-page":"190","article-title":"Collecting highly parallel data for paraphrase evaluation","volume-title":"Proc. 49th Annu. Meeting Assoc. Comput. Linguistics: Hum. Lang. Technol.","author":"Chen"},{"key":"ref35","first-page":"1929","article-title":"Variational structured semantic inference for diverse image captioning","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Chen"},{"key":"ref36","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-11024-6_43"},{"key":"ref37","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-01234-2_32"},{"key":"ref38","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2019.00802"},{"key":"ref39","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR42600.2020.01081"},{"key":"ref40","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2017.667"},{"key":"ref41","first-page":"1691","article-title":"Generative pretraining from pixels","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Chen"},{"key":"ref42","doi-asserted-by":"publisher","DOI":"10.1109\/TIP.2020.3009820"},{"key":"ref43","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v33i01.33018199"},{"key":"ref44","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-58565-5_36"},{"key":"ref45","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-58577-8_7"},{"key":"ref46","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2017.81"},{"key":"ref47","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR46437.2021.00831"},{"key":"ref48","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-54184-6_6"},{"key":"ref49","first-page":"7365","article-title":"Unsupervised cross-modal alignment of speech and text embedding spaces","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Chung"},{"key":"ref50","first-page":"1282","article-title":"Quantifying generalization in reinforcement learning","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Cobbe"},{"key":"ref51","doi-asserted-by":"publisher","DOI":"10.1007\/BF00994018"},{"key":"ref52","doi-asserted-by":"publisher","DOI":"10.1007\/11736790_9"},{"key":"ref53","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2017.323"},{"key":"ref54","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2005.177"},{"key":"ref55","first-page":"753","article-title":"Scaling egocentric vision: The epic-kitchens dataset","volume-title":"Proc. Eur. Conf. Comput. Vis.","author":"Damen"},{"key":"ref56","doi-asserted-by":"publisher","DOI":"10.1016\/j.cviu.2017.10.001"},{"key":"ref57","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2017.121"},{"key":"ref58","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-58577-8_25"},{"key":"ref59","doi-asserted-by":"publisher","DOI":"10.1145\/3197517.3201371"},{"key":"ref60","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2017.475"},{"key":"ref61","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2018.00808"},{"key":"ref62","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2009.5206848"},{"key":"ref63","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV48922.2021.00179"},{"key":"ref64","first-page":"1174","article-title":"Stochastic video generation with a learned prior","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Denton"},{"key":"ref65","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR46437.2021.01101"},{"key":"ref66","article-title":"BERT: Pre-training of deep bidirectional transformers for language understanding","author":"Devlin","year":"2018"},{"key":"ref67","article-title":"Jukebox: A generative model for music","author":"Dhariwal","year":"2020"},{"key":"ref68","first-page":"8780","article-title":"Diffusion models beat GANs on image synthesis","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Dhariwal"},{"key":"ref69","doi-asserted-by":"publisher","DOI":"10.1145\/3474085.3475195"},{"key":"ref70","first-page":"2012","article-title":"MuST-C: A multilingual speech translation corpus","volume-title":"Proc. Conf. North Amer. Chapter Assoc. Comput. Linguistics: Hum. Lang. Technol.","author":"Di Gangi"},{"key":"ref71","first-page":"19822","article-title":"CogView: Mastering text-to-image generation via transformers","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Ding"},{"key":"ref72","article-title":"Pypianoroll: Open source Python package for handling multitrack pianoroll","volume-title":"Proc. Int. Soc. Music Inf. Retrieval Conf.","author":"Dong"},{"key":"ref73","doi-asserted-by":"publisher","DOI":"10.1109\/WACV48630.2021.00406"},{"key":"ref74","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR46437.2021.00276"},{"key":"ref75","doi-asserted-by":"publisher","DOI":"10.1109\/ICCVW.2017.61"},{"key":"ref76","doi-asserted-by":"publisher","DOI":"10.1109\/ICASSP.2017.7953127"},{"key":"ref77","first-page":"6340","article-title":"auDeep: Unsupervised learning of representations from audio with deep recurrent neural networks","volume":"18","author":"Freitag","year":"2017","journal-title":"J. Mach. Learn. Res."},{"key":"ref78","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D16-1044"},{"key":"ref79","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-58621-8_44"},{"key":"ref80","article-title":"ESB: A benchmark for multi-domain end-to-end speech recognition","author":"Gandhi","year":"2022"},{"key":"ref81","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2017.563"},{"key":"ref82","first-page":"776","article-title":"Audio set: An ontology and human-labeled dataset for audio events","volume-title":"Proc. IEEE Int. Conf. Acoust. Speech Signal Process.","author":"Jort"},{"key":"ref83","doi-asserted-by":"publisher","DOI":"10.5555\/2969033.2969125"},{"key":"ref84","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2017.670"},{"key":"ref85","doi-asserted-by":"publisher","DOI":"10.31390\/gradschool_dissertations.4601"},{"key":"ref86","doi-asserted-by":"publisher","DOI":"10.1121\/10.0011809"},{"key":"ref87","article-title":"Mamba: Linear-time sequence modeling with selective state spaces","author":"Gu","year":"2023"},{"key":"ref88","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2018.00750"},{"key":"ref89","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52688.2022.01043"},{"key":"ref90","first-page":"5767","article-title":"Improved training of Wasserstein GANs","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Gulrajani"},{"key":"ref91","doi-asserted-by":"publisher","DOI":"10.1109\/ACCESS.2019.2916887"},{"key":"ref92","doi-asserted-by":"publisher","DOI":"10.1109\/ASRU.2015.7404800"},{"key":"ref93","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-20074-8_25"},{"key":"ref94","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52688.2022.01553"},{"key":"ref95","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2016.90"},{"key":"ref96","doi-asserted-by":"publisher","DOI":"10.1109\/ICASSP.2017.7952132"},{"key":"ref97","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2021.emnlp-main.595"},{"key":"ref98","first-page":"6626","article-title":"GANs trained by a two time-scale update rule converge to a local Nash equilibrium","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Heusel"},{"key":"ref99","article-title":"beta-VAE: Learning basic visual concepts with a constrained variational framework","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Higgins"},{"key":"ref100","doi-asserted-by":"publisher","DOI":"10.1126\/science.1127647"},{"key":"ref101","first-page":"6840","article-title":"Denoising diffusion probabilistic models","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Ho"},{"key":"ref102","first-page":"47:1","article-title":"Cascaded diffusion models for high fidelity image generation","volume":"23","author":"Ho","year":"2022","journal-title":"J. Mach. Learn. Res."},{"key":"ref103","article-title":"Classifier-free diffusion guidance","author":"Ho","year":"2022"},{"key":"ref104","article-title":"Video diffusion models","author":"Ho","year":"2022"},{"key":"ref105","doi-asserted-by":"publisher","DOI":"10.1162\/neco.1997.9.8.1735"},{"key":"ref106","doi-asserted-by":"publisher","DOI":"10.1109\/TPAMI.2019.2911066"},{"key":"ref107","doi-asserted-by":"publisher","DOI":"10.1109\/ICASSP.2019.8682583"},{"key":"ref108","doi-asserted-by":"publisher","DOI":"10.1016\/0004-3702(81)90024-2"},{"key":"ref109","doi-asserted-by":"publisher","DOI":"10.1109\/cvpr52688.2022.01121"},{"key":"ref110","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2016.493"},{"key":"ref111","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52688.2022.01768"},{"key":"ref112","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v36i1.19983"},{"key":"ref113","doi-asserted-by":"publisher","DOI":"10.1109\/ICASSP49357.2023.10096198"},{"key":"ref114","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2017.632"},{"key":"ref115","doi-asserted-by":"publisher","DOI":"10.1016\/j.cviu.2006.10.019"},{"key":"ref116","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2018.00603"},{"key":"ref117","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v36i10.21327"},{"key":"ref118","doi-asserted-by":"publisher","DOI":"10.1109\/TASLP.2016.2585878"},{"key":"ref119","first-page":"29192","article-title":"Embracing consistency: A one-stage approach for spatio-temporal video grounding","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Jin"},{"key":"ref120","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2017.215"},{"key":"ref121","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV48922.2021.00180"},{"key":"ref122","first-page":"1889","article-title":"Deep fragment embeddings for bidirectional image sentence mapping","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Karpathy"},{"key":"ref123","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52688.2022.00246"},{"key":"ref124","first-page":"2758","article-title":"Lip to speech synthesis with visual context attentional GAN","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Kim"},{"key":"ref125","first-page":"21696","article-title":"Variational diffusion models","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Kingma"},{"key":"ref126","article-title":"Auto-encoding variational bayes","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Kingma"},{"key":"ref127","first-page":"17022","article-title":"HiFi-GAN: Generative adversarial networks for efficient and high fidelity speech synthesis","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Kong"},{"key":"ref128","article-title":"DiffWave: A versatile diffusion model for audio synthesis","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Kong"},{"key":"ref129","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2017.83"},{"key":"ref130","doi-asserted-by":"publisher","DOI":"10.1007\/s11263-016-0981-7"},{"key":"ref131","first-page":"1106","article-title":"ImageNet classification with deep convolutional neural networks","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Krizhevsky"},{"key":"ref132","first-page":"14881","article-title":"MelGAN: Generative adversarial networks for conditional waveform synthesis","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Kumar"},{"key":"ref133","article-title":"Diffusion models already have a semantic latent space","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Kwon"},{"key":"ref134","doi-asserted-by":"publisher","DOI":"10.21437\/Interspeech.2021-36"},{"key":"ref135","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR46437.2021.00725"},{"key":"ref136","doi-asserted-by":"publisher","DOI":"10.1145\/3581783.3612293"},{"key":"ref137","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52688.2022.01852"},{"key":"ref138","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV48922.2021.01315"},{"key":"ref139","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2019.01245"},{"key":"ref140","first-page":"5927","article-title":"Visual grounding of learned physical models","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Li"},{"key":"ref141","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v32i1.12233"},{"key":"ref142","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2017.142"},{"key":"ref143","doi-asserted-by":"publisher","DOI":"10.1145\/3656580"},{"key":"ref144","first-page":"74","article-title":"ROUGE: A package for automatic evaluation of summaries","volume-title":"Proc. Text Summarization Branches Out","author":"Lin"},{"key":"ref145","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-10602-1_48"},{"key":"ref146","first-page":"11449","article-title":"Exploring cross-video and cross-modality signals for weakly-supervised audio-visual video parsing","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Lin"},{"key":"ref147","first-page":"274","article-title":"Audiovisual transformer with instance attention for audio-visual event localization","volume-title":"Proc. Asian Conf. Comput. Vis.","author":"Lin"},{"key":"ref148","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2019.00477"},{"key":"ref149","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-19790-1_26"},{"key":"ref150","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2017.100"},{"key":"ref151","article-title":"Use what you have: Video retrieval using representations from collaborative experts","author":"Liu","year":"2019"},{"key":"ref152","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR46437.2021.00556"},{"key":"ref153","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v34i07.6833"},{"key":"ref154","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2015.7298965"},{"key":"ref155","doi-asserted-by":"publisher","DOI":"10.1145\/2816795.2818013"},{"key":"ref156","doi-asserted-by":"publisher","DOI":"10.1023\/b:visi.0000029664.99615.94"},{"key":"ref157","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-46448-0_51"},{"key":"ref158","first-page":"13","article-title":"ViLBERT: Pretraining task-agnostic visiolinguistic representations for vision-and-language tasks","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Lu"},{"key":"ref159","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2017.345"},{"key":"ref160","article-title":"UniVL: A unified video and language pre-training model for multimodal understanding and generation","author":"Luo","year":"2020"},{"key":"ref161","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2016.9"},{"key":"ref162","article-title":"Deep captioning with multimodal recurrent neural networks (m-RNN)","author":"Mao","year":"2014"},{"key":"ref163","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2019.00331"},{"key":"ref164","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52688.2022.00357"},{"key":"ref165","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR46437.2021.00993"},{"key":"ref166","doi-asserted-by":"publisher","DOI":"10.21437\/Interspeech.2020-1026"},{"key":"ref167","article-title":"Learning a text-video embedding from incomplete and heterogeneous data","author":"Miech","year":"2018"},{"key":"ref168","article-title":"Efficient estimation of word representations in vector space","author":"Mikolov","year":"2013"},{"key":"ref169","doi-asserted-by":"publisher","DOI":"10.21437\/interspeech.2022-10770"},{"key":"ref170","article-title":"Conditional generative adversarial nets","author":"Mirza","year":"2014"},{"key":"ref171","article-title":"Symbolic music generation with diffusion models","author":"Mittal","year":"2021"},{"key":"ref172","article-title":"ASL alphabet","author":"Nagaraj","year":"2018"},{"key":"ref173","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR46437.2021.00397"},{"key":"ref174","article-title":"GLIDE: Towards photorealistic image generation and editing with text-guided diffusion models","author":"Nichol","year":"2021"},{"key":"ref175","first-page":"8162","article-title":"Improved denoising diffusion probabilistic models","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Nichol"},{"key":"ref176","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR46437.2021.01251"},{"key":"ref177","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2011.5995586"},{"key":"ref178","doi-asserted-by":"publisher","DOI":"10.21437\/Interspeech.2021-2227"},{"key":"ref179","first-page":"6306","article-title":"Neural discrete representation learning","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"van den Oord"},{"key":"ref180","first-page":"119","article-title":"Do we need sound for sound source localization?","volume-title":"Proc. Asian Conf. Comput. Vis.","author":"Oya"},{"key":"ref181","doi-asserted-by":"publisher","DOI":"10.1145\/3588432.3591500"},{"key":"ref182","first-page":"311","article-title":"Bleu: A method for automatic evaluation of machine translation","volume-title":"Proc. Annu. Meeting Assoc. Comput. Linguistics","author":"Papineni"},{"key":"ref183","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2016.85"},{"key":"ref184","article-title":"Technology, science, and culture: A global vision","author":"Picazo-Vela","year":"2020"},{"key":"ref185","doi-asserted-by":"publisher","DOI":"10.1109\/TASL.2006.872621"},{"key":"ref186","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2017.213"},{"key":"ref187","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR42600.2020.01381"},{"key":"ref188","first-page":"77","article-title":"PointNet: Deep learning on point sets for 3D classification and segmentation","volume-title":"Proc. IEEE\/CVF Conf. Comput. Vis. Pattern Recognit.","author":"Qi"},{"key":"ref189","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR42600.2020.01087"},{"key":"ref190","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-58565-5_18"},{"key":"ref191","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV48922.2021.00194"},{"key":"ref192","first-page":"8748","article-title":"Learning transferable visual models from natural language supervision","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Radford"},{"key":"ref193","article-title":"Language models are unsupervised multitask learners","volume":"1","author":"Radford","year":"2019","journal-title":"OpenAI Blog"},{"key":"ref194","article-title":"Hierarchical text-conditional image generation with CLIP latents","author":"Ramesh","year":"2022"},{"key":"ref195","first-page":"8821","article-title":"Zero-shot text-to-image generation","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Ramesh"},{"key":"ref196","doi-asserted-by":"publisher","DOI":"10.1016\/j.eswa.2020.113794"},{"key":"ref197","first-page":"14837","article-title":"Generating diverse high-fidelity images with VQ-VAE-2","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Razavi"},{"key":"ref198","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2016.13"},{"key":"ref199","first-page":"1060","article-title":"Generative adversarial text to image synthesis","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Reed"},{"key":"ref200","doi-asserted-by":"publisher","DOI":"10.1109\/ICASSP.2001.941023"},{"key":"ref201","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52688.2022.01042"},{"key":"ref202","doi-asserted-by":"publisher","DOI":"10.1007\/s11263-015-0816-y"},{"key":"ref203","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v33i01.330110023"},{"key":"ref204","first-page":"2226","article-title":"Improved techniques for training GANs","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Salimans"},{"key":"ref205","doi-asserted-by":"publisher","DOI":"10.1109\/TNN.2008.2005605"},{"key":"ref206","article-title":"LAION-5B: An open large-scale dataset for training next generation image-text models","author":"Schuhmann","year":"2022"},{"key":"ref207","article-title":"LAION-400M: Open dataset of CLIP-filtered 400 million image-text pairs","author":"Schuhmann","year":"2021"},{"key":"ref208","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2019.01283"},{"key":"ref209","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2018.00458"},{"key":"ref210","doi-asserted-by":"publisher","DOI":"10.1109\/WACV51458.2022.00065"},{"key":"ref211","first-page":"3719","article-title":"Visual reference resolution using attention memory for visual dialog","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Seo"},{"key":"ref212","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/P18-1238"},{"key":"ref213","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2016.499"},{"key":"ref214","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR42600.2020.01086"},{"key":"ref215","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-46448-0_31"},{"key":"ref216","article-title":"Very deep convolutional networks for large-scale image recognition","author":"Simonyan","year":"2014"},{"key":"ref217","first-page":"2256","article-title":"Deep unsupervised learning using nonequilibrium thermodynamics","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Sohl-Dickstein"},{"key":"ref218","first-page":"1849","article-title":"Improved deep metric learning with multi-class n-pair loss objective","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Sohn"},{"key":"ref219","first-page":"1415","article-title":"Maximum likelihood training of score-based diffusion models","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Song"},{"key":"ref220","first-page":"12438","article-title":"Improved techniques for training score-based generative models","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Song"},{"key":"ref221","article-title":"Score-based generative modeling through stochastic differential equations","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Song"},{"key":"ref222","doi-asserted-by":"publisher","DOI":"10.24963\/ijcai.2019\/129"},{"key":"ref223","first-page":"3222","article-title":"Self-supervised predictive learning: A negative-free method for sound source localization in visual scenes","volume-title":"Proc. IEEE\/CVF Conf. Comput. Vis. Pattern Recognit.","author":"Song"},{"key":"ref224","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2018.00017"},{"key":"ref225","doi-asserted-by":"publisher","DOI":"10.1145\/3404835.3463257"},{"key":"ref226","doi-asserted-by":"publisher","DOI":"10.1109\/ICRA48506.2021.9560926"},{"key":"ref227","first-page":"3325","article-title":"Audeo: Audio generation for a silent performance video","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Su"},{"key":"ref228","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV48922.2021.00156"},{"key":"ref229","doi-asserted-by":"publisher","DOI":"10.1109\/TPAMI.2021.3058684"},{"key":"ref230","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2018.00151"},{"key":"ref231","first-page":"2063","article-title":"Discovery of latent 3D keypoints via end-to-end geometric reasoning","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Suwajanakorn"},{"key":"ref232","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2016.308"},{"key":"ref233","doi-asserted-by":"publisher","DOI":"10.1109\/ICASSP.2010.5495701"},{"key":"ref234","doi-asserted-by":"publisher","DOI":"10.1109\/tpami.2024.3356232\/mm1"},{"key":"ref235","article-title":"Improved vector quantized diffusion models","author":"Tang","year":"2022"},{"key":"ref236","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-58536-5_24"},{"key":"ref237","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-20059-5_23"},{"key":"ref238","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-58580-8_26"},{"key":"ref239","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-01216-8_16"},{"key":"ref240","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV48922.2021.00114"},{"key":"ref241","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2016.423"},{"key":"ref242","first-page":"501","article-title":"AIST dance video database: Multi-genre, multi-dancer, and multi-camera database for dance information processing","volume-title":"Proc. Int. Soc. Music Inf. Retrieval Conf.","author":"Tsuchida"},{"key":"ref243","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2018.00165"},{"key":"ref244","article-title":"FVD: A new metric for video generation","author":"Unterthiner","year":"2019","journal-title":"ICLR 2019 Workshop"},{"key":"ref245","doi-asserted-by":"publisher","DOI":"10.48550\/ARXIV.1706.03762"},{"key":"ref246","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2015.7299087"},{"key":"ref247","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2015.7298935"},{"key":"ref248","doi-asserted-by":"publisher","DOI":"10.21437\/Interspeech.2019-1445"},{"key":"ref249","doi-asserted-by":"publisher","DOI":"10.1007\/s11263-019-01251-8"},{"key":"ref250","article-title":"The Caltech-UCSD Birds-200\u20132011 dataset","author":"Wah","year":"2011"},{"key":"ref251","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2015.281"},{"key":"ref252","doi-asserted-by":"publisher","DOI":"10.1007\/s00221-002-1014-z"},{"key":"ref253","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV51070.2023.00366"},{"key":"ref254","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2018.00795"},{"key":"ref255","doi-asserted-by":"publisher","DOI":"10.1145\/3123266.3123326"},{"key":"ref256","article-title":"A comprehensive survey on cross-modal retrieval","author":"Wang","year":"2016"},{"key":"ref257","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2019.00879"},{"key":"ref258","doi-asserted-by":"publisher","DOI":"10.1109\/TPAMI.2018.2797921"},{"key":"ref259","first-page":"5756","article-title":"Diverse and accurate image description using a variational auto-encoder with an additive Gaussian encoding space","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Wang"},{"key":"ref260","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2019.00206"},{"key":"ref261","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2019.00042"},{"key":"ref262","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2019.00468"},{"key":"ref263","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR46437.2021.00504"},{"key":"ref264","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2019.00826"},{"key":"ref265","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2019.00586"},{"key":"ref266","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-19781-9_14"},{"key":"ref267","doi-asserted-by":"publisher","DOI":"10.1016\/0169-7439(87)80084-9"},{"key":"ref268","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2018.00639"},{"key":"ref269","article-title":"The jazz transformer on the front line: Exploring the shortcomings of AI-composed music through quantitative measures","author":"Wu","year":"2020"},{"key":"ref270","doi-asserted-by":"publisher","DOI":"10.1109\/TPAMI.2022.3144984"},{"key":"ref271","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR46437.2021.00138"},{"key":"ref272","doi-asserted-by":"publisher","DOI":"10.1145\/3240508.3240640"},{"key":"ref273","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2019.00639"},{"key":"ref274","doi-asserted-by":"publisher","DOI":"10.1109\/TNNLS.2020.2978386"},{"key":"ref275","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2015.7298801"},{"key":"ref276","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2017.558"},{"key":"ref277","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR46437.2021.00965"},{"key":"ref278","article-title":"Visual entailment task for visually-grounded language learning","author":"Xie","year":"2018"},{"key":"ref279","article-title":"Visual entailment: A novel task for fine-grained image understanding","author":"Xie","year":"2019"},{"key":"ref280","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-46478-7_28"},{"key":"ref281","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2016.571"},{"key":"ref282","doi-asserted-by":"publisher","DOI":"10.5555\/3045118.3045336"},{"key":"ref283","doi-asserted-by":"publisher","DOI":"10.1109\/TPAMI.2023.3275156"},{"key":"ref284","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2018.00143"},{"key":"ref285","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v34i01.5361"},{"key":"ref286","doi-asserted-by":"publisher","DOI":"10.1109\/ICASSP39728.2021.9414040"},{"key":"ref287","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52688.2022.01595"},{"key":"ref288","doi-asserted-by":"publisher","DOI":"10.1145\/3503161.3548291"},{"key":"ref289","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2019.00474"},{"key":"ref290","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2017.538"},{"key":"ref291","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-58568-6_23"},{"key":"ref292","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2019.00478"},{"key":"ref293","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2016.10"},{"key":"ref294","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2016.503"},{"key":"ref295","volume-title":"Automatic Speech Recognition","volume":"1","author":"Yu","year":"2016"},{"key":"ref296","doi-asserted-by":"publisher","DOI":"10.1145\/3503161.3547869"},{"key":"ref297","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-46475-6_5"},{"key":"ref298","doi-asserted-by":"publisher","DOI":"10.24963\/ijcai.2018\/155"},{"key":"ref299","doi-asserted-by":"publisher","DOI":"10.1109\/TPAMI.2022.3204236"},{"key":"ref300","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR46437.2021.00692"},{"key":"ref301","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV48922.2021.00204"},{"key":"ref302","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2019.00377"},{"key":"ref303","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2017.629"},{"key":"ref304","doi-asserted-by":"publisher","DOI":"10.1109\/TPAMI.2018.2856256"},{"key":"ref305","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR46437.2021.01248"},{"key":"ref306","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR46437.2021.00366"},{"key":"ref307","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR42600.2020.01068"},{"key":"ref308","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2019.00182"},{"key":"ref309","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-01246-5_35"},{"key":"ref310","doi-asserted-by":"publisher","DOI":"10.1109\/TNNLS.2018.2876865"},{"key":"ref311","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2019.01064"},{"key":"ref312","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR46437.2021.00416"},{"key":"ref313","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2019.00674"},{"key":"ref314","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2018.00374"},{"key":"ref315","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-58520-4_35"},{"key":"ref316","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2019.00595"},{"key":"ref317","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-19836-6_11"},{"key":"ref318","first-page":"78319","article-title":"Boundary guided learning-free semantic control with diffusion models","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Zhu"},{"key":"ref319","doi-asserted-by":"publisher","DOI":"10.1109\/ICASSP39728.2021.9414296"},{"key":"ref320","article-title":"Discrete contrastive diffusion for cross-modal and conditional generation","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Zhu"},{"key":"ref321","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-58592-1_10"},{"key":"ref322","doi-asserted-by":"publisher","DOI":"10.1109\/TPAMI.2021.3093360"}],"container-title":["IEEE Transactions on Pattern Analysis and Machine Intelligence"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx8\/34\/10746266\/10577498.pdf?arnumber=10577498","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,11,26]],"date-time":"2024-11-26T19:29:44Z","timestamp":1732649384000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/10577498\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,12]]},"references-count":322,"journal-issue":{"issue":"12"},"URL":"https:\/\/doi.org\/10.1109\/tpami.2024.3420239","relation":{},"ISSN":["0162-8828","2160-9292","1939-3539"],"issn-type":[{"value":"0162-8828","type":"print"},{"value":"2160-9292","type":"electronic"},{"value":"1939-3539","type":"electronic"}],"subject":[],"published":{"date-parts":[[2024,12]]}}}