{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,21]],"date-time":"2026-07-21T09:41:45Z","timestamp":1784626905419,"version":"3.55.0"},"reference-count":200,"publisher":"Institute of Electrical and Electronics Engineers (IEEE)","issue":"7","license":[{"start":{"date-parts":[[2025,7,1]],"date-time":"2025-07-01T00:00:00Z","timestamp":1751328000000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/ieeexplore.ieee.org\/Xplorehelp\/downloads\/license-information\/IEEE.html"},{"start":{"date-parts":[[2025,7,1]],"date-time":"2025-07-01T00:00:00Z","timestamp":1751328000000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2025,7,1]],"date-time":"2025-07-01T00:00:00Z","timestamp":1751328000000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"}],"funder":[{"name":"Yongshuo Zong is supported by the United Kingdom Research and Innovation","award":["EP\/S02431X\/1"],"award-info":[{"award-number":["EP\/S02431X\/1"]}]},{"name":"UKRI Centre for Doctoral Training in Biomedical AI at the University of Edinburgh"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["IEEE Trans. Pattern Anal. Mach. Intell."],"published-print":{"date-parts":[[2025,7]]},"DOI":"10.1109\/tpami.2024.3429301","type":"journal-article","created":{"date-parts":[[2024,8,7]],"date-time":"2024-08-07T17:43:56Z","timestamp":1723052636000},"page":"5299-5318","source":"Crossref","is-referenced-by-count":83,"title":["Self-Supervised Multimodal Learning: A Survey"],"prefix":"10.1109","volume":"47","author":[{"ORCID":"https:\/\/orcid.org\/0009-0002-6726-8776","authenticated-orcid":false,"given":"Yongshuo","family":"Zong","sequence":"first","affiliation":[{"name":"School of Informatics, University of Edinburgh, Edinburgh, U.K."}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-5787-5073","authenticated-orcid":false,"given":"Oisin Mac","family":"Aodha","sequence":"additional","affiliation":[{"name":"School of Informatics, University of Edinburgh, Edinburgh, U.K."}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-4867-7486","authenticated-orcid":false,"given":"Timothy M.","family":"Hospedales","sequence":"additional","affiliation":[{"name":"School of Informatics, University of Edinburgh, Edinburgh, U.K."}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"263","reference":[{"key":"ref1","doi-asserted-by":"publisher","DOI":"10.1109\/TPAMI.2018.2798607"},{"key":"ref2","doi-asserted-by":"publisher","DOI":"10.1145\/3656580"},{"key":"ref3","doi-asserted-by":"publisher","DOI":"10.1016\/j.inffus.2021.07.009"},{"key":"ref4","doi-asserted-by":"publisher","DOI":"10.1016\/j.inffus.2021.06.003"},{"key":"ref5","doi-asserted-by":"publisher","DOI":"10.24963\/ijcai.2019\/877"},{"key":"ref6","doi-asserted-by":"publisher","DOI":"10.1038\/s41591-022-01981-2"},{"key":"ref7","article-title":"Multi-modal sensor fusion for auto driving perception: A survey","author":"Huang","year":"2022"},{"key":"ref8","doi-asserted-by":"publisher","DOI":"10.1146\/annurev.psych.59.103006.093639"},{"key":"ref9","doi-asserted-by":"publisher","DOI":"10.1109\/TPAMI.2020.2992393"},{"key":"ref10","doi-asserted-by":"publisher","DOI":"10.1109\/MSP.2021.3134634"},{"key":"ref11","doi-asserted-by":"publisher","DOI":"10.1109\/MSP.2017.2738401"},{"key":"ref12","doi-asserted-by":"publisher","DOI":"10.1109\/TPAMI.2023.3275156"},{"key":"ref13","doi-asserted-by":"publisher","DOI":"10.1109\/TKDE.2021.3090866"},{"key":"ref14","doi-asserted-by":"publisher","DOI":"10.1561\/0600000105"},{"key":"ref15","article-title":"Beyond just vision: A review on self-supervised representation learning on multimodal and temporal data","author":"Deldari","year":"2022"},{"key":"ref16","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2016.90"},{"key":"ref17","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-46466-4_5"},{"key":"ref18","doi-asserted-by":"publisher","DOI":"10.5555\/3524938.3525087"},{"key":"ref19","doi-asserted-by":"publisher","DOI":"10.1109\/cvpr52688.2022.01553"},{"key":"ref20","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-10602-1_48"},{"key":"ref21","first-page":"8748","article-title":"Learning transferable visual models from natural language supervision","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Radford"},{"key":"ref22","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2017.73"},{"key":"ref23","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-58621-8_45"},{"key":"ref24","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2005.202"},{"key":"ref25","doi-asserted-by":"publisher","DOI":"10.1109\/MSP.2017.2732900"},{"key":"ref26","doi-asserted-by":"publisher","DOI":"10.1007\/s11263-016-0981-7"},{"key":"ref27","article-title":"Pixel-BERT: Aligning image pixels with text by deep multi-modal transformers","author":"Huang","year":"2020"},{"key":"ref28","first-page":"5583","article-title":"ViLT: Vision-and-language transformer without convolution or region supervision","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Kim"},{"key":"ref29","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52729.2023.01838"},{"key":"ref30","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52688.2022.01558"},{"key":"ref31","doi-asserted-by":"publisher","DOI":"10.1109\/TKDE.2022.3193569"},{"key":"ref32","first-page":"2949","article-title":"Multimodal learning with deep Boltzmann machines","volume":"15","author":"Srivastava","year":"2014","journal-title":"J. Mach. Learn. Res."},{"key":"ref33","first-page":"8821","article-title":"Zero-shot text-to-image generation","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Ramesh"},{"key":"ref34","article-title":"VisualBERT: A simple and performant baseline for vision and language","author":"Li","year":"2019"},{"key":"ref35","first-page":"1083","article-title":"On deep multi-view representation learning","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Wang"},{"key":"ref36","first-page":"4116","article-title":"Contrastive multi-view representation learning on graphs","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Hassani"},{"key":"ref37","doi-asserted-by":"publisher","DOI":"10.1109\/TNNLS.2015.2498149"},{"key":"ref38","doi-asserted-by":"publisher","DOI":"10.5555\/2969033.2969125"},{"key":"ref39","first-page":"6840","article-title":"Denoising diffusion probabilistic models","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Ho"},{"key":"ref40","doi-asserted-by":"publisher","DOI":"10.1109\/tpami.2023.3305243"},{"key":"ref41","doi-asserted-by":"publisher","DOI":"10.1016\/j.neunet.2021.07.019"},{"key":"ref42","doi-asserted-by":"publisher","DOI":"10.1145\/3626235"},{"key":"ref43","first-page":"7774","article-title":"Cooperative learning of audio and video models from self-supervised synchronization","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Korbar"},{"key":"ref44","first-page":"4733","article-title":"Learning representations from audio-visual spatial alignment","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Morgado"},{"key":"ref45","first-page":"25","article-title":"Self-supervised multimodal versatile networks","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Alayrac"},{"key":"ref46","first-page":"24206","article-title":"VATT: Transformers for multimodal self-supervised learning from raw video, audio and text","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Akbari"},{"key":"ref47","doi-asserted-by":"publisher","DOI":"10.1109\/ICASSP43922.2022.9747631"},{"key":"ref48","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2021.emnlp-main.544"},{"key":"ref49","doi-asserted-by":"publisher","DOI":"10.1016\/j.neucom.2022.07.028"},{"key":"ref50","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52688.2022.00836"},{"key":"ref51","first-page":"4904","article-title":"Scaling up visual and vision-language representation learning with noisy text supervision","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Jia"},{"key":"ref52","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52729.2023.01059"},{"key":"ref53","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52729.2023.02240"},{"key":"ref54","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-19809-0_30"},{"key":"ref55","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV48922.2021.00148"},{"key":"ref56","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52688.2022.00967"},{"key":"ref57","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR46437.2021.01229"},{"key":"ref58","first-page":"435","article-title":"Objects that sound","volume-title":"Proc. Eur. Conf. Comput. Vis.","author":"Arandjelovi"},{"key":"ref59","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-01231-1_39"},{"key":"ref60","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-01246-5_35"},{"key":"ref61","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-19836-6_22"},{"key":"ref62","doi-asserted-by":"publisher","DOI":"10.1109\/TPAMI.2021.3137988"},{"key":"ref63","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2019.00182"},{"key":"ref64","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR42600.2020.01049"},{"key":"ref65","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-58577-8_7"},{"key":"ref66","first-page":"13","article-title":"Vilbert: Pretraining task-agnostic visiolinguistic representations for vision-and-language tasks","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Lu"},{"key":"ref67","first-page":"12888","article-title":"BLIP: Bootstrapping language-image pre-training for unified vision-language understanding and generation","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Li"},{"key":"ref68","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52688.2022.01519"},{"key":"ref69","first-page":"9694","article-title":"Align before fuse: Vision and language representation learning with momentum distillation","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Li"},{"key":"ref70","first-page":"8765","article-title":"Debiased contrastive learning","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Chuang"},{"key":"ref71","first-page":"9904","article-title":"A theoretical analysis of contrastive unsupervised representation learning","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Saunshi"},{"key":"ref72","article-title":"Contrastive learning with hard negative samples","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Robinson"},{"key":"ref73","first-page":"21798","article-title":"Hard negative mixing for contrastive learning","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Kalantidis"},{"key":"ref74","article-title":"Towards democratizing joint-embedding self-supervised learning","author":"Bordes","year":"2023"},{"key":"ref75","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-01264-9_9"},{"key":"ref76","first-page":"9758","article-title":"Self-supervised learning by cross-modal audio-video clustering","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Alwassel"},{"key":"ref77","first-page":"4660","article-title":"Labelling unlabelled videos from scratch with multi-modal self-supervision","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Asano"},{"key":"ref78","article-title":"Self-labelling via simultaneous clustering and representation learning","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Asano"},{"key":"ref79","doi-asserted-by":"publisher","DOI":"10.1109\/cvpr.2019.00947"},{"key":"ref80","article-title":"Learning audio-visual speech representation by masked multimodal cluster prediction","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Shi"},{"key":"ref81","first-page":"21157","article-title":"U-HuBERT: Unified mixed-modal speech pretraining and zero-shot transfer to unlabeled modality","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Hsu"},{"key":"ref82","first-page":"4171","article-title":"BERT: Pre-training of deep bidirectional transformers for language understanding","volume-title":"Proc. Conf. North Amer. Chapter Assoc. Comput. Linguistics","author":"Devlin"},{"key":"ref83","article-title":"Improving language understanding with unsupervised learning","author":"Radford","year":"2018"},{"key":"ref84","article-title":"VL-BEiT: Generative vision-language pretraining","author":"Bao","year":"2022"},{"key":"ref85","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2019.00756"},{"key":"ref86","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR46437.2021.00560"},{"key":"ref87","article-title":"Unified-IO: A unified model for vision, language, and multi-modal tasks","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Lu"},{"key":"ref88","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR42600.2020.00877"},{"key":"ref89","first-page":"4797","article-title":"Conditional image generation with PixelCNN decoders","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Van den Oord"},{"key":"ref90","article-title":"SimVLM: Simple visual language model pretraining with weak supervision","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Wang"},{"key":"ref91","article-title":"OPT: Omni-perception pre-trainer for cross-modal understanding and generation","author":"Liu","year":"2021"},{"key":"ref92","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2021.acl-long.202"},{"key":"ref93","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV48922.2021.00791"},{"key":"ref94","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52688.2022.01032"},{"key":"ref95","first-page":"32897","article-title":"VLMo: Unified vision-language pre-training with mixture-of-modality-experts","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Wang"},{"key":"ref96","article-title":"UniViLM: A unified video and language pre-training model for multimodal understanding and generation","author":"Luo","year":"2020"},{"key":"ref97","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52688.2022.01589"},{"key":"ref98","article-title":"Contrastive audio-visual masked autoencoder","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Gong"},{"key":"ref99","article-title":"MAViL: Masked audio-video learners","author":"Huang","year":"2022"},{"key":"ref100","first-page":"37309","article-title":"Deep bidirectional language-knowledge graph pretraining","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Yasunaga"},{"key":"ref101","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-58577-8_8"},{"key":"ref102","doi-asserted-by":"publisher","DOI":"10.1109\/LSP.2022.3224688"},{"key":"ref103","first-page":"22680","article-title":"VLMixer: Unpaired vision-language pre-training via cross-modal cutmix","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Wang"},{"key":"ref104","article-title":"Hierarchical perceiver","author":"Carreira","year":"2022"},{"key":"ref105","first-page":"4651","article-title":"Perceiver: General perception with iterative attention","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Jaegle"},{"key":"ref106","article-title":"One model, multiple modalities: A sparsely activated approach for text, sound, image, video and code","author":"Dai","year":"2022"},{"key":"ref107","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52688.2022.01939"},{"key":"ref108","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2021.findings-acl.370"},{"key":"ref109","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52688.2022.01743"},{"key":"ref110","doi-asserted-by":"publisher","DOI":"10.48550\/ARXIV.1706.03762"},{"key":"ref111","article-title":"On the opportunities and risks of foundation models","author":"Bommasani","year":"2021"},{"key":"ref112","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2015.7298701"},{"key":"ref113","first-page":"200","article-title":"Multimodal few-shot learning with frozen language models","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Tsimpoukelli"},{"key":"ref114","article-title":"Linearly mapping from image to text space","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Merullo"},{"key":"ref115","first-page":"17283","article-title":"Grounding language models to images for multimodal inputs and outputs","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Koh"},{"key":"ref116","first-page":"19730","article-title":"BLIP-2: Bootstrapping language-image pre-training with frozen image encoders and large language models","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Li"},{"key":"ref117","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52729.2023.01044"},{"key":"ref118","article-title":"ClipCap: Clip prefix for image captioning","author":"Mokady","year":"2021"},{"key":"ref119","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2022.findings-emnlp.179"},{"key":"ref120","first-page":"23716","article-title":"Flamingo: A visual language model for few-shot learning","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Alayrac"},{"key":"ref121","first-page":"1877","article-title":"Language models are few-shot learners","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Brown"},{"key":"ref122","first-page":"24824","article-title":"Chain-of-thought prompting elicits reasoning in large language models","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Wei"},{"key":"ref123","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR42600.2020.00990"},{"key":"ref124","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2021.naacl-main.420"},{"key":"ref125","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52688.2022.01599"},{"key":"ref126","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2019.01042"},{"key":"ref127","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2017.244"},{"key":"ref128","first-page":"5888","article-title":"DM2C: Deep mixed-modal clustering","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Jiang"},{"key":"ref129","first-page":"7892","article-title":"MACK: Multimodal aligned conceptual knowledge for unpaired image-text matching","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Huang"},{"key":"ref130","doi-asserted-by":"publisher","DOI":"10.1109\/TMM.2023.3275873"},{"key":"ref131","doi-asserted-by":"publisher","DOI":"10.1145\/3394171.3413869"},{"key":"ref132","first-page":"22605","article-title":"COOT: Cooperative hierarchical transformer for video-text representation learning","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Ging"},{"key":"ref133","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.emnlp-main.161"},{"key":"ref134","doi-asserted-by":"publisher","DOI":"10.1561\/2200000073"},{"key":"ref135","first-page":"1542","article-title":"Graph optimal transport for cross-domain alignment","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Chen"},{"key":"ref136","first-page":"433","article-title":"A fast proximal point method for computing exact wasserstein distance","volume-title":"Proc. Conf. Uncertainty Artif. Intell.","author":"Xie"},{"key":"ref137","doi-asserted-by":"publisher","DOI":"10.1162\/0899766042321814"},{"key":"ref138","first-page":"1247","article-title":"Deep canonical correlation analysis","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Andrew"},{"key":"ref139","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2018.00161"},{"key":"ref140","doi-asserted-by":"publisher","DOI":"10.1016\/j.inffus.2022.09.006"},{"key":"ref141","doi-asserted-by":"publisher","DOI":"10.1016\/S0004-3702(96)00034-3"},{"key":"ref142","doi-asserted-by":"publisher","DOI":"10.1109\/TPAMI.2016.2577031"},{"key":"ref143","article-title":"VL-BERT: Pre-training of generic visual-linguistic representations","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Su"},{"key":"ref144","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v34i07.6795"},{"key":"ref145","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D19-1514"},{"key":"ref146","article-title":"The information bottleneck method","author":"Tishby","year":"2000"},{"key":"ref147","doi-asserted-by":"publisher","DOI":"10.3390\/e26030252"},{"key":"ref148","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR46437.2021.01144"},{"key":"ref149","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52688.2022.00966"},{"key":"ref150","doi-asserted-by":"publisher","DOI":"10.1145\/3406095"},{"key":"ref151","doi-asserted-by":"publisher","DOI":"10.1016\/j.neunet.2018.07.006"},{"key":"ref152","doi-asserted-by":"publisher","DOI":"10.1109\/mcs.2025.3534477"},{"key":"ref153","doi-asserted-by":"crossref","DOI":"10.1109\/TNN.1998.712192","volume-title":"Introduction to Reinforcement Learning","author":"Sutton","year":"1998"},{"key":"ref154","first-page":"2863","article-title":"Action-conditional video prediction using deep networks in atari games","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Oh"},{"key":"ref155","first-page":"2455","article-title":"Recurrent world models facilitate policy evolution","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Ha"},{"key":"ref156","first-page":"2555","article-title":"Learning latent dynamics for planning from pixels","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Hafner"},{"key":"ref157","first-page":"5092","article-title":"Learning to poke by poking: Experiential learning of intuitive physics","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Agrawal"},{"key":"ref158","first-page":"564","article-title":"Learning predictive representations for deformable objects using contrastive estimation","volume-title":"Proc. Conf. Robot Learn.","author":"Yan"},{"key":"ref159","article-title":"Representation learning with contrastive predictive coding","author":"van den Oord","year":"2018"},{"key":"ref160","doi-asserted-by":"publisher","DOI":"10.1109\/icra.2019.8793485"},{"key":"ref161","article-title":"Contrastive learning of medical visual representations from paired images and text","author":"Zhang","year":"2020"},{"key":"ref162","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV48922.2021.00391"},{"key":"ref163","doi-asserted-by":"publisher","DOI":"10.1038\/s41551-022-00936-9"},{"key":"ref164","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2022.emnlp-main.256"},{"key":"ref165","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52688.2022.02024"},{"key":"ref166","first-page":"18433","article-title":"CoMIR: Contrastive multimodal image representation for registration","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Pielawski"},{"key":"ref167","doi-asserted-by":"publisher","DOI":"10.1109\/MGRS.2022.3198244"},{"key":"ref168","doi-asserted-by":"publisher","DOI":"10.1109\/TGRS.2021.3128072"},{"key":"ref169","doi-asserted-by":"publisher","DOI":"10.1109\/LGRS.2022.3195259"},{"key":"ref170","doi-asserted-by":"publisher","DOI":"10.1109\/TGRS.2021.3109957"},{"key":"ref171","doi-asserted-by":"publisher","DOI":"10.1016\/j.jag.2022.103130"},{"key":"ref172","article-title":"LLaMA: Open and efficient foundation language models","author":"Touvron","year":"2023"},{"key":"ref173","doi-asserted-by":"publisher","DOI":"10.1093\/nsr\/nwae403"},{"key":"ref174","article-title":"PaLM-E: An embodied multimodal language model","author":"Driess","year":"2023"},{"key":"ref175","article-title":"Making LLaMA SEE and draw with SEED tokenizer","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Ge"},{"key":"ref176","article-title":"Emu: Generative pretraining in multimodality","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Sun"},{"key":"ref177","article-title":"LAION-5B: An open large-scale dataset for training next generation image-text models","author":"Schuhmann","year":"2022"},{"key":"ref178","article-title":"The de-democratization of AI: Deep learning and the compute divide in artificial intelligence research","author":"Ahmed","year":"2020"},{"key":"ref179","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-20059-5_14"},{"key":"ref180","doi-asserted-by":"publisher","DOI":"10.1162\/tacl_a_00385"},{"key":"ref181","first-page":"19523","article-title":"Beyond neural scaling laws: Beating power law scaling via data pruning","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Sorscher"},{"key":"ref182","first-page":"26809","article-title":"Platon: Pruning large transformer models with upper confidence bound of weight importance","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Zhang"},{"key":"ref183","article-title":"Supervision exists everywhere: A data efficient contrastive language-image pre-training paradigm","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Li"},{"key":"ref184","first-page":"1196","article-title":"Learning with noisy labels","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Natarajan"},{"key":"ref185","doi-asserted-by":"publisher","DOI":"10.1109\/TNNLS.2022.3152527"},{"key":"ref186","article-title":"When and why vision-language models behave like bags-of-words, and what to do about it?","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Yuksekgonul"},{"key":"ref187","first-page":"24291","article-title":"Robustness in multimodal learning under train-test modality mismatch","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"McKinzie"},{"key":"ref188","article-title":"Extracting training data from diffusion models","author":"Carlini","year":"2023"},{"key":"ref189","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2022.gebnlp-1.10"},{"key":"ref190","article-title":"Evaluating clip: Towards characterization of broader capabilities and downstream implications","author":"Agarwal","year":"2021"},{"key":"ref191","article-title":"Multimodal datasets: Misogyny, pornography, and malignant stereotypes","author":"Birhane","year":"2021"},{"key":"ref192","first-page":"17612","article-title":"Mind the gap: Understanding the modality gap in multi-modal contrastive representation learning","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Liang"},{"key":"ref193","article-title":"Language modelling with pixels","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Rust"},{"key":"ref194","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2019.00677"},{"key":"ref195","first-page":"1298","article-title":"Data2vec: A general framework for self-supervised learning in speech, vision and language","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Baevski"},{"key":"ref196","doi-asserted-by":"publisher","DOI":"10.1038\/s41587-022-01618-2"},{"key":"ref197","article-title":"Emergent abilities of large language models","author":"Wei","year":"2022","journal-title":"Trans. Mach. Learn. Res."},{"key":"ref198","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52729.2023.01457"},{"key":"ref199","article-title":"A cookbook of self-supervised learning","author":"Balestriero","year":"2023"},{"key":"ref200","article-title":"Introducing ChatGPT","year":"2023"}],"container-title":["IEEE Transactions on Pattern Analysis and Machine Intelligence"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx8\/34\/11026037\/10630605.pdf?arnumber=10630605","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,6]],"date-time":"2025-06-06T04:14:51Z","timestamp":1749183291000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/10630605\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,7]]},"references-count":200,"journal-issue":{"issue":"7"},"URL":"https:\/\/doi.org\/10.1109\/tpami.2024.3429301","relation":{},"ISSN":["0162-8828","2160-9292","1939-3539"],"issn-type":[{"value":"0162-8828","type":"print"},{"value":"2160-9292","type":"electronic"},{"value":"1939-3539","type":"electronic"}],"subject":[],"published":{"date-parts":[[2025,7]]}}}