{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,31]],"date-time":"2026-10-31T20:36:43Z","timestamp":1793479003357,"version":"4.5.0"},"reference-count":310,"publisher":"Institute of Electrical and Electronics Engineers (IEEE)","issue":"10","license":[{"start":{"date-parts":[[2023,10,1]],"date-time":"2023-10-01T00:00:00Z","timestamp":1696118400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/legalcode"}],"funder":[{"name":"RAEng Research Chair"},{"name":"NIHR Research Professorship"},{"DOI":"10.13039\/501100013373","name":"NIHR Oxford Biomedical Research Centre","doi-asserted-by":"publisher","id":[{"id":"10.13039\/501100013373","id-type":"DOI","asserted-by":"publisher"}]},{"name":"InnoHK Centre for Cerebro-cardiovascular Health Engineering"},{"name":"Oxford Pandemic Sciences Institute"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["IEEE Trans. Pattern Anal. Mach. Intell."],"published-print":{"date-parts":[[2023,10]]},"DOI":"10.1109\/tpami.2023.3275156","type":"journal-article","created":{"date-parts":[[2023,5,11]],"date-time":"2023-05-11T13:31:12Z","timestamp":1683811872000},"page":"12113-12132","source":"Crossref","is-referenced-by-count":984,"title":["Multimodal Learning With Transformers: A Survey"],"prefix":"10.1109","volume":"45","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-6431-1110","authenticated-orcid":false,"given":"Peng","family":"Xu","sequence":"first","affiliation":[{"name":"Tsinghua University, Beijing, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-9284-2955","authenticated-orcid":false,"given":"Xiatian","family":"Zhu","sequence":"additional","affiliation":[{"name":"University of Surrey, Guildford, U.K."}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-9848-8555","authenticated-orcid":false,"given":"David A.","family":"Clifton","sequence":"additional","affiliation":[{"name":"University of Oxford, Oxford, U.K."}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"263","reference":[{"key":"ref57","article-title":"Survey: Transformer based video-language pre-training","author":"ruan","year":"2021"},{"key":"ref207","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR42600.2020.01315"},{"key":"ref56","article-title":"Video transformers: A survey","author":"selva","year":"2022"},{"key":"ref208","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-88480-2_63"},{"key":"ref59","article-title":"Vision-language intelligence: Tasks, representation learning, and large models","author":"li","year":"2022"},{"key":"ref205","article-title":"Zero-shot text-to-image generation","author":"ramesh","year":"2021"},{"key":"ref58","article-title":"VLP: A survey on vision-language pre-training","author":"chen","year":"2022"},{"key":"ref206","first-page":"4153","article-title":"UC$^{2}$2: Universal cross-lingual cross-modal vision-and-language pre-training","author":"zhou","year":"2021","journal-title":"Proc IEEE Conf Comput Vis Pattern Recognit"},{"key":"ref53","article-title":"A survey on vision transformer","author":"han","year":"2020"},{"key":"ref203","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR46437.2021.01278"},{"key":"ref52","article-title":"A survey of visual transformers","author":"liu","year":"2021"},{"key":"ref204","article-title":"KD-VLP: Improving end-to-end vision-and-language pretraining with object knowledge distillation","author":"liu","year":"2021"},{"key":"ref55","article-title":"Transformers in medical imaging: A survey","author":"shamshad","year":"2022"},{"key":"ref201","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-83527-9_27"},{"key":"ref54","doi-asserted-by":"publisher","DOI":"10.1007\/s41095-021-0247-3"},{"key":"ref202","first-page":"91","article-title":"Faster R-CNN: Towards real-time object detection with region proposal networks","author":"ren","year":"2015","journal-title":"Proc Int Conf Neural Inf Process"},{"key":"ref209","article-title":"FILIP: Fine-grained interactive language-image pre-training","author":"yao","year":"2021"},{"key":"ref210","article-title":"Large-scale adversarial training for vision-and-language representation learning","author":"gan","year":"2020"},{"key":"ref211","article-title":"ERNIE-ViLG: Unified generative pre-training for bidirectional vision-language generation","author":"zhang","year":"2021"},{"key":"ref51","article-title":"Transformers in vision: A survey","author":"khan","year":"2021"},{"key":"ref50","first-page":"270","article-title":"Visualizing transformers for NLP: A brief survey","author":"bra?oveanu","year":"2020","journal-title":"Proc of the Int Conf on Information Visualisation"},{"key":"ref46","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR42600.2020.00092"},{"key":"ref218","first-page":"214","article-title":"Multi-modal transformer for video retrieval","author":"gabeur","year":"2020","journal-title":"Proc Eur Conf Comput Vis"},{"key":"ref45","article-title":"Language-agnostic representation learning of source code from structure and context","author":"z\u00fcgner","year":"2021"},{"key":"ref219","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52688.2022.01939"},{"key":"ref48","article-title":"A survey of transformers","author":"lin","year":"2021"},{"key":"ref216","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52688.2022.00116"},{"key":"ref47","doi-asserted-by":"publisher","DOI":"10.24963\/ijcai.2019\/825"},{"key":"ref217","doi-asserted-by":"publisher","DOI":"10.1109\/ICASSP39728.2021.9414638"},{"key":"ref42","article-title":"LayoutLMv2: Multi-modal pre-training for visually-rich document understanding","author":"xu","year":"2020"},{"key":"ref214","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52688.2022.01180"},{"key":"ref41","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR46437.2021.00199"},{"key":"ref215","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR46437.2021.00700"},{"key":"ref44","article-title":"GraphCodeBERT: Pre-training code representations with data flow","author":"guo","year":"2020"},{"key":"ref212","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR46437.2021.01246"},{"key":"ref43","article-title":"Longformer: The long-document transformer","author":"beltagy","year":"2020"},{"key":"ref213","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52688.2022.01187"},{"key":"ref49","article-title":"Efficient transformers: A survey","author":"tay","year":"2020"},{"key":"ref8","first-page":"6621","article-title":"Speech-T: Transducer for text to speech and beyond","author":"chen","year":"2021","journal-title":"Proc Int Conf Neural Inf Process"},{"key":"ref7","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2019.00756"},{"key":"ref9","article-title":"Learning transferable visual models from natural language supervision","author":"radford","year":"2021"},{"key":"ref4","article-title":"BERT: Pre-training of deep bidirectional transformers for language understanding","author":"devlin","year":"2018"},{"key":"ref3","first-page":"4651","article-title":"Perceiver: General perception with iterative attention","author":"jaegle","year":"2021","journal-title":"Proc Int Conf Mach Learn"},{"key":"ref6","first-page":"213","article-title":"End-to-end object detection with transformers","author":"carion","year":"2020","journal-title":"Proc Eur Conf Comput Vis"},{"key":"ref5","article-title":"An image is worth 16x16 words: Transformers for image recognition at scale","author":"dosovitskiy","year":"2020"},{"key":"ref100","first-page":"12116","article-title":"Do vision transformers see like convolutional neural networks?","author":"raghu","year":"2021","journal-title":"Proc Int Conf Neural Inf Process"},{"key":"ref221","article-title":"End-to-end referring video object segmentation with multimodal transformers","author":"botach","year":"2021"},{"key":"ref101","article-title":"How to understand masked autoencoders","author":"cao","year":"2022"},{"key":"ref222","doi-asserted-by":"publisher","DOI":"10.1145\/3404835.3462838"},{"key":"ref40","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2019.00226"},{"key":"ref220","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-19781-9_14"},{"key":"ref35","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2018.00844"},{"key":"ref306","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52688.2022.02072"},{"key":"ref34","doi-asserted-by":"publisher","DOI":"10.1109\/TKDE.2022.3224228"},{"key":"ref307","article-title":"SLIP: Self-supervision meets language-image pre-training","author":"mu","year":"2021"},{"key":"ref37","doi-asserted-by":"publisher","DOI":"10.1109\/TCSVT.2020.3014491"},{"key":"ref304","article-title":"VALSE: A task-independent benchmark for vision and language models centered on linguistic phenomena","author":"parcalabescu","year":"2021"},{"key":"ref36","doi-asserted-by":"publisher","DOI":"10.1109\/TCSVT.2020.3003048"},{"key":"ref305","article-title":"VL-CheckList: Evaluating pre-trained vision-language models with objects, attributes and relations","author":"zhao","year":"2022"},{"key":"ref31","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV48922.2021.00292"},{"key":"ref302","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2021.emnlp-main.775"},{"key":"ref30","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV48922.2021.01135"},{"key":"ref303","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV48922.2021.00045"},{"key":"ref33","article-title":"Learning knowledge graph-based world models of textual environments","author":"ammanabrolu","year":"2021"},{"key":"ref300","doi-asserted-by":"publisher","DOI":"10.1162\/tacl_a_00385"},{"key":"ref32","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR46437.2021.01389"},{"key":"ref301","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2021.findings-acl.318"},{"key":"ref39","article-title":"FaceFormer: Speech-driven 3D facial animation with transformers","author":"fan","year":"2021"},{"key":"ref38","doi-asserted-by":"publisher","DOI":"10.1145\/3528223.3530068"},{"key":"ref308","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2021.findings-acl.370"},{"key":"ref309","article-title":"BLIP: Bootstrapping language-image pre-training for unified vision-language understanding and generation","author":"li","year":"2022"},{"key":"ref310","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR46437.2021.00553"},{"key":"ref24","first-page":"7664","article-title":"SADGA: Structure-aware dual graph aggregation network for text-to-SQL","author":"cai","year":"2021","journal-title":"Proc Int Conf Neural Inf Process"},{"key":"ref23","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR46437.2021.01268"},{"key":"ref26","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR46437.2021.01522"},{"key":"ref25","article-title":"Speech-to-SQL: Towards speech-driven SQL query generation from natural language question","author":"song","year":"2022"},{"key":"ref20","article-title":"LayoutTransformer: Layout generation and completion with self-attention","author":"gupta","year":"2020"},{"key":"ref22","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52688.2022.01888"},{"key":"ref21","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR46437.2021.00373"},{"key":"ref28","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR46437.2021.00137"},{"key":"ref27","first-page":"17021","article-title":"ProTo: Program-guided transformer for program-guided tasks","author":"zhao","year":"2021","journal-title":"Proc Int Conf Neural Inf Process"},{"key":"ref29","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR46437.2021.01658"},{"key":"ref200","article-title":"MAM: Masked acoustic modeling for end-to-end speech-to-text translation","author":"chen","year":"2020"},{"key":"ref128","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV48922.2021.00128"},{"key":"ref249","first-page":"4514","article-title":"Probing inter-modality: Visual parsing with self-attention for vision-and-language pre-training","author":"xue","year":"2021","journal-title":"Proc Int Conf Neural Inf Process"},{"key":"ref129","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR46437.2021.01628"},{"key":"ref97","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV48922.2021.00951"},{"key":"ref126","doi-asserted-by":"publisher","DOI":"10.1007\/s11263-016-0981-7"},{"key":"ref247","doi-asserted-by":"publisher","DOI":"10.1109\/5.664274"},{"key":"ref96","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV48922.2021.00950"},{"key":"ref127","first-page":"1143","article-title":"Im2Text: Describing images using 1 million captioned photographs","author":"ordonez","year":"2011","journal-title":"Proc Int Conf Neural Inf Process"},{"key":"ref248","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-01231-1_39"},{"key":"ref99","article-title":"Vision transformers are robust learners","author":"paul","year":"2021"},{"key":"ref124","first-page":"740","article-title":"Microsoft COCO: Common objects in context","author":"lin","year":"2014","journal-title":"Proc Eur Conf Comput Vis"},{"key":"ref245","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR46437.2021.00693"},{"key":"ref98","article-title":"BEiT: BERT pre-training of image transformers","author":"bao","year":"2021"},{"key":"ref125","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2015.279"},{"key":"ref246","doi-asserted-by":"publisher","DOI":"10.1145\/3447548.3467206"},{"key":"ref93","first-page":"10347","article-title":"Training data-efficient image transformers & distillation through attention","author":"touvron","year":"2021","journal-title":"Proc Int Conf Mach Learn"},{"key":"ref133","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR46437.2021.01115"},{"key":"ref254","doi-asserted-by":"publisher","DOI":"10.1109\/WACV51458.2022.00217"},{"key":"ref92","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR46437.2021.01212"},{"key":"ref134","doi-asserted-by":"publisher","DOI":"10.1109\/TPAMI.2018.2889052"},{"key":"ref255","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR42600.2020.01043"},{"key":"ref95","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV48922.2021.00986"},{"key":"ref131","article-title":"MultiModalQA: Complex question answering over text, tables and images","author":"talmor","year":"2021"},{"key":"ref252","article-title":"COOT: Cooperative hierarchical transformer for video-text representation learning","author":"ging","year":"2020"},{"key":"ref94","article-title":"Toward transformer-based object detection","author":"beal","year":"2020"},{"key":"ref132","article-title":"VALUE: A multi-task benchmark for video-and-language understanding evaluation","author":"li","year":"2021"},{"key":"ref253","article-title":"Support-set bottlenecks for video-text representation learning","author":"patrick","year":"2020"},{"key":"ref250","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV48922.2021.00114"},{"key":"ref130","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV48922.2021.00205"},{"key":"ref251","doi-asserted-by":"publisher","DOI":"10.21437\/Interspeech.2020-3139"},{"key":"ref91","first-page":"1691","article-title":"Generative pretraining from pixels","author":"chen","year":"2020","journal-title":"Proc Int Conf Mach Learn"},{"key":"ref90","article-title":"XLNet: Generalized autoregressive pretraining for language understanding","author":"yang","year":"2019","journal-title":"Proc Int Conf Neural Inf Process"},{"key":"ref89","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/P19-1285"},{"key":"ref139","article-title":"WenLan: Bridging vision and language by large-scale multi-modal pre-training","author":"huo","year":"2021"},{"key":"ref86","article-title":"PaLi: A jointly-scaled multilingual language-image model","author":"chen","year":"2022"},{"key":"ref137","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV48922.2021.01157"},{"key":"ref258","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52688.2022.01509"},{"key":"ref85","article-title":"Image as a foreign language: BEiT pretraining for all vision and vision-language tasks","author":"wang","year":"2022"},{"key":"ref138","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR46437.2021.00356"},{"key":"ref259","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52688.2022.01595"},{"key":"ref88","article-title":"Improving language understanding by generative pre-training","author":"radford","year":"2018"},{"key":"ref135","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2017.83"},{"key":"ref256","first-page":"18513","article-title":"Explainable semantic space by grounding language to vision with cross-modal contrastive learning","author":"zhang","year":"2021","journal-title":"Proc Int Conf Neural Inf Process"},{"key":"ref87","article-title":"BART: Denoising sequence-to-sequence pre-training for natural language generation, translation, and comprehension","author":"lewis","year":"2019"},{"key":"ref136","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2017.121"},{"key":"ref257","first-page":"28442","article-title":"End-to-end multi-modal video temporal grounding","author":"chen","year":"2021","journal-title":"Proc Int Conf Neural Inf Process"},{"key":"ref82","first-page":"200","article-title":"Multimodal few-shot learning with frozen language models","author":"tsimpoukelli","year":"2021","journal-title":"Proc Int Conf Neural Inf Process"},{"key":"ref144","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV48922.2021.00204"},{"key":"ref265","article-title":"CLIP2Video: Mastering video-text retrieval via image CLIP","author":"fang","year":"2021"},{"key":"ref81","article-title":"DeepChange: A large long-term person re-identification benchmark with clothes change","author":"xu","year":"2021"},{"key":"ref145","article-title":"Learning representations from audio-visual spatial alignment","author":"morgado","year":"2020"},{"key":"ref266","first-page":"13988","article-title":"CLIP-It! Language-guided video summarization","author":"narasimhan","year":"2021","journal-title":"Proc Int Conf Neural Inf Process"},{"key":"ref84","first-page":"23716","article-title":"Flamingo: A visual language model for few-shot learning","author":"alayrac","year":"2022","journal-title":"Proc Int Conf Neural Inf Process"},{"key":"ref142","article-title":"Scaling up vision-language pre-training for image captioning","author":"hu","year":"2021"},{"key":"ref263","article-title":"Align and prompt: Video-and-language pre-training with entity prompts","author":"li","year":"2021"},{"key":"ref83","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52688.2022.00516"},{"key":"ref143","article-title":"LAION-400M: Open dataset of CLIP-filtered 400 million image-text pairs","author":"schuhmann","year":"2021"},{"key":"ref264","article-title":"CLIP4Clip: An empirical study of CLIP for end to end video clip retrieval","author":"luo","year":"2021"},{"key":"ref140","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV48922.2021.00171"},{"key":"ref261","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR46437.2021.00725"},{"key":"ref141","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2019.00272"},{"key":"ref262","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV48922.2021.01136"},{"key":"ref80","doi-asserted-by":"publisher","DOI":"10.1109\/BIBM49941.2020.9313289"},{"key":"ref260","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2021.emnlp-main.544"},{"key":"ref79","doi-asserted-by":"publisher","DOI":"10.1038\/s41598-020-62922-y"},{"key":"ref108","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v34i07.6795"},{"key":"ref229","first-page":"847","article-title":"TVT: Two-view transformer network for video captioning","author":"chen","year":"2018","journal-title":"Proc 10th Asian Conf Mach Learn"},{"key":"ref78","doi-asserted-by":"publisher","DOI":"10.1109\/TPAMI.2022.3148853"},{"key":"ref109","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D19-1219"},{"key":"ref106","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-58577-8_7"},{"key":"ref227","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2021.findings-acl.223"},{"key":"ref107","article-title":"Learning video representations using contrastive bidirectional transformer","author":"sun","year":"2019"},{"key":"ref228","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV48922.2021.01343"},{"key":"ref75","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV48922.2021.00168"},{"key":"ref104","article-title":"VisualBERT: A simple and performant baseline for vision and language","author":"li","year":"2019"},{"key":"ref225","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV48922.2021.01560"},{"key":"ref74","first-page":"13328","article-title":"Curriculum learning for vision-and-language navigation","author":"zhang","year":"2021","journal-title":"Proc Int Conf Neural Inf Process"},{"key":"ref105","article-title":"VL-BERT: Pre-training of generic visual-linguistic representations","author":"su","year":"2019"},{"key":"ref226","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV48922.2021.01563"},{"key":"ref77","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR46437.2021.01312"},{"key":"ref102","article-title":"ViLBERT: Pretraining task-agnostic visiolinguistic representations for vision-and-language tasks","author":"lu","year":"2019"},{"key":"ref223","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV48922.2021.01359"},{"key":"ref76","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR46437.2021.01526"},{"key":"ref103","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D19-1514"},{"key":"ref224","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR42600.2020.00380"},{"key":"ref71","doi-asserted-by":"publisher","DOI":"10.1145\/3534678.3539151"},{"key":"ref111","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR42600.2020.01045"},{"key":"ref232","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV48922.2021.00677"},{"key":"ref70","doi-asserted-by":"publisher","DOI":"10.1109\/ICCVW54120.2021.00357"},{"key":"ref112","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-58577-8_8"},{"key":"ref233","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2019.00473"},{"key":"ref73","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR46437.2021.00169"},{"key":"ref230","article-title":"SwinBERT: End-to-end transformers with sparse attention for video captioning","author":"lin","year":"2021"},{"key":"ref72","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR46437.2021.01112"},{"key":"ref110","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v34i07.7005"},{"key":"ref231","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR46437.2021.00030"},{"key":"ref68","article-title":"MERLOT: Multimodal neural script knowledge models","author":"zellers","year":"2021"},{"key":"ref119","article-title":"Scaling up visual and vision-language representation learning with noisy text supervision","author":"jia","year":"2021"},{"key":"ref67","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR46437.2021.00258"},{"key":"ref117","article-title":"UniVL: A unified video and language pre-training model for multimodal understanding and generation","author":"luo","year":"2020"},{"key":"ref238","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v33i01.33016706"},{"key":"ref69","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v35i14.17534"},{"key":"ref118","article-title":"A simple baseline for zero-shot semantic segmentation with pre-trained vision-language model","author":"xu","year":"2021"},{"key":"ref239","article-title":"CogView: Mastering text-to-image generation via transformers","author":"ding","year":"2021"},{"key":"ref64","doi-asserted-by":"publisher","DOI":"10.1109\/TITS.2020.2972974"},{"key":"ref115","article-title":"ImageBERT: Cross-modal pre-training with large-scale weak-supervised image-text data","author":"qi","year":"2020"},{"key":"ref236","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v35i3.16328"},{"key":"ref63","author":"lazarus","year":"1976","journal-title":"Multimodal Behavior Therapy"},{"key":"ref116","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.emnlp-main.161"},{"key":"ref237","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR46437.2021.01245"},{"key":"ref66","first-page":"7357","article-title":"SOAT: A scene-and object-aware transformer for vision-and-language navigation","author":"moudgil","year":"2021","journal-title":"Proc Int Conf Neural Inf Process"},{"key":"ref113","article-title":"Pixel-BERT: Aligning image pixels with text by deep multi-modal transformers","author":"huang","year":"2020"},{"key":"ref234","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR42600.2020.01098"},{"key":"ref65","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR46437.2021.00749"},{"key":"ref114","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR42600.2020.00877"},{"key":"ref235","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR46437.2021.00972"},{"key":"ref60","doi-asserted-by":"publisher","DOI":"10.1109\/6046.807953"},{"key":"ref122","article-title":"CoCa: Contrastive captioners are image-text foundation models","author":"yu","year":"2022"},{"key":"ref243","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v35i2.16231"},{"key":"ref123","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/P18-1238"},{"key":"ref244","doi-asserted-by":"publisher","DOI":"10.1109\/TVCG.2021.3114683"},{"key":"ref62","doi-asserted-by":"publisher","DOI":"10.1109\/35.41402"},{"key":"ref120","article-title":"CLIP-TD: CLIP targeted distillation for vision-language tasks","author":"wang","year":"2022"},{"key":"ref241","article-title":"Dance revolution: Long-term dance generation with music via curriculum learning","author":"huang","year":"2020"},{"key":"ref61","doi-asserted-by":"publisher","DOI":"10.1109\/ACCESS.2019.2916887"},{"key":"ref121","first-page":"9694","article-title":"Align before fuse: Vision and language representation learning with momentum distillation","author":"li","year":"2021","journal-title":"Proc Int Conf Neural Inf Process"},{"key":"ref242","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV48922.2021.00184"},{"key":"ref240","article-title":"CLIP-forge: Towards zero-shot text-to-shape generation","author":"sanghi","year":"2021"},{"key":"ref168","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2018.00911"},{"key":"ref289","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.coling-main.529"},{"key":"ref169","article-title":"RAT-SQL: Relation-aware schema encoding and linking for Text-to-SQL parsers","author":"wang","year":"2019"},{"key":"ref290","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52688.2022.01764"},{"key":"ref170","article-title":"SGEITL: Scene graph enhanced image-text learning for visual commonsense reasoning","author":"wang","year":"2021"},{"key":"ref291","first-page":"11041","article-title":"Robust visual reasoning via language guided neural module networks","author":"akula","year":"2021","journal-title":"Proc Int Conf Neural Inf Process"},{"key":"ref177","article-title":"Meta-StyleSpeech: Multi-speaker adaptive text-to-speech generation","author":"min","year":"2021"},{"key":"ref298","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52688.2022.01563"},{"key":"ref178","article-title":"Learning audio-visual speech representation by masked multimodal cluster prediction","author":"shi","year":"2022"},{"key":"ref299","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-58539-6_34"},{"key":"ref175","first-page":"14200","article-title":"Attention bottlenecks for multimodal fusion","author":"nagrani","year":"2021","journal-title":"Proc Int Conf Neural Inf Process"},{"key":"ref296","article-title":"OmniNet: A unified architecture for multi-modal multi-task learning","author":"pramanik","year":"2019"},{"key":"ref176","first-page":"15748","article-title":"Multimodal and multilingual embeddings for large-scale speech mining","author":"duquenne","year":"2021","journal-title":"Proc Int Conf Neural Inf Process"},{"key":"ref297","article-title":"Unifying architectures, tasks, and modalities through a simple sequence-to-sequence learning framework","author":"wang","year":"2022"},{"key":"ref173","first-page":"11449","article-title":"Exploring cross-video and cross-modality signals for weakly-supervised audio-visual video parsing","author":"lin","year":"2021","journal-title":"Proc Int Conf Neural Inf Process"},{"key":"ref294","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR46437.2021.00697"},{"key":"ref174","article-title":"VATT: Transformers for multimodal self-supervised learning from raw video, audio and text","author":"akbari","year":"2021"},{"key":"ref295","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV48922.2021.00163"},{"key":"ref171","first-page":"315","article-title":"Deep sparse rectifier neural networks","author":"glorot","year":"2011","journal-title":"Proc 14th Int Conf Artif Intell Statist"},{"key":"ref292","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.acl-main.586"},{"key":"ref172","article-title":"Gaussian error linear units (GELUs)","author":"hendrycks","year":"2016"},{"key":"ref293","article-title":"A closer look at the robustness of vision-and-language pre-trained models","author":"li","year":"2020"},{"key":"ref179","doi-asserted-by":"publisher","DOI":"10.1109\/ICASSP39728.2021.9414053"},{"key":"ref180","article-title":"Fused acoustic and text encoding for multimodal bilingual pretraining and speech translation","author":"zheng","year":"2021"},{"key":"ref181","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2021.acl-long.28"},{"key":"ref188","article-title":"InterBERT: Vision-and-language interaction for multi-modal pretraining","author":"lin","year":"2020"},{"key":"ref189","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/P19-1656"},{"key":"ref186","article-title":"Rethinking spatiotemporal feature learning for video understanding","author":"xie","year":"2017"},{"key":"ref187","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2018.00675"},{"key":"ref184","doi-asserted-by":"publisher","DOI":"10.1038\/s41746-021-00455-y"},{"key":"ref185","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV48922.2021.00398"},{"key":"ref182","article-title":"Towards robust vision transformer","author":"mao","year":"2021"},{"key":"ref183","first-page":"9774","article-title":"TriBERT: Human-centric audio-visual representation learning","author":"rahman","year":"2021","journal-title":"Proc Int Conf Neural Inf Process"},{"key":"ref148","article-title":"MultiBench: Multiscale benchmarks for multimodal representation learning","author":"liang","year":"2021"},{"key":"ref269","article-title":"Open-vocabulary object detection via vision and language knowledge distillation","author":"gu","year":"2021"},{"key":"ref149","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV48922.2021.00213"},{"key":"ref146","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV48922.2021.01315"},{"key":"ref267","doi-asserted-by":"publisher","DOI":"10.1145\/3560815"},{"key":"ref147","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR46437.2021.01140"},{"key":"ref268","doi-asserted-by":"publisher","DOI":"10.1109\/TPAMI.2018.2857768"},{"key":"ref155","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v32i1.12342"},{"key":"ref276","article-title":"Towards a unified foundation model: Jointly pre-training transformers on unpaired images and text","author":"li","year":"2021"},{"key":"ref156","doi-asserted-by":"publisher","DOI":"10.3115\/v1\/N15-1015"},{"key":"ref277","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR46437.2021.00397"},{"key":"ref153","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2021.acl-long.249"},{"key":"ref274","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.acl-main.214"},{"key":"ref154","article-title":"The hateful memes challenge: Detecting hate speech in multimodal memes","author":"kiela","year":"2020"},{"key":"ref275","doi-asserted-by":"publisher","DOI":"10.1109\/AVSS52988.2021.9663793"},{"key":"ref151","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2021.acl-long.526"},{"key":"ref272","article-title":"Supervising the transfer of reasoning patterns in VQA","author":"kervadec","year":"2021"},{"key":"ref152","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV48922.2021.00962"},{"key":"ref273","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR46437.2021.00419"},{"key":"ref270","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.emnlp-main.707"},{"key":"ref150","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV48922.2021.00166"},{"key":"ref271","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2021.acl-long.42"},{"key":"ref159","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2016.90"},{"key":"ref157","article-title":"Geometric deep learning: Grids, groups, graphs, geodesics, and gauges","author":"bronstein","year":"2021"},{"key":"ref278","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR46437.2021.00970"},{"key":"ref158","article-title":"A generalization of transformer networks to graphs","author":"dwivedi","year":"2020"},{"key":"ref279","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV48922.2021.00221"},{"key":"ref280","article-title":"Parameter efficient multimodal transformers for video representation learning","author":"lee","year":"2020"},{"key":"ref166","article-title":"Position information in transformers: An overview","author":"dufter","year":"2021"},{"key":"ref287","first-page":"17723","article-title":"Long-short transformer: Efficient transformers for language and vision","author":"zhu","year":"2021","journal-title":"Proc Int Conf Neural Inf Process"},{"key":"ref167","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2018.00813"},{"key":"ref288","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52688.2022.00333"},{"key":"ref164","doi-asserted-by":"publisher","DOI":"10.1007\/s41095-021-0229-5"},{"key":"ref285","first-page":"16344","article-title":"FlashAttention: Fast and memory-efficient exact attention with IO-awareness","author":"dao","year":"2022","journal-title":"Proc Int Conf Neural Inf Process"},{"key":"ref165","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV48922.2021.01474"},{"key":"ref286","article-title":"Generating long sequences with sparse transformers","author":"child","year":"2019"},{"key":"ref162","article-title":"On layer normalization in the transformer architecture","author":"xiong","year":"2020","journal-title":"Proc Int Conf Mach Learn"},{"key":"ref283","article-title":"Playing lottery tickets with vision and language","author":"gan","year":"2021"},{"key":"ref163","doi-asserted-by":"publisher","DOI":"10.1109\/TNNLS.2021.3069230"},{"key":"ref284","first-page":"4150","article-title":"PipeTransformer: Automated elastic pipelining for distributed training of large-scale models","author":"he","year":"2021","journal-title":"Proc Int Conf Mach Learn"},{"key":"ref160","first-page":"448","article-title":"Batch normalization: Accelerating deep network training by reducing internal covariate shift","author":"ioffe","year":"2015","journal-title":"Proc Int Conf Mach Learn"},{"key":"ref281","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2021.acl-long.199"},{"key":"ref161","article-title":"Layer normalization","author":"ba","year":"2016"},{"key":"ref282","article-title":"Supervision exists everywhere: A data efficient contrastive language-image pre-training paradigm","author":"li","year":"2021"},{"key":"ref13","volume":"2","author":"hastie","year":"2009","journal-title":"The Elements of Statistical Learning Data Mining Inference and Prediction"},{"key":"ref12","doi-asserted-by":"publisher","DOI":"10.1016\/j.inffus.2021.12.003"},{"key":"ref15","article-title":"Cross-modality fusion transformer for multispectral object detection","author":"qingyun","year":"2021"},{"key":"ref14","doi-asserted-by":"publisher","DOI":"10.1109\/WACV51458.2022.00221"},{"key":"ref11","doi-asserted-by":"publisher","DOI":"10.1109\/JSTSP.2020.2987728"},{"key":"ref10","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52688.2022.01593"},{"key":"ref17","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR42600.2020.01033"},{"key":"ref16","article-title":"wav2vec 2.0: A framework for self-supervised learning of speech representations","author":"baevski","year":"2020","journal-title":"Proc Int Conf Neural Inf Process"},{"key":"ref19","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV48922.2021.01607"},{"key":"ref18","article-title":"Open question answering over tables and text","author":"chen","year":"2020"},{"key":"ref2","first-page":"5998","article-title":"Attention is all you need","author":"vaswani","year":"2017","journal-title":"Proc Int Conf Neural Inf Process"},{"key":"ref1","doi-asserted-by":"publisher","DOI":"10.1109\/TPAMI.2018.2798607"},{"key":"ref191","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v35i10.17034"},{"key":"ref192","first-page":"5583","article-title":"ViLT: Vision-and-language transformer without convolution or region supervision","author":"kim","year":"2021","journal-title":"Proc Int Conf Mach Learn"},{"key":"ref190","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-58523-5_20"},{"key":"ref199","article-title":"SimVLM: Simple visual language model pretraining with weak supervision","author":"wang","year":"2021"},{"key":"ref197","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR42600.2020.00990"},{"key":"ref198","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52688.2022.00292"},{"key":"ref195","first-page":"36067","article-title":"GLIPv2: Unifying localization and vision-language understanding","author":"zhang","year":"2022","journal-title":"Proc Int Conf Neural Inf Process"},{"key":"ref196","article-title":"SemVLP: Vision-language pre-training by aligning semantics at multiple levels","author":"li","year":"2021"},{"key":"ref193","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52688.2022.01522"},{"key":"ref194","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52688.2022.01069"}],"container-title":["IEEE Transactions on Pattern Analysis and Machine Intelligence"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx7\/34\/10241246\/10123038.pdf?arnumber=10123038","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,6,6]],"date-time":"2024-06-06T13:08:04Z","timestamp":1717679284000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/10123038\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023,10]]},"references-count":310,"journal-issue":{"issue":"10"},"URL":"https:\/\/doi.org\/10.1109\/tpami.2023.3275156","relation":{},"ISSN":["0162-8828","2160-9292","1939-3539"],"issn-type":[{"value":"0162-8828","type":"print"},{"value":"2160-9292","type":"electronic"},{"value":"1939-3539","type":"electronic"}],"subject":[],"published":{"date-parts":[[2023,10]]}}}