{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,3,17]],"date-time":"2026-03-17T18:25:19Z","timestamp":1773771919210,"version":"3.50.1"},"reference-count":91,"publisher":"Institute of Electrical and Electronics Engineers (IEEE)","license":[{"start":{"date-parts":[[2023,1,1]],"date-time":"2023-01-01T00:00:00Z","timestamp":1672531200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/ieeexplore.ieee.org\/Xplorehelp\/downloads\/license-information\/IEEE.html"},{"start":{"date-parts":[[2023,1,1]],"date-time":"2023-01-01T00:00:00Z","timestamp":1672531200000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2023,1,1]],"date-time":"2023-01-01T00:00:00Z","timestamp":1672531200000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"}],"funder":[{"name":"NICT"},{"name":"NICT"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["IEEE Trans. Pattern Anal. Mach. Intell."],"published-print":{"date-parts":[[2023]]},"DOI":"10.1109\/tpami.2023.3234170","type":"journal-article","created":{"date-parts":[[2023,1,4]],"date-time":"2023-01-04T18:36:35Z","timestamp":1672857395000},"page":"1-18","source":"Crossref","is-referenced-by-count":21,"title":["Universal Multimodal Representation for Language Understanding"],"prefix":"10.1109","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-4183-3645","authenticated-orcid":false,"given":"Zhuosheng","family":"Zhang","sequence":"first","affiliation":[{"name":"Department of Computer Science and Engineering, Shanghai Jiao Tong University, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-4346-7618","authenticated-orcid":false,"given":"Kehai","family":"Chen","sequence":"additional","affiliation":[{"name":"Key Laboratory of Shanghai Education Commission for Intelligent Interaction and Cognitive Engineering, Shanghai Jiao Tong University, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-8007-2503","authenticated-orcid":false,"given":"Rui","family":"Wang","sequence":"additional","affiliation":[{"name":"Department of Computer Science and Engineering, Shanghai Jiao Tong University, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-1111-9245","authenticated-orcid":false,"given":"Masao","family":"Utiyama","sequence":"additional","affiliation":[{"name":"National Institute of Information and Communications Technology (NICT), Japan"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-1028-4399","authenticated-orcid":false,"given":"Eiichiro","family":"Sumita","sequence":"additional","affiliation":[{"name":"National Institute of Information and Communications Technology (NICT), Japan"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-0436-8446","authenticated-orcid":false,"given":"Zuchao","family":"Li","sequence":"additional","affiliation":[{"name":"Department of Computer Science and Engineering, Shanghai Jiao Tong University, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-7290-0487","authenticated-orcid":false,"given":"Hai","family":"Zhao","sequence":"additional","affiliation":[{"name":"Department of Computer Science and Engineering, Shanghai Jiao Tong University, China"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"263","reference":[{"key":"ref13","doi-asserted-by":"publisher","DOI":"10.1109\/TPAMI.2019.2911066"},{"key":"ref57","article-title":"GLUE: A multi-task benchmark and analysis platform for natural language understanding","author":"wang","year":"2019","journal-title":"Proc 7th Int Conf Learn Representations"},{"key":"ref12","doi-asserted-by":"publisher","DOI":"10.1109\/TPAMI.2018.2798607"},{"key":"ref56","doi-asserted-by":"publisher","DOI":"10.1007\/s11263-017-1016-8"},{"key":"ref15","article-title":"VL-BERT: Pre-training of generic visual-linguistic representations","author":"su","year":"2020","journal-title":"Proc 8th Int Conf Learn Representations"},{"key":"ref59","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/W17-5301"},{"key":"ref14","doi-asserted-by":"publisher","DOI":"10.1109\/TPAMI.2020.3029008"},{"key":"ref58","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D15-1075"},{"key":"ref53","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2016.513"},{"key":"ref52","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2017.634"},{"key":"ref11","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/P19-1180"},{"key":"ref55","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2015.7298682"},{"key":"ref10","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/P19-1653"},{"key":"ref54","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2014.180"},{"key":"ref17","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D19-1514"},{"key":"ref16","first-page":"13","article-title":"ViLBERT: Pretraining task-agnostic visiolinguistic representations for vision-and-language tasks","author":"lu","year":"2019","journal-title":"Proc Int Conf Neural Inf Process"},{"key":"ref19","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v34i07.7005"},{"key":"ref18","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v34i07.6795"},{"key":"ref51","doi-asserted-by":"publisher","DOI":"10.1109\/TASLP.2019.2937190"},{"key":"ref50","article-title":"RoBERTa: A robustly optimized BERT pretraining approach","author":"liu","year":"2019"},{"key":"ref91","article-title":"ELECTRA: Pre-training text encoders as discriminators rather than generators","author":"clark","year":"2020","journal-title":"Proc 8th Int Conf Learn Representations"},{"key":"ref90","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v34i05.6510"},{"key":"ref46","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2018.00419"},{"key":"ref45","doi-asserted-by":"publisher","DOI":"10.1109\/TPAMI.2018.2797921"},{"key":"ref89","doi-asserted-by":"publisher","DOI":"10.1007\/s11263-016-0981-7"},{"key":"ref48","article-title":"Image search using multilingual texts: A cross-modal learning approach between image and text Maxime Portaz Qwant research","author":"portaz","year":"2019"},{"key":"ref47","doi-asserted-by":"publisher","DOI":"10.1007\/978-1-4612-4380-9_14"},{"key":"ref42","doi-asserted-by":"publisher","DOI":"10.1109\/TPAMI.2016.2598339"},{"key":"ref86","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.acl-main.385"},{"key":"ref41","doi-asserted-by":"publisher","DOI":"10.1145\/2964284.2967212"},{"key":"ref85","first-page":"56","article-title":"An exploration of word embedding initialization in deep-learning tasks","author":"kocmi","year":"2017","journal-title":"Proc 14th Int Conf Natural Lang Process"},{"key":"ref44","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D16-1089"},{"key":"ref88","doi-asserted-by":"publisher","DOI":"10.13053\/cys-23-4-3294"},{"key":"ref43","doi-asserted-by":"publisher","DOI":"10.1109\/TPAMI.2021.3132034"},{"key":"ref87","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/P18-1198"},{"key":"ref49","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2016.90"},{"key":"ref8","doi-asserted-by":"publisher","DOI":"10.1037\/0278-7393.12.1.54"},{"key":"ref7","doi-asserted-by":"publisher","DOI":"10.1109\/ICDM.2018.00090"},{"key":"ref9","author":"meier","year":"2000","journal-title":"The Accelerated Learning Handbook A Creative Guide to Designing and Delivering Faster More Effective Training Programs"},{"key":"ref4","article-title":"Improving language understanding by generative pre-training","author":"radford","year":"2018"},{"key":"ref3","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/N18-1202"},{"key":"ref6","first-page":"5754","article-title":"XLNet: Generalized autoregressive pretraining for language understanding","author":"yang","year":"2019","journal-title":"Proc Int Conf Neural Inf Process"},{"key":"ref5","first-page":"4171","article-title":"BERT: Pre-training of deep bidirectional transformers for language understanding","author":"devlin","year":"2019","journal-title":"Proc Conf North Amer Chapter Assoc Comput Linguistics Hum Lang Technol"},{"key":"ref82","article-title":"Train neural networks with noise to reduce overfitting","author":"brownlee","year":"2019","journal-title":"Machine Learning Mastery"},{"key":"ref81","first-page":"5109","article-title":"Regularizing deep neural networks by noise: Its interpretation and optimization","author":"noh","year":"2017","journal-title":"Proc Int Conf Neural Inf Process"},{"key":"ref40","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2015.7298932"},{"key":"ref84","first-page":"99","article-title":"A bag of useful tricks for practical neural machine translation: Embedding layer initialization and large batch size","author":"neishi","year":"2017","journal-title":"Proc 4th Workshop Asian Transl"},{"key":"ref83","doi-asserted-by":"crossref","first-page":"146","DOI":"10.1080\/00437956.1954.11659520","article-title":"Distributional structure","volume":"10","author":"harris","year":"1954","journal-title":"Word"},{"key":"ref80","doi-asserted-by":"publisher","DOI":"10.1023\/A:1024218913435"},{"key":"ref35","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D19-1437"},{"key":"ref79","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/N19-1422"},{"key":"ref34","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D18-1549"},{"key":"ref78","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/W18-6439"},{"key":"ref37","article-title":"Neural machine translation with universal visual representation","author":"zhang","year":"2020","journal-title":"Proc 8th Int Conf Learn Representations"},{"key":"ref36","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D19-1143"},{"key":"ref31","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/P17-1012"},{"key":"ref75","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/N19-4009"},{"key":"ref30","article-title":"Neural machine translation by jointly learning to align and translate","author":"bahdanau","year":"2015","journal-title":"Proc 3rd Int Conf Learn Representations"},{"key":"ref74","article-title":"Adam: A method for stochastic optimization","author":"kingma","year":"2015","journal-title":"Proc 3rd Int Conf Learn Representations"},{"key":"ref33","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/P17-2090"},{"key":"ref77","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/W17-4753"},{"key":"ref32","first-page":"5998","article-title":"Attention is all you need","author":"vaswani","year":"2017","journal-title":"Proc Int Conf Neural Inf Process"},{"key":"ref76","first-page":"388","article-title":"Statistical significance tests for machine translation evaluation","author":"koehn","year":"2004","journal-title":"Proc Conf Empir Methods Natural Lang Process"},{"key":"ref2","doi-asserted-by":"publisher","DOI":"10.3115\/v1\/D14-1162"},{"key":"ref1","first-page":"3111","article-title":"Distributed representations of words and phrases and their compositionality","author":"mikolov","year":"2013","journal-title":"Proc 27th Int Conf Neural Inf Process Syst"},{"key":"ref39","first-page":"2121","article-title":"DeViSE: A deep visual-semantic embedding model","author":"frome","year":"2013","journal-title":"Proc Int Conf Neural Inf Process"},{"key":"ref38","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-58577-8_8"},{"key":"ref71","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/P19-1441"},{"key":"ref70","doi-asserted-by":"publisher","DOI":"10.1145\/3394171.3413715"},{"key":"ref73","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV48922.2021.00147"},{"key":"ref72","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.emnlp-main.162"},{"key":"ref24","doi-asserted-by":"publisher","DOI":"10.1109\/TPAMI.2020.2972281"},{"key":"ref68","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2021.acl-long.480"},{"key":"ref23","doi-asserted-by":"crossref","first-page":"87","DOI":"10.1109\/TPAMI.2022.3152247","article-title":"A survey on vision transformer","volume":"45","author":"han","year":"2023","journal-title":"IEEE Trans Pattern Anal Mach Intell"},{"key":"ref67","first-page":"740","article-title":"Microsoft COCO: Common objects in context","author":"lin","year":"2014","journal-title":"Proc Eur Conf Comput Vis"},{"key":"ref26","doi-asserted-by":"publisher","DOI":"10.1109\/TMM.2019.2896494"},{"key":"ref25","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/W16-3210"},{"key":"ref69","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.acl-main.273"},{"key":"ref20","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2019.00756"},{"key":"ref64","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/S17-2001"},{"key":"ref63","article-title":"First Quora dataset release: Question pairs","author":"iyer","year":"2017"},{"key":"ref22","first-page":"2742","article-title":"Glyce: Glyph-vectors for chinese character representations","author":"meng","year":"2019","journal-title":"Proc Int Conf Neural Inf Process"},{"key":"ref66","first-page":"1631","article-title":"Recursive deep models for semantic compositionality over a sentiment treebank","author":"socher","year":"2013","journal-title":"Proc Conf Empir Methods Natural Lang Process"},{"key":"ref21","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v32i1.11939"},{"key":"ref65","doi-asserted-by":"publisher","DOI":"10.1162\/tacl_a_00290"},{"key":"ref28","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/P16-1168"},{"key":"ref27","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/P17-2066"},{"key":"ref29","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/P18-1239"},{"key":"ref60","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D16-1264"},{"key":"ref62","article-title":"Automatically constructing a corpus of sentential paraphrases","author":"dolan","year":"2005","journal-title":"Proc 3rd Int Workshop Paraphrasing"},{"key":"ref61","article-title":"The fifth PASCAL recognizing textual entailment challenge","author":"bentivogli","year":"2009","journal-title":"Proc 2nd Text Anal Conf"}],"container-title":["IEEE Transactions on Pattern Analysis and Machine Intelligence"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx7\/34\/4359286\/10005816.pdf?arnumber=10005816","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2023,6,7]],"date-time":"2023-06-07T01:22:11Z","timestamp":1686100931000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/10005816\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023]]},"references-count":91,"URL":"https:\/\/doi.org\/10.1109\/tpami.2023.3234170","relation":{},"ISSN":["0162-8828","2160-9292","1939-3539"],"issn-type":[{"value":"0162-8828","type":"print"},{"value":"2160-9292","type":"electronic"},{"value":"1939-3539","type":"electronic"}],"subject":[],"published":{"date-parts":[[2023]]}}}