{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,2]],"date-time":"2026-07-02T13:51:12Z","timestamp":1783000272784,"version":"3.54.5"},"reference-count":139,"publisher":"Institute of Electrical and Electronics Engineers (IEEE)","license":[{"start":{"date-parts":[[2023,1,1]],"date-time":"2023-01-01T00:00:00Z","timestamp":1672531200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by-nc-nd\/4.0\/"},{"start":{"date-parts":[[2023,1,1]],"date-time":"2023-01-01T00:00:00Z","timestamp":1672531200000},"content-version":"am","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by-nc-nd\/4.0\/"}],"funder":[{"DOI":"10.13039\/100000001","name":"NSF","doi-asserted-by":"publisher","award":["1802588"],"award-info":[{"award-number":["1802588"]}],"id":[{"id":"10.13039\/100000001","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["IEEE Access"],"published-print":{"date-parts":[[2023]]},"DOI":"10.1109\/access.2023.3299877","type":"journal-article","created":{"date-parts":[[2023,7,28]],"date-time":"2023-07-28T17:35:24Z","timestamp":1690565724000},"page":"80624-80646","source":"Crossref","is-referenced-by-count":11,"title":["Attention-Based Multimodal Deep Learning on Vision-Language Data: Models, Datasets, Tasks, Evaluation Metrics and Applications"],"prefix":"10.1109","volume":"11","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-7943-1422","authenticated-orcid":false,"given":"Priyankar","family":"Bose","sequence":"first","affiliation":[{"name":"Department of Computer Science, Virginia Commonwealth University, Richmond, VA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Pratip","family":"Rana","sequence":"additional","affiliation":[{"name":"Department of Computer Science, Virginia Commonwealth University, Richmond, VA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-3880-5886","authenticated-orcid":false,"given":"Preetam","family":"Ghosh","sequence":"additional","affiliation":[{"name":"Department of Computer Science, Virginia Commonwealth University, Richmond, VA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"263","reference":[{"key":"ref57","doi-asserted-by":"publisher","DOI":"10.1016\/j.comnet.2020.107366"},{"key":"ref56","doi-asserted-by":"publisher","DOI":"10.3115\/981732.981751"},{"key":"ref59","doi-asserted-by":"publisher","DOI":"10.1038\/s41598-022-26090-5"},{"key":"ref58","doi-asserted-by":"publisher","DOI":"10.1109\/ACCESS.2021.3052415"},{"key":"ref53","doi-asserted-by":"publisher","DOI":"10.1145\/3447685"},{"key":"ref52","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-01231-1_3"},{"key":"ref55","article-title":"A survey on recent advances in named entity recognition from deep learning models","author":"yadav","year":"2019","journal-title":"arXiv 1910 11470"},{"key":"ref54","doi-asserted-by":"publisher","DOI":"10.1109\/TCYB.2020.2985716"},{"key":"ref51","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/W19-3502"},{"key":"ref50","article-title":"E-SNLI-VE: Corrected visual-textual entailment with natural language explanations","author":"do","year":"2020","journal-title":"arXiv 2004 03744"},{"key":"ref46","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2017.215"},{"key":"ref45","doi-asserted-by":"publisher","DOI":"10.1145\/2733373.2806246"},{"key":"ref48","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2019.00688"},{"key":"ref47","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-40303-3_18"},{"key":"ref42","doi-asserted-by":"publisher","DOI":"10.1145\/2812802"},{"key":"ref41","doi-asserted-by":"publisher","DOI":"10.1007\/s11263-016-0981-7"},{"key":"ref44","doi-asserted-by":"publisher","DOI":"10.1007\/11526346_54"},{"key":"ref43","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2016.540"},{"key":"ref49","article-title":"A corpus for reasoning about natural language grounded in photographs","author":"suhr","year":"2018","journal-title":"arXiv 1811 00491"},{"key":"ref8","doi-asserted-by":"publisher","DOI":"10.1109\/ACCESS.2021.3050296"},{"key":"ref7","doi-asserted-by":"publisher","DOI":"10.1109\/TPAMI.2021.3126387"},{"key":"ref9","first-page":"6","article-title":"Multi-modal classification using images and text","volume":"3","author":"miller","year":"2020","journal-title":"SMU Data Sci Rev"},{"key":"ref4","doi-asserted-by":"publisher","DOI":"10.1016\/j.heliyon.2018.e00938"},{"key":"ref3","doi-asserted-by":"publisher","DOI":"10.1016\/j.inffus.2021.12.003"},{"key":"ref6","doi-asserted-by":"publisher","DOI":"10.1109\/ACCESS.2021.3082108"},{"key":"ref5","doi-asserted-by":"publisher","DOI":"10.1145\/3459930.3469518"},{"key":"ref100","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR42600.2020.01095"},{"key":"ref101","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR42600.2020.00448"},{"key":"ref40","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2015.303"},{"key":"ref35","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2015.279"},{"key":"ref34","first-page":"1","article-title":"The IAPR TC-12 benchmark: A new evaluation resource for visual information systems","volume":"2","author":"grubinger","year":"2006","journal-title":"Proc Int Workshop OntoImage"},{"key":"ref37","doi-asserted-by":"publisher","DOI":"10.1109\/TPAMI.2018.2828437"},{"key":"ref36","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2017.670"},{"key":"ref31","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2017.143"},{"key":"ref30","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2016.9"},{"key":"ref33","doi-asserted-by":"publisher","DOI":"10.3115\/v1\/D14-1086"},{"key":"ref32","first-page":"69","article-title":"Modeling context in referring expressions","author":"yu","year":"2016","journal-title":"Vision Computer"},{"key":"ref39","first-page":"1","article-title":"VQA-Med: Overview of the medical visual question answering task at ImageCLEF 2019","volume":"2","author":"abacha","year":"2019","journal-title":"Proc CLEF"},{"key":"ref38","first-page":"1","article-title":"A multi-world approach to question answering about real-world scenes based on uncertain input","volume":"27","author":"malinowski","year":"2014","journal-title":"Proc Adv Neural Inf Process Syst"},{"key":"ref24","article-title":"Self-supervised multimodal learning: A survey","author":"zong","year":"2023","journal-title":"arXiv 2304 01008"},{"key":"ref23","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-46448-0_7"},{"key":"ref26","doi-asserted-by":"publisher","DOI":"10.1016\/j.neucom.2019.02.003"},{"key":"ref25","first-page":"3523","article-title":"Image segmentation using deep learning: A survey","volume":"44","author":"minaee","year":"2022","journal-title":"IEEE Trans Pattern Anal Mach Intell"},{"key":"ref20","first-page":"817","article-title":"Grounding of textual phrases in images by reconstruction","author":"rohrbach","year":"2016","journal-title":"Vision Computer"},{"key":"ref22","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2016.493"},{"key":"ref21","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D16-1044"},{"key":"ref28","first-page":"740","article-title":"Microsoft COCO: Common objects in context","author":"lin","year":"2014","journal-title":"Vision Computer"},{"key":"ref27","doi-asserted-by":"publisher","DOI":"10.1109\/IJCNN48605.2020.9207697"},{"key":"ref29","article-title":"Exploring models and data for image question answering","author":"ren","year":"2015","journal-title":"arXiv 1505 02074"},{"key":"ref13","doi-asserted-by":"publisher","DOI":"10.1109\/TIP.2022.3175605"},{"key":"ref12","doi-asserted-by":"publisher","DOI":"10.1109\/TPAMI.2022.3222871"},{"key":"ref15","doi-asserted-by":"publisher","DOI":"10.1016\/j.knosys.2021.107650"},{"key":"ref128","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2019.00683"},{"key":"ref14","doi-asserted-by":"publisher","DOI":"10.1109\/TMM.2020.3035277"},{"key":"ref129","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2019.00214"},{"key":"ref97","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2019.00587"},{"key":"ref126","doi-asserted-by":"crossref","first-page":"61","DOI":"10.1109\/TNN.2008.2005605","article-title":"The graph neural network model","volume":"20","author":"scarselli","year":"2009","journal-title":"IEEE Trans Neural Netw"},{"key":"ref96","article-title":"Referring expression object segmentation with caption-aware consistency","author":"chen","year":"2019","journal-title":"arXiv 1910 04748"},{"key":"ref127","doi-asserted-by":"publisher","DOI":"10.1145\/3394486.3403182"},{"key":"ref11","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2009.5206848"},{"key":"ref99","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2019.00265"},{"key":"ref124","article-title":"BERT: Pre-training of deep bidirectional transformers for language understanding","author":"devlin","year":"2018","journal-title":"arXiv 1810 04805"},{"key":"ref10","doi-asserted-by":"publisher","DOI":"10.1145\/3065386"},{"key":"ref98","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2019.01068"},{"key":"ref125","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2019.00686"},{"key":"ref17","doi-asserted-by":"publisher","DOI":"10.1038\/s41598-021-98390-1"},{"key":"ref16","article-title":"Stacked attention networks for image question answering","author":"yang","year":"2015","journal-title":"arXiv 1511 02274"},{"key":"ref19","doi-asserted-by":"publisher","DOI":"10.1109\/TITS.2022.3232242"},{"key":"ref18","doi-asserted-by":"publisher","DOI":"10.1016\/j.inffus.2023.02.005"},{"key":"ref93","article-title":"Bilinear attention networks","volume":"31","author":"kim","year":"2018","journal-title":"Proc Adv Neural Inf Process Syst"},{"key":"ref133","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2021.maiworkshop-1.12"},{"key":"ref92","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2018.00637"},{"key":"ref134","doi-asserted-by":"publisher","DOI":"10.1016\/j.imavis.2021.104316"},{"key":"ref95","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2019.01075"},{"key":"ref131","doi-asserted-by":"publisher","DOI":"10.1016\/j.inffus.2021.02.006"},{"key":"ref94","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-01252-6_39"},{"key":"ref132","doi-asserted-by":"publisher","DOI":"10.3390\/s21092911"},{"key":"ref130","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR42600.2020.01007"},{"key":"ref91","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2018.00142"},{"key":"ref90","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2017.375"},{"key":"ref89","article-title":"Hierarchical question-image co-attention for visual question answering","author":"lu","year":"2016","journal-title":"arXiv 1606 00061"},{"key":"ref139","doi-asserted-by":"publisher","DOI":"10.1109\/CVPRW56347.2022.00506"},{"key":"ref86","article-title":"A multi-world approach to question answering about real-world scenes based on uncertain input","author":"malinowski","year":"2014","journal-title":"arXiv 1410 0210"},{"key":"ref137","article-title":"Contrastive language-image pre-training with knowledge graphs","author":"pan","year":"2022","journal-title":"arXiv 2210 08901"},{"key":"ref85","first-page":"1","article-title":"Faster R-CNN: Towards real-time object detection with region proposal networks","volume":"28","author":"ren","year":"2015","journal-title":"Proc Adv Neural Inf Process Syst"},{"key":"ref138","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52688.2022.00503"},{"key":"ref88","first-page":"2397","article-title":"Dynamic memory networks for visual and textual question answering","volume":"48","author":"xiong","year":"2016","journal-title":"Proc 33rd Int Conf Mach Learn"},{"key":"ref135","article-title":"VQA-GNN: Reasoning with multimodal semantic graph for visual question answering","author":"wang","year":"2022","journal-title":"arXiv 2205 11501"},{"key":"ref87","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-46478-7_28"},{"key":"ref136","doi-asserted-by":"publisher","DOI":"10.1145\/3579051.3579053"},{"key":"ref82","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-01225-0_13"},{"key":"ref81","doi-asserted-by":"publisher","DOI":"10.24963\/ijcai.2018\/155"},{"key":"ref84","first-page":"8748","article-title":"Learning transferable visual models from natural language supervision","author":"radford","year":"2021","journal-title":"Proc Int Conf Mach Learn"},{"key":"ref83","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2018.00602"},{"key":"ref80","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2015.9"},{"key":"ref79","article-title":"Multimodal unified attention networks for vision-and-language interactions","author":"yu","year":"2019","journal-title":"arXiv 1908 04107"},{"key":"ref108","doi-asserted-by":"crossref","first-page":"15","DOI":"10.1007\/978-3-319-27674-8_2","article-title":"Sentiment analysis on multi-view social data","author":"niu","year":"2016","journal-title":"Multimedia Modeling"},{"key":"ref78","article-title":"Utilizing large scale vision and text datasets for image segmentation from referring expressions","author":"hu","year":"2016","journal-title":"arXiv 1608 08305"},{"key":"ref109","article-title":"VisualBERT: A simple and performant baseline for vision and language","author":"harold li","year":"2019","journal-title":"arXiv 1908 03557"},{"key":"ref106","first-page":"1378","article-title":"Ask me anything: Dynamic memory networks for natural language processing","volume":"48","author":"kumar","year":"2016","journal-title":"Proc 33rd Int Conf Mach Learn"},{"key":"ref107","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v33i01.33019324"},{"key":"ref75","doi-asserted-by":"publisher","DOI":"10.1109\/IROS.2018.8593786"},{"key":"ref104","doi-asserted-by":"publisher","DOI":"10.1109\/JSYST.2020.3026879"},{"key":"ref74","first-page":"654","article-title":"Standardizing radiotherapy structure names with multimodal data: Deep learning approach","volume":"49","author":"bose","year":"2022","journal-title":"Med Phys"},{"key":"ref105","doi-asserted-by":"publisher","DOI":"10.1117\/12.2623371"},{"key":"ref77","doi-asserted-by":"publisher","DOI":"10.1007\/s13735-017-0143-x"},{"key":"ref102","doi-asserted-by":"publisher","DOI":"10.1109\/TIP.2020.2992888"},{"key":"ref76","doi-asserted-by":"publisher","DOI":"10.1145\/3123266.3123454"},{"key":"ref103","doi-asserted-by":"publisher","DOI":"10.1145\/3394171.3414006"},{"key":"ref2","article-title":"Multimodal learning with transformers: A survey","author":"xu","year":"2022","journal-title":"arXiv 2206 06488"},{"key":"ref1","doi-asserted-by":"publisher","DOI":"10.1016\/j.imavis.2020.104042"},{"key":"ref71","article-title":"Multi-modal fusion using fine-tuned self-attention and transfer learning for veracity analysis of web information","author":"meel","year":"2021","journal-title":"arXiv 2109 12547"},{"key":"ref111","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D19-1514"},{"key":"ref70","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR42600.2020.00359"},{"key":"ref112","article-title":"ImageBERT: Cross-modal pre-training with large-scale weak-supervised image-text data","author":"qi","year":"2020","journal-title":"arXiv 2001 07966"},{"key":"ref73","doi-asserted-by":"publisher","DOI":"10.3390\/cancers13081796"},{"key":"ref72","doi-asserted-by":"publisher","DOI":"10.1007\/s41095-020-0199-z"},{"key":"ref110","first-page":"1","article-title":"ViLBERT: Pretraining task-agnostic visiolinguistic representations for vision-and-language tasks","volume":"32","author":"lu","year":"2019","journal-title":"Proc Adv Neural Inf Process Syst"},{"key":"ref68","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2019.00755"},{"key":"ref119","first-page":"4904","article-title":"Scaling up visual and vision-language representation learning with noisy text supervision","volume":"139","author":"jia","year":"2021","journal-title":"Proc 38th Int Conf Mach Learn"},{"key":"ref67","doi-asserted-by":"publisher","DOI":"10.1016\/j.knosys.2019.01.019"},{"key":"ref117","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.emnlp-main.269"},{"key":"ref69","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR42600.2020.01005"},{"key":"ref118","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR46437.2021.00864"},{"key":"ref64","doi-asserted-by":"publisher","DOI":"10.1109\/TMM.2020.3046855"},{"key":"ref115","article-title":"UNITER: UNiversal image-TExt representation learning","author":"chen","year":"2019","journal-title":"arXiv 1909 11740"},{"key":"ref63","article-title":"Neural machine translation by jointly learning to align and translate","author":"bahdanau","year":"2014","journal-title":"arXiv 1409 0473"},{"key":"ref116","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-58586-0_14"},{"key":"ref66","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2017.122"},{"key":"ref113","article-title":"VL-BERT: Pre-training of generic visual-linguistic representations","author":"su","year":"2019","journal-title":"arXiv 1908 08530"},{"key":"ref65","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2017.202"},{"key":"ref114","article-title":"Pixel-BERT: Aligning image pixels with text by deep multi-modal transformers","author":"huang","year":"2020","journal-title":"arXiv 2004 00849"},{"key":"ref60","doi-asserted-by":"publisher","DOI":"10.1108\/14684520510598066"},{"key":"ref122","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52688.2022.00501"},{"key":"ref123","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52688.2022.01757"},{"key":"ref62","first-page":"285","article-title":"GRAVL-BERT: Graphical visual-linguistic representations for multimodal coreference resolution","author":"guo","year":"2022","journal-title":"Proc 29th Int Conf Comput Linguistics"},{"key":"ref120","doi-asserted-by":"publisher","DOI":"10.1109\/ICASSP43922.2022.9747769"},{"key":"ref61","year":"2023","journal-title":"Connected Papers"},{"key":"ref121","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52688.2022.01605"}],"container-title":["IEEE Access"],"original-title":[],"link":[{"URL":"https:\/\/ieeexplore.ieee.org\/ielam\/6287639\/10005208\/10197152-aam.pdf","content-type":"application\/pdf","content-version":"am","intended-application":"syndication"},{"URL":"http:\/\/xplorestaging.ieee.org\/ielx7\/6287639\/10005208\/10197152.pdf?arnumber=10197152","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2023,8,28]],"date-time":"2023-08-28T18:05:35Z","timestamp":1693245935000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/10197152\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023]]},"references-count":139,"URL":"https:\/\/doi.org\/10.1109\/access.2023.3299877","relation":{},"ISSN":["2169-3536"],"issn-type":[{"value":"2169-3536","type":"electronic"}],"subject":[],"published":{"date-parts":[[2023]]}}}