{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,4]],"date-time":"2026-07-04T17:11:29Z","timestamp":1783185089612,"version":"3.54.6"},"reference-count":56,"publisher":"Springer Science and Business Media LLC","issue":"3","license":[{"start":{"date-parts":[[2022,5,12]],"date-time":"2022-05-12T00:00:00Z","timestamp":1652313600000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2022,5,12]],"date-time":"2022-05-12T00:00:00Z","timestamp":1652313600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"funder":[{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["61966004"],"award-info":[{"award-number":["61966004"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100004607","name":"Natural Science Foundation of Guangxi Province","doi-asserted-by":"publisher","award":["No. 2019GXNSFDA245018"],"award-info":[{"award-number":["No. 2019GXNSFDA245018"]}],"id":[{"id":"10.13039\/501100004607","id-type":"DOI","asserted-by":"publisher"}]},{"name":"Graduate Innovation Project of Guangxi Graduate Education","award":["No.XYCBZ2021002"],"award-info":[{"award-number":["No.XYCBZ2021002"]}]}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Appl Intell"],"published-print":{"date-parts":[[2023,2]]},"DOI":"10.1007\/s10489-022-03624-y","type":"journal-article","created":{"date-parts":[[2022,5,12]],"date-time":"2022-05-12T21:23:03Z","timestamp":1652390583000},"page":"2706-2722","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":28,"title":["Enhance understanding and reasoning ability for image captioning"],"prefix":"10.1007","volume":"53","author":[{"given":"Jiahui","family":"Wei","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-5313-6134","authenticated-orcid":false,"given":"Zhixin","family":"Li","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Jianwei","family":"Zhu","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Huifang","family":"Ma","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2022,5,12]]},"reference":[{"issue":"8","key":"3624_CR1","doi-asserted-by":"publisher","first-page":"1798","DOI":"10.1109\/TPAMI.2013.50","volume":"35","author":"Y Bengio","year":"2013","unstructured":"Bengio Y, Courville A, Vincent P (2013) Representation learning: a review and new perspectives. IEEE Trans Pattern Anal Mach Intell 35(8):1798\u20131828","journal-title":"IEEE Trans Pattern Anal Mach Intell"},{"key":"3624_CR2","doi-asserted-by":"crossref","unstructured":"Mikolov T, Karafi\u00e1t M., Burget L, \u010cernocky\u0300 J, Khudanpur S (2010) Recurrent neural network based language model. In: Proceedings of the annual conference of the international speech communication association, pp 1045\u20131048","DOI":"10.21437\/Interspeech.2010-343"},{"issue":"12","key":"3624_CR3","doi-asserted-by":"publisher","first-page":"2313","DOI":"10.1007\/s10994-020-05919-y","volume":"109","author":"F Huang","year":"2020","unstructured":"Huang F, Li Z, Wei H, Zhang C, Ma H (2020) Boost image captioning with knowledge reasoning. Mach Learn 109(12):2313\u20132332","journal-title":"Mach Learn"},{"key":"3624_CR4","unstructured":"Vaswani A, Shazeer N, Parmar N, Uszkoreit J, Jones L, Gomez AN, Kaiser \u0141, Polosukhin I (2017) Attention is all you need. In: Advances in Neural Information Processing Systems, pp 5998\u20136008"},{"key":"3624_CR5","doi-asserted-by":"crossref","unstructured":"Chen S, Li Z, Huang F, Zhang C, Ma H (2020) Improving object detection with relation mining network. In: Proceedings of the 2020 IEEE international conference on data mining, pp 52\u201361","DOI":"10.1109\/ICDM50108.2020.00014"},{"key":"3624_CR6","doi-asserted-by":"publisher","first-page":"409","DOI":"10.1613\/jair.4900","volume":"55","author":"R Bernardi","year":"2016","unstructured":"Bernardi R, Cakici R, Elliott D, Erdem A, Erdem E, Ikizler-Cinbis N, Keller F, Muscat A, Plank B (2016) Automatic description generation from images: a survey of models, datasets, and evaluation measures. J Artif Intell Res 55:409\u2013442","journal-title":"J Artif Intell Res"},{"key":"3624_CR7","doi-asserted-by":"publisher","first-page":"207","DOI":"10.1162\/tacl_a_00177","volume":"2","author":"R Socher","year":"2014","unstructured":"Socher R, Karpathy A, Le QV, Manning CD, Ng AY (2014) Grounded compositional semantics for finding and describing images with sentences. Transactions of the Association for Computational Linguistics 2:207\u2013218","journal-title":"Transactions of the Association for Computational Linguistics"},{"key":"3624_CR8","unstructured":"Karpathy A, Joulin A, Li FF (2014) Deep fragment embeddings for bidirectional image sentence mapping. In: Advances in neural information processing systems, pp 1889\u20131897"},{"key":"3624_CR9","doi-asserted-by":"crossref","unstructured":"Ma L, Lu Z, Shang L, Li H (2015) Multimodal convolutional neural networks for matching image and sentence. In: Proceedings of the IEEE international conference on computer vision, pp 2623\u20132631","DOI":"10.1109\/ICCV.2015.301"},{"issue":"12","key":"3624_CR10","doi-asserted-by":"publisher","first-page":"2891","DOI":"10.1109\/TPAMI.2012.162","volume":"35","author":"G Kulkarni","year":"2013","unstructured":"Kulkarni G, Premraj V, Ordonez V, Dhar S, Li S, Choi Y, Berg AC, Berg TL (2013) Babytalk: Understanding and generating simple image descriptions. IEEE Trans Pattern Anal Mach Intell 35 (12):2891\u20132903","journal-title":"IEEE Trans Pattern Anal Mach Intell"},{"key":"3624_CR11","unstructured":"Lebret R, Pinheiro P, Collobert R (2015) Phrase-based image captioning. In: Proceedings of the international conference on machine learning, PMLR, pp 2085\u20132094"},{"key":"3624_CR12","doi-asserted-by":"crossref","unstructured":"Lu J, Yang J, Batra D, Parikh D (2018) Neural baby talk. In: Proceedings of the IEEE conference on computer vision and pattern recognition, pp 7219\u20137228","DOI":"10.1109\/CVPR.2018.00754"},{"key":"3624_CR13","doi-asserted-by":"crossref","unstructured":"Vinyals O, Toshev A, Bengio S, Erhan D (2015) Show and tell: a neural image caption generator. In: Proceedings of the IEEE conference on computer vision and pattern recognition, pp 3156\u20133164","DOI":"10.1109\/CVPR.2015.7298935"},{"key":"3624_CR14","doi-asserted-by":"publisher","first-page":"85","DOI":"10.1016\/j.neunet.2014.09.003","volume":"61","author":"J Schmidhuber","year":"2015","unstructured":"Schmidhuber J (2015) Deep learning in neural networks: an overview. Neural Netw 61:85\u2013117","journal-title":"Neural Netw"},{"key":"3624_CR15","unstructured":"Xingjian S, Chen Z, Wang H, Yeung D-Y, Wong W-K, Woo W-c (2015) Convolutional lstm network: A machine learning approach for precipitation nowcasting. In: Advances in neural information processing systems, pp 802\u2013810"},{"key":"3624_CR16","unstructured":"Xu K, Ba J, Kiros R, Cho K, Courville A, Salakhudinov R, Zemel R, Bengio Y (2015) Show, attend and tell: Neural image caption generation with visual attention. In: Proceedings of the international conference on machine learning, pp 2048\u20132057"},{"issue":"2","key":"3624_CR17","doi-asserted-by":"publisher","first-page":"52:1","DOI":"10.1145\/3439734","volume":"17","author":"H Wei","year":"2021","unstructured":"Wei H, Li Z, Huang F, Zhang C, Ma H, Shi Z (2021) Integrating scene semantic knowledge into image captioning. ACM Transactions on Multimedia Computing Communications, and Applications 17(2):52:1\u201352:22","journal-title":"ACM Transactions on Multimedia Computing Communications, and Applications"},{"key":"3624_CR18","doi-asserted-by":"crossref","unstructured":"Qin Y, Du J, Zhang Y, Lu H (2019) Look back and predict forward in image captioning. In: Proceedings of the IEEE conference on computer vision and pattern recognition, pp 8367\u20138375","DOI":"10.1109\/CVPR.2019.00856"},{"key":"3624_CR19","doi-asserted-by":"publisher","first-page":"103068:1","DOI":"10.1016\/j.cviu.2020.103068","volume":"201","author":"H Wei","year":"2020","unstructured":"Wei H, Li Z, Zhang C, Ma H (2020) The synergy of double attention: Combine sentence-level and word-level attention for image captioning. Computer Vision and Image Understanding 201:103068:1\u2013103068:10","journal-title":"Computer Vision and Image Understanding"},{"key":"3624_CR20","unstructured":"Sutskever I, Vinyals O, Le QV (2014) Sequence to sequence learning with neural networks, in Advances in Neural Information Processing Systems, pp 3104\u20133112"},{"key":"3624_CR21","unstructured":"Lin T, Wang Y, Liu X, Qiu X (2021) A survey of transformers. arXiv:2106.04554"},{"key":"3624_CR22","doi-asserted-by":"crossref","unstructured":"Carion N, Massa F, Synnaeve G, Usunier N, Kirillov A, Zagoruyko S (2020) End-to-end object detection with transformers. In: Proceedings of the European conference on computer vision, pp 213\u2013229","DOI":"10.1007\/978-3-030-58452-8_13"},{"key":"3624_CR23","unstructured":"Parmar N, Vaswani A, Uszkoreit J, Kaiser L, Shazeer N, Ku A, Tran D (2018) Image transformer. In: Proceedings of the international conference on machine learning, pp 4055\u20134064"},{"key":"3624_CR24","doi-asserted-by":"crossref","unstructured":"Dong L, Xu S, Xu B (2018) Speech-transformer: A no-recurrence sequence-to-sequence model for speech recognition. In: Proceedings of the 2018 IEEE international conference on acoustics, speech and signal processing, pp 5884\u20135888","DOI":"10.1109\/ICASSP.2018.8462506"},{"issue":"9","key":"3624_CR25","doi-asserted-by":"publisher","first-page":"1572","DOI":"10.1021\/acscentsci.9b00576","volume":"5","author":"P Schwaller","year":"2019","unstructured":"Schwaller P, Laino T, Gaudin T, Bolgar P, Hunter CA, Bekas C, Lee AA (2019) Molecular transformer: a model for uncertainty-calibrated chemical reaction prediction. ACS Central Sci 5(9):1572\u20131583","journal-title":"ACS Central Sci"},{"key":"3624_CR26","unstructured":"Devlin J, Chang M-W, Lee K, Toutanova K (2018) Bert: Pre-training of deep bidirectional transformers for language understanding. arXiv:1810.04805"},{"key":"3624_CR27","doi-asserted-by":"crossref","unstructured":"Ji J, Luo Y, Sun X, Chen F, Luo G, Wu Y, Gao Y, Ji R (2021) Improving image captioning by leveraging intra-and inter-layer global representation in transformer network. In: Proceedings of the AAAI conference on artificial intelligence, pp 1655\u20131663","DOI":"10.1609\/aaai.v35i2.16258"},{"key":"3624_CR28","doi-asserted-by":"crossref","unstructured":"Huang L, Wang W, Chen J, Wei X-Y (2019) Attention on attention for image captioning. In: Proceedings of the IEEE international conference on computer vision, pp 4634\u20134643","DOI":"10.1109\/ICCV.2019.00473"},{"key":"3624_CR29","doi-asserted-by":"crossref","unstructured":"Cornia M, Stefanini M, Baraldi L, Cucchiara R (2020) Meshed-memory transformer for image captioning. In: Proceedings of the IEEE conference on computer vision and pattern recognition, pp 10578\u201310587","DOI":"10.1109\/CVPR42600.2020.01059"},{"key":"3624_CR30","doi-asserted-by":"crossref","unstructured":"Hu H, Gu J, Zhang Z, Dai J, Wei Y (2018) Relation networks for object detection. In: Proceedings of the IEEE conference on computer vision and pattern recognition, pp 3588\u20133597","DOI":"10.1109\/CVPR.2018.00378"},{"key":"3624_CR31","unstructured":"Herdade S, Kappeler A, Boakye K, Soares J (2019) Image captioning: Transforming objects into words. In: Advances in neural information processing systems, pp 11137\u201311147"},{"key":"3624_CR32","doi-asserted-by":"crossref","unstructured":"Guo L, Liu J, Zhu X, Yao P, Lu S, Lu H (2020) Normalized and geometry-aware self-attention network for image captioning. In: Proceedings of the IEEE conference on computer vision and pattern recognition, pp 10327\u201310336","DOI":"10.1109\/CVPR42600.2020.01034"},{"key":"3624_CR33","doi-asserted-by":"crossref","unstructured":"Li X, Yin X, Li C, Zhang P, Hu X, Zhang L, Wang L, Hu H, Dong L, Wei F et al (2020) Oscar: Object-semantics aligned pre-training for vision-language tasks. In: European conference on computer vision, pp 121\u2013137","DOI":"10.1007\/978-3-030-58577-8_8"},{"key":"3624_CR34","doi-asserted-by":"crossref","unstructured":"Zhang P, Li X, Hu X, Yang J, Zhang L, Wang L, Choi Y, Gao J (2021) Vinvl: Revisiting visual representations in vision-language models. In: Proceedings of the IEEE conference on computer vision and pattern recognition, pp 5579\u20135588","DOI":"10.1109\/CVPR46437.2021.00553"},{"key":"3624_CR35","unstructured":"Ren S, He K, Girshick R, Sun J (2015) Faster r-CNN: Towards real-time object detection with region proposal networks. In: Advances in neural information processing systems, pp 91\u201399"},{"key":"3624_CR36","doi-asserted-by":"crossref","unstructured":"He K, Zhang X, Ren S, Sun J (2016) Deep residual learning for image recognition. In: Proceedings of the IEEE conference on computer vision and pattern recognition, pp 770\u2013778","DOI":"10.1109\/CVPR.2016.90"},{"key":"3624_CR37","doi-asserted-by":"crossref","unstructured":"Speer R, Chin J, Havasi C (2017) Conceptnet 5.5: An open multilingual graph of general knowledge. In: Proceedings of the AAAI conference on artificial intelligence, pp 4444\u20134451","DOI":"10.1609\/aaai.v31i1.11164"},{"key":"3624_CR38","doi-asserted-by":"crossref","unstructured":"Johnson J, Karpathy A, Fei-Fei L (2016) Densecap: Fully convolutional localization networks for dense captioning. In: Proceedings of the IEEE conference on computer vision and pattern recognition, pp 4565\u20134574","DOI":"10.1109\/CVPR.2016.494"},{"key":"3624_CR39","doi-asserted-by":"crossref","unstructured":"Papineni K, Roukos S, Ward T, Zhu W-J (2002) Bleu: a method for automatic evaluation of machine translation. In: Proceedings of the annual meeting of the association for computational linguistics, pp 311\u2013318","DOI":"10.3115\/1073083.1073135"},{"key":"3624_CR40","doi-asserted-by":"crossref","unstructured":"Lin C-Y, Hovy E (2003) Automatic evaluation of summaries using n-gram co-occurrence statistics. In: Proceedings of the 2003 human language technology conference of the north american chapter of the association for computational linguistics, pp 150\u2013157","DOI":"10.3115\/1073445.1073465"},{"key":"3624_CR41","unstructured":"Banerjee S, Lavie A (2005) Meteor: an automatic metric for mt evaluation with improved correlation with human judgments. In: Proceedings of the ACL workshop on intrinsic and extrinsic evaluation measures for machine translation and\/or summarization, pp 65\u201372"},{"key":"3624_CR42","doi-asserted-by":"crossref","unstructured":"Vedantam R, Lawrence Zitnick C, Parikh D (2015) Cider: Consensus-based image description evaluation. In: Proceedings of the IEEE conference on computer vision and pattern recognition, pp 4566\u20134575","DOI":"10.1109\/CVPR.2015.7299087"},{"key":"3624_CR43","doi-asserted-by":"crossref","unstructured":"Anderson P, He X, Buehler C, Teney D, Johnson M, Gould S, Zhang L (2018) Bottom-up and top-down attention for image captioning and visual question answering. In: Proceedings of the IEEE conference on computer vision and pattern recognition, pp 6077\u20136086","DOI":"10.1109\/CVPR.2018.00636"},{"key":"3624_CR44","doi-asserted-by":"crossref","unstructured":"Yao T, Pan Y, Li Y, Mei T (2018) Exploring visual relationship for image captioning. In: Proceedings of the European conference on computer vision, pp 684\u2013699","DOI":"10.1007\/978-3-030-01264-9_42"},{"key":"3624_CR45","doi-asserted-by":"crossref","unstructured":"Ke L, Pei W, Li R, Shen X, Tai Y-W (2019) Reflective decoding network for image captioning. In: Proceedings of the IEEE international conference on computer vision, pp 8888\u20138897","DOI":"10.1109\/ICCV.2019.00898"},{"key":"3624_CR46","doi-asserted-by":"crossref","unstructured":"Li G, Zhu L, Liu P, Yang Y (2019) Entangled transformer for image captioning. In: Proceedings of the IEEE international conference on computer vision, pp 8928\u20138937","DOI":"10.1109\/ICCV.2019.00902"},{"key":"3624_CR47","doi-asserted-by":"crossref","unstructured":"Wang L, Bai Z, Zhang Y, Lu H (2020) Show, recall, and tell: Image captioning with recall mechanism. In: Proceedings of the AAAI conference on artificial intelligence, pp 12176\u201312183","DOI":"10.1609\/aaai.v34i07.6898"},{"key":"3624_CR48","doi-asserted-by":"crossref","unstructured":"Zhou Y, Wang M, Liu D, Hu Z, Zhang H (2020) More grounded image captioning by distilling image-text matching model. In: Proceedings of the IEEE conference on computer vision and pattern recognition, pp 4777\u20134786","DOI":"10.1109\/CVPR42600.2020.00483"},{"key":"3624_CR49","doi-asserted-by":"crossref","unstructured":"Wang J, Xu W, Wang Q, Chan AB (2020) Compare and reweight: Distinctive image captioning using similar images sets. In: Proceedings of the European conference on computer vision, pp 370\u2013386","DOI":"10.1007\/978-3-030-58452-8_22"},{"key":"3624_CR50","doi-asserted-by":"crossref","unstructured":"Wu L, Xu M, Sang L, Yao T, Mei T (2020) Noise augmented double-stream graph convolutional networks for image captioning. In: IEEE Transactions on Circuits and Systems for Video Technology","DOI":"10.1109\/TCSVT.2020.3036860"},{"key":"3624_CR51","doi-asserted-by":"crossref","unstructured":"Wang X, Ma L, Fu Y, Xue X (2021) Neural symbolic representation learning for image captioning. In: Proceedings of the 2021 international conference on multimedia retrieval, pp 312\u2013321","DOI":"10.1145\/3460426.3463637"},{"key":"3624_CR52","doi-asserted-by":"crossref","unstructured":"Nie W, Li J, Xu N, Liu A-A, Li X, Zhang Y (2021) Triangle-reward reinforcement learning: a visual-linguistic semantic alignment for image captioning. In: Proceedings of the ACM international conference on multimedia, pp 4510\u20134518","DOI":"10.1145\/3474085.3475604"},{"key":"3624_CR53","doi-asserted-by":"crossref","unstructured":"Liu D, Zha Z-J, Zhang H, Zhang Y, Wu F (2018) Context-aware visual policy network for sequence-level image captioning. In: Proceedings of the ACM international conference on multimedia, pp 1416\u20131424","DOI":"10.1145\/3240508.3240632"},{"key":"3624_CR54","doi-asserted-by":"crossref","unstructured":"Yang X, Tang K, Zhang H, Cai J (2019) Auto-encoding scene graphs for image captioning. In: Proceedings of the IEEE conference on computer vision and pattern recognition, pp 10685\u201310694","DOI":"10.1109\/CVPR.2019.01094"},{"key":"3624_CR55","doi-asserted-by":"crossref","unstructured":"Guo L, Liu J, Tang J, Li J, Luo W, Lu H (2019) Aligning linguistic words and visual semantic units for image captioning. In: Proceedings of the ACM international conference on multimedia, pp 765\u2013773","DOI":"10.1145\/3343031.3350943"},{"key":"3624_CR56","first-page":"1","volume":"7","author":"J Dem\u0161ar","year":"2006","unstructured":"Dem\u0161ar J (2006) Statistical comparisons of classifiers over multiple data sets. J Machine Learn Res 7:1\u201330","journal-title":"J Machine Learn Res"}],"container-title":["Applied Intelligence"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10489-022-03624-y.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s10489-022-03624-y\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10489-022-03624-y.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2023,1,11]],"date-time":"2023-01-11T11:38:21Z","timestamp":1673437101000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s10489-022-03624-y"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2022,5,12]]},"references-count":56,"journal-issue":{"issue":"3","published-print":{"date-parts":[[2023,2]]}},"alternative-id":["3624"],"URL":"https:\/\/doi.org\/10.1007\/s10489-022-03624-y","relation":{},"ISSN":["0924-669X","1573-7497"],"issn-type":[{"value":"0924-669X","type":"print"},{"value":"1573-7497","type":"electronic"}],"subject":[],"published":{"date-parts":[[2022,5,12]]},"assertion":[{"value":"12 April 2022","order":1,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"12 May 2022","order":2,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Declarations"}},{"value":"We declare that we have no financial and personal relationships with other people or organizations that can inappropriately influence our work. There is no professional or other personal interest of any nature or kind in any product, service and\/or company that could be construed as influencing the position presented in, or the review of, the manuscript entitled.","order":2,"name":"Ethics","group":{"name":"EthicsHeading","label":"<!--Emphasis Type='Bold' removed-->Conflict of Interests"}}]}}