{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,6,26]],"date-time":"2026-06-26T03:43:30Z","timestamp":1782445410334,"version":"3.54.5"},"reference-count":36,"publisher":"Springer Science and Business Media LLC","issue":"12","license":[{"start":{"date-parts":[[2025,9,5]],"date-time":"2025-09-05T00:00:00Z","timestamp":1757030400000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2025,9,5]],"date-time":"2025-09-05T00:00:00Z","timestamp":1757030400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["SIViP"],"published-print":{"date-parts":[[2025,12]]},"DOI":"10.1007\/s11760-025-04611-9","type":"journal-article","created":{"date-parts":[[2025,9,5]],"date-time":"2025-09-05T16:04:55Z","timestamp":1757088295000},"update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":2,"title":["Multimodal knowledge retrieval of layout image text based on CLIP and ViT"],"prefix":"10.1007","volume":"19","author":[{"given":"Bowen","family":"Zeng","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Rong","family":"Lu","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Guanghu","family":"Mao","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2025,9,5]]},"reference":[{"issue":"5","key":"4611_CR1","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1145\/3580501","volume":"19","author":"D Feng","year":"2023","unstructured":"Feng, D., He, X., Peng, Y.: MKVSE: Multimodal knowledge enhanced visual-semantic embedding for image-text retrieval[J]. ACM Trans. Multimedia Comput. Commun. Appl. 19(5), 1\u201321 (2023)","journal-title":"ACM Trans. Multimedia Comput. Commun. Appl."},{"issue":"10","key":"4611_CR2","doi-asserted-by":"publisher","first-page":"5486","DOI":"10.1109\/TCSVT.2023.3257193","volume":"33","author":"W Ma","year":"2023","unstructured":"Ma, W., Chen, Q., Zhou, T., et al.: Using multimodal contrastive knowledge distillation for video-text retrieval[J]. IEEE Trans. Circuits Syst. Video Technol. 33(10), 5486\u20135497 (2023)","journal-title":"IEEE Trans. Circuits Syst. Video Technol."},{"issue":"2","key":"4611_CR3","doi-asserted-by":"publisher","first-page":"715","DOI":"10.1109\/TKDE.2022.3224228","volume":"36","author":"X Zhu","year":"2022","unstructured":"Zhu, X., Li, Z., Wang, X., et al.: Multi-modal knowledge graph construction and application: A survey[J]. IEEE Trans. Knowl. Data Eng. 36(2), 715\u2013735 (2022)","journal-title":"IEEE Trans. Knowl. Data Eng."},{"issue":"7","key":"4611_CR4","first-page":"9169","volume":"45","author":"Z Zhang","year":"2023","unstructured":"Zhang, Z., Chen, K., Wang, R., et al.: Universal multimodal representation for Language understanding[J]. IEEE Trans. Pattern Anal. Mach. Intell. 45(7), 9169\u20139185 (2023)","journal-title":"IEEE Trans. Pattern Anal. Mach. Intell."},{"issue":"5","key":"4611_CR5","doi-asserted-by":"publisher","first-page":"3156","DOI":"10.1109\/TPAMI.2023.3339661","volume":"46","author":"Y Zeng","year":"2023","unstructured":"Zeng, Y., Zhang, X., Li, H., et al.: X $^{2} $2-VLM: All-in-One Pre-Trained model for Vision-Language Tasks[J]. IEEE Trans. Pattern Anal. Mach. Intell. 46(5), 3156\u20133168 (2023)","journal-title":"IEEE Trans. Pattern Anal. Mach. Intell."},{"issue":"1","key":"4611_CR6","doi-asserted-by":"publisher","first-page":"38","DOI":"10.1007\/s11633-022-1369-5","volume":"20","author":"FL Chen","year":"2023","unstructured":"Chen, F.L., Zhang, D.Z., Han, M.L., et al.: Vlp: A survey on vision-language pre-training[J]. Mach. Intell. Res. 20(1), 38\u201356 (2023)","journal-title":"Mach. Intell. Res."},{"issue":"7","key":"4611_CR7","doi-asserted-by":"publisher","first-page":"6559","DOI":"10.1109\/TCSVT.2024.3360530","volume":"34","author":"L Chen","year":"2024","unstructured":"Chen, L., Deng, Z., Liu, L., et al.: Multilevel semantic interaction alignment for video\u2013text cross-modal retrieval[J]. IEEE Trans. Circuits Syst. Video Technol. 34(7), 6559\u20136575 (2024)","journal-title":"IEEE Trans. Circuits Syst. Video Technol."},{"issue":"9","key":"4611_CR8","doi-asserted-by":"publisher","first-page":"5318","DOI":"10.1109\/TCSVT.2023.3247822","volume":"33","author":"M Hou","year":"2023","unstructured":"Hou, M., Zhang, Z., Liu, C., et al.: Semantic alignment network for multi-modal emotion recognition[J]. IEEE Trans. Circuits Syst. Video Technol. 33(9), 5318\u20135329 (2023)","journal-title":"IEEE Trans. Circuits Syst. Video Technol."},{"issue":"5","key":"4611_CR9","doi-asserted-by":"publisher","first-page":"2465","DOI":"10.1109\/TCSVT.2022.3220297","volume":"33","author":"Z Liu","year":"2022","unstructured":"Liu, Z., Chen, F., Xu, J., et al.: Image-text retrieval with cross-modal semantic importance consistency[J]. IEEE Trans. Circuits Syst. Video Technol. 33(5), 2465\u20132476 (2022)","journal-title":"IEEE Trans. Circuits Syst. Video Technol."},{"issue":"4","key":"4611_CR10","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1145\/3499027","volume":"18","author":"Y Cheng","year":"2022","unstructured":"Cheng, Y., Zhu, X., Qian, J., et al.: Cross-modal graph matching network for image-text retrieval[J]. ACM Trans. Multimedia Comput. Commun. Appl. (TOMM). 18(4), 1\u201323 (2022)","journal-title":"ACM Trans. Multimedia Comput. Commun. Appl. (TOMM)"},{"issue":"2","key":"4611_CR11","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1145\/3606368","volume":"42","author":"X Chen","year":"2023","unstructured":"Chen, X., Song, X., Jing, L., et al.: Multimodal dialog systems with dual knowledge-enhanced generative pretrained Language model[J]. ACM Trans. Inform. Syst. 42(2), 1\u201325 (2023)","journal-title":"ACM Trans. Inform. Syst."},{"issue":"12","key":"4611_CR12","doi-asserted-by":"publisher","first-page":"1447","DOI":"10.1038\/s42256-023-00759-6","volume":"5","author":"S Liu","year":"2023","unstructured":"Liu, S., Nie, W., Wang, C., et al.: Multi-modal molecule structure\u2013text model for text-based retrieval and editing[J]. Nat. Mach. Intell. 5(12), 1447\u20131457 (2023)","journal-title":"Nat. Mach. Intell."},{"issue":"1","key":"4611_CR13","doi-asserted-by":"publisher","first-page":"641","DOI":"10.1109\/TPAMI.2022.3148470","volume":"45","author":"K Li","year":"2022","unstructured":"Li, K., Zhang, Y., Li, K., et al.: Image-text embedding learning via visual and textual semantic reasoning[J]. IEEE Trans. Pattern Anal. Mach. Intell. 45(1), 641\u2013656 (2022)","journal-title":"IEEE Trans. Pattern Anal. Mach. Intell."},{"issue":"9","key":"4611_CR14","doi-asserted-by":"publisher","first-page":"8983","DOI":"10.1109\/TKDE.2022.3220625","volume":"35","author":"Q Fang","year":"2022","unstructured":"Fang, Q., Zhang, X., Hu, J., et al.: Contrastive multi-modal knowledge graph representation learning[J]. IEEE Trans. Knowl. Data Eng. 35(9), 8983\u20138996 (2022)","journal-title":"IEEE Trans. Knowl. Data Eng."},{"issue":"1","key":"4611_CR15","doi-asserted-by":"publisher","first-page":"239","DOI":"10.1109\/TKDE.2023.3282921","volume":"36","author":"L Zhu","year":"2023","unstructured":"Zhu, L., Zheng, C., Guan, W., et al.: Multi-modal hashing for efficient multimedia retrieval: A survey[J]. IEEE Trans. Knowl. Data Eng. 36(1), 239\u2013260 (2023)","journal-title":"IEEE Trans. Knowl. Data Eng."},{"issue":"9","key":"4611_CR16","doi-asserted-by":"publisher","first-page":"6289","DOI":"10.1109\/TNNLS.2021.3135420","volume":"34","author":"T Hoang","year":"2022","unstructured":"Hoang, T., Do, T.T., Nguyen, T.V., et al.: Multimodal mutual information maximization: A novel approach for unsupervised deep cross-modal hashing[J]. IEEE Trans. Neural Networks Learn. Syst. 34(9), 6289\u20136302 (2022)","journal-title":"IEEE Trans. Neural Networks Learn. Syst."},{"issue":"2","key":"4611_CR17","doi-asserted-by":"publisher","first-page":"367","DOI":"10.1007\/s10844-022-00764-y","volume":"61","author":"SK Uppada","year":"2023","unstructured":"Uppada, S.K., Patel, P.: An image and text-based multimodal model for detecting fake news in OSN\u2019s[J]. J. Intell. Inform. Syst. 61(2), 367\u2013393 (2023)","journal-title":"J. Intell. Inform. Syst."},{"issue":"7","key":"4611_CR18","doi-asserted-by":"publisher","first-page":"3516","DOI":"10.1109\/TCSVT.2022.3233042","volume":"33","author":"W Li","year":"2022","unstructured":"Li, W., Ma, Z., Deng, L.J., et al.: Neuron-based spiking transmission and reasoning network for robust image-text retrieval[J]. IEEE Trans. Circuits Syst. Video Technol. 33(7), 3516\u20133528 (2022)","journal-title":"IEEE Trans. Circuits Syst. Video Technol."},{"issue":"12","key":"4611_CR19","doi-asserted-by":"publisher","first-page":"6070","DOI":"10.1109\/JBHI.2022.3207502","volume":"26","author":"JH Moon","year":"2022","unstructured":"Moon, J.H., Lee, H., Shin, W., et al.: Multi-modal Understanding and generation for medical images and text via vision-language pre-training[J]. IEEE J. Biomedical Health Inf. 26(12), 6070\u20136080 (2022)","journal-title":"IEEE J. Biomedical Health Inf."},{"issue":"4","key":"4611_CR20","doi-asserted-by":"publisher","first-page":"717","DOI":"10.1007\/s13735-022-00262-5","volume":"11","author":"SI Papadopoulos","year":"2022","unstructured":"Papadopoulos, S.I., Koutlis, C., Papadopoulos, S., et al.: Multimodal Quasi-AutoRegression: Forecasting the visual popularity of new fashion products[J]. Int. J. Multimedia Inform. Retr. 11(4), 717\u2013729 (2022)","journal-title":"Int. J. Multimedia Inform. Retr."},{"issue":"1","key":"4611_CR21","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1145\/3475867","volume":"22","author":"A Yadav","year":"2021","unstructured":"Yadav, A., Vishwakarma, D.K.: A language-independent network to analyze the impact of COVID-19 on the world via sentiment analysis[J]. ACM Trans. Internet Technol. (TOIT). 22(1), 1\u201330 (2021)","journal-title":"ACM Trans. Internet Technol. (TOIT)"},{"issue":"1","key":"4611_CR22","doi-asserted-by":"publisher","first-page":"504","DOI":"10.1109\/TCSVT.2023.3287296","volume":"34","author":"P Guo","year":"2023","unstructured":"Guo, P., Song, Y., Deng, Y., et al.: DCMAI: A dynamical cross-modal alignment interaction framework for document key information extraction[J]. IEEE Trans. Circuits Syst. Video Technol. 34(1), 504\u2013517 (2023)","journal-title":"IEEE Trans. Circuits Syst. Video Technol."},{"issue":"3","key":"4611_CR23","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1145\/3715330","volume":"21","author":"J Wang","year":"2025","unstructured":"Wang, J., Lin, Z., Huang, D., et al.: LiLTv2: Language-substitutable layout-image transformer for visual information extraction[J]. ACM Trans. Multimedia Comput. Commun. Appl. 21(3), 1\u201327 (2025)","journal-title":"ACM Trans. Multimedia Comput. Commun. Appl."},{"issue":"3","key":"4611_CR24","doi-asserted-by":"publisher","first-page":"149","DOI":"10.1007\/s10032-021-00375-3","volume":"24","author":"M Hole\u010dek","year":"2021","unstructured":"Hole\u010dek, M.: Learning from similarity and information extraction from structured documents[J]. Int. J. Doc. Anal. Recognit. (IJDAR). 24(3), 149\u2013165 (2021)","journal-title":"Int. J. Doc. Anal. Recognit. (IJDAR)"},{"issue":"3","key":"4611_CR25","doi-asserted-by":"publisher","first-page":"310","DOI":"10.1002\/path.6232","volume":"262","author":"D Truhn","year":"2024","unstructured":"Truhn, D., Loeffler, C.M.L., M\u00fcller-Franzes, G., et al.: Extracting structured information from unstructured histopathology reports using generative pre\u2010trained transformer 4 (GPT\u20104)[J]. J. Pathol. 262(3), 310\u2013319 (2024)","journal-title":"J. Pathol."},{"issue":"1","key":"4611_CR26","first-page":"55","volume":"6","author":"E Blash","year":"2023","unstructured":"Blash, E., HIGH-LEVEL INFORMATION FUSION SITUATIONAL, DEVELOPMENTS[J]: Perspect. Inform. Fusion. 6(1), 55\u201356 (2023)","journal-title":"Perspect. Inform. Fusion"},{"issue":"3","key":"4611_CR27","doi-asserted-by":"publisher","first-page":"447","DOI":"10.1007\/s10032-024-00483-w","volume":"27","author":"S Saifullah","year":"2024","unstructured":"Saifullah, S., Agne, S., Dengel, A., et al.: DocXclassifier: Towards a robust and interpretable deep neural network for document image classification[J]. Int. J. Doc. Anal. Recognit. (IJDAR). 27(3), 447\u2013473 (2024)","journal-title":"Int. J. Doc. Anal. Recognit. (IJDAR)"},{"issue":"1","key":"4611_CR28","doi-asserted-by":"publisher","first-page":"448","DOI":"10.3390\/make6010023","volume":"6","author":"S Dutta","year":"2024","unstructured":"Dutta, S., Adhikary, S., Dwivedi, A.D.: Visformers\u2014combining vision and Transformers for enhanced complex document classification[J]. Mach. Learn. Knowl. Extr. 6(1), 448\u2013463 (2024)","journal-title":"Mach. Learn. Knowl. Extr."},{"issue":"10","key":"4611_CR29","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1145\/3561970","volume":"55","author":"N Rethmeier","year":"2023","unstructured":"Rethmeier, N., Augenstein, I.: A primer on contrastive pretraining in Language processing: Methods, lessons learned, and perspectives[J]. ACM Comput. Surveys. 55(10), 1\u201317 (2023)","journal-title":"ACM Comput. Surveys"},{"issue":"4","key":"4611_CR30","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1145\/3592097","volume":"42","author":"T Ao","year":"2023","unstructured":"Ao, T., Zhang, Z., Liu, L., Gesturediffuclip: Gesture diffusion model with clip latents[J]. ACM Trans. Graphics (TOG). 42(4), 1\u201318 (2023)","journal-title":"ACM Trans. Graphics (TOG)"},{"key":"4611_CR31","doi-asserted-by":"crossref","unstructured":"Gan, Z., Li, L., Li, C., et al.: Vision-language pre-training: Basics, recent advances, and future trends[J]. Foundations and Trends\u00ae in Computer Graphics and Vision, 14(3\u20134): 163\u2013352. (2022)","DOI":"10.1561\/0600000105"},{"issue":"39","key":"4611_CR32","doi-asserted-by":"publisher","first-page":"86833","DOI":"10.1007\/s11042-024-19667-x","volume":"83","author":"ML ABIMOULOUD","year":"2024","unstructured":"ABIMOULOUD, M.L., BENSID, K., Elleuch, M., et al.: Vision transformer based convolutional neural network for breast cancer histopathological images classification[J]. Multimedia Tools Appl. 83(39), 86833\u201386868 (2024)","journal-title":"Multimedia Tools Appl."},{"issue":"12","key":"4611_CR33","doi-asserted-by":"publisher","first-page":"12192","DOI":"10.1002\/int.23082","volume":"37","author":"X Ren","year":"2022","unstructured":"Ren, X., Zheng, X., Zhou, H., et al.: Contrastive hashing with vision transformer for image retrieval[J]. Int. J. Intell. Syst. 37(12), 12192\u201312211 (2022)","journal-title":"Int. J. Intell. Syst."},{"issue":"6","key":"4611_CR34","doi-asserted-by":"publisher","first-page":"2888","DOI":"10.1109\/TVCG.2023.3261935","volume":"29","author":"Y Li","year":"2023","unstructured":"Li, Y., Wang, J., Dai, X., et al.: How does attention work in vision transformers? A visual analytics attempt[J]. IEEE Trans. Vis. Comput. Graph. 29(6), 2888\u20132900 (2023)","journal-title":"IEEE Trans. Vis. Comput. Graph."},{"issue":"7","key":"4611_CR35","doi-asserted-by":"publisher","first-page":"3841","DOI":"10.1007\/s11042-024-19220-w","volume":"84","author":"U Kosarkar","year":"2025","unstructured":"Kosarkar, U., Sakarkar, G.: Design an efficient VARMA LSTM GRU model for identification of deep-fake images via dynamic window-based spatio-temporal analysis[J]. Multimedia Tools Appl. 84(7), 3841\u20133857 (2025)","journal-title":"Multimedia Tools Appl."},{"issue":"10","key":"4611_CR36","doi-asserted-by":"publisher","first-page":"1195","DOI":"10.1038\/s41565-023-01422-2","volume":"18","author":"Y Yang","year":"2023","unstructured":"Yang, Y., Chen, Y., Pei, P., et al.: Fluorescence-amplified nanocrystals in the second near-infrared window for in vivo real-time dynamic multiplexed imaging[J]. Nat. Nanotechnol. 18(10), 1195\u20131204 (2023)","journal-title":"Nat. Nanotechnol."}],"container-title":["Signal, Image and Video Processing"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s11760-025-04611-9.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s11760-025-04611-9\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s11760-025-04611-9.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,9,22]],"date-time":"2025-09-22T13:16:25Z","timestamp":1758546985000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s11760-025-04611-9"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,9,5]]},"references-count":36,"journal-issue":{"issue":"12","published-print":{"date-parts":[[2025,12]]}},"alternative-id":["4611"],"URL":"https:\/\/doi.org\/10.1007\/s11760-025-04611-9","relation":{},"ISSN":["1863-1703","1863-1711"],"issn-type":[{"value":"1863-1703","type":"print"},{"value":"1863-1711","type":"electronic"}],"subject":[],"published":{"date-parts":[[2025,9,5]]},"assertion":[{"value":"22 May 2025","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"14 July 2025","order":2,"name":"revised","label":"Revised","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"1 August 2025","order":3,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"5 September 2025","order":4,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Declarations"}},{"value":"The authors declare no competing interests.","order":2,"name":"Ethics","group":{"name":"EthicsHeading","label":"Competing interests"}},{"value":"All authors declare that they have no conflicts of interest.","order":3,"name":"Ethics","group":{"name":"EthicsHeading","label":"Conflict of interest"}}],"article-number":"1001"}}