{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,9]],"date-time":"2026-07-09T06:07:53Z","timestamp":1783577273772,"version":"3.55.0"},"reference-count":47,"publisher":"Springer Science and Business Media LLC","issue":"10","license":[{"start":{"date-parts":[[2025,3,11]],"date-time":"2025-03-11T00:00:00Z","timestamp":1741651200000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2025,3,11]],"date-time":"2025-03-11T00:00:00Z","timestamp":1741651200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Vis Comput"],"published-print":{"date-parts":[[2025,8]]},"DOI":"10.1007\/s00371-025-03829-5","type":"journal-article","created":{"date-parts":[[2025,3,11]],"date-time":"2025-03-11T16:33:01Z","timestamp":1741710781000},"page":"7657-7670","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":5,"title":["Multimodal retrieval-augmented generation for financial documents: image-centric analysis of charts and tables with large language models"],"prefix":"10.1007","volume":"41","author":[{"given":"Cheng","family":"Jiang","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Pengle","family":"Zhang","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Ying","family":"Ni","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Xiaoli","family":"Wang","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Hanghang","family":"Peng","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Sen","family":"Liu","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Mengdi","family":"Fei","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Yuxin","family":"He","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Yaxuan","family":"Xiao","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Jin","family":"Huang","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Xingyu","family":"Ma","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Tian","family":"Yang","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2025,3,11]]},"reference":[{"issue":"1","key":"3829_CR1","doi-asserted-by":"publisher","first-page":"33","DOI":"10.1007\/s42001-019-00035-x","volume":"2","author":"W Souma","year":"2019","unstructured":"Souma, W., Vodenska, I., Aoyama, H.: Enhanced news sentiment analysis using deep learning methods. J. Comput. Soc. Sci. 2(1), 33\u201346 (2019)","journal-title":"J. Comput. Soc. Sci."},{"key":"3829_CR2","doi-asserted-by":"crossref","unstructured":"Liu, Z., Huang, D., Huang, K., Li, Z., Zhao, J.: FinBERT: a pre-trained financial language representation model for financial text mining. In: Proceedings of the Twenty-Ninth International Conference on International Joint Conferences on Artificial Intelligence pp. 4513\u20134519 (2021)","DOI":"10.24963\/ijcai.2020\/622"},{"key":"3829_CR3","unstructured":"Yang, Y., Tang, Y., Tam, K.Y.: InvestLM: a large language model for investment using financial domain instruction tuning (2023). arxiv preprint arxiv:2309.13064"},{"key":"3829_CR4","doi-asserted-by":"crossref","unstructured":"Chapman, C.L., Hillebrand, L., Stenzel, M.R., Deu\u00dfer, T., Biesner, D., Bauckhage, C., Sifa, R.: Towards generating financial reports from tabular data using transformers. In: International Cross-Domain Conference for Machine Learning and Knowledge Extraction, pp. 221\u2013232. Springer, Cham (2022)","DOI":"10.1007\/978-3-031-14463-9_14"},{"key":"3829_CR5","unstructured":"La Quatra, M., Cagliero, L.: End-to-end training for financial report summarization. In: Proceedings of the 1st Joint Workshop on Financial Narrative Processing and MultiLing Financial Summarisation, pp. 118\u2013123 (2020)"},{"key":"3829_CR6","unstructured":"Kang, H., Liu, X.Y.: Deficiency of large language models in finance: an empirical examination of hallucination. In: I Can\u2019t Believe It\u2019s Not Better Workshop: Failure Modes in the Age of Foundation Models (2023)"},{"key":"3829_CR7","doi-asserted-by":"publisher","first-page":"157","DOI":"10.1162\/tacl_a_00638","volume":"12","author":"NF Liu","year":"2024","unstructured":"Liu, N.F., Lin, K., Hewitt, J., Paranjape, A., Bevilacqua, M., Petroni, F., Liang, P.: Lost in the middle: how language models use long contexts. Trans. Assoc. Comput. Linguist. 12, 157\u2013173 (2024)","journal-title":"Trans. Assoc. Comput. Linguist."},{"key":"3829_CR8","first-page":"9459","volume":"33","author":"P Lewis","year":"2020","unstructured":"Lewis, P., Perez, E., Piktus, A., Petroni, F., Karpukhin, V., Goyal, N., Kiela, D.: Retrieval-augmented generation for knowledge-intensive nlp tasks. Adv. Neural Inf. Process. Syst. 33, 9459\u20139474 (2020)","journal-title":"Adv. Neural Inf. Process. Syst."},{"key":"3829_CR9","doi-asserted-by":"crossref","unstructured":"Zhang, B., Yang, H., Zhou, T., Ali Babar, M., Liu, X.Y.: Enhancing financial sentiment analysis via retrieval augmented large language models. In: Proceedings of the Fourth ACM International Conference on AI in Finance, pp. 349\u2013356 (2023)","DOI":"10.1145\/3604237.3626866"},{"key":"3829_CR10","unstructured":"Yepes, A.J., You, Y., Milczek, J., Laverde, S., Li, R.: Financial report chunking for effective retrieval augmented generation (2024). arxiv preprint arxiv:2402.05131"},{"key":"3829_CR11","doi-asserted-by":"publisher","first-page":"50","DOI":"10.1109\/TMM.2021.3120873","volume":"25","author":"X Lin","year":"2021","unstructured":"Lin, X., Sun, S., Huang, W., Sheng, B., Li, P., Feng, D.D.: EAPT: efficient attention pyramid transformer for image processing. IEEE Trans. Multimed. 25, 50\u201361 (2021)","journal-title":"IEEE Trans. Multimed."},{"issue":"4","key":"3829_CR12","doi-asserted-by":"publisher","first-page":"1083","DOI":"10.1109\/TMI.2022.3223683","volume":"42","author":"R Liu","year":"2022","unstructured":"Liu, R., Wang, T., Li, H., Zhang, P., Li, J., Yang, X., Sheng, B.: TMM-Nets: transferred multi-to mono-modal generation for lupus retinopathy diagnosis. IEEE Trans. Med. Imaging 42(4), 1083\u20131094 (2022)","journal-title":"IEEE Trans. Med. Imaging"},{"issue":"8","key":"3829_CR13","doi-asserted-by":"publisher","first-page":"3183","DOI":"10.1007\/s00371-023-02965-0","volume":"39","author":"D Meng","year":"2023","unstructured":"Meng, D., Li, S., Sheng, B., Wu, H., Tian, S., Ma, W., Yan, X.: 3D reconstruction-oriented fully automatic multi-modal tumor segmentation by dual attention-guided VNet. Vis. Comput. 39(8), 3183\u20133196 (2023)","journal-title":"Vis. Comput."},{"key":"3829_CR14","doi-asserted-by":"crossref","unstructured":"Chen, W., Hu, H., Chen, X., Verga, P., Cohen, W.W.: MuRAG: multimodal retrieval-augmented generator for open question answering over images and text (2022). arxiv preprint arxiv:2210.02928","DOI":"10.18653\/v1\/2022.emnlp-main.375"},{"key":"3829_CR15","doi-asserted-by":"crossref","unstructured":"Yuan, Z., Q., Tan, C., Zhao, Z., Yuan, H., Huang, F., Huang, S.: RAMM: retrieval-augmented biomedical visual question answering with multi-modal pre-training. In: Proceedings of the 31st ACM International Conference on Multimedia, pp. 547\u2013556 (2023)","DOI":"10.1145\/3581783.3611830"},{"key":"3829_CR16","doi-asserted-by":"crossref","unstructured":"Hu, A., Xu, H., Ye, J., Yan, M., Zhang, L., Zhang, B., Zhou, J.: mPLUG-DocOwl 1.5: unified structure learning for OCR-free document understanding (2024). arxiv preprint arxiv:2403.12895","DOI":"10.18653\/v1\/2024.findings-emnlp.175"},{"key":"3829_CR17","doi-asserted-by":"crossref","unstructured":"Chen, Z., Wu, J., Wang, W., Su, W., Chen, G., Xing, S., Dai, J.: InternVL: scaling up vision foundation models and aligning for generic visual-linguistic tasks. In: Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition, pp. 24185\u201324198 (2024)","DOI":"10.1109\/CVPR52733.2024.02283"},{"key":"3829_CR18","unstructured":"Es, S., James, J., Espinosa-Anke, L., Schockaert, S.: RAGAS: automated evaluation of retrieval augmented generation (2023). arxiv preprint arxiv:2309.15217"},{"key":"3829_CR19","doi-asserted-by":"crossref","unstructured":"Yao, J., Chen, J., Niu, L., Sheng, B.: Scene-aware human pose generation using transformer. In: Proceedings of the 31st ACM International Conference on Multimedia, pp. 2847\u20132855 (2023)","DOI":"10.1145\/3581783.3612439"},{"key":"3829_CR20","first-page":"1877","volume":"33","author":"T Brown","year":"2020","unstructured":"Brown, T., Mann, B., Ryder, N., Subbiah, M., Kaplan, J.D., Dhariwal, P., Amodei, D.: Language models are few-shot learners. Adv. Neural Inf. Process. Syst. 33, 1877\u20131901 (2020)","journal-title":"Adv. Neural Inf. Process. Syst."},{"key":"3829_CR21","unstructured":"Kenton, J.D.M.W.C., Toutanova, L.K.: BERT: pre-training of deep bidirectional transformers for language understanding. In: Proceedings of NAACL-HLT, vol. 1, p. 2 (2019)"},{"key":"3829_CR22","first-page":"27730","volume":"35","author":"L Ouyang","year":"2022","unstructured":"Ouyang, L., Wu, J., Jiang, X., Almeida, D., Wainwright, C., Mishkin, P., Lowe, R.: Training language models to follow instructions with human feedback. Adv. Neural Inf. Process. Syst. 35, 27730\u201327744 (2022)","journal-title":"Adv. Neural Inf. Process. Syst."},{"key":"3829_CR23","unstructured":"Wu, S., Irsoy, O., Lu, S., Dabravolski, V., Dredze, M., Gehrmann, S., Mann, G.: BloombergGPT: a large language model for finance (2023). arXiv preprint arXiv:2303.17564"},{"key":"3829_CR24","unstructured":"Xie, Q., Han, W., Zhang, X., Lai, Y., Peng, M., Lopez-Lira, A., Huang, J.: PIXIU: a large language model, instruction data and evaluation benchmark for finance (2023). arXiv preprint arXiv:2306.05443"},{"key":"3829_CR25","doi-asserted-by":"crossref","unstructured":"Zhang, B., Yang, H., Liu, X.Y.: Instruct-FinGPT: financial sentiment analysis by instruction tuning of general-purpose large language models (2023). arXiv preprint arXiv:2306.12659","DOI":"10.2139\/ssrn.4489831"},{"key":"3829_CR26","doi-asserted-by":"crossref","unstructured":"Rajpoot, P.K., Parikh, A.: GPT-FinRE: In-context learning for financial relation extraction using large language models (2023). arXiv preprint arXiv:2306.17519","DOI":"10.18653\/v1\/2023.finnlp-2.5"},{"key":"3829_CR27","doi-asserted-by":"crossref","unstructured":"Li, Y., Wang, S., Ding, H., Chen, H.: Large language models in finance: a survey. In: Proceedings of the Fourth ACM International Conference on AI in Finance, pp. 374\u2013382 (2023)","DOI":"10.1145\/3604237.3626869"},{"key":"3829_CR28","doi-asserted-by":"crossref","unstructured":"Guo, Y., Xu, Z., Yang, Y.: Is ChatGPT a financial expert? Evaluating language models on financial natural language processing (2023). arXiv preprint arXiv:2310.12664","DOI":"10.18653\/v1\/2023.findings-emnlp.58"},{"key":"3829_CR29","unstructured":"Chu, Z., Guo, H., Zhou, X., Wang, Y., Yu, F., Chen, H., Li, S.: Data-centric financial large language models (2023). arXiv preprint arXiv:2310.17784"},{"key":"3829_CR30","doi-asserted-by":"publisher","first-page":"8294","DOI":"10.1007\/s11227-019-03101-3","volume":"76","author":"SI Lee","year":"2020","unstructured":"Lee, S.I., Yoo, S.J.: Multimodal deep learning for finance: integrating and forecasting international stock markets. J. Supercomput. 76, 8294\u20138312 (2020)","journal-title":"J. Supercomput."},{"key":"3829_CR31","first-page":"1","volume":"36","author":"H Liu","year":"2024","unstructured":"Liu, H., Li, C., Wu, Q., Lee, Y.J.: Visual instruction tuning. Adv. Neural Inf. Process. Syst. 36, 1\u201324 (2024)","journal-title":"Adv. Neural Inf. Process. Syst."},{"key":"3829_CR32","unstructured":"Li, J., Li, D., Savarese, S., Hoi, S.: BLIP-2: bootstrapping language-image pre-training with frozen image encoders and large language models. In: International Conference on Machine Learning, pp. 19730\u201319742. PMLR (2023)"},{"key":"3829_CR33","unstructured":"Wang, Z., Li, Y., Wu, J., Soon, J., Zhang, X.: FinVis-GPT: a multimodal large language model for financial chart analysis (2023). arXiv preprint arXiv:2308.01430"},{"key":"3829_CR34","doi-asserted-by":"crossref","unstructured":"Bhatia, G., Nagoudi, E.M.B., Cavusoglu, H., Abdul-Mageed, M.: FinTral: a family of GPT-4 level multimodal financial large language models (2024). arXiv preprint arXiv:2402.10986","DOI":"10.18653\/v1\/2024.findings-acl.774"},{"key":"3829_CR35","unstructured":"Jiang, A.Q., Sablayrolles, A., Mensch, A., Bamford, C., Chaplot, D.S., Casas, D.D.L., Sayed, W.E.: Mistral 7B (2023). arXiv preprint arXiv:2310.06825"},{"key":"3829_CR36","unstructured":"Yao, Y., Yu, T., Zhang, A., Wang, C., Cui, J., Zhu, H., Sun, M.: MiniCPM-V: a GPT-4V level MLLM on your phone (2024). arXiv preprint arXiv:2408.01800"},{"key":"3829_CR37","unstructured":"Yao, L., Peng, J., Mao, C., Luo, Y.: Exploring large language models for knowledge graph completion (2023). arXiv preprint arXiv:2308.13916"},{"key":"3829_CR38","doi-asserted-by":"crossref","unstructured":"Baek, J., Aji, A.F., Saffari, A.: Knowledge-augmented language model prompting for zero-shot knowledge graph question answering (2023). arXiv preprint arXiv:2306.04136","DOI":"10.18653\/v1\/2023.nlrse-1.7"},{"key":"3829_CR39","unstructured":"Kang, M., Kwak, J.M., Baek, J., Hwang, S.J.: Knowledge graph-augmented language models for knowledge-grounded dialogue generation (2023). arXiv preprint arXiv:2305.18846"},{"key":"3829_CR40","doi-asserted-by":"crossref","unstructured":"Wang, Y., Lipka, N., Rossi, R.A., Siu, A., Zhang, R., Derr, T.: Knowledge graph prompting for multi-document question answering. In: Proceedings of the AAAI Conference on Artificial Intelligence, Vol. 38, No. 17, pp. 19206\u201319214 (2024)","DOI":"10.1609\/aaai.v38i17.29889"},{"key":"3829_CR41","doi-asserted-by":"crossref","unstructured":"Chen, J., Xiao, S., Zhang, P., Luo, K., Lian, D., Liu, Z.: BGE M3-embedding: multi-lingual, multi-functionality, multi-granularity text embeddings through self-knowledge distillation (2024). arXiv preprint arXiv:2402.03216","DOI":"10.18653\/v1\/2024.findings-acl.137"},{"key":"3829_CR42","doi-asserted-by":"crossref","unstructured":"Wang, J., Yi, X., Guo, R., Jin, H., Xu, P., Li, S., Xie, C.: Milvus: a purpose-built vector data management system. In: Proceedings of the 2021 International Conference on Management of Data, pp. 2614\u20132627 (2021)","DOI":"10.1145\/3448016.3457550"},{"key":"3829_CR43","doi-asserted-by":"crossref","unstructured":"Guo, R., Luan, X., Xiang, L., Yan, X., Yi, X., Luo, J., Xie, C.: Manu: a cloud native vector database management system (2022). arXiv preprint arXiv:2206.13843","DOI":"10.14778\/3554821.3554843"},{"key":"3829_CR44","doi-asserted-by":"crossref","unstructured":"Xiao, S., Liu, Z., Zhang, P., Muennighoff, N., Lian, D., Nie, J.Y.: C-pack: packed resources for general Chinese embeddings. In: Proceedings of the 47th International ACM SIGIR Conference on Research and Development in Information Retrieval, pp. 641\u2013649 (2024)","DOI":"10.1145\/3626772.3657878"},{"key":"3829_CR45","unstructured":"Wu, M., Yi, X., Yu, H., Liu, Y., Wang, Y.: Nebula graph: an open source distributed graph database (2022). arXiv preprint arXiv:2206.07278"},{"key":"3829_CR46","unstructured":"Bai, J., Bai, S., Chu, Y., Cui, Z., Dang, K., Deng, X., Zhu, T.: Qwen technical report (2023). arXiv preprint arXiv:2309.16609"},{"key":"3829_CR47","unstructured":"Thakur, N., Reimers, N., R\u00fcckl\u00e9, A., Srivastava, A., Gurevych, I.: BEIR: a heterogenous benchmark for zero-shot evaluation of information retrieval models (2021). arXiv preprint arXiv:2104.08663"}],"container-title":["The Visual Computer"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s00371-025-03829-5.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s00371-025-03829-5\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s00371-025-03829-5.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,9,6]],"date-time":"2025-09-06T07:36:17Z","timestamp":1757144177000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s00371-025-03829-5"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,3,11]]},"references-count":47,"journal-issue":{"issue":"10","published-print":{"date-parts":[[2025,8]]}},"alternative-id":["3829"],"URL":"https:\/\/doi.org\/10.1007\/s00371-025-03829-5","relation":{},"ISSN":["0178-2789","1432-2315"],"issn-type":[{"value":"0178-2789","type":"print"},{"value":"1432-2315","type":"electronic"}],"subject":[],"published":{"date-parts":[[2025,3,11]]},"assertion":[{"value":"21 January 2025","order":1,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"11 March 2025","order":2,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}}]}}