{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,18]],"date-time":"2026-04-18T11:49:33Z","timestamp":1776512973028,"version":"3.51.2"},"publisher-location":"Cham","reference-count":37,"publisher":"Springer Nature Switzerland","isbn-type":[{"value":"9783032095299","type":"print"},{"value":"9783032095305","type":"electronic"}],"license":[{"start":{"date-parts":[[2025,10,29]],"date-time":"2025-10-29T00:00:00Z","timestamp":1761696000000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2025,10,29]],"date-time":"2025-10-29T00:00:00Z","timestamp":1761696000000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2026]]},"DOI":"10.1007\/978-3-032-09530-5_1","type":"book-chapter","created":{"date-parts":[[2025,10,28]],"date-time":"2025-10-28T23:03:29Z","timestamp":1761692609000},"page":"3-21","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":0,"title":["mmRAG: A Modular Benchmark for\u00a0Retrieval-Augmented Generation over\u00a0Text, Tables, and\u00a0Knowledge Graphs"],"prefix":"10.1007","author":[{"ORCID":"https:\/\/orcid.org\/0009-0002-3410-9664","authenticated-orcid":false,"given":"Chuan","family":"Xu","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0009-0002-0610-7725","authenticated-orcid":false,"given":"Qiaosheng","family":"Chen","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0009-0004-1553-1485","authenticated-orcid":false,"given":"Yutong","family":"Feng","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-3539-7776","authenticated-orcid":false,"given":"Gong","family":"Cheng","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2025,10,29]]},"reference":[{"key":"1_CR1","doi-asserted-by":"publisher","unstructured":"Chen, J., Lin, H., Han, X., Sun, L.: Benchmarking large language models in retrieval-augmented generation. In: Wooldridge, M.J., Dy, J.G., Natarajan, S. (eds.) Thirty-Eighth AAAI Conference on Artificial Intelligence, AAAI 2024, Thirty-Sixth Conference on Innovative Applications of Artificial Intelligence, IAAI 2024, Fourteenth Symposium on Educational Advances in Artificial Intelligence, EAAI 2014, February 20-27, 2024, Vancouver, Canada. pp. 17754\u201317762. AAAI Press (2024). https:\/\/doi.org\/10.1609\/AAAI.V38I16.29728","DOI":"10.1609\/AAAI.V38I16.29728"},{"key":"1_CR2","unstructured":"Chen, W., Chang, M., Schlinger, E., Wang, W.Y., Cohen, W.W.: Open question answering over tables and text. CoRR abs\/2010.10439 (2020). https:\/\/arxiv.org\/abs\/2010.10439"},{"key":"1_CR3","doi-asserted-by":"publisher","unstructured":"Chen, W., Zha, H., Chen, Z., Xiong, W., Wang, H., Wang, W.Y.: Hybridqa: a dataset of multi-hop question answering over tabular and textual data. In: Findings of the Association for Computational Linguistics: EMNLP 2020, Online Event, 16-20 November 2020. Findings of ACL, vol. EMNLP 2020, pp. 1026\u20131036. Association for Computational Linguistics (2020). https:\/\/doi.org\/10.18653\/V1\/2020.FINDINGS-EMNLP.91","DOI":"10.18653\/V1\/2020.FINDINGS-EMNLP.91"},{"key":"1_CR4","doi-asserted-by":"publisher","unstructured":"Christmann, P., Roy, R.S., Weikum, G.: Compmix: A benchmark for heterogeneous question answering. In: Companion Proceedings of the ACM on Web Conference 2024, WWW 2024, Singapore, Singapore, May 13-17, 2024. pp. 1091\u20131094. ACM (2024). https:\/\/doi.org\/10.1145\/3589335.3651444","DOI":"10.1145\/3589335.3651444"},{"key":"1_CR5","unstructured":"DeepSeek-AI: Deepseek-v3 technical report (2024). https:\/\/arxiv.org\/abs\/2412.19437"},{"key":"1_CR6","doi-asserted-by":"publisher","unstructured":"Fan, W., et al.: A survey on RAG meeting llms: Towards retrieval-augmented large language models. In: Proceedings of the 30th ACM SIGKDD Conference on Knowledge Discovery and Data Mining, KDD 2024, Barcelona, Spain, August 25-29, 2024. pp. 6491\u20136501. ACM (2024). https:\/\/doi.org\/10.1145\/3637528.3671470","DOI":"10.1145\/3637528.3671470"},{"key":"1_CR7","doi-asserted-by":"publisher","unstructured":"Friel, R., Belyi, M., Sanyal, A.: Ragbench: explainable benchmark for retrieval-augmented generation systems. CoRR abs\/2407.11005 (2024). https:\/\/doi.org\/10.48550\/ARXIV.2407.11005","DOI":"10.48550\/ARXIV.2407.11005"},{"key":"1_CR8","unstructured":"GLM, T., et al.: ChatGLM: A family of large language models from GLM-130b to GLM-4 all tools (2024)"},{"key":"1_CR9","doi-asserted-by":"publisher","unstructured":"Gupta, S., Ranjan, R., Singh, S.N.: A comprehensive survey of retrieval-augmented generation (RAG): evolution, current landscape and future directions. CoRR abs\/2410.12837 (2024). https:\/\/doi.org\/10.48550\/ARXIV.2410.12837","DOI":"10.48550\/ARXIV.2410.12837"},{"key":"1_CR10","unstructured":"He, X., et al.: G-retriever: Retrieval-augmented generation for textual graph understanding and question answering. In: Advances in Neural Information Processing Systems 38: Annual Conference on Neural Information Processing Systems 2024, NeurIPS 2024, Vancouver, BC, Canada, December 10\u201315, 2024 (2024). http:\/\/papers.nips.cc\/paper_files\/paper\/2024\/hash\/efaf1c9726648c8ba363a5c927440529-Abstract-Conference.html"},{"key":"1_CR11","doi-asserted-by":"publisher","unstructured":"Izacard, G., Caron, M., Hosseini, L., Riedel, S., Bojanowski, P., Joulin, A., Grave, E.: Unsupervised dense information retrieval with contrastive learning (2021). https:\/\/doi.org\/10.48550\/ARXIV.2112.09118","DOI":"10.48550\/ARXIV.2112.09118"},{"key":"1_CR12","unstructured":"Joshi, M., Choi, E., Weld, D.S., Zettlemoyer, L.: Triviaqa: A large scale distantly supervised challenge dataset for reading comprehension. CoRR abs\/1705.03551 (2017). http:\/\/arxiv.org\/abs\/1705.03551"},{"key":"1_CR13","doi-asserted-by":"publisher","unstructured":"Karpukhin, V., et al.: Dense passage retrieval for open-domain question answering. In: Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing, EMNLP 2020, Online, November 16-20, 2020, pp. 6769\u20136781. Association for Computational Linguistics (2020). https:\/\/doi.org\/10.18653\/V1\/2020.EMNLP-MAIN.550","DOI":"10.18653\/V1\/2020.EMNLP-MAIN.550"},{"key":"1_CR14","doi-asserted-by":"publisher","unstructured":"Kwiatkowski, T., et al.: Natural questions: a benchmark for question answering research. Trans. Assoc. Comput. Linguist. 7, 452\u2013466 (2019). https:\/\/doi.org\/10.1162\/TACL_A_00276","DOI":"10.1162\/TACL_A_00276"},{"key":"1_CR15","unstructured":"Li, Z., Zhang, X., Zhang, Y., Long, D., Xie, P., Zhang, M.: Towards general text embeddings with multi-stage contrastive learning. arXiv preprint arXiv:2308.03281 (2023)"},{"key":"1_CR16","doi-asserted-by":"publisher","unstructured":"Luo, H., et al.: Chatkbqa: a generate-then-retrieve framework for knowledge base question answering with fine-tuned large language models. In: Findings of the Association for Computational Linguistics, ACL 2024, Bangkok, Thailand and virtual meeting, August 11-16, 2024, pp. 2039\u20132056. Association for Computational Linguistics (2024). https:\/\/doi.org\/10.18653\/V1\/2024.FINDINGS-ACL.122","DOI":"10.18653\/V1\/2024.FINDINGS-ACL.122"},{"key":"1_CR17","doi-asserted-by":"publisher","unstructured":"Marino, K., Rastegari, M., Farhadi, A., Mottaghi, R.: OK-VQA: a visual question answering benchmark requiring external knowledge. In: IEEE Conference on Computer Vision and Pattern Recognition, CVPR 2019, Long Beach, CA, USA, June 16-20, 2019, pp. 3195\u20133204. Computer Vision Foundation \/ IEEE (2019). https:\/\/doi.org\/10.1109\/CVPR.2019.00331, http:\/\/openaccess.thecvf.com\/content_CVPR_2019\/html\/Marino_OK-VQA_A_Visual_Question_Answering_Benchmark_Requiring_External_Knowledge_CVPR_2019_paper.html","DOI":"10.1109\/CVPR.2019.00331"},{"key":"1_CR18","doi-asserted-by":"publisher","unstructured":"Petroni, F., et al.: KILT: a benchmark for knowledge intensive language tasks. In: Proceedings of the 2021 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, NAACL-HLT 2021, Online, June 6-11, 2021, pp. 2523\u20132544. Association for Computational Linguistics (2021). https:\/\/doi.org\/10.18653\/V1\/2021.NAACL-MAIN.200","DOI":"10.18653\/V1\/2021.NAACL-MAIN.200"},{"key":"1_CR19","doi-asserted-by":"crossref","unstructured":"Rau, D., et al.: BERGEN: A benchmarking library for retrieval-augmented generation. In: Al-Onaizan, Y., Bansal, M., Chen, Y. (eds.) Findings of the Association for Computational Linguistics: EMNLP 2024, Miami, Florida, USA, November 12-16, 2024. pp. 7640\u20137663. Association for Computational Linguistics (2024). https:\/\/aclanthology.org\/2024.findings-emnlp.449","DOI":"10.18653\/v1\/2024.findings-emnlp.449"},{"key":"1_CR20","doi-asserted-by":"publisher","unstructured":"Samarinas, C., Zamani, H.: Procis: a benchmark for proactive retrieval in conversations. In: Proceedings of the 47th International ACM SIGIR Conference on Research and Development in Information Retrieval, SIGIR 2024, Washington DC, USA, July 14-18, 2024, pp. 830\u2013840. ACM (2024). https:\/\/doi.org\/10.1145\/3626772.3657869","DOI":"10.1145\/3626772.3657869"},{"key":"1_CR21","doi-asserted-by":"publisher","unstructured":"Shah, S., Mishra, A., Yadati, N., Talukdar, P.P.: KVQA: knowledge-aware visual question answering. In: The Thirty-Third AAAI Conference on Artificial Intelligence, AAAI 2019, The Thirty-First Innovative Applications of Artificial Intelligence Conference, IAAI 2019, The Ninth AAAI Symposium on Educational Advances in Artificial Intelligence, EAAI 2019, Honolulu, Hawaii, USA, January 27 - February 1, 2019, pp. 8876\u20138884. AAAI Press (2019). https:\/\/doi.org\/10.1609\/AAAI.V33I01.33018876","DOI":"10.1609\/AAAI.V33I01.33018876"},{"key":"1_CR22","doi-asserted-by":"crossref","unstructured":"Sun, W., et al.: MAIR: A massive benchmark for evaluating instructed retrieval. In: Proceedings of the 2024 Conference on Empirical Methods in Natural Language Processing, EMNLP 2024, Miami, FL, USA, November 12-16, 2024, pp. 14044\u201314067. Association for Computational Linguistics (2024), https:\/\/aclanthology.org\/2024.emnlp-main.778","DOI":"10.18653\/v1\/2024.emnlp-main.778"},{"key":"1_CR23","doi-asserted-by":"publisher","unstructured":"Talmor, A., Berant, J.: The web as a knowledge-base for answering complex questions. In: Proceedings of the 2018 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, NAACL-HLT 2018, New Orleans, Louisiana, USA, June 1-6, 2018, Volume 1 (Long Papers), pp. 641\u2013651. Association for Computational Linguistics (2018). https:\/\/doi.org\/10.18653\/V1\/N18-1059","DOI":"10.18653\/V1\/N18-1059"},{"key":"1_CR24","unstructured":"Talmor, A., et al.: Multimodalqa: complex question answering over text, tables and images. In: 9th International Conference on Learning Representations, ICLR 2021, Virtual Event, Austria, May 3-7, 2021. OpenReview.net (2021). https:\/\/openreview.net\/forum?id=ee6W5UgQLa"},{"key":"1_CR25","unstructured":"Team, Q.: Qwen2.5: A party of foundation models (Sept 2024). https:\/\/qwenlm.github.io\/blog\/qwen2.5\/"},{"key":"1_CR26","doi-asserted-by":"publisher","unstructured":"V, V., Prabhu, D., Anand, A.: DEXTER: A benchmark for open-domain complex question answering using LLMs. CoRR abs\/2406.17158 (2024). https:\/\/doi.org\/10.48550\/ARXIV.2406.17158","DOI":"10.48550\/ARXIV.2406.17158"},{"key":"1_CR27","doi-asserted-by":"publisher","unstructured":"Wang, P., Wu, Q., Shen, C., Dick, A.R., van\u00a0den Hengel, A.: FVQA: fact-based visual question answering. IEEE Trans. Pattern Anal. Mach. Intell. 40(10), 2413\u20132427 (2018). https:\/\/doi.org\/10.1109\/TPAMI.2017.2754246","DOI":"10.1109\/TPAMI.2017.2754246"},{"key":"1_CR28","doi-asserted-by":"crossref","unstructured":"Xiao, S., Liu, Z., Zhang, P., Muennighoff, N.: C-pack: Packaged resources to advance general Chinese embedding (2023)","DOI":"10.1145\/3626772.3657878"},{"key":"1_CR29","doi-asserted-by":"publisher","unstructured":"Xu, C., Chen, Q., Feng, Y., Cheng, G.: mmrag_benchmark (revision 72f010b) (2025). https:\/\/doi.org\/10.57967\/hf\/5475, https:\/\/huggingface.co\/datasets\/Askio\/mmrag_benchmark","DOI":"10.57967\/hf\/5475"},{"key":"1_CR30","unstructured":"Yang, A., et al.: Qwen2 technical report. arXiv preprint arXiv:2407.10671 (2024)"},{"key":"1_CR31","doi-asserted-by":"publisher","unstructured":"Yang, Z., et al.: Hotpotqa: a dataset for diverse, explainable multi-hop question answering. In: Proceedings of the 2018 Conference on Empirical Methods in Natural Language Processing, Brussels, Belgium, October 31 - November 4, 2018, pp. 2369\u20132380. Association for Computational Linguistics (2018). https:\/\/doi.org\/10.18653\/V1\/D18-1259","DOI":"10.18653\/V1\/D18-1259"},{"key":"1_CR32","unstructured":"Yeo, W., Kim, K., Jeong, S., Baek, J., Hwang, S.J.: Universalrag: Retrieval-augmented generation over multiple corpora with diverse modalities and granularities (2025). https:\/\/arxiv.org\/abs\/2504.20734"},{"key":"1_CR33","doi-asserted-by":"publisher","unstructured":"Yih, W., Richardson, M., Meek, C., Chang, M., Suh, J.: The value of semantic parse labeling for knowledge base question answering. In: Proceedings of the 54th Annual Meeting of the Association for Computational Linguistics, ACL 2016, August 7-12, 2016, Berlin, Germany, Volume 2: Short Papers. The Association for Computer Linguistics (2016). https:\/\/doi.org\/10.18653\/V1\/P16-2033","DOI":"10.18653\/V1\/P16-2033"},{"key":"1_CR34","doi-asserted-by":"publisher","unstructured":"Yu, S., et al.: Visrag: Vision-based retrieval-augmented generation on multi-modality documents. CoRR abs\/2410.10594 (2024). https:\/\/doi.org\/10.48550\/ARXIV.2410.10594","DOI":"10.48550\/ARXIV.2410.10594"},{"key":"1_CR35","doi-asserted-by":"publisher","unstructured":"Zhang, L., et al.: A survey on complex factual question answering. AI Open 4, 1\u201312 (2023). https:\/\/doi.org\/10.1016\/J.AIOPEN.2022.12.003, https:\/\/doi.org\/10.1016\/j.aiopen.2022.12.003","DOI":"10.1016\/J.AIOPEN.2022.12.003"},{"key":"1_CR36","doi-asserted-by":"crossref","unstructured":"Zhang, X, et\u00a0al.: mGTE: Generalized long-context text representation and reranking models for multilingual text retrieval. arXiv preprint arXiv:2407.19669 (2024)","DOI":"10.18653\/v1\/2024.emnlp-industry.103"},{"key":"1_CR37","doi-asserted-by":"publisher","unstructured":"Zhu, F., et al.: TAT-QA: A question answering benchmark on a hybrid of tabular and textual content in finance. In: Proceedings of the 59th Annual Meeting of the Association for Computational Linguistics and the 11th International Joint Conference on Natural Language Processing, ACL\/IJCNLP 2021, (Volume 1: Long Papers), Virtual Event, August 1-6, 2021, pp. 3277\u20133287. Association for Computational Linguistics (2021). https:\/\/doi.org\/10.18653\/V1\/2021.ACL-LONG.254","DOI":"10.18653\/V1\/2021.ACL-LONG.254"}],"container-title":["Lecture Notes in Computer Science","The Semantic Web \u2013 ISWC 2025"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-032-09530-5_1","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,4,18]],"date-time":"2026-04-18T11:13:31Z","timestamp":1776510811000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-3-032-09530-5_1"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,10,29]]},"ISBN":["9783032095299","9783032095305"],"references-count":37,"URL":"https:\/\/doi.org\/10.1007\/978-3-032-09530-5_1","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"value":"0302-9743","type":"print"},{"value":"1611-3349","type":"electronic"}],"subject":[],"published":{"date-parts":[[2025,10,29]]},"assertion":[{"value":"29 October 2025","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"The mmRAG benchmark data is available from Hugging Face\u00a0[\n                      \n                      ]. The source code related to mmRAG is available from GitHub at\n                      \n                      . All resources are available under the Apache License 2.0.","order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Resource Availability Statement"}},{"value":"ISWC","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"International Semantic Web Conference","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Nara","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Japan","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2025","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2 November 2025","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"6 November 2025","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"24","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"semweb2025","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"https:\/\/iswc2025.semanticweb.org\/","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}}]}}