{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,8,7]],"date-time":"2026-08-07T16:40:44Z","timestamp":1786120844490,"version":"build-2736575974"},"publisher-location":"New York, NY, USA","reference-count":51,"publisher":"ACM","license":[{"start":{"date-parts":[[2026,8,8]],"date-time":"2026-08-08T00:00:00Z","timestamp":1786147200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/legalcode"}],"funder":[{"name":"National Science Foundation","award":["CCF-1845763"],"award-info":[{"award-number":["CCF-1845763"]}]},{"name":"National Science Foundation","award":["CCF-2316235"],"award-info":[{"award-number":["CCF-2316235"]}]},{"name":"National Science Foundation","award":["CCF-2403237"],"award-info":[{"award-number":["CCF-2403237"]}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2026,8,9]]},"DOI":"10.1145\/3770855.3817481","type":"proceedings-article","created":{"date-parts":[[2026,8,6]],"date-time":"2026-08-06T17:15:47Z","timestamp":1786036547000},"page":"9403-9414","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":0,"title":["How Much Reasoning Do Retrieval-Augmented Models Add beyond LLMs? A Benchmarking Framework for Multi-Hop Inference over Hybrid Knowledge"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0009-0002-9121-7098","authenticated-orcid":false,"given":"Junhong","family":"Lin","sequence":"first","affiliation":[{"name":"EECS, Massachusetts Institute of Technology, Cambridge, MA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-1911-8466","authenticated-orcid":false,"given":"Bing","family":"Zhang","sequence":"additional","affiliation":[{"name":"IBM Research, Yorktown Heights, NY, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-1273-7694","authenticated-orcid":false,"given":"Song","family":"Wang","sequence":"additional","affiliation":[{"name":"University of Central Florida, Orlando, FL, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0002-5945-4923","authenticated-orcid":false,"given":"Ziyan","family":"Liu","sequence":"additional","affiliation":[{"name":"Harrisburg University of Science and Technology, Harrisburg, PA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-5101-4443","authenticated-orcid":false,"given":"Dan","family":"Gutfreund","sequence":"additional","affiliation":[{"name":"IBM Research, Yorktown Heights, NY, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-6163-6625","authenticated-orcid":false,"given":"Julian","family":"Shun","sequence":"additional","affiliation":[{"name":"EECS, Massachusetts Institute of Technology, Cambridge, MA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-3338-6371","authenticated-orcid":false,"given":"Yada","family":"Zhu","sequence":"additional","affiliation":[{"name":"IBM Research, Yorktown Heights, NY, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2026,8,8]]},"reference":[{"key":"e_1_3_2_1_1_1","volume-title":"Herd: Architecture, Training, Evaluation, and Deployment Notes. arXiv preprint arXiv:2601.11659","author":"Adcock Aaron","year":"2026","unstructured":"Aaron Adcock, Aayushi Srivastava, Abhimanyu Dubey, Abhinav Jauhri, Abhinav Pande, Abhinav Pandey, Abhinav Sharma, Abhishek Kadian, Abhishek Kumawat, Adam Kelsey, et al., 2026. The Llama 4 Herd: Architecture, Training, Evaluation, and Deployment Notes. arXiv preprint arXiv:2601.11659 (2026)."},{"key":"e_1_3_2_1_2_1","first-page":"9112","volume-title":"International Conference on Learning Representations (ICLR)","volume":"2024","author":"Asai Akari","year":"2024","unstructured":"Akari Asai, Zeqiu Wu, Yizhong Wang, Avi Sil, and Hannaneh Hajishirzi. 2024. Self-rag: Learning to retrieve, generate, and critique through self-reflection. In International Conference on Learning Representations (ICLR), Vol. 2024. 9112-9141."},{"key":"e_1_3_2_1_3_1","unstructured":"Shuai Bai Yuxuan Cai Ruizhe Chen Keqin Chen Xionghui Chen Zesen Cheng Lianghao Deng Wei Ding Chang Gao Chunjiang Ge et al. 2025. Qwen3-VL technical report. arXiv preprint arXiv:2511.21631 (2025)."},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D13-1160"},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1145\/1376616.1376746"},{"key":"e_1_3_2_1_6_1","first-page":"1877","article-title":". Language Models Are Few-Shot Learners","volume":"33","author":"Brown Tom","year":"2020","unstructured":"Tom Brown, Benjamin Mann, Nick Ryder, Melanie Subbiah, Jared D Kaplan, Prafulla Dhariwal, Arvind Neelakantan, Pranav Shyam, Girish Sastry, Amanda Askell, et al., 2020. Language Models Are Few-Shot Learners. Advances in Neural Information Processing Systems (NeurIPS), Vol. 33 (2020), 1877-1901.","journal-title":"Advances in Neural Information Processing Systems (NeurIPS)"},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2022.acl-long.422"},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"crossref","unstructured":"Liyi Chen Panrong Tong Zhongming Jin Ying Sun Jieping Ye and Hui Xiong. 2024. Plan-on-Graph: Self-Correcting Adaptive Planning of Large Language Model on Knowledge Graphs. In Advances in neural information processing systems (NeurIPS).","DOI":"10.52202\/079017-1189"},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2023.acl-long.637"},{"key":"e_1_3_2_1_10_1","volume-title":"International Conference on Machine Learning (ICML).","author":"Chiang Wei-Lin","year":"2024","unstructured":"Wei-Lin Chiang, Lianmin Zheng, Ying Sheng, Anastasios Nikolas Angelopoulos, Tianle Li, Dacheng Li, Banghua Zhu, Hao Zhang, Michael Jordan, Joseph E Gonzalez, et al., 2024. Chatbot arena: An open platform for evaluating LLMs by human preference. In International Conference on Machine Learning (ICML)."},{"key":"e_1_3_2_1_11_1","volume-title":"Proceedings of the 48th International ACM SIGIR Conference on Research and Development in Information Retrieval. 3498-3508","author":"Srikar Dammu Preetam Prabhu","year":"2025","unstructured":"Preetam Prabhu Srikar Dammu, Himanshu Naidu, and Chirag Shah. 2025. Dynamic-kgqa: A scalable framework for generating adaptive question answering datasets. In Proceedings of the 48th International ACM SIGIR Conference on Research and Development in Information Retrieval. 3498-3508."},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2024.findings-acl.716"},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-30796-7_5"},{"key":"e_1_3_2_1_14_1","volume-title":"Robert Osazuwa Ness, and Jonathan Larson","author":"Edge Darren","year":"2024","unstructured":"Darren Edge, Ha Trinh, Newman Cheng, Joshua Bradley, Alex Chao, Apurva Mody, Steven Truitt, Dasha Metropolitansky, Robert Osazuwa Ness, and Jonathan Larson. 2024. From local to global: A graph RAG approach to query-focused summarization. arXiv preprint arXiv:2404.16130 (2024)."},{"key":"e_1_3_2_1_15_1","unstructured":"Cl\u00e9mentine Fourrier Nathan Habib Alina Lozovskaya Konrad Szafer and Thomas Wolf. 2024. Open LLM Leaderboard v2. https:\/\/huggingface.co\/spaces\/open-llm-leaderboard\/open_llm_leaderboard."},{"key":"e_1_3_2_1_16_1","unstructured":"Aaron Grattafiori Abhimanyu Dubey Abhinav Jauhri Abhinav Pandey Abhishek Kadian Ahmad Al-Dahle Aiesha Letman Akhil Mathur Alan Schelten Alex Vaughan et al. 2024. The Llama 3 herd of models. arXiv preprint arXiv:2407.21783 (2024)."},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"crossref","unstructured":"Jiawei Gu Xuhui Jiang Zhichao Shi Hexiang Tan Xuehao Zhai Chengjin Xu Wei Li Yinghan Shen Shengjie Ma Honghao Liu et al. 2026. A survey on LLM-as-a-Judge. The Innovation (2026) 101253.","DOI":"10.1016\/j.xinn.2025.101253"},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1145\/3442381.3449992"},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2025.findings-emnlp.568"},{"key":"e_1_3_2_1_20_1","volume-title":"International Conference on Machine Learning (ICML).","author":"Guti\u00e9rrez Bernal Jim\u00e9nez","year":"2025","unstructured":"Bernal Jim\u00e9nez Guti\u00e9rrez, Yiheng Shu, Weijian Qi, Sizhe Zhou, and Yu Su. 2025. From RAG to Memory: Non-Parametric Continual Learning for Large Language Models. In International Conference on Machine Learning (ICML)."},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.1145\/3459637.3482416"},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2025.findings-ijcnlp.29"},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/P17-1147"},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"publisher","DOI":"10.3233\/SW-140134"},{"key":"e_1_3_2_1_25_1","first-page":"9459","article-title":"Retrieval-augmented generation for knowledge-intensive NLP tasks","volume":"33","author":"Lewis Patrick","year":"2020","unstructured":"Patrick Lewis, Ethan Perez, Aleksandra Piktus, Fabio Petroni, Vladimir Karpukhin, Naman Goyal, Heinrich K\u00fcttler, Mike Lewis, Wen-tau Yih, Tim Rockt\u00e4schel, et al., 2020. Retrieval-augmented generation for knowledge-intensive NLP tasks. Advances in Neural Information Processing Systems (NeurIPS), Vol. 33 (2020), 9459-9474.","journal-title":"Advances in Neural Information Processing Systems (NeurIPS)"},{"key":"e_1_3_2_1_26_1","volume-title":"International Conference on Learning Representations (ICLR).","author":"Li Xingxuan","year":"2024","unstructured":"Xingxuan Li, Ruochen Zhao, Yew Ken Chia, Bosheng Ding, Shafiq Joty, Soujanya Poria, and Lidong Bing. 2024b. Chain-of-Knowledge: Grounding Large Language Models via Dynamic Knowledge Adapting over Heterogeneous Sources. In International Conference on Learning Representations (ICLR)."},{"key":"e_1_3_2_1_27_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2024.findings-emnlp.30"},{"key":"e_1_3_2_1_28_1","volume-title":"On-Demand, Open Knowledge Benchmarking. arXiv preprint arXiv:2511.08598","author":"Li Yanhong","year":"2025","unstructured":"Yanhong Li, Tianyang Xu, Kenan Tang, Karen Livescu, David McAllester, and Jiawei Zhou. 2025. OKBench: Democratizing LLM Evaluation with Fully Automated, On-Demand, Open Knowledge Benchmarking. arXiv preprint arXiv:2511.08598 (2025)."},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v40i44.41098"},{"key":"e_1_3_2_1_30_1","volume-title":"Temporal reasoning with large language models augmented by evolving knowledge graphs. arXiv preprint arXiv:2509.15464","author":"Lin Junhong","year":"2025","unstructured":"Junhong Lin, Song Wang, Xiaojie Guo, Julian Shun, and Yada Zhu. 2025. Temporal reasoning with large language models augmented by evolving knowledge graphs. arXiv preprint arXiv:2509.15464 (2025)."},{"key":"e_1_3_2_1_31_1","unstructured":"Aixin Liu Aoxue Mei Bangcai Lin Bing Xue Bingxuan Wang Bingzheng Xu Bochao Wu Bowei Zhang Chaofan Lin Chen Dong et al. 2025. DeepSeek-V3.2: Pushing the frontier of open large language models. arXiv preprint arXiv:2512.02556 (2025)."},{"key":"e_1_3_2_1_32_1","volume-title":"Reasoning on Graphs: Faithful and Interpretable Large Language Model Reasoning. In International Conference on Learning Representations (ICLR).","author":"Luo Linhao","year":"2024","unstructured":"Linhao Luo, Yuan-Fang Li, Reza Haf, and Shirui Pan. 2024. Reasoning on Graphs: Faithful and Interpretable Large Language Model Reasoning. In International Conference on Learning Representations (ICLR)."},{"key":"e_1_3_2_1_33_1","volume-title":"International Conference on Learning Representations (ICLR).","author":"Ma Shengjie","year":"2025","unstructured":"Shengjie Ma, Chengjin Xu, Xuhui Jiang, Muzhi Li, Huaren Qu, Cehao Yang, Jiaxin Mao, and Jian Guo. 2025. Think-on-Graph 2.0: Deep and Faithful Large Language Model Reasoning with Knowledge-guided Retrieval Augmented Generation. In International Conference on Learning Representations (ICLR)."},{"key":"e_1_3_2_1_34_1","doi-asserted-by":"publisher","DOI":"10.1109\/TPAMI.2018.2889473"},{"key":"e_1_3_2_1_35_1","volume-title":"KGGen: Extracting Knowledge Graphs from Plain Text with Language Models. Advances in Neural Information Processing Systems (NeurIPS)","author":"Mo Belinda","year":"2025","unstructured":"Belinda Mo, Kyssen Yu, Joshua Kazdan, Proud Mpala, Lisa Yu, Chris Cundy, Charilaos Kanatsoulis, and Sanmi Koyejo. 2025. KGGen: Extracting Knowledge Graphs from Plain Text with Language Models. Advances in Neural Information Processing Systems (NeurIPS) (2025)."},{"key":"e_1_3_2_1_36_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2024.conda-1.3"},{"key":"e_1_3_2_1_37_1","unstructured":"Aaditya Singh Adam Fry Adam Perelman Adam Tart Adi Ganesh Ahmed El-Kishky Aidan McLaughlin Aiden Low AJ Ostrow Akhila Ananthram et al. 2025. OpenAI GPT-5 system card. arXiv preprint arXiv:2601.03267 (2025)."},{"key":"e_1_3_2_1_38_1","volume-title":"Andrew Poulton, David Esiobu, Maria Lomeli, Gergely Szilvasy, and Dieuwke Hupkes.","author":"Singh Aaditya K","year":"2024","unstructured":"Aaditya K Singh, Muhammed Yusuf Kocyigit, Andrew Poulton, David Esiobu, Maria Lomeli, Gergely Szilvasy, and Dieuwke Hupkes. 2024. Evaluation data contamination in LLMs: how do we measure it and (when) does it matter? arXiv preprint arXiv:2411.03923 (2024)."},{"key":"e_1_3_2_1_39_1","volume-title":"Think-on-Graph: Deep and Responsible Reasoning of Large Language Model on Knowledge Graph. In International Conference on Learning Representations (ICLR).","author":"Sun Jiashuo","year":"2024","unstructured":"Jiashuo Sun, Chengjin Xu, Lumingyuan Tang, Saizhuo Wang, Chen Lin, Yeyun Gong, Lionel Ni, Heung-Yeung Shum, and Jian Guo. 2024. Think-on-Graph: Deep and Responsible Reasoning of Large Language Model on Knowledge Graph. In International Conference on Learning Representations (ICLR)."},{"key":"e_1_3_2_1_40_1","first-page":"641","article-title":"The Web as a Knowledge-base for Answering Complex Questions","author":"Talmor Alon","year":"2018","unstructured":"Alon Talmor and Jonathan Berant. 2018. The Web as a Knowledge-base for Answering Complex Questions. In Proceedings of NAACL-HLT. 641-651.","journal-title":"Proceedings of NAACL-HLT."},{"key":"e_1_3_2_1_41_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-68204-4_22"},{"key":"e_1_3_2_1_42_1","doi-asserted-by":"publisher","DOI":"10.1145\/2629489"},{"key":"e_1_3_2_1_43_1","volume-title":"Self-Consistency Improves Chain of Thought Reasoning in Language Models. In International Conference on Learning Representations (ICLR).","author":"Wang Xuezhi","year":"2023","unstructured":"Xuezhi Wang, Jason Wei, Dale Schuurmans, Quoc V Le, Ed H Chi, Sharan Narang, Aakanksha Chowdhery, and Denny Zhou. 2023. Self-Consistency Improves Chain of Thought Reasoning in Language Models. In International Conference on Learning Representations (ICLR)."},{"key":"e_1_3_2_1_44_1","volume-title":"Denny Zhou, et al.","author":"Wei Jason","year":"2022","unstructured":"Jason Wei, Xuezhi Wang, Dale Schuurmans, Maarten Bosma, Fei Xia, Ed Chi, Quoc V Le, Denny Zhou, et al., 2022. Chain-of-thought prompting elicits reasoning in large language models. Advances in neural information processing systems (NeurIPS), Vol. 35 (2022), 24824-24837."},{"key":"e_1_3_2_1_45_1","unstructured":"Cheng Xu Shuhao Guan Derek Greene M Kechadi et al. 2024. Benchmark data contamination of large language models: A survey. arXiv preprint arXiv:2406.04244 (2024)."},{"key":"e_1_3_2_1_46_1","volume-title":"arXiv preprint arXiv:2412.15115","author":"Yang An","year":"2024","unstructured":"An Yang, Baosong Yang, Beichen Zhang, Binyuan Hui, Bo Zheng, Bowen Yu, Chengyuan Li, Dayiheng Liu, Fei Huang, Haoran Wei, Huan Lin, Jian Yang, Jianhong Tu, Jianwei Zhang, Jianxin Yang, Jiaxi Yang, Jingren Zhou, Junyang Lin, Kai Dang, Keming Lu, Keqin Bao, Kexin Yang, Le Yu, Mei Li, Mingfeng Xue, Pei Zhang, Qin Zhu, Rui Men, Runji Lin, Tianhao Li, Tingyu Xia, Xingzhang Ren, Xuancheng Ren, Yang Fan, Yang Su, Yichang Zhang, Yu Wan, Yuqiong Liu, Zeyu Cui, Zhenru Zhang, and Zihan Qiu. 2024b. Qwen2.5 Technical Report. arXiv preprint arXiv:2412.15115 (2024)."},{"key":"e_1_3_2_1_47_1","doi-asserted-by":"publisher","DOI":"10.52202\/079017-0335"},{"key":"e_1_3_2_1_48_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D18-1259"},{"key":"e_1_3_2_1_49_1","volume-title":"Proceedings of the 54th Annual Meeting of the Association for Computational Linguistics (Volume 2: Short Papers). 201-206","author":"Richardson Matthew","year":"2016","unstructured":"Wen-tau Yih, Matthew Richardson, Christopher Meek, Ming-Wei Chang, and Jina Suh. 2016. The value of semantic parse labeling for knowledge base question answering. In Proceedings of the 54th Annual Meeting of the Association for Computational Linguistics (Volume 2: Short Papers). 201-206."},{"key":"e_1_3_2_1_50_1","doi-asserted-by":"crossref","unstructured":"Liangliang Zhang Zhuorui Jiang Hongliang Chi Haoyang Chen Mohammed Elkoumy Fali Wang Qiong Wu Zhengyi Zhou Shirui Pan Suhang Wang et al. 2025. Diagnosing and Addressing Pitfalls in KG-RAG Datasets: Toward More Reliable Benchmarking. Advances in Neural Information Processing Systems (NeurIPS) (2025).","DOI":"10.52202\/085713-2274"},{"key":"e_1_3_2_1_51_1","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v32i1.12057"}],"event":{"name":"KDD '26: The 32nd ACM SIGKDD Conference on Knowledge Discovery and Data Mining","location":"Jeju Island Republic of Korea","sponsor":["SIGMOD ACM Special Interest Group on Management of Data","SIGKDD ACM Special Interest Group on Knowledge Discovery in Data"]},"container-title":["Proceedings of the 32nd ACM SIGKDD Conference on Knowledge Discovery and Data Mining V.2"],"original-title":[],"deposited":{"date-parts":[[2026,8,7]],"date-time":"2026-08-07T01:31:21Z","timestamp":1786066281000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3770855.3817481"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026,8,8]]},"references-count":51,"alternative-id":["10.1145\/3770855.3817481","10.1145\/3770855"],"URL":"https:\/\/doi.org\/10.1145\/3770855.3817481","relation":{},"subject":[],"published":{"date-parts":[[2026,8,8]]},"assertion":[{"value":"2026-08-08","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}