{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,15]],"date-time":"2026-07-15T18:06:11Z","timestamp":1784138771558,"version":"3.55.0"},"publisher-location":"New York, NY, USA","reference-count":37,"publisher":"ACM","license":[{"start":{"date-parts":[[2026,7,19]],"date-time":"2026-07-19T00:00:00Z","timestamp":1784419200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/legalcode"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2026,7,20]]},"DOI":"10.1145\/3805712.3808585","type":"proceedings-article","created":{"date-parts":[[2026,7,15]],"date-time":"2026-07-15T17:06:26Z","timestamp":1784135186000},"page":"3109-3116","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":0,"title":["<i>LiveRAG:<\/i>\n                    A Diverse Q&amp;A Dataset with Varying Difficulty Level for RAG Evaluation"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-1161-7084","authenticated-orcid":false,"given":"David","family":"Carmel","sequence":"first","affiliation":[{"name":"Technology Innovation Institute (TII), Haifa, Israel"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0002-6735-9950","authenticated-orcid":false,"given":"Simone","family":"Filice","sequence":"additional","affiliation":[{"name":"Technology Innovation Institute (TII), Haifa, Israel"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0001-5093-7235","authenticated-orcid":false,"given":"Guy","family":"Horowitz","sequence":"additional","affiliation":[{"name":"Technology Innovation Institute (TII), Haifa, Israel"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-3160-7115","authenticated-orcid":false,"given":"Yoelle","family":"Maarek","sequence":"additional","affiliation":[{"name":"Technology Innovation Institute (TII), Haifa, Israel"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0000-1147-3872","authenticated-orcid":false,"given":"Alexander","family":"Shtoff","sequence":"additional","affiliation":[{"name":"Technology Innovation Institute (TII), Haifa, Israel"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0003-8454-5568","authenticated-orcid":false,"given":"Oren","family":"Somekh","sequence":"additional","affiliation":[{"name":"Technology Innovation Institute (TII), Haifa, Israel"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0000-5279-9596","authenticated-orcid":false,"given":"Ran","family":"Tavory","sequence":"additional","affiliation":[{"name":"Technology Innovation Institute (TII), Haifa, Israel"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2026,7,19]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/P19-1620"},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D13-1160"},{"key":"e_1_3_2_1_3_1","unstructured":"David Carmel Simone Filice Guy Horowitz Yoelle Maarek Alex Shtoff Oren Somekh and Ran Tavory. 2025a. LiveRAG: A diverse Q&A dataset with varying difficulty level for RAG evaluation. arXiv:2511.14531 [cs.CL] https:\/\/arxiv.org\/abs\/2511.14531"},{"key":"e_1_3_2_1_4_1","volume-title":"SIGIR 2025 - LiveRAG Challenge Report. arXiv:2507","author":"Carmel David","year":"2025","unstructured":"David Carmel, Simone Filice, Guy Horowitz, Yoelle Maarek, Oren Somekh, Ran Tavory, Mehdi Ghissassi, Edo Liberty, and Roy Miara. 2025b. SIGIR 2025 - LiveRAG Challenge Report. arXiv:2507.04942 [cs.CL] https:\/\/arxiv.org\/abs\/2507.04942"},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1145\/1148170.1148238"},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2024.eacl-demo.16"},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/P19-1346"},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2025.acl-industry.33"},{"key":"e_1_3_2_1_9_1","volume-title":"Ragbench: Explainable benchmark for retrieval-augmented generation systems. arXiv preprint arXiv:2407.11005","author":"Friel Robert","year":"2024","unstructured":"Robert Friel, Masha Belyi, and Atindriyo Sanyal. 2024. Ragbench: Explainable benchmark for retrieval-augmented generation systems. arXiv preprint arXiv:2407.11005 (2024)."},{"key":"e_1_3_2_1_10_1","unstructured":"Jiawei Gu Xuhui Jiang Zhichao Shi Hexiang Tan Xuehao Zhai Chengjin Xu Wei Li Yinghan Shen Shengjie Ma Honghao Liu et al. 2024. A survey on llm-as-a-judge. arXiv preprint arXiv:2411.15594 (2024)."},{"key":"e_1_3_2_1_11_1","unstructured":"Zishan Guo Renren Jin Chuang Liu Yufei Huang Dan Shi Supryadi Linhao Yu Yan Liu Jiaxuan Li Bojian Xiong and Deyi Xiong. 2023. Evaluating Large Language Models: A Comprehensive Survey. arXiv:2310.19736 [cs.CL]"},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-13021-7_1"},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.coling-main.580"},{"key":"e_1_3_2_1_14_1","first-page":"1","article-title":"Atlas: Few-shot learning with retrieval augmented language models","volume":"24","author":"Izacard Gautier","year":"2023","unstructured":"Gautier Izacard, Patrick Lewis, Maria Lomeli, Lucas Hosseini, Fabio Petroni, Timo Schick, Jane Dwivedi-Yu, Armand Joulin, Sebastian Riedel, and Edouard Grave. 2023. Atlas: Few-shot learning with retrieval augmented language models. Journal of Machine Learning Research, Vol. 24, 251 (2023), 1-43.","journal-title":"Journal of Machine Learning Research"},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/P17-1147"},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.1162\/tacl_a_00276"},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.1287\/ijoc.2022.1250"},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D18-1500"},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/P19-1484"},{"key":"e_1_3_2_1_20_1","first-page":"9459","article-title":"Retrieval-augmented generation for knowledge-intensive NLP tasks","volume":"33","author":"Lewis Patrick","year":"2020","unstructured":"Patrick Lewis, Ethan Perez, Aleksandra Piktus, Fabio Petroni, Vladimir Karpukhin, Naman Goyal, Heinrich K\u00fcttler, Mike Lewis, Wen-tau Yih, Tim Rockt\u00e4schel, et al., 2020. Retrieval-augmented generation for knowledge-intensive NLP tasks. Advances in Neural Information Processing Systems, Vol. 33 (2020), 9459-9474.","journal-title":"Advances in Neural Information Processing Systems"},{"key":"e_1_3_2_1_21_1","volume-title":"Challenges in Generalization in Open Domain Question Answering. In Findings of the Association for Computational Linguistics: NAACL 2022. 2014","author":"Liu Linqing","year":"2022","unstructured":"Linqing Liu, Patrick Lewis, Sebastian Riedel, and Pontus Stenetorp. 2022. Challenges in Generalization in Open Domain Question Answering. In Findings of the Association for Computational Linguistics: NAACL 2022. 2014-2029."},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.4324\/9780203056615"},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.1145\/3615354"},{"key":"e_1_3_2_1_24_1","first-page":"9802","article-title":"When Not to Trust Language Models","author":"Mallen Alex","year":"2023","unstructured":"Alex Mallen, Akari Asai, Victor Zhong, Rajarshi Das, Daniel Khashabi, and Hannaneh Hajishirzi. 2023. When Not to Trust Language Models: Investigating Effectiveness of Parametric and Non-Parametric Memories. In Proceeding of ACL. 9802-9822.","journal-title":"Investigating Effectiveness of Parametric and Non-Parametric Memories. In Proceeding of ACL."},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2021.naacl-main.200"},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2021.acl-long.346"},{"key":"e_1_3_2_1_27_1","unstructured":"Chantal Shaib Joe Barrow Jiuding Sun Alexa F. Siu Byron C. Wallace and Ani Nenkova. 2025. Standardizing the Measurement of Text Diversity: A Tool and a Comparative Analysis of Scores. arXiv:2403.00553 [cs.CL] https:\/\/arxiv.org\/abs\/2403.00553"},{"key":"e_1_3_2_1_28_1","volume-title":"An instance level analysis of data complexity. Machine learning","author":"Smith Michael R","year":"2014","unstructured":"Michael R Smith, Tony Martinez, and Christophe Giraud-Carrier. 2014. An instance level analysis of data complexity. Machine learning, Vol. 95 (2014), 225-256."},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.1007\/s11263-022-01611-x"},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2022.emnlp-main.566"},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2022.acl-long.479"},{"key":"e_1_3_2_1_32_1","volume-title":"Support Evaluation for the TREC 2024 RAG Track: Comparing Human versus LLM Judges. arXiv preprint arXiv:2504","author":"Thakur Nandan","year":"2025","unstructured":"Nandan Thakur, Ronak Pradeep, Shivani Upadhyay, Daniel Campos, Nick Craswell, and Jimmy Lin. 2025. Support Evaluation for the TREC 2024 RAG Track: Comparing Human versus LLM Judges. arXiv preprint arXiv:2504.15205 (2025)."},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2021.acl-long.92"},{"key":"e_1_3_2_1_34_1","volume-title":"Yunxin Joy Jiao, Spencer Papay, Amelia Glaese, John Schulman, and William Fedus.","author":"Wei Jason","year":"2024","unstructured":"Jason Wei, Nguyen Karina, Hyung Won Chung, Yunxin Joy Jiao, Spencer Papay, Amelia Glaese, John Schulman, and William Fedus. 2024. Measuring short-form factuality in large language models. arXiv:2411.04368 [cs.CL] https:\/\/arxiv.org\/abs\/2411.04368"},{"key":"e_1_3_2_1_35_1","doi-asserted-by":"publisher","DOI":"10.52202\/079017-0335"},{"key":"e_1_3_2_1_36_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D18-1259"},{"key":"e_1_3_2_1_37_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D18-125910.18653\/v1\/D18-1259"}],"event":{"name":"SIGIR '26: The 49th International ACM SIGIR Conference on Research and Development in Information Retrieval","location":"Melbourne VIC Australia","sponsor":["SIGIR ACM Special Interest Group on Information Retrieval"]},"container-title":["Proceedings of the 49th International ACM SIGIR Conference on Research and Development in Information Retrieval"],"original-title":[],"deposited":{"date-parts":[[2026,7,15]],"date-time":"2026-07-15T17:22:08Z","timestamp":1784136128000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3805712.3808585"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026,7,19]]},"references-count":37,"alternative-id":["10.1145\/3805712.3808585","10.1145\/3805712"],"URL":"https:\/\/doi.org\/10.1145\/3805712.3808585","relation":{},"subject":[],"published":{"date-parts":[[2026,7,19]]},"assertion":[{"value":"2026-07-19","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}