{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,15]],"date-time":"2026-07-15T18:07:10Z","timestamp":1784138830490,"version":"3.55.0"},"publisher-location":"New York, NY, USA","reference-count":55,"publisher":"ACM","license":[{"start":{"date-parts":[[2026,7,19]],"date-time":"2026-07-19T00:00:00Z","timestamp":1784419200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/legalcode"}],"funder":[{"name":"ARC Centre of Excellence for Automated Decision-Making and Society (ADM+S)","award":["CE200100005"],"award-info":[{"award-number":["CE200100005"]}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2026,7,20]]},"DOI":"10.1145\/3805712.3808601","type":"proceedings-article","created":{"date-parts":[[2026,7,15]],"date-time":"2026-07-15T17:06:26Z","timestamp":1784135186000},"page":"3159-3166","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":0,"title":["Auto-Judge: A Cross-Task Benchmark for Comparing LLM Judges for Citation-Grounded RAG Systems"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0009-0000-3297-8888","authenticated-orcid":false,"given":"Naghmeh","family":"Farzi","sequence":"first","affiliation":[{"name":"University of New Hampshire, Durham, NH, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0000-4854-7249","authenticated-orcid":false,"given":"Tim","family":"Hagen","sequence":"additional","affiliation":[{"name":"University of Kassel, Kassel, Germany and hessian.AI, Darmstadt, Germany"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-0051-1535","authenticated-orcid":false,"given":"Eugene","family":"Yang","sequence":"additional","affiliation":[{"name":"Johns Hopkins University, Baltimore, MD, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-1003-981X","authenticated-orcid":false,"given":"Maik","family":"Fr\u00f6be","sequence":"additional","affiliation":[{"name":"Friedrich-Schiller-Universit\u00e4t Jena, Jena, Germany"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-6296-601X","authenticated-orcid":false,"given":"Ronak","family":"Pradeep","sequence":"additional","affiliation":[{"name":"University of Waterloo, Waterloo, Canada"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-2779-4942","authenticated-orcid":false,"given":"Hossein A.","family":"Rahmani","sequence":"additional","affiliation":[{"name":"University College London, London, United Kingdom"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-5936-9919","authenticated-orcid":false,"given":"Xi","family":"Wang","sequence":"additional","affiliation":[{"name":"University of Sheffield, Sheffield, United Kingdom"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-1535-0989","authenticated-orcid":false,"given":"Oleg","family":"Zendel","sequence":"additional","affiliation":[{"name":"RMIT University, Melbourne, Australia"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-2451-0665","authenticated-orcid":false,"given":"Martin","family":"Potthast","sequence":"additional","affiliation":[{"name":"University of Kassel, Kassel, Germany, hessian.AI, Darmstadt, Germany, and ScaDS.AI, Leipzig, Germany"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-1624-3907","authenticated-orcid":false,"given":"Laura","family":"Dietz","sequence":"additional","affiliation":[{"name":"University of New Hampshire, Durham, NH, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2026,7,19]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"publisher","DOI":"10.1145\/3673791.3698431"},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.48550\/ARXIV.2507.02186"},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1145\/3397271.3401036"},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-72240-1_51"},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1145\/3477495.3531738"},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-88720-8_58"},{"key":"e_1_3_2_1_7_1","first-page":"11","article-title":"Epistemolog\u00eda y ciencia de la informaci\u00f3n","volume":"4","author":"Capurro Rafael","year":"2007","unstructured":"Rafael Capurro. 2007. Epistemolog\u00eda y ciencia de la informaci\u00f3n. Enlace, Vol. 4, 1 (2007), 11-29. http:\/\/ve.scielo.org\/scielo.php?script=sci_arttext&pid=S1690-75152007000100002&lng=en&tlng=es","journal-title":"Enlace"},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2024.naacl-long.176"},{"key":"e_1_3_2_1_9_1","volume-title":"Proceedings of the 18th NTCIR Conference on Evaluation of Information Access Technologies","author":"Chen Junjie","year":"2025","unstructured":"Junjie Chen, Haitao Li, Zhumin Chu, Yiqun Liu, and Qingyao Ai. 2025. Overview of the NTCIR-18 Automatic Evaluation of LLMs (AEOLLM) Task. In Proceedings of the 18th NTCIR Conference on Evaluation of Information Access Technologies. Tokyo, Japan."},{"key":"e_1_3_2_1_10_1","volume-title":"Automated Rubrics for Reliable Evaluation of Medical Dialogue Systems. arXiv preprint arXiv:2601.15161","author":"Chen Yinzhu","year":"2026","unstructured":"Yinzhu Chen, Abdine Maiga, Hossein A Rahmani, and Emine Yilmaz. 2026. Automated Rubrics for Reliable Evaluation of Medical Dialogue Systems. arXiv preprint arXiv:2601.15161 (2026)."},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2023.acl-long.870"},{"key":"e_1_3_2_1_12_1","volume-title":"Proceedings of the 41st International Conference on Machine Learning (Proceedings of Machine Learning Research","volume":"8388","author":"Chiang Wei-Lin","year":"2024","unstructured":"Wei-Lin Chiang, Lianmin Zheng, Ying Sheng, Anastasios Nikolas Angelopoulos, Tianle Li, Dacheng Li, Banghua Zhu, Hao Zhang, Michael I. Jordan, Joseph E. Gonzalez, and Ion Stoica. 2024. Chatbot Arena: An Open Platform for Evaluating LLMs by Human Preference. In Proceedings of the 41st International Conference on Machine Learning (Proceedings of Machine Learning Research, Vol. 235). PMLR, 8359-8388."},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.20736\/0002002105"},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.1108\/eb049778"},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.1108\/eb050097"},{"key":"e_1_3_2_1_16_1","volume-title":"Technical Report PB169574. Association","author":"Cleverdon Cyril W.","year":"1966","unstructured":"Cyril W. Cleverdon, J. Mills, and M. Keen. 1966. Factors Determining the Performance of Indexing Systems. Volume I. Design. Part 2. Appendices. Technical Report PB169574. Association of Special Libraries and Information Bureau, Cranfield (England). https:\/\/ntrl.ntis.gov\/NTRL\/dashboard\/searchResults\/titleDetail\/PB169574.xhtml Num Pages: 261."},{"key":"e_1_3_2_1_17_1","volume-title":"Overview of the TREC 2023 Deep Learning Track. In Text REtrieval Conference (TREC). NIST, TREC. https:\/\/www.microsoft.com\/en-us\/research\/publication\/overview-of-the-trec-2023-deep-learning-track\/","author":"Craswell Nick","year":"2024","unstructured":"Nick Craswell, Bhaskar Mitra, Emine Yilmaz, Hossein A. Rahmani, Daniel Campos, Jimmy Lin, Ellen M. Voorhees, and Ian Soboroff. 2024. Overview of the TREC 2023 Deep Learning Track. In Text REtrieval Conference (TREC). NIST, TREC. https:\/\/www.microsoft.com\/en-us\/research\/publication\/overview-of-the-trec-2023-deep-learning-track\/"},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-032-21289-4_37"},{"key":"e_1_3_2_1_19_1","unstructured":"Laura Dietz Ian Soboroff and Coordinators of various TREC tracks. 2025a. TREC AutoJudge Pilot Data. trec-auto-judge.cs.unh.edu Licensed under CC BY-SA 3.0. Based on content from TREC DRAGUN TREC RAG TREC RAGTIME their license applies."},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.1145\/3731120.3744588"},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.1145\/3578337.3605136"},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.1145\/3664190.3672511"},{"key":"e_1_3_2_1_23_1","volume-title":"The Information Retrieval Experiment Platform. In 46th International ACM SIGIR Conference on Research and Development in Information Retrieval (SIGIR","author":"Fr\u00f6be Maik","year":"2023","unstructured":"Maik Fr\u00f6be, Jan Heinrich Reimer, Sean MacAvaney, Niklas Deckers, Simon Reich, Janek Bevendorff, Benno Stein, Matthias Hagen, and Martin Potthast. 2023. The Information Retrieval Experiment Platform. In 46th International ACM SIGIR Conference on Research and Development in Information Retrieval (SIGIR 2023)."},{"key":"e_1_3_2_1_24_1","unstructured":"Tianyu Gao Howard Yen Jiatong Yu and Danqi Chen. 2023. Enabling Large Language Models to Generate Text with Citations. In Empirical Methods in Natural Language Processing (EMNLP)."},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2025.acl-long.34"},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.1145\/3726302.3730297"},{"key":"e_1_3_2_1_27_1","volume-title":"Overview of the TREC 2025 RAGTIME Track. In TREC.","author":"Lawrie Dawn","year":"2026","unstructured":"Dawn Lawrie, Sean MacAvaney, James Mayfield, Luca Soldaini, Eugene Yang, and Andrew Yates. 2026. Overview of the TREC 2025 RAGTIME Track. In TREC."},{"key":"e_1_3_2_1_28_1","volume-title":"How to correctly report llm-as-a-judge evaluations. arXiv preprint arXiv:2511.21140","author":"Lee Chungpa","year":"2025","unstructured":"Chungpa Lee, Thomas Zeng, Jongwon Jeong, Jy-yong Sohn, and Kangwook Lee. 2025. How to correctly report llm-as-a-judge evaluations. arXiv preprint arXiv:2511.21140 (2025)."},{"key":"e_1_3_2_1_29_1","volume-title":"Judging Against the Reference: Uncovering Knowledge-Driven Failures in LLM-Judges on QA Evaluation. arXiv preprint arXiv:2601.07506","author":"Lee Dongryeol","year":"2026","unstructured":"Dongryeol Lee, Yerin Hwang, Taegwan Kang, Minwoo Lee, Younhyung Chae, and Kyomin Jung. 2026. Judging Against the Reference: Uncovering Knowledge-Driven Failures in LLM-Judges on QA Evaluation. arXiv preprint arXiv:2601.07506 (2026)."},{"key":"e_1_3_2_1_30_1","volume-title":"Who Judges the Judge? LLM Jury-on-Demand: Building Trustworthy LLM Evaluation Systems. arXiv preprint arXiv:2512.01786","author":"Li Xiaochuan","year":"2025","unstructured":"Xiaochuan Li, Ke Wang, Girija Gouda, Shubham Choudhary, Yaqun Wang, Linwei Hu, Joel Vaughan, and Freddy Lecue. 2025. Who Judges the Judge? LLM Jury-on-Demand: Building Trustworthy LLM Evaluation Systems. arXiv preprint arXiv:2512.01786 (2025)."},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2023.emnlp-main.153"},{"key":"e_1_3_2_1_32_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2024.findings-acl.753"},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"publisher","DOI":"10.1145\/3404835.3463254"},{"key":"e_1_3_2_1_34_1","doi-asserted-by":"publisher","DOI":"10.52202\/079017-2197"},{"key":"e_1_3_2_1_35_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2021.naacl-main.200"},{"key":"e_1_3_2_1_36_1","doi-asserted-by":"publisher","DOI":"10.1145\/3726302.3730090"},{"key":"e_1_3_2_1_37_1","doi-asserted-by":"publisher","DOI":"10.1145\/3626772.3657942"},{"key":"e_1_3_2_1_38_1","doi-asserted-by":"publisher","DOI":"10.1145\/3746252.3760908"},{"key":"e_1_3_2_1_39_1","volume-title":"Proceedings of The First Workshop on Large Language Models for Evaluation in Information Retrieval (LLM4Eval 2024) co-located with SIGIR 2024 (CEUR Workshop Proceedings","volume":"3","author":"Rahmani Hossein A.","year":"2024","unstructured":"Hossein A. Rahmani, Emine Yilmaz, Nick Craswell, Bhaskar Mitra, Paul Thomas, Charles L. A. Clarke, Mohammad Aliannejadi, Clemencia Siro, and Guglielmo Faggioli. 2024b. LLMJudge: LLMs for Relevance Judgments. In Proceedings of The First Workshop on Large Language Models for Evaluation in Information Retrieval (LLM4Eval 2024) co-located with SIGIR 2024 (CEUR Workshop Proceedings, Vol. 3752). CEUR-WS.org, 1-3. https:\/\/ceur-ws.org\/Vol-3752\/paper8.pdf"},{"key":"e_1_3_2_1_40_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2024.emnlp-main.427"},{"key":"e_1_3_2_1_41_1","volume-title":"Are LLM Evaluators Really Narcissists? Sanity Checking Self-Preference Evaluations. arXiv preprint arXiv:2601.22548","author":"Roytburg Dani","year":"2026","unstructured":"Dani Roytburg, Matthew Bozoukov, Matthew Nguyen, Mackenzie Puig-Hall, and Narmeen Oozeer. 2026. Are LLM Evaluators Really Narcissists? Sanity Checking Self-Preference Evaluations. arXiv preprint arXiv:2601.22548 (2026)."},{"key":"e_1_3_2_1_42_1","volume-title":"Rexhina Blloshmi, Christopher Davis, and Adri\u00e0 de Gispert.","author":"Sorodoc Ionut Teodor","year":"2025","unstructured":"Ionut Teodor Sorodoc, Leonardo FR Ribeiro, Rexhina Blloshmi, Christopher Davis, and Adri\u00e0 de Gispert. 2025. Garage: A benchmark with grounding annotations for rag evaluation. In Findings of the Association for Computational Linguistics: ACL 2025. 17030-17049."},{"key":"e_1_3_2_1_43_1","volume-title":"Proceedings of the Fourth Workshop on Generation, Evaluation and Metrics (GEM$^2$). 404-430","author":"Thakur Aman Singh","year":"2025","unstructured":"Aman Singh Thakur, Kartik Choudhary, Venkat Srinik Ramayapally, Sankaran Vaidyanathan, and Dieuwke Hupkes. 2025. Judging the judges: Evaluating alignment and vulnerabilities in llms-as-judges. In Proceedings of the Fourth Workshop on Generation, Evaluation and Metrics (GEM$^2$). 404-430."},{"key":"e_1_3_2_1_44_1","doi-asserted-by":"publisher","DOI":"10.1145\/3626772.3657707"},{"key":"e_1_3_2_1_45_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/N18-1074"},{"key":"e_1_3_2_1_46_1","volume-title":"Proceedings of the Thirty-Fourth Text REtrieval Conference (TREC","author":"Upadhyay Shivani","year":"2026","unstructured":"Shivani Upadhyay, Ronak Pradeep, Nandan Thakur, Daniel Campos, Nick Craswell, and Jimmy Lin. 2026. Overview of the TREC 2025 RAG Track. In Proceedings of the Thirty-Fourth Text REtrieval Conference (TREC 2025). National Institute of Standards and Technology (NIST)."},{"key":"e_1_3_2_1_47_1","volume-title":"UMBRELA: UMbrela is the (Open-Source Reproduction of the) Bing RELevance Assessor. arXiv preprint arXiv:2406.06519","author":"Upadhyay Shivani","year":"2024","unstructured":"Shivani Upadhyay, Ronak Pradeep, Nandan Thakur, Nick Craswell, and Jimmy Lin. 2024. UMBRELA: UMbrela is the (Open-Source Reproduction of the) Bing RELevance Assessor. arXiv preprint arXiv:2406.06519 (2024)."},{"key":"e_1_3_2_1_48_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.emnlp-main.609"},{"key":"e_1_3_2_1_49_1","doi-asserted-by":"publisher","DOI":"10.1145\/3805712.3808384"},{"key":"e_1_3_2_1_50_1","volume-title":"The Thirteenth International Conference on Learning Representations (ICLR","author":"Wang Xinyi","year":"2025","unstructured":"Xinyi Wang, Antonis Antoniades, Yanai Elazar, Alfonso Amayuelas, Alon Albalak, Kexun Zhang, and William Yang Wang. 2025. Generalization v.s. Memorization: Tracing Language Models' Capabilities Back to Pretraining Data. In The Thirteenth International Conference on Learning Representations (ICLR 2025)."},{"key":"e_1_3_2_1_51_1","volume-title":"Self-preference bias in llm-as-a-judge. arXiv preprint arXiv:2410.21819","author":"Wataoka Koki","year":"2024","unstructured":"Koki Wataoka, Tsubasa Takahashi, and Ryokan Ri. 2024. Self-preference bias in llm-as-a-judge. arXiv preprint arXiv:2410.21819 (2024)."},{"key":"e_1_3_2_1_52_1","doi-asserted-by":"publisher","DOI":"10.1145\/1390334.1390437"},{"key":"e_1_3_2_1_53_1","doi-asserted-by":"publisher","DOI":"10.1145\/3746252.3761382"},{"key":"e_1_3_2_1_54_1","volume-title":"Overview of the TREC 2025 DRAGUN Track: Detection, Retrieval, and Augmented Generation for Understanding News. In The Thirty-Fourth Text REtrieval Conference Proceedings (TREC 2025)","author":"Zhang Dake","unstructured":"Dake Zhang, Mark D. Smucker, and Charles L. A. Clarke. 2025. Overview of the TREC 2025 DRAGUN Track: Detection, Retrieval, and Augmented Generation for Understanding News. In The Thirty-Fourth Text REtrieval Conference Proceedings (TREC 2025) (NIST Special Publication, ). National Institute of Standards and Technology (NIST)."},{"key":"e_1_3_2_1_55_1","doi-asserted-by":"crossref","unstructured":"Lianmin Zheng Wei-Lin Chiang Ying Sheng Siyuan Zhuang Zhanghao Wu Yonghao Zhuang Zi Lin Zhuohan Li Dacheng Li Eric Xing et al. 2023. Judging llm-as-a-judge with mt-bench and chatbot arena. Advances in neural information processing systems Vol. 36 (2023) 46595-46623.","DOI":"10.52202\/075280-2020"}],"event":{"name":"SIGIR '26: The 49th International ACM SIGIR Conference on Research and Development in Information Retrieval","location":"Melbourne VIC Australia","sponsor":["SIGIR ACM Special Interest Group on Information Retrieval"]},"container-title":["Proceedings of the 49th International ACM SIGIR Conference on Research and Development in Information Retrieval"],"original-title":[],"deposited":{"date-parts":[[2026,7,15]],"date-time":"2026-07-15T17:18:53Z","timestamp":1784135933000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3805712.3808601"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026,7,19]]},"references-count":55,"alternative-id":["10.1145\/3805712.3808601","10.1145\/3805712"],"URL":"https:\/\/doi.org\/10.1145\/3805712.3808601","relation":{},"subject":[],"published":{"date-parts":[[2026,7,19]]},"assertion":[{"value":"2026-07-19","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}