{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,11]],"date-time":"2026-07-11T02:36:27Z","timestamp":1783737387006,"version":"3.55.0"},"publisher-location":"New York, NY, USA","reference-count":23,"publisher":"ACM","funder":[{"name":"MICIU\/AEI\/10.13039\/501100011033","award":["PID2022-137061OB-C21"],"award-info":[{"award-number":["PID2022-137061OB-C21"]}]},{"name":"Conseller\u00eda de Cultura, Educaci\u00f3n, Formaci\u00f3n Profesional e Universidades","award":["ED431G 2023\/01 and ED431C 2025\/49"],"award-info":[{"award-number":["ED431G 2023\/01 and ED431C 2025\/49"]}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2025,7,13]]},"DOI":"10.1145\/3726302.3730221","type":"proceedings-article","created":{"date-parts":[[2025,7,14]],"date-time":"2025-07-14T01:38:52Z","timestamp":1752457132000},"page":"2545-2549","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":4,"title":["Limitations of Automatic Relevance Assessments with Large Language Models for Fair and Reliable Retrieval Evaluation"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-1139-0449","authenticated-orcid":false,"given":"David","family":"Otero","sequence":"first","affiliation":[{"name":"IRlab, CITIC, Universidade da Coru\u00f1a, A Coru\u00f1a, Spain"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-5997-8252","authenticated-orcid":false,"given":"Javier","family":"Parapar","sequence":"additional","affiliation":[{"name":"IRlab, CITIC, Universidade da Coru\u00f1a, A Coru\u00f1a, Spain"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-6698-2946","authenticated-orcid":false,"given":"\u00c1lvaro","family":"Barreiro","sequence":"additional","affiliation":[{"name":"IRlab, CITIC, University of A Coru\u00f1a, A Coru\u00f1a, Spain"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2025,7,13]]},"reference":[{"key":"e_1_3_2_1_1_1","volume-title":"Can we use large language models to fill relevance judgment holes?","author":"Abbasiantaeb Zahra","year":"2024","unstructured":"Zahra Abbasiantaeb, Chuan Meng, Leif Azzopardi, and Mohammad Aliannejadi. 2024. Can we use large language models to fill relevance judgment holes? (2024). https:\/\/arxiv.org\/abs\/2405.05600 arXiv: 2405.05600 [cs.IR] (cited on p. 1)."},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.1145\/3673791.3698431"},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1145\/1390334.1390445"},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.1145\/3578337.3605136"},{"key":"e_1_3_2_1_5_1","volume-title":"Rank Correlation Methods","author":"Kendall Maurice G.","unstructured":"Maurice G. Kendall. 1948. Rank Correlation Methods. Charles Griffin and Company Limited (cited on p. 1)."},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"publisher","DOI":"10.1145\/3269206.3271751"},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"publisher","DOI":"10.1145\/3539618.3592032"},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.1177\/01655515231171085"},{"key":"e_1_3_2_1_9_1","volume-title":"Advances in Information Retrieval","author":"Otero David","unstructured":"David Otero, Javier Parapar, and \u00c1lvaro Barreiro. 2025. Towards reliable testing for multiple information retrieval system comparisons. In Advances in Information Retrieval. Springer Nature Switzerland, Cham, 424-439. isbn: 978-3-031-88711-6 (cited on p. 3)."},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.1145\/3583780.3614916"},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.1145\/3412841.3441945"},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.1145\/3626772.3657942"},{"key":"e_1_3_2_1_13_1","volume-title":"Syndl: a large-scale synthetic test collection for passage retrieval. (2024). https:\/\/arxiv.org\/abs\/2408.16312 arXiv: 2408.16312 [cs.IR] (cited on","author":"Rahmani Hossein A.","unstructured":"Hossein A. Rahmani, Xi Wang, Emine Yilmaz, Nick Craswell, Bhaskar Mitra, and Paul Thomas. 2024. Syndl: a large-scale synthetic test collection for passage retrieval. (2024). https:\/\/arxiv.org\/abs\/2408.16312 arXiv: 2408.16312 [cs.IR] (cited on p. 2)."},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.1145\/1076034.1076064"},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.1145\/3626772.3657707"},{"key":"e_1_3_2_1_16_1","volume-title":"Llms can patch up missing relevance judgments in evaluation. (2024). https:\/\/arxiv.org\/abs\/2405.04727 arXiv: 2405.04727 [cs.IR] (cited on","author":"Upadhyay Shivani","unstructured":"Shivani Upadhyay, Ehsan Kamalloo, and Jimmy Lin. 2024. Llms can patch up missing relevance judgments in evaluation. (2024). https:\/\/arxiv.org\/abs\/2405.04727 arXiv: 2405.04727 [cs.IR] (cited on p. 1)."},{"key":"e_1_3_2_1_17_1","volume-title":"Hoa Trang Dang, and Jimmy Lin","author":"Upadhyay Shivani","year":"2024","unstructured":"Shivani Upadhyay, Ronak Pradeep, Nandan Thakur, Daniel Campos, Nick Craswell, Ian Soboroff, Hoa Trang Dang, and Jimmy Lin. 2024. A large-scale study of relevance assessments with large language models: an initial look. (2024). https:\/\/arxiv.org\/abs\/2411.08275 arXiv: 2411.08275 [cs.IR] (cited on pp. 1, 2)."},{"key":"e_1_3_2_1_18_1","volume-title":"Umbrela: umbrela is the (open-source reproduction of the) bing relevance assessor. arXiv:2406.06519 (cited on","author":"Upadhyay Shivani","unstructured":"Shivani Upadhyay, Ronak Pradeep, Nandan Thakur, Nick Craswell, and Jimmy Lin. 2024. Umbrela: umbrela is the (open-source reproduction of the) bing relevance assessor. arXiv:2406.06519 (cited on p. 1)."},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","DOI":"10.1145\/3240323.3240347"},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.1145\/3269206.3271766"},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.1145\/1852102.1852106"},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.1145\/1458082.1458158"},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.1145\/1390334.1390435"}],"event":{"name":"SIGIR '25: The 48th International ACM SIGIR Conference on Research and Development in Information Retrieval","location":"Padua Italy","acronym":"SIGIR '25","sponsor":["SIGIR ACM Special Interest Group on Information Retrieval"]},"container-title":["Proceedings of the 48th International ACM SIGIR Conference on Research and Development in Information Retrieval"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3726302.3730221","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,8,22]],"date-time":"2025-08-22T12:07:31Z","timestamp":1755864451000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3726302.3730221"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,7,13]]},"references-count":23,"alternative-id":["10.1145\/3726302.3730221","10.1145\/3726302"],"URL":"https:\/\/doi.org\/10.1145\/3726302.3730221","relation":{},"subject":[],"published":{"date-parts":[[2025,7,13]]},"assertion":[{"value":"2025-07-13","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}