{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,3,25]],"date-time":"2026-03-25T00:41:49Z","timestamp":1774399309660,"version":"3.50.1"},"publisher-location":"New York, NY, USA","reference-count":15,"publisher":"ACM","license":[{"start":{"date-parts":[[2025,5,8]],"date-time":"2025-05-08T00:00:00Z","timestamp":1746662400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"funder":[{"name":"Engineering and Physical Sciences Research Council","award":["EP\/S021566\/1"],"award-info":[{"award-number":["EP\/S021566\/1"]}]},{"name":"EPSRC Fellowship titled Task Based Information Retrieval","award":["EP\/P024289\/1"],"award-info":[{"award-number":["EP\/P024289\/1"]}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2025,5,8]]},"DOI":"10.1145\/3701716.3715536","type":"proceedings-article","created":{"date-parts":[[2025,5,23]],"date-time":"2025-05-23T16:06:11Z","timestamp":1748016371000},"page":"1268-1272","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":3,"title":["JudgeBlender: Ensembling Automatic Relevance Judgments"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-2779-4942","authenticated-orcid":false,"given":"Hossein A.","family":"Rahmani","sequence":"first","affiliation":[{"name":"University College London, London, United Kingdom"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-4734-4532","authenticated-orcid":false,"given":"Emine","family":"Yilmaz","sequence":"additional","affiliation":[{"name":"University College London, London, United Kingdom"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-9351-8137","authenticated-orcid":false,"given":"Nick","family":"Craswell","sequence":"additional","affiliation":[{"name":"Microsoft, Seattle, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-5270-5550","authenticated-orcid":false,"given":"Bhaskar","family":"Mitra","sequence":"additional","affiliation":[{"name":"Microsoft, Montr\u00e9al, Canada"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2025,5,23]]},"reference":[{"key":"e_1_3_2_1_1_1","volume-title":"Overview of the TREC 2023 Deep Learning Track. In Text REtrieval Conference (TREC). NIST, TREC.","author":"Craswell Nick","year":"2024","unstructured":"Nick Craswell, Bhaskar Mitra, Emine Yilmaz, Hossein A. Rahmani, Daniel Campos, Jimmy Lin, Ellen M. Voorhees, and Ian Soboroff. 2024. Overview of the TREC 2023 Deep Learning Track. In Text REtrieval Conference (TREC). NIST, TREC."},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.5555\/648054.743935"},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1145\/3578337.3605136"},{"key":"e_1_3_2_1_4_1","volume-title":"Best in Tau@ LLMJudge: Criteria-Based Relevance Evaluation with Llama3. arXiv preprint arXiv:2410.14044","author":"Farzi Naghmeh","year":"2024","unstructured":"Naghmeh Farzi and Laura Dietz. 2024a. Best in Tau@ LLMJudge: Criteria-Based Relevance Evaluation with Llama3. arXiv preprint arXiv:2410.14044 (2024)."},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1145\/3664190.3672511"},{"key":"e_1_3_2_1_6_1","volume-title":"Proceedings of the 61th Annual Meeting of the Association for Computational Linguistics (ACL","author":"Jiang Dongfu","year":"2023","unstructured":"Dongfu Jiang, Xiang Ren, and Bill Yuchen Lin. 2023. LLM-Blender: Ensembling Large Language Models with Pairwise Comparison and Generative Fusion. In Proceedings of the 61th Annual Meeting of the Association for Computational Linguistics (ACL 2023)."},{"key":"e_1_3_2_1_7_1","volume-title":"Query performance prediction using relevance judgments generated by large language models. arXiv preprint arXiv:2404.01012","author":"Meng Chuan","year":"2024","unstructured":"Chuan Meng, Negar Arabzadeh, Arian Askari, Mohammad Aliannejadi, and Maarten de Rijke. 2024. Query performance prediction using relevance judgments generated by large language models. arXiv preprint arXiv:2404.01012 (2024)."},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.1145\/3626772.3657942"},{"key":"e_1_3_2_1_9_1","volume-title":"SynDL: A Large-Scale Synthetic Test Collection for Passage Retrieval. arXiv preprint arXiv:2408.16312","author":"Rahmani Hossein A","year":"2024","unstructured":"Hossein A Rahmani, Xi Wang, Emine Yilmaz, Nick Craswell, Bhaskar Mitra, and Paul Thomas. 2024b. SynDL: A Large-Scale Synthetic Test Collection for Passage Retrieval. arXiv preprint arXiv:2408.16312 (2024)."},{"key":"e_1_3_2_1_10_1","volume-title":"Mohammad Aliannejadi, Clemencia Siro, and Guglielmo Faggioli.","author":"Rahmani Hossein A","year":"2024","unstructured":"Hossein A Rahmani, Emine Yilmaz, Nick Craswell, Bhaskar Mitra, Paul Thomas, Charles LA Clarke, Mohammad Aliannejadi, Clemencia Siro, and Guglielmo Faggioli. 2024c. LLMJudge: LLMs for Relevance Judgments. arXiv preprint arXiv:2408.08896 (2024)."},{"key":"e_1_3_2_1_11_1","volume-title":"Ensemble learning: A survey","author":"Sagi Omer","year":"2018","unstructured":"Omer Sagi and Lior Rokach. 2018. Ensemble learning: A survey. Wiley interdisciplinary reviews: data mining and knowledge discovery, Vol. 8, 4 (2018), e1249."},{"key":"e_1_3_2_1_12_1","volume-title":"Is ChatGPT good at search? investigating large language models as re-ranking agents. arXiv preprint arXiv:2304.09542","author":"Sun Weiwei","year":"2023","unstructured":"Weiwei Sun, Lingyong Yan, Xinyu Ma, Shuaiqiang Wang, Pengjie Ren, Zhumin Chen, Dawei Yin, and Zhaochun Ren. 2023. Is ChatGPT good at search? investigating large language models as re-ranking agents. arXiv preprint arXiv:2304.09542 (2023)."},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1145\/3626772.3657707"},{"key":"e_1_3_2_1_14_1","volume-title":"UMBRELA: UMbrela is the (Open-Source Reproduction of the) Bing RELevance Assessor. arXiv preprint arXiv:2406.06519","author":"Upadhyay Shivani","year":"2024","unstructured":"Shivani Upadhyay, Ronak Pradeep, Nandan Thakur, Nick Craswell, and Jimmy Lin. 2024. UMBRELA: UMbrela is the (Open-Source Reproduction of the) Bing RELevance Assessor. arXiv preprint arXiv:2406.06519 (2024)."},{"key":"e_1_3_2_1_15_1","volume-title":"Replacing Judges with Juries: Evaluating LLM Generations with a Panel of Diverse Models. arXiv preprint arXiv:2404.18796","author":"Verga Pat","year":"2024","unstructured":"Pat Verga, Sebastian Hofstatter, Sophia Althammer, Yixuan Su, Aleksandra Piktus, Arkady Arkhangorodsky, Minjie Xu, Naomi White, and Patrick Lewis. 2024. Replacing Judges with Juries: Evaluating LLM Generations with a Panel of Diverse Models. arXiv preprint arXiv:2404.18796 (2024)."}],"event":{"name":"WWW '25: The ACM Web Conference 2025","location":"Sydney NSW Australia","acronym":"WWW '25","sponsor":["SIGWEB ACM Special Interest Group on Hypertext, Hypermedia, and Web"]},"container-title":["Companion Proceedings of the ACM on Web Conference 2025"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3701716.3715536","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3701716.3715536","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,10,7]],"date-time":"2025-10-07T17:38:51Z","timestamp":1759858731000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3701716.3715536"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,5,8]]},"references-count":15,"alternative-id":["10.1145\/3701716.3715536","10.1145\/3701716"],"URL":"https:\/\/doi.org\/10.1145\/3701716.3715536","relation":{},"subject":[],"published":{"date-parts":[[2025,5,8]]},"assertion":[{"value":"2025-05-23","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}