{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,15]],"date-time":"2026-07-15T18:05:14Z","timestamp":1784138714331,"version":"3.55.0"},"publisher-location":"New York, NY, USA","reference-count":31,"publisher":"ACM","license":[{"start":{"date-parts":[[2026,7,19]],"date-time":"2026-07-19T00:00:00Z","timestamp":1784419200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by-nc-nd\/4.0\/legalcode"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2026,7,20]]},"DOI":"10.1145\/3805712.3809905","type":"proceedings-article","created":{"date-parts":[[2026,7,15]],"date-time":"2026-07-15T17:06:26Z","timestamp":1784135186000},"page":"4322-4327","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":0,"title":["When LLM Judges Inflate Scores: Exploring Overrating in Relevance Assessment"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0009-0008-5150-3843","authenticated-orcid":false,"given":"Chuting","family":"Yu","sequence":"first","affiliation":[{"name":"The University of Queensland, Brisbane, QLD, Australia"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-5317-7227","authenticated-orcid":false,"given":"Hang","family":"Li","sequence":"additional","affiliation":[{"name":"The University of Queensland, Brisbane, QLD, Australia"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-0271-5563","authenticated-orcid":false,"given":"Guido","family":"Zuccon","sequence":"additional","affiliation":[{"name":"The University of Queensland, Brisbane, QLD, Australia"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-7992-4633","authenticated-orcid":false,"given":"Joel","family":"Mackenzie","sequence":"additional","affiliation":[{"name":"The University of Queensland, Brisbane, QLD, Australia"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-8117-0612","authenticated-orcid":false,"given":"Teerapong","family":"Leelanupab","sequence":"additional","affiliation":[{"name":"The University of Queensland, Brisbane, QLD, Australia"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2026,7,19]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"publisher","DOI":"10.1145\/3673791.3698431"},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.1145\/3726302.3730305"},{"key":"e_1_3_2_1_3_1","volume-title":"MS MARCO: A Human Generated MAchine Reading COmprehension Dataset. arXiv:1611.09268v3","author":"Bajaj Payal","year":"2018","unstructured":"Payal Bajaj, Daniel Campos, Nick Craswell, Li Deng, Jianfeng Gao, Xiaodong Liu, Rangan Majumder, Andrew McNamara, Bhaskar Mitra, Tri Nguyen, Mir Rosenberg, Xia Song, Alina Stoica, Saurabh Tiwary, and Tong Wang. 2018. MS MARCO: A Human Generated MAchine Reading COmprehension Dataset. arXiv:1611.09268v3 (2018)."},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.1145\/3726302.3730348"},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.48550\/arXiv.2412.17156"},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"publisher","DOI":"10.1108\/eb050097"},{"key":"e_1_3_2_1_7_1","volume-title":"Proceedings of the 28th Text REtrieval Conference, TREC.","author":"Craswell Nick","year":"2020","unstructured":"Nick Craswell, Bhaskar Mitra, Emine Yilmaz, Daniel Campos, and Ellen M Voorhees. 2020. Overview of the TREC 2019 Deep Learning Track. In Proceedings of the 28th Text REtrieval Conference, TREC."},{"key":"e_1_3_2_1_8_1","volume-title":"Proceedings of the 29th Text REtrieval Conference, TREC.","author":"Craswell Nick","year":"2021","unstructured":"Nick Craswell, Bhaskar Mitra, Emine Yilmaz, Daniel Campos, and Ellen M Voorhees. 2021. Overview of the TREC 2020 Deep Learning Track. In Proceedings of the 29th Text REtrieval Conference, TREC."},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1145\/3637528.3671882"},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.1145\/3077136.3080729"},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.1145\/3578337.3605136"},{"key":"e_1_3_2_1_12_1","volume-title":"Fowler's dictionary of modern English usage","author":"Fowler Henry Watson","unstructured":"Henry Watson Fowler. 2015. Fowler's dictionary of modern English usage. Oxford University Press."},{"key":"e_1_3_2_1_13_1","unstructured":"Aaron Grattafiori Abhimanyu Dubey Abhinav Jauhri Abhinav Pandey Abhishek Kadian Ahmad Al-Dahle Aiesha Letman Akhil Mathur Alan Schelten Alex Vaughan et al. 2024. The Llama 3 Herd of Models. arXiv preprint arXiv:2407.21783 (2024). https:\/\/arxiv.org\/abs\/2407.21783"},{"key":"e_1_3_2_1_14_1","volume-title":"Evaluating Large Language Models: A Comprehensive Survey. arXiv preprint arXiv:2310.19736","author":"Guo Zishan","year":"2023","unstructured":"Zishan Guo, Renren Jin, Chuang Liu, Yufei Huang, Dan Shi, Supryadi, Linhao Yu, Yan Liu, Jiaxuan Li, Bojian Xiong, and Deyi Xiong. 2023. Evaluating Large Language Models: A Comprehensive Survey. arXiv preprint arXiv:2310.19736 (2023). https:\/\/arxiv.org\/abs\/2310.19736"},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.6028\/NIST.SP.500-249.web-overview"},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","unstructured":"Albert Q. Jiang Alexandre Sablayrolles Arthur Mensch Chris Bamford Devendra Singh Chaplot Diego de las Casas Florian Bressand Gianna Lengyel Guillaume Lample Lucile Saulnier L\u00e9lio Renard Lavaud Marie-Anne Lachaux Pierre Stock Teven Le Scao Thibaut Lavril Thomas Wang Timoth\u00e9e Lacroix and William El Sayed. 2023. Mistral 7B. arXiv preprint arXiv:2310.06825 (2023). https:\/\/doi.org\/10.48550\/arXiv.2310.06825","DOI":"10.48550\/arXiv.2310.06825"},{"key":"e_1_3_2_1_17_1","first-page":"59","article-title":"Passive voice and active voice in sentence structure","volume":"2","author":"Lubis M Yusri Ali","year":"2024","unstructured":"M Yusri Ali Lubis, Reysha Miranti, and Yani Lubis. 2024. Passive voice and active voice in sentence structure. Journal of Psychology, Counseling and Education, Vol. 2, 1 (2024), 59-64.","journal-title":"Journal of Psychology, Counseling and Education"},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1145\/3736402"},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-032-21300-6_5"},{"key":"e_1_3_2_1_20_1","volume-title":"Judging the Judges: A Collection of LLM-Generated Relevance Judgements. arXiv preprint arXiv:2502.13908","author":"Rahmani Hossein A.","year":"2025","unstructured":"Hossein A. Rahmani, Clemencia Siro, Mohammad Aliannejadi, Nick Craswell, Charles L. A. Clarke, Guglielmo Faggioli, Bhaskar Mitra, Paul Thomas, and Emine Yilmaz. 2025. Judging the Judges: A Collection of LLM-Generated Relevance Judgements. arXiv preprint arXiv:2502.13908 (2025). https:\/\/arxiv.org\/abs\/2502.13908"},{"key":"e_1_3_2_1_21_1","volume-title":"Verbosity bias in preference labeling by large language models. arXiv preprint arXiv:2310.10076","author":"Saito Keita","year":"2023","unstructured":"Keita Saito, Akifumi Wachi, Koki Wataoka, and Youhei Akimoto. 2023. Verbosity bias in preference labeling by large language models. arXiv preprint arXiv:2310.10076 (2023). https:\/\/arxiv.org\/abs\/2310.10076"},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.54195\/irrj.19625"},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.48550\/arXiv.2501.18536"},{"key":"e_1_3_2_1_24_1","unstructured":"Gemma Team. 2025. Gemma 3. https:\/\/goo.gle\/Gemma3Report"},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.1145\/3626772.3657707"},{"key":"e_1_3_2_1_26_1","volume-title":"Hoa Trang Dang, and Jimmy Lin","author":"Upadhyay Shivani","year":"2024","unstructured":"Shivani Upadhyay, Ronak Pradeep, Nandan Thakur, Daniel Campos, Nick Craswell, Ian Soboroff, Hoa Trang Dang, and Jimmy Lin. 2024a. A Large-Scale Study of Relevance Assessments with Large Language Models: An Initial Look. arXiv preprint arXiv:2411.08275 (2024). https:\/\/arxiv.org\/abs\/2411.08275"},{"key":"e_1_3_2_1_27_1","doi-asserted-by":"publisher","DOI":"10.1145\/3731120.3744605"},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"publisher","DOI":"10.48550\/arXiv.2406.06519"},{"key":"e_1_3_2_1_29_1","volume-title":"Overview of the TREC 2005 Robust Retrieval Track. Text Retrieval Conference (TREC)","author":"Voorhees Ellen M","year":"2006","unstructured":"Ellen M Voorhees. 2006. Overview of the TREC 2005 Robust Retrieval Track. Text Retrieval Conference (TREC) (2006). https:\/\/tsapps.nist.gov\/publication\/get_pdf.cfm?pub_id=150643"},{"key":"e_1_3_2_1_30_1","unstructured":"An Yang Anfeng Li Baosong Yang Beichen Zhang Binyuan Hui Bo Zheng Bowen Yu Chang Gao Chengen Huang Chenxu Lv Chujie Zheng Dayiheng Liu Fan Zhou Fei Huang Feng Hu Hao Ge Haoran Wei Huan Lin... and Zihan Qiu. 2025. Qwen3 Technical Report. arXiv preprint arXiv:2505.09388 (2025). https:\/\/doi.org\/10.48550\/arXiv.2505.09388"},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"publisher","DOI":"10.52202\/075280-2020"}],"event":{"name":"SIGIR '26: The 49th International ACM SIGIR Conference on Research and Development in Information Retrieval","location":"Melbourne VIC Australia","sponsor":["SIGIR ACM Special Interest Group on Information Retrieval"]},"container-title":["Proceedings of the 49th International ACM SIGIR Conference on Research and Development in Information Retrieval"],"original-title":[],"deposited":{"date-parts":[[2026,7,15]],"date-time":"2026-07-15T17:20:06Z","timestamp":1784136006000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3805712.3809905"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026,7,19]]},"references-count":31,"alternative-id":["10.1145\/3805712.3809905","10.1145\/3805712"],"URL":"https:\/\/doi.org\/10.1145\/3805712.3809905","relation":{},"subject":[],"published":{"date-parts":[[2026,7,19]]},"assertion":[{"value":"2026-07-19","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}