{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,15]],"date-time":"2026-07-15T18:08:59Z","timestamp":1784138939955,"version":"3.55.0"},"publisher-location":"New York, NY, USA","reference-count":51,"publisher":"ACM","license":[{"start":{"date-parts":[[2026,7,19]],"date-time":"2026-07-19T00:00:00Z","timestamp":1784419200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/legalcode"}],"funder":[{"name":"German Research Foundation (DFG)","award":["407518790"],"award-info":[{"award-number":["407518790"]}]},{"name":"German Federal Ministry of Education and Research (BMBF) and Joint Science Conference (GWK)","award":["03FHP109"],"award-info":[{"award-number":["03FHP109"]}]},{"name":"German Research Foundation (DFG)","award":["492813820"],"award-info":[{"award-number":["492813820"]}]},{"name":"Klaus Tschira Stiftung","award":["00.003.2020"],"award-info":[{"award-number":["00.003.2020"]}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2026,7,20]]},"DOI":"10.1145\/3805712.3809561","type":"proceedings-article","created":{"date-parts":[[2026,7,10]],"date-time":"2026-07-10T14:28:19Z","timestamp":1783693699000},"page":"778-789","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":0,"title":["Formalized Information Needs Improve Large-Language-Model Relevance Judgments"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-9392-8646","authenticated-orcid":false,"given":"J\u00fcri","family":"Keller","sequence":"first","affiliation":[{"name":"TH K\u00f6ln, Cologne, Germany"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-1003-981X","authenticated-orcid":false,"given":"Maik","family":"Fr\u00f6be","sequence":"additional","affiliation":[{"name":"Friedrich-Schiller-Universit\u00e4t Jena, Jena, Germany"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0000-7074-9066","authenticated-orcid":false,"given":"Bj\u00f6rn","family":"Engelmann","sequence":"additional","affiliation":[{"name":"TH K\u00f6ln, Cologne, Germany"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-3392-7860","authenticated-orcid":false,"given":"Fabian","family":"Haak","sequence":"additional","affiliation":[{"name":"TH K\u00f6ln, Cologne, Germany"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-1765-2449","authenticated-orcid":false,"given":"Timo","family":"Breuer","sequence":"additional","affiliation":[{"name":"TH K\u00f6ln, Cologne, Germany"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-3622-2698","authenticated-orcid":false,"given":"Birger","family":"Larsen","sequence":"additional","affiliation":[{"name":"Aalborg University, Copenhagen, Denmark"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-8817-4632","authenticated-orcid":false,"given":"Philipp","family":"Schaer","sequence":"additional","affiliation":[{"name":"TH K\u00f6ln, Cologne, Germany"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2026,7,19]]},"reference":[{"key":"e_1_3_2_1_1_1","first-page":"390","volume-title":"Sanderson","author":"Alaofi M.","year":"2025","unstructured":"Alaofi, M., Ferro, N., Thomas, P., Scholer, F., Sanderson, M.: Demographically-inspired query variants using an LLM. In: Zamani, H., Dietz, L., Piwowarski, B., Bruch, S. (eds.) Proceedings of the 2025 International ACM SIGIR Conference on Innovative Concepts and Theories in Information Retrieval, ICTIR 2025, Padua, Italy, 18 July 2025. pp. 390-400. ACM (2025). 10.1145\/3731120.3744608"},{"key":"e_1_3_2_1_2_1","first-page":"1869","volume-title":"Thomas","author":"Alaofi M.","year":"2023","unstructured":"Alaofi, M., Gallagher, L., Sanderson, M., Scholer, F., Thomas, P.: Can generative llms create query variants for test collections? an exploratory study. In: Chen, H., Duh, W.E., Huang, H., Kato, M.P., Mothe, J., Poblete, B. (eds.) Proceedings of the 46th International ACM SIGIR Conference on Research and Development in Information Retrieval, SIGIR 2023, Taipei, Taiwan, July 23-27, 2023. pp. 1869-1873. ACM (2023). 10.1145\/3539618.3591960"},{"key":"e_1_3_2_1_3_1","first-page":"32","volume-title":"Sanderson","author":"Alaofi M.","year":"2024","unstructured":"Alaofi, M., Thomas, P., Scholer, F., Sanderson, M.: Llms can be fooled into labelling a document as relevant: best caf\u00e9 near me; this paper is perfectly relevant. In: Sakai, T., Ishita, E., Ohshima, H., Hasibi, F., Mao, J., Jose, J.M. (eds.) Proceedings of the 2024 Annual International ACM SIGIR Conference on Research and Development in Information Retrieval in the Asia Pacific Region, SIGIR-AP 2024, Tokyo, Japan, December 9-12, 2024. pp. 32-41. ACM (2024). 10.1145\/3673791.3698431"},{"key":"e_1_3_2_1_4_1","first-page":"3194","volume-title":"48th International ACM SIGIR, Conference on Research and Development in Information Retrieval, SIGIR, 2025","author":"Arabzadeh N.","year":"2025","unstructured":"Arabzadeh, N., Clarke, C.L.A.: Benchmarking LLM-based, relevance judgment methods. In: Ferro, N., Maistro, M., Pasi, G., Alonso, O., Trotman, A., Verberne, S. (eds.) Proceedings of the 48th International ACM SIGIR, Conference on Research and Development in Information Retrieval, SIGIR, 2025, Padua, Italy, July 13-18, 2025. pp. 3194-3204. ACM (2025). 10.1145\/3726302.3730305"},{"key":"e_1_3_2_1_5_1","first-page":"3865","volume-title":"Qin","author":"Balog K.","year":"2025","unstructured":"Balog, K., Metzler, D., Qin, Z.: Rankers, judges, and assistants: Towards understanding the interplay of llms in information retrieval evaluation. In: Ferro, N., Maistro, M., Pasi, G., Alonso, O., Trotman, A., Verberne, S. (eds.) Proceedings of the 48th International ACM SIGIR Conference on Research and Development in Information Retrieval, SIGIR 2025, Padua, Italy, July 13-18, 2025. pp. 3865-3875. ACM (2025). 10.1145\/3726302.3730348"},{"key":"e_1_3_2_1_6_1","unstructured":"Belkin N.J.: Anomalous states of knowledge as a basis for information retrieval. Canadian journal of information science textbf5(1) 133-143 (1980)"},{"key":"e_1_3_2_1_7_1","first-page":"1","volume-title":"Culpepper","author":"Benham R.","year":"2017","unstructured":"Benham, R., Culpepper, J.S.: Risk-reward trade-offs in rank fusion. In: Koopman, B., Zuccon, G., Carman, M.J. (eds.) Proceedings of the 22nd Australasian Document Computing Symposium, ADCS, 2017, Brisbane, QLD, Australia, December 7-8, 2017. pp. 1:1-1:8. ACM (2017). 10.1145\/3166072.3166084"},{"key":"e_1_3_2_1_8_1","first-page":"185","volume-title":"Smucker","author":"Clarke C.L.A.","year":"2020","unstructured":"Clarke, C.L.A., Vtyurina, A., Smucker, M.D.: Offline evaluation without gain. In: Balog, K., Setty, V., Lioma, C., Liu, Y., Zhang, M., Berberich, K. (eds.) ICTIR '20: The 2020 ACM SIGIR International Conference on the Theory of Information Retrieval, Virtual Event, Norway, September 14-17, 2020. pp. 185-192. ACM (2020). 10.1145\/3409256.3409816"},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1145\/122860.122861"},{"key":"e_1_3_2_1_10_1","volume-title":"A coefficient of agreement for nominal scales. Educational and Psychological Measurement textbf20(1), 37-46","author":"Cohen J.","year":"1960","unstructured":"Cohen, J.: A coefficient of agreement for nominal scales. Educational and Psychological Measurement textbf20(1), 37-46 (1960). 10.1177\/001316446002000104"},{"key":"e_1_3_2_1_11_1","volume-title":"Campos","author":"Craswell N.","year":"2020","unstructured":"Craswell, N., Mitra, B., Yilmaz, E., Campos, D.: Overview of the TREC 2020 deep learning track. In: Voorhees, E.M., Ellis, A. (eds.) Proceedings of the Twenty-Ninth Text REtrieval Conference, TREC 2020, Virtual Event [Gaithersburg, Maryland, USA], November 16-20, 2020. NIST Special Publication, National Institute of Standards and Technology (NIST) (2020), https:\/\/trec.nist.gov\/pubs\/trec29\/papers\/OVERVIEW.DL.pdf"},{"key":"e_1_3_2_1_12_1","volume-title":"Voorhees","author":"Craswell N.","year":"2019","unstructured":"Craswell, N., Mitra, B., Yilmaz, E., Campos, D., Voorhees, E.M.: Overview of the TREC 2019 deep learning track. CoRR textbfabs\/2003.07820 (2020), https:\/\/arxiv.org\/abs\/2003.07820"},{"key":"e_1_3_2_1_13_1","first-page":"2369","volume-title":"Soboroff","author":"Craswell N.","year":"2021","unstructured":"Craswell, N., Mitra, B., Yilmaz, E., Campos, D., Voorhees, E.M., Soboroff, I.: TREC deep learning track: Reusable test collections in the large data regime. In: Diaz, F., Shah, C., Suel, T., Castells, P., Jones, R., Sakai, T. (eds.) SIGIR '21: The 44th International ACM SIGIR Conference on Research and Development in Information Retrieval, Virtual Event, Canada, July 11-15, 2021. pp. 2369-2375. ACM (2021). 10.1145\/3404835.3463249"},{"key":"e_1_3_2_1_14_1","volume-title":"I.: Overview of the TREC 2023 deep learning track. In: Soboroff, I., Ellis, A. (eds.) The Thirty-Second Text REtrieval Conference Proceedings (TREC 2023","author":"Craswell N.","year":"2023","unstructured":"Craswell, N., Mitra, B., Yilmaz, E., Rahmani, H.A., Campos, D., Lin, J., Voorhees, E.M., Soboroff, I.: Overview of the TREC 2023 deep learning track. In: Soboroff, I., Ellis, A. (eds.) The Thirty-Second Text REtrieval Conference Proceedings (TREC 2023), Gaithersburg, MD, USA, November 14-17, 2023. NIST Special Publication, vol. 500-xxx. National Institute of Standards and Technology (NIST) (2023), https:\/\/trec.nist.gov\/pubs\/trec32\/papers\/Overview_deep.pdf"},{"key":"e_1_3_2_1_15_1","first-page":"173","volume-title":"Fuhr","author":"Engelmann B.","year":"2024","unstructured":"Engelmann, B., Breuer, T., Friese, J.I., Schaer, P., Fuhr, N.: Context-driven interactive query simulations based on generative large language models. In: Goharian, N., Tonellotto, N., He, Y., Lipani, A., McDonald, G., Macdonald, C., Ounis, I. (eds.) Advances in Information Retrieval - 46th European Conference on Information Retrieval, ECIR 2024, Glasgow, UK, March 24-28, 2024, Proceedings, Part II. Lecture Notes in Computer Science, vol. 14609, pp. 173-188. Springer (2024). 10.1007\/978-3-031-56060-6_12"},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.1145\/3578337.3605136"},{"key":"e_1_3_2_1_17_1","first-page":"254","volume-title":"Dietz","author":"Farzi N.","year":"2025","unstructured":"Farzi, N., Dietz, L.: Criteria-based LLM relevance judgments. In: Zamani, H., Dietz, L., Piwowarski, B., Bruch, S. (eds.) Proceedings of the 2025 International ACM SIGIR Conference on Innovative Concepts and Theories in Information Retrieval, ICTIR 2025, Padua, Italy, 18 July 2025. pp. 254-263. ACM (2025). 10.1145\/3731120.3744591"},{"key":"e_1_3_2_1_18_1","first-page":"3214","volume-title":"Dietz","author":"Farzi N.","year":"2025","unstructured":"Farzi, N., Dietz, L.: Does UMBRELA work on other llms? In: Ferro, N., Maistro, M., Pasi, G., Alonso, O., Trotman, A., Verberne, S. (eds.) Proceedings of the 48th International ACM SIGIR Conference on Research and Development in Information Retrieval, SIGIR 2025, Padua, Italy, July 13-18, 2025. pp. 3214-3222. ACM (2025). 10.1145\/3726302.3730317"},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"crossref","unstructured":"Fleiss J.L.: Measuring nominal scale agreement among many raters. Psychological Bulletin textbf76 378-382 (1971)","DOI":"10.1037\/h0031619"},{"key":"e_1_3_2_1_20_1","first-page":"2858","volume-title":"48th International ACM SIGIR, Conference on Research and Development in Information Retrieval, SIGIR, 2025","author":"Fr\u00f6be M.","year":"2025","unstructured":"Fr\u00f6be, M., Parry, A., Schlatt, F., MacAvaney, S., Stein, B., Potthast, M., Hagen, M.: Large language model relevance assessors agree with one another more than with human assessors. In: Ferro, N., Maistro, M., Pasi, G., Alonso, O., Trotman, A., Verberne, S. (eds.) Proceedings of the 48th International ACM SIGIR, Conference on Research and Development in Information Retrieval, SIGIR, 2025, Padua, Italy, July 13-18, 2025. pp. 2858-2863. ACM (2025). 10.1145\/3726302.3730218"},{"key":"e_1_3_2_1_21_1","first-page":"36","volume-title":"Proceedings of the 16th Annual International ACM-SIGIR Conference on Research and Development in Information Retrieval.","author":"Harman D.","year":"1993","unstructured":"Harman, D.: Overview of the first text retrieval conference. In: Korfhage, R.R., Rasmussen, E.M., Willett, P. (eds.) Proceedings of the 16th Annual International ACM-SIGIR Conference on Research and Development in Information Retrieval. Pittsburgh, PA, USA, June 27 - July 1, 1993. pp. 36-47. ACM (1993). 10.1145\/160688.160692"},{"key":"e_1_3_2_1_22_1","first-page":"149","volume-title":"Peleteiro-Ramallo","author":"Hosseini K.","year":"2025","unstructured":"Hosseini, K., Kober, T., Krapac, J., Vollgraf, R., Cheng, W., Peleteiro-Ramallo, A.: Retrieve, annotate, evaluate, repeat: Leveraging multimodal llms for large-scale product retrieval evaluation. In: Hauff, C., Macdonald, C., Jannach, D., Kazai, G., Nardini, F.M., Pinelli, F., Silvestri, F., Tonellotto, N. (eds.) Advances in Information Retrieval - 47th European Conference on Information Retrieval, ECIR 2025, Lucca, Italy, April 6-10, 2025, Proceedings, Part I. pp. 149-163. Lecture Notes in Computer Science, Springer (2025). 10.1007\/978-3-031-88708-6_10"},{"key":"e_1_3_2_1_23_1","first-page":"2931","volume-title":"Schaer","author":"Kruff A.K.","year":"2025","unstructured":"Kruff, A.K., Breuer, T., Schaer, P.: Evaluating contrastive feedback for effective user simulations. In: Ferro, N., Maistro, M., Pasi, G., Alonso, O., Trotman, A., Verberne, S. (eds.) Proceedings of the 48th International ACM SIGIR Conference on Research and Development in Information Retrieval, SIGIR 2025, Padua, Italy, July 13-18, 2025. pp. 2931-2935. ACM (2025). 10.1145\/3726302.3730189"},{"key":"e_1_3_2_1_24_1","first-page":"2321","volume-title":"Bendersky","author":"Li M.","year":"2024","unstructured":"Li, M., Zhuang, H., Hui, K., Qin, Z., Lin, J., Jagerman, R., Wang, X., Bendersky, M.: Can query expansion improve generalization of strong cross-encoder rankers? In: Yang, G.H., Wang, H., Han, S., Hauff, C., Zuccon, G., Zhang, Y. (eds.) Proceedings of the 47th International ACM SIGIR Conference on Research and Development in Information Retrieval, SIGIR 2024, Washington DC, USA, July 14-18, 2024. pp. 2321-2326. ACM (2024). 10.1145\/3626772.3657979"},{"key":"e_1_3_2_1_25_1","first-page":"74","volume-title":"Text Summarization Branches Out.","author":"Lin C.Y.","year":"2004","unstructured":"Lin, C.Y.: ROUGE: A package for automatic evaluation of summaries. In: Text Summarization Branches Out. pp. 74-81. Association for Computational Linguistics, Barcelona, Spain (Jul 2004), https:\/\/aclanthology.org\/W04-1013\/"},{"key":"e_1_3_2_1_26_1","volume-title":"de Rijke","author":"Meng C.","year":"2025","unstructured":"Meng, C., Arabzadeh, N., Askari, A., Aliannejadi, M., de Rijke, M.: Query performance prediction using relevance judgments generated by large language models. ACM Trans. Inf. Syst. textbf43(4), 106:1-106:35 (2025). 10.1145\/3736402"},{"key":"e_1_3_2_1_27_1","first-page":"1759","volume-title":"Bailey","author":"Moffat A.","year":"2015","unstructured":"Moffat, A., Scholer, F., Thomas, P., Bailey, P.: Pooled evaluation over query variations: Users are as diverse as systems. In: Bailey, J., Moffat, A., Aggarwal, C.C., de Rijke, M., Kumar, R., Murdock, V., Sellis, T.K., Yu, J.X. (eds.) Proceedings of the 24th ACM International Conference on Information and Knowledge Management, CIKM 2015, Melbourne, VIC, Australia, October 19 - 23, 2015. pp. 1759-1762. ACM (2015). 10.1145\/2806416.2806606"},{"key":"e_1_3_2_1_28_1","volume-title":"Deng","author":"Nguyen T.","year":"2016","unstructured":"Nguyen, T., Rosenberg, M., Song, X., Gao, J., Tiwary, S., Majumder, R., Deng, L.: MS MARCO: A human generated machine reading comprehension dataset. In: Besold, T.R., Bordes, A., d'Avila Garcez, A.S., Wayne, G. (eds.) Proceedings of the Workshop on Cognitive Computation: Integrating neural and symbolic approaches 2016 co-located with the 30th Annual Conference on Neural Information Processing Systems (NIPS 2016), Barcelona, Spain, December 9, 2016. CEUR Workshop Proceedings, CEUR-WS.org (2016), https:\/\/ceur-ws.org\/Vol-1773\/CoCoNIPS_2016_paper9.pdf"},{"key":"e_1_3_2_1_29_1","volume-title":"Cho","author":"Nogueira R.","year":"1904","unstructured":"Nogueira, R., Yang, W., Lin, J., Cho, K.: Document expansion by query prediction. CoRR textbfabs\/1904.08375 (2019), http:\/\/arxiv.org\/abs\/1904.08375"},{"key":"e_1_3_2_1_30_1","volume-title":"gpt-oss-120b & gpt-oss-20b model card. CoRR textbfabs\/2508.10925","author":"Open AI","year":"2025","unstructured":"OpenAI: gpt-oss-120b & gpt-oss-20b model card. CoRR textbfabs\/2508.10925 (2025). 10.48550\/ARXIV.2508.10925"},{"key":"e_1_3_2_1_31_1","first-page":"2647","volume-title":"47th International ACM SIGIR, Conference on Research and Development in Information Retrieval, SIGIR, 2024","author":"Rahmani H.A.","year":"2024","unstructured":"Rahmani, H.A., Craswell, N., Yilmaz, E., Mitra, B., Campos, D.: Synthetic test collections for retrieval evaluation. In: Yang, G.H., Wang, H., Han, S., Hauff, C., Zuccon, G., Zhang, Y. (eds.) Proceedings of the 47th International ACM SIGIR, Conference on Research and Development in Information Retrieval, SIGIR, 2024, Washington DC, USA, July 14-18, 2024. pp. 2647-2651. ACM (2024). 10.1145\/3626772.3657942"},{"key":"e_1_3_2_1_32_1","first-page":"781","volume-title":"Thomas","author":"Rahmani H.A.","year":"2025","unstructured":"Rahmani, H.A., Wang, X., Yilmaz, E., Craswell, N., Mitra, B., Thomas, P.: Syndl: A large-scale synthetic test collection for passage retrieval. In: Long, G., Blumestein, M., Chang, Y., Lewin-Eytan, L., Huang, Z.H., Yom-Tov, E. (eds.) Companion Proceedings of the ACM on Web Conference 2025, WWW 2025, Sydney, NSW, Australia, 28 April 2025 - 2 May 2025. pp. 781-784. ACM (2025). 10.1145\/3701716.3715311"},{"key":"e_1_3_2_1_33_1","volume-title":"Test collection based evaluation of information retrieval systems. Found. Trends Inf. Retr. textbf4(4), 247-375","author":"Sanderson M.","year":"2010","unstructured":"Sanderson, M.: Test collection based evaluation of information retrieval systems. Found. Trends Inf. Retr. textbf4(4), 247-375 (2010). 10.1561\/1500000009"},{"key":"e_1_3_2_1_34_1","volume-title":"Don't use llms to make relevance judgments. Inf. Retr. Res. J. textbf1(1), 29-46","author":"Soboroff I.","year":"2025","unstructured":"Soboroff, I.: Don't use llms to make relevance judgments. Inf. Retr. Res. J. textbf1(1), 29-46 (2025). 10.54195\/IRRJ.19625"},{"key":"e_1_3_2_1_35_1","first-page":"95","volume-title":"Soboroff","author":"Takehi R.","year":"2025","unstructured":"Takehi, R., Voorhees, E.M., Sakai, T., Soboroff, I.: Llm-assisted relevance assessments: When should we ask llms for help? In: Ferro, N., Maistro, M., Pasi, G., Alonso, O., Trotman, A., Verberne, S. (eds.) Proceedings of the 48th International ACM SIGIR Conference on Research and Development in Information Retrieval, SIGIR 2025, Padua, Italy, July 13-18, 2025. pp. 95-105. ACM (2025). 10.1145\/3726302.3729916"},{"key":"e_1_3_2_1_36_1","volume-title":"A note on the pseudo-mathematics of relevance. American Documentation textbf16(2), 69-72","author":"Taube M.","year":"1965","unstructured":"Taube, M.: A note on the pseudo-mathematics of relevance. American Documentation textbf16(2), 69-72 (1965). 10.1002\/asi.5090160204"},{"key":"e_1_3_2_1_37_1","first-page":"1930","volume-title":"Mitra","author":"Thomas P.","year":"2024","unstructured":"Thomas, P., Spielman, S., Craswell, N., Mitra, B.: Large language models can accurately predict searcher preferences. In: Yang, G.H., Wang, H., Han, S., Hauff, C., Zuccon, G., Zhang, Y. (eds.) Proceedings of the 47th International ACM SIGIR Conference on Research and Development in Information Retrieval, SIGIR 2024, Washington DC, USA, July 14-18, 2024. pp. 1930-1940. ACM (2024). 10.1145\/3626772.3657707"},{"key":"e_1_3_2_1_38_1","doi-asserted-by":"publisher","DOI":"10.1145\/3626772.3657707"},{"key":"e_1_3_2_1_39_1","volume-title":"Lin, J.: A large-scale study of relevance assessments with large language models: An, initial look. CoRR textbfabs\/2411.08275","author":"Upadhyay S.","year":"2024","unstructured":"Upadhyay, S., Pradeep, R., Thakur, N., Campos, D., Craswell, N., Soboroff, I., Dang, H.T., Lin, J.: A large-scale study of relevance assessments with large language models: An, initial look. CoRR textbfabs\/2411.08275 (2024). 10.48550\/ARXIV.2411.08275"},{"key":"e_1_3_2_1_40_1","volume-title":"Lin","author":"Upadhyay S.","year":"2024","unstructured":"Upadhyay, S., Pradeep, R., Thakur, N., Craswell, N., Lin, J.: UMBRELA: umbrela is the (open-source reproduction of the) bing relevance assessor. CoRR textbfabs\/2406.06519 (2024). 10.48550\/ARXIV.2406.06519"},{"key":"e_1_3_2_1_41_1","first-page":"321","volume-title":"ICTIR 2017","author":"Urbano J.","year":"2017","unstructured":"Urbano, J., Marrero, M.: The treatment of ties in AP correlation. In: Kamps, J., Kanoulas, E., de Rijke, M., Fang, H., Yilmaz, E. (eds.) Proceedings of the ACM SIGIR International Conference on Theory of Information Retrieval, ICTIR 2017, Amsterdam, The Netherlands, October 1-4, 2017. pp. 321-324. ACM (2017). 10.1145\/3121050.3121106"},{"key":"e_1_3_2_1_42_1","volume-title":"TREC 2004","author":"Voorhees E.M.","year":"2004","unstructured":"Voorhees, E.M.: Overview of the TREC 2004 robust track. In: Voorhees, E.M., Buckland, L.P. (eds.) Proceedings of the Thirteenth Text REtrieval Conference, TREC 2004, Gaithersburg, Maryland, USA, November 16-19, 2004. NIST Special Publication, vol. 500-261. National Institute of Standards and Technology (NIST) (2004), http:\/\/trec.nist.gov\/pubs\/trec13\/papers\/ROBUST.OVERVIEW.pdf"},{"key":"e_1_3_2_1_43_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-22948-1_2"},{"key":"e_1_3_2_1_44_1","first-page":"2970","volume-title":"Lin","author":"Voorhees E.M.","year":"2022","unstructured":"Voorhees, E.M., Craswell, N., Lin, J.: Too many relevants: Whither cranfield test collections? In: Amig\u00f3, E., Castells, P., Gonzalo, J., Carterette, B., Culpepper, J.S., Kazai, G. (eds.) SIGIR '22: The 45th International ACM SIGIR Conference on Research and Development in Information Retrieval, Madrid, Spain, July 11 - 15, 2022. pp. 2970-2980. ACM (2022). 10.1145\/3477495.3531728"},{"key":"e_1_3_2_1_45_1","volume-title":"Harman","author":"Voorhees E.M.","year":"2005","unstructured":"Voorhees, E.M., Harman, D.K.: TREC: Experiment and Evaluation in Information Retrieval (Digital Libraries and Electronic Publishing). The MIT Press (2005)"},{"key":"e_1_3_2_1_46_1","series-title":"J.: A similarity measure for indefinite rankings. ACM Trans. Inf. Syst. textbf28(4)","first-page":"1","volume-title":"Zobel","author":"Webber W.","year":"2010","unstructured":"Webber, W., Moffat, A., Zobel, J.: A similarity measure for indefinite rankings. ACM Trans. Inf. Syst. textbf28(4), 20:1-20:38 (2010). 10.1145\/1852102.1852106"},{"key":"e_1_3_2_1_47_1","volume-title":"Qiu","author":"Yang A.","year":"2025","unstructured":"Yang, A., Li, A., Yang, B., Zhang, B., Hui, B., Zheng, B., Yu, B., Gao, C., Huang, C., Lv, C., Zheng, C., Liu, D., Zhou, F., Huang, F., Hu, F., Ge, H., Wei, H., Lin, H., Tang, J., Yang, J., Tu, J., Zhang, J., Yang, J., Yang, J., Zhou, J., Lin, J., Dang, K., Bao, K., Yang, K., Yu, L., Deng, L., Li, M., Xue, M., Li, M., Zhang, P., Wang, P., Zhu, Q., Men, R., Gao, R., Liu, S., Luo, S., Li, T., Tang, T., Yin, W., Ren, X., Wang, X., Zhang, X., Ren, X., Fan, Y., Su, Y., Zhang, Y., Zhang, Y., Wan, Y., Liu, Y., Wang, Z., Cui, Z., Zhang, Z., Zhou, Z., Qiu, Z.: Qwen3 technical report. CoRR textbfabs\/2505.09388 (2025). 10.48550\/ARXIV.2505.09388"},{"key":"e_1_3_2_1_48_1","first-page":"587","volume-title":"S.: A new rank correlation coefficient for information retrieval","author":"Yilmaz E.","year":"2008","unstructured":"Yilmaz, E., Aslam, J.A., Robertson, S.: A new rank correlation coefficient for information retrieval. In: Myaeng, S., Oard, D.W., Sebastiani, F., Chua, T., Leong, M. (eds.) Proceedings of the 31st Annual International ACM SIGIR Conference on Research and Development in Information Retrieval, SIGIR 2008, Singapore, July 20-24, 2008. pp. 587-594. ACM (2008). 10.1145\/1390334.1390435"},{"key":"e_1_3_2_1_49_1","volume-title":"8th International Conference on Learning Representations, ICLR 2020","author":"Zhang T.","year":"2020","unstructured":"Zhang, T., Kishore, V., Wu, F., Weinberger, K.Q., Artzi, Y.: Bertscore: Evaluating text generation with BERT. In: 8th International Conference on Learning Representations, ICLR 2020, Addis Ababa, Ethiopia, April 26-30, 2020. OpenReview.net (2020), https:\/\/openreview.net\/forum?id=SkeHuCVFDr"},{"key":"e_1_3_2_1_50_1","volume-title":"Stoica","author":"Zheng L.","year":"2023","unstructured":"Zheng, L., Chiang, W., Sheng, Y., Zhuang, S., Wu, Z., Zhuang, Y., Lin, Z., Li, Z., Li, D., Xing, E.P., Zhang, H., Gonzalez, J.E., Stoica, I.: Judging llm-as-a-judge with mt-bench and chatbot arena. In: Oh, A., Naumann, T., Globerson, A., Saenko, K., Hardt, M., Levine, S. (eds.) Advances in Neural Information Processing Systems 36: Annual Conference on Neural Information Processing Systems 2023, NeurIPS 2023, New Orleans, LA, USA, December 10 - 16, 2023 (2023), http:\/\/papers.nips.cc\/paper_files\/paper\/2023\/hash\/91f18a1287b398d378ef22505bf41832-Abstract-Datasets_and_Benchmarks.html"},{"key":"e_1_3_2_1_51_1","doi-asserted-by":"publisher","DOI":"10.1145\/290941.291014"}],"event":{"name":"SIGIR '26: The 49th International ACM SIGIR Conference on Research and Development in Information Retrieval","location":"Melbourne VIC Australia","sponsor":["SIGIR ACM Special Interest Group on Information Retrieval"]},"container-title":["Proceedings of the 49th International ACM SIGIR Conference on Research and Development in Information Retrieval"],"original-title":[],"deposited":{"date-parts":[[2026,7,15]],"date-time":"2026-07-15T17:32:26Z","timestamp":1784136746000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3805712.3809561"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026,7,19]]},"references-count":51,"alternative-id":["10.1145\/3805712.3809561","10.1145\/3805712"],"URL":"https:\/\/doi.org\/10.1145\/3805712.3809561","relation":{},"subject":[],"published":{"date-parts":[[2026,7,19]]},"assertion":[{"value":"2026-07-19","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}