{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,11]],"date-time":"2026-07-11T02:36:22Z","timestamp":1783737382980,"version":"3.55.0"},"publisher-location":"New York, NY, USA","reference-count":29,"publisher":"ACM","content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2025,7,13]]},"DOI":"10.1145\/3726302.3730252","type":"proceedings-article","created":{"date-parts":[[2025,7,14]],"date-time":"2025-07-14T01:25:28Z","timestamp":1752456328000},"page":"2812-2816","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":2,"title":["System Comparison Using Automated Generation of Relevance Judgements in Multiple Languages"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-2425-3136","authenticated-orcid":false,"given":"Paul","family":"Thomas","sequence":"first","affiliation":[{"name":"Microsoft, Adelaide, SA, Australia"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-1696-0407","authenticated-orcid":false,"given":"Douglas W.","family":"Oard","sequence":"additional","affiliation":[{"name":"University of Maryland, College Park, MD, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-0051-1535","authenticated-orcid":false,"given":"Eugene","family":"Yang","sequence":"additional","affiliation":[{"name":"Johns Hopkins University HLTCOE, Baltimore, MD, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-7347-7086","authenticated-orcid":false,"given":"Dawn","family":"Lawrie","sequence":"additional","affiliation":[{"name":"Johns Hopkins University HLTCOE, Baltimore, MD, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-3866-3013","authenticated-orcid":false,"given":"James","family":"Mayfield","sequence":"additional","affiliation":[{"name":"Johns Hopkins University HLTCOE, Baltimore, MD, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2025,7,13]]},"reference":[{"key":"e_1_3_2_1_1_1","volume-title":"Proc. SIGIR-AP.","author":"Alaofi Marwah","year":"2024","unstructured":"Marwah Alaofi, Paul Thomas, Falk Scholer, and Mark Sanderson. 2024. LLMs Can be Fooled into Labelling a Document as Relevant. In Proc. SIGIR-AP."},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.1145\/860435.860501"},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1145\/1277741.1277755"},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.1145\/1189702.1189703"},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1177\/001316446002000104"},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"publisher","DOI":"10.1145\/290941.291009"},{"key":"e_1_3_2_1_7_1","first-page":"251","article-title":"The Treatment of Ties in Rank-Biased Overlap","author":"Corsi Matteo","year":"2024","unstructured":"Matteo Corsi and Juli\u00e1n Urbano. 2024. The Treatment of Ties in Rank-Biased Overlap. In Proc. SIGIR. 251-260.","journal-title":"Proc. SIGIR."},{"key":"e_1_3_2_1_8_1","first-page":"1089","article-title":"Gauging the Quality of Relevance Assessments Using Inter-Rater Agreement","author":"Damessie Tadele T.","year":"2017","unstructured":"Tadele T. Damessie, Taho P. Nghiem, Falk Scholer, and J. Shane Culpepper. 2017. Gauging the Quality of Relevance Assessments Using Inter-Rater Agreement. In Proc. SIGIR. 1089-1092.","journal-title":"Proc. SIGIR."},{"key":"e_1_3_2_1_9_1","first-page":"39","article-title":"Perspectives on Large Language Models for Relevance Judgment","author":"Faggioli Guglielmo","year":"2023","unstructured":"Guglielmo Faggioli, Laura Dietz, Charles LA Clarke, Gianluca Demartini, Matthias Hagen, Claudia Hauff, Noriko Kando, Evangelos Kanoulas, Martin Potthast, Benno Stein, et al. 2023. Perspectives on Large Language Models for Relevance Judgment. In Proc. SIGIR. 39-50.","journal-title":"Proc. SIGIR."},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"crossref","unstructured":"Nicola Ferro and Carol Peters (Eds.). 2019. Information Retrieval Evaluation in a Changing World Lessons Learned from 20 Years of CLEF. Springer.","DOI":"10.1007\/978-3-030-22948-1"},{"key":"e_1_3_2_1_11_1","volume-title":"Cross-Language Retrieval","author":"Galu\u0161\u010d\u00e1kov\u00e1 Petra","year":"2024","unstructured":"Petra Galu\u0161\u010d\u00e1kov\u00e1, Douglas W Oard, and Suraj Nair. 2024. Cross-Language Retrieval. Information Retrieval: Advanced Topics and Techniques (2024), 321-357."},{"key":"e_1_3_2_1_12_1","first-page":"941","article-title":"Pearson rank","author":"Gao Ning","year":"2016","unstructured":"Ning Gao, Mossaab Bagdouri, and Douglas W Oard. 2016. Pearson rank: A Head-Weighted Gap-Sensitive Score-Based Correlation Coefficient. In Proc. SIGIR. 941-944.","journal-title":"A Head-Weighted Gap-Sensitive Score-Based Correlation Coefficient. In Proc. SIGIR."},{"key":"e_1_3_2_1_13_1","volume-title":"Proc","author":"Gao Ning","unstructured":"Ning Gao, William Webber, and Douglas W Oard. 2014. Reducing Reliance on Relevance judgments for System Comparison by Using Expectation-Maximization. In Proc. ECIR. Springer, 1-12."},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.1093\/biomet\/30.1-2.81"},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.1177\/001316447003000105"},{"key":"e_1_3_2_1_16_1","volume-title":"Proc. TREC.","author":"Lawrie Dawn","year":"2022","unstructured":"Dawn Lawrie, Sean MacAvaney, James Mayfield, Paul McNamee, Douglas W. Oard, Luca Soldaini, and Eugene Yang. 2022. Overview of the TREC 2022 NeuCLIR Track. In Proc. TREC."},{"key":"e_1_3_2_1_17_1","volume-title":"Proc. TREC.","author":"Lawrie Dawn","year":"2023","unstructured":"Dawn Lawrie, Sean MacAvaney, James Mayfield, Paul McNamee, Douglas W Oard, Luca Soldaini, and Eugene Yang. 2023. Overview of the TREC 2023 NeuCLIR Track. In Proc. TREC."},{"key":"e_1_3_2_1_18_1","volume-title":"Proc. TREC.","author":"Lawrie Dawn","year":"2024","unstructured":"Dawn Lawrie, Sean MacAvaney, James Mayfield, Paul McNamee, Douglas W Oard, Luca Soldaini, and Eugene Yang. 2024. Overview of the TREC 2024 NeuCLIR Track. In Proc. TREC."},{"key":"e_1_3_2_1_19_1","first-page":"2230","article-title":"One-Shot Labeling for Automatic Relevance Estimation","author":"MacAvaney Sean","year":"2023","unstructured":"Sean MacAvaney and Luca Soldaini. 2023. One-Shot Labeling for Automatic Relevance Estimation. In Proc. SIGIR. 2230-2235.","journal-title":"Proc. SIGIR."},{"key":"e_1_3_2_1_20_1","volume-title":"Cross-Language Information Retrieval","author":"Nie Jian-Yun","unstructured":"Jian-Yun Nie. 2010. Cross-Language Information Retrieval. Morgan & Claypool Publishers."},{"key":"e_1_3_2_1_21_1","unstructured":"OpenAI. 2024. Hello GPT-4o. https:\/\/openai.com\/index\/hello-gpt-4o\/"},{"key":"e_1_3_2_1_22_1","first-page":"347","article-title":"VII. Note on Regression and Inheritance in the Case of Two Parents","volume":"58","author":"Pearson Karl","year":"1895","unstructured":"Karl Pearson. 1895. VII. Note on Regression and Inheritance in the Case of Two Parents. Proceedings of the Royal Society of London, Vol. 58, 347-352 ( 1895), 240-242.","journal-title":"Proceedings of the Royal Society of London"},{"key":"e_1_3_2_1_23_1","first-page":"3040","article-title":"LLM4EVAL","author":"Rahmani Hossein A","year":"2024","unstructured":"Hossein A Rahmani, Clemencia Siro, Mohammad Aliannejadi, Nick Craswell, Charles LA Clarke, Guglielmo Faggioli, Bhaskar Mitra, Paul Thomas, and Emine Yilmaz. 2024a. LLM4EVAL: Large Language Model for Evaluation in IR. In Proc. SIGIR. 3040-3043.","journal-title":"Large Language Model for Evaluation in IR. In Proc. SIGIR."},{"key":"e_1_3_2_1_24_1","volume-title":"Report on the 1st Workshop on Large Language Model for Evaluation in Information Retrieval (LLM4EVAL 2024) at SIGIR 2024. arXiv preprint arXiv:2408","author":"Rahmani Hossein A","year":"2024","unstructured":"Hossein A Rahmani, Clemencia Siro, Mohammad Aliannejadi, Nick Craswell, Charles LA Clarke, Guglielmo Faggioli, Bhaskar Mitra, Paul Thomas, and Emine Yilmaz. 2024b. Report on the 1st Workshop on Large Language Model for Evaluation in Information Retrieval (LLM4EVAL 2024) at SIGIR 2024. arXiv preprint arXiv:2408.05388 (2024)."},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.1145\/383952.383961"},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.1145\/3626772.3657707"},{"key":"e_1_3_2_1_27_1","doi-asserted-by":"publisher","DOI":"10.1016\/S0306-4573(00)00010-8"},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"publisher","DOI":"10.1145\/1852102.1852106"},{"key":"e_1_3_2_1_29_1","first-page":"587","article-title":"A New Rank Correlation Coefficient for Information Retrieval","author":"Yilmaz Emine","year":"2008","unstructured":"Emine Yilmaz, Javed A Aslam, and Stephen Robertson. 2008. A New Rank Correlation Coefficient for Information Retrieval. In Proc. SIGIR. 587-594.","journal-title":"Proc. SIGIR."}],"event":{"name":"SIGIR '25: The 48th International ACM SIGIR Conference on Research and Development in Information Retrieval","location":"Padua Italy","acronym":"SIGIR '25","sponsor":["SIGIR ACM Special Interest Group on Information Retrieval"]},"container-title":["Proceedings of the 48th International ACM SIGIR Conference on Research and Development in Information Retrieval"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3726302.3730252","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,8,22]],"date-time":"2025-08-22T12:16:43Z","timestamp":1755865003000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3726302.3730252"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,7,13]]},"references-count":29,"alternative-id":["10.1145\/3726302.3730252","10.1145\/3726302"],"URL":"https:\/\/doi.org\/10.1145\/3726302.3730252","relation":{},"subject":[],"published":{"date-parts":[[2025,7,13]]},"assertion":[{"value":"2025-07-13","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}