{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,8,28]],"date-time":"2026-08-28T06:15:56Z","timestamp":1787897756703,"version":"build-2784847793"},"publisher-location":"New York, NY, USA","reference-count":101,"publisher":"ACM","license":[{"start":{"date-parts":[[2024,7,10]],"date-time":"2024-07-10T00:00:00Z","timestamp":1720569600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2024,7,10]]},"DOI":"10.1145\/3626772.3657845","type":"proceedings-article","created":{"date-parts":[[2024,7,11]],"date-time":"2024-07-11T12:40:05Z","timestamp":1720701605000},"page":"282-292","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":8,"title":["What Matters in a Measure? A Perspective from Large-Scale Search Evaluation"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-2425-3136","authenticated-orcid":false,"given":"Paul","family":"Thomas","sequence":"first","affiliation":[{"name":"Microsoft, Adelaide, Australia"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0002-5158-6630","authenticated-orcid":false,"given":"Gabriella","family":"Kazai","sequence":"additional","affiliation":[{"name":"Amazon, London, United Kingdom"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-9351-8137","authenticated-orcid":false,"given":"Nick","family":"Craswell","sequence":"additional","affiliation":[{"name":"Microsoft, Seattle, WA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-5089-7632","authenticated-orcid":false,"given":"Seth","family":"Spielman","sequence":"additional","affiliation":[{"name":"Microsoft, Boulder, CO, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2024,7,11]]},"reference":[{"key":"e_1_3_2_1_1_1","volume-title":"The practice of crowdsourcing","author":"Alonso Omar","unstructured":"Omar Alonso. 2022. The practice of crowdsourcing. Springer Nature."},{"key":"e_1_3_2_1_2_1","volume-title":"The practice of crowdsourcing","author":"Alonso Omar","unstructured":"Omar Alonso and Gary Marchionini. 2019. The practice of crowdsourcing. Morgan & Claypool Publishers."},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1145\/3366423.3380281"},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.1145\/1645953.1646031"},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1145\/3209978.3210027"},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"publisher","DOI":"10.1145\/1328964.1328969"},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"publisher","DOI":"10.1145\/1390334.1390447"},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.1145\/2766462.2767728"},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1287\/mksc.2018.1104"},{"key":"e_1_3_2_1_10_1","volume-title":"A systematic review of fairness, accountability, transparency and ethics in information retrieval. Comput. Surveys (Dec","author":"Bernard Nolwenn","year":"2023","unstructured":"Nolwenn Bernard and Krisztian Balog. 2023. A systematic review of fairness, accountability, transparency and ethics in information retrieval. Comput. Surveys (Dec. 2023). To appear."},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.1038\/nature11421"},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.1145\/1008992.1009000"},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1145\/3543507.3583551"},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.1145\/3477495.3531953"},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.1145\/3077136.3080804"},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"crossref","unstructured":"Cheng-Han Chiang and Hung yi Lee. 2023. Can large language models be an alternative to human evaluation?arxiv: 2305.01937v1 [cs.CL]","DOI":"10.18653\/v1\/2023.acl-long.870"},{"key":"e_1_3_2_1_17_1","volume-title":"Mizen","author":"Alec Chrystal K.","year":"2001","unstructured":"K. Alec Chrystal and Paul D. Mizen. 2001. Goodhart's law: Its origins, meaning and implications for monetary policy. Prepared for the Festschrift in honour of Charles Goodhart."},{"key":"e_1_3_2_1_18_1","volume-title":"Design. Aslib Cranfield Research Project.","author":"Cleverdon Cyril W","unstructured":"Cyril W Cleverdon, Jack Mills, and E Michael Keen. 1966. Factors determining the performance of indexing systems, volume 1: Design. Aslib Cranfield Research Project."},{"key":"e_1_3_2_1_19_1","volume-title":"Introducing holdouts. https:\/\/blog.statsig.com\/introducing-holdouts-4bcfc1821d1c. Accessed","author":"Coleman Alex","year":"2024","unstructured":"Alex Coleman. 2021. Introducing holdouts. https:\/\/blog.statsig.com\/introducing-holdouts-4bcfc1821d1c. Accessed January 2024."},{"key":"e_1_3_2_1_20_1","volume-title":"Proceedings of the International ACM SIGIR Conference on Research and Development in Information Retrieval. 1566--1576","author":"Craswell Nick","year":"2021","unstructured":"Nick Craswell, Bhaskar Mitra, Emine Yilmaz, Daniel Campos, and Jimmy Lin. 2021. MSMARCO: Benchmarking ranking models in the large-data regime. In Proceedings of the International ACM SIGIR Conference on Research and Development in Information Retrieval. 1566--1576."},{"key":"e_1_3_2_1_21_1","article-title":"Topic difficulty: Collection and query formulation effects","volume":"40","author":"Culpepper J Shane","year":"2021","unstructured":"J Shane Culpepper, Guglielmo FAggioli, Nicola Ferro, and Oren Kurland. 2021. Topic difficulty: Collection and query formulation effects. ACM Transactions on Information Systems, Vol. 40, 1, Article 19 (2021).","journal-title":"ACM Transactions on Information Systems"},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.1145\/3148148"},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.1145\/2939672.2939700"},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"publisher","DOI":"10.1145\/3459637.3482032"},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.1145\/2983323.2983356"},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.1145\/2736277.2741116"},{"key":"e_1_3_2_1_27_1","doi-asserted-by":"publisher","DOI":"10.1145\/2433396.2433418"},{"key":"e_1_3_2_1_28_1","volume-title":"Long-term impact of badging. https:\/\/jwegan.com\/growth-hacking\/long-term-impact-badging\/. Accessed","author":"Egan John","year":"2024","unstructured":"John Egan. 2015. Long-term impact of badging. https:\/\/jwegan.com\/growth-hacking\/long-term-impact-badging\/. Accessed January 2024."},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.1073\/pnas.1419828112"},{"key":"e_1_3_2_1_30_1","volume-title":"Article 19 (Aug.","author":"Ferro Nicola","year":"2017","unstructured":"Nicola Ferro. 2017. What does affect the correlation among evaluation measures? ACM Transactions on Information Systems, Article 19 (Aug. 2017)."},{"key":"e_1_3_2_1_31_1","volume-title":"Proceedings of the ACM International Conference on Web Search and Data Mining. 280--288","author":"Ferro Nicola","year":"2022","unstructured":"Nicola Ferro and Mark Sanderson. 2022. How do you test a test? A multifaceted examination of significance tests. In Proceedings of the ACM International Conference on Web Search and Data Mining. 280--288."},{"key":"e_1_3_2_1_32_1","doi-asserted-by":"publisher","DOI":"10.1145\/2566486.2568015"},{"key":"e_1_3_2_1_33_1","first-page":"4","article-title":"The Netflix recommender system: Algorithms, business value, and innovation","volume":"6","author":"Gomez-Uribe Carlos A.","year":"2015","unstructured":"Carlos A. Gomez-Uribe and Neil Hunt. 2015. The Netflix recommender system: Algorithms, business value, and innovation. ACM Transactions on Management Information Systems, Vol. 6, 4 (Dec. 2015).","journal-title":"ACM Transactions on Management Information Systems"},{"key":"e_1_3_2_1_34_1","volume-title":"Papers in Monetary Economics.","author":"Goodhart Charles A E","unstructured":"Charles A E Goodhart. 1975. Problems of monetary management: The UK experience. In Papers in Monetary Economics. Vol. 1. Reserve Bank of Australia."},{"key":"e_1_3_2_1_35_1","volume-title":"STEDII properties of a good metric. https:\/\/www.microsoft.com\/en-us\/research\/group\/experimentation-platform-exp\/articles\/stedii-properties-of-a-good-metric\/. Accessed","author":"Gupta Somit","year":"2024","unstructured":"Somit Gupta and Widad Machmouchi. 2022. STEDII properties of a good metric. https:\/\/www.microsoft.com\/en-us\/research\/group\/experimentation-platform-exp\/articles\/stedii-properties-of-a-good-metric\/. Accessed January 2024."},{"key":"e_1_3_2_1_36_1","doi-asserted-by":"crossref","unstructured":"Donna Harman. 2011. Information Retrieval Evaluation. Number 19 in Synthesis Lectures on Information Concepts Retrieval and Services. Morgan and Claypool.","DOI":"10.1007\/978-3-031-02276-0"},{"key":"e_1_3_2_1_37_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10791-009-9101-4"},{"key":"e_1_3_2_1_38_1","volume-title":"Simulating information retrieval test collections. Synthesis lectures on information concepts, retrieval, and services","author":"Hawking David","unstructured":"David Hawking, Bodo Billerbeck, Paul Thomas, and Nick Craswell. 2020. Simulating information retrieval test collections. Synthesis lectures on information concepts, retrieval, and services, Vol. 71. Morgan and Claypool."},{"key":"e_1_3_2_1_39_1","volume-title":"OKCupid: we experiment on users. Everyone does. The Guardian (29","author":"Hern Alex","year":"2014","unstructured":"Alex Hern. 2014. OKCupid: we experiment on users. Everyone does. The Guardian (29 July 2014). https:\/\/www.theguardian.com\/technology\/2014\/jul\/29\/okcupid-experiment-human-beings-dating"},{"key":"e_1_3_2_1_40_1","volume-title":"Accountability: Power, ethos and technologies of managing","author":"Hoskin Keith","unstructured":"Keith Hoskin. 1996. The `awful' idea of accountability: Inscribing people into the measurement of objects. In Accountability: Power, ethos and technologies of managing, R Munro and J Mouritsen (Eds.). International Thompson Business Press, London."},{"key":"e_1_3_2_1_41_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10551-021-05004-x"},{"key":"e_1_3_2_1_42_1","doi-asserted-by":"publisher","DOI":"10.1145\/582415.582418"},{"key":"e_1_3_2_1_43_1","doi-asserted-by":"publisher","DOI":"10.1145\/2854946.2855005"},{"key":"e_1_3_2_1_44_1","doi-asserted-by":"publisher","DOI":"10.1145\/2766462.2767824"},{"key":"e_1_3_2_1_45_1","doi-asserted-by":"publisher","DOI":"10.1145\/2396761.2396779"},{"key":"e_1_3_2_1_46_1","doi-asserted-by":"publisher","DOI":"10.1145\/2396761.2398697"},{"key":"e_1_3_2_1_47_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10791-012-9205-0"},{"key":"e_1_3_2_1_48_1","doi-asserted-by":"publisher","DOI":"10.1561\/1500000012"},{"key":"e_1_3_2_1_49_1","doi-asserted-by":"publisher","DOI":"10.1145\/959258.959260"},{"key":"e_1_3_2_1_50_1","doi-asserted-by":"publisher","DOI":"10.1145\/2484028.2484168"},{"key":"e_1_3_2_1_51_1","doi-asserted-by":"publisher","DOI":"10.1145\/3077136.3080837"},{"key":"e_1_3_2_1_52_1","volume-title":"Proceedings of the International ACM SIGIR Conference on Research and Development in Information Retrieval. 829--832","author":"Kim Jin Young","year":"2018","unstructured":"Jin Young Kim, Jaime Teevan, and Nick Craswell. 2018. Explicit in situ user feedback for web search results. In Proceedings of the International ACM SIGIR Conference on Research and Development in Information Retrieval. 829--832."},{"key":"e_1_3_2_1_53_1","volume-title":"Tamir Melamed, and Juan M. Lavista Ferres.","author":"Kohavi Ronny","year":"2009","unstructured":"Ronny Kohavi, Thomas Crook, Roger Longbotham, Brian Frasca, Randy Henne, Juan Lavista Ferres, Tamir Melamed, and Juan M. Lavista Ferres. 2009a. Online experimentation at Microsoft. https:\/\/www.microsoft.com\/en-us\/research\/publication\/online-experimentation-at-microsoft\/. Accessed January 2024."},{"key":"e_1_3_2_1_54_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10618-008-0114-1"},{"key":"e_1_3_2_1_55_1","doi-asserted-by":"publisher","DOI":"10.1073\/pnas.1320040111"},{"key":"e_1_3_2_1_56_1","volume-title":"You are not a gadget: A manifesto","author":"Lanier Jaron","unstructured":"Jaron Lanier. 2010. You are not a gadget: A manifesto. Alfred A Knopf, New York."},{"key":"e_1_3_2_1_57_1","doi-asserted-by":"publisher","DOI":"10.1145\/3445029"},{"key":"e_1_3_2_1_58_1","unstructured":"Natasha Lomas. 2017. Google to ramp up AI efforts to ID extremism on YouTube. https:\/\/techcrunch.com\/2017\/06\/19\/google-to-ramp-up-ai-efforts-to-id-extremism-on-youtube\/."},{"key":"e_1_3_2_1_59_1","first-page":"507","article-title":"Bounties, grants, and market-making entrepreneurship","volume":"22","author":"Lucas David S","year":"2018","unstructured":"David S Lucas and Caleb S Fuller. 2018. Bounties, grants, and market-making entrepreneurship. The Independent Review, Vol. 22, 4 (2018), 507--528.","journal-title":"The Independent Review"},{"key":"e_1_3_2_1_60_1","doi-asserted-by":"publisher","DOI":"10.1145\/3132847.3132850"},{"key":"e_1_3_2_1_61_1","doi-asserted-by":"publisher","DOI":"10.1145\/2911451.2926731"},{"key":"e_1_3_2_1_62_1","doi-asserted-by":"publisher","DOI":"10.1145\/3404835.3462894"},{"key":"e_1_3_2_1_63_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-45068-6_1"},{"key":"e_1_3_2_1_64_1","doi-asserted-by":"crossref","unstructured":"Alistair Moffat. 2022. Batch evaluation metrics in information retrieval: Measures scales and meaning. arxiv: 2207.03103 [cs.IR]","DOI":"10.1109\/ACCESS.2022.3211668"},{"key":"e_1_3_2_1_65_1","doi-asserted-by":"publisher","DOI":"10.3844\/jcssp.2005.437.444"},{"key":"e_1_3_2_1_66_1","article-title":"The development and evaluation of a survey to measure user engagement","volume":"61","author":"O'Brien Heather L","year":"2010","unstructured":"Heather L O'Brien and Elaine G Toms. 2010. The development and evaluation of a survey to measure user engagement. Journal of the Association for Information Science and Technology, Vol. 61 (2010). Issue 1.","journal-title":"Journal of the Association for Information Science and Technology"},{"key":"e_1_3_2_1_67_1","first-page":"198","article-title":"Think-aloud protocols. In Applications of social research methods to questions in information and library science 2 ed.), Barbara M Wildemuth (Ed.). Libraries Unlimited, Santa Barbara, California","volume":"21","author":"Oh Sanghee","year":"2017","unstructured":"Sanghee Oh and Barbara M Wildemuth. 2017. Think-aloud protocols. In Applications of social research methods to questions in information and library science 2 ed.), Barbara M Wildemuth (Ed.). Libraries Unlimited, Santa Barbara, California, Chapter 21, 198--208.","journal-title":"Chapter"},{"key":"e_1_3_2_1_68_1","first-page":"2","article-title":"FACTS-IR: fairness, accountability, confidentiality, transparency, and safety in information retrieval","volume":"53","author":"Olteanu Alexandra","year":"2021","unstructured":"Alexandra Olteanu, Jean Garcia-Gathright, Maarten de Rijke, Michael D. Ekstrand, Adam Roegiest, Aldo Lipani, Alex Beutel, Alexandra Olteanu, Ana Lucic, Ana-Andreea Stoica, Anubrata Das, Asia Biega, Bart Voorn, Claudia Hauff, Damiano Spina, David Lewis, Douglas W. Oard, Emine Yilmaz, Faegheh Hasibi, Gabriella Kazai, Graham McDonald, Hinda Haned, Iadh Ounis, Ilse van der Linden, Jean Garcia-Gathright, Joris Baan, Kamuela N. Lau, Krisztian Balog, Maarten de Rijke, Mahmoud Sayed, Maria Panteli, Mark Sanderson, Matthew Lease, Michael D. Ekstrand, Preethi Lahoti, and Toshihiro Kamishima. 2021. FACTS-IR: fairness, accountability, confidentiality, transparency, and safety in information retrieval. ACM SIGIR Forum, Vol. 53, 2 (March 2021), 20--43.","journal-title":"ACM SIGIR Forum"},{"key":"e_1_3_2_1_69_1","doi-asserted-by":"publisher","DOI":"10.1145\/2816815"},{"key":"e_1_3_2_1_70_1","volume-title":"Proceedings of the International Working Conference on Advanced Visual Interfaces. 405--408","author":"Rahmanian Bahareh","unstructured":"Bahareh Rahmanian and Joseph G. Davis. 2014. User interface design for crowdsourcing systems. In Proceedings of the International Working Conference on Advanced Visual Interfaces. 405--408."},{"key":"e_1_3_2_1_71_1","volume-title":"Auditing radicalization pathways on YouTube. arxiv","author":"Ribeiro Manoel Horta","year":"1908","unstructured":"Manoel Horta Ribeiro, Robert West, Raphael Ottoni, Virg\u00edlio A. F. Almeida, and Wagner Meira Jr. 2021. Auditing radicalization pathways on YouTube. arxiv: 1908.08313 [cs.CY]"},{"key":"e_1_3_2_1_72_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.ipm.2021.102688"},{"key":"e_1_3_2_1_73_1","unstructured":"Kevin Roose. 2020. Rabbit Hole. The New York Times: https:\/\/www.nytimes.com\/column\/rabbit-hole."},{"key":"e_1_3_2_1_74_1","doi-asserted-by":"publisher","DOI":"10.1145\/1277741.1277756"},{"key":"e_1_3_2_1_75_1","doi-asserted-by":"publisher","DOI":"10.1145\/2641383.2641385"},{"key":"e_1_3_2_1_76_1","doi-asserted-by":"publisher","DOI":"10.1145\/3077136.3080766"},{"key":"e_1_3_2_1_77_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10791-008-9059-7"},{"key":"e_1_3_2_1_78_1","doi-asserted-by":"publisher","DOI":"10.1109\/INFRKM.2016.7806344"},{"key":"e_1_3_2_1_79_1","doi-asserted-by":"publisher","DOI":"10.1561\/1500000009"},{"key":"e_1_3_2_1_80_1","unstructured":"Barry Schwarz. 2010. Microsoft records their Q1 2010 earnings Bing takes loss again. Search Engine Land: https:\/\/searchengineland.com\/microsoft-records-their-q1-2010-earnings-bing-takes-loss-again-54281."},{"key":"e_1_3_2_1_81_1","doi-asserted-by":"publisher","DOI":"10.1145\/1321440.1321528"},{"key":"e_1_3_2_1_82_1","doi-asserted-by":"publisher","DOI":"10.1609\/aimag.v42i3.18140"},{"key":"e_1_3_2_1_83_1","doi-asserted-by":"publisher","DOI":"10.1145\/3498366.3505815"},{"key":"e_1_3_2_1_84_1","doi-asserted-by":"publisher","DOI":"10.1145\/3291992.3292002"},{"key":"e_1_3_2_1_85_1","doi-asserted-by":"publisher","DOI":"10.1145\/3626772.3657707"},{"key":"e_1_3_2_1_86_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.ipm.2019.02.001"},{"key":"e_1_3_2_1_87_1","volume-title":"Assessing ranking metrics in top-N recommendation. Information Retrieval Journal","author":"Valcarce Daniel","year":"2020","unstructured":"Daniel Valcarce, Alejandro Bellog\u00edn, Javier Parapar, and Pablo Castells. 2020. Assessing ranking metrics in top-N recommendation. Information Retrieval Journal (2020), 411--448."},{"key":"e_1_3_2_1_88_1","doi-asserted-by":"publisher","DOI":"10.1353\/fch.2003.0027"},{"key":"e_1_3_2_1_89_1","volume-title":"TREC: Experiment and evaluation in information retrieval","author":"Voorhees Ellen M.","year":"2005","unstructured":"Ellen M. Voorhees and Donna K. Harman (Eds.). 2005. TREC: Experiment and evaluation in information retrieval. The MIT Press, Cambridge, MA."},{"key":"e_1_3_2_1_90_1","doi-asserted-by":"publisher","DOI":"10.1145\/3580305.3599522"},{"key":"e_1_3_2_1_91_1","unstructured":"Craig Whitlock Leslie Shapiro and Armand Emamdjomeh. 2019. Mohammed Ehsan Zia lessons learned interview. The Washington Post: https:\/\/www.washingtonpost.com\/graphics\/2019\/investigations\/afghanistan-papers\/documents-database\/?tid=a_inl_manual&document=background_ll_04_xx4_04122016."},{"key":"e_1_3_2_1_92_1","doi-asserted-by":"publisher","DOI":"10.1145\/3336191.3371799"},{"key":"e_1_3_2_1_93_1","volume-title":"Information Processing and Management","volume":"58","author":"Wicaksono Alfan Farizki","year":"2021","unstructured":"Alfan Farizki Wicaksono and Alistair Moffat. 2021. Modeling search and session effectiveness. Information Processing and Management, Vol. 58 (2021)."},{"key":"e_1_3_2_1_94_1","doi-asserted-by":"publisher","DOI":"10.1145\/1183614.1183633"},{"key":"e_1_3_2_1_95_1","doi-asserted-by":"publisher","DOI":"10.1145\/2661829.2661953"},{"key":"e_1_3_2_1_96_1","doi-asserted-by":"publisher","DOI":"10.1177\/0894439313506838"},{"key":"e_1_3_2_1_97_1","volume-title":"Evaluating recommender systems: Survey and Framework. Comput. Surveys","author":"Zangerle Eva","year":"2022","unstructured":"Eva Zangerle and Christine Bauer. 2022. Evaluating recommender systems: Survey and Framework. Comput. Surveys (2022)."},{"key":"e_1_3_2_1_98_1","unstructured":"Cheng Xiang Zhai. 2022. Information retrieval evaluation as search simulation. Talk presented at the NTCIR-16 conference."},{"key":"e_1_3_2_1_99_1","doi-asserted-by":"publisher","DOI":"10.1145\/3077136.3080841"},{"key":"e_1_3_2_1_100_1","doi-asserted-by":"publisher","DOI":"10.1145\/3397271.3401162"},{"key":"e_1_3_2_1_101_1","doi-asserted-by":"crossref","unstructured":"Honglei Zhuang Zhen Qin Kai Hui Junru Wu Le Yan Xuanhui Wang and Michael Bendersky. 2023. Beyond yes and no: Improving zero-shot LLM rankers via scoring fine-grained relevance labels. arxiv: 2310.14122","DOI":"10.18653\/v1\/2024.naacl-short.31"}],"event":{"name":"SIGIR 2024: The 47th International ACM SIGIR Conference on Research and Development in Information Retrieval","location":"Washington DC USA","acronym":"SIGIR 2024","sponsor":["SIGIR ACM Special Interest Group on Information Retrieval"]},"container-title":["Proceedings of the 47th International ACM SIGIR Conference on Research and Development in Information Retrieval"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3626772.3657845","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3626772.3657845","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,8,22]],"date-time":"2025-08-22T05:43:57Z","timestamp":1755841437000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3626772.3657845"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,7,10]]},"references-count":101,"alternative-id":["10.1145\/3626772.3657845","10.1145\/3626772"],"URL":"https:\/\/doi.org\/10.1145\/3626772.3657845","relation":{},"subject":[],"published":{"date-parts":[[2024,7,10]]},"assertion":[{"value":"2024-07-11","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}