{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,6,18]],"date-time":"2025-06-18T04:19:39Z","timestamp":1750220379441,"version":"3.41.0"},"publisher-location":"New York, NY, USA","reference-count":31,"publisher":"ACM","license":[{"start":{"date-parts":[[2021,7,11]],"date-time":"2021-07-11T00:00:00Z","timestamp":1625961600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"name":"Natural Sciences and Engineering Research Council of Canada","award":["Discovery Grant"],"award-info":[{"award-number":["Discovery Grant"]}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2021,7,11]]},"DOI":"10.1145\/3404835.3462947","type":"proceedings-article","created":{"date-parts":[[2021,7,12]],"date-time":"2021-07-12T03:08:41Z","timestamp":1626059321000},"page":"1534-1543","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":3,"title":["Evaluation Measures Based on Preference Graphs"],"prefix":"10.1145","author":[{"given":"Charles L.A.","family":"Clarke","sequence":"first","affiliation":[{"name":"University of Waterloo, Waterloo, ON, Canada"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Chengxi","family":"Luo","sequence":"additional","affiliation":[{"name":"University of Waterloo, Waterloo, ON, Canada"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Mark D.","family":"Smucker","sequence":"additional","affiliation":[{"name":"University of Waterloo, Waterloo, ON, Canada"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2021,7,11]]},"reference":[{"volume-title":"2nd ACM International Conference on Web Search and Data Mining. 172--181","author":"Agrawal R.","key":"e_1_3_2_2_1_1","unstructured":"R. Agrawal, A. Halverson, K. Kenthapadi, N. Mishra, and P. Tsaparas. 2009. Generating Labels from Clicks. In 2nd ACM International Conference on Web Search and Data Mining. 172--181."},{"key":"e_1_3_2_2_2_1","doi-asserted-by":"publisher","DOI":"10.1145\/1277741.1277902"},{"volume-title":"36th International ACM SIGIR Conference on Research and Development in Information Retrieval","author":"Bashir Maryam","key":"e_1_3_2_2_3_1","unstructured":"Maryam Bashir, Jesse Anderton, Jie Wu, Peter B Golbus, Virgil Pavlu, and Javed A. Aslam. 2013. A Document Rating System for Preference Judgements. In 36th International ACM SIGIR Conference on Research and Development in Information Retrieval. Dublin, Ireland, 909--912."},{"key":"e_1_3_2_2_5_1","volume-title":"SIGIR 2008 Workshop on Beyond Binary Relevance: Preferences, Diversity, and Set-Level Judgments","author":"Carterette Ben","year":"2008","unstructured":"Ben Carterette, Paul Bennett, and Olivier Chapelle. 2008a. A Test Collection of Preference Judgments. In SIGIR 2008 Workshop on Beyond Binary Relevance: Preferences, Diversity, and Set-Level Judgments. Singapore."},{"volume-title":"Evaluation Measures For Preference Judgments. In 31st Annual International ACM SIGIR Conference on Research and Development in Information Retrieval","author":"Carterette Ben","key":"e_1_3_2_2_6_1","unstructured":"Ben Carterette and Paul N. Bennett. 2008. Evaluation Measures For Preference Judgments. In 31st Annual International ACM SIGIR Conference on Research and Development in Information Retrieval. Singapore, 685--686."},{"key":"e_1_3_2_2_7_1","volume-title":"David Maxwell Chickering, and Susan T. Dumais","author":"Carterette Ben","year":"2008","unstructured":"Ben Carterette, Paul N. Bennett, David Maxwell Chickering, and Susan T. Dumais. 2008b. Here or there: Preference Judgments for Relevance. Computer Science Department Faculty Publication Series 46. University of Massachusetts Amherst."},{"key":"e_1_3_2_2_8_1","volume-title":"Preference Based Evaluation Measures for Novelty and Diversity. In 36th International ACM SIGIR Conference on Research and Development in Information Retrieval","author":"Chandar Praveen","year":"2013","unstructured":"Praveen Chandar and Ben Carterette. 2013. Preference Based Evaluation Measures for Novelty and Diversity. In 36th International ACM SIGIR Conference on Research and Development in Information Retrieval. Dublin, Ireland, 413--422."},{"key":"e_1_3_2_2_9_1","volume-title":"Expected Reciprocal Rank for Graded Relevance. In 18th ACM Conference on Information and Knowledge Management","author":"Chapelle Olivier","year":"2009","unstructured":"Olivier Chapelle, Donald Metlzer, Ya Zhang, and Pierre Grinspan. 2009. Expected Reciprocal Rank for Graded Relevance. In 18th ACM Conference on Information and Knowledge Management. Hong Kong, China, 621--630."},{"volume-title":"2020 ACM SIGIR on International Conference on Theory of Information Retrieval. 185--192","author":"Clarke Charles L.A.","key":"e_1_3_2_2_10_1","unstructured":"Charles L.A. Clarke, Alexandra Vtyurina, and Mark D. Smucker. 2020 b. Offline Evaluation without Gain. In 2020 ACM SIGIR on International Conference on Theory of Information Retrieval. 185--192."},{"key":"e_1_3_2_2_11_1","doi-asserted-by":"publisher","DOI":"10.1145\/3340531.3411915"},{"key":"e_1_3_2_2_12_1","volume-title":"Smucker","author":"Clarke Charles L. A.","year":"2021","unstructured":"Charles L. A. Clarke, Alexandra Vtyurina, and Mark D. Smucker. 2021. Assessing Top-k Preferences. ACM Transactions on Information Systems (2021). To appear."},{"key":"e_1_3_2_2_13_1","volume-title":"28th Text REtrieval Conference","author":"Dalton Jeffrey","year":"2019","unstructured":"Jeffrey Dalton, Chenyan Xiong, and Jamie Callan. 2019. CAsT 2019: The Conversational Assistance Track Overview. In 28th Text REtrieval Conference. Gaithersburg, Maryland."},{"key":"e_1_3_2_2_14_1","doi-asserted-by":"publisher","DOI":"10.1016\/0020-0190(93)90079-O"},{"key":"e_1_3_2_2_15_1","doi-asserted-by":"crossref","unstructured":"Marco Ferrante Nicola Ferro and Norbert Fuhr. 2021. Towards Meaningful Statements in IR Evaluation. Mapping Evaluation Measures to Interval Scales. arxiv: cs.IR\/2101.02668","DOI":"10.1109\/ACCESS.2021.3116857"},{"key":"e_1_3_2_2_16_1","doi-asserted-by":"publisher","DOI":"10.1016\/0306-4573(91)90046-O"},{"key":"e_1_3_2_2_17_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-56608-5_58"},{"key":"e_1_3_2_2_18_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-56608-5_19"},{"key":"e_1_3_2_2_19_1","doi-asserted-by":"publisher","DOI":"10.1145\/582415.582418"},{"volume-title":"Information Retrieval Evaluation Measures Based on Preference Graphs. Master's thesis","author":"Luo Chengxi","key":"e_1_3_2_2_20_1","unstructured":"Chengxi Luo. 2021. Information Retrieval Evaluation Measures Based on Preference Graphs. Master's thesis. University of Waterloo, Canada."},{"key":"e_1_3_2_2_21_1","doi-asserted-by":"publisher","DOI":"10.1145\/3002172"},{"key":"e_1_3_2_2_22_1","volume-title":"Evaluation Methods and Query Complexity. In 4th ACM International Conference on Web Search and Data Mining","author":"Radinsky Kira","year":"2011","unstructured":"Kira Radinsky and Nir Ailon. 2011. Ranking from Pairs and Triplets: Information Quality, Evaluation Methods and Query Complexity. In 4th ACM International Conference on Web Search and Data Mining. Hong Kong, China, 105--114."},{"key":"e_1_3_2_2_23_1","doi-asserted-by":"publisher","DOI":"10.1002\/(SICI)1097-4571(199012)41:8<590::AID-ASI5>3.0.CO;2-T"},{"key":"e_1_3_2_2_24_1","volume-title":"Evaluating Diversified Search Results Using Per-Intent Graded Relevance. In 34th International ACM SIGIR Conference on Research and Development in Information Retrieval. 1043--1052","author":"Sakai Tetsuya","year":"2011","unstructured":"Tetsuya Sakai and Ruihua Song. 2011. Evaluating Diversified Search Results Using Per-Intent Graded Relevance. In 34th International ACM SIGIR Conference on Research and Development in Information Retrieval. 1043--1052."},{"key":"e_1_3_2_2_25_1","doi-asserted-by":"publisher","DOI":"10.1145\/3331184.3331215"},{"key":"e_1_3_2_2_26_1","volume-title":"Good Evaluation Measures Based on Document Preferences. In 43rd International ACM SIGIR Conference on Research and Development in Information Retrieval. 359--368","author":"Sakai Tetsuya","year":"2020","unstructured":"Tetsuya Sakai and Zhaohao Zeng. 2020. Good Evaluation Measures Based on Document Preferences. In 43rd International ACM SIGIR Conference on Research and Development in Information Retrieval. 359--368."},{"key":"e_1_3_2_2_27_1","doi-asserted-by":"publisher","DOI":"10.14778\/3021924.3021930"},{"key":"e_1_3_2_2_28_1","doi-asserted-by":"publisher","DOI":"10.1145\/1852102.1852106"},{"key":"e_1_3_2_2_29_1","volume-title":"Preference-Based Evaluation Metrics for Web Image Search. In 43rd International ACM SIGIR Conference on Research and Development in Information Retrieval. 369--378","author":"Xie Xiaohui","year":"2020","unstructured":"Xiaohui Xie, Jiaxin Mao, Yiqun Liu, Maarten de Rijke, Haitian Chen, Min Zhang, and Shaoping Ma. 2020. Preference-Based Evaluation Metrics for Web Image Search. In 43rd International ACM SIGIR Conference on Research and Development in Information Retrieval. 369--378."},{"key":"e_1_3_2_2_30_1","doi-asserted-by":"publisher","DOI":"10.1145\/3291992.3291995"},{"key":"e_1_3_2_2_31_1","doi-asserted-by":"publisher","DOI":"10.5555\/208098.208105"},{"key":"e_1_3_2_2_32_1","volume-title":"An Analysis of Assessor Behavior in Crowdsourced Preference Judgments. In SIGIR 2010 Workshop on Crowdsourcing for Search Evaluation.","author":"Zhu Dongqing","year":"2010","unstructured":"Dongqing Zhu and Ben Carterette. 2010. An Analysis of Assessor Behavior in Crowdsourced Preference Judgments. In SIGIR 2010 Workshop on Crowdsourcing for Search Evaluation."}],"event":{"name":"SIGIR '21: The 44th International ACM SIGIR Conference on Research and Development in Information Retrieval","sponsor":["SIGIR ACM Special Interest Group on Information Retrieval"],"location":"Virtual Event Canada","acronym":"SIGIR '21"},"container-title":["Proceedings of the 44th International ACM SIGIR Conference on Research and Development in Information Retrieval"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3404835.3462947","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3404835.3462947","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T20:18:20Z","timestamp":1750191500000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3404835.3462947"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2021,7,11]]},"references-count":31,"alternative-id":["10.1145\/3404835.3462947","10.1145\/3404835"],"URL":"https:\/\/doi.org\/10.1145\/3404835.3462947","relation":{},"subject":[],"published":{"date-parts":[[2021,7,11]]},"assertion":[{"value":"2021-07-11","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}