{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,8,22]],"date-time":"2025-08-22T05:03:38Z","timestamp":1755839018978,"version":"3.41.0"},"publisher-location":"New York, NY, USA","reference-count":47,"publisher":"ACM","license":[{"start":{"date-parts":[[2020,9,14]],"date-time":"2020-09-14T00:00:00Z","timestamp":1600041600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"DOI":"10.13039\/501100000038","name":"Natural Sciences and Engineering Research Council of Canada","doi-asserted-by":"publisher","award":["Discovery grant"],"award-info":[{"award-number":["Discovery grant"]}],"id":[{"id":"10.13039\/501100000038","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2020,9,14]]},"DOI":"10.1145\/3409256.3409816","type":"proceedings-article","created":{"date-parts":[[2020,9,5]],"date-time":"2020-09-05T10:13:32Z","timestamp":1599300812000},"page":"185-192","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":12,"title":["Offline Evaluation without Gain"],"prefix":"10.1145","author":[{"given":"Charles L.A.","family":"Clarke","sequence":"first","affiliation":[{"name":"University of Waterloo, Waterloo, ON, Canada"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Alexandra","family":"Vtyurina","sequence":"additional","affiliation":[{"name":"University of Waterloo, Waterloo, ON, Canada"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Mark D.","family":"Smucker","sequence":"additional","affiliation":[{"name":"University of Waterloo, Waterloo, ON, Canada"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2020,9,14]]},"reference":[{"volume-title":"28th ACM International Conference on Information and Knowledge Management. 1833--1842","author":"Abualsaud Mustafa","key":"e_1_3_2_1_1_1"},{"volume-title":"Text REtrieval Conference.","year":"1999","author":"Adi Tom","key":"e_1_3_2_1_2_1"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_3_1","DOI":"10.1145\/1390334.1390447"},{"volume-title":"36th International ACM SIGIR Conference on Research and Development in Information Retrieval. 909--912","author":"Bashir Maryam","key":"e_1_3_2_1_4_1"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_5_1","DOI":"10.1145\/1008992.1009093"},{"volume-title":"27th Annual International ACM SIGIR Conference on Research and Development in Information Retrieval. 25--32","author":"Buckley Chris","key":"e_1_3_2_1_6_1"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_8_1","DOI":"10.1145\/2009916.2010037"},{"volume-title":"SIGIR 2008 Workshop on Beyond Binary Relevance: Preferences, Diversity, and Set-Level Judgments.","year":"2008","author":"Carterette Ben","key":"e_1_3_2_1_9_1"},{"volume-title":"Evaluation Measures for Preference Judgments. In 31st Annual International ACM SIGIR Conference on Research and Development in Information Retrieval. 685--686","author":"Carterette Ben","key":"e_1_3_2_1_10_1"},{"unstructured":"Ben Carterette Paul N. Bennett David Maxwell Chickering and Susan T. Dumais. 2008b. Here or there: Preference judgments for relevance. Computer Science Department Faculty Publication Series 46. University of Massachusetts Amherst.  Ben Carterette Paul N. Bennett David Maxwell Chickering and Susan T. Dumais. 2008b. Here or there: Preference judgments for relevance. Computer Science Department Faculty Publication Series 46. University of Massachusetts Amherst.","key":"e_1_3_2_1_11_1"},{"volume-title":"Preference Based Evaluation Measures for Novelty and Diversity. In 36th International ACM SIGIR Conference on Research and Development in Information Retrieval. 413--422","year":"2013","author":"Chandar Praveen","key":"e_1_3_2_1_12_1"},{"key":"e_1_3_2_1_13_1","volume-title":"Proceedings of the Learning to Rank Challenge (Proceedings of Machine Learning Research),, Olivier Chapelle, Yi Chang, and Tie-Yan Liu (Eds.)","volume":"14","author":"Chapelle Olivier","year":"2011"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_14_1","DOI":"10.1145\/1645953.1646033"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_15_1","DOI":"10.1145\/2433396.2433420"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_16_1","DOI":"10.1145\/3340531.3411915"},{"unstructured":"Charles L. A. Clarke Alexandra Vtyurina and Mark D. Smucker. 2020 b. Assessing top-k preferences. arxiv: cs.IR\/2007.11682  Charles L. A. Clarke Alexandra Vtyurina and Mark D. Smucker. 2020 b. Assessing top-k preferences. arxiv: cs.IR\/2007.11682","key":"e_1_3_2_1_17_1"},{"volume-title":"TREC 2013 Web Track overview. In Text REtrieval Conference.","author":"Collins-Thompson Kevyn","key":"e_1_3_2_1_18_1"},{"volume-title":"TREC 2014 Web Track overview. In Text REtrieval Conference.","author":"Collins-Thompson Kevyn","key":"e_1_3_2_1_19_1"},{"volume-title":"Overview of the TREC 2019 Deep Learning Track. In Text REtrieval Conference.","year":"2019","author":"Craswell Nick","key":"e_1_3_2_1_20_1"},{"volume-title":"21st Australasian Document Computing Symposium. 17--24","author":"Culpepper J. Shane","key":"e_1_3_2_1_21_1"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_22_1","DOI":"10.1145\/3298689.3347058"},{"volume-title":"Text REtrieval Conference.","year":"2019","author":"Dalton Jeffrey","key":"e_1_3_2_1_23_1"},{"volume-title":"Text REtrieval Conference.","year":"2019","author":"Dietz Laura","key":"e_1_3_2_1_24_1"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_25_1","DOI":"10.1016\/0306-4573(91)90046-O"},{"key":"e_1_3_2_1_26_1","volume-title":"ACM Transactions on Information Systems","volume":"20","author":"Jaana Kalervo","year":"2002"},{"volume-title":"18th ACM Conference on Information and Knowledge Management. 611--620","author":"Kanoulas Evangelos","key":"e_1_3_2_1_27_1"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_28_1","DOI":"10.1145\/1772690.1772749"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_29_1","DOI":"10.1145\/3308774.3308781"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_30_1","DOI":"10.1145\/3159652.3159676"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_31_1","DOI":"10.1145\/3002172"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_32_1","DOI":"10.1007\/978-3-642-04769-5_7"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_33_1","DOI":"10.1145\/1416950.1416952"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_34_1","DOI":"10.1145\/2348283.2348384"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_35_1","DOI":"10.1145\/1935826.1935850"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_36_1","DOI":"10.1002\/(SICI)1097-4571(199012)41:8<590::AID-ASI5>3.0.CO;2-T"},{"volume-title":"Evaluating Evaluation Metrics Based on the Bootstrap. In 29th Annual International ACM SIGIR Conference on Research and Development in Information Retrieval. 525--532","year":"2006","author":"Sakai Tetsuya","key":"e_1_3_2_1_37_1"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_38_1","DOI":"10.1145\/1277741.1277756"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_39_1","DOI":"10.1145\/1835449.1835542"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_40_1","DOI":"10.1145\/1772690.1772785"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_41_1","DOI":"10.1109\/TKDE.2015.2448541"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_42_1","DOI":"10.1145\/290941.291017"},{"volume-title":"Text REtrieval Conference.","author":"Ellen","key":"e_1_3_2_1_43_1"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_44_1","DOI":"10.1145\/1852102.1852106"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_45_1","DOI":"10.1145\/3291992.3291995"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_46_1","DOI":"10.5555\/208098.208105"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_47_1","DOI":"10.1145\/1871437.1871672"},{"volume-title":"SIGIR 2010 Workshop on Crowdsourcing for Search Evaluation.","year":"2010","author":"Zhu Dongqing","key":"e_1_3_2_1_48_1"}],"event":{"sponsor":["SIGIR ACM Special Interest Group on Information Retrieval"],"acronym":"ICTIR '20","name":"ICTIR '20: The 2020 ACM SIGIR International Conference on the Theory of Information Retrieval","location":"Virtual Event Norway"},"container-title":["Proceedings of the 2020 ACM SIGIR on International Conference on Theory of Information Retrieval"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3409256.3409816","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3409256.3409816","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T22:01:32Z","timestamp":1750197692000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3409256.3409816"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2020,9,14]]},"references-count":47,"alternative-id":["10.1145\/3409256.3409816","10.1145\/3409256"],"URL":"https:\/\/doi.org\/10.1145\/3409256.3409816","relation":{},"subject":[],"published":{"date-parts":[[2020,9,14]]},"assertion":[{"value":"2020-09-14","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}