{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,1,16]],"date-time":"2026-01-16T19:29:16Z","timestamp":1768591756607,"version":"3.49.0"},"publisher-location":"New York, NY, USA","reference-count":31,"publisher":"ACM","license":[{"start":{"date-parts":[[2017,9,12]],"date-time":"2017-09-12T00:00:00Z","timestamp":1505174400000},"content-version":"vor","delay-in-days":365,"URL":"http:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"DOI":"10.13039\/100000001","name":"National Science Foundation","doi-asserted-by":"publisher","award":["IIS-1247637"],"award-info":[{"award-number":["IIS-1247637"]}],"id":[{"id":"10.13039\/100000001","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000181","name":"Air Force Office of Scientific Research","doi-asserted-by":"publisher","award":["FA9550-15-1-0038"],"award-info":[{"award-number":["FA9550-15-1-0038"]}],"id":[{"id":"10.13039\/100000181","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2016,9,12]]},"DOI":"10.1145\/2970398.2970410","type":"proceedings-article","created":{"date-parts":[[2016,9,9]],"date-time":"2016-09-09T10:28:57Z","timestamp":1473416937000},"page":"109-118","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":10,"title":["Unbiased Comparative Evaluation of Ranking Functions"],"prefix":"10.1145","author":[{"given":"Tobias","family":"Schnabel","sequence":"first","affiliation":[{"name":"Cornell University, Ithaca, NY, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Adith","family":"Swaminathan","sequence":"additional","affiliation":[{"name":"Cornell University, Ithaca, NY, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Peter I.","family":"Frazier","sequence":"additional","affiliation":[{"name":"Cornell University, Ithaca, NY, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Thorsten","family":"Joachims","sequence":"additional","affiliation":[{"name":"Cornell University, Ithaca, NY, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2016,9,12]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"publisher","DOI":"10.1145\/1183614.1183773"},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.1145\/1148170.1148263"},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1145\/1008992.1009000"},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.1145\/1277741.1277755"},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1145\/1148170.1148219"},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-00958-7_27"},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"publisher","DOI":"10.1145\/122860.122861"},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.1145\/1148170.1148262"},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1145\/1629096.1629099"},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-28997-2_16"},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.1287\/opre.1.5.263"},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.1145\/1935826.1935878"},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-30671-1_20"},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.1145\/2851613.2851692"},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.1145\/1277741.1277806"},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.1145\/1416950.1416952"},{"key":"e_1_3_2_1_17_1","volume-title":"Draft","author":"Owen A. B.","year":"2013","unstructured":"A. B. Owen. Monte Carlo theory, methods and examples. Draft, 2013."},{"key":"e_1_3_2_1_18_1","volume-title":"A practical sampling strategy for efficient retrieval evaluation. Technical report","author":"Pavlu V.","year":"2007","unstructured":"V. Pavlu and J. Aslam. A practical sampling strategy for efficient retrieval evaluation. Technical report, Northeastern University, 2007."},{"key":"e_1_3_2_1_19_1","volume-title":"ICML Workshop: Machine Learning Meets Crowdsourcing","author":"Peng Ye D. D.","year":"2013","unstructured":"D. D. Peng Ye. Combining preference and absolute judgements in a crowd-sourced setting. In ICML Workshop: Machine Learning Meets Crowdsourcing, 2013."},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.1145\/2348283.2348402"},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10791-008-9059-7"},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.1561\/1500000009"},{"key":"e_1_3_2_1_23_1","volume-title":"Recommendations as treatments: Debiasing learning and evaluation. CoRR, abs\/1602.05352","author":"Schnabel T.","year":"2016","unstructured":"T. Schnabel, A. Swaminathan, A. Singh, N. Chandak, and T. Joachims. Recommendations as treatments: Debiasing learning and evaluation. CoRR, abs\/1602.05352, 2016."},{"key":"e_1_3_2_1_24_1","volume-title":"Report on the need for and provision of an 'ideal' information retrieval test collection. Technical report","author":"Sparck-Jones K.","year":"1975","unstructured":"K. Sparck-Jones and C. J. V. Rijsbergen. Report on the need for and provision of an 'ideal' information retrieval test collection. Technical report, University of Cambridge, 1975."},{"key":"e_1_3_2_1_25_1","volume-title":"NIST Special Publication 500--246","author":"Voorhees E. M.","year":"1999","unstructured":"E. M. Voorhees and D. Harman. Overview of the ninth text retrieval conference (TREC-8). In NIST Special Publication 500--246, 1999."},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.1145\/1571941.1572018"},{"key":"e_1_3_2_1_27_1","doi-asserted-by":"publisher","DOI":"10.1145\/1183614.1183633"},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"publisher","DOI":"10.1145\/1390334.1390437"},{"key":"e_1_3_2_1_29_1","first-page":"799","volume-title":"FLAIRS","author":"Yuan C.","year":"2005","unstructured":"C. Yuan and M. J. Druzdzel. How heavy should the tails be? In FLAIRS, pages 799--805, 2005."},{"key":"e_1_3_2_1_30_1","first-page":"1","volume-title":"ICML","author":"Zhao P.","year":"2015","unstructured":"P. Zhao and T. Zhang. Stochastic optimization with importance sampling for regularized loss minimization. In ICML, pages 1--9, 2015."},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"publisher","DOI":"10.1145\/290941.291014"}],"event":{"name":"ICTIR '16: ACM SIGIR International Conference on the Theory of Information Retrieval","location":"Newark Delaware USA","acronym":"ICTIR '16","sponsor":["SIGIR ACM Special Interest Group on Information Retrieval"]},"container-title":["Proceedings of the 2016 ACM International Conference on the Theory of Information Retrieval"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/2970398.2970410","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/2970398.2970410","content-type":"application\/pdf","content-version":"vor","intended-application":"syndication"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/2970398.2970410","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,11,18]],"date-time":"2025-11-18T09:31:01Z","timestamp":1763458261000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/2970398.2970410"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2016,9,12]]},"references-count":31,"alternative-id":["10.1145\/2970398.2970410","10.1145\/2970398"],"URL":"https:\/\/doi.org\/10.1145\/2970398.2970410","relation":{},"subject":[],"published":{"date-parts":[[2016,9,12]]},"assertion":[{"value":"2016-09-12","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}