{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,6,18]],"date-time":"2025-06-18T04:24:16Z","timestamp":1750220656248,"version":"3.41.0"},"publisher-location":"New York, NY, USA","reference-count":11,"publisher":"ACM","license":[{"start":{"date-parts":[[2020,10,19]],"date-time":"2020-10-19T00:00:00Z","timestamp":1603065600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"DOI":"10.13039\/501100009023","name":"Precursory Research for Embryonic Science and Technology","doi-asserted-by":"publisher","award":["JPMJPR1853"],"award-info":[{"award-number":["JPMJPR1853"]}],"id":[{"id":"10.13039\/501100009023","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100001691","name":"Japan Society for the Promotion of Science","doi-asserted-by":"publisher","award":["JP18H03243, JP18H03244, JP18H03494"],"award-info":[{"award-number":["JP18H03243, JP18H03244, JP18H03494"]}],"id":[{"id":"10.13039\/501100001691","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2020,10,19]]},"DOI":"10.1145\/3340531.3412143","type":"proceedings-article","created":{"date-parts":[[2020,10,19]],"date-time":"2020-10-19T06:32:45Z","timestamp":1603089165000},"page":"2081-2084","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":0,"title":["What Rankers Can be Statistically Distinguished in Multileaved Comparisons?"],"prefix":"10.1145","author":[{"given":"Makoto P.","family":"Kato","sequence":"first","affiliation":[{"name":"University of Tsukuba, Tsukuba, Japan"}]},{"given":"Akiomi","family":"Nishida","sequence":"additional","affiliation":[{"name":"Yahoo Japan Corporation, Chiyoda, Japan"}]},{"given":"Tomohiro","family":"Manabe","sequence":"additional","affiliation":[{"name":"Yahoo Japan Corporation, Chiyoda, Japan"}]},{"given":"Sumio","family":"Fujita","sequence":"additional","affiliation":[{"name":"Yahoo Japan Corporation, Chiyoda, Japan"}]},{"given":"Takehiro","family":"Yamamoto","sequence":"additional","affiliation":[{"name":"University of Hyogo, Kobe, Japan"}]}],"member":"320","published-online":{"date-parts":[[2020,10,19]]},"reference":[{"key":"e_1_3_2_2_1_1","doi-asserted-by":"publisher","DOI":"10.1145\/2094072.2094078"},{"key":"e_1_3_2_2_2_1","doi-asserted-by":"crossref","unstructured":"William Hersh Andrew Turpin Susan Price Benjamin Chan Dale Kramer Lynetta Sacherek and Daniel Olson. 2000. Do Batch and User Evaluations Give the Same Results?. In SIGIR. 17--24.  William Hersh Andrew Turpin Susan Price Benjamin Chan Dale Kramer Lynetta Sacherek and Daniel Olson. 2000. Do Batch and User Evaluations Give the Same Results?. In SIGIR. 17--24.","DOI":"10.1145\/345508.345539"},{"key":"e_1_3_2_2_3_1","doi-asserted-by":"crossref","unstructured":"Makoto P. Kato Tomohiro Manabe Sumio Fujita Akiomi Nishida and Takehiro Yamamoto. 2018. Challenges of Multileaved Comparison in Practice: Lessons from NTCIR-13 OpenLiveQ Task. In CIKM. 1515--1518.  Makoto P. Kato Tomohiro Manabe Sumio Fujita Akiomi Nishida and Takehiro Yamamoto. 2018. Challenges of Multileaved Comparison in Practice: Lessons from NTCIR-13 OpenLiveQ Task. In CIKM. 1515--1518.","DOI":"10.1145\/3269206.3269318"},{"key":"e_1_3_2_2_4_1","doi-asserted-by":"crossref","unstructured":"Makoto P. Kato Akiomi Nishida Tomohiro Manabe Sumio Fujita and Takehiro Yamamoto. 2019 a. Final Report of the NTCIR-14 OpenLiveQ-2 Task. In NII Testbeds and Community for Information Access Research. 45--56.  Makoto P. Kato Akiomi Nishida Tomohiro Manabe Sumio Fujita and Takehiro Yamamoto. 2019 a. Final Report of the NTCIR-14 OpenLiveQ-2 Task. In NII Testbeds and Community for Information Access Research. 45--56.","DOI":"10.1007\/978-3-030-36805-0_4"},{"key":"e_1_3_2_2_5_1","volume-title":"NTCIR-14 Conference.","author":"Kato Makoto P.","year":"2019","unstructured":"Makoto P. Kato , Akiomi Nishida , Tomohiro Manabe , Sumio Fujita , and Takehiro Yamamoto . 2019 b. Overview of the NTCIR-14 OpenLiveQ-2 Task . In NTCIR-14 Conference. Makoto P. Kato, Akiomi Nishida, Tomohiro Manabe, Sumio Fujita, and Takehiro Yamamoto. 2019 b. Overview of the NTCIR-14 OpenLiveQ-2 Task. In NTCIR-14 Conference."},{"key":"e_1_3_2_2_6_1","doi-asserted-by":"crossref","unstructured":"Harrie Oosterhuis and Maarten de Rijke. 2017. Sensitive and Scalable Online Evaluation with Theoretical Guarantees. In CIKM. 77--86.  Harrie Oosterhuis and Maarten de Rijke. 2017. Sensitive and Scalable Online Evaluation with Theoretical Guarantees. In CIKM. 77--86.","DOI":"10.1145\/3132847.3132895"},{"key":"e_1_3_2_2_7_1","doi-asserted-by":"crossref","unstructured":"Tetsuya Sakai and Ruihua Song. 2011. Evaluating diversified search results using per-intent graded relevance. In SIGIR. 1043--1052.  Tetsuya Sakai and Ruihua Song. 2011. Evaluating diversified search results using per-intent graded relevance. In SIGIR. 1043--1052.","DOI":"10.1145\/2009916.2010055"},{"key":"e_1_3_2_2_8_1","doi-asserted-by":"crossref","unstructured":"Anne Schuth Katja Hofmann and Filip Radlinski. 2015. Predicting search satisfaction metrics with interleaved comparisons. In SIGIR. 463--472.  Anne Schuth Katja Hofmann and Filip Radlinski. 2015. Predicting search satisfaction metrics with interleaved comparisons. In SIGIR. 463--472.","DOI":"10.1145\/2766462.2767695"},{"key":"e_1_3_2_2_9_1","doi-asserted-by":"crossref","unstructured":"Anne Schuth Floor Sietsma Shimon Whiteson Damien Lefortier and Maarten de Rijke. 2014. Multileaved Comparisons for Fast Online Evaluation. In CIKM. 71--80.  Anne Schuth Floor Sietsma Shimon Whiteson Damien Lefortier and Maarten de Rijke. 2014. Multileaved Comparisons for Fast Online Evaluation. In CIKM. 71--80.","DOI":"10.1145\/2661829.2661952"},{"key":"e_1_3_2_2_10_1","doi-asserted-by":"crossref","unstructured":"Andrew Turpin and Falk Scholer. 2006. User Performance versus Precision Measures for Simple Search Tasks. In SIGIR. 11--18.  Andrew Turpin and Falk Scholer. 2006. User Performance versus Precision Measures for Simple Search Tasks. In SIGIR. 11--18.","DOI":"10.1145\/1148170.1148176"},{"key":"e_1_3_2_2_11_1","volume-title":"Turpin and William Hersh","author":"Andrew","year":"2001","unstructured":"Andrew H. Turpin and William Hersh . 2001 . Why Batch and User Evaluations Do Not Give the Same Results. In SIGIR. 225--231. Andrew H. Turpin and William Hersh. 2001. Why Batch and User Evaluations Do Not Give the Same Results. In SIGIR. 225--231."}],"event":{"name":"CIKM '20: The 29th ACM International Conference on Information and Knowledge Management","sponsor":["SIGWEB ACM Special Interest Group on Hypertext, Hypermedia, and Web","SIGIR ACM Special Interest Group on Information Retrieval"],"location":"Virtual Event Ireland","acronym":"CIKM '20"},"container-title":["Proceedings of the 29th ACM International Conference on Information & Knowledge Management"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3340531.3412143","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3340531.3412143","content-type":"application\/pdf","content-version":"vor","intended-application":"syndication"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3340531.3412143","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T22:02:30Z","timestamp":1750197750000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3340531.3412143"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2020,10,19]]},"references-count":11,"alternative-id":["10.1145\/3340531.3412143","10.1145\/3340531"],"URL":"https:\/\/doi.org\/10.1145\/3340531.3412143","relation":{},"subject":[],"published":{"date-parts":[[2020,10,19]]},"assertion":[{"value":"2020-10-19","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}