{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,5,12]],"date-time":"2026-05-12T10:08:41Z","timestamp":1778580521414,"version":"3.51.4"},"publisher-location":"New York, NY, USA","reference-count":27,"publisher":"ACM","license":[{"start":{"date-parts":[[2014,11,3]],"date-time":"2014-11-03T00:00:00Z","timestamp":1414972800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"DOI":"10.13039\/100006112","name":"Microsoft Research","doi-asserted-by":"publisher","id":[{"id":"10.13039\/100006112","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2014,11,3]]},"DOI":"10.1145\/2661829.2661893","type":"proceedings-article","created":{"date-parts":[[2014,11,7]],"date-time":"2014-11-07T17:10:54Z","timestamp":1415380254000},"page":"61-70","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":8,"title":["Designing Test Collections for Comparing Many Systems"],"prefix":"10.1145","author":[{"given":"Tetsuya","family":"Sakai","sequence":"first","affiliation":[{"name":"Waseda University, Tokyo, Japan"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2014,11,3]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"publisher","DOI":"10.1145\/1277741.1277805"},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.5555\/2040317.2040333"},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1145\/1390334.1390445"},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.1145\/1321440.1321530"},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10791-011-9167-7"},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"publisher","DOI":"10.1145\/1935826.1935847"},{"key":"e_1_3_2_1_7_1","volume-title":"Overview of the TREC 2011 web track. In Proceedings of TREC 2011","author":"Clarke C. L. A.","year":"2012","unstructured":"C. L. A. Clarke, N. Craswell, I. Soboroff, and E. M. Voorhees. Overview of the TREC 2011 web track. In Proceedings of TREC 2011, 2012."},{"key":"e_1_3_2_1_8_1","volume-title":"Overview of the TREC 2012 web track. In Proceedings of TREC 2012","author":"Clarke C. L. A.","year":"2013","unstructured":"C. L. A. Clarke, N. Craswell, and E. M. Voorhees. Overview of the TREC 2012 web track. In Proceedings of TREC 2012, 2013."},{"key":"e_1_3_2_1_9_1","volume-title":"Statistical Power Analysis for the Behavioral Sciences","author":"Cohen J.","year":"1988","unstructured":"J. Cohen. Statistical Power Analysis for the Behavioral Sciences (Second Edition). Lawrence Erlbaum Associates, 1988."},{"key":"e_1_3_2_1_10_1","volume-title":"Confidence Intervals, and Meta-Analysis","author":"Cumming G.","year":"2012","unstructured":"G. Cumming. Understanding the New Statistics: Effect Sizes, Confidence Intervals, and Meta-Analysis. Routledge, 2012."},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.1017\/CBO9780511761676"},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.socec.2004.09.035"},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1080\/01621459.1995.10476572"},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.1111\/j.0956-7976.2005.01538.x"},{"key":"e_1_3_2_1_15_1","volume-title":"Asakura Shoten","author":"Nagata Y.","year":"2003","unstructured":"Y. Nagata. How to Design the Sample Size (in Japanese). Asakura Shoten, 2003."},{"key":"e_1_3_2_1_16_1","first-page":"393","volume-title":"Proceedings of CAIS\/ASCI'98","author":"Nelson M. J.","year":"1998","unstructured":"M. J. Nelson. Statistical power and effect size in information retrieval experiments. In Proceedings of CAIS\/ASCI'98, pages 393--400, 1998."},{"key":"e_1_3_2_1_17_1","volume-title":"Keiso Shobo","author":"Okubo M.","year":"2012","unstructured":"M. Okubo and K. Okada. Psychological Statistics to Tell Your Story: Effect Size, Confidence Interval (in Japanese). Keiso Shobo, 2012."},{"key":"e_1_3_2_1_18_1","volume-title":"Proceedings of TREC 2011","author":"Ounis I.","year":"2012","unstructured":"I. Ounis, C. Macdonald, J. Lin, and I. Soboroff. Overview of the TREC-2011 microblog track. In Proceedings of TREC 2011, 2012."},{"key":"e_1_3_2_1_19_1","first-page":"116","volume-title":"PROMISE Winter School 2013: Bridging between Information Retrieval and Databases (LNCS 8173)","author":"Sakai T.","year":"2014","unstructured":"T. Sakai. Metrics, statistics, tests. In PROMISE Winter School 2013: Bridging between Information Retrieval and Databases (LNCS 8173), pages 116--163, 2014."},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.1145\/2641383.2641385"},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.1145\/2009916.2010055"},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10791-012-9208-x"},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.1145\/2484028.2484038"},{"key":"e_1_3_2_1_24_1","volume-title":"Overview of the TREC 2003 robust retrieval track. In Proceeings of TREC 2003","author":"Voorhees E. M.","year":"2004","unstructured":"E. M. Voorhees. Overview of the TREC 2003 robust retrieval track. In Proceeings of TREC 2003, 2004."},{"key":"e_1_3_2_1_25_1","volume-title":"Overview of the TREC 2004 robust retrieval track. In Proceeings of TREC 2004","author":"Voorhees E. M.","year":"2005","unstructured":"E. M. Voorhees. Overview of the TREC 2004 robust retrieval track. In Proceeings of TREC 2004, 2005."},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.1145\/1571941.1572138"},{"key":"e_1_3_2_1_27_1","doi-asserted-by":"publisher","DOI":"10.1145\/1458082.1458158"}],"event":{"name":"CIKM '14: 2014 ACM Conference on Information and Knowledge Management","location":"Shanghai China","acronym":"CIKM '14","sponsor":["SIGWEB ACM Special Interest Group on Hypertext, Hypermedia, and Web","SIGIR ACM Special Interest Group on Information Retrieval"]},"container-title":["Proceedings of the 23rd ACM International Conference on Conference on Information and Knowledge Management"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/2661829.2661893","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/2661829.2661893","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,18]],"date-time":"2025-06-18T19:04:11Z","timestamp":1750273451000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/2661829.2661893"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2014,11,3]]},"references-count":27,"alternative-id":["10.1145\/2661829.2661893","10.1145\/2661829"],"URL":"https:\/\/doi.org\/10.1145\/2661829.2661893","relation":{},"subject":[],"published":{"date-parts":[[2014,11,3]]},"assertion":[{"value":"2014-11-03","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}