{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,8,20]],"date-time":"2026-08-20T15:02:47Z","timestamp":1787238167962,"version":"3.56.0"},"reference-count":47,"publisher":"Springer Science and Business Media LLC","issue":"3","license":[{"start":{"date-parts":[[2015,10,27]],"date-time":"2015-10-27T00:00:00Z","timestamp":1445904000000},"content-version":"unspecified","delay-in-days":0,"URL":"http:\/\/creativecommons.org\/licenses\/by\/4.0"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Inf Retrieval J"],"published-print":{"date-parts":[[2016,6]]},"DOI":"10.1007\/s10791-015-9273-z","type":"journal-article","created":{"date-parts":[[2015,10,27]],"date-time":"2015-10-27T07:13:12Z","timestamp":1445929992000},"page":"256-283","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":38,"title":["Topic set size design"],"prefix":"10.1007","volume":"19","author":[{"given":"Tetsuya","family":"Sakai","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2015,10,27]]},"reference":[{"key":"9273_CR1","doi-asserted-by":"crossref","unstructured":"Bodoff, D., & Li, P. (2007). Test theory for assessing IR test collections. In Proceedings of ACM SIGIR 2007 (pp. 367\u2013374).","DOI":"10.1145\/1277741.1277805"},{"key":"9273_CR2","doi-asserted-by":"crossref","unstructured":"Boytsov, L., Belova, A., & Westfall, P. (2013). Deciding on an adjustment for multiplicity in IR experiments. In Proceedings of ACM SIGIR 2013 (pp. 403\u2013412).","DOI":"10.1145\/2484028.2484034"},{"key":"9273_CR3","doi-asserted-by":"crossref","unstructured":"Carterette, B. (2011). Model-based inference about IR systems. In ICTIR 2011 (LNCS 6931) (pp. 101\u2013112).","DOI":"10.1007\/978-3-642-23318-0_11"},{"key":"9273_CR4","doi-asserted-by":"publisher","unstructured":"Carterette, B. (2012). Multiple testing in statistical analysis of systems-based information retrieval experiments. ACM TOIS. doi: 10.1145\/2094072.2094076 .","DOI":"10.1145\/2094072.2094076"},{"key":"9273_CR5","doi-asserted-by":"crossref","unstructured":"Carterette, B., & Smucker, M. D. (2007). Hypothesis testing with incomplete relevance judgments. In Proceedings of ACM CIKM 2007 (pp. 643\u2013652).","DOI":"10.1145\/1321440.1321530"},{"key":"9273_CR6","doi-asserted-by":"crossref","unstructured":"Carterette, B., Pavlu, V., Kanoulas, E., Aslam, J. A., & Allan, J. (2008). Evaluation over thousands of queries. In Proceedings of ACM SIGIR 2008 (pp. 651\u2013658).","DOI":"10.1145\/1390334.1390445"},{"issue":"6","key":"9273_CR7","doi-asserted-by":"crossref","first-page":"572","DOI":"10.1007\/s10791-011-9167-7","volume":"14","author":"O Chapelle","year":"2011","unstructured":"Chapelle, O., Ji, S., Liao, C., Velipasaoglu, E., Lai, L., & Wu, S. L. (2011). Intent-based diversification of web search results: Metrics and algorithms. Information Retrieval, 14(6), 572\u2013592.","journal-title":"Information Retrieval"},{"key":"9273_CR8","unstructured":"Clarke, C. L. A., Kolla, M., Cormack, G. V., Vechtomova, O., Ashkan, A., B\u00fcttcher, S., et al. (2009). Novelty and diversity in information retrieval evaluation. In Proceedings of ACM SIGIR 2008 (pp. 659\u2013666)."},{"key":"9273_CR9","unstructured":"Clarke, C. L. A., Craswell, N., Soboroff, I., & Voorhees, E. M. (2012). Overview of the TREC 2011 web track. In Proceedings of TREC 2011."},{"key":"9273_CR10","unstructured":"Clarke, C. L. A., Craswell, N., & Voorhees, E. M. (2013). Overview of the TREC 2012 web track. In Proceedings of TREC 2012."},{"key":"9273_CR11","volume-title":"Statistical power analysis for the behavioral sciences","author":"J Cohen","year":"1988","unstructured":"Cohen, J. (1988). Statistical power analysis for the behavioral sciences (2nd ed.). London: Lawrence Erlbaum Associates.","edition":"2"},{"issue":"1","key":"9273_CR12","doi-asserted-by":"crossref","first-page":"1","DOI":"10.20982\/tqmp.07.1.p001","volume":"7","author":"D Cousineau","year":"2011","unstructured":"Cousineau, D., & Laurencelle, L. (2011). Non-central t distribution and the power of the t test: A rejoinder. Tutorials in Quantitative Methods for Psychology, 7(1), 1\u20134.","journal-title":"Tutorials in Quantitative Methods for Psychology"},{"key":"9273_CR13","volume-title":"Understanding the new statistics: Effect sizes, confidence intervals, and meta-analysis","author":"G Cumming","year":"2012","unstructured":"Cumming, G. (2012). Understanding the new statistics: Effect sizes, confidence intervals, and meta-analysis. London: Routledge."},{"key":"9273_CR14","doi-asserted-by":"crossref","DOI":"10.1017\/CBO9780511761676","volume-title":"The essential guide to effect sizes","author":"PD Ellis","year":"2010","unstructured":"Ellis, P. D. (2010). The essential guide to effect sizes. Cambridge: Cambridge University Press."},{"key":"9273_CR15","doi-asserted-by":"crossref","first-page":"87","DOI":"10.2307\/2340521","volume":"85","author":"RA Fisher","year":"1922","unstructured":"Fisher, R. A. (1922). On the interpretation of $$\\chi ^2$$ \u03c7 2 from contingency tables and calculation of p. Journal of the Royal Statistical Society, Series A, 85, 87\u201394.","journal-title":"Journal of the Royal Statistical Society, Series A"},{"key":"9273_CR16","unstructured":"Gilbert, H., & Jones, K. S. (1979). Statistical bases of relevance assessment for the \u2018IDEAL\u2019 information retrieval test collection. In Tech. rep., Computer Laboratory, University of Cambridge, British Library Research and Development Report No. 5481."},{"issue":"4","key":"9273_CR17","doi-asserted-by":"crossref","first-page":"422","DOI":"10.1145\/582415.582418","volume":"20","author":"K J\u00e4rvelin","year":"2002","unstructured":"J\u00e4rvelin, K., & Kek\u00e4l\u00e4inen, J. (2002). Cumulated gain-based evaluation of IR techniques. ACM TOIS, 20(4), 422\u2013446.","journal-title":"ACM TOIS"},{"key":"9273_CR18","doi-asserted-by":"crossref","first-page":"362","DOI":"10.1093\/biomet\/31.3-4.362","volume":"31","author":"NL Johnson","year":"1940","unstructured":"Johnson, N. L., & Welch, B. L. (1940). Applications of the non-central t-distribution. Biometrika, 31, 362\u2013389.","journal-title":"Biometrika"},{"key":"9273_CR38","unstructured":"Jones, K. S., & Van Rijsbergen, C. J. (1975) Report on the need for and provision of an \u2018ideal\u2019 information retrieval test collection. Tech. rep., Computer Laboratory, University of Cambridge, British Library Research and Development Report No. 5266."},{"key":"9273_CR39","volume-title":"Readings in information retrieval","year":"1997","unstructured":"Jones, K. S., Willet, P., et al. (Eds.). (1997). Readings in information retrieval. Los Altos: Morgan Kaufmann."},{"issue":"430","key":"9273_CR19","doi-asserted-by":"crossref","first-page":"773","DOI":"10.1080\/01621459.1995.10476572","volume":"90","author":"RE Kass","year":"1995","unstructured":"Kass, R. E., & Raftery, A. E. (1995). Bayes factors. Journal of the American Statistical Association, 90(430), 773\u2013795.","journal-title":"Journal of the American Statistical Association"},{"issue":"1\u20132","key":"9273_CR20","first-page":"1","volume":"3","author":"D Kelly","year":"2009","unstructured":"Kelly, D. (2009). Methods for evaluating interactive information retrieval systems with users. Foundation and Trends in Information Retrieval, 3(1\u20132), 1\u2013224.","journal-title":"Foundation and Trends in Information Retrieval"},{"key":"9273_CR21","doi-asserted-by":"crossref","first-page":"345","DOI":"10.1111\/j.0956-7976.2005.01538.x","volume":"16","author":"PR Killeen","year":"2005","unstructured":"Killeen, P. R. (2005). An alternative to null hypothesis significance tests. Psychological Science, 16, 345\u2013353.","journal-title":"Psychological Science"},{"issue":"2","key":"9273_CR22","doi-asserted-by":"crossref","first-page":"93","DOI":"10.5023\/jappstat.27.93","volume":"27","author":"Y Nagata","year":"1998","unstructured":"Nagata, Y. (1998). How to use multiple comparison procedures. Japanese Journal of Applied Statistics, 27(2), 93\u2013108. (in Japanese).","journal-title":"Japanese Journal of Applied Statistics"},{"key":"9273_CR23","volume-title":"How to design the sample size","author":"Y Nagata","year":"2003","unstructured":"Nagata, Y. (2003). How to design the sample size. Tokyo: Asakura Shoten. (in Japanese)."},{"key":"9273_CR24","unstructured":"Nelson, M. J. (1998). Statistical power and effect size in information retrieval experiments. In Proceedings of CAIS\/ASCI\u201998 (pp. 393\u2013400)."},{"key":"9273_CR25","volume-title":"Psychological statistics to tell your story: Effect size, confidence interval","author":"M Okubo","year":"2012","unstructured":"Okubo, M., & Okada, K. (2012). Psychological statistics to tell your story: Effect size, confidence interval. Tokyo: Keiso Shobo. (in Japanese)."},{"key":"9273_CR26","unstructured":"Ounis, I., Macdonald, C., Lin, J., & Soboroff, I. (2012). Overview of the TREC-2011 microblog track. In Proceedings of TREC 2011."},{"key":"9273_CR27","first-page":"202","volume":"36","author":"PB Patnaik","year":"1949","unstructured":"Patnaik, P. B. (1949). The non-central $$\\chi ^2$$ \u03c7 2 - and $${F}$$ F -distributions and their applications. Biometrika, 36, 202\u2013232.","journal-title":"Biometrika"},{"key":"9273_CR28","doi-asserted-by":"crossref","unstructured":"Sakai, T. (2005). Ranking the NTCIR systems based on multigrade relevance. In Proceedings of AIRS 2004 (LNCS 3411) (pp. 251\u2013262).","DOI":"10.1007\/978-3-540-31871-2_22"},{"key":"9273_CR29","doi-asserted-by":"crossref","unstructured":"Sakai, T. (2006). Evaluating evaluation metrics based on the bootstrap. In Proceedings of ACM SIGIR 2006 (pp. 525\u2013532).","DOI":"10.1145\/1148170.1148261"},{"key":"9273_CR30","doi-asserted-by":"crossref","unstructured":"Sakai, T. (2014a). Designing test collections for comparing many systems. In Proceedings of ACM CIKM 2014 (pp. 61\u201370).","DOI":"10.1145\/2661829.2661893"},{"key":"9273_CR31","unstructured":"Sakai, T. (2014b). Designing test collections that provide tight confidence intervals. In Forum on information technology 2014 RD-003 (Vol. 2, pp.15\u201318)."},{"key":"9273_CR32","doi-asserted-by":"crossref","unstructured":"Sakai, T. (2014c). Metrics, statistics, tests. In PROMISE Winter School 2013: Bridging between information retrieval and databases (LNCS 8173) (pp. 116\u2013163).","DOI":"10.1007\/978-3-642-54798-0_6"},{"issue":"1","key":"9273_CR33","doi-asserted-by":"crossref","first-page":"3","DOI":"10.1145\/2641383.2641385","volume":"48","author":"T Sakai","year":"2014","unstructured":"Sakai, T. (2014d). Statistical reform in information retrieval? SIGIR Forum, 48(1), 3\u201312.","journal-title":"SIGIR Forum"},{"key":"9273_CR34","unstructured":"Sakai, T. (2014e). Topic set size design with variance estimates from two-way ANOVA. In Proceedings of EVIA 2014 (pp. 1\u20138)."},{"key":"9273_CR35","doi-asserted-by":"crossref","unstructured":"Sakai, T., & Song, R. (2011). Evaluating diversified search results using per-intent graded relevance. In Proceedings of ACM SIGIR 2011 (pp. 1043\u20131042).","DOI":"10.1145\/2009916.2010055"},{"key":"9273_CR36","doi-asserted-by":"crossref","unstructured":"Smucker, M. D., Allan, J., & Carterette, B. (2007). A comparison of statistical significance tests for information retrieval evaluation. In Proceedings of ACM CIKM 2007 (pp. 623\u2013632).","DOI":"10.1145\/1321440.1321528"},{"key":"9273_CR37","unstructured":"Soboroff, I. (2014). Computing confidence intervals for common IR measures. In Proceedings of EVIA 2014 (pp. 25\u201328)."},{"key":"9273_CR40","doi-asserted-by":"crossref","unstructured":"Urbano, J., Marrero, M., & Mart\u00edn, D. (2013). On the measurement of test collection reliability. In Proceedings of ACM SIGIR 2013 (pp. 393\u2013402).","DOI":"10.1145\/2484028.2484038"},{"key":"9273_CR41","volume-title":"Information retrieval","author":"CJ Rijsbergen Van","year":"1979","unstructured":"Van Rijsbergen, C. J. (1979). Information retrieval (2nd ed.). London: Butterworths.","edition":"2"},{"key":"9273_CR42","unstructured":"Voorhees, E. M. (2004). Overview of the TREC 2003 robust retrieval track. In Proceeings of TREC 2003."},{"key":"9273_CR43","unstructured":"Voorhees, E.M. (2005). Overview of the TREC 2004 robust retrieval track. In Proceeings of TREC 2004"},{"key":"9273_CR44","doi-asserted-by":"crossref","unstructured":"Voorhees, E. M. (2009). Topic set size redux. In Proceedings of ACM SIGIR 2009 (pp. 806\u2013807).","DOI":"10.1145\/1571941.1572138"},{"key":"9273_CR45","doi-asserted-by":"crossref","unstructured":"Webber, W., Moffat, A., & Zobel, J. (2008). Score standardization for inter-collection comparison of retrieval systems. In Proceedings of ACM SIGIR 2008 (pp. 51\u201358).","DOI":"10.1145\/1390334.1390346"},{"key":"9273_CR46","doi-asserted-by":"crossref","unstructured":"Webber, W., Moffat, A., & Zobel, J. (2008). Statistical power in retrieval experimentation. In Proceedings of ACM CIKM 2008 (pp. 571\u2013580).","DOI":"10.1145\/1458082.1458158"},{"key":"9273_CR47","doi-asserted-by":"crossref","unstructured":"Zobel, J. (1998). How reliable are the results of large-scale information retrieval experiments? In Proceedings of ACM SIGIR 1998 (pp. 307\u2013314).","DOI":"10.1145\/290941.291014"}],"container-title":["Information Retrieval Journal"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s10791-015-9273-z.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/article\/10.1007\/s10791-015-9273-z\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s10791-015-9273-z","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2019,3,26]],"date-time":"2019-03-26T15:06:23Z","timestamp":1553612783000},"score":1,"resource":{"primary":{"URL":"http:\/\/link.springer.com\/10.1007\/s10791-015-9273-z"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2015,10,27]]},"references-count":47,"journal-issue":{"issue":"3","published-print":{"date-parts":[[2016,6]]}},"alternative-id":["9273"],"URL":"https:\/\/doi.org\/10.1007\/s10791-015-9273-z","relation":{},"ISSN":["1386-4564","1573-7659"],"issn-type":[{"value":"1386-4564","type":"print"},{"value":"1573-7659","type":"electronic"}],"subject":[],"published":{"date-parts":[[2015,10,27]]}}}