{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,1,20]],"date-time":"2026-01-20T06:57:18Z","timestamp":1768892238797,"version":"3.49.0"},"publisher-location":"New York, NY, USA","reference-count":25,"publisher":"ACM","license":[{"start":{"date-parts":[[2020,10,19]],"date-time":"2020-10-19T00:00:00Z","timestamp":1603065600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2020,10,19]]},"DOI":"10.1145\/3340531.3411998","type":"proceedings-article","created":{"date-parts":[[2020,10,19]],"date-time":"2020-10-19T06:18:51Z","timestamp":1603088331000},"page":"1933-1952","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":9,"title":["Corpus Bootstrapping for Assessment of the Properties of Effectiveness Measures"],"prefix":"10.1145","author":[{"given":"Justin","family":"Zobel","sequence":"first","affiliation":[{"name":"University of Melbourne, Parkville, VIC, Australia"}]},{"given":"Lida","family":"Rashidi","sequence":"additional","affiliation":[{"name":"University of Melbourne, Parkville, VIC, Australia"}]}],"member":"320","published-online":{"date-parts":[[2020,10,19]]},"reference":[{"key":"e_1_3_2_2_1_1","doi-asserted-by":"publisher","DOI":"10.1145\/2094072.2094076"},{"key":"e_1_3_2_2_2_1","volume-title":"Proceedings of the 29th International ACM SIGIR Conference on Research and Development in Information Retrieval. ACM, 533--540","author":"Cormack G. V.","unstructured":"G. V. Cormack and T. R. Lynam . 2006. Statistical precision of information retrieval evaluation . In Proceedings of the 29th International ACM SIGIR Conference on Research and Development in Information Retrieval. ACM, 533--540 . G. V. Cormack and T. R. Lynam. 2006. Statistical precision of information retrieval evaluation. In Proceedings of the 29th International ACM SIGIR Conference on Research and Development in Information Retrieval. ACM, 533--540."},{"key":"e_1_3_2_2_3_1","doi-asserted-by":"crossref","unstructured":"B. Efron and R. Tibshirani. 1986. Bootstrap methods for standard errors confidence intervals and other measures of statistical accuracy. Statist. Sci. (1986) 54--75.  B. Efron and R. Tibshirani. 1986. Bootstrap methods for standard errors confidence intervals and other measures of statistical accuracy. Statist. Sci. (1986) 54--75.","DOI":"10.1214\/ss\/1177013815"},{"key":"e_1_3_2_2_4_1","doi-asserted-by":"crossref","unstructured":"B. Efron and R. J. Tibshirani. 1994. An introduction to the bootstrap. CRC press.  B. Efron and R. J. Tibshirani. 1994. An introduction to the bootstrap. CRC press.","DOI":"10.1201\/9780429246593"},{"key":"e_1_3_2_2_5_1","doi-asserted-by":"publisher","DOI":"10.1145\/3190580.3190586"},{"key":"e_1_3_2_2_6_1","volume-title":"ACM Trans. on Information Systems (TOIS)","volume":"20","author":"K.","year":"2002","unstructured":"K. J\"arvelin and J. Kek\"al\"ainen. 2002 . Cumulated gain-based evaluation of IR techniques . ACM Trans. on Information Systems (TOIS) , Vol. 20 , 4 (2002), 422--446. K. J\"arvelin and J. Kek\"al\"ainen. 2002. Cumulated gain-based evaluation of IR techniques. ACM Trans. on Information Systems (TOIS), Vol. 20, 4 (2002), 422--446."},{"key":"e_1_3_2_2_7_1","volume-title":"Proceedings of the 17th Australasian Document Computing Symposium. 47--54","author":"Moffat A.","unstructured":"A. Moffat , F. Scholer , and P. Thomas . 2012. Models and metrics: IR evaluation as a user process . In Proceedings of the 17th Australasian Document Computing Symposium. 47--54 . A. Moffat, F. Scholer, and P. Thomas. 2012. Models and metrics: IR evaluation as a user process. In Proceedings of the 17th Australasian Document Computing Symposium. 47--54."},{"key":"e_1_3_2_2_8_1","doi-asserted-by":"publisher","DOI":"10.1145\/1416950.1416952"},{"key":"e_1_3_2_2_9_1","unstructured":"C. J. Van Rijsbergen. 1979. Information Retrieval.  C. J. Van Rijsbergen. 1979. Information Retrieval."},{"key":"e_1_3_2_2_10_1","doi-asserted-by":"publisher","DOI":"10.1145\/1183614.1183630"},{"key":"e_1_3_2_2_11_1","volume-title":"Proceedings of the 35th International ACM SIGIR Conference on Research and Development in Information Retrieval. ACM, 891--900","author":"Robertson S. E.","unstructured":"S. E. Robertson and E. Kanoulas . 2012. On per-topic variance in IR evaluation . In Proceedings of the 35th International ACM SIGIR Conference on Research and Development in Information Retrieval. ACM, 891--900 . S. E. Robertson and E. Kanoulas. 2012. On per-topic variance in IR evaluation. In Proceedings of the 35th International ACM SIGIR Conference on Research and Development in Information Retrieval. ACM, 891--900."},{"key":"e_1_3_2_2_12_1","doi-asserted-by":"publisher","DOI":"10.1145\/1148170.1148261"},{"key":"e_1_3_2_2_13_1","volume-title":"Laboratory experiments in information retrieval: Sample sizes, effect sizes, and statistical power","author":"Sakai T.","unstructured":"T. Sakai . 2018. Laboratory experiments in information retrieval: Sample sizes, effect sizes, and statistical power . Vol. 40 . Springer . T. Sakai. 2018. Laboratory experiments in information retrieval: Sample sizes, effect sizes, and statistical power. Vol. 40. Springer."},{"key":"e_1_3_2_2_14_1","doi-asserted-by":"publisher","DOI":"10.1145\/3451964.3451976"},{"key":"e_1_3_2_2_15_1","doi-asserted-by":"publisher","DOI":"10.1561\/1500000009"},{"key":"e_1_3_2_2_16_1","volume-title":"Proceedings of the 28th International ACM SIGIR Conference on Research and Development in Information Retrieval. ACM, 162--169","author":"Sanderson M.","unstructured":"M. Sanderson and J. Zobel . 2005. Information retrieval system evaluation: effort, sensitivity, and reliability . In Proceedings of the 28th International ACM SIGIR Conference on Research and Development in Information Retrieval. ACM, 162--169 . M. Sanderson and J. Zobel. 2005. Information retrieval system evaluation: effort, sensitivity, and reliability. In Proceedings of the 28th International ACM SIGIR Conference on Research and Development in Information Retrieval. ACM, 162--169."},{"key":"e_1_3_2_2_17_1","doi-asserted-by":"publisher","DOI":"10.1016\/S0306-4573(97)00027-7"},{"key":"e_1_3_2_2_18_1","volume-title":"Proceedings of the 16th ACM Conference on Information and Knowledge Management. ACM, 623--632","author":"Smucker M. D.","unstructured":"M. D. Smucker , J. Allan , and B. Carterette . 2007. A comparison of statistical significance tests for information retrieval evaluation . In Proceedings of the 16th ACM Conference on Information and Knowledge Management. ACM, 623--632 . M. D. Smucker, J. Allan, and B. Carterette. 2007. A comparison of statistical significance tests for information retrieval evaluation. In Proceedings of the 16th ACM Conference on Information and Knowledge Management. ACM, 623--632."},{"key":"e_1_3_2_2_19_1","doi-asserted-by":"publisher","DOI":"10.1145\/1008992.1009105"},{"key":"e_1_3_2_2_20_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10791-015-9274-y"},{"key":"e_1_3_2_2_21_1","doi-asserted-by":"publisher","DOI":"10.1016\/S0306-4573(00)00010-8"},{"key":"e_1_3_2_2_22_1","volume-title":"Proceedings of the 25th International ACM SIGIR Conference on Research and Development in Information Retrieval. ACM, 316--323","author":"Voorhees E. M.","unstructured":"E. M. Voorhees and C. Buckley . 2002. The effect of topic set size on retrieval experiment error . In Proceedings of the 25th International ACM SIGIR Conference on Research and Development in Information Retrieval. ACM, 316--323 . E. M. Voorhees and C. Buckley. 2002. The effect of topic set size on retrieval experiment error. In Proceedings of the 25th International ACM SIGIR Conference on Research and Development in Information Retrieval. ACM, 316--323."},{"key":"e_1_3_2_2_23_1","volume-title":"TREC: Experiment and evaluation in information retrieval.","author":"Voorhees E. M.","year":"2005","unstructured":"E. M. Voorhees , D. K. Harman , 2005 . TREC: Experiment and evaluation in information retrieval. Vol. 63 . MIT press Cambridge . E. M. Voorhees, D. K. Harman, et al. 2005. TREC: Experiment and evaluation in information retrieval. Vol. 63. MIT press Cambridge."},{"key":"e_1_3_2_2_24_1","volume-title":"Proceedings of the 31st International ACM SIGIR Conference on Research and Development in Information Retrieval. ACM, 695--696","author":"Webber W.","unstructured":"W. Webber , A. Moffat , J. Zobel , and T. Sakai . 2008. Precision-at-ten considered redundant . In Proceedings of the 31st International ACM SIGIR Conference on Research and Development in Information Retrieval. ACM, 695--696 . W. Webber, A. Moffat, J. Zobel, and T. Sakai. 2008. Precision-at-ten considered redundant. In Proceedings of the 31st International ACM SIGIR Conference on Research and Development in Information Retrieval. ACM, 695--696."},{"key":"e_1_3_2_2_25_1","doi-asserted-by":"publisher","DOI":"10.1145\/290941.291014"}],"event":{"name":"CIKM '20: The 29th ACM International Conference on Information and Knowledge Management","location":"Virtual Event Ireland","acronym":"CIKM '20","sponsor":["SIGWEB ACM Special Interest Group on Hypertext, Hypermedia, and Web","SIGIR ACM Special Interest Group on Information Retrieval"]},"container-title":["Proceedings of the 29th ACM International Conference on Information & Knowledge Management"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3340531.3411998","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3340531.3411998","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T22:01:23Z","timestamp":1750197683000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3340531.3411998"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2020,10,19]]},"references-count":25,"alternative-id":["10.1145\/3340531.3411998","10.1145\/3340531"],"URL":"https:\/\/doi.org\/10.1145\/3340531.3411998","relation":{},"subject":[],"published":{"date-parts":[[2020,10,19]]},"assertion":[{"value":"2020-10-19","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}