{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,6,18]],"date-time":"2025-06-18T04:30:30Z","timestamp":1750221030854,"version":"3.41.0"},"publisher-location":"New York, NY, USA","reference-count":40,"publisher":"ACM","license":[{"start":{"date-parts":[[2019,9,26]],"date-time":"2019-09-26T00:00:00Z","timestamp":1569456000000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"DOI":"10.13039\/100000001","name":"NSF","doi-asserted-by":"publisher","award":["IIS-1350799"],"award-info":[{"award-number":["IIS-1350799"]}],"id":[{"id":"10.13039\/100000001","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2019,9,26]]},"DOI":"10.1145\/3341981.3358959","type":"proceedings-article","created":{"date-parts":[[2019,9,27]],"date-time":"2019-09-27T12:34:07Z","timestamp":1569587647000},"page":"257-259","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":1,"title":["Statistical Significance Testing in Theory and in Practice"],"prefix":"10.1145","author":[{"given":"Ben","family":"Carterette","sequence":"first","affiliation":[{"name":"Spotify, New York, NY, USA"}]}],"member":"320","published-online":{"date-parts":[[2019,9,26]]},"reference":[{"key":"e_1_3_2_1_1_1","volume-title":"Report on the sigir 2015 workshop on reproducibility, inexplicability, nd generalizability of results (rigor). SIGIR Forum, 49(2)","author":"Arguello Jaime","year":"2015","unstructured":"Jaime Arguello , Matt Crane , Fernando Diaz , Jimmy Lin , and Andrew Trotman . Report on the sigir 2015 workshop on reproducibility, inexplicability, nd generalizability of results (rigor). SIGIR Forum, 49(2) , 2015 . Jaime Arguello, Matt Crane, Fernando Diaz, Jimmy Lin, and Andrew Trotman. Report on the sigir 2015 workshop on reproducibility, inexplicability, nd generalizability of results (rigor). SIGIR Forum, 49(2), 2015."},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_2_1","DOI":"10.1145\/1645953.1646031"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_3_1","DOI":"10.1214\/ss\/1056397485"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_4_1","DOI":"10.1145\/2484028.2484034"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_5_1","DOI":"10.5555\/2040317.2040333"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_6_1","DOI":"10.1145\/2094072.2094076"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_7_1","DOI":"10.1145\/2499178.2499204"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_8_1","DOI":"10.1145\/2600428.2602292"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_9_1","DOI":"10.1145\/2808194.2809469"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_10_1","DOI":"10.1145\/2808194.2809445"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_11_1","DOI":"10.1145\/3077136.3082065"},{"key":"e_1_3_2_1_12_1","volume-title":"Proceedings of SIGIR","author":"Carterette Ben","year":"2010","unstructured":"Ben Carterette , Evangelos Kanoulas , Virgil Pavlu , and Hui Fang . Building reusable test collections through experimental design . In Proceedings of SIGIR , 2010 . Ben Carterette, Evangelos Kanoulas, Virgil Pavlu, and Hui Fang. Building reusable test collections through experimental design. In Proceedings of SIGIR, 2010."},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_13_1","DOI":"10.1145\/2063576.2063668"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_14_1","DOI":"10.1145\/2396761.2396782"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_15_1","DOI":"10.1145\/1321440.1321530"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_16_1","DOI":"10.1145\/1148170.1148262"},{"key":"e_1_3_2_1_17_1","first-page":"8","article-title":"Reproducibility challenges in information retrieval","author":"Ferro Nicola","year":"2017","unstructured":"Nicola Ferro . Reproducibility challenges in information retrieval . Journal of Data and Information Quality , 8 , 2017 . Nicola Ferro. Reproducibility challenges in information retrieval. Journal of Data and Information Quality, 8, 2017.","journal-title":"Journal of Data and Information Quality"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_18_1","DOI":"10.1145\/2911451.2911530"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_19_1","DOI":"10.1007\/978-3-642-15998-5_14"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_20_1","DOI":"10.1001\/jama.294.2.218"},{"key":"e_1_3_2_1_21_1","volume-title":"Why most published research findings are false. PLoS Medicine, 2(8)","author":"Ioannidis John P. A.","year":"2005","unstructured":"John P. A. Ioannidis . Why most published research findings are false. PLoS Medicine, 2(8) , 2005 . John P. A. Ioannidis. Why most published research findings are false. PLoS Medicine, 2(8), 2005."},{"key":"e_1_3_2_1_23_1","volume-title":"Buttersworth","author":"Jones Karen Sparck","year":"1981","unstructured":"Karen Sparck Jones , editor. Information Retrieval Experiment . Buttersworth , 1981 . Karen Sparck Jones, editor. Information Retrieval Experiment. Buttersworth, 1981."},{"key":"e_1_3_2_1_24_1","volume-title":"Readings in Information Retrieval","author":"Jones Karen Sparck","year":"1997","unstructured":"Karen Sparck Jones and Peter Willett , editors. Readings in Information Retrieval . Morgan Kaufmann Publishers , 1997 . Karen Sparck Jones and Peter Willett, editors. Readings in Information Retrieval. Morgan Kaufmann Publishers, 1997."},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_25_1","DOI":"10.1007\/978-3-540-30480-7_1"},{"key":"e_1_3_2_1_26_1","first-page":"506","article-title":"Statistical errors","author":"Nuzzo Regina","year":"2014","unstructured":"Regina Nuzzo . Statistical errors . Nature News , 506 , 2014 . Regina Nuzzo. Statistical errors. Nature News, 506, 2014.","journal-title":"Nature News"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_27_1","DOI":"10.1145\/2970398.2970443"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_28_1","DOI":"10.1561\/1500000009"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_29_1","DOI":"10.1145\/1076034.1076064"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_30_1","DOI":"10.1145\/1321440.1321528"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_31_1","DOI":"10.1145\/1571941.1572050"},{"doi-asserted-by":"crossref","unstructured":"Victoria Stodden Marcia McNutt David H. Bailey Ewa Deelman Yolanda Gil Brooks Hanson Michael A. Heroux John P. A. Ioannidis and Michela Taufer. Enhancing reproducibility for computational methods. Science 354 016.  Victoria Stodden Marcia McNutt David H. Bailey Ewa Deelman Yolanda Gil Brooks Hanson Michael A. Heroux John P. A. Ioannidis and Michela Taufer. Enhancing reproducibility for computational methods. Science 354 016.","key":"e_1_3_2_1_32_1","DOI":"10.1126\/science.aah6168"},{"key":"e_1_3_2_1_33_1","first-page":"59","volume-title":"Jones [23]","author":"Tague Jean","unstructured":"Jean Tague . The pragmatics of information retrieval evaluation . In Jones [23] , pages 59 -- 102 . Jean Tague. The pragmatics of information retrieval evaluation. In Jones [23], pages 59--102."},{"key":"e_1_3_2_1_34_1","first-page":"205","volume-title":"Jones and Willett [24]","author":"Tague-Sutcliffe Jean","unstructured":"Jean Tague-Sutcliffe . The pragmatics of information retrieval evaluation revisited . In Jones and Willett [24] , pages 205 -- 216 . Jean Tague-Sutcliffe. The pragmatics of information retrieval evaluation revisited. In Jones and Willett [24], pages 205--216."},{"key":"e_1_3_2_1_35_1","first-page":"385","volume-title":"Proceedings of the 3rd Text REtrieval Conference (TREC)","author":"Tague-Sutcliffe Jean","year":"1994","unstructured":"Jean Tague-Sutcliffe and James Blustein . A statistical analysis of the TREC-3 data . In Proceedings of the 3rd Text REtrieval Conference (TREC) , pages 385 -- 399 , 1994 . Jean Tague-Sutcliffe and James Blustein. A statistical analysis of the TREC-3 data. In Proceedings of the 3rd Text REtrieval Conference (TREC), pages 385--399, 1994."},{"key":"e_1_3_2_1_36_1","volume-title":"Proceedings of SIGIR","author":"Urbano Juli\u00e1n","year":"2013","unstructured":"Juli\u00e1n Urbano , M\u00f3nica Marrero , and Diego Mart\u00edn . Acomparison of the optimality of statistical significance tests for information retrieval evaluation . In Proceedings of SIGIR , 2013 . Juli\u00e1n Urbano, M\u00f3nica Marrero, and Diego Mart\u00edn. Acomparison of the optimality of statistical significance tests for information retrieval evaluation. In Proceedings of SIGIR, 2013."},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_37_1","DOI":"10.1145\/290941.291017"},{"key":"e_1_3_2_1_38_1","first-page":"316","volume-title":"Proceedings of SIGIR","author":"Ellen","year":"2002","unstructured":"Ellen M. Voorhees and Chris Buckley. The effect of topic set size on retrieval experiment error . In Proceedings of SIGIR , pages 316 -- 323 , 2002 . Ellen M. Voorhees and Chris Buckley. The effect of topic set size on retrieval experiment error. In Proceedings of SIGIR, pages 316--323, 2002."},{"key":"e_1_3_2_1_39_1","volume-title":"TREC: Experiments and evaluation in information retrieval","author":"Voorhees Ellen M.","year":"2005","unstructured":"Ellen M. Voorhees and Donna K. Harman . TREC: Experiments and evaluation in information retrieval . The MIT Press , 2005 . Ellen M. Voorhees and Donna K. Harman. TREC: Experiments and evaluation in information retrieval. The MIT Press, 2005."},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_40_1","DOI":"10.1145\/1458082.1458158"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_41_1","DOI":"10.1145\/290941.291014"}],"event":{"sponsor":["SIGIR ACM Special Interest Group on Information Retrieval"],"acronym":"ICTIR '19","name":"ICTIR '19: The 2019 ACM SIGIR International Conference on the Theory of Information Retrieval","location":"Santa Clara CA USA"},"container-title":["Proceedings of the 2019 ACM SIGIR International Conference on Theory of Information Retrieval"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3341981.3358959","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3341981.3358959","content-type":"application\/pdf","content-version":"vor","intended-application":"syndication"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3341981.3358959","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,18]],"date-time":"2025-06-18T00:43:25Z","timestamp":1750207405000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3341981.3358959"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2019,9,26]]},"references-count":40,"alternative-id":["10.1145\/3341981.3358959","10.1145\/3341981"],"URL":"https:\/\/doi.org\/10.1145\/3341981.3358959","relation":{},"subject":[],"published":{"date-parts":[[2019,9,26]]},"assertion":[{"value":"2019-09-26","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}