{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,6,19]],"date-time":"2025-06-19T04:08:56Z","timestamp":1750306136479,"version":"3.41.0"},"publisher-location":"New York, NY, USA","reference-count":40,"publisher":"ACM","license":[{"start":{"date-parts":[[2017,8,7]],"date-time":"2017-08-07T00:00:00Z","timestamp":1502064000000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"DOI":"10.13039\/100000001","name":"NSF","doi-asserted-by":"publisher","award":["IIS-1350799"],"award-info":[{"award-number":["IIS-1350799"]}],"id":[{"id":"10.13039\/100000001","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2017,8,7]]},"DOI":"10.1145\/3077136.3082065","type":"proceedings-article","created":{"date-parts":[[2017,7,28]],"date-time":"2017-07-28T19:35:01Z","timestamp":1501270501000},"page":"1387-1389","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":6,"title":["Statistical Significance Testing in Information Retrieval"],"prefix":"10.1145","author":[{"given":"Ben","family":"Carterette","sequence":"first","affiliation":[{"name":"University of Delaware, Newark, DE, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2017,8,7]]},"reference":[{"key":"e_1_3_2_1_1_1","volume-title":"Report on the sigir 2015 workshop on reproducibility, inexplicability, and generalizability of results (rigor). SIGIR Forum, 49(2)","author":"Arguello Jaime","year":"2015","unstructured":"Jaime Arguello , Matt Crane , Fernando Diaz , Jimmy Lin , and Andrew Trotman . Report on the sigir 2015 workshop on reproducibility, inexplicability, and generalizability of results (rigor). SIGIR Forum, 49(2) , 2015 . Jaime Arguello, Matt Crane, Fernando Diaz, Jimmy Lin, and Andrew Trotman. Report on the sigir 2015 workshop on reproducibility, inexplicability, and generalizability of results (rigor). SIGIR Forum, 49(2), 2015."},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.1145\/1645953.1646031"},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1214\/ss\/1056397485"},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.1145\/2484028.2484034"},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-23318-0_11"},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"publisher","DOI":"10.1145\/2094072.2094076"},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"publisher","DOI":"10.1145\/2499178.2499204"},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.1145\/2600428.2602292"},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1145\/2808194.2809469"},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.1145\/2808194.2809445"},{"key":"e_1_3_2_1_11_1","volume-title":"Proceedings of SIGIR","author":"Carterette Ben","year":"2010","unstructured":"Ben Carterette , Evangelos Kanoulas , Virgil Pavlu , and Hui Fang . Building reusable test collections through experimental design . In Proceedings of SIGIR , 2010 . Ben Carterette, Evangelos Kanoulas, Virgil Pavlu, and Hui Fang. Building reusable test collections through experimental design. In Proceedings of SIGIR, 2010."},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.1145\/2063576.2063668"},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1145\/2396761.2396782"},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.1145\/1321440.1321530"},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.1145\/1148170.1148262"},{"key":"e_1_3_2_1_16_1","first-page":"8","article-title":"Reproducibility challenges in information retrieval","author":"Ferro Nicola","year":"2017","unstructured":"Nicola Ferro . Reproducibility challenges in information retrieval . Journal of Data and Information Quality , 8 , 2017 . Nicola Ferro. Reproducibility challenges in information retrieval. Journal of Data and Information Quality, 8, 2017.","journal-title":"Journal of Data and Information Quality"},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.1145\/2911451.2911530"},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-15998-5_14"},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","DOI":"10.1001\/jama.294.2.218"},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.1371\/journal.pmed.0020124"},{"key":"e_1_3_2_1_22_1","volume-title":"Buttersworth","author":"Jones Karen Sparck","year":"1981","unstructured":"Karen Sparck Jones , editor. Information Retrieval Experiment . Buttersworth , 1981 . Karen Sparck Jones, editor. Information Retrieval Experiment. Buttersworth, 1981."},{"key":"e_1_3_2_1_23_1","volume-title":"Readings in Information Retrieval","author":"Jones Karen Sparck","year":"1997","unstructured":"Karen Sparck Jones and Peter Willett , editors. Readings in Information Retrieval . Morgan Kaufmann Publishers , 1997 . Karen Sparck Jones and Peter Willett, editors. Readings in Information Retrieval. Morgan Kaufmann Publishers, 1997."},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-540-30480-7_1"},{"key":"e_1_3_2_1_25_1","first-page":"506","article-title":"Statistical errors","author":"Nuzzo Regina","year":"2014","unstructured":"Regina Nuzzo . Statistical errors . Nature News , 506 , 2014 . Regina Nuzzo. Statistical errors. Nature News, 506, 2014.","journal-title":"Nature News"},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.1145\/2970398.2970443"},{"key":"e_1_3_2_1_27_1","doi-asserted-by":"publisher","DOI":"10.1561\/1500000009"},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"publisher","DOI":"10.1145\/1076034.1076064"},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.1145\/1321440.1321528"},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"publisher","DOI":"10.1145\/1571941.1572050"},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"crossref","unstructured":"Victoria Stodden Marcia McNutt David H. Bailey Ewa Deelman Yolanda Gil Brooks Hanson Michael A. Heroux John P. A. Ioannidis and Michela Taufer. Enhancing reproducibility for computational methods. Science 354 016.  Victoria Stodden Marcia McNutt David H. Bailey Ewa Deelman Yolanda Gil Brooks Hanson Michael A. Heroux John P. A. Ioannidis and Michela Taufer. Enhancing reproducibility for computational methods. Science 354 016.","DOI":"10.1126\/science.aah6168"},{"key":"e_1_3_2_1_32_1","first-page":"59","volume-title":"JonesciteJones:81","author":"Tague Jean","unstructured":"Jean Tague . The pragmatics of information retrieval evaluation . In JonesciteJones:81 , pages 59 -- 102 . Jean Tague. The pragmatics of information retrieval evaluation. In JonesciteJones:81, pages 59--102."},{"key":"e_1_3_2_1_33_1","unstructured":"Jean Tague-Sutcliffe. The pragmatics of information retrieval evaluation revisited. In Jones and WillettciteJones  Jean Tague-Sutcliffe. The pragmatics of information retrieval evaluation revisited. In Jones and WillettciteJones"},{"key":"e_1_3_2_1_34_1","unstructured":"Willett:97 pages 205--216.  Willett:97 pages 205--216."},{"key":"e_1_3_2_1_35_1","first-page":"385","volume-title":"Proceedings of the 3rd Text REtrieval Conference (TREC)","author":"Tague-Sutcliffe Jean","year":"1994","unstructured":"Jean Tague-Sutcliffe and James Blustein . A statistical analysis of the TREC-3 data . In Proceedings of the 3rd Text REtrieval Conference (TREC) , pages 385 -- 399 , 1994 . Jean Tague-Sutcliffe and James Blustein. A statistical analysis of the TREC-3 data. In Proceedings of the 3rd Text REtrieval Conference (TREC), pages 385--399, 1994."},{"key":"e_1_3_2_1_36_1","doi-asserted-by":"publisher","DOI":"10.1145\/2484028.2484163"},{"key":"e_1_3_2_1_37_1","doi-asserted-by":"publisher","DOI":"10.1145\/290941.291017"},{"key":"e_1_3_2_1_38_1","doi-asserted-by":"publisher","DOI":"10.1145\/564376.564432"},{"key":"e_1_3_2_1_39_1","volume-title":"TREC: Experiments and evaluation in information retrieval","author":"Voorhees Ellen M.","year":"2005","unstructured":"Ellen M. Voorhees and Donna K. Harman . TREC: Experiments and evaluation in information retrieval . The MIT Press , 2005 . Ellen M. Voorhees and Donna K. Harman. TREC: Experiments and evaluation in information retrieval. The MIT Press, 2005."},{"key":"e_1_3_2_1_40_1","doi-asserted-by":"publisher","DOI":"10.1145\/1458082.1458158"},{"key":"e_1_3_2_1_41_1","doi-asserted-by":"publisher","DOI":"10.1145\/290941.291014"}],"event":{"name":"SIGIR '17: The 40th International ACM SIGIR conference on research and development in Information Retrieval","sponsor":["SIGIR ACM Special Interest Group on Information Retrieval"],"location":"Shinjuku Tokyo Japan","acronym":"SIGIR '17"},"container-title":["Proceedings of the 40th International ACM SIGIR Conference on Research and Development in Information Retrieval"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3077136.3082065","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3077136.3082065","content-type":"application\/pdf","content-version":"vor","intended-application":"syndication"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3077136.3082065","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,18]],"date-time":"2025-06-18T03:37:19Z","timestamp":1750217839000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3077136.3082065"}},"subtitle":["Theory and Practice"],"short-title":[],"issued":{"date-parts":[[2017,8,7]]},"references-count":40,"alternative-id":["10.1145\/3077136.3082065","10.1145\/3077136"],"URL":"https:\/\/doi.org\/10.1145\/3077136.3082065","relation":{},"subject":[],"published":{"date-parts":[[2017,8,7]]},"assertion":[{"value":"2017-08-07","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}