{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,16]],"date-time":"2026-07-16T05:18:54Z","timestamp":1784179134080,"version":"3.55.0"},"publisher-location":"New York, NY, USA","reference-count":15,"publisher":"ACM","license":[{"start":{"date-parts":[[2017,8,7]],"date-time":"2017-08-07T00:00:00Z","timestamp":1502064000000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"DOI":"10.13039\/100000001","name":"NSF","doi-asserted-by":"publisher","award":["IIS-1350799"],"award-info":[{"award-number":["IIS-1350799"]}],"id":[{"id":"10.13039\/100000001","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2017,8,7]]},"DOI":"10.1145\/3077136.3080738","type":"proceedings-article","created":{"date-parts":[[2017,7,28]],"date-time":"2017-07-28T19:35:01Z","timestamp":1501270501000},"page":"1125-1128","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":9,"title":["But Is It Statistically Significant?"],"prefix":"10.1145","author":[{"given":"Ben","family":"Carterette","sequence":"first","affiliation":[{"name":"University of Delaware, Newark, DE, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2017,8,7]]},"reference":[{"key":"e_1_3_2_1_1_1","volume-title":"Deciding on an Adjustment for Multiplicity in IR Experiments Proc. SIGIR","author":"Boytsov Leonid","unstructured":"Leonid Boytsov , Anna Belova , and Peter Westfall . 2013. Deciding on an Adjustment for Multiplicity in IR Experiments Proc. SIGIR . Leonid Boytsov, Anna Belova, and Peter Westfall. 2013. Deciding on an Adjustment for Multiplicity in IR Experiments Proc. SIGIR."},{"key":"e_1_3_2_1_2_1","volume-title":"Smucker","author":"Carterette Ben","year":"2007","unstructured":"Ben Carterette and Mark D . Smucker 2007 . Hypothesis Testing with Incomplete Relevance Judgments Proceedings of CIKM. 643--652. Ben Carterette and Mark D. Smucker 2007. Hypothesis Testing with Incomplete Relevance Judgments Proceedings of CIKM. 643--652."},{"key":"e_1_3_2_1_3_1","volume-title":"The Testing of Index Language Devices","author":"Cleverdon Cyril W.","unstructured":"Cyril W. Cleverdon and J. Mills 1963. The Testing of Index Language Devices . Morgan Kaufmann Publishers , 98--110. Cyril W. Cleverdon and J. Mills 1963. The Testing of Index Language Devices. Morgan Kaufmann Publishers, 98--110."},{"key":"e_1_3_2_1_4_1","volume-title":"Using Statistical Testing in the Evaluation of Retrieval Experiments Proceedings of SIGIR. 329--338","author":"Hull David A.","year":"1993","unstructured":"David A. Hull . 1993 . Using Statistical Testing in the Evaluation of Retrieval Experiments Proceedings of SIGIR. 329--338 . David A. Hull. 1993. Using Statistical Testing in the Evaluation of Retrieval Experiments Proceedings of SIGIR. 329--338."},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1080\/17470218.2012.711335"},{"key":"e_1_3_2_1_6_1","volume-title":"Evaluating Evaluation Metrics Based on the Bootstrap Proceedings of SIGIR. 525--532","author":"Sakai Tetsuya","year":"2006","unstructured":"Tetsuya Sakai . 2006 . Evaluating Evaluation Metrics Based on the Bootstrap Proceedings of SIGIR. 525--532 . Tetsuya Sakai. 2006. Evaluating Evaluation Metrics Based on the Bootstrap Proceedings of SIGIR. 525--532."},{"key":"e_1_3_2_1_7_1","volume-title":"Designing Test Collections for Comparing Many Systems Proc. CIKM.","author":"Sakai Tetsuya","year":"2014","unstructured":"Tetsuya Sakai . 2014 . Designing Test Collections for Comparing Many Systems Proc. CIKM. Tetsuya Sakai. 2014. Designing Test Collections for Comparing Many Systems Proc. CIKM."},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.1145\/2911451.2911492"},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1016\/S0306-4573(97)00027-7"},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"crossref","unstructured":"Mark Smucker James Allan and Ben Carterette. 2007. A Comparison of Statistical Significance Tests for Information Retrieval Evaluation Proceedings of CIKM. 623--632.  Mark Smucker James Allan and Ben Carterette. 2007. A Comparison of Statistical Significance Tests for Information Retrieval Evaluation Proceedings of CIKM. 623--632.","DOI":"10.1145\/1321440.1321528"},{"key":"e_1_3_2_1_11_1","unstructured":"Jean Tague. 1981. The Pragmatics of Information Retrieval Evaluation. Buttersworth 59--102.  Jean Tague. 1981. The Pragmatics of Information Retrieval Evaluation. Buttersworth 59--102."},{"key":"e_1_3_2_1_12_1","volume-title":"Information Retrieval","author":"van Rijsbergen C. J.","unstructured":"C. J. van Rijsbergen . 1979. Information Retrieval . Butterworths , London, UK . C. J. van Rijsbergen. 1979. Information Retrieval. Butterworths, London, UK."},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1145\/1458082.1458158"},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.1177\/016555159402000405"},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.1145\/290941.291014"}],"event":{"name":"SIGIR '17: The 40th International ACM SIGIR conference on research and development in Information Retrieval","location":"Shinjuku Tokyo Japan","acronym":"SIGIR '17","sponsor":["SIGIR ACM Special Interest Group on Information Retrieval"]},"container-title":["Proceedings of the 40th International ACM SIGIR Conference on Research and Development in Information Retrieval"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3077136.3080738","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3077136.3080738","content-type":"application\/pdf","content-version":"vor","intended-application":"syndication"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3077136.3080738","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,18]],"date-time":"2025-06-18T19:04:07Z","timestamp":1750273447000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3077136.3080738"}},"subtitle":["Statistical Significance in IR Research, 1995-2014"],"short-title":[],"issued":{"date-parts":[[2017,8,7]]},"references-count":15,"alternative-id":["10.1145\/3077136.3080738","10.1145\/3077136"],"URL":"https:\/\/doi.org\/10.1145\/3077136.3080738","relation":{},"subject":[],"published":{"date-parts":[[2017,8,7]]},"assertion":[{"value":"2017-08-07","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}