{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,16]],"date-time":"2026-07-16T05:19:19Z","timestamp":1784179159500,"version":"3.55.0"},"publisher-location":"New York, NY, USA","reference-count":22,"publisher":"ACM","license":[{"start":{"date-parts":[[2007,11,6]],"date-time":"2007-11-06T00:00:00Z","timestamp":1194307200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2007,11,6]]},"DOI":"10.1145\/1321440.1321528","type":"proceedings-article","created":{"date-parts":[[2007,11,30]],"date-time":"2007-11-30T14:39:06Z","timestamp":1196433546000},"page":"623-632","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":418,"title":["A comparison of statistical significance tests for information retrieval evaluation"],"prefix":"10.1145","author":[{"given":"Mark D.","family":"Smucker","sequence":"first","affiliation":[{"name":"University of Massachusetts Amherst, Amherst, MA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"James","family":"Allan","sequence":"additional","affiliation":[{"name":"University of Massachusetts Amherst, Amherst, MA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Ben","family":"Carterette","sequence":"additional","affiliation":[{"name":"University of Massachusetts Amherst, Amherst, MA"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2007,11,6]]},"reference":[{"key":"e_1_3_2_2_1_1","volume-title":"Statistics for Experimenters","author":"Box G. E. P.","year":"1978","unstructured":"G. E. P. Box , W. G. Hunter , and J. S. Hunter . Statistics for Experimenters . John Wiley & amp; Sons, 1978 . G. E. P. Box, W. G. Hunter, and J. S. Hunter. Statistics for Experimenters. John Wiley & Sons, 1978."},{"key":"e_1_3_2_2_2_1","volume-title":"Distribution-Free Statistical Tests","author":"Bradley J. V.","year":"1968","unstructured":"J. V. Bradley . Distribution-Free Statistical Tests . Prentice-Hall , 1968 . J. V. Bradley. Distribution-Free Statistical Tests. Prentice-Hall, 1968."},{"key":"e_1_3_2_2_3_1","unstructured":"C. Buckley. trec_eval. http:\/\/trec.nist.gov\/trec_eval\/trec_eval.8.0.tar.gz.  C. Buckley. trec_eval. http:\/\/trec.nist.gov\/trec_eval\/trec_eval.8.0.tar.gz."},{"key":"e_1_3_2_2_4_1","volume-title":"Empirical methods for artificial intelligence","author":"Cohen P. R.","year":"1995","unstructured":"P. R. Cohen . Empirical methods for artificial intelligence . MIT Press , 1995 . P. R. Cohen. Empirical methods for artificial intelligence. MIT Press, 1995."},{"key":"e_1_3_2_2_5_1","doi-asserted-by":"publisher","DOI":"10.1145\/1277741.1277892"},{"key":"e_1_3_2_2_6_1","doi-asserted-by":"publisher","DOI":"10.1145\/1148170.1148262"},{"key":"e_1_3_2_2_7_1","unstructured":"E. S. Edgington. Randomization Tests. Marcel Dekker 1995.   E. S. Edgington. Randomization Tests. Marcel Dekker 1995."},{"key":"e_1_3_2_2_8_1","volume-title":"An Introduction to the Bootstrap","author":"Efron B.","year":"1998","unstructured":"B. Efron and R. J. Tibshirani . An Introduction to the Bootstrap . Chapman & amp; Hall\/CRC, 1998 . B. Efron and R. J. Tibshirani. An Introduction to the Bootstrap. Chapman & Hall\/CRC, 1998."},{"key":"e_1_3_2_2_9_1","volume-title":"The Design of Experiments. Oliver and Boyd","author":"Fisher R. A.","year":"1935","unstructured":"R. A. Fisher . The Design of Experiments. Oliver and Boyd , first edition, 1935 . R. A. Fisher. The Design of Experiments. Oliver and Boyd, first edition, 1935."},{"key":"e_1_3_2_2_10_1","doi-asserted-by":"publisher","DOI":"10.1145\/160688.160758"},{"key":"e_1_3_2_2_11_1","doi-asserted-by":"publisher","DOI":"10.1093\/biomet\/56.2.231"},{"key":"e_1_3_2_2_12_1","doi-asserted-by":"publisher","DOI":"10.1145\/272991.272995"},{"key":"e_1_3_2_2_13_1","volume-title":"Mathematical Statistics with Applications","author":"Mendenhall W.","year":"1990","unstructured":"W. Mendenhall , D. D. Wackerly , and R. L. Scheaffer . Mathematical Statistics with Applications . PWS-KENT Publishing Company , 1990 . W. Mendenhall, D. D. Wackerly, and R. L. Scheaffer. Mathematical Statistics with Applications. PWS-KENT Publishing Company, 1990."},{"key":"e_1_3_2_2_14_1","volume-title":"Computer Intensive Methods for Testing Hypotheses","author":"Noreen E. W.","year":"1989","unstructured":"E. W. Noreen . Computer Intensive Methods for Testing Hypotheses . John Wiley , 1989 . E. W. Noreen. Computer Intensive Methods for Testing Hypotheses. John Wiley, 1989."},{"key":"e_1_3_2_2_15_1","volume-title":"R: A language and environment for statistical computing","author":"Development Core Team R","year":"2004","unstructured":"R Development Core Team . R: A language and environment for statistical computing . R Foundation for Statistical Computing , Vienna, Austria , 2004 . 3-900051-07-0. R Development Core Team. R: A language and environment for statistical computing. R Foundation for Statistical Computing, Vienna, Austria, 2004. 3-900051-07-0."},{"key":"e_1_3_2_2_16_1","doi-asserted-by":"publisher","DOI":"10.1145\/1148170.1148261"},{"key":"e_1_3_2_2_17_1","doi-asserted-by":"publisher","DOI":"10.1145\/1076034.1076064"},{"key":"e_1_3_2_2_18_1","doi-asserted-by":"publisher","DOI":"10.1016\/S0306-4573(97)00027-7"},{"key":"e_1_3_2_2_19_1","volume-title":"Information Retrieval. Butterworths","author":"van Rijsbergen C. J.","year":"1979","unstructured":"C. J. van Rijsbergen . Information Retrieval. Butterworths , second edition, 1979 . http:\/\/www.dcs.gla.ac.uk\/Keith\/Preface.html. C. J. van Rijsbergen. Information Retrieval. Butterworths, second edition, 1979. http:\/\/www.dcs.gla.ac.uk\/Keith\/Preface.html."},{"key":"e_1_3_2_2_20_1","volume-title":"TREC","author":"Voorhees E. M.","year":"2005","unstructured":"E. M. Voorhees and D. K. Harman , editors . TREC . MIT Press , 2005 . E. M. Voorhees and D. K. Harman, editors. TREC. MIT Press, 2005."},{"key":"e_1_3_2_2_21_1","doi-asserted-by":"publisher","DOI":"10.1177\/016555159402000405"},{"key":"e_1_3_2_2_22_1","doi-asserted-by":"publisher","DOI":"10.2307\/3001968"}],"event":{"name":"CIKM07: Conference on Information and Knowledge Management","location":"Lisbon Portugal","acronym":"CIKM07","sponsor":["SIGWEB ACM Special Interest Group on Hypertext, Hypermedia, and Web","ACM Association for Computing Machinery","SIGIR ACM Special Interest Group on Information Retrieval"]},"container-title":["Proceedings of the sixteenth ACM conference on Conference on information and knowledge management"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/1321440.1321528","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/1321440.1321528","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,18]],"date-time":"2025-06-18T13:56:33Z","timestamp":1750254993000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/1321440.1321528"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2007,11,6]]},"references-count":22,"alternative-id":["10.1145\/1321440.1321528","10.1145\/1321440"],"URL":"https:\/\/doi.org\/10.1145\/1321440.1321528","relation":{},"subject":[],"published":{"date-parts":[[2007,11,6]]},"assertion":[{"value":"2007-11-06","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}