{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,11,3]],"date-time":"2026-11-03T13:33:58Z","timestamp":1793712838683,"version":"4.6.1"},"publisher-location":"New York, NY, USA","reference-count":43,"publisher":"ACM","license":[{"start":{"date-parts":[[2024,7,10]],"date-time":"2024-07-10T00:00:00Z","timestamp":1720569600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"funder":[{"name":"Australian Research Council, Centre of Excellence for Automated Decision-Making and Society","award":["CE200100005"],"award-info":[{"award-number":["CE200100005"]}]},{"name":"CAMEO, PRIN 2022, MUR Ministero dell'Universit\u00e0 e della Ricerca","award":["2022ZLL7MW"],"award-info":[{"award-number":["2022ZLL7MW"]}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2024,7,10]]},"DOI":"10.1145\/3626772.3657827","type":"proceedings-article","created":{"date-parts":[[2024,7,11]],"date-time":"2024-07-11T12:40:05Z","timestamp":1720701605000},"page":"261-270","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":5,"title":["Uncontextualized significance considered dangerous"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0001-9219-6239","authenticated-orcid":false,"given":"Nicola","family":"Ferro","sequence":"first","affiliation":[{"name":"University of Padova, Padova, Italy"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-0487-9609","authenticated-orcid":false,"given":"Mark","family":"Sanderson","sequence":"additional","affiliation":[{"name":"RMIT University, Melbourne, VIC, Australia"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2024,7,11]]},"reference":[{"key":"e_1_3_2_1_1_1","volume-title":"TREC 2017 Common Core Track Overview. In The Twenty-Sixth Text REtrieval Conference Proceedings (TREC","author":"Allan J.","year":"2017","unstructured":"J. Allan, D. K. Harman, E. Kanoulas, D. Li, C. Van Gysel, and E. M. Voorhees. 2018. TREC 2017 Common Core Track Overview. In The Twenty-Sixth Text REtrieval Conference Proceedings (TREC 2017), E. M. Voorhees and A. Ellis (Eds.). National Institute of Standards and Technology (NIST), Special Publication 500--324, Washington, USA."},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.1023\/A:1009984519381"},{"key":"e_1_3_2_1_3_1","volume-title":"The Twenty-Seventh Text REtrieval Conference Proceedings (TREC","author":"Benham R.","year":"2018","unstructured":"R. Benham, L. Gallagher, J. Mackenzie, B. Liu, X. Lu, F. Scholer, A. Moffat, and J. S. Culpepper. 2019. RMIT at the 2018 TREC CORE Track. In The Twenty-Seventh Text REtrieval Conference Proceedings (TREC 2018), E. M. Voorhees and A. Ellis (Eds.). National Institute of Standards and Technology (NIST), Special Publication 500--331, Washington, USA."},{"key":"e_1_3_2_1_4_1","unstructured":"Roi Blanco and Hugo Zaragoza. 2011. Beware of relatively large but meaningless improvements. Technical Report. Yahoo! Research 2011--001."},{"key":"e_1_3_2_1_5_1","unstructured":"C. E. Bonferroni. 1936. Teoria statistica delle classi e calcolo delle probabilit\u00e0 . Number 8 in Pubblicazioni del R. Istituto Superiore di Scienze Economiche e Commerciali di Firenze. Libreria internazionale Seeber Firenze Italia."},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"crossref","unstructured":"L. Boytsov A. Belova and P. Westfall. 2013. Deciding on an Adjustment for Multiplicity in IR Experiments See citeNzz-SIGIR2013 403--412.","DOI":"10.1145\/2484028.2484034"},{"key":"e_1_3_2_1_7_1","unstructured":"C. Buckley and E. M. Voorhees. 2005. Retrieval System Evaluation. In TREC. Experiment and Evaluation in Information Retrieval D. K. Harman and E. M. Voorhees (Eds.). MIT Press Cambridge (MA) USA 53--78."},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.1145\/2094072.2094076"},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1145\/3077136.3080738"},{"key":"e_1_3_2_1_10_1","volume-title":"Sampling Techniques","author":"Cochran W. G.","unstructured":"W. G. Cochran. 1977. Sampling Techniques 3rd ed.). John Wiley & Sons, New York, USA.","edition":"3"},{"key":"e_1_3_2_1_11_1","volume-title":"The Lancet","volume":"337","author":"Easterbrook P. J.","year":"1991","unstructured":"P. J. Easterbrook, R. Gopalan, J. A. Berlin, and D. R. Matthews. 1991. Publication bias in clinical research. The Lancet, Vol. 337, 8746 (1991), 867--872."},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.1145\/3331184.3338062"},{"key":"e_1_3_2_1_13_1","volume-title":"Proc. 15th ACM International Conference on Web Searching and Data Mining (WSDM","author":"Ferro N.","year":"2022","unstructured":"N. Ferro and M. Sanderson. 2022. How do you Test a Test? A Multifaceted Examination of Significance Tests. In Proc. 15th ACM International Conference on Web Searching and Data Mining (WSDM 2022), , K. S. Candan, H. Liu, L. Akoglu, X. L. Dong, and J. Tang (Eds.). ACM Press, New York, USA, 280--288."},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"crossref","unstructured":"N. Ferro and G. Silvello. 2016. A General Linear Mixed Models Approach to Study System Component Effects See citeNzz-SIGIR2016 25--34.","DOI":"10.1145\/2911451.2911530"},{"key":"e_1_3_2_1_15_1","volume-title":"Statistical Methods for Research Workers. Oliver & Boyd","author":"Fisher R. A.","unstructured":"R. A. Fisher. 1925. Statistical Methods for Research Workers. Oliver & Boyd, Edinburgh, UK."},{"key":"e_1_3_2_1_16_1","first-page":"3","article-title":"Some Common Mistakes In IR Evaluation, And How They Can Be Avoided","volume":"51","author":"Fuhr N.","year":"2017","unstructured":"N. Fuhr. 2017. Some Common Mistakes In IR Evaluation, And How They Can Be Avoided. SIGIR Forum , Vol. 51, 3 (December 2017), 32--41.","journal-title":"SIGIR Forum"},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"crossref","unstructured":"Y. Hochberg and A. C. Tamhane. 1987. Multiple Comparison Procedures. John Wiley & Sons USA.","DOI":"10.1002\/9780470316672"},{"key":"e_1_3_2_1_18_1","volume-title":"Theory and methods","author":"Hsu J. C.","unstructured":"J. C. Hsu. 1996. Multiple Comparisons. Theory and methods. Chapman and Hall\/CRC, USA."},{"key":"e_1_3_2_1_19_1","volume-title":"ACM Transactions on Information Systems (TOIS)","volume":"20","author":"K.","year":"2002","unstructured":"K. J\"arvelin and J. Kek\"al\"ainen. 2002. Cumulated Gain-Based Evaluation of IR Techniques. ACM Transactions on Information Systems (TOIS) , Vol. 20, 4 (October 2002), 422--446."},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.1145\/2484028"},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"crossref","unstructured":"S. E. Maxwell H. D. Delaney and K. Kelly. 2017. Designing Experiments and Analyzing Data: A Model Comparison Perspective. Routledge Taylor & Francis Group UK.","DOI":"10.4324\/9781315642956"},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.1093\/biomet\/31.1-2.20"},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.1145\/3412841.3441945"},{"key":"e_1_3_2_1_24_1","first-page":"1","article-title":"Using Score Distributions to Compare Statistical Significance Tests for Information Retrieval Evaluation","volume":"71","author":"Parapar J.","year":"2020","unstructured":"J. Parapar, D. E. Losada, M. A. Presedo-Quindimil, and A. Barreiro. 2020. Using Score Distributions to Compare Statistical Significance Tests for Information Retrieval Evaluation. Journal of the American Society for Information Science and Technology (JASIST) , Vol. 71, 1 (January 2020), 98--113.","journal-title":"Journal of the American Society for Information Science and Technology (JASIST)"},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.1145\/2911451"},{"key":"e_1_3_2_1_26_1","volume-title":"Mathematical Statistics and Data Analysis","author":"Rice J. A.","unstructured":"J. A. Rice. 2007. Mathematical Statistics and Data Analysis 3rd ed.). Thomson, Belmont (CA), USA.","edition":"3"},{"key":"e_1_3_2_1_27_1","volume-title":"Proc. 35th Annual International ACM SIGIR Conference on Research and Development in Information Retrieval (SIGIR","author":"Robertson S. E.","year":"2012","unstructured":"S. E. Robertson and E. Kanoulas. 2012. On Per-topic Variance in IR Evaluation. In Proc. 35th Annual International ACM SIGIR Conference on Research and Development in Information Retrieval (SIGIR 2012), W. Hersh, J. Callan, Y. Maarek, and M. Sanderson (Eds.). ACM Press, New York, USA, 891--900."},{"key":"e_1_3_2_1_28_1","volume-title":"A GLM Approach","author":"Rutherford A.","unstructured":"A. Rutherford. 2011. ANOVA and ANCOVA. A GLM Approach 2nd ed.). John Wiley & Sons, New York, USA.","edition":"2"},{"key":"e_1_3_2_1_29_1","volume-title":"Statistical Significance, Power, and Sample Sizes: A Systematic Review of SIGIR and TOIS","author":"Sakai T.","year":"2006","unstructured":"T. Sakai. 2016. Statistical Significance, Power, and Sample Sizes: A Systematic Review of SIGIR and TOIS, 2006--2015, See citeNzz-SIGIR2016, 5--14."},{"key":"e_1_3_2_1_30_1","series-title":"The Information Retrieval Series","volume-title":"Laboratory Experiments in Information Retrieval","author":"Sakai T.","unstructured":"T. Sakai. 2018. Laboratory Experiments in Information Retrieval. The Information Retrieval Series, Vol. 40. Springer Singapore."},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"publisher","DOI":"10.1145\/3451964.3451976"},{"key":"e_1_3_2_1_32_1","doi-asserted-by":"publisher","DOI":"10.1561\/1500000009"},{"key":"e_1_3_2_1_33_1","volume-title":"Proc. 28th Annual International ACM SIGIR Conference on Research and Development in Information Retrieval (SIGIR","author":"Sanderson M.","year":"2005","unstructured":"M. Sanderson and J. Zobel. 2005. Information Retrieval System Evaluation: Effort, Sensitivity, and Reliability. In Proc. 28th Annual International ACM SIGIR Conference on Research and Development in Information Retrieval (SIGIR 2005), , R. Baeza-Yates, N. Ziviani, G. Marchionini, A. Moffat, and J. Tait (Eds.). ACM Press, New York, USA, 162--169."},{"key":"e_1_3_2_1_34_1","volume-title":"Practical Business Statistics","author":"Siegel A. F.","unstructured":"A. F. Siegel. 2016. Practical Business Statistics 7th ed.). Elsevier, The Netherlands.","edition":"7"},{"key":"e_1_3_2_1_35_1","doi-asserted-by":"publisher","DOI":"10.2307\/2331554"},{"key":"e_1_3_2_1_36_1","volume-title":"The Third Text REtrieval Conference (TREC-3), , D. K. Harman (Ed.). National Institute of Standards and Technology (NIST), Special Publication 500--225","author":"Tague-Sutcliffe J. M.","unstructured":"J. M. Tague-Sutcliffe and J. Blustein. 1995. A Statistical Analysis of the TREC-3 Data. In The Third Text REtrieval Conference (TREC-3), , D. K. Harman (Ed.). National Institute of Standards and Technology (NIST), Special Publication 500--225, Washington, USA, 385--398."},{"key":"e_1_3_2_1_37_1","doi-asserted-by":"publisher","DOI":"10.2307\/3001913"},{"key":"e_1_3_2_1_38_1","doi-asserted-by":"crossref","unstructured":"J. Urbano M. Marrero and D. Mart'in. 2013. A Comparison of the Optimality of Statistical Significance Tests for Information Retrieval Evaluation See citeNzz-SIGIR2013 925--928.","DOI":"10.1145\/2484028.2484163"},{"key":"e_1_3_2_1_39_1","volume-title":"Overview of the TREC 2004 Robust Track. In The Thirteenth Text REtrieval Conference Proceedings (TREC","author":"Voorhees E. M.","year":"2005","unstructured":"E. M. Voorhees. 2005. Overview of the TREC 2004 Robust Track. In The Thirteenth Text REtrieval Conference Proceedings (TREC 2004), E. M. Voorhees and L. P. Buckland (Eds.). National Institute of Standards and Technology (NIST), Special Publication 500--261, Washington, USA."},{"key":"e_1_3_2_1_40_1","doi-asserted-by":"publisher","DOI":"10.1145\/564376.564432"},{"key":"e_1_3_2_1_41_1","doi-asserted-by":"publisher","DOI":"10.1145\/3086701"},{"key":"e_1_3_2_1_42_1","doi-asserted-by":"publisher","DOI":"10.2307\/3001968"},{"key":"e_1_3_2_1_43_1","doi-asserted-by":"publisher","DOI":"10.1145\/290941.291014"}],"event":{"name":"SIGIR 2024: The 47th International ACM SIGIR Conference on Research and Development in Information Retrieval","location":"Washington DC USA","acronym":"SIGIR 2024","sponsor":["SIGIR ACM Special Interest Group on Information Retrieval"]},"container-title":["Proceedings of the 47th International ACM SIGIR Conference on Research and Development in Information Retrieval"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3626772.3657827","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3626772.3657827","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,8,22]],"date-time":"2025-08-22T05:38:21Z","timestamp":1755841101000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3626772.3657827"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,7,10]]},"references-count":43,"alternative-id":["10.1145\/3626772.3657827","10.1145\/3626772"],"URL":"https:\/\/doi.org\/10.1145\/3626772.3657827","relation":{},"subject":[],"published":{"date-parts":[[2024,7,10]]},"assertion":[{"value":"2024-07-11","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}