{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,3,25]],"date-time":"2026-03-25T00:44:07Z","timestamp":1774399447944,"version":"3.50.1"},"publisher-location":"Cham","reference-count":29,"publisher":"Springer Nature Switzerland","isbn-type":[{"value":"9783031887109","type":"print"},{"value":"9783031887116","type":"electronic"}],"license":[{"start":{"date-parts":[[2025,1,1]],"date-time":"2025-01-01T00:00:00Z","timestamp":1735689600000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2025,1,1]],"date-time":"2025-01-01T00:00:00Z","timestamp":1735689600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2025]]},"DOI":"10.1007\/978-3-031-88711-6_27","type":"book-chapter","created":{"date-parts":[[2025,4,4]],"date-time":"2025-04-04T17:15:25Z","timestamp":1743786925000},"page":"424-439","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":2,"title":["Towards Reliable Testing for\u00a0Multiple Information Retrieval System Comparisons"],"prefix":"10.1007","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-1139-0449","authenticated-orcid":false,"given":"David","family":"Otero","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-5997-8252","authenticated-orcid":false,"given":"Javier","family":"Parapar","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-6698-2946","authenticated-orcid":false,"given":"\u00c1lvaro","family":"Barreiro","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2025,4,4]]},"reference":[{"issue":"1","key":"27_CR1","doi-asserted-by":"publisher","first-page":"289","DOI":"10.1111\/j.2517-6161.1995.tb02031.x","volume":"57","author":"Y Benjamini","year":"1995","unstructured":"Benjamini, Y., Hochberg, Y.: Controlling the false discovery rate: a practical and powerful approach to multiple testing. J. Roy. Stat. Soc.: Ser. B (Methodol.) 57(1), 289\u2013300 (1995). https:\/\/doi.org\/10.1111\/j.2517-6161.1995.tb02031.x","journal-title":"J. Roy. Stat. Soc.: Ser. B (Methodol.)"},{"issue":"4","key":"27_CR2","doi-asserted-by":"publisher","first-page":"1165","DOI":"10.1214\/aos\/1013699998","volume":"29","author":"Y Benjamini","year":"2001","unstructured":"Benjamini, Y., Yekutieli, D.: The control of the false discovery rate in multiple testing under dependency. Ann. Stat. 29(4), 1165\u20131188 (2001). https:\/\/doi.org\/10.1214\/aos\/1013699998","journal-title":"Ann. Stat."},{"key":"27_CR3","doi-asserted-by":"publisher","unstructured":"Boytsov, L., Belova, A., Westfall, P.: Deciding on an adjustment for multiplicity in IR experiments. In: Proceedings of the 36th International ACM SIGIR Conference on Research and Development in Information Retrieval, pp. 403\u2013412. Association for Computing Machinery, New York, NY, USA (2013). https:\/\/doi.org\/10.1145\/2484028.2484034","DOI":"10.1145\/2484028.2484034"},{"key":"27_CR4","doi-asserted-by":"publisher","unstructured":"Carterette, B.: Multiple testing in statistical analysis of systems-based information retrieval experiments. ACM Trans. Inf. Syst. 30(1) (2012). https:\/\/doi.org\/10.1145\/2094072.2094076","DOI":"10.1145\/2094072.2094076"},{"key":"27_CR5","doi-asserted-by":"publisher","unstructured":"Carterette, B., Pavlu, V., Kanoulas, E., Aslam, J.A., Allan, J.: Evaluation over thousands of queries. In: Proceedings of the 31st Annual International ACM SIGIR Conference on Research and Development in Information Retrieval, pp. 651\u2013658. Association for Computing Machinery, New York, NY, USA (2008). https:\/\/doi.org\/10.1145\/1390334.1390445","DOI":"10.1145\/1390334.1390445"},{"key":"27_CR6","doi-asserted-by":"crossref","unstructured":"Carterette, B., Pavlu, V., Fang, H., Kanoulas, E.: Million query track 2009 overview. In: Proceedings of The Eighteenth Text REtrieval Conference, TREC 2009, Gaithersburg, Maryland, USA, November 17-20, 2009. NIST Special Publication, vol. 500\u2013278. National Institute of Standards and Technology (NIST) (2009). http:\/\/trec.nist.gov\/pubs\/trec18\/papers\/MQ09OVERVIEW.pdf","DOI":"10.6028\/NIST.SP.500-278.million-query-overview"},{"key":"27_CR7","unstructured":"Cohen, J.: Statistical Power Analysis for the Behavioral Sciences. Lawrence Erlbaum Associates (1988)"},{"issue":"293","key":"27_CR8","doi-asserted-by":"publisher","first-page":"52","DOI":"10.1080\/01621459.1961.10482090","volume":"56","author":"OJ Dunn","year":"1961","unstructured":"Dunn, O.J.: Multiple comparisons among means. J. Am. Stat. Assoc. 56(293), 52\u201364 (1961). https:\/\/doi.org\/10.1080\/01621459.1961.10482090","journal-title":"J. Am. Stat. Assoc."},{"key":"27_CR9","doi-asserted-by":"publisher","unstructured":"Ferro, N., Sanderson, M.: How do you test a test? A multifaceted examination of significance tests. In: Proceedings of the Fifteenth ACM International Conference on Web Search and Data Mining, pp. 280\u2013288. WSDM \u201922, Association for Computing Machinery, New York, NY, USA (2022). https:\/\/doi.org\/10.1145\/3488560.3498406","DOI":"10.1145\/3488560.3498406"},{"key":"27_CR10","doi-asserted-by":"publisher","unstructured":"Ferro, N., Sanderson, M.: Uncontextualized significance considered dangerous. In: Proceedings of the 47th International ACM SIGIR Conference on Research and Development in Information Retrieval, pp. 261\u2013270. Association for Computing Machinery, New York, NY, USA (2024). https:\/\/doi.org\/10.1145\/3626772.3657827","DOI":"10.1145\/3626772.3657827"},{"issue":"3","key":"27_CR11","doi-asserted-by":"publisher","first-page":"32","DOI":"10.1145\/3190580.3190586","volume":"51","author":"N Fuhr","year":"2018","unstructured":"Fuhr, N.: Some common mistakes in IR evaluation, and how they can be avoided. SIGIR Forum 51(3), 32\u201341 (2018). https:\/\/doi.org\/10.1145\/3190580.3190586","journal-title":"SIGIR Forum"},{"issue":"4","key":"27_CR12","doi-asserted-by":"publisher","first-page":"800","DOI":"10.2307\/2336325","volume":"75","author":"Y Hochberg","year":"1988","unstructured":"Hochberg, Y.: A sharper Bonferroni procedure for multiple tests of significance. Biometrika 75(4), 800\u2013802 (1988). https:\/\/doi.org\/10.2307\/2336325","journal-title":"Biometrika"},{"key":"27_CR13","unstructured":"Holm, S.: A simple sequentially rejective multiple test procedure. Scandinavian J. Stat. 6(2), 65\u201370 (1979). https:\/\/www.jstor.org\/stable\/4615733"},{"issue":"2","key":"27_CR14","doi-asserted-by":"publisher","first-page":"383","DOI":"10.1093\/biomet\/75.2.383","volume":"75","author":"G Hommel","year":"1988","unstructured":"Hommel, G.: A Stagewise rejective multiple test procedure based on a modified Bonferroni test. Biometrika 75(2), 383\u2013386 (1988)","journal-title":"Biometrika"},{"issue":"3","key":"27_CR15","doi-asserted-by":"publisher","first-page":"624","DOI":"10.1093\/biomet\/76.3.624","volume":"76","author":"G Hommel","year":"1989","unstructured":"Hommel, G.: A comparison of two modified Bonferroni procedures. Biometrika 76(3), 624\u2013625 (1989). https:\/\/doi.org\/10.1093\/biomet\/76.3.624","journal-title":"Biometrika"},{"key":"27_CR16","doi-asserted-by":"crossref","unstructured":"Ihemelandu, N., Ekstrand, M.D.: Multiple testing for IR and recommendation system experiments. In: Proceedings of 46th European Conference on IR Research (ECIR 2024), pp. 449\u2013457. ECIR \u201924, Springer Nature Switzerland (2024)","DOI":"10.1007\/978-3-031-56063-7_37"},{"key":"27_CR17","doi-asserted-by":"crossref","unstructured":"James, G., Witten, D., Hastie, T., Tibshirani, R.: An Introduction to Statistical Learning: with Applications in R (2013)","DOI":"10.1007\/978-1-4614-7138-7"},{"key":"27_CR18","doi-asserted-by":"publisher","unstructured":"Parapar, J., Losada, D.E., Barreiro, A.: Testing the tests: simulation of rankings to compare statistical significance tests in information retrieval evaluation. In: Proceedings of the 36th Annual ACM Symposium on Applied Computing, pp. 655\u2013664. SAC \u201921, Association for Computing Machinery, New York, NY, USA (2021). https:\/\/doi.org\/10.1145\/3412841.3441945","DOI":"10.1145\/3412841.3441945"},{"issue":"1","key":"27_CR19","doi-asserted-by":"publisher","first-page":"98","DOI":"10.1002\/asi.24203","volume":"71","author":"J Parapar","year":"2020","unstructured":"Parapar, J., Losada, D.E., Presedo-Quindimil, M.A., Barreiro, A.: Using score distributions to compare statistical significance tests for information retrieval evaluation. J. Am. Soc. Inf. Sci. 71(1), 98\u2013113 (2020). https:\/\/doi.org\/10.1002\/asi.24203","journal-title":"J. Am. Soc. Inf. Sci."},{"key":"27_CR20","doi-asserted-by":"publisher","unstructured":"Robertson, S., Kanoulas, E., Yilmaz, E.: Modelling score distributions without actual scores. In: Proceedings of the 2013 Conference on the Theory of Information Retrieval, pp. 85\u201392. Association for Computing Machinery, New York, NY, USA (2013). https:\/\/doi.org\/10.1145\/2499178.2499181","DOI":"10.1145\/2499178.2499181"},{"key":"27_CR21","doi-asserted-by":"crossref","unstructured":"Rom, D.M.: A sequentially rejective test procedure based on a modified Bonferroni inequality. Biometrika 77(3), 663\u2013665 (1990). http:\/\/www.jstor.org\/stable\/2337008","DOI":"10.1093\/biomet\/77.3.663"},{"key":"27_CR22","doi-asserted-by":"publisher","unstructured":"Sakai, T.: Laboratory Experiments in Information Retrieval. The Information Retrieval Series, vol.\u00a040. Springer (2018). https:\/\/doi.org\/10.1007\/978-981-13-1199-4","DOI":"10.1007\/978-981-13-1199-4"},{"key":"27_CR23","doi-asserted-by":"publisher","unstructured":"Sakai, T.: On Fuhr\u2019s guideline for IR evaluation. SIGIR Forum 54(1) (2021).https:\/\/doi.org\/10.1145\/3451964.3451976","DOI":"10.1145\/3451964.3451976"},{"key":"27_CR24","doi-asserted-by":"publisher","unstructured":"Sanderson, M., Zobel, J.: Information retrieval system evaluation: effort, sensitivity, and reliability. In: Proceedings of the 28th Annual International ACM SIGIR Conference on Research and Development in Information Retrieval, pp. 162\u2013169. SIGIR \u201905, Association for Computing Machinery, New York, NY, USA (2005). https:\/\/doi.org\/10.1145\/1076034.1076064","DOI":"10.1145\/1076034.1076064"},{"issue":"3","key":"27_CR25","doi-asserted-by":"publisher","first-page":"751","DOI":"10.2307\/2336545","volume":"73","author":"RJ Simes","year":"1986","unstructured":"Simes, R.J.: An improved Bonferroni procedure for multiple tests of significance. Biometrika 73(3), 751\u2013754 (1986). https:\/\/doi.org\/10.2307\/2336545","journal-title":"Biometrika"},{"key":"27_CR26","doi-asserted-by":"publisher","unstructured":"Smucker, M.D., Allan, J., Carterette, B.: A comparison of statistical significance tests for information retrieval evaluation. In: Proceedings of the Sixteenth ACM Conference on Conference on Information and Knowledge Management, pp. 623\u2013632. CIKM \u201907, Association for Computing Machinery, New York, NY, USA (2007). https:\/\/doi.org\/10.1145\/1321440.1321528","DOI":"10.1145\/1321440.1321528"},{"key":"27_CR27","doi-asserted-by":"publisher","unstructured":"Urbano, J., Corsi, M., Hanjalic, A.: How do metric score distributions affect the type i error rate of statistical significance tests in information retrieval? In: Proceedings of the 2021 ACM SIGIR International Conference on Theory of Information Retrieval, pp. 245\u2013250. Association for Computing Machinery, New York, NY, USA (2021). https:\/\/doi.org\/10.1145\/3471158.3472242","DOI":"10.1145\/3471158.3472242"},{"key":"27_CR28","doi-asserted-by":"publisher","unstructured":"Urbano, J., Lima, H., Hanjalic, A.: Statistical significance testing in information retrieval: An empirical analysis of type I, type II and type III errors. In: Proceedings of the 42nd International ACM SIGIR Conference on Research and Development in Information Retrieval, pp. 505\u2013514. SIGIR\u201919, Association for Computing Machinery, New York, NY, USA (2019). https:\/\/doi.org\/10.1145\/3331184.3331259","DOI":"10.1145\/3331184.3331259"},{"key":"27_CR29","doi-asserted-by":"publisher","unstructured":"Webber, W., Moffat, A., Zobel, J.: Statistical power in retrieval experimentation. In: Proceedings of the 17th ACM Conference on Information and Knowledge Management, pp. 57100580. CIKM \u201908, Association for Computing Machinery, New York, NY, USA (2008). https:\/\/doi.org\/10.1145\/1458082.1458158","DOI":"10.1145\/1458082.1458158"}],"container-title":["Lecture Notes in Computer Science","Advances in Information Retrieval"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-031-88711-6_27","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,4,4]],"date-time":"2025-04-04T17:15:39Z","timestamp":1743786939000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-3-031-88711-6_27"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025]]},"ISBN":["9783031887109","9783031887116"],"references-count":29,"URL":"https:\/\/doi.org\/10.1007\/978-3-031-88711-6_27","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"value":"0302-9743","type":"print"},{"value":"1611-3349","type":"electronic"}],"subject":[],"published":{"date-parts":[[2025]]},"assertion":[{"value":"4 April 2025","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"ECIR","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"European Conference on Information Retrieval","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Lucca","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Italy","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2025","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"7 April 2025","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"11 April 2025","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"47","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"ecir2025","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"https:\/\/ecir2025.eu\/","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}}]}}