{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,8,18]],"date-time":"2026-08-18T01:45:39Z","timestamp":1787017539699,"version":"build-2736575974"},"publisher-location":"New York, NY, USA","reference-count":46,"publisher":"ACM","license":[{"start":{"date-parts":[[2019,7,18]],"date-time":"2019-07-18T00:00:00Z","timestamp":1563408000000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"DOI":"10.13039\/501100007601","name":"Horizon 2020 Framework Programme","doi-asserted-by":"publisher","award":["770376-2"],"award-info":[{"award-number":["770376-2"]}],"id":[{"id":"10.13039\/501100007601","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2019,7,18]]},"DOI":"10.1145\/3331184.3331259","type":"proceedings-article","created":{"date-parts":[[2019,7,19]],"date-time":"2019-07-19T13:40:26Z","timestamp":1563543626000},"page":"505-514","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":42,"title":["Statistical Significance Testing in Information Retrieval"],"prefix":"10.1145","author":[{"given":"Juli\u00e1n","family":"Urbano","sequence":"first","affiliation":[{"name":"Delft University of Technology, Delft, Netherlands"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Harlley","family":"Lima","sequence":"additional","affiliation":[{"name":"Delft University of Technology, Delft, Netherlands"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Alan","family":"Hanjalic","sequence":"additional","affiliation":[{"name":"Delft University of Technology, Delft, Netherlands"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2019,7,18]]},"reference":[{"key":"e_1_3_2_2_1_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.ipm.2012.01.004"},{"key":"e_1_3_2_2_2_1","doi-asserted-by":"publisher","DOI":"10.1145\/1390334.1390447"},{"key":"e_1_3_2_2_3_1","doi-asserted-by":"publisher","DOI":"10.1145\/2094072.2094076"},{"key":"e_1_3_2_2_4_1","doi-asserted-by":"publisher","DOI":"10.1145\/2808194.2809469"},{"key":"e_1_3_2_2_5_1","doi-asserted-by":"publisher","DOI":"10.1145\/3077136.3080738"},{"key":"e_1_3_2_2_6_1","doi-asserted-by":"publisher","DOI":"10.1145\/1390334.1390445"},{"key":"e_1_3_2_2_7_1","doi-asserted-by":"publisher","DOI":"10.1080\/01621459.1973.10481460"},{"key":"e_1_3_2_2_8_1","volume-title":"Practical Nonparametric Statistics","author":"Conover William J.","unstructured":"William J. Conover. 1999. Practical Nonparametric Statistics. Wiley."},{"key":"e_1_3_2_2_9_1","doi-asserted-by":"publisher","DOI":"10.1080\/01621459.1978.10480025"},{"key":"e_1_3_2_2_10_1","doi-asserted-by":"publisher","DOI":"10.1145\/1148170.1148262"},{"key":"e_1_3_2_2_11_1","doi-asserted-by":"publisher","DOI":"10.1145\/1277741.1277892"},{"key":"e_1_3_2_2_12_1","volume-title":"Tibshirani","author":"Efron Bradley","year":"1998","unstructured":"Bradley Efron and Robert J. Tibshirani. 1998. An Introduction to the Bootstrap. Chapman & Hall\/CRC."},{"key":"e_1_3_2_2_13_1","doi-asserted-by":"publisher","DOI":"10.1145\/3121050.3121058"},{"key":"e_1_3_2_2_14_1","unstructured":"Ronald A. Fisher. 1935. The Design of Experiments. Oliver and Boyd."},{"key":"e_1_3_2_2_15_1","doi-asserted-by":"publisher","DOI":"10.1007\/s001800000040"},{"key":"e_1_3_2_2_16_1","doi-asserted-by":"publisher","DOI":"10.1145\/160688.160758"},{"key":"e_1_3_2_2_17_1","doi-asserted-by":"publisher","DOI":"10.1201\/b17116"},{"key":"e_1_3_2_2_18_1","doi-asserted-by":"publisher","DOI":"10.1145\/2661829.2661945"},{"key":"e_1_3_2_2_19_1","doi-asserted-by":"publisher","DOI":"10.1037\/h0047595"},{"key":"e_1_3_2_2_20_1","doi-asserted-by":"publisher","DOI":"10.1002\/(SICI)1097-4571(199010)41:7<495::AID-ASI3>3.0.CO;2-S"},{"key":"e_1_3_2_2_21_1","volume-title":"Using score distributions to compare statistical significance tests for information retrieval evaluation. Journal of the Association for Information Science and Technology","author":"Parapar Javier","year":"2019","unstructured":"Javier Parapar, David E. Losada, Manuel A. Presedo Quindimil, and Alvaro Barreiro. 2019. Using score distributions to compare statistical significance tests for information retrieval evaluation. Journal of the Association for Information Science and Technology (2019)."},{"key":"e_1_3_2_2_22_1","doi-asserted-by":"publisher","DOI":"10.2307\/2984124"},{"key":"e_1_3_2_2_23_1","doi-asserted-by":"publisher","DOI":"10.1080\/01621459.1959.10501526"},{"key":"e_1_3_2_2_24_1","doi-asserted-by":"publisher","DOI":"10.1145\/2348283.2348402"},{"key":"e_1_3_2_2_25_1","doi-asserted-by":"publisher","DOI":"10.1145\/1148170.1148261"},{"key":"e_1_3_2_2_26_1","doi-asserted-by":"publisher","DOI":"10.1145\/2641383.2641385"},{"key":"e_1_3_2_2_27_1","doi-asserted-by":"publisher","DOI":"10.1145\/2911451.2911492"},{"key":"e_1_3_2_2_28_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10791-015-9273-z"},{"key":"e_1_3_2_2_29_1","doi-asserted-by":"publisher","DOI":"10.1145\/2911451.2914684"},{"key":"e_1_3_2_2_30_1","doi-asserted-by":"publisher","DOI":"10.1145\/2396761.2398553"},{"key":"e_1_3_2_2_31_1","doi-asserted-by":"publisher","DOI":"10.1145\/1076034.1076064"},{"key":"e_1_3_2_2_32_1","doi-asserted-by":"publisher","DOI":"10.1016\/S0306-4573(97)00027-7"},{"key":"e_1_3_2_2_33_1","doi-asserted-by":"publisher","DOI":"10.1145\/1321440.1321528"},{"key":"e_1_3_2_2_34_1","doi-asserted-by":"publisher","DOI":"10.1145\/1571941.1572050"},{"key":"e_1_3_2_2_35_1","doi-asserted-by":"publisher","DOI":"10.2307\/2331554"},{"key":"e_1_3_2_2_36_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10791-015-9274-y"},{"key":"e_1_3_2_2_37_1","doi-asserted-by":"publisher","DOI":"10.1145\/2484028.2484163"},{"key":"e_1_3_2_2_38_1","doi-asserted-by":"publisher","DOI":"10.1145\/2484028.2484038"},{"key":"e_1_3_2_2_39_1","doi-asserted-by":"publisher","DOI":"10.1145\/3209978.3210043"},{"key":"e_1_3_2_2_40_1","doi-asserted-by":"publisher","unstructured":"Cornelis J. van Rijsbergen. 1979. Information Retrieval. Butterworths.","DOI":"10.5555\/539927"},{"key":"e_1_3_2_2_41_1","doi-asserted-by":"publisher","DOI":"10.1016\/S0306-4573(00)00010-8"},{"key":"e_1_3_2_2_42_1","doi-asserted-by":"publisher","DOI":"10.1145\/1571941.1572138"},{"key":"e_1_3_2_2_43_1","doi-asserted-by":"publisher","DOI":"10.1145\/564376.564432"},{"key":"e_1_3_2_2_44_1","doi-asserted-by":"publisher","DOI":"10.1177\/016555159402000405"},{"key":"e_1_3_2_2_45_1","doi-asserted-by":"publisher","DOI":"10.2307\/3001968"},{"key":"e_1_3_2_2_46_1","doi-asserted-by":"publisher","DOI":"10.1145\/290941.291014"}],"event":{"name":"SIGIR '19: The 42nd International ACM SIGIR Conference on Research and Development in Information Retrieval","location":"Paris France","acronym":"SIGIR '19","sponsor":["SIGIR ACM Special Interest Group on Information Retrieval"]},"container-title":["Proceedings of the 42nd International ACM SIGIR Conference on Research and Development in Information Retrieval"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3331184.3331259","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3331184.3331259","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,1,16]],"date-time":"2026-01-16T00:54:45Z","timestamp":1768524885000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3331184.3331259"}},"subtitle":["An Empirical Analysis of Type I, Type II and Type III Errors"],"short-title":[],"issued":{"date-parts":[[2019,7,18]]},"references-count":46,"alternative-id":["10.1145\/3331184.3331259","10.1145\/3331184"],"URL":"https:\/\/doi.org\/10.1145\/3331184.3331259","relation":{},"subject":[],"published":{"date-parts":[[2019,7,18]]},"assertion":[{"value":"2019-07-18","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}