{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,10]],"date-time":"2026-07-10T15:54:00Z","timestamp":1783698840130,"version":"3.55.0"},"publisher-location":"New York, NY, USA","reference-count":29,"publisher":"ACM","license":[{"start":{"date-parts":[[2023,7,18]],"date-time":"2023-07-18T00:00:00Z","timestamp":1689638400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"DOI":"10.13039\/100000001","name":"National Science Foundation","doi-asserted-by":"publisher","award":["17-51278"],"award-info":[{"award-number":["17-51278"]}],"id":[{"id":"10.13039\/100000001","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2023,7,19]]},"DOI":"10.1145\/3539618.3592004","type":"proceedings-article","created":{"date-parts":[[2023,7,19]],"date-time":"2023-07-19T00:22:23Z","timestamp":1689726143000},"page":"2087-2091","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":3,"title":["Inference at Scale: Significance Testing for Large Search and Recommendation Experiments"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-8468-1581","authenticated-orcid":false,"given":"Ngozi","family":"Ihemelandu","sequence":"first","affiliation":[{"name":"Boise State University, Boise, ID, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-2467-0108","authenticated-orcid":false,"given":"Michael D.","family":"Ekstrand","sequence":"additional","affiliation":[{"name":"Boise State University, Boise, ID, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2023,7,18]]},"reference":[{"key":"e_1_3_2_1_1_1","unstructured":"Payal Bajaj Daniel Campos Nick Craswell Li Deng Jianfeng Gao Xiaodong Liu Rangan Majumder Andrew McNamara Bhaskar Mitra Tri Nguyen et al. 2016. Ms marco: A human generated machine reading comprehension dataset. arXiv preprint arXiv:1611.09268 (2016)."},{"key":"e_1_3_2_1_2_1","unstructured":"George EP Box William H Hunter Stuart Hunter et al. 1978. Statistics for experimenters. Vol. 664. John Wiley and sons New York."},{"key":"e_1_3_2_1_3_1","volume-title":"Borah: Dell HPC Intel (High Performance Computing Cluster).","author":"Byrne Kelly A","year":"2020","unstructured":"Kelly A Byrne. 2020. Borah: Dell HPC Intel (High Performance Computing Cluster). (2020)."},{"key":"e_1_3_2_1_4_1","volume-title":"Aslib proceedings","author":"Cleverdon Cyril","unstructured":"Cyril Cleverdon. 1967. The Cranfield tests on index language devices. In Aslib proceedings. MCB UP Ltd."},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1145\/3404835.3462804"},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"publisher","DOI":"10.1145\/3340531.3412778"},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"publisher","DOI":"10.1136\/bmj.1.3923.554-a"},{"key":"e_1_3_2_1_8_1","volume-title":"Acm sigir forum","author":"Fuhr Norbert","unstructured":"Norbert Fuhr. 2018. Some common mistakes in IR evaluation, and how they can be avoided. In Acm sigir forum, Vol. 51. ACM New York, NY, USA, 32--41."},{"key":"e_1_3_2_1_9_1","volume-title":"The movielens datasets: History and context. Acm transactions on interactive intelligent systems (tiis) 5, 4","author":"Maxwell Harper F","year":"2015","unstructured":"F Maxwell Harper and Joseph A Konstan. 2015. The movielens datasets: History and context. Acm transactions on interactive intelligent systems (tiis) 5, 4 (2015), 1--19."},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.1145\/2872427.2883037"},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.1145\/160688.160758"},{"key":"e_1_3_2_1_12_1","volume-title":"Statistical Inference: The Missing Piece of RecSys Experiment Reliability Discourse. arXiv preprint arXiv:2109.06424","author":"Ihemelandu Ngozi","year":"2021","unstructured":"Ngozi Ihemelandu and Michael D Ekstrand. 2021. Statistical Inference: The Missing Piece of RecSys Experiment Reliability Discourse. arXiv preprint arXiv:2109.06424 (2021)."},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1145\/3404835.3463034"},{"key":"e_1_3_2_1_14_1","volume-title":"Computer-intensive methods for testing hypotheses","author":"Noreen Eric W","unstructured":"Eric W Noreen. 1989. Computer-intensive methods for testing hypotheses. Wiley New York."},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.1145\/3412841.3441945"},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.1002\/asi.24203"},{"key":"e_1_3_2_1_17_1","volume-title":"Mathematical statistics with applications in R","author":"Ramachandran Kandethody M","unstructured":"Kandethody M Ramachandran and Chris P Tsokos. 2020. Mathematical statistics with applications in R. Academic Press."},{"key":"e_1_3_2_1_18_1","unstructured":"V Rijsbergen and C Keith Joost. 1979. Information Retrieval Butterworths London. (1979)."},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","DOI":"10.1145\/2911451.2911492"},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-981-13-1199-4"},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.1145\/1076034.1076064"},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.1145\/1321440.1321528"},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.1145\/3471158.3472242"},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"publisher","DOI":"10.1145\/3331184.3331259"},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.1145\/3209978.3210043"},{"key":"e_1_3_2_1_26_1","volume-title":"Workshop of the cross-language evaluation forum for european languages. Springer, 355--370","author":"Voorhees Ellen M","year":"2001","unstructured":"Ellen M Voorhees. 2001. The philosophy of information retrieval evaluation. In Workshop of the cross-language evaluation forum for european languages. Springer, 355--370."},{"key":"e_1_3_2_1_27_1","unstructured":"Dennis Wackerly William Mendenhall and Richard L Scheaffer. 2014. Mathematical statistics with applications. Cengage Learning."},{"key":"e_1_3_2_1_28_1","first-page":"0","volume-title":"Moving to a world beyond","author":"Wasserstein Ronald L","unstructured":"Ronald L Wasserstein, Allen L Schirm, and Nicole A Lazar. 2019. Moving to a world beyond p 0.05\"., 19 pages."},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.1145\/290941.291014"}],"event":{"name":"SIGIR '23: The 46th International ACM SIGIR Conference on Research and Development in Information Retrieval","location":"Taipei Taiwan","acronym":"SIGIR '23","sponsor":["SIGIR ACM Special Interest Group on Information Retrieval"]},"container-title":["Proceedings of the 46th International ACM SIGIR Conference on Research and Development in Information Retrieval"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3539618.3592004","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/abs\/10.1145\/3539618.3592004","content-type":"text\/html","content-version":"vor","intended-application":"syndication"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3539618.3592004","content-type":"application\/pdf","content-version":"vor","intended-application":"syndication"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3539618.3592004","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T16:37:48Z","timestamp":1750178268000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3539618.3592004"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023,7,18]]},"references-count":29,"alternative-id":["10.1145\/3539618.3592004","10.1145\/3539618"],"URL":"https:\/\/doi.org\/10.1145\/3539618.3592004","relation":{},"subject":[],"published":{"date-parts":[[2023,7,18]]},"assertion":[{"value":"2023-07-18","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}