{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,8,24]],"date-time":"2025-08-24T00:02:26Z","timestamp":1755993746090,"version":"3.44.0"},"publisher-location":"New York, NY, USA","reference-count":14,"publisher":"ACM","license":[{"start":{"date-parts":[[2024,6,18]],"date-time":"2024-06-18T00:00:00Z","timestamp":1718668800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"name":"M\u00e9tropole Europ\u00e9enne de Lille"},{"DOI":"10.13039\/501100006374","name":"Agence Nationale de la Recherche","doi-asserted-by":"publisher","id":[{"id":"10.13039\/501100006374","id-type":"DOI","asserted-by":"publisher"}]},{"name":"Inria"},{"name":"R\u00e9gion Hauts-de-France"},{"name":"I-Site LNE","award":["R-PILOTE-19-004-APPRENF"],"award-info":[{"award-number":["R-PILOTE-19-004-APPRENF"]}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2024,6,18]]},"DOI":"10.1145\/3641525.3663618","type":"proceedings-article","created":{"date-parts":[[2024,7,11]],"date-time":"2024-07-11T19:02:19Z","timestamp":1720724539000},"page":"20-24","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":0,"title":["Statistical comparison in empirical computer science with minimal computation usage"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0009-0006-3628-174X","authenticated-orcid":false,"given":"Timoth\u00e9e","family":"Mathieu","sequence":"first","affiliation":[{"name":"Scool research group, INRIA, France and CRIStAL, Universit\u00e9 de Lille, France"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-2067-2838","authenticated-orcid":false,"given":"Philippe","family":"Preux","sequence":"additional","affiliation":[{"name":"CRIStAL, Universite de Lille, France and Scool research group, INRIA, France"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2024,7,11]]},"reference":[{"key":"e_1_3_2_1_1_1","volume-title":"Functional Package Management with Guix. In European Lisp Symposium. https:\/\/arxiv.org\/abs\/1305","author":"Court\u00e8s Ludovic","year":"2013","unstructured":"Ludovic Court\u00e8s. 2013. Functional Package Management with Guix. In European Lisp Symposium. https:\/\/arxiv.org\/abs\/1305.4584"},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-27308-2_47"},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.swevo.2011.02.002"},{"key":"e_1_3_2_1_4_1","unstructured":"Dimitrov Martin and Strickland Carl and Kim Seung-Woo and Kumar Karthik and Doshi Kshitij. [n. d.]. Intel\u00ae Power Governor. https:\/\/software.intel.com\/ en-us\/articles\/intel-power-governor"},{"key":"e_1_3_2_1_5_1","volume-title":"The statistical crisis in science. The best writing on mathematics (Pitici M, ed)","author":"Gelman Andrew","year":"2016","unstructured":"Andrew Gelman and Eric Loken. 2016. The statistical crisis in science. The best writing on mathematics (Pitici M, ed) (2016), 305\u2013318."},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"publisher","unstructured":"Nikolaus Hansen Yoshihikoueno ARF1 Gabriela Kadlecov\u00e1 Kento Nozawa Luca Rolshoven Matthew Chan Youhei Akimoto Brieglhostis and Dimo Brockhoff. 2023. CMA-ES\/pycma: r3.3.0. https:\/\/doi.org\/10.5281\/ZENODO.7573532","DOI":"10.5281\/ZENODO.7573532"},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"publisher","DOI":"10.1126\/science.aal3618"},{"key":"e_1_3_2_1_8_1","unstructured":"Timoth\u00e9e Mathieu Riccardo\u00a0Della Vecchia Alena Shilova Matheus\u00a0Centa de Medeiros Hector Kohler Odalric-Ambrym Maillard and Philippe Preux. 2023. AdaStop: sequential testing for efficient and reliable comparisons of Deep RL Agents. arxiv:2306.10882\u00a0[cs.LG] submitted."},{"key":"e_1_3_2_1_9_1","first-page":"25404","article-title":"Benchopt: Reproducible, efficient and collaborative optimization benchmarks","volume":"35","author":"Moreau Thomas","year":"2022","unstructured":"Thomas Moreau, Mathurin Massias, Alexandre Gramfort, Pierre Ablin, Pierre-Antoine Bannier, Benjamin Charlier, Mathieu Dagr\u00e9ou, Tom Dupre\u00a0la Tour, Ghislain Durif, Cassio\u00a0F Dantas, 2022. Benchopt: Reproducible, efficient and collaborative optimization benchmarks. Advances in Neural Information Processing Systems 35 (2022), 25404\u201325421.","journal-title":"Advances in Neural Information Processing Systems"},{"key":"e_1_3_2_1_10_1","volume-title":"Benchopt: Reproducible, efficient and collaborative optimization benchmarks. In NeurIPS. https:\/\/arxiv.org\/abs\/2206.13424","author":"Moreau Th.","year":"2022","unstructured":"Th. Moreau, M. Massias, A. Gramfort, P. Ablin, P-A. Bannier, B. Charlier, M. Dagr\u00e9ou, T.\u00a0Dupr\u00e9 la Tour, G. Durif, C.\u00a0F. Dantas, Q. Klopfenstein, J. Larsson, E. Lai, T. Lefort, B. Mal\u00e9zieux, B. Moufad, B.\u00a0T. Nguyen, A. Rakotomamonjy, Z. Ramzi, J. Salmon, and S. Vaiter. 2022. Benchopt: Reproducible, efficient and collaborative optimization benchmarks. In NeurIPS. https:\/\/arxiv.org\/abs\/2206.13424"},{"key":"e_1_3_2_1_11_1","volume-title":"Engineering, and Medicine","author":"National Academies of Sciences","year":"2019","unstructured":"National Academies of Sciences, Engineering, and Medicine. 2019. Reproducibility and Replicability in Science. The National Academies Press."},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.1145\/3136014.3136031"},{"key":"e_1_3_2_1_13_1","volume-title":"Problem definitions and evaluation criteria for the CEC 2005 special session on real-parameter optimization. KanGAL report 2005005","author":"Suganthan N","year":"2005","unstructured":"Ponnuthurai\u00a0N Suganthan, Nikolaus Hansen, Jing\u00a0J Liang, Kalyanmoy Deb, Ying-Ping Chen, Anne Auger, and Santosh Tiwari. 2005. Problem definitions and evaluation criteria for the CEC 2005 special session on real-parameter optimization. KanGAL report 2005005, 2005 (2005), 2005."},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.1038\/s41592-019-0686-2"}],"event":{"name":"ACM REP '24: ACM Conference on Reproducibility and Replicability","sponsor":["EIGREP Emerging Interest Group on Reproducibility and Replicability"],"location":"Rennes France","acronym":"ACM REP '24"},"container-title":["Proceedings of the 2nd ACM Conference on Reproducibility and Replicability"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3641525.3663618","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3641525.3663618","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,8,23]],"date-time":"2025-08-23T02:04:37Z","timestamp":1755914677000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3641525.3663618"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,6,18]]},"references-count":14,"alternative-id":["10.1145\/3641525.3663618","10.1145\/3641525"],"URL":"https:\/\/doi.org\/10.1145\/3641525.3663618","relation":{},"subject":[],"published":{"date-parts":[[2024,6,18]]},"assertion":[{"value":"2024-07-11","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}