{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,6,30]],"date-time":"2026-06-30T03:09:27Z","timestamp":1782788967076,"version":"3.54.5"},"publisher-location":"New York, NY, USA","reference-count":34,"publisher":"ACM","funder":[{"DOI":"10.13039\/501100004329","name":"Javna Agencija za Raziskovalno Dejavnost RS","doi-asserted-by":"publisher","award":["P2-0098"],"award-info":[{"award-number":["P2-0098"]}],"id":[{"id":"10.13039\/501100004329","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2025,7,14]]},"DOI":"10.1145\/3712256.3726336","type":"proceedings-article","created":{"date-parts":[[2025,7,8]],"date-time":"2025-07-08T12:26:58Z","timestamp":1751977618000},"page":"1181-1189","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":1,"title":["The Pitfalls of Benchmarking in Algorithm Selection: What We Are Getting Wrong"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0001-5929-5761","authenticated-orcid":false,"given":"Ga\u0161per","family":"Petelin","sequence":"first","affiliation":[{"name":"Jozef Stefan Institute, Ljubljana, Slovenia"},{"name":"Jozef Stefan International Postgraduate School, Ljubljana, Slovenia"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-2723-0821","authenticated-orcid":false,"given":"Gjorgjina","family":"Cenikj","sequence":"additional","affiliation":[{"name":"Jozef Stefan Institute, Ljubljana, Slovenia"},{"name":"Jozef Stefan International Postgraduate School, Ljubljana, Slovenia"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2025,7,13]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-56855-8_19"},{"key":"e_1_3_2_1_2_1","volume-title":"Artificial Evolution: European Conference, AE 95 Brest, France, September 4\u20136","author":"B\u00e4ck Thomas","year":"2005","unstructured":"Thomas B\u00e4ck. 2005. Evolution strategies: An alternative evolutionary algorithm. In Artificial Evolution: European Conference, AE 95 Brest, France, September 4\u20136, 1995 Selected Papers. Springer, 1\u201320."},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1007\/s11081-017-9366-1"},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.1109\/ACCESS.2020.2990567"},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.swevo.2024.101534"},{"key":"e_1_3_2_1_6_1","volume-title":"Landscape Features in Single-Objective Continuous Optimization: Have We Hit a Wall in Algorithm Selection Generalization? arXiv preprint arXiv:2501.17663","author":"Cenikj Gjorgjina","year":"2025","unstructured":"Gjorgjina Cenikj, Ga\u0161per Petelin, Moritz Seiler, Nikola Cenikj, and Tome Eftimov. 2025. Landscape Features in Single-Objective Continuous Optimization: Have We Hit a Wall in Algorithm Selection Generalization? arXiv preprint arXiv:2501.17663 (2025)."},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"publisher","DOI":"10.1080\/10556788.2020.1808977"},{"key":"e_1_3_2_1_8_1","volume-title":"Completely derandomized self-adaptation in evolution strategies. Evolutionary computation 9, 2","author":"Hansen Nikolaus","year":"2001","unstructured":"Nikolaus Hansen and Andreas Ostermeier. 2001. Completely derandomized self-adaptation in evolution strategies. Evolutionary computation 9, 2 (2001), 159\u2013195."},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10618-022-00894-5"},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.1007\/BF02430364"},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.1145\/3377930.3390183"},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.1007\/s11042-020-10139-6"},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICNN.1995.488968"},{"key":"e_1_3_2_1_14_1","volume-title":"Automated algorithm selection on continuous black-box problems by combining exploratory landscape analysis and machine learning. Evolutionary computation 27, 1","author":"Kerschke Pascal","year":"2019","unstructured":"Pascal Kerschke and Heike Trautmann. 2019. Automated algorithm selection on continuous black-box problems by combining exploratory landscape analysis and machine learning. Evolutionary computation 27, 1 (2019), 99\u2013127."},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.1145\/3319619.3322051"},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-30229-9_25"},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.1145\/2001576.2001690"},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.3390\/a14010019"},{"key":"e_1_3_2_1_19_1","volume-title":"Pervasive label errors in test sets destabilize machine learning benchmarks. arXiv preprint arXiv:2103.14749","author":"Northcutt Curtis G","year":"2021","unstructured":"Curtis G Northcutt, Anish Athalye, and Jonas Mueller. 2021. Pervasive label errors in test sets destabilize machine learning benchmarks. arXiv preprint arXiv:2103.14749 (2021)."},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.5555\/1953048.2078195"},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.1145\/3638530.3654124"},{"key":"e_1_3_2_1_22_1","unstructured":"Ga\u0161per Petelin and Gjorgjina Cenikj. 2025. Benchmarking Pitfalls. https:\/\/github.com\/gasperpetelin\/benchmarking-as-problems. Accessed: 2025-01-20."},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.eswa.2022.119023"},{"key":"e_1_3_2_1_24_1","volume-title":"Pflacco: Feature-based landscape analysis of continuous and constrained optimization problems in Python. Evolutionary Computation","author":"Prager Raphael Patrick","year":"2023","unstructured":"Raphael Patrick Prager and Heike Trautmann. 2023. Pflacco: Feature-based landscape analysis of continuous and constrained optimization problems in Python. Evolutionary Computation (2023), 1\u201325."},{"key":"e_1_3_2_1_25_1","volume-title":"Differential evolution: a practical approach to global optimization","author":"Price Kenneth","unstructured":"Kenneth Price, Rainer M Storn, and Jouni A Lampinen. 2006. Differential evolution: a practical approach to global optimization. Springer Science & Business Media."},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-72699-7_2"},{"key":"e_1_3_2_1_27_1","doi-asserted-by":"publisher","DOI":"10.1145\/2939672.2939778"},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.asoc.2020.106138"},{"key":"e_1_3_2_1_29_1","volume-title":"Transfer learning analysis of multi-class classification for landscape-aware algorithm selection. mathematics 10, 3","author":"\u0160kvorc Urban","year":"2022","unstructured":"Urban \u0160kvorc, Tome Eftimov, and Peter Koro\u0161ec. 2022. Transfer learning analysis of multi-class classification for landscape-aware algorithm selection. mathematics 10, 3 (2022), 432."},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10489-024-05715-4"},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"publisher","DOI":"10.1109\/TEVC.2022.3169770"},{"key":"e_1_3_2_1_32_1","volume-title":"Problem Definitions and Evaluation Criteria for the CEC 2017 Competition and Special Session on Constrained Single Objective Real-Parameter Optimization. Computational Intelligence Laboratory","author":"Wu Guohua","year":"2016","unstructured":"Guohua Wu, Rammohan Mallipeddi, and Ponnuthurai Suganthan. 2016. Problem Definitions and Evaluation Criteria for the CEC 2017 Competition and Special Session on Constrained Single Objective Real-Parameter Optimization. Computational Intelligence Laboratory, Zhengzhou University, Zhengzhou China and Technical Report, Nanyang Technological University, Singapore. (10 2016)."},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"publisher","DOI":"10.1109\/TKDE.2021.3112126"},{"key":"e_1_3_2_1_34_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.asoc.2020.106138"}],"event":{"name":"GECCO '25: Genetic and Evolutionary Computation Conference","location":"NH Malaga Hotel Malaga Spain","acronym":"GECCO '25","sponsor":["SIGEVO ACM Special Interest Group on Genetic and Evolutionary Computation"]},"container-title":["Proceedings of the Genetic and Evolutionary Computation Conference"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3712256.3726336","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,10,7]],"date-time":"2025-10-07T20:41:41Z","timestamp":1759869701000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3712256.3726336"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,7,13]]},"references-count":34,"alternative-id":["10.1145\/3712256.3726336","10.1145\/3712256"],"URL":"https:\/\/doi.org\/10.1145\/3712256.3726336","relation":{},"subject":[],"published":{"date-parts":[[2025,7,13]]},"assertion":[{"value":"2025-07-13","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}