{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,2,21]],"date-time":"2026-02-21T18:20:31Z","timestamp":1771698031538,"version":"3.50.1"},"publisher-location":"New York, NY, USA","reference-count":42,"publisher":"ACM","license":[{"start":{"date-parts":[[2022,7,8]],"date-time":"2022-07-08T00:00:00Z","timestamp":1657238400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"DOI":"10.13039\/501100004329","name":"Javna Agencija za Raziskovalno Dejavnost RS","doi-asserted-by":"publisher","award":["P2-0098, N2-0239"],"award-info":[{"award-number":["P2-0098, N2-0239"]}],"id":[{"id":"10.13039\/501100004329","id-type":"DOI","asserted-by":"publisher"}]},{"name":"Ad Futura grant for postgraduate study","award":["GC"],"award-info":[{"award-number":["GC"]}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2022,7,8]]},"DOI":"10.1145\/3512290.3528809","type":"proceedings-article","created":{"date-parts":[[2022,7,8]],"date-time":"2022-07-08T20:02:57Z","timestamp":1657310577000},"page":"620-629","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":20,"title":["SELECTOR"],"prefix":"10.1145","author":[{"given":"Gjorgjina","family":"Cenikj","sequence":"first","affiliation":[{"name":"Jo\u017eef Stefan Institute, Slovenia"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Ryan Dieter","family":"Lang","sequence":"additional","affiliation":[{"name":"Stellenbosch University, South Africa"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Andries Petrus","family":"Engelbrecht","sequence":"additional","affiliation":[{"name":"Stellenbosch University, South Africa"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Carola","family":"Doerr","sequence":"additional","affiliation":[{"name":"Sorbonne Universit\u00e9, Paris, France"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Peter","family":"Koro\u0161ec","sequence":"additional","affiliation":[{"name":"Jo\u017eef Stefan Institute, Slovenia"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Tome","family":"Eftimov","sequence":"additional","affiliation":[{"name":"Jo\u017eef Stefan Institute, Slovenia"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2022,7,8]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"publisher","DOI":"10.1016\/0012-365X(78)90105-X"},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","unstructured":"B. Beachkofski and R. Grandhi. 2002. Improved Distributed Hypercube Sampling. In 43rd AIAA\/ASME\/ASCE\/AHS\/ASC Structures Structural Dynamics and Materials Conference. American Institute of Aeronautics and Astronautics. 10.2514\/6.2002-1274","DOI":"10.2514\/6.2002-1274"},{"key":"e_1_3_2_1_3_1","volume-title":"scmamp: Statistical Comparison of Multiple Algorithms in Multiple Problems. The R Journal Accepted for publication","author":"Calvo Borja","year":"2015","unstructured":"Borja Calvo and Guzman Santafe. 2015. scmamp: Statistical Comparison of Multiple Algorithms in Multiple Problems. The R Journal Accepted for publication (2015)."},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.5555\/1756006.1859921"},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1145\/3205651.3205747"},{"key":"e_1_3_2_1_6_1","volume-title":"Statistical Comparisons of Classifiers over Multiple Data Sets. 7 (dec","author":"Dem\u0161ar Janez","year":"2006","unstructured":"Janez Dem\u0161ar. 2006. Statistical Comparisons of Classifiers over Multiple Data Sets. 7 (dec 2006), 1--30."},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.ins.2017.07.015"},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.1109\/SSCI47803.2020.9308180"},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10732-008-9080-4"},{"key":"e_1_3_2_1_10_1","volume-title":"Swart","author":"Hagberg Aric A.","year":"2008","unstructured":"Aric A. Hagberg, Daniel A. Schult, and Pieter J. Swart. 2008. Exploring Network Structure, Dynamics, and Function using NetworkX. (2008), 11 -- 15."},{"key":"e_1_3_2_1_11_1","volume-title":"COCO: A platform for comparing continuous optimizers in a black-box setting. Optimization Methods and Software","author":"Hansen Nikolaus","year":"2020","unstructured":"Nikolaus Hansen, Anne Auger, Raymond Ros, Olaf Mersmann, Tea Tu\u0161ar, and Dimo Brockhoff. 2020. COCO: A platform for comparing continuous optimizers in a black-box setting. Optimization Methods and Software (2020), 1--31."},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1162\/106365601750190398"},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.1007\/s00446-016-0269-8"},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICNN.1995.488968"},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.1145\/2739480.2754642"},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-25147-5_7"},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1145\/3319619.3322051"},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","DOI":"10.3390\/a14030078"},{"key":"e_1_3_2_1_20_1","volume-title":"Problem definitions and evaluation criteria for the CEC 2014 special session and competition on single objective real-parameter numerical optimization. Computational Intelligence Laboratory","author":"Liang J.J.","year":"2013","unstructured":"J.J. Liang, B. Qu, and Ponnuthurai Suganthan. 2013. Problem definitions and evaluation criteria for the CEC 2014 special session and competition on single objective real-parameter numerical optimization. Computational Intelligence Laboratory, Zhengzhou University, Zhengzhou China and Technical Report, Nanyang Technological University, Singapore. (12 2013)."},{"key":"e_1_3_2_1_21_1","unstructured":"J.J. Liang B.Y. Qu P.N. Suganthan and Q. Chen. 2014. Problem definitions and evaluation criteria for the CEC 2015 competition on learning-based real-parameter single objective optimization. Computational Intelligence Laboratory Zhengzhou University Zhengzhou China and Technical Report Nanyang Technological University Singapore. (2014)."},{"key":"e_1_3_2_1_22_1","volume-title":"Problem Definitions and Evaluation Criteria for the CEC 2013 Special Session on Real-Parameter Optimization. Computational Intelligence Laboratory","author":"Liang J.J.","year":"2013","unstructured":"J.J. Liang, B. Qu, Ponnuthurai Suganthan, and Alfredo Hern\u00e1ndez-D\u00edaz. 2013. Problem Definitions and Evaluation Criteria for the CEC 2013 Special Session on Real-Parameter Optimization. Computational Intelligence Laboratory, Zhengzhou University, Zhengzhou China and Technical Report, Nanyang Technological University, Singapore. (01 2013)."},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.1145\/1143997.1144085"},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"publisher","DOI":"10.1145\/2001576.2001690"},{"key":"e_1_3_2_1_25_1","volume-title":"Baptiste Roziere, Jeremy Rapin, Olivier Teytaud, Antoine Moreau, and Carola Doerr.","author":"Meunier Laurent","year":"2021","unstructured":"Laurent Meunier, Herilalaina Rakotoarison, Pak Kan Wong, Baptiste Roziere, Jeremy Rapin, Olivier Teytaud, Antoine Moreau, and Carola Doerr. 2021. Blackbox optimization revisited: Improving algorithm selection wizards through massive benchmarking. IEEE Transactions on Evolutionary Computation (2021)."},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.1109\/TEVC.2014.2302006"},{"key":"e_1_3_2_1_27_1","doi-asserted-by":"publisher","DOI":"10.1162\/evco_a_00262"},{"key":"e_1_3_2_1_28_1","volume-title":"Distribution-free Multiple Comparisons","author":"Nemenyi P.","unstructured":"P. Nemenyi. 1963. Distribution-free Multiple Comparisons. Princeton University. https:\/\/books.google.si\/books?id=nhDMtgAACAAJ"},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.5555\/1953048.2078195"},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"crossref","unstructured":"Thorsten Pohlert. 2014. The Pairwise Multiple Comparison of Mean Ranks Package (PMCMR). https:\/\/CRAN.R-project.org\/package=PMCMR R package.","DOI":"10.32614\/CRAN.package.PMCMR"},{"key":"e_1_3_2_1_31_1","unstructured":"J. Rapin and O. Teytaud. 2018. Nevergrad - A gradient-free optimization platform. https:\/\/GitHub.com\/FacebookResearch\/Nevergrad."},{"key":"e_1_3_2_1_32_1","doi-asserted-by":"publisher","DOI":"10.1371\/journal.pone.0144059"},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.asoc.2020.106138"},{"key":"e_1_3_2_1_34_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.asoc.2020.106138"},{"key":"e_1_3_2_1_35_1","volume-title":"A Complementarity Analysis of the COCO Benchmark Problems and Artificially Generated Problems. CoRR abs\/2104.13060","author":"Skvorc Urban","year":"2021","unstructured":"Urban Skvorc, Tome Eftimov, and Peter Korosec. 2021. A Complementarity Analysis of the COCO Benchmark Problems and Artificially Generated Problems. CoRR abs\/2104.13060 (2021). arXiv:2104.13060 https:\/\/arxiv.org\/abs\/2104.13060"},{"key":"e_1_3_2_1_36_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.cor.2013.11.015"},{"key":"e_1_3_2_1_37_1","doi-asserted-by":"publisher","DOI":"10.1023\/A:1008202821328"},{"key":"e_1_3_2_1_38_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.asoc.2020.106269"},{"key":"e_1_3_2_1_39_1","volume-title":"Problem Definitions and Evaluation Criteria for the CEC 2017 Competition and Special Session on Constrained Single Objective Real-Parameter Optimization. Computational Intelligence Laboratory","author":"Wu Guohua","year":"2016","unstructured":"Guohua Wu, Rammohan Mallipeddi, and Ponnuthurai Suganthan. 2016. Problem Definitions and Evaluation Criteria for the CEC 2017 Competition and Special Session on Constrained Single Objective Real-Parameter Optimization. Computational Intelligence Laboratory, Zhengzhou University, Zhengzhou China and Technical Report, Nanyang Technological University, Singapore. (10 2016)."},{"key":"e_1_3_2_1_40_1","doi-asserted-by":"publisher","DOI":"10.1109\/cec48606.2020.9185664"},{"key":"e_1_3_2_1_41_1","volume-title":"Parallel Problem Solving from Nature - PPSN VIII, Xin Yao, Edmund K","author":"Yuan Bo","unstructured":"Bo Yuan and Marcus Gallagher. 2004. Statistical Racing Techniques for Improved Empirical Evaluation of Evolutionary Algorithms. In Parallel Problem Solving from Nature - PPSN VIII, Xin Yao, Edmund K. Burke, Jos\u00e9 A. Lozano, Jim Smith, Juan Juli\u00e1n Merelo-Guerv\u00f3s, John A. Bullinaria, Jonathan E. Rowe, Peter Ti\u0148o, Ata Kab\u00e1n, and Hans-Paul Schwefel (Eds.). Springer Berlin Heidelberg, Berlin, Heidelberg, 172--181."},{"key":"e_1_3_2_1_42_1","doi-asserted-by":"publisher","DOI":"10.1109\/CEC.2019.8790048"},{"key":"e_1_3_2_1_43_1","doi-asserted-by":"publisher","DOI":"10.3390\/math10030432"}],"event":{"name":"GECCO '22: Genetic and Evolutionary Computation Conference","location":"Boston Massachusetts","acronym":"GECCO '22","sponsor":["SIGEVO ACM Special Interest Group on Genetic and Evolutionary Computation"]},"container-title":["Proceedings of the Genetic and Evolutionary Computation Conference"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3512290.3528809","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3512290.3528809","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T18:09:56Z","timestamp":1750183796000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3512290.3528809"}},"subtitle":["selecting a representative benchmark suite for reproducible statistical comparison"],"short-title":[],"issued":{"date-parts":[[2022,7,8]]},"references-count":42,"alternative-id":["10.1145\/3512290.3528809","10.1145\/3512290"],"URL":"https:\/\/doi.org\/10.1145\/3512290.3528809","relation":{},"subject":[],"published":{"date-parts":[[2022,7,8]]},"assertion":[{"value":"2022-07-08","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}