{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,11,12]],"date-time":"2025-11-12T14:20:47Z","timestamp":1762957247300,"version":"3.40.3"},"publisher-location":"Cham","reference-count":50,"publisher":"Springer International Publishing","isbn-type":[{"type":"print","value":"9783031252624"},{"type":"electronic","value":"9783031252631"}],"license":[{"start":{"date-parts":[[2023,1,1]],"date-time":"2023-01-01T00:00:00Z","timestamp":1672531200000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2023,1,1]],"date-time":"2023-01-01T00:00:00Z","timestamp":1672531200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2023]]},"DOI":"10.1007\/978-3-031-25263-1_6","type":"book-chapter","created":{"date-parts":[[2023,7,28]],"date-time":"2023-07-28T14:02:22Z","timestamp":1690552942000},"page":"149-179","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":1,"title":["Benchmarking"],"prefix":"10.1007","author":[{"given":"Vanessa","family":"Volz","sequence":"first","affiliation":[]},{"given":"Dani","family":"Irawan","sequence":"additional","affiliation":[]},{"given":"Koen","family":"van der Blom","sequence":"additional","affiliation":[]},{"given":"Boris","family":"Naujoks","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2023,7,29]]},"reference":[{"key":"6_CR1","unstructured":"T.\u00a0Bartz-Beielstein, C.\u00a0Doerr, J.\u00a0Bossek, S.\u00a0Chandrasekaran, T.\u00a0Eftimov, A.\u00a0Fischbach, P.\u00a0Kerschke, M.\u00a0Lopez-Ibanez, K.\u00a0M. Malan, J.H. Moore, B.\u00a0Naujoks, P.\u00a0Orzechowski, V.\u00a0Volz, M.\u00a0Wagner, T.\u00a0Weise, Benchmarking in optimization: Best practice and open issues (2020)"},{"key":"6_CR2","doi-asserted-by":"crossref","unstructured":"L.C.T. Bezerra, M.\u00a0L\u00f3pez-Ib\u00e1\u00f1ez, T.\u00a0St\u00fctzle, An empirical assessment of the properties of inverted generational distance indicators on multi- and many-objective optimization, in Evolutionary Multi-criterion Optimization (EMO) (2017), pp. 31\u201345","DOI":"10.1007\/978-3-319-54157-0_3"},{"key":"6_CR3","unstructured":"M.\u00a0Chiarandini, L.\u00a0Paquete, M.\u00a0Preuss, E.\u00a0Ridge, Experiments on metaheuristics: Methodological overview and open issues. Technical Report DMF-2007-03-003, The Danish Mathematical Society, Denmark (2007)"},{"key":"6_CR4","doi-asserted-by":"crossref","unstructured":"S.J. Daniels, A.A. Rahat, R.M. Everson, G.R. Tabor, J.E. Fieldsend, A suite of computationally expensive shape optimisation problems using computational fluid dynamics, in Parallel Problem Solving from Nature (PPSN) (Springer, 2018), pp. 296\u2013307","DOI":"10.1007\/978-3-319-99259-4_24"},{"key":"6_CR5","unstructured":"K.\u00a0Deb, Evolutionary algorithms for multi-criterion optimization in engineering design, in Evolutionary Algorithms in Engineering and Computer Science (EUROGEN) (1999), pp. 135\u2013161"},{"key":"6_CR6","doi-asserted-by":"crossref","unstructured":"K.\u00a0Deb, C.\u00a0Myburgh, Breaking the billion-variable barrier in real-world optimization using a customized evolutionary algorithm, in Genetic and Evolutionary Computation Conference (GECCO) (ACM Press, 2016), pp. 653\u2013660","DOI":"10.1145\/2908812.2908952"},{"key":"6_CR7","doi-asserted-by":"crossref","unstructured":"K.\u00a0Deb, L.\u00a0Thiele, M.\u00a0Laumanns, E.\u00a0Zitzler, Scalable multi-objective optimization test problems, in Congress on Evolutionary Computation (CEC) (IEEE Press, 2002), pp. 825\u2013830","DOI":"10.1109\/CEC.2002.1007032"},{"key":"6_CR8","doi-asserted-by":"crossref","unstructured":"T.\u00a0Eftimov, P.\u00a0Koro\u0161ec, Identifying practical significance through statistical comparison of meta-heuristic stochastic optimization algorithms. Appl. Soft Comput. 85(105862) (2019)","DOI":"10.1016\/j.asoc.2019.105862"},{"key":"6_CR9","doi-asserted-by":"crossref","unstructured":"T.\u00a0Eftimov, P.\u00a0Koro\u0161ec, The impact of statistics for benchmarking in evolutionary computation research, in Genetic and Evolutionary Computation Conference (GECCO) Companion (ACM Press, 2018), pp. 1329\u20131336","DOI":"10.1145\/3205651.3208232"},{"key":"6_CR10","doi-asserted-by":"crossref","unstructured":"T.\u00a0Eftimov, G.\u00a0Petelin, P.\u00a0Koro\u0161ec, Dsctool: a web-service-based framework for statistical comparison of stochastic optimization algorithms. Appl. Soft Comput. 87(105977) (2019)","DOI":"10.1016\/j.asoc.2019.105977"},{"key":"6_CR11","doi-asserted-by":"publisher","first-page":"861","DOI":"10.1613\/jair.1.11420","volume":"64","author":"K Eggensperger","year":"2019","unstructured":"K. Eggensperger, M. Lindauer, F. Hutter, Pitfalls and best practices in algorithm configuration. J. Artif. Intell. Res. 64, 861\u2013893 (2019)","journal-title":"J. Artif. Intell. Res."},{"issue":"1","key":"6_CR12","doi-asserted-by":"publisher","first-page":"19","DOI":"10.1016\/j.swevo.2011.02.001","volume":"1","author":"A Eiben","year":"2011","unstructured":"A. Eiben, S. Smit, Parameter tuning for configuring and analyzing evolutionary algorithms. Swarm Evol. Comput. 1(1), 19\u201331 (2011)","journal-title":"Swarm Evol. Comput."},{"key":"6_CR13","unstructured":"X.\u00a0Gandibleux, The MOCO numerical instances library. http:\/\/xgandibleux.free.fr\/MOCOlib\/, Accessed 20 July 2020"},{"key":"6_CR14","unstructured":"T.\u00a0Glasmachers, M.T.M. Emmerich, EMO\u20192017 Real-World Problems. https:\/\/www.ini.rub.de\/PEOPLE\/glasmtbl\/projects\/bbcomp\/. Online, accessed 22 August 2020"},{"key":"6_CR15","unstructured":"T.\u00a0Glasmachers, I.\u00a0Loshchilov, Black Box Optimization Competition BBComp. https:\/\/www.ini.rub.de\/PEOPLE\/glasmtbl\/projects\/bbcomp\/. Online, Accessed 22 August 2020"},{"key":"6_CR16","doi-asserted-by":"publisher","first-page":"114","DOI":"10.1080\/10556788.2020.1808977","volume":"36","author":"N Hansen","year":"2021","unstructured":"N. Hansen, A. Auger, O. Mersmann, T. Tu\u0161ar, D. Brockhoff, COCO: a platform for comparing continuous optimizers in a black-box setting. Optim. Methods Softw. 36, 114\u2013144 (2021)","journal-title":"Optim. Methods Softw."},{"key":"6_CR17","unstructured":"N.\u00a0Hansen, D.\u00a0Brockhoff, O.\u00a0Mersmann, T.\u00a0Tusar, D.\u00a0Tusar, O.A. ElHara, P.R. Sampaio, A.\u00a0Atamna, K.\u00a0Varelas, U.\u00a0Batu, D.M. Nguyen, F.\u00a0Matzner, A.\u00a0Auger, COmparing Continuous Optimizers: numbbo\/COCO on Github (2019)"},{"key":"6_CR18","unstructured":"N.\u00a0Hansen, S.\u00a0Finck, R.\u00a0Ros, A.\u00a0Auger, Real-parameter black-box optimization benchmarking 2009: Noiseless functions definitions. Technical Report RR-6829, Inria, France (2009). [Updated February 2010]"},{"issue":"5","key":"6_CR19","doi-asserted-by":"publisher","first-page":"477","DOI":"10.1109\/TEVC.2005.861417","volume":"10","author":"S Huband","year":"2006","unstructured":"S. Huband, P. Hingston, L. Barone, L. While, A review of multiobjective test problems and a scalable test problem toolkit. Trans. Evol. Comput. 10(5), 477\u2013506 (2006)","journal-title":"Trans. Evol. Comput."},{"key":"6_CR20","doi-asserted-by":"crossref","unstructured":"E.J. Hughes, Radar waveform optimisation as a many-objective application benchmark, in Evolutionary Multi-criterion Optimization (EMO) (Springer, 2007), pp. 700\u2013714","DOI":"10.1007\/978-3-540-70928-2_53"},{"key":"6_CR21","doi-asserted-by":"crossref","unstructured":"H.\u00a0Ishibuchi, L.\u00a0He, K.\u00a0Shang, Regular Pareto front shape is not realistic, in Congress on Evolutionary Computation (CEC) (IEEE Press, 2019), pp. 2034\u20132041","DOI":"10.1109\/CEC.2019.8790342"},{"issue":"2","key":"6_CR22","doi-asserted-by":"publisher","first-page":"169","DOI":"10.1109\/TEVC.2016.2587749","volume":"21","author":"H Ishibuchi","year":"2017","unstructured":"H. Ishibuchi, Y. Setoguchi, H. Masuda, Y. Nojima, Performance of decomposition-based many-objective algorithms strongly depends on pareto front shapes. IEEE Trans. Evol. Comput. 21(2), 169\u2013190 (2017)","journal-title":"IEEE Trans. Evol. Comput."},{"key":"6_CR23","doi-asserted-by":"crossref","unstructured":"H.\u00a0Ishibuchi, N.\u00a0Tsukamoto, Y.\u00a0Nojima, Evolutionary many-objective optimization: a short review, in Congress on Evolutionary Computation (CEC) (IEEE Press, 2008), pp. 2419\u20132426","DOI":"10.1109\/CEC.2008.4631121"},{"key":"6_CR24","doi-asserted-by":"crossref","unstructured":"H.\u00a0Jain, K.\u00a0Deb, An improved adaptive approach for elitist nondominated sorting genetic algorithm for many-objective optimization, in Evolutionary Multi-Criterion Optimization (EMO) (Springer, 2013), pp. 307\u2013321","DOI":"10.1007\/978-3-642-37140-0_25"},{"issue":"6","key":"6_CR25","doi-asserted-by":"publisher","first-page":"2814","DOI":"10.1109\/TCYB.2019.2896021","volume":"50","author":"S Jiang","year":"2020","unstructured":"S. Jiang, M. Kaiser, S. Yang, S. Kollias, N. Krasnogor, A scalable test suite for continuous dynamic multiobjective optimization. IEEE Trans. Cybernet. 50(6), 2814\u20132826 (2020)","journal-title":"IEEE Trans. Cybernet."},{"key":"6_CR26","doi-asserted-by":"crossref","unstructured":"P.\u00a0Kerschke, H.\u00a0Trautmann, Comprehensive Feature-based Landscape Analysis of Continuous and Constrained Optimization Problems Using the R-package flacco, in Applications in Statistical Computing (Springer, 2019), pp. 93\u00a0\u2013\u00a0123","DOI":"10.1007\/978-3-030-25147-5_7"},{"key":"6_CR27","doi-asserted-by":"crossref","unstructured":"T.\u00a0Kohira, H.\u00a0Kemmotsu, O.\u00a0Akira, T.\u00a0Tatsukawa, Proposal of benchmark problem based on real-world car structure design optimization, in Genetic and Evolutionary Computation Conference (GECCO) (ACM Press, 2018), pp. 183\u2013184","DOI":"10.1145\/3205651.3205702"},{"key":"6_CR28","doi-asserted-by":"publisher","first-page":"104","DOI":"10.1016\/j.swevo.2019.02.003","volume":"46","author":"H Li","year":"2019","unstructured":"H. Li, K. Deb, Q. Zhang, P. Suganthan, L. Chen, Comparison between MOEA\/D and NSGA-III on a set of novel many and multi-objective benchmark problems with challenging difficulties. Swarm Evol. Comput. 46, 104\u2013117 (2019)","journal-title":"Swarm Evol. Comput."},{"key":"6_CR29","unstructured":"J.\u00a0Liang, C.\u00a0Yue, G.\u00a0Li, B.\u00a0Qu, P.N. Suganthan, K.\u00a0Yu, Problem definitions and evaluation criteria for the CEC 2021 on multimodal multiobjective path planning optimization. Technical report, Computational Intelligence Laboratory - Zhengzhou Universit, China and Nanyang Technological University, Singapore (2020)"},{"key":"6_CR30","unstructured":"S.\u00a0Liu, Q.\u00a0Lin, K.C. Tan, Q.\u00a0Li, Benchmark problems for CEC2021 competition on evolutionary transfer multiobjectve optimization. Technical report, City University of Hong Kong (2021)"},{"key":"6_CR31","doi-asserted-by":"crossref","unstructured":"Y.\u00a0Marca, H.\u00a0Aguirre, S.\u00a0Z. Martinez, A.\u00a0Liefooghe, B.\u00a0Derbel, S.\u00a0Verel, K.\u00a0Tanaka, Approximating Pareto set topology by cubic interpolation on bi-objective problems, in Evolutionary Multi-criterion Optimization (EMO) (Springer, 2019), pp. 386\u2013398","DOI":"10.1007\/978-3-030-12598-1_31"},{"key":"6_CR32","doi-asserted-by":"crossref","unstructured":"H.\u00a0Masuda, Y.\u00a0Nojima, H.\u00a0Ishibuchi, Common properties of scalable multiobjective problems and a new framework of test problems, in 2016 IEEE Congress on Evolutionary Computation (CEC) (2016), pp. 3011\u20133018","DOI":"10.1109\/CEC.2016.7744170"},{"key":"6_CR33","doi-asserted-by":"crossref","unstructured":"T.\u00a0Matsumoto, N.\u00a0Masuyama, Y.\u00a0Nojima, H.\u00a0Ishibuchi, A multiobjective test suite with hexagon Pareto fronts and various feasible regions, in Congress on Evolutionary Computation (CEC) (IEEE Press, 2019), pp. 2058\u20132065","DOI":"10.1109\/CEC.2019.8790277"},{"key":"6_CR34","doi-asserted-by":"publisher","DOI":"10.1016\/j.asoc.2020.106139","volume":"90","author":"IR Meneghini","year":"2020","unstructured":"I.R. Meneghini, M.A. Alves, A. Gaspar-Cunha, F.G. Guimar\u00e3es, Scalable and customizable benchmark problems for many-objective optimization. Appl. Soft Comput. 90, 106139 (2020)","journal-title":"Appl. Soft Comput."},{"key":"6_CR35","doi-asserted-by":"crossref","unstructured":"O.\u00a0Mersmann, B.\u00a0Bischl, H.\u00a0Trautmann, M.\u00a0Preuss, C.\u00a0Weihs, G.\u00a0Rudolph, Exploratory landscape analysis, in Conference on Genetic and Evolutionary Computation (GECCO) (ACM Press, 2011), pp. 829\u2013836","DOI":"10.1145\/2001576.2001690"},{"key":"6_CR36","doi-asserted-by":"crossref","unstructured":"Y.\u00a0Nojima, T.\u00a0Fukase, Y.\u00a0Liu, N.\u00a0Masuyama, H.\u00a0Ishibuchi, Constrained multiobjective distance minimization problems, in Genetic and Evolutionary Computation Conference (GECCO) (ACM Press, 2019), pp. 586\u2013594","DOI":"10.1145\/3321707.3321878"},{"issue":"2","key":"6_CR37","doi-asserted-by":"publisher","first-page":"141","DOI":"10.1080\/03052150210915","volume":"34","author":"T Ray","year":"2002","unstructured":"T. Ray, K. Liew, A swarm metaphor for multiobjective design optimization. Eng. Optim. 34(2), 141\u2013153 (2002)","journal-title":"Eng. Optim."},{"key":"6_CR38","unstructured":"L.\u00a0Relund, Multi-objective optimization repository (MOrepo). https:\/\/github.com\/MCDMSociety\/MOrepo, Accessed 20 July 2020"},{"key":"6_CR39","doi-asserted-by":"crossref","unstructured":"R.\u00a0Tanabe, H.\u00a0Ishibuchi, An easy-to-use real-world multi-objective optimization problem suite. Appl. Soft Comput. 89, 106078 (2020). https:\/\/github.com\/ryojitanabe\/reproblems, Accessed 15 April 2020","DOI":"10.1016\/j.asoc.2020.106078"},{"key":"6_CR40","unstructured":"K. Tang, X. Li, P.N. Suganthan, Z. Yang, T. Weise, Benchmark functions for the CEC\u20192010 special session and competition on large-scale global optimization. Technical report, Nature Inspired Computation and Applications Laboratory (2009)"},{"key":"6_CR41","unstructured":"The Benchmarking Network, Benchmarking Network Homepage (2019). https:\/\/sites.google.com\/view\/benchmarking-network, Accessed 13 September 2020"},{"key":"6_CR42","unstructured":"The Japanese Society of Evolutionary Computation (JSEC), The 3rd Evolutionary Computation Competition - Wind Turbine Design Optimization (2019). http:\/\/www.jpnsec.org\/files\/competition2019\/EC-Symposium-2019-Competition-English.html, Accessed 1 September 2020"},{"key":"6_CR43","unstructured":"The Task Force on Benchmarking. IEEE CIS Task Force on Benchmarking Homepage (2019). https:\/\/cmte.ieee.org\/cis-benchmarking\/, Accessed 8 October 2020"},{"key":"6_CR44","doi-asserted-by":"crossref","unstructured":"T.\u00a0Tu\u0161ar, D.\u00a0Brockhoff, N.\u00a0Hansen, Mixed-integer benchmark problems for single- and bi-objective optimization, in Genetic and Evolutionary Computation Conference (GECCO) (ACM Press, 2019), pp. 718\u2013726","DOI":"10.1145\/3321707.3321868"},{"key":"6_CR45","unstructured":"D.A. Van\u00a0Veldhuizen, Multiobjective Evolutionary Algorithms: Classifications, Analyses, and New Innovations. Ph.D. thesis, Air University, USA, Air Force Institute of Technology, Ohio (1999)"},{"key":"6_CR46","doi-asserted-by":"crossref","unstructured":"M.\u00a0Vasile, Robust optimisation of trajectories intercepting dangerous neo, in AIAA\/AAS Astrodynamics Specialist Conference and Exhibit. AIAA (2002)","DOI":"10.2514\/6.2002-4719"},{"key":"6_CR47","doi-asserted-by":"crossref","unstructured":"V.\u00a0Volz, B.\u00a0Naujoks, Towards game-playing AI benchmarks via performance reporting standards, in Conference on Games (CoG) (IEEE Press, 2020) pp. 764\u2013777","DOI":"10.1109\/CoG47356.2020.9231705"},{"key":"6_CR48","doi-asserted-by":"crossref","unstructured":"V.\u00a0Volz, B.\u00a0Naujoks, P.\u00a0Kerschke, T.\u00a0Tu\u0161ar, Single- and multi-objective game-benchmark for evolutionary algorithms, in Genetic and Evolutionary Computation Conference (GECCO) (ACM Press, 2019), pp. 647\u2013655. http:\/\/www.gm.fh-koeln.de\/~naujoks\/gbea\/, Accessed 8 October 2020","DOI":"10.1145\/3321707.3321805"},{"key":"6_CR49","unstructured":"H.\u00a0Wang, D.\u00a0Vermettern, F.\u00a0Ye, C.\u00a0Doerr, T.\u00a0B\u00e4ck, IOHanalyzer: Performance Analysis for Iterative Optimization Heuristic (2020). arXiv:2007.03953"},{"issue":"2","key":"6_CR50","doi-asserted-by":"publisher","first-page":"173","DOI":"10.1162\/106365600568202","volume":"8","author":"E Zitzler","year":"2000","unstructured":"E. Zitzler, K. Deb, L. Thiele, Comparison of multiobjective evolutionary algorithms: empirical results. Evol. Comput. 8(2), 173\u2013195 (2000)","journal-title":"Evol. Comput."}],"container-title":["Natural Computing Series","Many-Criteria Optimization and Decision Analysis"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-031-25263-1_6","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,10,25]],"date-time":"2024-10-25T07:24:17Z","timestamp":1729841057000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-3-031-25263-1_6"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023]]},"ISBN":["9783031252624","9783031252631"],"references-count":50,"URL":"https:\/\/doi.org\/10.1007\/978-3-031-25263-1_6","relation":{},"ISSN":["1619-7127","2627-6461"],"issn-type":[{"type":"print","value":"1619-7127"},{"type":"electronic","value":"2627-6461"}],"subject":[],"published":{"date-parts":[[2023]]},"assertion":[{"value":"29 July 2023","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}}]}}