{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,3,25]],"date-time":"2025-03-25T14:39:24Z","timestamp":1742913564124,"version":"3.40.3"},"publisher-location":"Cham","reference-count":38,"publisher":"Springer Nature Switzerland","isbn-type":[{"type":"print","value":"9783031302282"},{"type":"electronic","value":"9783031302299"}],"license":[{"start":{"date-parts":[[2023,1,1]],"date-time":"2023-01-01T00:00:00Z","timestamp":1672531200000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2023,1,1]],"date-time":"2023-01-01T00:00:00Z","timestamp":1672531200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2023]]},"DOI":"10.1007\/978-3-031-30229-9_28","type":"book-chapter","created":{"date-parts":[[2023,4,8]],"date-time":"2023-04-08T19:02:39Z","timestamp":1680980559000},"page":"426-441","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":0,"title":["A Robust Statistical Framework for\u00a0the\u00a0Analysis of\u00a0the\u00a0Performances of\u00a0Stochastic Optimization Algorithms Using the\u00a0Principles of\u00a0Severity"],"prefix":"10.1007","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-5304-6411","authenticated-orcid":false,"given":"Sowmya","family":"Chandrasekaran","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-5938-5158","authenticated-orcid":false,"given":"Thomas","family":"Bartz-Beielstein","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2023,4,9]]},"reference":[{"key":"28_CR1","unstructured":"Bartz-Beielstein, T., et al.: Benchmarking in optimization: best practice and open issues. arXiv preprint arXiv:2007.03488 (2020)"},{"key":"28_CR2","doi-asserted-by":"publisher","unstructured":"Bartz-Beielstein, T., Mersmann, O., Chandrasekaran, S.: Ranking and result aggregation. In: Bartz, E., Bartz-Beielstein, T., Zaefferer, M., Mersmann, O. (eds.) Hyperparameter Tuning for Machine and Deep Learning with R: A Practical Guide, chap. 5, pp. 121\u2013161. Springer Nature (2023). https:\/\/doi.org\/10.1007\/978-981-19-5170-1_5","DOI":"10.1007\/978-981-19-5170-1_5"},{"issue":"56","key":"28_CR3","doi-asserted-by":"publisher","first-page":"2815","DOI":"10.21105\/joss.02815","volume":"5","author":"MS Ben-Shachar","year":"2020","unstructured":"Ben-Shachar, M.S., L\u00fcdecke, D., Makowski, D.: Effectsize: estimation of effect size indices and standardized parameters. J. Open Source Softw. 5(56), 2815 (2020)","journal-title":"J. Open Source Softw."},{"issue":"1","key":"28_CR4","first-page":"2653","volume":"18","author":"A Benavoli","year":"2017","unstructured":"Benavoli, A., Corani, G., Dem\u0161ar, J., Zaffalon, M.: Time for a change: a tutorial for comparing multiple classifiers through Bayesian analysis. J. Mach. Learn. Res. 18(1), 2653\u20132688 (2017)","journal-title":"J. Mach. Learn. Res."},{"issue":"397","key":"28_CR5","first-page":"112","volume":"82","author":"JO Berger","year":"1987","unstructured":"Berger, J.O., Sellke, T.: Testing a point null hypothesis: The irreconcilability of p values and evidence. J. Am. Stat. Assoc. 82(397), 112\u2013122 (1987)","journal-title":"J. Am. Stat. Assoc."},{"key":"28_CR6","doi-asserted-by":"crossref","unstructured":"Calvo, B., Shir, O.M., Ceberio, J., Doerr, C., Wang, H., B\u00e4ck, T., Lozano, J.A.: Bayesian performance analysis for black-box optimization benchmarking. In: Proceedings of the Genetic and Evolutionary Computation Conference Companion, pp. 1789\u20131797 (2019)","DOI":"10.1145\/3319619.3326888"},{"issue":"6","key":"28_CR7","doi-asserted-by":"publisher","first-page":"848","DOI":"10.1109\/TEVC.2010.2069567","volume":"15","author":"EG Carrano","year":"2011","unstructured":"Carrano, E.G., Wanner, E.F., Takahashi, R.H.: A multicriteria statistical based comparison methodology for evaluating evolutionary algorithms. IEEE Trans. Evol. Comput. 15(6), 848\u2013870 (2011)","journal-title":"IEEE Trans. Evol. Comput."},{"key":"28_CR8","doi-asserted-by":"crossref","unstructured":"Cenikj, G., Lang, R.D., Engelbrecht, A.P., Doerr, C., Koro\u0161ec, P., Eftimov, T.: Selector: selecting a representative benchmark suite for reproducible statistical comparison. arXiv preprint arXiv:2204.11527 (2022)","DOI":"10.1145\/3512290.3528809"},{"key":"28_CR9","unstructured":"Christensen, S., Wineberg, M.: Using appropriate statistics-statistics for artificial intelligence. In: Tutorial Program of the Genetic and Evolutionary Computation Conference, Seattle, WA, pp. 544\u2013564 (2004)"},{"key":"28_CR10","unstructured":"Cohen, J.: Statistical power analysis for the behavioral sciences (revised ed.) (1977)"},{"issue":"1","key":"28_CR11","doi-asserted-by":"publisher","first-page":"3","DOI":"10.1016\/j.swevo.2011.02.002","volume":"1","author":"J Derrac","year":"2011","unstructured":"Derrac, J., Garc\u00eda, S., Molina, D., Herrera, F.: A practical tutorial on the use of nonparametric statistical tests as a methodology for comparing evolutionary and swarm intelligence algorithms. Swarm Evol. Comput. 1(1), 3\u201318 (2011)","journal-title":"Swarm Evol. Comput."},{"key":"28_CR12","doi-asserted-by":"publisher","DOI":"10.1016\/j.asoc.2019.106027","volume":"88","author":"C Doerr","year":"2020","unstructured":"Doerr, C., Ye, F., Horesh, N., Wang, H., Shir, O.M., B\u00e4ck, T.: Benchmarking discrete optimization heuristics with IOHprofiler. Appl. Soft Comput. 88, 106027 (2020)","journal-title":"Appl. Soft Comput."},{"key":"28_CR13","doi-asserted-by":"publisher","DOI":"10.1201\/9780429246593","volume-title":"An Introduction to the Bootstrap","author":"B Efron","year":"1994","unstructured":"Efron, B., Tibshirani, R.J.: An Introduction to the Bootstrap. CRC Press, Boca Raton (1994)"},{"key":"28_CR14","doi-asserted-by":"publisher","DOI":"10.1016\/j.asoc.2019.105862","volume":"85","author":"T Eftimov","year":"2019","unstructured":"Eftimov, T., Koro\u0161ec, P.: Identifying practical significance through statistical comparison of meta-heuristic stochastic optimization algorithms. Appl. Soft Comput. 85, 105862 (2019)","journal-title":"Appl. Soft Comput."},{"key":"28_CR15","doi-asserted-by":"publisher","first-page":"255","DOI":"10.1016\/j.ins.2019.03.049","volume":"489","author":"T Eftimov","year":"2019","unstructured":"Eftimov, T., Koro\u0161ec, P.: A novel statistical approach for comparing meta-heuristic stochastic optimization algorithms according to the distribution of solutions in the search space. Inf. Sci. 489, 255\u2013273 (2019)","journal-title":"Inf. Sci."},{"issue":"6","key":"28_CR16","doi-asserted-by":"publisher","first-page":"617","DOI":"10.1007\/s10732-008-9080-4","volume":"15","author":"S Garc\u00eda","year":"2009","unstructured":"Garc\u00eda, S., Molina, D., Lozano, M., Herrera, F.: A study on the use of non-parametric tests for analyzing the evolutionary algorithms\u2019 behaviour: a case study on the cec\u20192005 special session on real parameter optimization. J. Heuristics 15(6), 617\u2013644 (2009)","journal-title":"J. Heuristics"},{"issue":"3","key":"28_CR17","first-page":"445","volume":"3","author":"A Gelman","year":"2008","unstructured":"Gelman, A.: Objections to Bayesian statistics. Bayesian. Analysis 3(3), 445\u2013449 (2008)","journal-title":"Analysis"},{"issue":"1","key":"28_CR18","doi-asserted-by":"publisher","first-page":"114","DOI":"10.1080\/10556788.2020.1808977","volume":"36","author":"N Hansen","year":"2021","unstructured":"Hansen, N., Auger, A., Ros, R., Mersmann, O., Tu\u0161ar, T., Brockhoff, D.: COCO: a platform for comparing continuous optimizers in a black-box setting. Optim. Methods Softw. 36(1), 114\u2013144 (2021)","journal-title":"Optim. Methods Softw."},{"key":"28_CR19","unstructured":"Hansen, N., Finck, S., Ros, R., Auger, A.: Real-parameter black-box optimization benchmarking 2009: Noiseless functions definitions. Ph.D. thesis, INRIA (2009)"},{"issue":"3","key":"28_CR20","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1371\/journal.pbio.1002106","volume":"13","author":"ML Head","year":"2015","unstructured":"Head, M.L., Holman, L., Lanfear, R., Kahn, A.T., Jennions, M.D.: The extent and consequences of p-hacking in science. PLOS Bio. 13(3), 1\u201315 (2015)","journal-title":"PLOS Bio."},{"key":"28_CR21","volume-title":"Statistical Methods for Meta-Analysis","author":"LV Hedges","year":"1985","unstructured":"Hedges, L.V., Olkin, I.: Statistical Methods for Meta-Analysis. Academic Press, New York (1985)"},{"issue":"3","key":"28_CR22","doi-asserted-by":"publisher","first-page":"399","DOI":"10.1111\/j.1751-5823.2001.tb00466.x","volume":"69","author":"B Lecoutre","year":"2001","unstructured":"Lecoutre, B., Lecoutre, M.P., Poitevineau, J.: Uses, abuses and misuses of significance tests in the scientific community: won\u2019t the Bayesian choice be unavoidable? Int. Stat. Rev. 69(3), 399\u2013417 (2001)","journal-title":"Int. Stat. Rev."},{"key":"28_CR23","doi-asserted-by":"publisher","DOI":"10.1007\/0-387-27605-X","volume-title":"Testing Statistical Hypotheses","author":"EL Lehmann","year":"2006","unstructured":"Lehmann, E.L., Romano, J.P.: Testing Statistical Hypotheses. Springer, New York (2006). https:\/\/doi.org\/10.1007\/0-387-27605-X"},{"key":"28_CR24","unstructured":"Liang, J.J., Qu, B.Y., Suganthan, P.N.: Problem definitions and evaluation criteria for the CEC 2014 special session and competition on single objective real-parameter numerical optimization. In: Computational Intelligence Laboratory, Zhengzhou University, Zhengzhou China and Technical Report, Nanyang Technological University, Singapore, vol. 635, p. 490 (2013)"},{"key":"28_CR25","unstructured":"Liang, J.J., Qu, B., Suganthan, P.N., Hern\u00e1ndez-D\u00edaz, A.G.: Problem definitions and evaluation criteria for the CEC 2013 special session on real-parameter optimization. In: Computational Intelligence Laboratory, Zhengzhou University, Zhengzhou, China and Nanyang Technological University, Singapore, Technical Report, vol. 201212, iss. 34, pp. 281\u2013295 (2013)"},{"key":"28_CR26","unstructured":"Liang, J., Qu, B., Suganthan, P., Chen, Q.: Problem definitions and evaluation criteria for the cec 2015 competition on learning-based real-parameter single objective optimization. In: Technical Report201411A, Computational Intelligence Laboratory, Zhengzhou University, Zhengzhou China and Technical Report, Nanyang Technological University, Singapore, vol. 29, pp. 625\u2013640 (2014)"},{"issue":"2","key":"28_CR27","doi-asserted-by":"publisher","first-page":"545","DOI":"10.11144\/Javeriana.upsy10-2.cdcp","volume":"10","author":"G Macbeth","year":"2011","unstructured":"Macbeth, G., Razumiejczyk, E., Ledesma, R.D.: Cliff\u2019s delta calculator: a non-parametric effect size program for two groups of observations. Universitas Psychologica 10(2), 545\u2013555 (2011)","journal-title":"Universitas Psychologica"},{"key":"28_CR28","unstructured":"Mammen, E., Nandi, S.: Bootstrap and resampling 111.2. Handbook of Computational Statistics: Concepts and Methods, p. 467 (2004)"},{"issue":"2","key":"28_CR29","doi-asserted-by":"publisher","first-page":"323","DOI":"10.1093\/bjps\/axl003","volume":"57","author":"DG Mayo","year":"2006","unstructured":"Mayo, D.G., Spanos, A.: Severe testing as a basic concept in a neyman-pearson philosophy of induction. British J. Philos. Sci. 57(2), 323\u2013357 (2006)","journal-title":"British J. Philos. Sci."},{"key":"28_CR30","doi-asserted-by":"crossref","unstructured":"Molina, D., LaTorre, A.: Toolkit for the automatic comparison of optimizers: comparing large-scale global optimizers made easy. In: 2018 IEEE Congress on Evolutionary Computation (CEC), pp. 1\u20138. IEEE (2018)","DOI":"10.1109\/CEC.2018.8477924"},{"key":"28_CR31","doi-asserted-by":"crossref","unstructured":"Neyman, J., Pearson, E.S.: On the use and interpretation of certain test criteria for purposes of statistical inference: Part i. Biometrika, pp. 175\u2013240 (1928)","DOI":"10.1093\/biomet\/20A.1-2.175"},{"issue":"6","key":"28_CR32","doi-asserted-by":"publisher","first-page":"1281","DOI":"10.1109\/TEVC.2022.3208110","volume":"26","author":"J Rojas-Delgado","year":"2022","unstructured":"Rojas-Delgado, J., Ceberio, J., Calvo, B., Lozano, J.A.: Bayesian performance analysis for algorithm ranking comparison. IEEE Trans. Evol. Comput. 26(6), 1281\u20131292 (2022)","journal-title":"IEEE Trans. Evol. Comput."},{"key":"28_CR33","doi-asserted-by":"crossref","unstructured":"Senn, S.S.: Statistical issues in drug development, vol. 69. John Wiley & Sons (2008)","DOI":"10.1002\/9780470723586"},{"issue":"14","key":"28_CR34","doi-asserted-by":"publisher","first-page":"2870","DOI":"10.1016\/j.ins.2008.03.007","volume":"178","author":"D Shilane","year":"2008","unstructured":"Shilane, D., Martikainen, J., Dudoit, S., Ovaska, S.J.: A general framework for statistical performance comparison of evolutionary computation algorithms. Inf. Sci. 178(14), 2870\u20132879 (2008)","journal-title":"Inf. Sci."},{"issue":"4","key":"28_CR35","doi-asserted-by":"publisher","first-page":"341","DOI":"10.1023\/A:1008202821328","volume":"11","author":"R Storn","year":"1997","unstructured":"Storn, R., Price, K.: Differential evolution-a simple and efficient heuristic for global optimization over continuous spaces. J. Global Optim. 11(4), 341\u2013359 (1997)","journal-title":"J. Global Optim."},{"key":"28_CR36","doi-asserted-by":"publisher","first-page":"656","DOI":"10.1016\/j.ins.2014.02.154","volume":"277","author":"N Ve\u010dek","year":"2014","unstructured":"Ve\u010dek, N., Mernik, M., \u010crepin\u0161ek, M.: A chess rating system for evolutionary algorithms: a new method for the comparison and ranking of evolutionary algorithms. Inf. Sci. 277, 656\u2013679 (2014)","journal-title":"Inf. Sci."},{"issue":"2","key":"28_CR37","doi-asserted-by":"publisher","first-page":"129","DOI":"10.1080\/00031305.2016.1154108","volume":"70","author":"RL Wasserstein","year":"2016","unstructured":"Wasserstein, R.L., Lazar, N.A.: The ASA\u2019s statement on p-values: context, process, and purpose. Am. Stat. 70(2), 129\u2013133 (2016)","journal-title":"Am. Stat."},{"key":"28_CR38","unstructured":"Wu, G., Mallipeddi, R., Suganthan, P.N.: Problem definitions and evaluation criteria for the CEC 2017 competition on constrained real-parameter optimization. National University of Defense Technology, Changsha, Hunan, PR China and Kyungpook National University, Daegu, South Korea and Nanyang Technological University, Singapore, Technical Report (2017)"}],"container-title":["Lecture Notes in Computer Science","Applications of Evolutionary Computation"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-031-30229-9_28","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2023,4,9]],"date-time":"2023-04-09T23:09:54Z","timestamp":1681081794000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-3-031-30229-9_28"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023]]},"ISBN":["9783031302282","9783031302299"],"references-count":38,"URL":"https:\/\/doi.org\/10.1007\/978-3-031-30229-9_28","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"type":"print","value":"0302-9743"},{"type":"electronic","value":"1611-3349"}],"subject":[],"published":{"date-parts":[[2023]]},"assertion":[{"value":"9 April 2023","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"EvoApplications","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"International Conference on the Applications of Evolutionary Computation (Part of EvoStar)","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Brno","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Czech Republic","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2023","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"12 April 2023","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"14 April 2023","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"26","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"evoapplications2023","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"https:\/\/www.evostar.org\/2023\/evoapps\/","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Double-blind","order":1,"name":"type","label":"Type","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"EasyChair","order":2,"name":"conference_management_system","label":"Conference Management System","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"78","order":3,"name":"number_of_submissions_sent_for_review","label":"Number of Submissions Sent for Review","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"37","order":4,"name":"number_of_full_papers_accepted","label":"Number of Full Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"14","order":5,"name":"number_of_short_papers_accepted","label":"Number of Short Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"47% - The value is computed by the equation \"Number of Full Papers Accepted \/ Number of Submissions Sent for Review * 100\" and then rounded to a whole number.","order":6,"name":"acceptance_rate_of_full_papers","label":"Acceptance Rate of Full Papers","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"3","order":7,"name":"average_number_of_reviews_per_paper","label":"Average Number of Reviews per Paper","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"3","order":8,"name":"average_number_of_papers_per_reviewer","label":"Average Number of Papers per Reviewer","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"No","order":9,"name":"external_reviewers_involved","label":"External Reviewers Involved","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}}]}}