{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,8,17]],"date-time":"2026-08-17T18:22:24Z","timestamp":1786990944682,"version":"build-2736575974"},"reference-count":51,"publisher":"Elsevier BV","license":[{"start":{"date-parts":[[2017,11,1]],"date-time":"2017-11-01T00:00:00Z","timestamp":1509494400000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.elsevier.com\/tdm\/userlicense\/1.0\/"}],"funder":[{"DOI":"10.13039\/501100004329","name":"Javna Agencija za Raziskovalno Dejavnost RS","doi-asserted-by":"publisher","award":["P2-0098"],"award-info":[{"award-number":["P2-0098"]}],"id":[{"id":"10.13039\/501100004329","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100004963","name":"Seventh Framework Programme","doi-asserted-by":"publisher","award":["621329"],"award-info":[{"award-number":["621329"]}],"id":[{"id":"10.13039\/501100004963","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["elsevier.com","sciencedirect.com"],"crossmark-restriction":true},"short-container-title":["Information Sciences"],"published-print":{"date-parts":[[2017,11]]},"DOI":"10.1016\/j.ins.2017.07.015","type":"journal-article","created":{"date-parts":[[2017,7,14]],"date-time":"2017-07-14T19:15:11Z","timestamp":1500059711000},"page":"186-215","update-policy":"https:\/\/doi.org\/10.1016\/elsevier_cm_policy","source":"Crossref","is-referenced-by-count":89,"special_numbering":"C","title":["A Novel Approach to statistical comparison of meta-heuristic stochastic optimization algorithms using deep statistics"],"prefix":"10.1016","volume":"417","author":[{"given":"Tome","family":"Eftimov","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Peter","family":"Koro\u0161ec","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Barbara","family":"Korou\u0161i\u0107 Seljak","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"78","reference":[{"key":"10.1016\/j.ins.2017.07.015_bib0001","series-title":"Proceedings of the Companion Publication of the 2015 on Genetic and Evolutionary Computation Conference","first-page":"1135","article-title":"Benchmarking ipop-cma-es-tpa and ipop-cma-es-msr on the bbob noiseless testbed","author":"Atamna","year":"2015"},{"key":"10.1016\/j.ins.2017.07.015_bib0002","series-title":"Proceedings of the Companion Publication of the 2015 on Genetic and Evolutionary Computation Conference","first-page":"1143","article-title":"Benchmarking gaussian processes and random forests surrogate models on the bbob noiseless testbed","author":"Bajer","year":"2015"},{"key":"10.1016\/j.ins.2017.07.015_bib0003","series-title":"Outliers in statistical data","volume":"3(1)","author":"Barnett","year":"1994"},{"key":"10.1016\/j.ins.2017.07.015_bib0004","series-title":"Multiple Hypothesenpr\u00fcfung\/Multiple Hypotheses Testing","first-page":"100","article-title":"Improvements of general multiple test procedures for redundant systems of hypotheses","author":"Bergmann","year":"1988"},{"key":"10.1016\/j.ins.2017.07.015_bib0005","unstructured":"Black-Box Optimization Benchmarking at CEC'2015 (CEC-BBOB), URL: (http:\/\/coco.gforge.inria.fr\/doku.php?id=cec-bbob-2015). Accessed: 2016-02-01."},{"key":"10.1016\/j.ins.2017.07.015_bib0006","series-title":"Proceedings of the Companion Publication of the 2015 on Genetic and Evolutionary Computation Conference","first-page":"1159","article-title":"The impact of initial designs on the performance of matsumoto on the noiseless bbob-2015 testbed: a preliminary study","author":"Brockhoff","year":"2015"},{"key":"10.1016\/j.ins.2017.07.015_sbref0005","article-title":"scmamp: Statistical comparison of multiple algorithms in multiple problems","author":"Calvo","year":"2016","journal-title":"The R Journal, vol. 8\/1"},{"issue":"4","key":"10.1016\/j.ins.2017.07.015_bib0008","doi-asserted-by":"crossref","first-page":"316","DOI":"10.1080\/00031305.1990.10475751","article-title":"A suggestion for using powerful and informative tests of normality","volume":"44","author":"D\u2019agostino","year":"1990","journal-title":"Am. Stat."},{"key":"10.1016\/j.ins.2017.07.015_bib0009","first-page":"1","article-title":"Statistical comparisons of classifiers over multiple data sets","volume":"7","author":"Dem\u0161ar","year":"2006","journal-title":"J. Mach. Learn. Res."},{"issue":"1","key":"10.1016\/j.ins.2017.07.015_bib0010","doi-asserted-by":"crossref","first-page":"3","DOI":"10.1016\/j.swevo.2011.02.002","article-title":"A practical tutorial on the use of nonparametric statistical tests as a methodology for comparing evolutionary and swarm intelligence algorithms","volume":"1","author":"Derrac","year":"2011","journal-title":"Swarm. Evol. Comput."},{"key":"10.1016\/j.ins.2017.07.015_bib0011","doi-asserted-by":"crossref","first-page":"41","DOI":"10.1016\/j.ins.2014.06.009","article-title":"Analyzing convergence performance of evolutionary algorithms: a statistical approach","volume":"289","author":"Derrac","year":"2014","journal-title":"Inf. Sci. (Ny)"},{"key":"10.1016\/j.ins.2017.07.015_bib0012","series-title":"Probability and statistics for engineering and the sciences","author":"Devore","year":"2015"},{"issue":"293","key":"10.1016\/j.ins.2017.07.015_bib0013","doi-asserted-by":"crossref","first-page":"52","DOI":"10.1080\/01621459.1961.10482090","article-title":"Multiple comparisons among means","volume":"56","author":"Dunn","year":"1961","journal-title":"J. Am. Stat. Assoc."},{"key":"10.1016\/j.ins.2017.07.015_bib0014","series-title":"Proceedings of the Bioinspired Optimizaiton Methods and their Applications, BIOMA 2016","first-page":"105","article-title":"Disadvantages of statistical comparison of stochastic optimization algorithms","author":"Eftimov","year":"2016"},{"issue":"Dec","key":"10.1016\/j.ins.2017.07.015_bib0015","first-page":"2677","article-title":"An extension on \u201cstatistical comparisons of classifiers over multiple data sets\u201d for all pairwise comparisons","volume":"9","author":"Garcia","year":"2008","journal-title":"J. Mach. Learn. Res."},{"issue":"6","key":"10.1016\/j.ins.2017.07.015_bib0016","doi-asserted-by":"crossref","first-page":"617","DOI":"10.1007\/s10732-008-9080-4","article-title":"A study on the use of non-parametric tests for analyzing the evolutionary algorithms\u2019 behaviour: a case study on the CEC\u20192005 special session on real parameter optimization","volume":"15","author":"Garc\u00eda","year":"2009","journal-title":"J. Heuristics"},{"issue":"10","key":"10.1016\/j.ins.2017.07.015_bib0017","doi-asserted-by":"crossref","first-page":"2044","DOI":"10.1016\/j.ins.2009.12.010","article-title":"Advanced nonparametric tests for multiple comparisons in the design of experiments in computational intelligence and data mining: experimental analysis of power","volume":"180","author":"Garc\u00eda","year":"2010","journal-title":"Inf. Sci. (Ny)"},{"key":"10.1016\/j.ins.2017.07.015_bib0018","unstructured":"J.L. Gastwirth, Y.R. Gel, W.L. Wallace Hui, V. Lyubchich, W. Miao, K. Noguchi, lawstat: Tools for Biostatistics, Public Policy, and Law, 2015. R package version 3.0, URL: https:\/\/CRAN.R-project.org\/package=lawstat."},{"issue":"3","key":"10.1016\/j.ins.2017.07.015_bib0019","doi-asserted-by":"crossref","first-page":"647","DOI":"10.1177\/106591299905200309","article-title":"The insignificance of null hypothesis significance testing","volume":"52","author":"Gill","year":"1999","journal-title":"Polit. Res. Q."},{"key":"10.1016\/j.ins.2017.07.015_bib0020","series-title":"ANOVA: Repeated Measures, 84","author":"Girden","year":"1992"},{"issue":"3","key":"10.1016\/j.ins.2017.07.015_bib0021","doi-asserted-by":"crossref","first-page":"187","DOI":"10.3102\/00028312003003187","article-title":"Testing homogeneity of variances","volume":"3","author":"Glass","year":"1966","journal-title":"Am. Educ. Res. J."},{"key":"10.1016\/j.ins.2017.07.015_bib0022","series-title":"Data Mining: Concepts and techniques","author":"Han","year":"2011"},{"key":"10.1016\/j.ins.2017.07.015_bib0023","unstructured":"N. Hansen, A. Auger, S. Finck, R. Ros, Real-parameter black-box optimization benchmarkingExper. Setup (2010)."},{"issue":"4","key":"10.1016\/j.ins.2017.07.015_bib0024","doi-asserted-by":"crossref","first-page":"800","DOI":"10.1093\/biomet\/75.4.800","article-title":"A sharper bonferroni procedure for multiple tests of significance","volume":"75","author":"Hochberg","year":"1988","journal-title":"Biometrika"},{"key":"10.1016\/j.ins.2017.07.015_bib0025","first-page":"65","article-title":"A simple sequentially rejective multiple test procedure","author":"Holm","year":"1979","journal-title":"Scand. J. Stat."},{"issue":"2","key":"10.1016\/j.ins.2017.07.015_bib0026","doi-asserted-by":"crossref","first-page":"383","DOI":"10.1093\/biomet\/75.2.383","article-title":"A stagewise rejective multiple test procedure based on a modified bonferroni test","volume":"75","author":"Hommel","year":"1988","journal-title":"Biometrika"},{"key":"10.1016\/j.ins.2017.07.015_bib0027","doi-asserted-by":"crossref","DOI":"10.1002\/9780471462422.eoct969","article-title":"Paired t test","author":"Hsu","year":"2008","journal-title":"Wiley Encycl. Clin. Trials"},{"issue":"1\u20133","key":"10.1016\/j.ins.2017.07.015_bib0028","doi-asserted-by":"crossref","first-page":"179","DOI":"10.1504\/IJVD.2017.082593","article-title":"A comparison of recent metaheuristic algorithms for crashworthiness optimisation of vehicle thin-walled tubes considering sheet metal forming effects","volume":"73","author":"Karag\u00f6z","year":"2017","journal-title":"Int. J. Veh. Des."},{"issue":"4","key":"10.1016\/j.ins.2017.07.015_bib0029","doi-asserted-by":"crossref","first-page":"723","DOI":"10.1007\/s11831-015-9155-y","article-title":"A comparative study of non-traditional methods for vehicle crashworthiness and NVH optimization","volume":"23","author":"Kiani","year":"2016","journal-title":"Arch. Comput. Methods Eng."},{"issue":"1","key":"10.1016\/j.ins.2017.07.015_bib0030","doi-asserted-by":"crossref","first-page":"1","DOI":"10.2202\/1544-6115.1041","article-title":"Multiple testing. Part ii. Step-down procedures for control of the family-wise error rate","volume":"3","author":"Laan van der","year":"2004","journal-title":"Stat. Appl. Genet. Mol. Biol."},{"issue":"2","key":"10.1016\/j.ins.2017.07.015_bib0031","doi-asserted-by":"crossref","first-page":"510","DOI":"10.1093\/biomet\/70.2.510","article-title":"A modified Wilcoxon rank sum test for paired data","volume":"70","author":"Lam","year":"1983","journal-title":"Biometrika"},{"key":"10.1016\/j.ins.2017.07.015_bib0032","series-title":"Testing statistical hypotheses","volume":"150","author":"Lehmann","year":"1986"},{"issue":"2","key":"10.1016\/j.ins.2017.07.015_bib0033","doi-asserted-by":"crossref","first-page":"171","DOI":"10.1111\/j.1468-2958.2008.00317.x","article-title":"A critical assessment of null hypothesis significance testing in quantitative communication research","volume":"34","author":"Levine","year":"2008","journal-title":"Hum. Commun. Res."},{"issue":"318","key":"10.1016\/j.ins.2017.07.015_bib0034","doi-asserted-by":"crossref","first-page":"399","DOI":"10.1080\/01621459.1967.10482916","article-title":"On the kolmogorov-smirnov test for normality with mean and variance unknown","volume":"62","author":"Lilliefors","year":"1967","journal-title":"J. Am. Stat. Assoc."},{"key":"10.1016\/j.ins.2017.07.015_bib0035","series-title":"Handbook of biological statistics","volume":"2","author":"McDonald","year":"2009"},{"key":"10.1016\/j.ins.2017.07.015_bib0036","series-title":"PhD Thesis","article-title":"Distribution-free multiple comparisons","author":"Nemenyi","year":"1963"},{"issue":"1\u20132","key":"10.1016\/j.ins.2017.07.015_bib0037","doi-asserted-by":"crossref","first-page":"99","DOI":"10.1023\/B:JMIV.0000011920.58935.9c","article-title":"A variational approach to remove outliers and impulse noise","volume":"20","author":"Nikolova","year":"2004","journal-title":"J. Math. Imaging Vis"},{"key":"10.1016\/j.ins.2017.07.015_bib0038","series-title":"Proceedings of the Companion Publication of the 2015 on Genetic and Evolutionary Computation Conference","first-page":"1151","article-title":"Dimension selection in axis-parallel brent-step method for black-box optimization of separable continuous functions","author":"Po\u0161\u00edk","year":"2015"},{"key":"10.1016\/j.ins.2017.07.015_sbref0035","series-title":"R: A Language and environment for statistical computing","author":"R Core Team","year":"2015"},{"issue":"1","key":"10.1016\/j.ins.2017.07.015_bib0040","doi-asserted-by":"crossref","first-page":"43","DOI":"10.1073\/pnas.42.1.43","article-title":"A central limit theorem and a strong mixing condition","volume":"42","author":"Rosenblatt","year":"1956","journal-title":"Proc. Nat. Acad. Sci."},{"issue":"2","key":"10.1016\/j.ins.2017.07.015_bib0041","first-page":"38","article-title":"Peirce\u2019S criterion for the elimination of suspect experimental data","volume":"20","author":"Ross","year":"2003","journal-title":"J. Eng. Technol."},{"issue":"4","key":"10.1016\/j.ins.2017.07.015_bib0042","first-page":"1","article-title":"A monte carlo simulation study for kolmogorov-smirnov two-sample test under the precondition of heterogeneity: upon the changes on the probabilities of statistical power and type i error rates with respect to skewness measure","volume":"2","author":"Senger","year":"2013","journal-title":"J. Stat. Econ. Meth."},{"issue":"337","key":"10.1016\/j.ins.2017.07.015_bib0043","doi-asserted-by":"crossref","first-page":"215","DOI":"10.1080\/01621459.1972.10481232","article-title":"An approximate analysis of variance test for normality","volume":"67","author":"Shapiro","year":"1972","journal-title":"J. Am. Stat. Assoc."},{"key":"10.1016\/j.ins.2017.07.015_bib0044","series-title":"Handbook of parametric and nonparametric statistical procedures","author":"Sheskin","year":"2003"},{"key":"10.1016\/j.ins.2017.07.015_bib0045","doi-asserted-by":"crossref","first-page":"656","DOI":"10.1016\/j.ins.2014.02.154","article-title":"A chess rating system for evolutionary algorithms: a new method for the comparison and ranking of evolutionary algorithms","volume":"277","author":"Ve\u010dek","year":"2014","journal-title":"Inf. Sci. (Ny)"},{"key":"10.1016\/j.ins.2017.07.015_bib0046","doi-asserted-by":"crossref","DOI":"10.1016\/j.asoc.2017.01.011","article-title":"On the influence of the number of algorithms, problems, and independent runs in the comparison of evolutionary algorithms","author":"Ve\u010dek","year":"2017","journal-title":"Appl. Soft. Comput."},{"issue":"2","key":"10.1016\/j.ins.2017.07.015_bib0047","doi-asserted-by":"crossref","first-page":"261","DOI":"10.1016\/j.rcim.2007.08.002","article-title":"A novel hybrid immune algorithm for global optimization in design and manufacturing","volume":"25","author":"Y\u0131ld\u0131z","year":"2009","journal-title":"Robot. Comput. Integr. Manuf."},{"issue":"1","key":"10.1016\/j.ins.2017.07.015_bib0048","doi-asserted-by":"crossref","first-page":"327","DOI":"10.1016\/j.engappai.2012.05.014","article-title":"Comparison of evolutionary-based optimization algorithms for structural design optimization","volume":"26","author":"Yildiz","year":"2013","journal-title":"Eng. Appl. Artif. Intell."},{"issue":"3","key":"10.1016\/j.ins.2017.07.015_bib0049","doi-asserted-by":"crossref","first-page":"1561","DOI":"10.1016\/j.asoc.2011.12.016","article-title":"A new hybrid differential evolution algorithm for the selection of optimal machining parameters in milling operations","volume":"13","author":"Yildiz","year":"2013","journal-title":"Appl. Soft. Comput."},{"issue":"1\u20133","key":"10.1016\/j.ins.2017.07.015_bib0050","doi-asserted-by":"crossref","first-page":"208","DOI":"10.1504\/IJVD.2017.082603","article-title":"A comparative investigation of eight recent population-based optimisation algorithms for mechanical and structural design problems","volume":"73","author":"Y\u0131ld\u0131z","year":"2017","journal-title":"Int. J. Veh. Des."},{"key":"10.1016\/j.ins.2017.07.015_bib0051","series-title":"Biostatistical Analysis","author":"Zar","year":"1999"}],"container-title":["Information Sciences"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/api.elsevier.com\/content\/article\/PII:S002002551631194X?httpAccept=text\/xml","content-type":"text\/xml","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/api.elsevier.com\/content\/article\/PII:S002002551631194X?httpAccept=text\/plain","content-type":"text\/plain","content-version":"vor","intended-application":"text-mining"}],"deposited":{"date-parts":[[2025,6,22]],"date-time":"2025-06-22T18:20:30Z","timestamp":1750616430000},"score":1,"resource":{"primary":{"URL":"https:\/\/linkinghub.elsevier.com\/retrieve\/pii\/S002002551631194X"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2017,11]]},"references-count":51,"alternative-id":["S002002551631194X"],"URL":"https:\/\/doi.org\/10.1016\/j.ins.2017.07.015","relation":{},"ISSN":["0020-0255"],"issn-type":[{"value":"0020-0255","type":"print"}],"subject":[],"published":{"date-parts":[[2017,11]]},"assertion":[{"value":"Elsevier","name":"publisher","label":"This article is maintained by"},{"value":"A Novel Approach to statistical comparison of meta-heuristic stochastic optimization algorithms using deep statistics","name":"articletitle","label":"Article Title"},{"value":"Information Sciences","name":"journaltitle","label":"Journal Title"},{"value":"https:\/\/doi.org\/10.1016\/j.ins.2017.07.015","name":"articlelink","label":"CrossRef DOI link to publisher maintained version"},{"value":"article","name":"content_type","label":"Content Type"},{"value":"\u00a9 2017 Elsevier Inc. All rights reserved.","name":"copyright","label":"Copyright"}]}}