{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,10,17]],"date-time":"2025-10-17T14:01:20Z","timestamp":1760709680263,"version":"3.37.3"},"reference-count":42,"publisher":"Springer Science and Business Media LLC","issue":"4-5","license":[{"start":{"date-parts":[[2018,8,10]],"date-time":"2018-08-10T00:00:00Z","timestamp":1533859200000},"content-version":"tdm","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"}],"funder":[{"name":"COMEX, Combinatorial Optimization: Metaheuristics & Exact Methods"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["J Heuristics"],"published-print":{"date-parts":[[2019,10]]},"DOI":"10.1007\/s10732-018-9388-7","type":"journal-article","created":{"date-parts":[[2018,8,10]],"date-time":"2018-08-10T11:45:34Z","timestamp":1533901534000},"page":"591-628","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":4,"title":["A combined approach for analysing heuristic algorithms"],"prefix":"10.1007","volume":"25","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-1717-5475","authenticated-orcid":false,"given":"Jeroen","family":"Corstjens","sequence":"first","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0002-2693-6953","authenticated-orcid":false,"given":"Nguyen","family":"Dang","sequence":"additional","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0003-4735-0609","authenticated-orcid":false,"given":"Beno\u00eet","family":"Depaire","sequence":"additional","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0001-6004-8776","authenticated-orcid":false,"given":"An","family":"Caris","sequence":"additional","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0002-6763-1945","authenticated-orcid":false,"given":"Patrick","family":"De Causmaecker","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2018,8,10]]},"reference":[{"key":"9388_CR1","doi-asserted-by":"crossref","unstructured":"Ans\u00f3tegui, C., Sellmann, M., Tierney, K.: A gender-based genetic algorithm for the automatic configuration of algorithms. In: International Conference on Principles and Practice of Constraint Programming, pp. 142\u2013157. Springer (2009)","DOI":"10.1007\/978-3-642-04244-7_14"},{"issue":"2","key":"9388_CR2","doi-asserted-by":"publisher","first-page":"413","DOI":"10.1002\/anac.200410007","volume":"1","author":"T Bartz-Beielstein","year":"2004","unstructured":"Bartz-Beielstein, T., Parsopoulos, K.E., Vrahatis, M.N.: Design and analysis of optimization algorithms using computational statistics. Appl. Numer. Anal. Comput. Math. 1(2), 413\u2013433 (2004)","journal-title":"Appl. Numer. Anal. Comput. Math."},{"issue":"2","key":"9388_CR3","doi-asserted-by":"publisher","first-page":"286","DOI":"10.1287\/opre.44.2.286","volume":"44","author":"DJ Bertsimas","year":"1996","unstructured":"Bertsimas, D.J., Simchi-Levi, D.: A new generation of vehicle routing research: robust algorithms, addressing uncertainty. Oper. Res. 44(2), 286 (1996)","journal-title":"Oper. Res."},{"key":"9388_CR4","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-00483-4","volume-title":"Tuning Metaheuristics, Studies in Computational Intelligence","author":"M Birattari","year":"2009","unstructured":"Birattari, M.: Tuning Metaheuristics, Studies in Computational Intelligence, vol. 197. Springer, Berlin (2009)"},{"key":"9388_CR5","unstructured":"Burke, E.K., Bykov, Y.: A late acceptance strategy in hill-climbing for exam timetabling problems. In: PATAT 2008 Conference, Montreal, Canada (2008)"},{"key":"9388_CR6","unstructured":"Bykov, Y., Petrovic, S.: An initial study of a novel step counting hill climbing heuristic applied to timetabling problems. In: Proceedings of 6th Multidisciplinary International Scheduling Conference (MISTA 2013) (2013)"},{"key":"9388_CR7","doi-asserted-by":"publisher","first-page":"225","DOI":"10.1007\/978-3-642-02538-9_10","volume":"1","author":"M Chiarandini","year":"2010","unstructured":"Chiarandini, M., Goegebeur, Y.: Mixed models for the analysis of optimization algorithms. Exp. Methods Anal. Optim. Algorithms 1, 225 (2010)","journal-title":"Exp. Methods Anal. Optim. Algorithms"},{"key":"9388_CR8","volume-title":"Learning and Intelligent Optimization","author":"J Corstjens","year":"2018","unstructured":"Corstjens, J., Caris, A., Depaire, B.: Explaining heuristic performance differences for vehicle routing problems with time windows. In: Kotsireas, S., Pardalos, P.M. (eds.) Learning and Intelligent Optimization. Lecture Notes in Computer Science. Springer, Berlin (2018). (in press)"},{"key":"9388_CR9","unstructured":"Corstjens, J., Depaire, B., Caris, A., S\u00f6rensen, K.: A multilevel evaluation method for heuristics with an application to the VRPTW. Manuscript submitted for publication (2017)"},{"issue":"1","key":"9388_CR10","doi-asserted-by":"publisher","first-page":"77","DOI":"10.1023\/A:1026569813391","volume":"7","author":"SP Coy","year":"2001","unstructured":"Coy, S.P., Golden, B.L., Runger, G.C., Wasil, E.A.: Using experimental design to find effective parameter settings for heuristics. J. Heuristics 7(1), 77\u201397 (2001)","journal-title":"J. Heuristics"},{"key":"9388_CR11","doi-asserted-by":"publisher","DOI":"10.1007\/978-0-387-73186-5","volume-title":"Handbook of Multilevel Analysis","author":"J Leeuw De","year":"2008","unstructured":"De Leeuw, J., Meijer, E., Goldstein, H.: Handbook of Multilevel Analysis. Springer, Berlin (2008)"},{"issue":"3","key":"9388_CR12","doi-asserted-by":"publisher","first-page":"326","DOI":"10.1145\/212094.212114","volume":"27","author":"T Dietterich","year":"1995","unstructured":"Dietterich, T.: Overfitting and undercomputing in machine learning. ACM Comput. Surv. CSUR 27(3), 326\u2013327 (1995)","journal-title":"ACM Comput. Surv. CSUR"},{"key":"9388_CR13","first-page":"1","volume":"22","author":"C Fawcett","year":"2015","unstructured":"Fawcett, C., Hoos, H.H.: Analysing differences between algorithm configurations through ablation. J. Heuristics 22, 1\u201328 (2015)","journal-title":"J. Heuristics"},{"key":"9388_CR14","doi-asserted-by":"publisher","DOI":"10.1017\/CBO9780511790942","volume-title":"Data Analysis Using Regression and Multilevel\/Hierarchical Models","author":"A Gelman","year":"2006","unstructured":"Gelman, A., Hill, J.: Data Analysis Using Regression and Multilevel\/Hierarchical Models. Cambridge University Press, Cambridge (2006)"},{"key":"9388_CR15","volume-title":"Multivariate Data Analysis: A Global Perspective","author":"JF Hair","year":"2010","unstructured":"Hair, J.F., Anderson, R.E., Babin, B.J., Black, W.C.: Multivariate Data Analysis: A Global Perspective, vol. 7. Pearson, Upper Saddle River, NJ (2010)"},{"key":"9388_CR16","doi-asserted-by":"publisher","first-page":"709","DOI":"10.1198\/106186007X237892","volume":"16","author":"G Hooker","year":"2012","unstructured":"Hooker, G.: Generalized functional anova diagnostics for high-dimensional functions of dependent variables. J. Comput. Graph. Stat. 16, 709\u2013732 (2012)","journal-title":"J. Comput. Graph. Stat."},{"issue":"1","key":"9388_CR17","doi-asserted-by":"publisher","first-page":"33","DOI":"10.1007\/BF02430364","volume":"1","author":"JN Hooker","year":"1995","unstructured":"Hooker, J.N.: Testing heuristics: we have it all wrong. J. Heuristics 1(1), 33\u201342 (1995)","journal-title":"J. Heuristics"},{"key":"9388_CR18","doi-asserted-by":"publisher","first-page":"507","DOI":"10.1007\/978-3-642-25566-3_40","volume-title":"Learning and Intelligent Optimization, Lecture Notes in Computer Science","author":"F Hutter","year":"2011","unstructured":"Hutter, F., Hoos, H.H., Leyton-Brown, K.: Sequential model-based optimization for general algorithm configuration. In: Coello, C.A.C. (ed.) Learning and Intelligent Optimization, Lecture Notes in Computer Science, vol. 6683, pp. 507\u2013523. Springer, Berlin (2011)"},{"key":"9388_CR19","doi-asserted-by":"crossref","unstructured":"Hutter, F., Hoos, H.H., Leyton-Brown, K.: Identifying key algorithm parameters and instance features using forward selection. In: International Conference on Learning and Intelligent Optimization, pp. 364\u2013381. Springer (2013)","DOI":"10.1007\/978-3-642-44973-4_40"},{"key":"9388_CR20","unstructured":"Hutter, F., Hoos, H.H., Leyton-Brown, K.: An efficient approach for assessing hyperparameter importance. In: International Conference on Machine Learning, pp. 754\u2013762 (2014)"},{"issue":"1","key":"9388_CR21","doi-asserted-by":"publisher","first-page":"267","DOI":"10.1613\/jair.2861","volume":"36","author":"F Hutter","year":"2009","unstructured":"Hutter, F., Hoos, H.H., Leyton-Brown, K., St\u00fctzle, T.: Paramils: an automatic algorithm configuration framework. J. Artif. Intell. Res. 36(1), 267\u2013306 (2009)","journal-title":"J. Artif. Intell. Res."},{"key":"9388_CR22","unstructured":"Jones, Z., Linder, F.: Exploratory data analysis using random forests. In: Prepared for the 73rd Annual MPSA Conference (2015)"},{"issue":"4598","key":"9388_CR23","first-page":"671","volume":"220","author":"S Kirkpatrick","year":"1983","unstructured":"Kirkpatrick, S., Gelatt, C.D., Vecchi, M.P.: Optimization by simulated annealing. science 220(4598), 671\u2013680 (1983)","journal-title":"Optimization by simulated annealing. science"},{"key":"9388_CR24","doi-asserted-by":"publisher","DOI":"10.1201\/9781315374529","volume-title":"Basic Experimental Strategies and Data Analysis for Science and Engineering","author":"J Lawson","year":"2016","unstructured":"Lawson, J., Erjavec, J.: Basic Experimental Strategies and Data Analysis for Science and Engineering. CRC Press, Boca Raton (2016)"},{"issue":"4","key":"9388_CR25","doi-asserted-by":"publisher","first-page":"22","DOI":"10.1145\/1538902.1538906","volume":"56","author":"K Leyton-Brown","year":"2009","unstructured":"Leyton-Brown, K., Nudelman, E., Shoham, Y.: Empirical hardness models: methodology and a case study on combinatorial auctions. J. ACM 56(4), 22 (2009)","journal-title":"J. ACM"},{"key":"9388_CR26","doi-asserted-by":"publisher","first-page":"43","DOI":"10.1016\/j.orp.2016.09.002","volume":"3","author":"M L\u00f3pez-Ib\u00e1\u00f1ez","year":"2016","unstructured":"L\u00f3pez-Ib\u00e1\u00f1ez, M., Dubois-Lacoste, J., C\u00e1ceres, L.P., Birattari, M., St\u00fctzle, T.: The IRACE package: iterated racing for automatic algorithm configuration. Oper. Res. Perspect. 3, 43\u201358 (2016)","journal-title":"Oper. Res. Perspect."},{"key":"9388_CR27","doi-asserted-by":"publisher","first-page":"320","DOI":"10.1007\/0-306-48056-5_11","volume-title":"Handbook of Metaheuristics","author":"HR Louren\u00e7o","year":"2003","unstructured":"Louren\u00e7o, H.R., Martin, O.C., St\u00fctzle, T.: Iterated local search. In: Glover, F.W., Kochenberger, G.A. (eds.) Handbook of Metaheuristics, pp. 320\u2013353. Springer, Berlin (2003)"},{"key":"9388_CR28","volume-title":"Design and Analysis of Experiments","author":"D Montgomery","year":"2012","unstructured":"Montgomery, D.: Design and Analysis of Experiments, 8th edn. Wiley, New York (2012)","edition":"8"},{"key":"9388_CR29","volume-title":"Introduction to the Practice of Statistics","author":"DS Moore","year":"2007","unstructured":"Moore, D.S., McCabe, G.P., Craig, B.A.: Introduction to the Practice of Statistics, 6th edn. W. H. Freeman, New York (2007)","edition":"6"},{"key":"9388_CR30","doi-asserted-by":"crossref","unstructured":"Nannen, V., Eiben, A.E.: Relevance estimation and value calibration of evolutionary algorithm parameters. In: International Joint Conference on Artificial Intelligence, vol. 7, pp. 975\u2013980 (2007)","DOI":"10.1109\/CEC.2007.4424460"},{"key":"9388_CR31","unstructured":"PassMark Software: CPU benchmarks. https:\/\/www.cpubenchmark.net\/ (2018). Accessed 26 Mar 2018"},{"key":"9388_CR32","unstructured":"Pellegrini, P., Birattari, M.: The relevance of tuning the parameters of metaheuristics. A case study: the vehicle routing problem with stochastic demand. Technical report TR\/IRIDIA\/2006-008, IRIDIA, Universit Libre de Bruxelles, Brussels, Belgium (2006)"},{"issue":"8","key":"9388_CR33","doi-asserted-by":"publisher","first-page":"2403","DOI":"10.1016\/j.cor.2005.09.012","volume":"34","author":"D Pisinger","year":"2007","unstructured":"Pisinger, D., Ropke, S.: A general heuristic for vehicle routing problems. Comput. Oper. Res. 34(8), 2403\u20132435 (2007)","journal-title":"Comput. Oper. Res."},{"issue":"3","key":"9388_CR34","doi-asserted-by":"publisher","first-page":"261","DOI":"10.1023\/A:1011319115230","volume":"7","author":"RL Rardin","year":"2001","unstructured":"Rardin, R.L., Uzsoy, R.: Experimental evaluation of heuristic optimization algorithms: a tutorial. J. Heuristics 7(3), 261\u2013304 (2001)","journal-title":"J. Heuristics"},{"key":"9388_CR35","doi-asserted-by":"crossref","first-page":"191","DOI":"10.1007\/978-94-017-9054-3_11","volume-title":"Modeling, Simulation and Optimization for Science and Technology","author":"J Rasku","year":"2014","unstructured":"Rasku, J., Musliu, N., K\u00e4rkk\u00e4inen, T.: Automating the parameter selection in VRP: an off-line parameter tuning tool comparison. In: Fitzgibbon, W., Kuznetsov, Y.A., Neittaanm\u00e4ki, P., Pironneau, O. (eds.) Modeling, Simulation and Optimization for Science and Technology, pp. 191\u2013209. Springer, Berlin (2014)"},{"key":"9388_CR36","doi-asserted-by":"publisher","unstructured":"Santos, H.G., Toffolo, T.A., Silva, C.L., Vanden\u00a0Berghe, G.: Analysis of stochastic local search methods for the unrelated parallel machine scheduling problem. Int. Trans. Oper. Res. (2016). https:\/\/doi.org\/10.1111\/itor.12316","DOI":"10.1111\/itor.12316"},{"issue":"11","key":"9388_CR37","doi-asserted-by":"publisher","first-page":"1359","DOI":"10.1177\/0956797611417632","volume":"22","author":"JP Simmons","year":"2011","unstructured":"Simmons, J.P., Nelson, L.D., Simonsohn, U.: False-positive psychology: undisclosed flexibility in data collection and analysis allows presenting anything as significant. Psychol. Sci. 22(11), 1359\u20131366 (2011)","journal-title":"Psychol. Sci."},{"key":"9388_CR38","doi-asserted-by":"publisher","first-page":"102","DOI":"10.1016\/j.cor.2015.04.022","volume":"63","author":"K Smith-Miles","year":"2015","unstructured":"Smith-Miles, K., Bowly, S.: Generating new test instances by evolving in instance space. Comput. Oper. Res. 63, 102\u2013113 (2015)","journal-title":"Comput. Oper. Res."},{"issue":"2","key":"9388_CR39","doi-asserted-by":"publisher","first-page":"254","DOI":"10.1287\/opre.35.2.254","volume":"35","author":"MM Solomon","year":"1987","unstructured":"Solomon, M.M.: Algorithms for the vehicle routing and scheduling problems with time window constraints. Oper. Res. 35(2), 254\u2013265 (1987)","journal-title":"Oper. Res."},{"key":"9388_CR40","volume-title":"Introduction to Econometrics","author":"J Stock","year":"2011","unstructured":"Stock, J., Watson, M.W.: Introduction to Econometrics. Prentice Hall, New York (2011)"},{"issue":"3","key":"9388_CR41","doi-asserted-by":"publisher","first-page":"279","DOI":"10.4300\/JGME-D-12-00156.1","volume":"4","author":"GM Sullivan","year":"2012","unstructured":"Sullivan, G.M., Feinn, R.: Using effect size\u2014or why the p value is not enough. J. Grad. Med. Educ. 4(3), 279\u2013282 (2012)","journal-title":"J. Grad. Med. Educ."},{"issue":"3","key":"9388_CR42","doi-asserted-by":"publisher","first-page":"612","DOI":"10.1016\/j.ejor.2011.01.011","volume":"211","author":"E Vallada","year":"2011","unstructured":"Vallada, E., Ruiz, R.: A genetic algorithm for the unrelated parallel machine scheduling problem with sequence dependent setup times. Eur. J. Oper. Res. 211(3), 612\u2013622 (2011)","journal-title":"Eur. J. Oper. Res."}],"container-title":["Journal of Heuristics"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s10732-018-9388-7.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/article\/10.1007\/s10732-018-9388-7\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s10732-018-9388-7.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2023,9,4]],"date-time":"2023-09-04T08:19:51Z","timestamp":1693815591000},"score":1,"resource":{"primary":{"URL":"http:\/\/link.springer.com\/10.1007\/s10732-018-9388-7"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2018,8,10]]},"references-count":42,"journal-issue":{"issue":"4-5","published-print":{"date-parts":[[2019,10]]}},"alternative-id":["9388"],"URL":"https:\/\/doi.org\/10.1007\/s10732-018-9388-7","relation":{},"ISSN":["1381-1231","1572-9397"],"issn-type":[{"type":"print","value":"1381-1231"},{"type":"electronic","value":"1572-9397"}],"subject":[],"published":{"date-parts":[[2018,8,10]]},"assertion":[{"value":"7 July 2017","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"17 July 2018","order":2,"name":"revised","label":"Revised","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"2 August 2018","order":3,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"10 August 2018","order":4,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}}]}}