{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,11]],"date-time":"2026-07-11T05:54:05Z","timestamp":1783749245424,"version":"3.55.0"},"publisher-location":"Cham","reference-count":34,"publisher":"Springer International Publishing","isbn-type":[{"value":"9783030581145","type":"print"},{"value":"9783030581152","type":"electronic"}],"license":[{"start":{"date-parts":[[2020,1,1]],"date-time":"2020-01-01T00:00:00Z","timestamp":1577836800000},"content-version":"tdm","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"},{"start":{"date-parts":[[2020,1,1]],"date-time":"2020-01-01T00:00:00Z","timestamp":1577836800000},"content-version":"vor","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2020]]},"DOI":"10.1007\/978-3-030-58115-2_10","type":"book-chapter","created":{"date-parts":[[2020,9,1]],"date-time":"2020-09-01T22:02:51Z","timestamp":1598997771000},"page":"139-153","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":58,"title":["Exploratory Landscape Analysis is Strongly Sensitive to the Sampling Strategy"],"prefix":"10.1007","author":[{"given":"Quentin","family":"Renau","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Carola","family":"Doerr","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Johann","family":"Dreo","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Benjamin","family":"Doerr","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2020,9,2]]},"reference":[{"key":"10_CR1","doi-asserted-by":"publisher","unstructured":"Beachkofski, B., Grandhi, R.: Improved distributed hypercube sampling. In: 43rd AIAA\/ASME\/ASCE\/AHS\/ASC Structures, Structural Dynamics, and Materials Conference. American Institute of Aeronautics and Astronautics (2002). \nhttps:\/\/doi.org\/10.2514\/6.2002-1274","DOI":"10.2514\/6.2002-1274"},{"key":"10_CR2","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"17","DOI":"10.1007\/978-3-319-50349-3_2","volume-title":"Learning and Intelligent Optimization","author":"N Belkhir","year":"2016","unstructured":"Belkhir, N., Dr\u00e9o, J., Sav\u00e9ant, P., Schoenauer, M.: Surrogate assisted feature computation for continuous problems. In: Festa, P., Sellmann, M., Vanschoren, J. (eds.) LION 2016. LNCS, vol. 10079, pp. 17\u201331. Springer, Cham (2016). \nhttps:\/\/doi.org\/10.1007\/978-3-319-50349-3_2"},{"key":"10_CR3","doi-asserted-by":"publisher","unstructured":"Belkhir, N., Dr\u00e9o, J., Sav\u00e9ant, P., Schoenauer, M.: Per instance algorithm configuration of CMA-ES with limited budget. In: Proceedings of Genetic and Evolutionary Computation Conference (GECCO), pp. 681\u2013688. ACM (2017). \nhttps:\/\/doi.org\/10.1145\/3071178.3071343","DOI":"10.1145\/3071178.3071343"},{"key":"10_CR4","unstructured":"Cauwet, M., et al.: Fully parallel hyperparameter search: reshaped space-filling. CoRR abs\/1910.08406 (2019). \nhttp:\/\/arxiv.org\/abs\/1910.08406"},{"key":"10_CR5","doi-asserted-by":"publisher","unstructured":"Derbel, B., Liefooghe, A., V\u00e9rel, S., Aguirre, H., Tanaka, K.: New features for continuous exploratory landscape analysis based on the SOO tree. In: Proceedings of Foundations of Genetic Algorithms (FOGA), pp. 72\u201386. ACM (2019). \nhttps:\/\/doi.org\/10.1145\/3299904.3340308","DOI":"10.1145\/3299904.3340308"},{"key":"10_CR6","doi-asserted-by":"publisher","DOI":"10.1017\/CBO9780511761188","volume-title":"Digital Nets and Sequences","author":"J Dick","year":"2010","unstructured":"Dick, J., Pillichshammer, F.: Digital Nets and Sequences. Cambridge University Press, Cambridge (2010)"},{"key":"10_CR7","doi-asserted-by":"publisher","unstructured":"Doerr, C., Dr\u00e9o, J., Kerschke, P.: Making a case for (hyper-)parameter tuning as benchmark problems. In: Proceedings of Genetic and Evolutionary Computation Conference (GECCO, Companion), pp. 1755\u20131764. AMC (2019). \nhttps:\/\/doi.org\/10.1145\/3319619.3326857","DOI":"10.1145\/3319619.3326857"},{"key":"10_CR8","doi-asserted-by":"publisher","unstructured":"Garden, R.W., Engelbrecht, A.P.: Analysis and classification of optimisation benchmark functions and benchmark suites. In: Proceedings of IEEE Congress on Evolutionary Computation (CEC 2014), pp. 1641\u20131649. IEEE (2014). \nhttps:\/\/doi.org\/10.1109\/CEC.2014.6900240","DOI":"10.1109\/CEC.2014.6900240"},{"key":"10_CR9","unstructured":"Hansen, N., Auger, A., Mersmann, O., Tusar, T., Brockhoff, D.: COCO: a platform for comparing continuous optimizers in a black-box setting. CoRR abs\/1603.08785 (2016). \nhttp:\/\/arxiv.org\/abs\/1603.08785"},{"key":"10_CR10","unstructured":"Hansen, N., Finck, A.A.S., Ros, R.: Real-Parameter Black-box Optimization Benchmarking: Experimental Setup. RR-7215, INRIA (2010). \nhttps:\/\/hal.inria.fr\/inria-00462481"},{"key":"10_CR11","series-title":"The Springer Series on Challenges in Machine Learning","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-05318-5","volume-title":"Automated Machine Learning","year":"2019","unstructured":"Hutter, F., Kotthoff, L., Vanschoren, J. (eds.): Automated Machine Learning. TSSCML. Springer, Cham (2019). \nhttps:\/\/doi.org\/10.1007\/978-3-030-05318-5"},{"key":"10_CR12","doi-asserted-by":"crossref","unstructured":"Jankovic, A., Doerr, C.: Adaptive landscape analysis. In: Proceedings of the Genetic and Evolutionary Computation Conference Companion (GECCO 2019), pp. 2032\u20132035. ACM (2019)","DOI":"10.1145\/3319619.3326905"},{"issue":"1","key":"10_CR13","doi-asserted-by":"publisher","first-page":"3","DOI":"10.1162\/evco_a_00242","volume":"27","author":"P Kerschke","year":"2019","unstructured":"Kerschke, P., Hoos, H., Neumann, F., Trautmann, H.: Automated algorithm selection: survey and Perspectives. Evol. Comput. 27(1), 3\u201345 (2019)","journal-title":"Evol. Comput."},{"key":"10_CR14","doi-asserted-by":"publisher","unstructured":"Kerschke, P., Preuss, M., Wessing, S., Trautmann, H.: Low-budget exploratory landscape analysis on multiple peaks models. In: Proceedings of Genetic and Evolutionary Computation Conference (GECCO), pp. 229\u2013236. ACM (2016). \nhttps:\/\/doi.org\/10.1145\/2908812.2908845","DOI":"10.1145\/2908812.2908845"},{"key":"10_CR15","unstructured":"Kerschke, P., Trautmann, H.: The R-package FLACCO for exploratory landscape analysis with applications to multi-objective optimization problems. In: Proceedings of CEC, pp. 5262\u20135269. IEEE (2016). flacco is available at \nhttp:\/\/kerschke.github.io\/flacco\/"},{"issue":"1","key":"10_CR16","doi-asserted-by":"publisher","first-page":"99","DOI":"10.1162\/evco_a_00236","volume":"27","author":"P Kerschke","year":"2019","unstructured":"Kerschke, P., Trautmann, H.: Automated algorithm selection on continuous black-box problems by combining exploratory landscape analysis and machine learning. Evol. Comput. 27(1), 99\u2013127 (2019). \nhttps:\/\/doi.org\/10.1162\/evco_a_00236","journal-title":"Evol. Comput."},{"key":"10_CR17","volume-title":"The Art of Computer Programming: Seminumerical Algorithms","author":"D Knuth","year":"1998","unstructured":"Knuth, D.: The Art of Computer Programming: Seminumerical Algorithms. Addison-Wesley, Boston (1998)"},{"key":"10_CR18","doi-asserted-by":"publisher","first-page":"148","DOI":"10.1016\/j.ins.2013.04.015","volume":"241","author":"K Malan","year":"2013","unstructured":"Malan, K., Engelbrecht, A.P.: A survey of techniques for characterising fitness landscapes and some possible ways forward. Inf. Sci. 241, 148\u2013163 (2013). \nhttps:\/\/doi.org\/10.1016\/j.ins.2013.04.015","journal-title":"Inf. Sci."},{"key":"10_CR19","volume-title":"Geometric Discrepancy","author":"J Matou\u0161ek","year":"2009","unstructured":"Matou\u0161ek, J.: Geometric Discrepancy, 2nd edn. Springer, Heidelberg (2009)","edition":"2"},{"issue":"1","key":"10_CR20","doi-asserted-by":"publisher","first-page":"3","DOI":"10.1145\/272991.272995","volume":"8","author":"M Matsumoto","year":"1998","unstructured":"Matsumoto, M., Nishimura, T.: Mersenne twister: A 623-dimensionally equidistributed uniform pseudo-random number generator. ACM Trans. Model. Comput. Simul. 8(1), 3\u201330 (1998). \nhttps:\/\/doi.org\/10.1145\/272991.272995","journal-title":"ACM Trans. Model. Comput. Simul."},{"key":"10_CR21","first-page":"239","volume":"21","author":"M McKay","year":"1979","unstructured":"McKay, M., Beckman, R., Conover, W.: A comparison of three methods for selecting values of input variables in the analysis of output from a computer code. Technometrics 21, 239\u2013245 (1979). \nhttp:\/\/www.jstor.org\/stable\/1268522","journal-title":"Technometrics"},{"key":"10_CR22","doi-asserted-by":"publisher","unstructured":"Mersmann, O., Bischl, B., Trautmann, H., Preuss, M., Weihs, C., Rudolph, G.: Exploratory landscape analysis. In: Proceedings of Genetic and Evolutionary Computation Conference (GECCO), pp. 829\u2013836. ACM (2011). \nhttps:\/\/doi.org\/10.1145\/2001576.2001690","DOI":"10.1145\/2001576.2001690"},{"issue":"3","key":"10_CR23","doi-asserted-by":"publisher","first-page":"456","DOI":"10.1109\/TEVC.2013.2281521","volume":"18","author":"R Morgan","year":"2014","unstructured":"Morgan, R., Gallagher, M.: Sampling techniques and distance metrics in high dimensional continuous landscape analysis: limitations and improvements. IEEE Trans. Evol. Comput. 18(3), 456\u2013461 (2014). \nhttps:\/\/doi.org\/10.1109\/TEVC.2013.2281521","journal-title":"IEEE Trans. Evol. Comput."},{"key":"10_CR24","doi-asserted-by":"publisher","first-page":"224","DOI":"10.1016\/j.ins.2015.05.010","volume":"317","author":"MA Mu\u00f1oz","year":"2015","unstructured":"Mu\u00f1oz, M.A., Sun, Y., Kirley, M., Halgamuge, S.K.: Algorithm selection for black-box continuous optimization problems: a survey on methods and challenges. Inf. Sci. 317, 224\u2013245 (2015). \nhttps:\/\/doi.org\/10.1016\/j.ins.2015.05.010","journal-title":"Inf. Sci."},{"key":"10_CR25","first-page":"2825","volume":"12","author":"F Pedregosa","year":"2011","unstructured":"Pedregosa, F., et al.: Scikit-learn: machine learning in Python. JMLR 12, 2825\u20132830 (2011)","journal-title":"JMLR"},{"key":"10_CR26","series-title":"Studies in Computational Intelligence","doi-asserted-by":"publisher","first-page":"161","DOI":"10.1007\/978-3-642-23229-9_8","volume-title":"Recent Advances in Intelligent Engineering Systems","author":"E Pitzer","year":"2012","unstructured":"Pitzer, E., Affenzeller, M.: A comprehensive survey on fitness landscape analysis. In: Fodor, J., Klempous, R., Su\u00e1rez Araujo, C.P. (eds.) Recent Advances in Intelligent Engineering Systems. Studies in Computational Intelligence, vol. 378, pp. 161\u2013191. Springer, Heidelberg (2012). \nhttps:\/\/doi.org\/10.1007\/978-3-642-23229-9_8"},{"key":"10_CR27","doi-asserted-by":"publisher","unstructured":"Renau, Q., Dreo, J., Doerr, C., Doerr, B.: Expressiveness and robustness of landscape features. In: Proceedings of Genetic and Evolutionary Computation Conference (GECCO, Companion), pp. 2048\u20132051. ACM (2019). \nhttps:\/\/doi.org\/10.1145\/3319619.3326913","DOI":"10.1145\/3319619.3326913"},{"key":"10_CR28","doi-asserted-by":"publisher","unstructured":"Renau, Q., Doerr, C., Dreo, J., Doerr, B.: Experimental data set for the study \u201cexploratory landscape analysis is strongly sensitive to the sampling strategy\u201d, June 2020. \nhttps:\/\/doi.org\/10.5281\/zenodo.3886816","DOI":"10.5281\/zenodo.3886816"},{"issue":"1","key":"10_CR29","doi-asserted-by":"publisher","first-page":"75","DOI":"10.1162\/evco_a_00247","volume":"27","author":"S Saleem","year":"2019","unstructured":"Saleem, S., Gallagher, M., Wood, I.: Direct feature evaluation in black-box optimization using problem transformations. Evol. Comput. 27(1), 75\u201398 (2019)","journal-title":"Evol. Comput."},{"key":"10_CR30","series-title":"Springer Series in Statistics","doi-asserted-by":"publisher","DOI":"10.1007\/978-1-4757-3799-8","volume-title":"The Design and Analysis of Computer Experiments","author":"T Santner","year":"2003","unstructured":"Santner, T., Williams, B., Notz, W.: The Design and Analysis of Computer Experiments. Springer Series in Statistics. Springer, New York (2003). \nhttps:\/\/doi.org\/10.1007\/978-1-4757-3799-8"},{"key":"10_CR31","doi-asserted-by":"publisher","unstructured":"Smith-Miles, K.A.: Cross-disciplinary perspectives on meta-learning for algorithm selection. ACM Comput. Surv. 41(1) (2009). \nhttps:\/\/doi.org\/10.1145\/1456650.1456656","DOI":"10.1145\/1456650.1456656"},{"issue":"4","key":"10_CR32","doi-asserted-by":"publisher","first-page":"86","DOI":"10.1016\/0041-5553(67)90144-9","volume":"7","author":"I Sobol\u2019","year":"1967","unstructured":"Sobol\u2019, I.: On the distribution of points in a cube and the approximate evaluation of integrals. USSR Comput. Math. Math. Phys. 7(4), 86\u2013112 (1967). \nhttps:\/\/doi.org\/10.1016\/0041-5553(67)90144-9","journal-title":"USSR Comput. Math. Math. Phys."},{"key":"10_CR33","unstructured":"Vallati, M., Hutter, F., Chrpa, L., McCluskey, T.: On the effective configuration of planning domain models. In: Proceedings of IJCAI 2015. AAAI (2015)"},{"key":"10_CR34","doi-asserted-by":"publisher","first-page":"565","DOI":"10.1613\/jair.2490","volume":"32","author":"L Xu","year":"2008","unstructured":"Xu, L., Hutter, F., Hoos, H., Leyton-Brown, K.: SATzilla: portfolio-based algorithm selection for sat. JAIR 32, 565\u2013606 (2008). \nhttp:\/\/dl.acm.org\/citation.cfm?id=1622673.1622687","journal-title":"JAIR"}],"container-title":["Lecture Notes in Computer Science","Parallel Problem Solving from Nature \u2013 PPSN XVI"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-030-58115-2_10","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2020,9,1]],"date-time":"2020-09-01T22:05:06Z","timestamp":1598997906000},"score":1,"resource":{"primary":{"URL":"http:\/\/link.springer.com\/10.1007\/978-3-030-58115-2_10"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2020]]},"ISBN":["9783030581145","9783030581152"],"references-count":34,"URL":"https:\/\/doi.org\/10.1007\/978-3-030-58115-2_10","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"value":"0302-9743","type":"print"},{"value":"1611-3349","type":"electronic"}],"subject":[],"published":{"date-parts":[[2020]]},"assertion":[{"value":"2 September 2020","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"PPSN","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"International Conference on Parallel Problem Solving from Nature","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Leiden","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"The Netherlands","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2020","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"5 September 2020","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"9 September 2020","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"16","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"ppsn2020","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"https:\/\/ppsn2020.liacs.leidenuniv.nl\/","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Single-blind","order":1,"name":"type","label":"Type","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"Easychair","order":2,"name":"conference_management_system","label":"Conference Management System","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"268","order":3,"name":"number_of_submissions_sent_for_review","label":"Number of Submissions Sent for Review","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"99","order":4,"name":"number_of_full_papers_accepted","label":"Number of Full Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"0","order":5,"name":"number_of_short_papers_accepted","label":"Number of Short Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"37% - The value is computed by the equation \"Number of Full Papers Accepted \/ Number of Submissions Sent for Review * 100\" and then rounded to a whole number.","order":6,"name":"acceptance_rate_of_full_papers","label":"Acceptance Rate of Full Papers","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"3","order":7,"name":"average_number_of_reviews_per_paper","label":"Average Number of Reviews per Paper","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"2.2","order":8,"name":"average_number_of_papers_per_reviewer","label":"Average Number of Papers per Reviewer","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"No","order":9,"name":"external_reviewers_involved","label":"External Reviewers Involved","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}}]}}