{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,6,16]],"date-time":"2026-06-16T07:51:05Z","timestamp":1781596265892,"version":"3.54.5"},"reference-count":55,"publisher":"Institute of Electrical and Electronics Engineers (IEEE)","issue":"6","license":[{"start":{"date-parts":[[2022,12,1]],"date-time":"2022-12-01T00:00:00Z","timestamp":1669852800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/legalcode"},{"start":{"date-parts":[[2022,12,1]],"date-time":"2022-12-01T00:00:00Z","timestamp":1669852800000},"content-version":"am","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/legalcode"}],"funder":[{"DOI":"10.13039\/501100020639","name":"Bavarian Ministry of Economic Affairs, Regional Development and Energy through the Center for Analytics-Data-Applications (ADACenter) within the Framework of BAYERN DIGITAL II","doi-asserted-by":"publisher","award":["20-3410-2-9-8"],"award-info":[{"award-number":["20-3410-2-9-8"]}],"id":[{"id":"10.13039\/501100020639","id-type":"DOI","asserted-by":"publisher"}]},{"name":"Research Center Trustworthy Data Science and Security"},{"DOI":"10.13039\/100000001","name":"NSF","doi-asserted-by":"publisher","award":["1813537"],"award-info":[{"award-number":["1813537"]}],"id":[{"id":"10.13039\/100000001","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["IEEE Trans. Evol. Computat."],"published-print":{"date-parts":[[2022,12]]},"DOI":"10.1109\/tevc.2022.3211336","type":"journal-article","created":{"date-parts":[[2022,10,6]],"date-time":"2022-10-06T19:39:25Z","timestamp":1665085165000},"page":"1336-1350","source":"Crossref","is-referenced-by-count":3,"title":["Automated Benchmark-Driven Design and Explanation of Hyperparameter Optimizers"],"prefix":"10.1109","volume":"26","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-0000-9297","authenticated-orcid":false,"given":"Julia","family":"Moosbauer","sequence":"first","affiliation":[{"name":"Department of Statistics, Ludwig-Maximilians-Universität München, Munich, Germany"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Martin","family":"Binder","sequence":"additional","affiliation":[{"name":"Department of Statistics, Ludwig-Maximilians-Universität München, Munich, Germany"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-4152-5308","authenticated-orcid":false,"given":"Lennart","family":"Schneider","sequence":"additional","affiliation":[{"name":"Department of Statistics, Ludwig-Maximilians-Universität München, Munich, Germany"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-8867-762X","authenticated-orcid":false,"given":"Florian","family":"Pfisterer","sequence":"additional","affiliation":[{"name":"Department of Statistics, Ludwig-Maximilians-Universität München, Munich, Germany"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Marc","family":"Becker","sequence":"additional","affiliation":[{"name":"Department of Statistics, Ludwig-Maximilians-Universität München, Munich, Germany"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Michel","family":"Lang","sequence":"additional","affiliation":[{"name":"Research Center Trustworthy Data Science and Security, Dortmund, Germany"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-4635-6873","authenticated-orcid":false,"given":"Lars","family":"Kotthoff","sequence":"additional","affiliation":[{"name":"Department of Computer Science, University of Wyoming, Laramie, WY, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-6002-6980","authenticated-orcid":false,"given":"Bernd","family":"Bischl","sequence":"additional","affiliation":[{"name":"Department of Statistics, Ludwig-Maximilians-Universität München, Munich, Germany"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"263","reference":[{"key":"ref39","first-page":"754","article-title":"An efficient approach for assessing hyperparameter importance","volume":"32","author":"hutter","year":"2014","journal-title":"Proc 31st Int Conf Mach Learn"},{"key":"ref38","doi-asserted-by":"publisher","DOI":"10.1214\/aoms\/1177730196"},{"key":"ref33","doi-asserted-by":"publisher","DOI":"10.1109\/SSCI.2016.7850138"},{"key":"ref32","doi-asserted-by":"publisher","DOI":"10.1023\/A:1006556606079"},{"key":"ref31","doi-asserted-by":"publisher","DOI":"10.1613\/jair.2861"},{"key":"ref30","doi-asserted-by":"publisher","DOI":"10.1145\/3071178.3071238"},{"key":"ref37","doi-asserted-by":"publisher","DOI":"10.1111\/0272-4332.00040"},{"key":"ref36","article-title":"SMAC3: A versatile Bayesian optimization package for hyperparameter optimization","author":"lindauer","year":"2021","journal-title":"arXiv 2109 09831"},{"key":"ref35","article-title":"Towards assessing the impact of Bayesian optimization’s own hyperparameters","author":"lindauer","year":"2019","journal-title":"arXiv 1908 06674"},{"key":"ref34","first-page":"5984","article-title":"Automating Bayesian optimization with Bayesian optimization","volume":"31","author":"malkomes","year":"2018","journal-title":"Proc Int Conf Adv Neural Inf Process Syst"},{"key":"ref28","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-02538-9_13"},{"key":"ref27","doi-asserted-by":"publisher","DOI":"10.1109\/TEVC.2015.2474158"},{"key":"ref29","doi-asserted-by":"publisher","DOI":"10.1016\/j.orp.2016.09.002"},{"key":"ref2","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-05318-5_4"},{"key":"ref1","article-title":"Hyperparameter optimization: Foundations, algorithms, best practices and open challenges","author":"bischl","year":"2021","journal-title":"arXiv 2107 05847"},{"key":"ref20","doi-asserted-by":"publisher","DOI":"10.1145\/2076450.2076469"},{"key":"ref22","doi-asserted-by":"publisher","DOI":"10.1017\/S0269888901000029"},{"key":"ref21","doi-asserted-by":"publisher","DOI":"10.1007\/BF00143877"},{"key":"ref24","first-page":"517","article-title":"SATenstein: Automatically building local search SAT solvers from components","author":"khudabukhsh","year":"2009","journal-title":"Proc 21st Int Joint Conf Artif Intell"},{"key":"ref23","doi-asserted-by":"publisher","DOI":"10.1109\/WICSA.2011.37"},{"key":"ref26","doi-asserted-by":"publisher","DOI":"10.1109\/TEVC.2011.2182651"},{"key":"ref25","doi-asserted-by":"publisher","DOI":"10.1007\/978-0-387-88843-9_3"},{"key":"ref50","article-title":"YAHPO gym—Design criteria and a new multifidelity benchmark for hyperparameter optimization","author":"pfisterer","year":"2021","journal-title":"arXiv 2109 03670"},{"key":"ref51","doi-asserted-by":"publisher","DOI":"10.1023\/A:1012771025575"},{"key":"ref55","first-page":"1","article-title":"Statistical comparisons of classifiers over multiple data sets","volume":"7","author":"dem\u0161ar","year":"2006","journal-title":"J Mach Learn Res"},{"key":"ref54","doi-asserted-by":"publisher","DOI":"10.1214\/12-AOS1049"},{"key":"ref53","doi-asserted-by":"publisher","DOI":"10.1023\/A:1010933404324"},{"key":"ref52","doi-asserted-by":"publisher","DOI":"10.1016\/j.ejor.2017.01.035"},{"key":"ref10","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-34413-8_5"},{"key":"ref11","first-page":"173","article-title":"MOI-MBO: Multiobjective infill for parallel model-based optimization","author":"bischl","year":"2014","journal-title":"Proc 8th Int Conf Learn Intell Optim"},{"key":"ref40","doi-asserted-by":"publisher","DOI":"10.1007\/s10732-014-9275-9"},{"key":"ref12","first-page":"648","article-title":"Batch Bayesian optimization via local penalization","volume":"51","author":"gonz\u00e1lez","year":"2016","journal-title":"Proc 19th Int Conf Artif Intell Stat (AISTATS)"},{"key":"ref13","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-44973-4_7"},{"key":"ref14","first-page":"1","article-title":"Hyperband: A novel bandit-based approach to hyperparameter optimization","volume":"18","author":"li","year":"2017","journal-title":"J Mach Learn Res"},{"key":"ref15","first-page":"230","article-title":"A system for massively parallel hyperparameter tuning","author":"li","year":"2020","journal-title":"Proc Int Conf Mach Learn Syst (MLSys)"},{"key":"ref16","first-page":"1436","article-title":"BOHB: Robust and efficient hyperparameter optimization at scale","volume":"80","author":"falkner","year":"2018","journal-title":"Proc 35th Int Conf Mach Learn (ICML)"},{"key":"ref17","article-title":"Model-based asynchronous hyperparameter optimization","author":"tiao","year":"2020","journal-title":"arXiv 2003 10865"},{"key":"ref18","first-page":"2503","article-title":"Hidden technical debt in machine learning systems","author":"sculley","year":"2015","journal-title":"Proc Annu Conf Neural Inf Process Syst"},{"key":"ref19","first-page":"240","article-title":"Non-stochastic best arm identification and hyperparameter optimization","volume":"51","author":"jamieson","year":"2016","journal-title":"Proc 19th Int Conf Artif Intell Stat (AISTATS)"},{"key":"ref4","doi-asserted-by":"publisher","DOI":"10.1023\/A:1008306431147"},{"key":"ref3","doi-asserted-by":"publisher","DOI":"10.1109\/4235.585893"},{"key":"ref6","article-title":"Freeze-thaw Bayesian optimization","author":"swersky","year":"2014","journal-title":"arXiv 1406 3896"},{"key":"ref5","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-25566-3_40"},{"key":"ref8","first-page":"2951","article-title":"Practical Bayesian optimization of machine learning algorithms","author":"snoek","year":"2012","journal-title":"Proc 25th Int Conf Neural Inf Process Syst Vol 2"},{"key":"ref7","first-page":"281","article-title":"Random search for hyper-parameter optimization","volume":"13","author":"bergstra","year":"2012","journal-title":"J Mach Learn Res"},{"key":"ref49","doi-asserted-by":"publisher","DOI":"10.1145\/3097983.3098043"},{"key":"ref9","first-page":"3","article-title":"Bayesian optimization is superior to random search for machine learning hyperparameter tuning: Analysis of the black-box optimization challenge 2020","volume":"133","author":"turner","year":"2020","journal-title":"Proc NeurIPS Competition Demonstration Track"},{"key":"ref46","first-page":"1238","article-title":"Almost optimal exploration in multi-armed bandits","author":"karnin","year":"2013","journal-title":"Proc Int Conf Mach Learn"},{"key":"ref45","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-00483-4"},{"key":"ref48","first-page":"528","article-title":"Fast Bayesian optimization of machine learning hyperparameters on large datasets","author":"klein","year":"2017","journal-title":"Proc Int Conf Artif Intell Stat"},{"key":"ref47","first-page":"2546","article-title":"Algorithms for hyper-parameter optimization","volume":"24","author":"bergstra","year":"2011","journal-title":"Proc Int Conf Adv Neural Inf Process Syst"},{"key":"ref42","doi-asserted-by":"publisher","DOI":"10.1007\/s11721-012-0070-7"},{"key":"ref41","doi-asserted-by":"publisher","DOI":"10.1145\/3437984.3458834"},{"key":"ref44","article-title":"Model-based asynchronous hyperparameter and neural architecture search","author":"klein","year":"2020","journal-title":"arXiv 2003 10865"},{"key":"ref43","doi-asserted-by":"publisher","DOI":"10.1145\/3449726.3463167"}],"container-title":["IEEE Transactions on Evolutionary Computation"],"original-title":[],"link":[{"URL":"https:\/\/ieeexplore.ieee.org\/ielam\/4235\/9967387\/9913342-aam.pdf","content-type":"application\/pdf","content-version":"am","intended-application":"syndication"},{"URL":"http:\/\/xplorestaging.ieee.org\/ielx7\/4235\/9967387\/09913342.pdf?arnumber=9913342","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2022,12,19]],"date-time":"2022-12-19T19:45:26Z","timestamp":1671479126000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/9913342\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2022,12]]},"references-count":55,"journal-issue":{"issue":"6"},"URL":"https:\/\/doi.org\/10.1109\/tevc.2022.3211336","relation":{},"ISSN":["1089-778X","1089-778X","1941-0026"],"issn-type":[{"value":"1089-778X","type":"print"},{"value":"1089-778X","type":"print"},{"value":"1941-0026","type":"electronic"}],"subject":[],"published":{"date-parts":[[2022,12]]}}}