{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,4]],"date-time":"2026-04-04T14:16:35Z","timestamp":1775312195698,"version":"3.50.1"},"reference-count":40,"publisher":"Springer Science and Business Media LLC","issue":"4","license":[{"start":{"date-parts":[[2025,2,17]],"date-time":"2025-02-17T00:00:00Z","timestamp":1739750400000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2025,2,17]],"date-time":"2025-02-17T00:00:00Z","timestamp":1739750400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Mach Learn"],"published-print":{"date-parts":[[2025,4]]},"DOI":"10.1007\/s10994-024-06630-y","type":"journal-article","created":{"date-parts":[[2025,2,17]],"date-time":"2025-02-17T22:48:52Z","timestamp":1739832532000},"update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":4,"title":["Reducing cross-validation variance through seed blocking in hyperparameter tuning"],"prefix":"10.1007","volume":"114","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-2539-9225","authenticated-orcid":false,"given":"Giovanni Maria","family":"Merola","sequence":"first","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2025,2,17]]},"reference":[{"issue":"1","key":"6630_CR1","doi-asserted-by":"crossref","first-page":"99","DOI":"10.1287\/opre.1050.0243","volume":"54","author":"B Adenso-D\u00edaz","year":"2006","unstructured":"Adenso-D\u00edaz, B., & Laguna, M. (2006). Fine-tuning of algorithms using fractional experimental designs and local search. Operations Research, 54(1), 99\u2013114.","journal-title":"Operations Research"},{"issue":"1","key":"6630_CR2","doi-asserted-by":"crossref","first-page":"125","DOI":"10.1080\/00401706.1974.10489157","volume":"16","author":"DS Allen","year":"1974","unstructured":"Allen, D. S. (1974). The relationship between variable selection and data agumentation and a method for prediction. Technometrics, 16(1), 125\u2013127.","journal-title":"Technometrics"},{"key":"6630_CR3","volume-title":"Introduction to machine learning","author":"E Alpaydin","year":"2020","unstructured":"Alpaydin, E. (2020). Introduction to machine learning (4th ed.). MIT Press.","edition":"4"},{"key":"6630_CR4","doi-asserted-by":"crossref","first-page":"40","DOI":"10.1214\/09-SS054","volume":"4","author":"S Arlot","year":"2010","unstructured":"Arlot, S., & Celisse, A. (2010). A survey of cross-validation procedures for model selection. Statistics Surveys, 4, 40\u20137.","journal-title":"Statistics Surveys"},{"key":"6630_CR6","unstructured":"Bates, S., Hastie, T., & Tibshirani, R. (2021). Cross-validation: What does it estimate and how well does it do it? arXiv preprint arXiv:2104.00673."},{"issue":"2","key":"6630_CR8","first-page":"281","volume":"13","author":"J Bergstra","year":"2012","unstructured":"Bergstra, J., & Bengio, Y. (2012). Random search for hyper-parameter optimization. Journal of Machine Learning Research, 13(2), 281\u2013305.","journal-title":"Journal of Machine Learning Research"},{"key":"6630_CR9","doi-asserted-by":"publisher","unstructured":"Bischl, B., Binder, M., Lang, M., Pielok, T., Richter, J., Coors, S., Thomas, J., Ullmann, T., Becker, M., Anne-Laure Boulesteix, D., & Deng, & M. Lindauer,. (2023). Hyperparameter optimization: Foundations, algorithms, best practices, and open challenges. WIREs Data Mining and Knowledge Discovery,13(2), e1484. https:\/\/doi.org\/10.1002\/widm.1484","DOI":"10.1002\/widm.1484"},{"key":"6630_CR11","unstructured":"Box, G.\u00a0E.\u00a0P., Hunter, J.\u00a0Stuart, & Hunter, William\u00a0G. (2005). Statistics for experimenters: Statistics For Experimenters: Design, Innovation, and discovery. Wiley series in probability and statistics. Wiley Hoboken, 2nd edition."},{"issue":"4","key":"6630_CR14","doi-asserted-by":"crossref","first-page":"547","DOI":"10.1016\/j.dss.2009.05.016","volume":"47","author":"P Cortez","year":"2009","unstructured":"Cortez, P., Cerdeira, Ant\u00f3nio, Almeida, F., Matos, T., & Reis, J. (2009). Modeling wine preferences by data mining from physicochemical properties. Decision Support Systems, 47(4), 547\u2013553.","journal-title":"Decision Support Systems"},{"key":"6630_CR16","doi-asserted-by":"crossref","DOI":"10.1201\/9781420035834","volume-title":"The theory of the design of experiments","author":"DR Cox","year":"2000","unstructured":"Cox, D. R., & Reid, N. (2000). The theory of the design of experiments. Chapman & Hall\/CRC."},{"issue":"7","key":"6630_CR17","doi-asserted-by":"crossref","first-page":"1895","DOI":"10.1162\/089976698300017197","volume":"10","author":"TG Dietterich","year":"1998","unstructured":"Dietterich, T. G. (1998). Approximate statistical tests for comparing supervised classification learning algorithms. Neural Computation, 10(7), 1895\u20131923.","journal-title":"Neural Computation"},{"key":"6630_CR19","doi-asserted-by":"crossref","unstructured":"Feurer, M., Hutter, F. (2019). Hyperparameter optimization. In F. Hutter, L. Kotthoff, & J. Vanschoren (Eds.), Hyperparameter optimization: Automated machine learning\u2014 methods, systems, challenges (pp. 3\u201333), Springer, Cham (CH). https:\/\/library.oapen.org\/bitstream\/handle\/20.500.12657\/23012\/1007149.pdf.","DOI":"10.1007\/978-3-030-05318-5_1"},{"key":"6630_CR20","volume-title":"The design of experiments","author":"RA Fisher","year":"1937","unstructured":"Fisher, R. A. (1937). The design of experiments. Oliver & Boyd."},{"key":"6630_CR21","doi-asserted-by":"crossref","unstructured":"Fix, E., & Hodges, J.\u00a0L. (1951). Discriminatory analysis. Nonparametric discrimination: Consistency properties. USAF School of Aviation Medicine, Randolph Field, 1951. https:\/\/web.archive.org\/web\/20200926212807\/https:\/\/apps.dtic.mil\/dtic\/tr\/fulltext\/u2\/a800276.pdf","DOI":"10.1037\/e471672008-001"},{"issue":"15","key":"6630_CR22","doi-asserted-by":"crossref","first-page":"1","DOI":"10.18637\/jss.v099.i15","volume":"99","author":"J Frossard","year":"2021","unstructured":"Frossard, J., & Renaud, O. (2021). Permutation tests for regression, ANOVA, and comparison of signals: The permuco package. Journal of Statistical Software, 99(15), 1\u201332.","journal-title":"Journal of Statistical Software"},{"key":"6630_CR24","volume-title":"Permutation tests: A practical guide to resampling methods for testing hypotheses","author":"P Good","year":"2013","unstructured":"Good, P. (2013). Permutation tests: A practical guide to resampling methods for testing hypotheses. Springer Science & Business Media."},{"key":"6630_CR25","doi-asserted-by":"crossref","first-page":"75","DOI":"10.1016\/0893-6080(88)90023-8","volume":"1","author":"RP Gorman","year":"1988","unstructured":"Gorman, R. P., & Sejnowski, T. J. (1988). Analysis of hidden units in a layered network trained to classify sonar targets. Neural Networks, 1, 75\u201389.","journal-title":"Neural Networks"},{"issue":"3","key":"6630_CR26","doi-asserted-by":"crossref","first-page":"e90081","DOI":"10.1371\/journal.pone.0090081","volume":"9","author":"KB Gorman","year":"2014","unstructured":"Gorman, K. B., Williams, T. D., & Fraser, W. R. (2014). Ecological sexual dimorphism and environmental variability within a community of antarctic penguins (genus pygoscelis). PLoS ONE, 9(3), e90081.","journal-title":"PLoS ONE"},{"key":"6630_CR27","doi-asserted-by":"crossref","first-page":"81","DOI":"10.1016\/0095-0696(78)90006-2","volume":"5","author":"D Harrison","year":"1978","unstructured":"Harrison, D., & Rubinfeld, D. L. (1978). Hedonic prices and the demand for clean air. Journal of Environmental Economics and Management, 5, 81\u2013102.","journal-title":"Journal of Environmental Economics and Management"},{"key":"6630_CR29","doi-asserted-by":"crossref","unstructured":"Henderson, P., Islam, R., Bachman, P., Pineau, J., Precup, D., & Meger, D. (2018). Deep reinforcement learning that matters. In Proceedings of the AAAI conference on artificial intelligence, vol. 32(1).","DOI":"10.1609\/aaai.v32i1.11694"},{"key":"6630_CR30","unstructured":"Hutter, F., Hoos, H.\u00a0H., & Leyton-Brown, K. (2014). An efficient approach for assessing hyperparameter importance. In International conference on machine learning, pp. 754\u2013762, PMLR."},{"key":"6630_CR31","doi-asserted-by":"crossref","first-page":"267","DOI":"10.1613\/jair.2861","volume":"36","author":"F Hutter","year":"2009","unstructured":"Hutter, F., Hoos, H. H., Leyton-Brown, K., & St\u00fctzle, T. (2009). ParamILS: An automatic algorithm configuration framework. Journal of Artificial Intelligence Research, 36, 267\u2013306.","journal-title":"Journal of Artificial Intelligence Research"},{"key":"6630_CR32","unstructured":"Islam, R., Henderson, P., Gomrokchi, M., Precup, D. (2017). Reproducibility of benchmarked deep reinforcement learning tasks for continuous control. arXiv preprint arXiv:1708.04133."},{"key":"6630_CR33","unstructured":"Ke, G., Meng, Q., Finley, T., Wang, T., Chen, W., Ma, W., Ye, Q., & Liu, T.-Y. (2017). LightGBM: A highly efficient gradient boosting decision tree. Advances in Neural Information Processing Systems, 30(NIPS 2017)."},{"issue":"1","key":"6630_CR34","doi-asserted-by":"crossref","first-page":"85","DOI":"10.1080\/07350015.1995.10524581","volume":"13","author":"FE Kennedy","year":"1995","unstructured":"Kennedy, F. E. (1995). Randomization tests in econometrics. Journal of Business & Economic Statistics, 13(1), 85\u201394.","journal-title":"Journal of Business & Economic Statistics"},{"issue":"11","key":"6630_CR35","doi-asserted-by":"crossref","first-page":"3735","DOI":"10.1016\/j.csda.2009.04.009","volume":"53","author":"J-H Kim","year":"2009","unstructured":"Kim, J.-H. (2009). Estimating classification error rate: Repeated cross-validation, repeated hold-out and bootstrap. Computational Statistics & Data Analysis, 53(11), 3735\u20133745.","journal-title":"Computational Statistics & Data Analysis"},{"key":"6630_CR37","unstructured":"Kohavi, R. (1996). Scaling up the accuracy of naive-bayes classifiers: A decision-tree hybrid. In Proceedings of the second international conference on knowledge discovery and data mining, KDD\u201996, pp. 202\u2013207, Portland, Oregon, 1996. AAAI Press. https:\/\/archive.ics.uci.edu\/ml\/datasets\/Adult."},{"key":"6630_CR41","unstructured":"Lantz, B. (2019). Machine learning with R: Expert techniques for predictive modeling, 3rd Edition. Packt Publishing, 2019. data https:\/\/www.kaggle.com\/datasets\/mirichoi0218\/insurance."},{"key":"6630_CR42","doi-asserted-by":"crossref","first-page":"195","DOI":"10.1016\/j.eswa.2018.05.024","volume":"109","author":"GA Lujan-Moreno","year":"2018","unstructured":"Lujan-Moreno, G. A., Howard, P. R., Rojas, O. G., & Montgomery, D. C. (2018). Design of experiments and response surface methodology to tune machine learning hyperparameters, with a random forest case-study. Expert Systems with Applications, 109, 195\u2013205.","journal-title":"Expert Systems with Applications"},{"key":"6630_CR43","doi-asserted-by":"publisher","first-page":"18","DOI":"10.1007\/s13721-016-0125-6","volume":"5","author":"G Luo","year":"2016","unstructured":"Luo, G. (2016). A review of automatic selection methods for machine learning algorithms and hyper-parameter values. Network Modeling Analysis in Health Informatics and Bioinformatics, 5, 18. https:\/\/doi.org\/10.1007\/s13721-016-0125-6","journal-title":"Network Modeling Analysis in Health Informatics and Bioinformatics"},{"issue":"15","key":"6630_CR44","doi-asserted-by":"crossref","first-page":"3301","DOI":"10.1093\/bioinformatics\/bti499","volume":"21","author":"AM Molinaro","year":"2005","unstructured":"Molinaro, A. M., Simon, R., & Pfeiffer, R. S. (2005). Prediction error estimation: A comparison of resampling methods. Bioinformatics, 21(15), 3301\u20133307.","journal-title":"Bioinformatics"},{"key":"6630_CR45","volume-title":"Design and analysis of experiments","author":"DC Montgomery","year":"2008","unstructured":"Montgomery, D. C. (2008). Design and analysis of experiments (7th ed.). Wiley.","edition":"7"},{"issue":"53","key":"6630_CR46","first-page":"1","volume":"20","author":"P Probst","year":"2019","unstructured":"Probst, P., Boulesteix, A.-L., & Bischl, B. (2019). Tunability: Importance of hyperparameters of machine learning algorithms. Journal of Machine Learning Research, 20(53), 1\u201332.","journal-title":"Journal of Machine Learning Research"},{"key":"6630_CR47","unstructured":"R Core Team. R: A Language and Environment for Statistical Computing. R Foundation for Statistical Computing, Vienna, Austria, 2020. URL https:\/\/www.R-project.org\/."},{"issue":"2","key":"6630_CR48","doi-asserted-by":"crossref","first-page":"111","DOI":"10.1111\/j.2517-6161.1974.tb00994.x","volume":"36","author":"M Stone","year":"1974","unstructured":"Stone, M. (1974). Cross-validatory choice and assessment of statistical predictions. Journal of the royal statistical society: Series B (Methodological), 36(2), 111\u2013133.","journal-title":"Journal of the royal statistical society: Series B (Methodological)"},{"key":"6630_CR51","unstructured":"Wheeler, B., & Torchiano, M. (2016). lmPerm: Permutation tests for linear models, 2016. R package version 2.1.0."},{"issue":"1","key":"6630_CR52","doi-asserted-by":"publisher","first-page":"1","DOI":"10.18637\/jss.v077.i01","volume":"77","author":"MN Wright","year":"2017","unstructured":"Wright, M. N., & Ziegler, A. (2017). ranger: A fast implementation of random forests for high dimensional data in C++ and R. Journal of Statistical Software, 77(1), 1\u201317. https:\/\/doi.org\/10.18637\/jss.v077.i01","journal-title":"Journal of Statistical Software"},{"key":"6630_CR53","volume-title":"Experiments: Planning, analysis, and optimization","author":"CFJ Wu","year":"2011","unstructured":"Wu, C. F. J., & Hamada, M. S. (2011). Experiments: Planning, analysis, and optimization (3rd ed.). Wiley.","edition":"3"},{"key":"6630_CR54","doi-asserted-by":"publisher","first-page":"295","DOI":"10.1016\/j.neucom.2020.07.061","volume":"415","author":"L Yang","year":"2020","unstructured":"Yang, L., & Shami, A. (2020). On hyperparameter optimization of machine learning algorithms: Theory and practice. Neurocomputing, 415, 295\u2013316. https:\/\/doi.org\/10.1016\/j.neucom.2020.07.061. ISSN 0925-2312.","journal-title":"Neurocomputing"},{"key":"6630_CR57","unstructured":"Yu, T., & Zhu, H. (2020). Hyper-parameter optimization: A review of algorithms and applications. arxiv:2003.05689."}],"container-title":["Machine Learning"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10994-024-06630-y.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s10994-024-06630-y","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10994-024-06630-y.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,2,17]],"date-time":"2026-02-17T01:02:54Z","timestamp":1771290174000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s10994-024-06630-y"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,2,17]]},"references-count":40,"journal-issue":{"issue":"4","published-print":{"date-parts":[[2025,4]]}},"alternative-id":["6630"],"URL":"https:\/\/doi.org\/10.1007\/s10994-024-06630-y","relation":{},"ISSN":["0885-6125","1573-0565"],"issn-type":[{"value":"0885-6125","type":"print"},{"value":"1573-0565","type":"electronic"}],"subject":[],"published":{"date-parts":[[2025,2,17]]},"assertion":[{"value":"14 August 2023","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"5 December 2024","order":2,"name":"revised","label":"Revised","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"12 December 2024","order":3,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"17 February 2025","order":4,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Declarations"}},{"value":"The author has no conflicting financial or non-financial interests that are directly or indirectly related to this work.","order":2,"name":"Ethics","group":{"name":"EthicsHeading","label":"Conflict of interest"}},{"value":"This article does not require permission for ethics approval as the numerical examples are based on publicly available datasets.","order":3,"name":"Ethics","group":{"name":"EthicsHeading","label":"Ethics approval"}},{"value":"This article does not require consent to participation as the numerical examples are based on publicly available datasets.","order":4,"name":"Ethics","group":{"name":"EthicsHeading","label":"Consent to participate"}},{"value":"The author of this manuscript consents to the publication of the manuscript.","order":5,"name":"Ethics","group":{"name":"EthicsHeading","label":"Consent for publication"}},{"value":"This content has been made available to all.","name":"free","label":"Free to read"}],"article-number":"89"}}