{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,2,21]],"date-time":"2025-02-21T13:38:54Z","timestamp":1740145134436,"version":"3.37.3"},"reference-count":32,"publisher":"Springer Science and Business Media LLC","issue":"3","license":[{"start":{"date-parts":[[2022,6,17]],"date-time":"2022-06-17T00:00:00Z","timestamp":1655424000000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2022,6,17]],"date-time":"2022-06-17T00:00:00Z","timestamp":1655424000000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"funder":[{"DOI":"10.13039\/100000001","name":"National Science Foundation","doi-asserted-by":"publisher","award":["1830478"],"award-info":[{"award-number":["1830478"]}],"id":[{"id":"10.13039\/100000001","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Optim Lett"],"published-print":{"date-parts":[[2023,4]]},"DOI":"10.1007\/s11590-022-01894-6","type":"journal-article","created":{"date-parts":[[2022,6,17]],"date-time":"2022-06-17T06:04:20Z","timestamp":1655445860000},"page":"545-560","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":2,"title":["The leave-worst-k-out criterion for cross validation"],"prefix":"10.1007","volume":"17","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-5527-4047","authenticated-orcid":false,"given":"Lizhi","family":"Wang","sequence":"first","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2022,6,17]]},"reference":[{"issue":"1","key":"1894_CR1","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1021\/ci0342472","volume":"44","author":"DM Hawkins","year":"2004","unstructured":"Hawkins, D.M.: The problem of overfitting. J. Chem. Inf. Comput. Sci. 44(1), 1\u201312 (2004)","journal-title":"J. Chem. Inf. Comput. Sci."},{"issue":"1","key":"1894_CR2","doi-asserted-by":"publisher","first-page":"402","DOI":"10.1214\/14-AOAS798","volume":"9","author":"L Trippa","year":"2015","unstructured":"Trippa, L., Waldron, L., Huttenhower, C., Parmigiani, G., et al.: Bayesian nonparametric cross-study validation of prediction methods. Ann. Appl. Stat. 9(1), 402\u2013428 (2015)","journal-title":"Ann. Appl. Stat."},{"issue":"1","key":"1894_CR3","doi-asserted-by":"publisher","first-page":"23","DOI":"10.1007\/s00265-010-1029-6","volume":"65","author":"KP Burnham","year":"2011","unstructured":"Burnham, K.P., Anderson, D.R., Huyvaert, K.P.: Aic model selection and multimodel inference in behavioral ecology: some background, observations, and comparisons. Behav. Ecol. Sociobiol. 65(1), 23\u201335 (2011)","journal-title":"Behav. Ecol. Sociobiol."},{"issue":"6","key":"1894_CR4","first-page":"2313","volume":"35","author":"E Candes","year":"2007","unstructured":"Candes, E., Tao, T., et al.: The dantzig selector: statistical estimation when p is much larger than n. Ann. Stat. 35(6), 2313\u20132351 (2007)","journal-title":"Ann. Stat."},{"key":"1894_CR5","doi-asserted-by":"crossref","unstructured":"Bartlett, P.L., Long, P.M., Lugosi, G., Tsigler, A.: Benign overfitting in linear regression. In: Proceedings of the National Academy of Sciences. (2020)","DOI":"10.1073\/pnas.1907378117"},{"issue":"22","key":"1894_CR6","doi-asserted-by":"publisher","first-page":"8930","DOI":"10.1073\/pnas.1119041110","volume":"110","author":"B Falkner","year":"2013","unstructured":"Falkner, B., Schr\u00f6der, G.F.: Cross-validation in cryo-EM-based structural modeling. Proc. Natl. Acad. Sci. 110(22), 8930\u20138935 (2013)","journal-title":"Proc. Natl. Acad. Sci."},{"issue":"9","key":"1894_CR7","doi-asserted-by":"publisher","first-page":"853","DOI":"10.1038\/nmeth.2115","volume":"9","author":"SH Scheres","year":"2012","unstructured":"Scheres, S.H., Chen, S.: Prevention of overfitting in cryo-EM structure determination. Nat. Methods 9(9), 853 (2012)","journal-title":"Nat. Methods"},{"issue":"5","key":"1894_CR8","doi-asserted-by":"publisher","first-page":"1413","DOI":"10.1007\/s11222-016-9696-4","volume":"27","author":"A Vehtari","year":"2017","unstructured":"Vehtari, A., Gelman, A., Gabry, J.: Practical Bayesian model evaluation using leave-one-out cross-validation and WAIC. Stat. Comput. 27(5), 1413\u20131432 (2017)","journal-title":"Stat. Comput."},{"issue":"5","key":"1894_CR9","doi-asserted-by":"publisher","first-page":"1879","DOI":"10.1214\/14-AOS1240","volume":"42","author":"A Celisse","year":"2014","unstructured":"Celisse, A., et al.: Optimal cross-validation in density estimation with the L$$\\{{2}\\}$$ -loss. Ann. Stat. 42(5), 1879\u20131910 (2014)","journal-title":"Ann. Stat."},{"issue":"4","key":"1894_CR10","doi-asserted-by":"publisher","first-page":"1828","DOI":"10.1016\/j.csda.2010.11.018","volume":"55","author":"A Airola","year":"2011","unstructured":"Airola, A., Pahikkala, T., Waegeman, W., De Baets, B., Salakoski, T.: An experimental comparison of cross-validation techniques for estimating the area under the roc curve. Comput. Stat. Data Anal. 55(4), 1828\u20131844 (2011)","journal-title":"Comput. Stat. Data Anal."},{"issue":"11","key":"1894_CR11","doi-asserted-by":"publisher","first-page":"2585","DOI":"10.1016\/S0031-3203(03)00136-5","volume":"36","author":"GC Cawley","year":"2003","unstructured":"Cawley, G.C., Talbot, N.L.: Efficient leave-one-out cross-validation of kernel fisher discriminant classifiers. Pattern Recogn. 36(11), 2585\u20132592 (2003)","journal-title":"Pattern Recogn."},{"issue":"1\u20132","key":"1894_CR12","doi-asserted-by":"publisher","first-page":"65","DOI":"10.1007\/s10994-014-5438-z","volume":"97","author":"D Homrighausen","year":"2014","unstructured":"Homrighausen, D., McDonald, D.J.: Leave-one-out cross-validation is risk consistent for lasso. Mach. Learn. 97(1\u20132), 65\u201378 (2014)","journal-title":"Mach. Learn."},{"issue":"3","key":"1894_CR13","doi-asserted-by":"publisher","first-page":"569","DOI":"10.1109\/TPAMI.2009.187","volume":"32","author":"JD Rodriguez","year":"2009","unstructured":"Rodriguez, J.D., Perez, A., Lozano, J.A.: Sensitivity analysis of k-fold cross validation in prediction error estimation. IEEE Trans. Pattern Anal. Mach. Intell. 32(3), 569\u2013575 (2009)","journal-title":"IEEE Trans. Pattern Anal. Mach. Intell."},{"issue":"2","key":"1894_CR14","doi-asserted-by":"publisher","first-page":"137","DOI":"10.1007\/s11222-009-9153-8","volume":"21","author":"T Fushiki","year":"2011","unstructured":"Fushiki, T.: Estimation of prediction error by using k-fold cross-validation. Stat. Comput. 21(2), 137\u2013146 (2011)","journal-title":"Stat. Comput."},{"key":"1894_CR15","doi-asserted-by":"crossref","unstructured":"Blum, A., Kalai, A., and Langford, J. Beating the hold-out: bounds for k-fold and progressive cross-validation. In: Proceedings of the Twelfth Annual Conference on Computational Learning Theory, pp.\u00a0203\u2013208. (1999)","DOI":"10.1145\/307400.307439"},{"key":"1894_CR16","unstructured":"Kohavi, R., et\u00a0al.: A study of cross-validation and bootstrap for accuracy estimation and model selection. In: IJCAI, vol.\u00a014, pp.\u00a01137\u20131145. (1995)"},{"key":"1894_CR17","unstructured":"Magnusson, M., Vehtari, A., Jonasson, J., Andersen, M.: Leave-one-out cross-validation for Bayesian model comparison in large data. In: International Conference on Artificial Intelligence and Statistics, pp.\u00a0341\u2013351. PMLR (2020)"},{"key":"1894_CR18","doi-asserted-by":"publisher","first-page":"29","DOI":"10.1016\/j.chemolab.2018.10.008","volume":"183","author":"L Xu","year":"2018","unstructured":"Xu, L., Hu, O., Guo, Y., Zhang, M., Lu, D., Cai, C.-B., Xie, S., Goodarzi, M., Fu, H.-Y., She, Y.-B.: Representative splitting cross validation. Chemom. Intell. Lab. Syst. 183, 29\u201335 (2018)","journal-title":"Chemom. Intell. Lab. Syst."},{"issue":"1","key":"1894_CR19","doi-asserted-by":"publisher","first-page":"197","DOI":"10.1080\/10485252.2017.1404598","volume":"30","author":"Y Jung","year":"2018","unstructured":"Jung, Y.: Multiple predicting k-fold cross-validation for model selection. J. Nonparametric Stat. 30(1), 197\u2013215 (2018)","journal-title":"J. Nonparametric Stat."},{"issue":"2","key":"1894_CR20","doi-asserted-by":"publisher","first-page":"185","DOI":"10.3390\/rs11020185","volume":"11","author":"A Ramezan","year":"2019","unstructured":"Ramezan, A., Warner, A., Maxwell, A.: Evaluation of sampling and cross-validation tuning strategies for regional-scale machine learning classification. Remote Sens. 11(2), 185 (2019)","journal-title":"Remote Sens."},{"key":"1894_CR21","doi-asserted-by":"publisher","first-page":"6","DOI":"10.1016\/j.patrec.2017.01.007","volume":"88","author":"E Duarte","year":"2017","unstructured":"Duarte, E., Wainer, J.: Empirical comparison of cross-validation and internal metrics for tuning svm hyperparameters. Pattern Recogn. Lett. 88, 6\u201311 (2017)","journal-title":"Pattern Recogn. Lett."},{"issue":"11","key":"1894_CR22","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1007\/s10916-018-1068-5","volume":"42","author":"R Sampath","year":"2018","unstructured":"Sampath, R., Indumathi, J.: Earlier detection of Alzheimer disease using n-fold cross validation approach. J. Med. Syst. 42(11), 1\u201311 (2018)","journal-title":"J. Med. Syst."},{"issue":"3","key":"1894_CR23","doi-asserted-by":"publisher","first-page":"431","DOI":"10.3390\/sym12030431","volume":"12","author":"T Horvat","year":"2020","unstructured":"Horvat, T., Hava\u0161, L., Srpak, D.: The impact of selecting a validation method in machine learning on predicting basketball game outcomes. Symmetry 12(3), 431 (2020)","journal-title":"Symmetry"},{"issue":"5","key":"1894_CR24","doi-asserted-by":"publisher","first-page":"2413","DOI":"10.1021\/acs.jcim.9b01121","volume":"60","author":"P Cossio","year":"2020","unstructured":"Cossio, P.: Need for cross-validation of single particle cryo-EM. J. Chem. Inf. Model. 60(5), 2413\u20132418 (2020)","journal-title":"J. Chem. Inf. Model."},{"issue":"2","key":"1894_CR25","doi-asserted-by":"publisher","first-page":"329","DOI":"10.3390\/en12020329","volume":"12","author":"RM Adnan","year":"2019","unstructured":"Adnan, R.M., Liang, Z., Yuan, X., Kisi, O., Akhlaq, M., Li, B.: Comparison of lssvr, m5rt, nf-gp, and nf-sc models for predictions of hourly wind speed and wind power based on cross-validation. Energies 12(2), 329 (2019)","journal-title":"Energies"},{"issue":"1","key":"1894_CR26","doi-asserted-by":"publisher","first-page":"76","DOI":"10.1007\/BF01584074","volume":"1","author":"M B\u00e9nichou","year":"1971","unstructured":"B\u00e9nichou, M., Gauthier, J.-M., Girodet, P., Hentges, G., Ribi\u00e8re, G., Vincent, O.: Experiments in mixed-integer linear programming. Math. Program. 1(1), 76\u201394 (1971)","journal-title":"Math. Program."},{"issue":"4","key":"1894_CR27","doi-asserted-by":"publisher","first-page":"756","DOI":"10.1287\/opre.1060.0286","volume":"54","author":"G Codato","year":"2006","unstructured":"Codato, G., Fischetti, M.: Combinatorial benders\u2019 cuts for mixed-integer linear programming. Oper. Res. 54(4), 756\u2013766 (2006)","journal-title":"Oper. Res."},{"key":"1894_CR28","doi-asserted-by":"publisher","first-page":"1456","DOI":"10.1109\/TAC.2019.2920812","volume":"65","author":"A Testa","year":"2019","unstructured":"Testa, A., Rucco, A., Notarstefano, G.: Distributed mixed-integer linear programming via cut generation and constraint exchange. IEEE Trans. Autom. Control 65, 1456\u20131467 (2019)","journal-title":"IEEE Trans. Autom. Control"},{"issue":"53","key":"1894_CR29","first-page":"157","volume":"46","author":"II Cplex","year":"2009","unstructured":"Cplex, I.I.: V12. 1: User\u2019s manual for cplex. Int. Bus. Mach. Corp. 46(53), 157 (2009)","journal-title":"Int. Bus. Mach. Corp."},{"key":"1894_CR30","unstructured":"Gurobi\u00a0Optimization, I. Gurobi Optimizer Reference Manual. URL http:\/\/www. gurobi. com. (2018)"},{"key":"1894_CR31","unstructured":"Comparative Evaluation of\u00a0Prediction Algorithms, C. http:\/\/www.coepra.org\/CoEPrA_regr.html. (2006)"},{"issue":"4","key":"1894_CR32","doi-asserted-by":"publisher","first-page":"1345","DOI":"10.1534\/genetics.115.181339","volume":"202","author":"P Mitteroecker","year":"2016","unstructured":"Mitteroecker, P., Cheverud, J., Pavlicev, M.: Multivariate analysis of genotype-phenotype association. Genetics 202(4), 1345\u20131363 (2016)","journal-title":"Genetics"}],"container-title":["Optimization Letters"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s11590-022-01894-6.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s11590-022-01894-6\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s11590-022-01894-6.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2023,3,8]],"date-time":"2023-03-08T12:16:41Z","timestamp":1678277801000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s11590-022-01894-6"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2022,6,17]]},"references-count":32,"journal-issue":{"issue":"3","published-print":{"date-parts":[[2023,4]]}},"alternative-id":["1894"],"URL":"https:\/\/doi.org\/10.1007\/s11590-022-01894-6","relation":{},"ISSN":["1862-4472","1862-4480"],"issn-type":[{"type":"print","value":"1862-4472"},{"type":"electronic","value":"1862-4480"}],"subject":[],"published":{"date-parts":[[2022,6,17]]},"assertion":[{"value":"8 February 2021","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"18 May 2022","order":2,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"17 June 2022","order":3,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}}]}}