{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,6,24]],"date-time":"2026-06-24T02:07:47Z","timestamp":1782266867307,"version":"3.54.5"},"reference-count":30,"publisher":"Elsevier BV","license":[{"start":{"date-parts":[[2017,9,1]],"date-time":"2017-09-01T00:00:00Z","timestamp":1504224000000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.elsevier.com\/tdm\/userlicense\/1.0\/"}],"funder":[{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["61673249"],"award-info":[{"award-number":["61673249"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["61503229"],"award-info":[{"award-number":["61503229"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100003398","name":"Shanxi Scholarship Council of China","doi-asserted-by":"publisher","award":["2016-004"],"award-info":[{"award-number":["2016-004"]}],"id":[{"id":"10.13039\/501100003398","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["elsevier.com","sciencedirect.com"],"crossmark-restriction":true},"short-container-title":["Pattern Recognition"],"published-print":{"date-parts":[[2017,9]]},"DOI":"10.1016\/j.patcog.2017.03.025","type":"journal-article","created":{"date-parts":[[2017,4,14]],"date-time":"2017-04-14T09:16:38Z","timestamp":1492161398000},"page":"94-106","update-policy":"https:\/\/doi.org\/10.1016\/elsevier_cm_policy","source":"Crossref","is-referenced-by-count":159,"special_numbering":"C","title":["Error estimation based on variance analysis of k -fold cross-validation"],"prefix":"10.1016","volume":"69","author":[{"given":"Gaoxia","family":"Jiang","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Wenjian","family":"Wang","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"78","reference":[{"key":"10.1016\/j.patcog.2017.03.025_bib0001","first-page":"1089","article-title":"No unbiased estimator of the variance of k-fold cross-validation","volume":"5","author":"Bengio","year":"2004","journal-title":"J. Mach. Learn. Res."},{"key":"10.1016\/j.patcog.2017.03.025_bib0002","first-page":"281","article-title":"Random search for hyper-parameter optimization","volume":"13","author":"Bergstra","year":"2012","journal-title":"J. Mach. Learn. Res."},{"issue":"11","key":"10.1016\/j.patcog.2017.03.025_bib0003","doi-asserted-by":"crossref","first-page":"2705","DOI":"10.1016\/j.patcog.2009.05.003","article-title":"On the sampling distribution of resubstitution and leave-one-out error estimators for linear classifiers","volume":"42","author":"Zollanvari","year":"2009","journal-title":"Pattern Recognit."},{"key":"10.1016\/j.patcog.2017.03.025_bib0004","series-title":"\u201cCross-validation.\u201d Encyclopedia of database systems","first-page":"532","author":"Refaeilzadeh","year":"2009"},{"issue":"3","key":"10.1016\/j.patcog.2017.03.025_bib0005","doi-asserted-by":"crossref","first-page":"855","DOI":"10.1016\/j.patcog.2012.09.007","article-title":"A general framework for the statistical analysis of the sources of variance for classification error estimators","volume":"46","author":"Rodriguez","year":"2013","journal-title":"Pattern Recognit."},{"issue":"1","key":"10.1016\/j.patcog.2017.03.025_bib0006","doi-asserted-by":"crossref","first-page":"37","DOI":"10.1111\/j.1467-9868.2011.01005.x","article-title":"Variance estimation using refitted cross-validation in ultrahigh dimensional regression","volume":"74","author":"Fan","year":"2012","journal-title":"J. R. Stat. Soc.(Stat. Methodol.)"},{"key":"10.1016\/j.patcog.2017.03.025_bib0007","doi-asserted-by":"crossref","first-page":"22","DOI":"10.1016\/j.knosys.2013.09.023","article-title":"Repeated double cross-validation for choosing a single solution in evolutionary multi-objective fuzzy classifier design","volume":"54","author":"Ishibuchi","year":"2013","journal-title":"Knowl. Based Syst."},{"key":"10.1016\/j.patcog.2017.03.025_bib0008","series-title":"An introduction to the bootstrap","first-page":"45","author":"Efron","year":"1994"},{"key":"10.1016\/j.patcog.2017.03.025_bib0009","series-title":"Proc. 14th Int. Joint Conf. on AI","first-page":"1137","article-title":"A study of cross-validation and bootstrap for accuracy estimation and model selection","author":"Kohavi","year":"1995"},{"issue":"11","key":"10.1016\/j.patcog.2017.03.025_bib0010","doi-asserted-by":"crossref","first-page":"3735","DOI":"10.1016\/j.csda.2009.04.009","article-title":"Estimating classification error rate: repeated cross-validation, repeated hold-out and bootstrap","volume":"53","author":"Kim","year":"2009","journal-title":"Comput. Stat. Data Anal."},{"issue":"1","key":"10.1016\/j.patcog.2017.03.025_bib0011","doi-asserted-by":"crossref","first-page":"10","DOI":"10.1186\/1758-2946-6-10","article-title":"Cross-validation pitfalls when selecting and assessing regression and classification models","volume":"6","author":"Krstajic","year":"2014","journal-title":"J. Cheminform."},{"issue":"11","key":"10.1016\/j.patcog.2017.03.025_bib0012","doi-asserted-by":"crossref","first-page":"3739","DOI":"10.1016\/j.patcog.2015.05.005","article-title":"Model selection for linear classifiers using bayesian error estimation","volume":"48","author":"Huttunen","year":"2015","journal-title":"Pattern Recognit."},{"issue":"9","key":"10.1016\/j.patcog.2017.03.025_bib0013","doi-asserted-by":"crossref","first-page":"2839","DOI":"10.1016\/j.patcog.2015.03.009","article-title":"Performance evaluation of classification algorithms by k-fold and leave-one-out cross validation","volume":"48","author":"Wong","year":"2015","journal-title":"Pattern Recognit."},{"key":"10.1016\/j.patcog.2017.03.025_bib0014","doi-asserted-by":"crossref","first-page":"97","DOI":"10.1016\/j.patcog.2016.12.018","article-title":"Parametric methods for comparing the performance of two classification algorithms evaluated by k-fold cross validation on multiple data sets","volume":"65","author":"Wong","year":"2017","journal-title":"Pattern Recognit."},{"issue":"7","key":"10.1016\/j.patcog.2017.03.025_bib0015","doi-asserted-by":"crossref","first-page":"1895","DOI":"10.1162\/089976698300017197","article-title":"Approximate statistical tests for comparing supervised classification learning algorithms","volume":"10","author":"Dietterich","year":"1998","journal-title":"Neural Comput."},{"issue":"8","key":"10.1016\/j.patcog.2017.03.025_bib0016","doi-asserted-by":"crossref","first-page":"1885","DOI":"10.1162\/089976699300016007","article-title":"Combined 5 \u00d7 2 cv f test for comparing supervised classification learning algorithms","volume":"11","author":"Alpaydin","year":"1999","journal-title":"Neural Comput."},{"key":"10.1016\/j.patcog.2017.03.025_bib0017","doi-asserted-by":"crossref","first-page":"192","DOI":"10.1016\/j.ins.2011.12.028","article-title":"On the use of cross-validation for time series predictor evaluation","volume":"191","author":"Bergmeir","year":"2012","journal-title":"Inf. Sci."},{"key":"10.1016\/j.patcog.2017.03.025_bib0018","first-page":"1127","article-title":"Analysis of variance of cross-validation estimators of the generalization error","volume":"6","author":"Markatou","year":"2005","journal-title":"J. Mach. Learn. Res."},{"issue":"3","key":"10.1016\/j.patcog.2017.03.025_bib0019","doi-asserted-by":"crossref","first-page":"239","DOI":"10.1023\/A:1024068626366","article-title":"Inference for the generalization error","volume":"52","author":"Nadeau","year":"2003","journal-title":"Mach. Learn."},{"issue":"3","key":"10.1016\/j.patcog.2017.03.025_bib0020","doi-asserted-by":"crossref","first-page":"569","DOI":"10.1109\/TPAMI.2009.187","article-title":"Sensitivity analysis of k-fold cross validation in prediction error estimation","volume":"32","author":"Rodriguez","year":"2010","journal-title":"IEEE Trans. Pattern Anal. Mach. Intell."},{"issue":"8","key":"10.1016\/j.patcog.2017.03.025_bib0021","doi-asserted-by":"crossref","first-page":"1304","DOI":"10.1109\/TNNLS.2012.2199516","article-title":"Study on the impact of partition induced dataset shift on k-fold cross-validation","volume":"23","author":"Moreno-Torres","year":"2012","journal-title":"IEEE Trans. Neural Netw. Learn. Syst."},{"key":"10.1016\/j.patcog.2017.03.025_bib0022","series-title":"Wrappers for performance enhancement and oblivious decision graphs","author":"Kohavi","year":"1995"},{"issue":"1","key":"10.1016\/j.patcog.2017.03.025_bib0023","doi-asserted-by":"crossref","first-page":"208","DOI":"10.1162\/NECO_a_00532","article-title":"Blocked 3\u00a0\u00d7 \u00a02 cross-validated t-test for comparing supervised classification learning algorithms","volume":"26","author":"Yu","year":"2014","journal-title":"Neural Comput."},{"issue":"14","key":"10.1016\/j.patcog.2017.03.025_bib0024","doi-asserted-by":"crossref","first-page":"1960","DOI":"10.1016\/j.patrec.2008.06.018","article-title":"Cross-validation and bootstrapping are unreliable in small sample classification","volume":"29","author":"Isaksson","year":"2008","journal-title":"Pattern Recognit. Lett."},{"key":"10.1016\/j.patcog.2017.03.025_bib0025","series-title":"Probability and Statistics","first-page":"337","author":"DeGroot","year":"2011"},{"key":"10.1016\/j.patcog.2017.03.025_bib0026","unstructured":"M. Lichman, UCI Machine Learning Repository, Irvine, CA: University of California, School of Information and Computer Science, <http:\/\/archive.ics.uci.edu\/ml\/>, 2014-12-8."},{"key":"10.1016\/j.patcog.2017.03.025_bib0027","series-title":"Applied statistics and probability for engineers","first-page":"238","author":"Montgomery","year":"2010"},{"issue":"8","key":"10.1016\/j.patcog.2017.03.025_bib0028","doi-asserted-by":"crossref","first-page":"2154","DOI":"10.1016\/j.patcog.2006.12.015","article-title":"Fast cross-validation algorithms for least squares support vector machine and kernel ridge regression","volume":"40","author":"An","year":"2007","journal-title":"Pattern Recognit."},{"key":"10.1016\/j.patcog.2017.03.025_bib0029","doi-asserted-by":"crossref","first-page":"260","DOI":"10.1016\/j.neucom.2016.02.058","article-title":"Efficient leave-one-out cross-validation-based regularized extreme learning machine","volume":"194","author":"Shao","year":"2016","journal-title":"Neurocomputing"},{"key":"10.1016\/j.patcog.2017.03.025_bib0030","unstructured":"C.-C. Chang, C.-J. Lin, LIBSVM data: classification, regression, and multilabel, <http:\/\/www.csie.ntu.edu.tw\/~cjlin\/libsvmtools\/datasets>, 2015-11-26."}],"container-title":["Pattern Recognition"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/api.elsevier.com\/content\/article\/PII:S003132031730136X?httpAccept=text\/xml","content-type":"text\/xml","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/api.elsevier.com\/content\/article\/PII:S003132031730136X?httpAccept=text\/plain","content-type":"text\/plain","content-version":"vor","intended-application":"text-mining"}],"deposited":{"date-parts":[[2018,9,2]],"date-time":"2018-09-02T18:20:35Z","timestamp":1535912435000},"score":1,"resource":{"primary":{"URL":"https:\/\/linkinghub.elsevier.com\/retrieve\/pii\/S003132031730136X"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2017,9]]},"references-count":30,"alternative-id":["S003132031730136X"],"URL":"https:\/\/doi.org\/10.1016\/j.patcog.2017.03.025","relation":{},"ISSN":["0031-3203"],"issn-type":[{"value":"0031-3203","type":"print"}],"subject":[],"published":{"date-parts":[[2017,9]]},"assertion":[{"value":"Elsevier","name":"publisher","label":"This article is maintained by"},{"value":"Error estimation based on variance analysis of k-fold cross-validation","name":"articletitle","label":"Article Title"},{"value":"Pattern Recognition","name":"journaltitle","label":"Journal Title"},{"value":"https:\/\/doi.org\/10.1016\/j.patcog.2017.03.025","name":"articlelink","label":"CrossRef DOI link to publisher maintained version"},{"value":"article","name":"content_type","label":"Content Type"},{"value":"\u00a9 2017 Elsevier Ltd. All rights reserved.","name":"copyright","label":"Copyright"}]}}