{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,5,2]],"date-time":"2026-05-02T10:53:39Z","timestamp":1777719219351,"version":"3.51.4"},"reference-count":41,"publisher":"Springer Science and Business Media LLC","issue":"11","license":[{"start":{"date-parts":[[2020,10,13]],"date-time":"2020-10-13T00:00:00Z","timestamp":1602547200000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"},{"start":{"date-parts":[[2020,10,13]],"date-time":"2020-10-13T00:00:00Z","timestamp":1602547200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Mach Learn"],"published-print":{"date-parts":[[2020,11]]},"DOI":"10.1007\/s10994-020-05910-7","type":"journal-article","created":{"date-parts":[[2020,10,13]],"date-time":"2020-10-13T22:02:42Z","timestamp":1602626562000},"page":"1997-2028","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":307,"title":["Evaluating time series forecasting models: an empirical study on performance estimation methods"],"prefix":"10.1007","volume":"109","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-9694-8423","authenticated-orcid":false,"given":"Vitor","family":"Cerqueira","sequence":"first","affiliation":[]},{"given":"Luis","family":"Torgo","sequence":"additional","affiliation":[]},{"given":"Igor","family":"Mozeti\u010d","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2020,10,13]]},"reference":[{"key":"5910_CR1","doi-asserted-by":"publisher","first-page":"40","DOI":"10.1214\/09-SS054","volume":"4","author":"S Arlot","year":"2010","unstructured":"Arlot, S., Celisse, A., et al. (2010). A survey of cross-validation procedures for model selection. Statistics Surveys, 4, 40\u201379.","journal-title":"Statistics Surveys"},{"issue":"1","key":"5910_CR2","first-page":"2653","volume":"18","author":"A Benavoli","year":"2017","unstructured":"Benavoli, A., Corani, G., Dem\u0161ar, J., & Zaffalon, M. (2017). Time for a change: A tutorial for comparing multiple classifiers through bayesian analysis. The Journal of Machine Learning Research, 18(1), 2653\u20132688.","journal-title":"The Journal of Machine Learning Research"},{"key":"5910_CR3","doi-asserted-by":"crossref","unstructured":"Bergmeir, C., & Benitez, J.M. (2011) Forecaster performance evaluation with cross-validation and variants. In: 2011 11th international conference on intelligent systems design and applications (ISDA), pp. 849\u2013854. IEEE.","DOI":"10.1109\/ISDA.2011.6121763"},{"key":"5910_CR4","doi-asserted-by":"publisher","first-page":"192","DOI":"10.1016\/j.ins.2011.12.028","volume":"191","author":"C Bergmeir","year":"2012","unstructured":"Bergmeir, C., & Ben\u00edtez, J. M. (2012). On the use of cross-validation for time series predictor evaluation. Information Sciences, 191, 192\u2013213.","journal-title":"Information Sciences"},{"key":"5910_CR5","doi-asserted-by":"publisher","first-page":"132","DOI":"10.1016\/j.csda.2014.02.001","volume":"76","author":"C Bergmeir","year":"2014","unstructured":"Bergmeir, C., Costantini, M., & Ben\u00edtez, J. M. (2014). On the usefulness of cross-validation for directional forecast evaluation. Computational Statistics & Data Analysis, 76, 132\u2013143.","journal-title":"Computational Statistics & Data Analysis"},{"key":"5910_CR6","doi-asserted-by":"publisher","first-page":"70","DOI":"10.1016\/j.csda.2017.11.003","volume":"120","author":"C Bergmeir","year":"2018","unstructured":"Bergmeir, C., Hyndman, R. J., & Koo, B. (2018). A note on the validity of cross-validation for evaluating autoregressive time series prediction. Computational Statistics & Data Analysis, 120, 70\u201383.","journal-title":"Computational Statistics & Data Analysis"},{"issue":"1","key":"5910_CR7","doi-asserted-by":"publisher","first-page":"5","DOI":"10.1023\/A:1010933404324","volume":"45","author":"L Breiman","year":"2001","unstructured":"Breiman, L. (2001). Random forests. Machine Learning, 45(1), 5\u201332.","journal-title":"Machine Learning"},{"key":"5910_CR8","doi-asserted-by":"publisher","DOI":"10.1201\/9781315139470","volume-title":"Classification and Regression Trees","author":"L Breiman","year":"2017","unstructured":"Breiman, L. (2017). Classification and Regression Trees. New York: Routledge."},{"key":"5910_CR9","unstructured":"Brockwell, P.J., & Davis, R.A. (2013). Time series: theory and methods. Springer Science & Business Media, Berlin"},{"issue":"6","key":"5910_CR10","doi-asserted-by":"publisher","first-page":"913","DOI":"10.1007\/s10994-018-05774-y","volume":"108","author":"V Cerqueira","year":"2019","unstructured":"Cerqueira, V., Torgo, L., Pinto, F., & Soares, C. (2019). Arbitrage of forecasting experts. Machine Learning, 108(6), 913\u2013944.","journal-title":"Machine Learning"},{"key":"5910_CR11","doi-asserted-by":"crossref","unstructured":"Cerqueira, V., Torgo, L., Smailovi\u0107, J., & Mozeti\u010d, I. (2017) A comparative study of performance estimation methods for time series forecasting. In 2017 IEEE international conference on data science and advanced analytics (DSAA) (pp. 529\u2013538). IEEE.","DOI":"10.1109\/DSAA.2017.7"},{"key":"5910_CR12","unstructured":"Cerqueira, V., Torgo, L., & Soares, C. (2019). Machine learning vs statistical methods for time series forecasting: Size matters. arXiv preprint arXiv:1909.13316."},{"issue":"2","key":"5910_CR13","doi-asserted-by":"publisher","first-page":"278","DOI":"10.2307\/2981683","volume":"147","author":"AP Dawid","year":"1984","unstructured":"Dawid, A. P. (1984). Present position and potential developments: Some personal views statistical theory the prequential approach. Journal of the Royal Statistical Society: Series A (General), 147(2), 278\u2013290.","journal-title":"Journal of the Royal Statistical Society: Series A (General)"},{"issue":"7","key":"5910_CR14","doi-asserted-by":"publisher","first-page":"1895","DOI":"10.1162\/089976698300017197","volume":"10","author":"TG Dietterich","year":"1998","unstructured":"Dietterich, T. G. (1998). Approximate statistical tests for comparing supervised classification learning algorithms. Neural Computation, 10(7), 1895\u20131923.","journal-title":"Neural Computation"},{"issue":"9","key":"5910_CR15","doi-asserted-by":"publisher","first-page":"1056","DOI":"10.1287\/mnsc.35.9.1056","volume":"35","author":"R Fildes","year":"1989","unstructured":"Fildes, R. (1989). Evaluation of aggregate and individual forecast method selection rules. Management Science, 35(9), 1056\u20131065.","journal-title":"Management Science"},{"issue":"1","key":"5910_CR16","doi-asserted-by":"publisher","first-page":"1","DOI":"10.18637\/jss.v033.i01","volume":"33","author":"J Friedman","year":"2010","unstructured":"Friedman, J., Hastie, T., & Tibshirani, R. (2010). Regularization paths for generalized linear models via coordinate descent. Journal of Statistical Software, 33(1), 1\u201322.","journal-title":"Journal of Statistical Software"},{"issue":"3","key":"5910_CR17","doi-asserted-by":"publisher","first-page":"317","DOI":"10.1007\/s10994-012-5320-9","volume":"90","author":"J Gama","year":"2013","unstructured":"Gama, J., Sebasti\u00e3o, R., & Rodrigues, P. P. (2013). On evaluating stream learning algorithms. Machine Learning, 90(3), 317\u2013346.","journal-title":"Machine Learning"},{"issue":"4","key":"5910_CR18","doi-asserted-by":"publisher","first-page":"44","DOI":"10.1145\/2523813","volume":"46","author":"J Gama","year":"2014","unstructured":"Gama, J., \u017dliobait\u0117, I., Bifet, A., Pechenizkiy, M., & Bouchachia, A. (2014). A survey on concept drift adaptation. ACM Computing Surveys (CSUR), 46(4), 44.","journal-title":"ACM Computing Surveys (CSUR)"},{"issue":"350","key":"5910_CR19","doi-asserted-by":"publisher","first-page":"320","DOI":"10.1080\/01621459.1975.10479865","volume":"70","author":"S Geisser","year":"1975","unstructured":"Geisser, S. (1975). The predictive sample reuse method with applications. Journal of the American statistical Association, 70(350), 320\u2013328.","journal-title":"Journal of the American statistical Association"},{"issue":"396","key":"5910_CR20","doi-asserted-by":"publisher","first-page":"1080","DOI":"10.1080\/01621459.1986.10478377","volume":"81","author":"JD Hart","year":"1986","unstructured":"Hart, J. D., & Wehrly, T. E. (1986). Kernel regression estimation using repeated measurements data. Journal of the American Statistical Association, 81(396), 1080\u20131088.","journal-title":"Journal of the American Statistical Association"},{"key":"5910_CR21","unstructured":"Hyndman, R., & Yang, Y. (2019) tsdl: Time series data library. https:\/\/github.com\/FinYang\/tsdl."},{"key":"5910_CR22","unstructured":"Hyndman, R.J., & Athanasopoulos, G. (2018). Forecasting: principles and practice. OTexts."},{"issue":"6","key":"5910_CR23","doi-asserted-by":"publisher","first-page":"3403","DOI":"10.1103\/PhysRevA.45.3403","volume":"45","author":"MB Kennel","year":"1992","unstructured":"Kennel, M. B., Brown, R., & Abarbanel, H. D. (1992). Determining embedding dimension for phase-space reconstruction using a geometrical construction. Physical Review A, 45(6), 3403.","journal-title":"Physical Review A"},{"key":"5910_CR24","unstructured":"Kuhn, M., Weston, S., & Keefer, C. (2014). code for Cubist\u00a0by Ross\u00a0Quinlan, N.C.C.: Cubist: Rule- and Instance-Based Regression Modeling. R package version 0.0.18."},{"issue":"2","key":"5910_CR25","doi-asserted-by":"publisher","first-page":"111","DOI":"10.1002\/for.3980010202","volume":"1","author":"S Makridakis","year":"1982","unstructured":"Makridakis, S., Andersen, A., Carbone, R., Fildes, R., Hibon, M., Lewandowski, R., et al. (1982). The accuracy of extrapolation (time series) methods: Results of a forecasting competition. Journal of Forecasting, 1(2), 111\u2013153.","journal-title":"Journal of Forecasting"},{"key":"5910_CR26","doi-asserted-by":"publisher","DOI":"10.1201\/9780203753736","volume-title":"Generalized linear models","author":"P McCullagh","year":"2019","unstructured":"McCullagh, P. (2019). Generalized linear models. New York: Routledge."},{"key":"5910_CR27","doi-asserted-by":"publisher","DOI":"10.1142\/3573","volume-title":"Regression and time series model selection","author":"AD McQuarrie","year":"1998","unstructured":"McQuarrie, A. D., & Tsai, C. L. (1998). Regression and time series model selection. Singapore: World Scientific."},{"key":"5910_CR28","unstructured":"Milborrow, S. (2018). rpart.plot: Plot \u2019rpart\u2019 Models: An Enhanced Version of \u2019plot.rpart\u2019. https:\/\/CRAN.R-project.org\/package=rpart.plot. R package version 3.0.6."},{"issue":"1","key":"5910_CR29","doi-asserted-by":"publisher","first-page":"5","DOI":"10.1023\/A:1007577530334","volume":"33","author":"DS Modha","year":"1998","unstructured":"Modha, D. S., & Masry, E. (1998). Prequential and cross-validated regression estimation. Machine Learning, 33(1), 5\u201339.","journal-title":"Machine Learning"},{"issue":"3","key":"5910_CR30","doi-asserted-by":"publisher","first-page":"e0194317","DOI":"10.1371\/journal.pone.0194317","volume":"13","author":"I Mozeti\u010d","year":"2018","unstructured":"Mozeti\u010d, I., Torgo, L., Cerqueira, V., & Smailovi\u0107, J. (2018). How to evaluate sentiment classifiers for Twitter time-ordered data? PLoS ONE, 13(3), e0194317.","journal-title":"PLoS ONE"},{"issue":"5","key":"5910_CR31","doi-asserted-by":"publisher","first-page":"879","DOI":"10.1111\/rssb.12015","volume":"75","author":"G Nason","year":"2013","unstructured":"Nason, G. (2013). A test for second-order stationarity and approximate confidence intervals for localized autocovariances for locally stationary time series. Journal of the Royal Statistical Society: Series B (Statistical Methodology), 75(5), 879\u2013904.","journal-title":"Journal of the Royal Statistical Society: Series B (Statistical Methodology)"},{"key":"5910_CR32","doi-asserted-by":"crossref","unstructured":"Oliveira, M., Torgo, L., & Costa, V.S. (2018) Evaluation procedures for forecasting with spatio-temporal data. In Joint European conference on machine learning and knowledge discovery in databases (pp. 703\u2013718). Berlin: Springer.","DOI":"10.1007\/978-3-030-10925-7_43"},{"key":"5910_CR33","doi-asserted-by":"crossref","unstructured":"Quinlan, J.R. (1993). Combining instance-based and model-based learning. In Proceedings of the tenth international conference on machine learning (pp. 236\u2013243).","DOI":"10.1016\/B978-1-55860-307-3.50037-X"},{"issue":"1","key":"5910_CR34","doi-asserted-by":"publisher","first-page":"39","DOI":"10.1016\/S0304-4076(00)00030-0","volume":"99","author":"J Racine","year":"2000","unstructured":"Racine, J. (2000). Consistent cross-validatory model-selection for dependent data: hv-block cross-validation. Journal of Econometrics, 99(1), 39\u201361.","journal-title":"Journal of Econometrics"},{"key":"5910_CR35","doi-asserted-by":"crossref","unstructured":"Snijders, T.A. (1988). On cross-validation for predictor evaluation in time series. In On model uncertainty and its statistical implications (pp. 56\u201369). Berlin: Springer.","DOI":"10.1007\/978-3-642-61564-1_4"},{"key":"5910_CR36","doi-asserted-by":"crossref","unstructured":"Stone, M. (1974). Cross-validation and multinomial prediction. Biometrika (pp. 509\u2013515).","DOI":"10.1093\/biomet\/61.3.509"},{"key":"5910_CR37","doi-asserted-by":"publisher","unstructured":"Takens, F. (1981). Dynamical systems and turbulence, Warwick 1980: Proceedings of a Symposium Held at the University of Warwick 1979\/80, chap. Detecting strange attractors in turbulence, pp. 366\u2013381. Springer Berlin Heidelberg, Berlin, Heidelberg. https:\/\/doi.org\/10.1007\/BFb0091924.","DOI":"10.1007\/BFb0091924"},{"issue":"4","key":"5910_CR38","doi-asserted-by":"publisher","first-page":"437","DOI":"10.1016\/S0169-2070(00)00065-0","volume":"16","author":"LJ Tashman","year":"2000","unstructured":"Tashman, L. J. (2000). Out-of-sample tests of forecasting accuracy: An analysis and review. International Journal of Forecasting, 16(4), 437\u2013450.","journal-title":"International Journal of Forecasting"},{"key":"5910_CR39","volume-title":"Sequential analysis","author":"A Wald","year":"1973","unstructured":"Wald, A. (1973). Sequential analysis. Philadelphia: Courier Corporation."},{"issue":"3","key":"5910_CR40","doi-asserted-by":"publisher","first-page":"335","DOI":"10.1007\/s10618-005-0039-x","volume":"13","author":"X Wang","year":"2006","unstructured":"Wang, X., Smith, K., & Hyndman, R. (2006). Characteristic-based clustering for time series data. Data Mining and Knowledge Discovery, 13(3), 335\u2013364.","journal-title":"Data Mining and Knowledge Discovery"},{"key":"5910_CR41","unstructured":"Wright MN (2015) Ranger: A fast implementation of random forests . R package version 0.3.0."}],"container-title":["Machine Learning"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10994-020-05910-7.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s10994-020-05910-7\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10994-020-05910-7.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2021,10,13]],"date-time":"2021-10-13T12:19:20Z","timestamp":1634127560000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s10994-020-05910-7"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2020,10,13]]},"references-count":41,"journal-issue":{"issue":"11","published-print":{"date-parts":[[2020,11]]}},"alternative-id":["5910"],"URL":"https:\/\/doi.org\/10.1007\/s10994-020-05910-7","relation":{},"ISSN":["0885-6125","1573-0565"],"issn-type":[{"value":"0885-6125","type":"print"},{"value":"1573-0565","type":"electronic"}],"subject":[],"published":{"date-parts":[[2020,10,13]]},"assertion":[{"value":"30 May 2019","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"1 June 2020","order":2,"name":"revised","label":"Revised","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"25 August 2020","order":3,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"13 October 2020","order":4,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"This content has been made available to all.","name":"free","label":"Free to read"}]}}