{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,5,16]],"date-time":"2026-05-16T00:19:07Z","timestamp":1778890747360,"version":"3.51.4"},"reference-count":39,"publisher":"Springer Science and Business Media LLC","issue":"5","license":[{"start":{"date-parts":[[2021,8,21]],"date-time":"2021-08-21T00:00:00Z","timestamp":1629504000000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"},{"start":{"date-parts":[[2021,8,21]],"date-time":"2021-08-21T00:00:00Z","timestamp":1629504000000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Appl Intell"],"published-print":{"date-parts":[[2022,3]]},"DOI":"10.1007\/s10489-021-02735-2","type":"journal-article","created":{"date-parts":[[2021,8,21]],"date-time":"2021-08-21T06:02:26Z","timestamp":1629525746000},"page":"5770-5783","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":21,"title":["Out-of-time cross-validation strategies for classification in the presence of dataset shift"],"prefix":"10.1007","volume":"52","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-7124-0437","authenticated-orcid":false,"given":"Sebasti\u00e1n","family":"Maldonado","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Julio","family":"L\u00f3pez","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Andr\u00e9s","family":"Iturriaga","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2021,8,21]]},"reference":[{"key":"2735_CR1","doi-asserted-by":"crossref","unstructured":"Anelli V, Di Noia T, Di Sciascio E, Ragone A, Trotta J (2019) Local popularity and time in top-n recommendation. In: Azzopardi L, Stein B, Fuhr N, Mayr P, Hauff C, Hiemstra D (eds) Advances in information retrieval. ECIR 2019. Springer International Publishing","DOI":"10.1007\/978-3-030-15712-8_63"},{"key":"2735_CR2","doi-asserted-by":"crossref","unstructured":"Baesens B, Roesch D, Scheule H (2016) Credit risk analytics: Measurement Techniques, Applications, and Examples in SAS. Wiley","DOI":"10.1002\/9781119449560"},{"key":"2735_CR3","doi-asserted-by":"crossref","unstructured":"Bergmeir C, Ben\u00edtez J (2012) On the use of cross-validation for time series predictor evaluation. Inf Sci 191:192\u2013213","DOI":"10.1016\/j.ins.2011.12.028"},{"key":"2735_CR4","doi-asserted-by":"crossref","unstructured":"Bifet A, de Francisci Morales G, Read J, Holmes G, Pfahringer B (2015) Efficient online evaluation of big data stream classifiers. In: Proceedings of the 21th ACM SIGKDD International Conference on Knowledge Discovery and Data Mining, Association for Computing Machinery, KDD \u201915, pp 59\u201368","DOI":"10.1145\/2783258.2783372"},{"issue":"11","key":"2735_CR5","doi-asserted-by":"publisher","first-page":"1895","DOI":"10.1057\/jors.2015.18","volume":"66","author":"C Bravo","year":"2015","unstructured":"Bravo C, Maldonado S (2015) Fieller stability measure: a novel model-dependent backtesting approach. J Oper Res Soc 66(11):1895\u20131905","journal-title":"J Oper Res Soc"},{"issue":"2","key":"2735_CR6","doi-asserted-by":"publisher","first-page":"358","DOI":"10.1016\/j.ejor.2012.10.040","volume":"227","author":"C Bravo","year":"2013","unstructured":"Bravo C, Maldonado S, Weber R (2013) Methodologies for granting and managing loans for micro-entrepreneurs: New developments and practical experiences. Eur J Oper Res 227(2):358\u2013366","journal-title":"Eur J Oper Res"},{"issue":"3","key":"2735_CR7","doi-asserted-by":"publisher","first-page":"359","DOI":"10.1057\/jors.2009.69","volume":"61","author":"G Castermans","year":"2010","unstructured":"Castermans G, Martens D, Gestel T, Hamers B, Baesens B (2010) An overview and framework for pd backtesting and benchmarking. J Oper Res Soc 61(3):359\u2013373","journal-title":"J Oper Res Soc"},{"key":"2735_CR8","doi-asserted-by":"crossref","unstructured":"Cieslak D, Chawla N (2007) Detecting fractures in classifier performance. In: Seventh IEEE International Conference on Data Mining (ICDM 2007). IEEE, pp 123\u2013132","DOI":"10.1109\/ICDM.2007.106"},{"key":"2735_CR9","first-page":"1","volume":"7","author":"J Dem\u0161ar","year":"2006","unstructured":"Dem\u0161ar J (2006) Statistical comparisons of classifiers over multiple data set. J Mach Learn Res 7:1\u201330","journal-title":"J Mach Learn Res"},{"issue":"4","key":"2735_CR10","doi-asserted-by":"publisher","first-page":"12","DOI":"10.1109\/MCI.2015.2471196","volume":"10","author":"G Ditzler","year":"2015","unstructured":"Ditzler G, Roveri M, Alippi C, Polikar R (2015) Learning in nonstationary environments: a survey. IEEE Comput Intell Mag 10(4):12\u201325","journal-title":"IEEE Comput Intell Mag"},{"key":"2735_CR11","unstructured":"Fr\u00fchwirth-Schnatter S (2006) Finite mixture and Markov switching models. Springer Science & Business Media"},{"issue":"3","key":"2735_CR12","doi-asserted-by":"publisher","first-page":"317","DOI":"10.1007\/s10994-012-5320-9","volume":"90","author":"J Gama","year":"2013","unstructured":"Gama J, Sebasti\u00e3o R, Rodrigues P (2013) On evaluating stream learning algorithms. Mach Learn 90(3):317\u2013346","journal-title":"Mach Learn"},{"issue":"4","key":"2735_CR13","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1145\/2523813","volume":"46","author":"J Gama","year":"2014","unstructured":"Gama J, \u017eliobaite I, Bifet A, Pechenizkiy M, Bouchachia A (2014) A survey on concept drift adaptation. ACM Comput Surv (CSUR) 46(4):1\u201337","journal-title":"ACM Comput Surv (CSUR)"},{"issue":"1","key":"2735_CR14","doi-asserted-by":"publisher","first-page":"377","DOI":"10.1016\/j.csda.2012.07.007","volume":"57","author":"V Hofer","year":"2013","unstructured":"Hofer V, Krempl G (2013) Drift mining in data: a framework for addressing drift in classification. Comput Stat Data Anal 57(1):377\u2013391","journal-title":"Comput Stat Data Anal"},{"key":"2735_CR15","doi-asserted-by":"crossref","unstructured":"Hyndman RJ, Athanasopoulos G (2018) Forecasting: principles and practice, 2nd edn. O Texts","DOI":"10.32614\/CRAN.package.fpp2"},{"key":"2735_CR16","doi-asserted-by":"crossref","unstructured":"Jiang G, Wang W (2017) Markov cross-validation for time series model evaluations. Inf Sci 375:219\u2013233","DOI":"10.1016\/j.ins.2016.09.061"},{"key":"2735_CR17","doi-asserted-by":"crossref","unstructured":"Kelly M, Hand D, Adams N (1999) The impact of changing populations on classifier performance. In: Proceedings of the fifth ACM SIGKDD international conference on Knowledge discovery and data mining, pp 367\u2013371","DOI":"10.1145\/312129.312285"},{"issue":"3","key":"2735_CR18","doi-asserted-by":"publisher","first-page":"281","DOI":"10.3233\/IDA-2004-8305","volume":"8","author":"R Klinkenberg","year":"2004","unstructured":"Klinkenberg R (2004) Learning drifting concepts: Example selection vs. example weighting. Intell Data Anal 8(3):281\u2013 300","journal-title":"Intell Data Anal"},{"key":"2735_CR19","first-page":"2755","volume":"8","author":"J Kolter","year":"2007","unstructured":"Kolter J, Maloof M (2007) Dynamic weighted majority: an ensemble method for drifting concepts. J Mach Learn Res 8:2755\u20132790","journal-title":"J Mach Learn Res"},{"key":"2735_CR20","doi-asserted-by":"crossref","unstructured":"L\u00f3pez J, Maldonado S (2019) Profit-based credit scoring based on robust optimization and feature selection. Inf Sci 500:190\u2013202","DOI":"10.1016\/j.ins.2019.05.093"},{"key":"2735_CR21","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1016\/j.ins.2013.09.038","volume":"257","author":"V L\u00f3pez","year":"2014","unstructured":"L\u00f3pez V, Fern\u00e1ndez A, Herrera F (2014) On the importance of the validation technique for classification with imbalanced datasets: Addressing covariate shift when data is skewed. Inf Sci 257:1\u201313","journal-title":"Inf Sci"},{"key":"2735_CR22","doi-asserted-by":"crossref","unstructured":"Lucas Y, Portier PE, Laporte L, Calabretto S, He-Guelton L, Obl\u00e9 F, Granitzer M (2019) Dataset shift quantification for credit card fraud detection. In: 2019 IEEE Second International Conference on Artificial Intelligence and Knowledge Engineering (AIKE). IEEE, pp 97\u2013100","DOI":"10.1109\/AIKE.2019.00024"},{"key":"2735_CR23","doi-asserted-by":"publisher","first-page":"84","DOI":"10.1016\/j.dss.2019.04.006","volume":"121","author":"S Maldonado","year":"2019","unstructured":"Maldonado S, Gonz\u00e1lez R, Quijada F, Ram\u00edrez A (2019) Analytics meets port logistics: a decision support system for container stacking operations. Decis Support Syst 121:84\u201393","journal-title":"Decis Support Syst"},{"issue":"1","key":"2735_CR24","doi-asserted-by":"publisher","first-page":"273","DOI":"10.1016\/j.ejor.2019.12.007","volume":"284","author":"S Maldonado","year":"2020","unstructured":"Maldonado S, L\u00f3pez J, Vairetti C (2020) Profit-based churn prediction based on minimax probability machines. Eur J Oper Res 284(1):273\u2013284","journal-title":"Eur J Oper Res"},{"key":"2735_CR25","doi-asserted-by":"crossref","unstructured":"Moreno-Torres J, Raeder T, Alaiz-Rodr\u00edguez R, Chawla N, Herrera F (2012s) A unifying view on dataset shift in classification. Pattern Recogn 45(1):521\u2013530","DOI":"10.1016\/j.patcog.2011.06.019"},{"key":"2735_CR26","doi-asserted-by":"crossref","unstructured":"Moreno-Torres J, S\u00e1ez J, Herrera F (2012b) Study on the impact of partition-induced dataset shift on k-fold cross-validation. IEEE Trans Neural Netw Learn Syst 23(8):1304\u20131312","DOI":"10.1109\/TNNLS.2012.2199516"},{"key":"2735_CR27","doi-asserted-by":"crossref","unstructured":"Quionero-Candela J, Sugiyama M, Schwaighofer A, Lawrence N (2009) Dataset shift in machine learning. The MIT Press","DOI":"10.7551\/mitpress\/9780262170055.001.0001"},{"key":"2735_CR28","doi-asserted-by":"crossref","unstructured":"S\u00e1nchez P, Bellog\u00edn A (2018) Time-aware novelty metrics for recommender systems. In: ECIR, pp 357\u2013370","DOI":"10.1007\/978-3-319-76941-7_27"},{"key":"2735_CR29","unstructured":"Siddiqi N (2007) Credit risk scorecards: developing and implementing intelligent credit scoring, vol 3. Wiley"},{"issue":"6","key":"2735_CR30","doi-asserted-by":"publisher","first-page":"2495","DOI":"10.1287\/mnsc.2019.3308","volume":"66","author":"D Simester","year":"2019","unstructured":"Simester D, Timoshenko A, Zoumpoulis S (2019) Targeting prospective customers: Robustness of machine-learning methods to typical data challenges. Manag Sci 66(6):2495\u2013 2522","journal-title":"Manag Sci"},{"issue":"2","key":"2735_CR31","doi-asserted-by":"publisher","first-page":"139","DOI":"10.1191\/1471082x02st029oa","volume":"2","author":"G Storvik","year":"2002","unstructured":"Storvik G, Frigessi A, Hirst D (2002) Stationary space-time gaussian fields and their time autoregressive representation. Stat Model 2(2):139\u2013161","journal-title":"Stat Model"},{"key":"2735_CR32","first-page":"985","volume":"8","author":"M Sugiyama","year":"2007","unstructured":"Sugiyama M, Krauledat M, M\u00fcller K R (2007) Covariate shift adaptation by importance weighted cross validation. J Mach Learn Res 8:985\u20131005","journal-title":"J Mach Learn Res"},{"issue":"4","key":"2735_CR33","doi-asserted-by":"publisher","first-page":"699","DOI":"10.1007\/s10463-008-0197-x","volume":"60","author":"M Sugiyama","year":"2008","unstructured":"Sugiyama M, Suzuki T, Nakajima S, Kashima H, von B\u00fcnau P, Kawanabe M (2008) Direct importance estimation for covariate shift adaptation. Ann Inst Stat Math 60(4):699\u2013746","journal-title":"Ann Inst Stat Math"},{"key":"2735_CR34","doi-asserted-by":"crossref","unstructured":"Sun J, Li H, Fujita H, Fu B, Ai W (2020) Class-imbalanced dynamic financial distress prediction based on adaboost-svm ensemble combined with smote and time weighting. Inf Fusion 54:128\u2013144","DOI":"10.1016\/j.inffus.2019.07.006"},{"issue":"18","key":"2735_CR35","doi-asserted-by":"publisher","first-page":"6034","DOI":"10.1080\/00207543.2017.1380326","volume":"56","author":"I Svetunkov","year":"2018","unstructured":"Svetunkov I, Petropoulos F (2018) Old dog, new tricks: a modelling view of simple moving averages. Int J Prod Res 56(18):6034\u20136047","journal-title":"Int J Prod Res"},{"issue":"4","key":"2735_CR36","doi-asserted-by":"publisher","first-page":"437","DOI":"10.1016\/S0169-2070(00)00065-0","volume":"16","author":"L Tashman","year":"2000","unstructured":"Tashman L (2000) Out-of-sample tests of forecasting accuracy: an analysis and review. Int J Forecast 16(4):437\u2013450","journal-title":"Int J Forecast"},{"key":"2735_CR37","unstructured":"Tibshirani R, Hastie T (2009) The Elements of Statistical Learning. Springer"},{"issue":"1","key":"2735_CR38","first-page":"69","volume":"23","author":"G Widmer","year":"1996","unstructured":"Widmer G, Kubat M (1996) Learning in the presence of concept drift and hidden contexts. Mach Learn 23(1):69\u2013101","journal-title":"Mach Learn"},{"issue":"1","key":"2735_CR39","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1080\/095281300146272","volume":"12","author":"X Zeng","year":"2000","unstructured":"Zeng X, Martinez TR (2000) Distribution-balanced stratified crossvalidation for accuracy estimation. J Exper Theor Artif Intell 12(1):1\u201312","journal-title":"J Exper Theor Artif Intell"}],"container-title":["Applied Intelligence"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10489-021-02735-2.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s10489-021-02735-2\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10489-021-02735-2.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,9,7]],"date-time":"2024-09-07T01:04:34Z","timestamp":1725671074000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s10489-021-02735-2"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2021,8,21]]},"references-count":39,"journal-issue":{"issue":"5","published-print":{"date-parts":[[2022,3]]}},"alternative-id":["2735"],"URL":"https:\/\/doi.org\/10.1007\/s10489-021-02735-2","relation":{},"ISSN":["0924-669X","1573-7497"],"issn-type":[{"value":"0924-669X","type":"print"},{"value":"1573-7497","type":"electronic"}],"subject":[],"published":{"date-parts":[[2021,8,21]]},"assertion":[{"value":"31 July 2021","order":1,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"21 August 2021","order":2,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}}]}}