{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,6,4]],"date-time":"2026-06-04T11:12:33Z","timestamp":1780571553572,"version":"3.54.1"},"reference-count":47,"publisher":"Springer Science and Business Media LLC","issue":"1","license":[{"start":{"date-parts":[[2017,12,21]],"date-time":"2017-12-21T00:00:00Z","timestamp":1513814400000},"content-version":"unspecified","delay-in-days":0,"URL":"http:\/\/creativecommons.org\/licenses\/by\/4.0"}],"funder":[{"DOI":"10.13039\/501100003958","name":"Stichting voor de Technische Wetenschappen","doi-asserted-by":"publisher","award":["612.001.206"],"award-info":[{"award-number":["612.001.206"]}],"id":[{"id":"10.13039\/501100003958","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100001659","name":"Deutsche Forschungsgemeinschaft","doi-asserted-by":"publisher","award":["Emmy Noether grant HU 1900\/2-1"],"award-info":[{"award-number":["Emmy Noether grant HU 1900\/2-1"]}],"id":[{"id":"10.13039\/501100001659","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Mach Learn"],"published-print":{"date-parts":[[2018,1]]},"DOI":"10.1007\/s10994-017-5686-9","type":"journal-article","created":{"date-parts":[[2017,12,21]],"date-time":"2017-12-21T22:03:26Z","timestamp":1513893806000},"page":"149-176","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":95,"title":["The online performance estimation framework: heterogeneous ensemble learning for data streams"],"prefix":"10.1007","volume":"107","author":[{"given":"Jan N.","family":"van Rijn","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Geoffrey","family":"Holmes","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Bernhard","family":"Pfahringer","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Joaquin","family":"Vanschoren","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2017,12,21]]},"reference":[{"issue":"2","key":"5686_CR1","doi-asserted-by":"crossref","first-page":"197","DOI":"10.1016\/S0167-739X(97)00021-6","volume":"13","author":"C Apt\u00e9","year":"1997","unstructured":"Apt\u00e9, C., & Weiss, S. (1997). Data mining with decision trees and decision rules. Future Generation Computer Systems, 13(2), 197\u2013210.","journal-title":"Future Generation Computer Systems"},{"issue":"6","key":"5686_CR2","doi-asserted-by":"crossref","first-page":"627","DOI":"10.3233\/IDA-2007-11604","volume":"11","author":"J Beringer","year":"2007","unstructured":"Beringer, J., & H\u00fcllermeier, E. (2007). Efficient instance-based learning on data streams. Intelligent Data Analysis, 11(6), 627\u2013650.","journal-title":"Intelligent Data Analysis"},{"issue":"2","key":"5686_CR3","first-page":"30","volume":"3","author":"A Bifet","year":"2012","unstructured":"Bifet, A., Frank, E., Holmes, G., & Pfahringer, B. (2012). Ensembles of restricted hoeffding trees. ACM Transactions on Intelligent Systems and Technology (TIST), 3(2), 30.","journal-title":"ACM Transactions on Intelligent Systems and Technology (TIST)"},{"key":"5686_CR4","first-page":"139","volume":"7","author":"A Bifet","year":"2007","unstructured":"Bifet, A., & Gavalda, R. (2007). Learning from time-changing data with adaptive windowing. SDM, SIAM, 7, 139\u2013148.","journal-title":"SDM, SIAM"},{"key":"5686_CR5","doi-asserted-by":"crossref","unstructured":"Bifet, A., & Gavald\u00e0, R. (2009). Adaptive learning from evolving data streams. In Advances in intelligent data analysis VIII (pp. 249\u2013260). Springer.","DOI":"10.1007\/978-3-642-03915-7_22"},{"key":"5686_CR6","first-page":"1601","volume":"11","author":"A Bifet","year":"2010","unstructured":"Bifet, A., Holmes, G., Kirkby, R., & Pfahringer, B. (2010a). MOA: Massive online analysis. Journal of Machine Learning Research, 11, 1601\u20131604.","journal-title":"Journal of Machine Learning Research"},{"key":"5686_CR7","doi-asserted-by":"crossref","unstructured":"Bifet, A., Holmes, G., & Pfahringer, B. (2010b). Leveraging bagging for evolving data streams. In Machine learning and knowledge discovery in databases, Lecture Notes in Computer Science (Vol. 6321, pp. 135\u2013150). Springer.","DOI":"10.1007\/978-3-642-15880-3_15"},{"key":"5686_CR8","doi-asserted-by":"crossref","unstructured":"Bottou, L. (2004). Stochastic learning. In Advanced lectures on machine learning (pp. 146\u2013168). Springer.","DOI":"10.1007\/978-3-540-28650-9_7"},{"key":"5686_CR9","doi-asserted-by":"crossref","unstructured":"Brazdil, P., Gama, J., & Henery, B. (1994). Characterizing the applicability of classification algorithms using meta-level learning. In Machine learning: ECML-94 (pp. 83\u2013102). Springer.","DOI":"10.1007\/3-540-57868-4_52"},{"issue":"3","key":"5686_CR10","doi-asserted-by":"crossref","first-page":"251","DOI":"10.1023\/A:1021713901879","volume":"50","author":"P Brazdil","year":"2003","unstructured":"Brazdil, P., Soares, C., & Da Costa, J. P. (2003). Ranking learning algorithms: Using IBL and meta-learning on accuracy and time results. Machine Learning, 50(3), 251\u2013277.","journal-title":"Machine Learning"},{"issue":"2","key":"5686_CR11","first-page":"123","volume":"24","author":"L Breiman","year":"1996","unstructured":"Breiman, L. (1996). Bagging predictors. Machine Learning, 24(2), 123\u2013140.","journal-title":"Machine Learning"},{"key":"5686_CR12","doi-asserted-by":"crossref","unstructured":"Caruana, R., Niculescu-Mizil, A., Crew, G., & Ksikes, A. (2004). Ensemble selection from libraries of models. In Proceedings of the twenty-first international conference on Machine learning (p. 18). ACM.","DOI":"10.1145\/1015330.1015432"},{"key":"5686_CR13","first-page":"1","volume":"7","author":"J Dem\u0161ar","year":"2006","unstructured":"Dem\u0161ar, J. (2006). Statistical comparisons of classifiers over multiple data sets. The Journal of Machine Learning Research, 7, 1\u201330.","journal-title":"The Journal of Machine Learning Research"},{"key":"5686_CR14","doi-asserted-by":"crossref","unstructured":"Domingos, P., & Hulten, G. (2000). Mining high-speed data streams. In Proceedings of the sixth ACM SIGKDD international conference on knowledge discovery and data mining (pp. 71\u201380).","DOI":"10.1145\/347090.347107"},{"issue":"4","key":"5686_CR15","doi-asserted-by":"crossref","first-page":"945","DOI":"10.1198\/1061860032544","volume":"12","author":"P Domingos","year":"2003","unstructured":"Domingos, P., & Hulten, G. (2003). A general framework for mining massive data streams. Journal of Computational and Graphical Statistics, 12(4), 945\u2013949.","journal-title":"Journal of Computational and Graphical Statistics"},{"issue":"3","key":"5686_CR16","doi-asserted-by":"crossref","first-page":"315","DOI":"10.1023\/A:1007652114878","volume":"41","author":"J Gama","year":"2000","unstructured":"Gama, J., & Brazdil, P. (2000). Cascade generalization. Machine Learning, 41(3), 315\u2013343.","journal-title":"Machine Learning"},{"issue":"3","key":"5686_CR17","doi-asserted-by":"crossref","first-page":"489","DOI":"10.1007\/s10115-013-0654-6","volume":"40","author":"J Gama","year":"2014","unstructured":"Gama, J., & Kosina, P. (2014). Recurrent concepts in data streams classification. Knowledge and Information Systems, 40(3), 489\u2013507.","journal-title":"Knowledge and Information Systems"},{"key":"5686_CR18","doi-asserted-by":"crossref","unstructured":"Gama, J., Medas, P., Castillo, G., & Rodrigues, P. (2004a). Learning with drift detection. In SBIA Brazilian symposium on artificial intelligence, Lecture Notes in Computer Science (Vol. 3171, pp. 286\u2013295). Springer.","DOI":"10.1007\/978-3-540-28645-5_29"},{"key":"5686_CR19","doi-asserted-by":"crossref","unstructured":"Gama, J., Medas, P., & Rocha, R. (2004b). Forest trees for on-line data. In Proceedings of the 2004 ACM symposium on applied computing (pp. 632\u2013636). ACM.","DOI":"10.1145\/967900.968033"},{"key":"5686_CR20","doi-asserted-by":"crossref","unstructured":"Gama, J., Sebasti\u00e3o, R., & Rodrigues, P.\u00a0P. (2009). Issues in evaluation of stream learning algorithms. In Proceedings of the 15th ACM SIGKDD international conference on knowledge discovery and data mining (pp. 329\u2013338). ACM.","DOI":"10.1145\/1557019.1557060"},{"issue":"3","key":"5686_CR21","doi-asserted-by":"crossref","first-page":"317","DOI":"10.1007\/s10994-012-5320-9","volume":"90","author":"J Gama","year":"2013","unstructured":"Gama, J., Sebasti\u00e3o, R., & Rodrigues, P. P. (2013). On evaluating stream learning algorithms. Machine Learning, 90(3), 317\u2013346.","journal-title":"Machine Learning"},{"issue":"1","key":"5686_CR22","doi-asserted-by":"crossref","first-page":"10","DOI":"10.1145\/1656274.1656278","volume":"11","author":"M Hall","year":"2009","unstructured":"Hall, M., Frank, E., Holmes, G., Pfahringer, B., Reutemann, P., & Witten, I. H. (2009). The WEKA data mining software: An update. ACM SIGKDD Explorations Newsletter, 11(1), 10\u201318.","journal-title":"ACM SIGKDD Explorations Newsletter"},{"issue":"10","key":"5686_CR23","doi-asserted-by":"crossref","first-page":"993","DOI":"10.1109\/34.58871","volume":"12","author":"L Hansen","year":"1990","unstructured":"Hansen, L., & Salamon, P. (1990). Neural network ensembles. IEEE Transactions on Pattern Analysis and Machine Intelligence, 12(10), 993\u20131001.","journal-title":"IEEE Transactions on Pattern Analysis and Machine Intelligence"},{"issue":"2","key":"5686_CR24","doi-asserted-by":"crossref","first-page":"181","DOI":"10.1080\/00031305.1998.10480559","volume":"52","author":"JL Hintze","year":"1998","unstructured":"Hintze, J. L., & Nelson, R. D. (1998). Violin plots: A box plot-density trace synergism. The American Statistician, 52(2), 181\u2013184.","journal-title":"The American Statistician"},{"key":"5686_CR25","first-page":"2755","volume":"8","author":"JZ Kolter","year":"2007","unstructured":"Kolter, J. Z., & Maloof, M. A. (2007). Dynamic weighted majority: An ensemble method for drifting concepts. Journal of Machine Learning Research, 8, 2755\u20132790.","journal-title":"Journal of Machine Learning Research"},{"issue":"1","key":"5686_CR26","doi-asserted-by":"crossref","first-page":"69","DOI":"10.1007\/BF00187434","volume":"10","author":"KK Ladha","year":"1993","unstructured":"Ladha, K. K. (1993). Condorcet\u2019s jury theorem in light of de Finetti\u2019s theorem. Social Choice and Welfare, 10(1), 69\u201385.","journal-title":"Social Choice and Welfare"},{"issue":"6","key":"5686_CR27","doi-asserted-by":"crossref","first-page":"827","DOI":"10.3233\/IDA-2011-0498","volume":"15","author":"JW Lee","year":"2011","unstructured":"Lee, J. W., & Giraud-Carrier, C. (2011). A metric for unsupervised metalearning. Intelligent Data Analysis, 15(6), 827\u2013841.","journal-title":"Intelligent Data Analysis"},{"issue":"2","key":"5686_CR28","doi-asserted-by":"crossref","first-page":"212","DOI":"10.1006\/inco.1994.1009","volume":"108","author":"N Littlestone","year":"1994","unstructured":"Littlestone, N., & Warmuth, M. (1994). The weighted majority algorithm. Information and Computation, 108(2), 212\u2013261.","journal-title":"Information and Computation"},{"key":"5686_CR29","doi-asserted-by":"crossref","unstructured":"Nguyen, H. L., Woon, Y. K., Ng, W. K., & Wan, L. (2012). Heterogeneous ensemble for feature drifts in data streams. In Advances in knowledge discovery and data mining (pp. 1\u201312). Springer.","DOI":"10.1007\/978-3-642-30220-6_1"},{"key":"5686_CR30","doi-asserted-by":"crossref","unstructured":"Oza, N. C. (2005). Online bagging and boosting. In 2005 IEEE international conference on systems, man and cybernetics (Vol.\u00a03, pp. 2340\u20132345). IEEE.","DOI":"10.1109\/ICSMC.2005.1571498"},{"key":"5686_CR31","unstructured":"Peterson, A. H., & Martinez, T. (2005). Estimating the potential for combining learning models. In Proceedings of the ICML workshop on meta-learning (pp. 68\u201375)."},{"key":"5686_CR32","unstructured":"Pfahringer, B., Bensusan, H., & Giraud-Carrier, C. (2000). Tell me who can learn you and I can tell you who you are: Landmarking Various learning algorithms. In Proceedings of the 17th international conference on machine learning (pp. 743\u2013750)."},{"key":"5686_CR33","doi-asserted-by":"crossref","unstructured":"Pfahringer, B., Holmes, G., & Kirkby, R. (2007). New options for hoeffding trees. In AI 2007: Advances in artificial intelligence (pp. 90\u201399). Springer.","DOI":"10.1007\/978-3-540-76928-6_11"},{"key":"5686_CR34","doi-asserted-by":"crossref","unstructured":"Read, J., Bifet, A., Pfahringer, B., & Holmes, G. (2012) Batch-incremental versus instance-incremental learning in dynamic and evolving data. In Advances in intelligent data analysis XI (pp. 313\u2013323). Springer.","DOI":"10.1007\/978-3-642-34156-4_29"},{"key":"5686_CR35","doi-asserted-by":"crossref","first-page":"65","DOI":"10.1016\/S0065-2458(08)60520-3","volume":"15","author":"JR Rice","year":"1976","unstructured":"Rice, J. R. (1976). The algorithm selection problem. Advances in Computers, 15, 65\u2013118.","journal-title":"Advances in Computers"},{"key":"5686_CR36","doi-asserted-by":"crossref","unstructured":"Rokach, L., & Maimon, O. (2005). Clustering methods. In Data mining and knowledge discovery handbook (pp. 321\u2013352). Springer.","DOI":"10.1007\/0-387-25465-X_15"},{"key":"5686_CR37","doi-asserted-by":"crossref","first-page":"52","DOI":"10.1016\/j.neucom.2013.05.048","volume":"127","author":"ALD Rossi","year":"2014","unstructured":"Rossi, A. L. D., de Leon Ferreira, A. C. P., Soares, C., & De Souza, B. F. (2014). MetaStream: A meta-learning based method for periodic algorithm selection in time-changing data. Neurocomputing, 127, 52\u201364.","journal-title":"Neurocomputing"},{"issue":"2","key":"5686_CR38","first-page":"197","volume":"5","author":"RE Schapire","year":"1990","unstructured":"Schapire, R. E. (1990). The strength of weak learnability. Machine Learning, 5(2), 197\u2013227.","journal-title":"Machine Learning"},{"key":"5686_CR39","doi-asserted-by":"crossref","first-page":"250","DOI":"10.1016\/j.neucom.2014.09.076","volume":"150","author":"A Shaker","year":"2015","unstructured":"Shaker, A., & H\u00fcllermeier, E. (2015). Recovery analysis for adaptive learning from non-stationary data streams: Experimental design and case study. Neurocomputing, 150, 250\u2013264.","journal-title":"Neurocomputing"},{"issue":"1","key":"5686_CR40","doi-asserted-by":"crossref","first-page":"3","DOI":"10.1007\/s10107-010-0420-4","volume":"127","author":"S Shalev-Shwartz","year":"2011","unstructured":"Shalev-Shwartz, S., Singer, Y., Srebro, N., & Cotter, A. (2011). Pegasos: Primal estimated sub-gradient solver for SVM. Mathematical Programming, 127(1), 3\u201330.","journal-title":"Mathematical Programming"},{"key":"5686_CR41","unstructured":"van Rijn, J. N. (2016). Massively collaborative machine learning. PhD thesis, Leiden University."},{"key":"5686_CR42","doi-asserted-by":"crossref","unstructured":"van Rijn, J. N., Holmes, G., Pfahringer, B., & Vanschoren, J. (2014). Algorithm selection on data streams. In Discovery Science, Lecture Notes in Computer Science (Vol. 8777, pp. 325\u2013336). Springer.","DOI":"10.1007\/978-3-319-11812-3_28"},{"key":"5686_CR43","doi-asserted-by":"crossref","unstructured":"van Rijn, J. N., Holmes, G., Pfahringer, B., & Vanschoren, J. (2015). Having a blast: Meta-learning and heterogeneous ensembles for data streams. In 2015 IEEE international conference on data mining (ICDM) (pp. 1003\u20131008). IEEE.","DOI":"10.1109\/ICDM.2015.55"},{"issue":"2","key":"5686_CR44","doi-asserted-by":"crossref","first-page":"49","DOI":"10.1145\/2641190.2641198","volume":"15","author":"J Vanschoren","year":"2014","unstructured":"Vanschoren, J., van Rijn, J. N., Bischl, B., & Torgo, L. (2014). OpenML: Networked science in machine learning. ACM SIGKDD Explorations Newsletter, 15(2), 49\u201360.","journal-title":"ACM SIGKDD Explorations Newsletter"},{"key":"5686_CR45","doi-asserted-by":"crossref","unstructured":"Wang, H., Fan, W., Yu, P. S., & Han, J. (2003). Mining concept-drifting data streams using ensemble classifiers. In KDD (pp. 226\u2013235).","DOI":"10.1145\/956750.956778"},{"issue":"2","key":"5686_CR46","doi-asserted-by":"crossref","first-page":"241","DOI":"10.1016\/S0893-6080(05)80023-1","volume":"5","author":"DH Wolpert","year":"1992","unstructured":"Wolpert, D. H. (1992). Stacked generalization. Neural Networks, 5(2), 241\u2013259.","journal-title":"Neural Networks"},{"key":"5686_CR47","doi-asserted-by":"crossref","unstructured":"Zhang, P., Gao, B. J., Zhu, X., & Guo, L. (2011) Enabling fast lazy learning for data streams. In 2011 IEEE 11th International conference on data mining (ICDM) (pp. 932\u2013941). IEEE.","DOI":"10.1109\/ICDM.2011.63"}],"container-title":["Machine Learning"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/link.springer.com\/article\/10.1007\/s10994-017-5686-9\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s10994-017-5686-9.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s10994-017-5686-9.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,29]],"date-time":"2025-06-29T03:51:12Z","timestamp":1751169072000},"score":1,"resource":{"primary":{"URL":"http:\/\/link.springer.com\/10.1007\/s10994-017-5686-9"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2017,12,21]]},"references-count":47,"journal-issue":{"issue":"1","published-print":{"date-parts":[[2018,1]]}},"alternative-id":["5686"],"URL":"https:\/\/doi.org\/10.1007\/s10994-017-5686-9","relation":{},"ISSN":["0885-6125","1573-0565"],"issn-type":[{"value":"0885-6125","type":"print"},{"value":"1573-0565","type":"electronic"}],"subject":[],"published":{"date-parts":[[2017,12,21]]}}}