{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,6,10]],"date-time":"2026-06-10T01:25:51Z","timestamp":1781054751547,"version":"3.54.1"},"reference-count":95,"publisher":"Springer Science and Business Media LLC","issue":"11-12","license":[{"start":{"date-parts":[[2024,12,1]],"date-time":"2024-12-01T00:00:00Z","timestamp":1733011200000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2024,12,1]],"date-time":"2024-12-01T00:00:00Z","timestamp":1733011200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"funder":[{"DOI":"10.13039\/501100010628","name":"Universidad de La Sabana","doi-asserted-by":"publisher","award":["ING-312-2023"],"award-info":[{"award-number":["ING-312-2023"]}],"id":[{"id":"10.13039\/501100010628","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Mach Learn"],"published-print":{"date-parts":[[2024,12]]},"DOI":"10.1007\/s10994-024-06619-7","type":"journal-article","created":{"date-parts":[[2024,12,23]],"date-time":"2024-12-23T11:26:02Z","timestamp":1734953162000},"page":"8371-8425","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":31,"title":["Learning curves for decision making in supervised machine learning: a survey"],"prefix":"10.1007","volume":"113","author":[{"given":"Felix","family":"Mohr","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Jan N.","family":"van Rijn","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2024,12,23]]},"reference":[{"key":"6619_CR1","first-page":"19858","volume":"36","author":"S Adriaensen","year":"2023","unstructured":"Adriaensen, S., Rakotoarison, H., M\u00fcller, S., & Hutter, F. (2023). Efficient Bayesian learning curve extrapolation using prior-data fitted networks. Advances in Neural Information Processing Systems, 36, 19858\u201319886.","journal-title":"Advances in Neural Information Processing Systems"},{"key":"6619_CR2","doi-asserted-by":"publisher","first-page":"167","DOI":"10.1016\/j.jocm.2018.07.002","volume":"28","author":"A Alwosheel","year":"2018","unstructured":"Alwosheel, A., van Cranenburgh, S., & Chorus, C. G. (2018). Is your dataset big enough? Sample size requirements when using artificial neural networks for discrete choice analysis. Journal of Choice Modelling, 28, 167\u2013182.","journal-title":"Journal of Choice Modelling"},{"issue":"1","key":"6619_CR3","doi-asserted-by":"publisher","first-page":"140","DOI":"10.1162\/neco.1993.5.1.140","volume":"5","author":"S Amari","year":"1993","unstructured":"Amari, S., & Murata, N. (1993). Statistical theory of learning curves under entropic loss criterion. Neural Computation, 5(1), 140\u2013153.","journal-title":"Neural Computation"},{"key":"6619_CR4","unstructured":"Baker, B., Gupta, O., Raskar, R., & Naik, N. (2018). Accelerating neural architecture search using performance prediction. In 6th International conference on learning representations, ICLR\u201918."},{"key":"6619_CR5","doi-asserted-by":"publisher","first-page":"25","DOI":"10.1016\/j.aca.2012.11.007","volume":"760","author":"C Beleites","year":"2013","unstructured":"Beleites, C., Neugebauer, U., Bocklitz, T., Krafft, C., & Popp, J. (2013). Sample size planning for classification models. Analytica Chimica Acta, 760, 25\u201333.","journal-title":"Analytica Chimica Acta"},{"key":"6619_CR6","doi-asserted-by":"publisher","DOI":"10.7551\/mitpress\/10654.001.0001","volume-title":"Machine learning for data streams: With practical examples in MOA","author":"A Bifet","year":"2018","unstructured":"Bifet, A., Gavald\u00e0, R., Holmes, G., & Pfahringer, B. (2018). Machine learning for data streams: With practical examples in MOA. MIT Press."},{"key":"6619_CR7","unstructured":"Bishop, C. (1995). Regularization and complexity control in feed-forward networks. In Proceedings international conference on artificial neural networks ICANN\u201995 (pp. 141\u2013148)."},{"key":"6619_CR8","doi-asserted-by":"crossref","unstructured":"Boonyanunta, N., & Zeephongsekul, P. (2004). Predicting the relationship between the size of training sample and the predictive power of classifiers. In Knowledge-based intelligent information and engineering systems, 8th international conference, KES 2004 (pp. 529\u2013535).","DOI":"10.1007\/978-3-540-30134-9_71"},{"key":"6619_CR9","unstructured":"Bornschein, J., Visin, F., & Osindero, S. (2020). Small data, big decisions: Model selection in the small-data regime. In Proceedings of the 37th international conference on machine learning (pp. 1035\u20131044)."},{"key":"6619_CR10","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-67024-5","volume-title":"Metalearning: Applications to automated machine learning and data mining","author":"P Brazdil","year":"2022","unstructured":"Brazdil, P., van Rijn, J. N., Soares, C., & Vanschoren, J. (2022). Metalearning: Applications to automated machine learning and data mining (2nd ed.). Springer.","edition":"2"},{"key":"6619_CR11","doi-asserted-by":"crossref","unstructured":"Cardona-Escobar, A. F., Giraldo-Forero, A. F., & Castro-Ospina, A. E., & Jaramillo-Garz\u00f3n, J. A. (2017). Efficient hyperparameter optimization in convolutional neural networks by learning curves prediction. In Progress in pattern recognition, image analysis, computer vision, and applications (pp. 143\u2013151).","DOI":"10.1007\/978-3-319-75193-1_18"},{"key":"6619_CR12","doi-asserted-by":"crossref","unstructured":"Chandrashekaran, A., & Lane, I. R. (2017). Speeding up hyper-parameter optimization by extrapolation of learning curves using previous builds. In Machine learning and knowledge discovery in databases\u2014European conference, ECML PKDD (vol. 2017, pp. 477\u2013492).","DOI":"10.1007\/978-3-319-71249-9_29"},{"key":"6619_CR13","unstructured":"Cho, J., Lee, K., & Shin, E., Choy, G., & Do, S. (2015). How much data is needed to train a medical image deep learning system to achieve necessary high accuracy? CoRR. arXiv:1511.06348"},{"key":"6619_CR14","first-page":"239","volume":"7","author":"C Cortes","year":"1994","unstructured":"Cortes, C., Jackel, L. D., & Chiang, W. (1994). Limits in learning machine accuracy imposed by data quality. Advances in Neural Information Processing Systems, 7, 239\u2013246.","journal-title":"Advances in Neural Information Processing Systems"},{"key":"6619_CR15","first-page":"327","volume":"6","author":"C Cortes","year":"1993","unstructured":"Cortes, C., Jackel, L. D., Solla, S. A., Vapnik, V., & Denker, J. (1993). Learning curves: Asymptotic values and rate of convergence. Advances in Neural Information Processing Systems, 6, 327\u2013334.","journal-title":"Advances in Neural Information Processing Systems"},{"key":"6619_CR16","doi-asserted-by":"publisher","first-page":"39","DOI":"10.1016\/j.eswa.2016.04.026","volume":"60","author":"FG da Costa","year":"2016","unstructured":"da Costa, F. G., Rios, R. A., & de Mello, R. F. (2016). Using dynamical systems tools to detect concept drift in data streams. Expert Systems with Applications, 60, 39\u201350.","journal-title":"Expert Systems with Applications"},{"key":"6619_CR17","unstructured":"Domhan, T., Springenberg, J. T., & Hutter, F. (2015). Speeding up automatic hyperparameter optimization of deep neural networks by extrapolation of learning curves. In Proceedings of the twenty-fourth international joint conference on artificial intelligence, IJCAI (vol. 2015, pp. 3460\u20133468)."},{"key":"6619_CR18","doi-asserted-by":"crossref","unstructured":"Domingos, P., & Hulten, G. (2000). Mining High-Speed Data Streams. In Proceedings of the sixth ACM SIGKDD international conference on Knowledge discovery and data mining (pp. 71\u201380).","DOI":"10.1145\/347090.347107"},{"key":"6619_CR19","unstructured":"Dong, X., & Yang, Y. (2020). Nas-bench-201: Extending the scope of reproducible neural architecture search. In 8th International conference on learning representations, ICLR 2020."},{"issue":"127","key":"6619_CR20","first-page":"964","volume":"597","author":"R Egele","year":"2024","unstructured":"Egele, R., Mohr, F., Viering, T., & Balaprakash, P. (2024). The unreasonable effectiveness of early discarding after one epoch in neural network hyperparameter optimization. Neurocomputing, 597(127), 964.","journal-title":"Neurocomputing"},{"issue":"1","key":"6619_CR21","doi-asserted-by":"publisher","first-page":"15","DOI":"10.1007\/s10994-017-5683-z","volume":"107","author":"K Eggensperger","year":"2018","unstructured":"Eggensperger, K., Lindauer, M., Hoos, H. H., Hutter, F., & Leyton-Brown, K. (2018). Efficient benchmarking of algorithm configurators via model-based surrogates. Machine Learning, 107(1), 15\u201341.","journal-title":"Machine Learning"},{"key":"6619_CR22","unstructured":"Eggensperger, K., M\u00fcller, P., Mallik, N., Feurer, M., Sass, R., Klein, A., Awad, N., Lindauer, M., & Hutter, F. (2021). HPOBench: A collection of reproducible multi-fidelity benchmark problems for HPO. In Proceedings of the neural information processing systems track on datasets and benchmarks."},{"key":"6619_CR23","doi-asserted-by":"publisher","first-page":"8","DOI":"10.1186\/1472-6947-12-8","volume":"12","author":"RL Figueroa","year":"2012","unstructured":"Figueroa, R. L., Zeng-Treitler, Q., Kandula, S., & Ngo, L. H. (2012). Predicting sample size required for classification performance. BMC Medical Informatics and Decision Making, 12, 8.","journal-title":"BMC Medical Informatics and Decision Making"},{"issue":"3","key":"6619_CR24","doi-asserted-by":"publisher","first-page":"747","DOI":"10.1162\/089976699300016647","volume":"11","author":"T Fine","year":"1999","unstructured":"Fine, T., & Mukherjee, S. (1999). Parameter convergence and learning curves for neural networks. Neural Computation, 11(3), 747\u2013769.","journal-title":"Neural Computation"},{"key":"6619_CR25","doi-asserted-by":"crossref","unstructured":"Forman, G., & Cohen, I. (2004). Learning from little: Comparison of classifiers given little training. In Knowledge discovery in databases: PKDD 2004, 8th European conference on principles and practice of knowledge discovery in databases (pp. 161\u2013172).","DOI":"10.1007\/978-3-540-30116-5_17"},{"key":"6619_CR26","unstructured":"Frey, L. J., & Fisher, D. H. (1999). Modeling decision tree performance with the power law. In Proceedings of the seventh international workshop on artificial intelligence and statistics, AISTATS 1999."},{"key":"6619_CR27","unstructured":"F\u00fcrnkranz, J., & Petrak, J. (2001). An evaluation of landmarking variants. In Working notes of the ECML\/PKDD 2000 workshop on integrating aspects of data mining, decision support and meta-learning (pp. 57\u201368)."},{"key":"6619_CR28","doi-asserted-by":"crossref","unstructured":"Gkioxari, G., Toshev, A., & Jaitly, N. (2016). Chained predictions using convolutional neural networks. In B. Leibe, J. Matas, N. Sebe, & M. Welling (Eds.), Computer vision\u2014ECCV 2016\u201414th European conference (pp. 728\u2013743).","DOI":"10.1007\/978-3-319-46493-0_44"},{"key":"6619_CR29","volume-title":"Deep learning","author":"I Goodfellow","year":"2016","unstructured":"Goodfellow, I., Bengio, Y., & Courville, A. (2016). Deep learning. MIT Press."},{"key":"6619_CR30","doi-asserted-by":"crossref","unstructured":"Gu, B., Hu, F., & Liu, H. (2001). Modelling classification performance for large data sets. In Advances in web-age information management, second international conference, WAIM (vol. 2001, pp. 317\u2013328).","DOI":"10.1007\/3-540-47714-4_29"},{"issue":"1","key":"6619_CR31","doi-asserted-by":"publisher","first-page":"65","DOI":"10.1053\/j.seminoncol.2009.12.002","volume":"37","author":"KR Hess","year":"2010","unstructured":"Hess, K. R., & Wei, C. (2010). Learning curves in classification with microarray data. Seminars in Oncology, 37(1), 65\u201368.","journal-title":"Seminars in Oncology"},{"key":"6619_CR32","unstructured":"Hollmann, N., M\u00fcller, S., & Eggensperger, K., & Hutter, F. (2023). Tabpfn: A transformer that solves small tabular classification problems in a second. In The eleventh international conference on learning representations, ICLR 2023."},{"issue":"1","key":"6619_CR33","doi-asserted-by":"publisher","first-page":"55","DOI":"10.1109\/TIT.1968.1054102","volume":"14","author":"GF Hughes","year":"1968","unstructured":"Hughes, G. F. (1968). On the mean accuracy of statistical pattern recognizers. IEEE Transactions on Information Theory, 14(1), 55\u201363.","journal-title":"IEEE Transactions on Information Theory"},{"issue":"3","key":"6619_CR34","doi-asserted-by":"publisher","first-page":"457","DOI":"10.1007\/s10994-021-05946-3","volume":"110","author":"E H\u00fcllermeier","year":"2021","unstructured":"H\u00fcllermeier, E., & Waegeman, W. (2021). Aleatoric and epistemic uncertainty in machine learning: An introduction to concepts and methods. Machine Learning, 110(3), 457\u2013506.","journal-title":"Machine Learning"},{"key":"6619_CR35","unstructured":"Jamieson, K. G., & Talwalkar, A. (2016). Non-stochastic best arm identification and hyperparameter optimization. In Proceedings of the 19th international conference on artificial intelligence and statistics, AISTATS 2016 (pp. 240\u2013248)."},{"key":"6619_CR36","unstructured":"John, G. H., & Langley, P. (1996). Static versus dynamic sampling for data mining. In Proceedings of the second international conference on knowledge discovery and data mining (KDD-96) (pp. 367\u2013370)."},{"key":"6619_CR37","doi-asserted-by":"crossref","unstructured":"Kielh\u00f6fer, L., Mohr, F., & van Rijn, J. N. (2024). Learning curve extrapolation methods across extrapolation settings. In Advances in intelligent data analysis XXII (pp. 145\u2013157).","DOI":"10.1007\/978-3-031-58553-1_12"},{"key":"6619_CR38","unstructured":"Klein, A., Falkner, S., & Bartels, S., Hennig, P., & Hutter, F. (2017a). Fast Bayesian optimization of machine learning hyperparameters on large datasets. In Proceedings of the 20th international conference on artificial intelligence and statistics, AISTATS 2017 (pp. 528\u2013536)."},{"key":"6619_CR39","unstructured":"Klein, A., Falkner, S., Springenberg, J. T., & Hutter, F. (2017b). Learning curve prediction with Bayesian neural networks. In 5th International conference on learning representations, ICLR\u201917."},{"key":"6619_CR40","unstructured":"Kolachina, P., Cancedda, N., Dymetman, M., & Venkatapathy, S. (2012). Prediction of learning curves in machine translation. In Proceedings of the 50th annual meeting of the association for computational linguistics, proceedings of the conference (pp. 22\u201330)."},{"key":"6619_CR41","unstructured":"Koshute, P., Zook, J., & McCulloh, I. (2021). Recommending training set sizes for classification. CoRR. arXiv:2102.09382"},{"key":"6619_CR42","doi-asserted-by":"crossref","unstructured":"Last, M. (2007). Predicting and optimizing classifier utility with the power law. In Workshops proceedings of the 7th IEEE international conference on data mining (ICDM 2007) (pp. 219\u2013224).","DOI":"10.1109\/ICDMW.2007.31"},{"key":"6619_CR43","doi-asserted-by":"crossref","unstructured":"Last, M. (2009). Improving data mining utility with projective sampling. In Proceedings of the 15th ACM SIGKDD international conference on knowledge discovery and data mining (pp. 487\u2013496).","DOI":"10.1145\/1557019.1557076"},{"key":"6619_CR44","doi-asserted-by":"crossref","unstructured":"Leite, R., & Brazdil, P. (2003). Improving progressive sampling via meta-learning. In Progress in artificial intelligence, 11th Protuguese conference on artificial intelligence, EPIA 2003 (pp. 313\u2013323).","DOI":"10.1007\/978-3-540-24580-3_37"},{"key":"6619_CR45","doi-asserted-by":"crossref","unstructured":"Leite, R., & Brazdil, P. (2004). Improving progressive sampling via meta-learning on learning curves. In Machine learning: ECML 2004, 15th European conference on machine learning (pp. 250\u2013261).","DOI":"10.1007\/978-3-540-30115-8_25"},{"key":"6619_CR46","doi-asserted-by":"crossref","unstructured":"Leite, R., & Brazdil, P. (2005). Predicting relative performance of classifiers from samples. In Machine learning, proceedings of the twenty-second international conference (ICML 2005) (pp. 497\u2013503).","DOI":"10.1145\/1102351.1102414"},{"key":"6619_CR47","doi-asserted-by":"crossref","unstructured":"Leite, R., & Brazdil, P. (2007). An iterative process for building learning curves and predicting relative performance of classifiers. In Progress in artificial intelligence, 13th Portuguese conference on artificial intelligence, EPIA 2007 (pp. 87\u201398).","DOI":"10.1007\/978-3-540-77002-2_8"},{"key":"6619_CR48","unstructured":"Leite, R., & Brazdil, P. (2008). Selecting classifiers using metalearning with sampling landmarks and data characterization. In Proceedings of the 2nd planning to learn workshop (PlanLearn) at ICML\/COLT\/UAI (pp. 35\u201341)."},{"key":"6619_CR49","doi-asserted-by":"crossref","unstructured":"Leite, R., & Brazdil, P. (2010). Active testing strategy to predict the best classification algorithm via sampling and metalearning. In ECAI 2010\u201419th European conference on artificial intelligence (pp. 309\u2013314).","DOI":"10.3233\/978-1-60750-606-5-309"},{"issue":"185","key":"6619_CR50","first-page":"1","volume":"18","author":"L Li","year":"2017","unstructured":"Li, L., Jamieson, K., DeSalvo, G., Rostamizadeh, A., & Talwalkar, A. (2017). Hyperband: A novel bandit-based approach to hyperparameter optimization. Journal of Machine Learning Research, 18(185), 1\u201352.","journal-title":"Journal of Machine Learning Research"},{"issue":"2","key":"6619_CR51","doi-asserted-by":"publisher","first-page":"80","DOI":"10.1049\/ccs.2019.0024","volume":"2","author":"D Long","year":"2020","unstructured":"Long, D., Zhang, S., & Zhang, Y. (2020). Performance prediction based on neural architecture features. Cognitive Computation and Systems, 2(2), 80\u201383.","journal-title":"Cognitive Computation and Systems"},{"key":"6619_CR52","doi-asserted-by":"crossref","unstructured":"Loog, M., & Duin, R. P. (2012). The dipping phenomenon. In Joint IAPR international workshops on statistical techniques in pattern recognition (SPR) and structural and syntactic pattern recognition (SSPR) (pp. 310\u2013317).","DOI":"10.1007\/978-3-642-34166-3_34"},{"key":"6619_CR53","first-page":"7476","volume":"32","author":"M Loog","year":"2019","unstructured":"Loog, M., Viering, T. J., & Mey, A. (2019). Minimizers of the empirical risk and risk monotonicity. Advances in Neural Information Processing Systems, 32, 7476\u20137485.","journal-title":"Advances in Neural Information Processing Systems"},{"key":"6619_CR54","doi-asserted-by":"crossref","unstructured":"Mao, Z., Jung, T., Lin, C., & Huang, Y. (2016). Predicting EEG sample size required for classification calibration. In Foundations of augmented cognition: Neuroergonomics and operational neuroscience\u201410th international conference, AC 2016 (pp. 57\u201368).","DOI":"10.1007\/978-3-319-39955-3_6"},{"key":"6619_CR55","first-page":"397","volume":"2","author":"C Meek","year":"2002","unstructured":"Meek, C., Thiesson, B., & Heckerman, D. (2002). The learning-curve sampling method applied to model-based clustering. Journal of Machine Learning Research, 2, 397\u2013418.","journal-title":"Journal of Machine Learning Research"},{"key":"6619_CR56","first-page":"10732","volume":"34","author":"Z Mhammedi","year":"2021","unstructured":"Mhammedi, Z., & Husain, H. (2021). Risk-monotonicity in statistical learning. Advances in Neural Information Processing Systems, 34, 10732\u201310744.","journal-title":"Advances in Neural Information Processing Systems"},{"key":"6619_CR57","unstructured":"Mohr, F., & van Rijn, J. N.: (2021). Towards model selection using learning curve cross-validation. In 8th ICML workshop on automated machine learning (AutoML)."},{"issue":"8","key":"6619_CR58","doi-asserted-by":"publisher","first-page":"9669","DOI":"10.1109\/TPAMI.2023.3251957","volume":"45","author":"F Mohr","year":"2023","unstructured":"Mohr, F., & van Rijn, J. N. (2023). Fast and informative model selection using learning curve cross-validation. IEEE Transactions on Pattern Analysis and Machine Intelligence, 45(8), 9669\u20139680.","journal-title":"IEEE Transactions on Pattern Analysis and Machine Intelligence"},{"key":"6619_CR59","doi-asserted-by":"crossref","unstructured":"Mohr, F., Viering, T. J., Loog, M., van Rijn, J. N. (2022) LCDB 1.0: An extensive learning curves database for classification tasks. In Machine learning and knowledge discovery in databases\u2014European conference, ECML PKDD (pp. 3\u201319).","DOI":"10.1007\/978-3-031-26419-1_1"},{"key":"6619_CR60","doi-asserted-by":"crossref","unstructured":"M\u00f8rch, N. J. S., Hansen, L. K., Strother, S. C., Svarer, C., Rottenberg, D. A., Lautrup, B., Savoy, R., & Paulson, O. B. (1997). Nonlinear versus linear models in functional neuroimaging: Learning curves and generalization crossover. In information processing in medical imaging, 15th international conference, IPMI\u201997 (pp. 259\u2013270).","DOI":"10.1007\/3-540-63046-5_20"},{"issue":"2","key":"6619_CR61","doi-asserted-by":"publisher","first-page":"119","DOI":"10.1089\/106652703321825928","volume":"10","author":"S Mukherjee","year":"2003","unstructured":"Mukherjee, S., Tamayo, P., Rogers, S., Rifkin, R., Engle, A., Campbell, C., Golub, T. R., & Mesirov, J. P. (2003). Estimating dataset size requirements for classifying DNA microarray data. Journal of Computational Biology, 10(2), 119\u2013142.","journal-title":"Journal of Computational Biology"},{"key":"6619_CR62","first-page":"607","volume":"5","author":"N Murata","year":"1992","unstructured":"Murata, N., Yoshizawa, S., & Amari, S. (1992). Learning curves, model selection and complexity of neural networks. Advances in Neural Information Processing Systems, 5, 607\u2013614.","journal-title":"Advances in Neural Information Processing Systems"},{"key":"6619_CR63","doi-asserted-by":"crossref","unstructured":"Nakkiran, P., Kaplun, G., Bansal, Y., Yang, T., Barak, B., & Sutskever, I. (2020). Deep double descent: Where bigger models and more data hurt. In 8th International conference on learning representations, ICLR\u201920.","DOI":"10.1088\/1742-5468\/ac3a74"},{"key":"6619_CR64","unstructured":"Nakkiran, P., Venkat, P., Kakade, S. M., & Ma, T. (2021). Optimal regularization can mitigate double descent. In 9th International conference on learning representations, ICLR 2021."},{"key":"6619_CR65","first-page":"841","volume":"14","author":"AY Ng","year":"2001","unstructured":"Ng, A. Y., & Jordan, M. I. (2001). On discriminative vs. generative classifiers: A comparison of logistic regression and naive bayes. Advances in Neural Information Processing Systems, 14, 841\u2013848.","journal-title":"Advances in Neural Information Processing Systems"},{"key":"6619_CR66","doi-asserted-by":"crossref","unstructured":"Ng, W., & Dash, M. (2006). An evaluation of progressive sampling for imbalanced data sets. In Workshops proceedings of the 6th IEEE international conference on data mining (ICDM 2006) (pp. 657\u2013661).","DOI":"10.1109\/ICDMW.2006.28"},{"key":"6619_CR67","doi-asserted-by":"crossref","unstructured":"Oyedare, T., & Park, J. J. (2019). Estimating the required training dataset size for transmitter classification using deep learning. In 2019 IEEE international symposium on dynamic spectrum access networks, DySPAN 2019 (pp. 1\u201310).","DOI":"10.1109\/DySPAN.2019.8935823"},{"key":"6619_CR68","first-page":"211","volume":"4","author":"C Perlich","year":"2003","unstructured":"Perlich, C., Provost, F. J., & Simonoff, J. S. (2003). Tree induction vs. logistic regression: A learning-curve analysis. Journal of Machine Learning Research, 4, 211\u2013255.","journal-title":"Journal of Machine Learning Research"},{"key":"6619_CR69","unstructured":"Petrak, J. (2000). Fast subsampling performance estimates for classification algorithm selection. In Proceedings of the ECML-00 workshop on meta-learning: Building automatic advice strategies for model selection and method combination (pp. 3\u201314)."},{"key":"6619_CR70","doi-asserted-by":"crossref","unstructured":"Pfisterer, F., Schneider, L., Moosbauer, J., Binder, M., & Bischl, B. (2022). YAHPO gym\u2014An efficient multi-objective multi-fidelity benchmark for hyperparameter optimization. In International conference on automated machine learning, AutoML (pp. 3\/1\u201339).","DOI":"10.1145\/3610536"},{"key":"6619_CR71","doi-asserted-by":"crossref","unstructured":"Provost, F. J., Jensen, D. D., & Oates, T. (1999). Efficient progressive sampling. In Proceedings of the fifth ACM SIGKDD international conference on knowledge discovery and data mining (pp. 23\u201332).","DOI":"10.1145\/312129.312188"},{"key":"6619_CR72","doi-asserted-by":"crossref","unstructured":"Richter, A. N., & Khoshgoftaar, T. M. (2019). Approximating learning curves for imbalanced big data with limited labels. In 31st IEEE international conference on tools with artificial intelligence, ICTAI 2019 (pp. 237\u2013242).","DOI":"10.1109\/ICTAI.2019.00041"},{"key":"6619_CR73","unstructured":"Ruhkopf, T., Mohan, A., Deng, D., Tornede, A., Hutter, F., & Lindauer, M. (2023). Masif: Meta-learned algorithm selection using implicit fidelity information. Transactions on Machine Learning Research."},{"key":"6619_CR74","doi-asserted-by":"crossref","unstructured":"Sabharwal, A., Samulowitz, H., & Tesauro, G. (2016). Selecting near-optimal learners via incremental data allocation. In Proceedings of the AAAI conference on artificial intelligence.","DOI":"10.1609\/aaai.v30i1.10316"},{"key":"6619_CR75","doi-asserted-by":"crossref","unstructured":"Sarkar, A., Guo, J., Siegmund, N., Apel, S., & Czarnecki, K. (2015). Cost-efficient sampling for performance prediction of configurable systems (T). In 30th IEEE\/ACM international conference on automated software engineering, ASE 2015 (pp. 342\u2013352).","DOI":"10.1109\/ASE.2015.45"},{"key":"6619_CR76","unstructured":"Settles, B. (2009). Active learning literature survey. Technical Report. University of Wisconsin."},{"issue":"8","key":"6619_CR77","doi-asserted-by":"publisher","first-page":"6056","DOI":"10.1103\/PhysRevA.45.6056","volume":"45","author":"HS Seung","year":"1992","unstructured":"Seung, H. S., Sompolinsky, H., & Tishby, N. (1992). Statistical mechanics of learning from examples. Physical Review A, 45(8), 6056.","journal-title":"Physical Review A"},{"key":"6619_CR78","unstructured":"Siems, J., Zimmer, L., Zela, A., Keuper, M. & Hutter, F. (2020). Nas-bench-301 and the case for surrogate benchmarks for neural architecture search. CoRR. arXiv:2008.09777"},{"key":"6619_CR79","unstructured":"Singh, S. (2005). Modeling performance of different classification methods: Deviation from the power law. Project Report, Department of Computer Science, Vanderbilt University, USA."},{"key":"6619_CR80","doi-asserted-by":"crossref","unstructured":"Strang, B., van\u00a0der Putten, P., van Rijn, J. N., & Hutter, F. (2018). Don\u2019t rule out simple models prematurely: A large scale benchmark comparing linear and non-linear classifiers in openml. In Advances in intelligent data analysis XVII (pp. 303\u2013315).","DOI":"10.1007\/978-3-030-01768-2_25"},{"key":"6619_CR81","unstructured":"Swersky, K., Snoek, J., & Adams, R. P. (2014). Freeze-thaw bayesian optimization. CoRR. arXiv:1406.3896"},{"key":"6619_CR82","unstructured":"Tomanek, K. (2010). Resource-aware annotation through active learning. PhD thesis, Dortmund University of Technology."},{"issue":"4","key":"6619_CR83","doi-asserted-by":"publisher","first-page":"315","DOI":"10.1209\/0295-5075\/9\/4\/003","volume":"9","author":"F Vallet","year":"1989","unstructured":"Vallet, F., Cailton, J. G., & Refregier, P. (1989). Linear and nonlinear extension of the pseudo-inverse solution for learning boolean functions. EPL (Europhysics Letters), 9(4), 315.","journal-title":"EPL (Europhysics Letters)"},{"key":"6619_CR84","unstructured":"Van\u00a0den Bosch, A. (2004). Wrapped progressive sampling search for optimizing learning algorithm parameters. In Proceedings of the 16th Belgian-Dutch conference on artificial intelligence (pp. 219\u2013226)."},{"key":"6619_CR85","doi-asserted-by":"crossref","unstructured":"van Rijn, J. N., Abdulrahman, S. M., Brazdil, P., & Vanschoren, J. (2015). Fast algorithm selection using learning curves. In Advances in intelligent data analysis XIV (pp. 298\u2013309).","DOI":"10.1007\/978-3-319-24465-5_26"},{"issue":"6","key":"6619_CR86","doi-asserted-by":"publisher","first-page":"7799","DOI":"10.1109\/TPAMI.2022.3220744","volume":"45","author":"TJ Viering","year":"2023","unstructured":"Viering, T. J., & Loog, M. (2023). The shape of learning curves: A review. IEEE Transactions on Pattern Analysis and Machine Intelligence, 45(6), 7799\u20137819.","journal-title":"IEEE Transactions on Pattern Analysis and Machine Intelligence"},{"key":"6619_CR87","doi-asserted-by":"crossref","unstructured":"Viering, T. J., Mey, A., & Loog, M. (2020). Making learners (more) monotone. In Advances in intelligent data analysis XVIII (pp. 535\u2013547).","DOI":"10.1007\/978-3-030-44584-3_42"},{"issue":"4","key":"6619_CR88","doi-asserted-by":"publisher","first-page":"390","DOI":"10.1109\/TAC.1965.1098193","volume":"10","author":"M Waltz","year":"1965","unstructured":"Waltz, M., & Fu, K. (1965). A heuristic approach to reinforcement learning control systems. IEEE Transactions on Automatic Control, 10(4), 390\u2013398.","journal-title":"IEEE Transactions on Automatic Control"},{"issue":"9","key":"6619_CR89","doi-asserted-by":"crossref","first-page":"4555","DOI":"10.1109\/TPAMI.2021.3072422","volume":"44","author":"X Wang","year":"2022","unstructured":"Wang, X., Chen, Y., & Zhu, W. (2022). A survey on curriculum learning. IEEE Transactions on Pattern Analysis and Machine Intelligence, 44(9), 4555\u20134576.","journal-title":"IEEE Transactions on Pattern Analysis and Machine Intelligence"},{"key":"6619_CR90","doi-asserted-by":"publisher","first-page":"315","DOI":"10.1613\/jair.1199","volume":"19","author":"GM Weiss","year":"2003","unstructured":"Weiss, G. M., & Provost, F. J. (2003). Learning when training data are costly: The effect of class distribution on tree induction. Journal of Artificial Intelligence Research, 19, 315\u2013354.","journal-title":"Journal of Artificial Intelligence Research"},{"issue":"2","key":"6619_CR91","doi-asserted-by":"publisher","first-page":"31","DOI":"10.1145\/1233321.1233325","volume":"8","author":"GM Weiss","year":"2006","unstructured":"Weiss, G. M., & Tian, Y. (2006). Maximizing classifier utility when training data is costly. SIGKDD Explorations, 8(2), 31\u201338.","journal-title":"SIGKDD Explorations"},{"issue":"2","key":"6619_CR92","doi-asserted-by":"publisher","first-page":"253","DOI":"10.1007\/s10618-007-0082-x","volume":"17","author":"GM Weiss","year":"2008","unstructured":"Weiss, G. M., & Tian, Y. (2008). Maximizing classifier utility when there are data acquisition and modeling costs. Data Mining and Knowledge Discovery, 17(2), 253\u2013282.","journal-title":"Data Mining and Knowledge Discovery"},{"key":"6619_CR93","unstructured":"White, C., Safari, M., Sukthanker, R., Ru, B., Elsken, T., Zela, A., Dey, D., & Hutter, F. (2023). Neural architecture search: Insights from 1000 papers. CoRR. arXiv:2301.08727"},{"key":"6619_CR94","unstructured":"Wistuba, M., & Pedapati, T. (2019). Inductive transfer for neural architecture optimization. CoRR. arXiv:1903.03536"},{"issue":"1","key":"6619_CR95","doi-asserted-by":"publisher","first-page":"2","DOI":"10.1007\/s13755-017-0023-z","volume":"5","author":"X Zeng","year":"2017","unstructured":"Zeng, X., & Luo, G. (2017). Progressive sampling-based Bayesian optimization for efficient and automatic machine learning model selection. Health Information Science and Systems, 5(1), 2.","journal-title":"Health Information Science and Systems"}],"container-title":["Machine Learning"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10994-024-06619-7.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s10994-024-06619-7","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10994-024-06619-7.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,12,23]],"date-time":"2025-12-23T01:03:03Z","timestamp":1766451783000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s10994-024-06619-7"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,12]]},"references-count":95,"journal-issue":{"issue":"11-12","published-print":{"date-parts":[[2024,12]]}},"alternative-id":["6619"],"URL":"https:\/\/doi.org\/10.1007\/s10994-024-06619-7","relation":{},"ISSN":["0885-6125","1573-0565"],"issn-type":[{"value":"0885-6125","type":"print"},{"value":"1573-0565","type":"electronic"}],"subject":[],"published":{"date-parts":[[2024,12]]},"assertion":[{"value":"4 December 2023","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"29 July 2024","order":2,"name":"revised","label":"Revised","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"30 August 2024","order":3,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"23 December 2024","order":4,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Declarations"}},{"value":"The author declare that they have no conflict of interest.","order":2,"name":"Ethics","group":{"name":"EthicsHeading","label":"Conflict of interest"}},{"value":"Not applicable.","order":3,"name":"Ethics","group":{"name":"EthicsHeading","label":"Ethics approval"}},{"value":"Not applicable.","order":4,"name":"Ethics","group":{"name":"EthicsHeading","label":"Consent to participate"}},{"value":"Not applicable.","order":5,"name":"Ethics","group":{"name":"EthicsHeading","label":"Consent for publication"}},{"value":"This content has been made available to all.","name":"free","label":"Free to read"}]}}