{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,2,21]],"date-time":"2025-02-21T03:40:16Z","timestamp":1740109216600,"version":"3.37.3"},"reference-count":51,"publisher":"Springer Science and Business Media LLC","issue":"3","license":[{"start":{"date-parts":[[2024,3,15]],"date-time":"2024-03-15T00:00:00Z","timestamp":1710460800000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2024,3,15]],"date-time":"2024-03-15T00:00:00Z","timestamp":1710460800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"funder":[{"DOI":"10.13039\/501100012165","name":"National Key R &D Program of China","doi-asserted-by":"publisher","award":["2022ZD0116315"],"award-info":[{"award-number":["2022ZD0116315"]}],"id":[{"id":"10.13039\/501100012165","id-type":"DOI","asserted-by":"publisher"}]},{"name":"Key Research and Development Program of Hubei Province","award":["2023BAB077"],"award-info":[{"award-number":["2023BAB077"]}]},{"DOI":"10.13039\/501100001711","name":"Schweizerischer Nationalfonds zur F\u00f6rderung der Wissenschaftlichen Forschung","doi-asserted-by":"publisher","award":["200021_184628"],"award-info":[{"award-number":["200021_184628"]}],"id":[{"id":"10.13039\/501100001711","id-type":"DOI","asserted-by":"publisher"}]},{"name":"Innosuisse\/SNF BRIDGE Discovery","award":["40B2-0_187132"],"award-info":[{"award-number":["40B2-0_187132"]}]},{"DOI":"10.13039\/100010661","name":"Horizon 2020 Framework Programme","doi-asserted-by":"publisher","award":["DAPHNE","957407"],"award-info":[{"award-number":["DAPHNE","957407"]}],"id":[{"id":"10.13039\/100010661","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["The VLDB Journal"],"published-print":{"date-parts":[[2024,5]]},"DOI":"10.1007\/s00778-024-00842-3","type":"journal-article","created":{"date-parts":[[2024,3,15]],"date-time":"2024-03-15T05:02:56Z","timestamp":1710478976000},"page":"833-857","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":0,"title":["How good are machine learning clouds? Benchmarking two snapshots over 5\u00a0years"],"prefix":"10.1007","volume":"33","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-0051-0046","authenticated-orcid":false,"given":"Jiawei","family":"Jiang","sequence":"first","affiliation":[]},{"given":"Yi","family":"Wei","sequence":"additional","affiliation":[]},{"given":"Yu","family":"Liu","sequence":"additional","affiliation":[]},{"given":"Wentao","family":"Wu","sequence":"additional","affiliation":[]},{"given":"Chuang","family":"Hu","sequence":"additional","affiliation":[]},{"given":"Zhigao","family":"Zheng","sequence":"additional","affiliation":[]},{"given":"Ziyi","family":"Zhang","sequence":"additional","affiliation":[]},{"given":"Yingxia","family":"Shao","sequence":"additional","affiliation":[]},{"given":"Ce","family":"Zhang","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2024,3,15]]},"reference":[{"key":"842_CR1","unstructured":"https:\/\/archive.ics.uci.edu\/ml\/datasets\/"},{"key":"842_CR2","unstructured":"Aguilar Melgar, L., et\u00a0al.: Ease.ml: a lifecycle management system for machine learning. In: 11th Annual Conference on Innovative Data Systems Research (CIDR 2021) (virtual). CIDR (2021)"},{"key":"842_CR3","unstructured":"Amazon: Amazon cloud. http:\/\/docs.aws.amazon.com\/machine-learning\/latest\/dg\/learning-algorithm.html (2021)"},{"key":"842_CR4","unstructured":"Amazon: Amazon sagemaker autopilot. https:\/\/aws.amazon.com\/sagemaker\/autopilot\/ (2021)"},{"key":"842_CR5","unstructured":"Auto, I.: Ibm autoai. https:\/\/www.ibm.com\/cloud\/watson-studio\/autoai (2021)"},{"key":"842_CR6","unstructured":"Azure, M.: Azure automated machine learning. https:\/\/aws.amazon.com\/sagemaker\/autopilot\/ (2021)"},{"key":"842_CR7","unstructured":"Balaji, A., Allen, A.: Benchmarking automatic machine learning frameworks. arXiv:1808.06492 (2018)"},{"key":"842_CR8","doi-asserted-by":"publisher","first-page":"105","DOI":"10.1023\/A:1007515423169","volume":"36","author":"E Bauer","year":"1998","unstructured":"Bauer, E., Kohavi, R.: An empirical comparison of voting classification algorithms: bagging, boosting, and variants. Mach Learn 36, 105\u2013139 (1998)","journal-title":"Mach Learn"},{"key":"842_CR9","doi-asserted-by":"crossref","unstructured":"Bergstra, J., et\u00a0al.: Hyperopt: a python library for optimizing the hyperparameters of machine learning algorithms. In: Proceedings of the 12th Python in science conference, vol.\u00a013, p.\u00a020. Citeseer (2013)","DOI":"10.25080\/Majora-8b375195-003"},{"key":"842_CR10","doi-asserted-by":"crossref","unstructured":"Caruana, R., et\u00a0al.: An empirical comparison of supervised learning algorithms. In: ICML (2006)","DOI":"10.1145\/1143844.1143865"},{"key":"842_CR11","doi-asserted-by":"crossref","unstructured":"Cooper, B.F., et\u00a0al.: Benchmarking cloud serving systems with YCSB. In: SoCC (2010)","DOI":"10.1145\/1807128.1807152"},{"key":"842_CR12","doi-asserted-by":"publisher","first-page":"273","DOI":"10.1007\/BF00994018","volume":"20","author":"C Cortes","year":"1995","unstructured":"Cortes, C., Vapnik, V.: Support-vector networks. Mach Learn 20, 273\u2013297 (1995)","journal-title":"Mach Learn"},{"key":"842_CR13","unstructured":"DeWitt, D.J.: The Wisconsin benchmark: past, present, and future. In: The Benchmark Handbook for Database and Transaction Systems (1993)"},{"key":"842_CR14","doi-asserted-by":"crossref","unstructured":"Domingos, P.: A few useful things to know about machine learning. In: CACM (2012)","DOI":"10.1145\/2347736.2347755"},{"key":"842_CR15","unstructured":"Erickson, N., Mueller, J., Shirkov, A., Zhang, H., Larroy, P., Li, M., Smola, A.: Autogluon-tabular: robust and accurate automl for structured data. arXiv:2003.06505 (2020)"},{"key":"842_CR16","unstructured":"Fern\u00e1ndez-Delgado, M., et\u00a0al.: Do we need hundreds of classifiers to solve real world classification problems. In: JMLR (2014)"},{"key":"842_CR17","doi-asserted-by":"crossref","unstructured":"Feurer, M., et\u00a0al.: Initializing Bayesian hyperparameter optimization via meta-learning. In: Proceedings of the AAAI Conference on Artificial Intelligence, vol.\u00a029 (2015)","DOI":"10.1609\/aaai.v29i1.9354"},{"key":"842_CR18","doi-asserted-by":"crossref","unstructured":"Feurer, M., et\u00a0al.: Auto-sklearn: efficient and robust automated machine learning. In: Automated Machine Learning, pp. 113\u2013134 (2019)","DOI":"10.1007\/978-3-030-05318-5_6"},{"key":"842_CR19","doi-asserted-by":"crossref","unstructured":"Freund, Y., Schapire, R.E.: A decision-theoretic generalization of on-line learning and an application to boosting. In: JCSS (1997)","DOI":"10.1006\/jcss.1997.1504"},{"key":"842_CR20","first-page":"3348","volume":"31","author":"N Fusi","year":"2018","unstructured":"Fusi, N., et al.: Probabilistic matrix factorization for automated machine learning. Adv. Neural Inf. Process. Syst. 31, 3348\u20133357 (2018)","journal-title":"Adv. Neural Inf. Process. Syst."},{"issue":"1","key":"842_CR21","doi-asserted-by":"publisher","first-page":"3","DOI":"10.1016\/j.neucom.2011.07.005","volume":"75","author":"TA Gomes","year":"2012","unstructured":"Gomes, T.A., et al.: Combining meta-learning and search techniques to select parameters for support vector machines. Neurocomputing 75(1), 3\u201313 (2012)","journal-title":"Neurocomputing"},{"key":"842_CR22","unstructured":"Google: Google cloud automl. https:\/\/cloud.google.com\/automl (2021)"},{"key":"842_CR23","volume-title":"Neural Networks: A Comprehensive Foundation","author":"S Haykin","year":"1998","unstructured":"Haykin, S.: Neural Networks: A Comprehensive Foundation, 2nd edn. Prentice Hall PTR, Hoboken (1998)","edition":"2"},{"key":"842_CR24","doi-asserted-by":"publisher","DOI":"10.1016\/j.knosys.2020.106622","volume":"212","author":"X He","year":"2021","unstructured":"He, X., et al.: Automl: a survey of the state-of-the-art. Knowl.-Based Syst. 212, 106622 (2021)","journal-title":"Knowl.-Based Syst."},{"key":"842_CR25","unstructured":"Herbrich, R., et\u00a0al.: Bayes point machines. In: JMLR (2001)"},{"key":"842_CR26","unstructured":"Ho, T.K.: Random decision forests. In: ICDAR (1995)"},{"key":"842_CR27","doi-asserted-by":"crossref","unstructured":"Hutter, F., et\u00a0al.: Sequential model-based optimization for general algorithm configuration. In: International Conference on Learning and Intelligent Optimization, pp. 507\u2013523. Springer (2011)","DOI":"10.1007\/978-3-642-25566-3_40"},{"key":"842_CR28","doi-asserted-by":"crossref","unstructured":"Jiang, J., Gan, S., Liu, Y., Wang, F., Alonso, G., Klimovic, A., Singla, A., Wu, W., Zhang, C.: Towards demystifying serverless machine learning training. In: Proceedings of the 2021 International Conference on Management of Data, pp. 857\u2013871 (2021)","DOI":"10.1145\/3448016.3459240"},{"key":"842_CR29","doi-asserted-by":"crossref","unstructured":"Kotthoff, L., et\u00a0al.: Auto-weka: automatic model selection and hyperparameter optimization in weka. In: Automated Machine Learning, pp. 81\u201395. Springer, Cham (2019)","DOI":"10.1007\/978-3-030-05318-5_4"},{"key":"842_CR30","unstructured":"LeDell, E., Poirier, S.: H2o automl: scalable automatic machine learning. In: Proceedings of the AutoML Workshop at ICML (2020)"},{"issue":"1","key":"842_CR31","first-page":"6765","volume":"18","author":"L Li","year":"2017","unstructured":"Li, L., Jamieson, K., DeSalvo, G., Rostamizadeh, A., Talwalkar, A.: Hyperband: a novel bandit-based approach to hyperparameter optimization. J. Mach. Learn. Res. 18(1), 6765\u20136816 (2017)","journal-title":"J. Mach. Learn. Res."},{"key":"842_CR32","doi-asserted-by":"crossref","unstructured":"Li, P., et\u00a0al.: Cleanml: a study for evaluating the impact of data cleaning on ml classification tasks. In: 36th IEEE International Conference on Data Engineering (ICDE 2020) (virtual) (2021)","DOI":"10.1109\/ICDE51399.2021.00009"},{"issue":"2","key":"842_CR33","doi-asserted-by":"publisher","first-page":"389","DOI":"10.1007\/s00778-022-00752-2","volume":"32","author":"Y Li","year":"2023","unstructured":"Li, Y., Shen, Y., Zhang, W., Zhang, C., Cui, B.: Volcanoml: speeding up end-to-end automl via scalable search space decomposition. VLDB J. 32(2), 389\u2013413 (2023)","journal-title":"VLDB J."},{"issue":"10","key":"842_CR34","doi-asserted-by":"publisher","first-page":"1220","DOI":"10.14778\/3231751.3231770","volume":"11","author":"Y Liu","year":"2018","unstructured":"Liu, Y., et al.: MLbench: benchmarking machine learning services against human experts. Proc. VLDB Endow. 11(10), 1220\u20131232 (2018)","journal-title":"Proc. VLDB Endow."},{"key":"842_CR35","doi-asserted-by":"publisher","first-page":"347","DOI":"10.1007\/s11704-012-2118-7","volume":"6","author":"C Luo","year":"2012","unstructured":"Luo, C., et al.: Cloudrank-d: benchmarking and ranking cloud computing systems for data processing applications. Front. Comput. Sci. 6, 347\u2013362 (2012)","journal-title":"Front. Comput. Sci."},{"key":"842_CR36","doi-asserted-by":"publisher","first-page":"291","DOI":"10.1016\/j.artint.2016.12.001","volume":"244","author":"M M\u0131s\u0131r","year":"2017","unstructured":"M\u0131s\u0131r, M., et al.: Alors: an algorithm recommender system. Artif. Intell. 244, 291\u2013314 (2017)","journal-title":"Artif. Intell."},{"key":"842_CR37","unstructured":"Olson, R.S., Moore, J.H.: Tpot: a tree-based pipeline optimization tool for automating machine learning. In: Workshop on Automatic Machine Learning, pp. 66\u201374. PMLR (2016)"},{"key":"842_CR38","doi-asserted-by":"crossref","unstructured":"Olson, R.S., et\u00a0al.: Evaluation of a tree-based pipeline optimization tool for automating data science. In: Proceedings of the Genetic and Evolutionary Computation Conference 2016, pp. 485\u2013492 (2016)","DOI":"10.1145\/2908812.2908918"},{"key":"842_CR39","unstructured":"Parry, P., et\u00a0al.: auto_ml. https:\/\/github.com\/ClimbsRocks\/auto_ml (2007)"},{"key":"842_CR40","unstructured":"Perrone, V., Shen, H., Seeger, M.W., Archambeau, C., Jenatton, R.: Learning search spaces for Bayesian optimization: another view of hyperparameter transfer learning. Adv. Neural Inf. Process. Syst. 32 (2019)"},{"key":"842_CR41","doi-asserted-by":"crossref","unstructured":"Quinlan, J.R.: Induction of decision trees. Mach. Learn. (1986)","DOI":"10.1007\/BF00116251"},{"issue":"3","key":"842_CR42","doi-asserted-by":"publisher","first-page":"357","DOI":"10.1007\/s10994-012-5286-7","volume":"87","author":"M Reif","year":"2012","unstructured":"Reif, M., et al.: Meta-learning for evolutionary parameter optimization of classifiers. Mach. Learn. 87(3), 357\u2013380 (2012)","journal-title":"Mach. Learn."},{"key":"842_CR43","unstructured":"Shotton, J., et\u00a0al.: Decision jungles: compact and rich models for classification. In: NIPS (2013)"},{"key":"842_CR44","unstructured":"Sun-Hosoya, L., et\u00a0al.: Activmetal: algorithm recommendation with active meta learning. In: IAL 2018 workshop, ECML PKDD (2018)"},{"key":"842_CR45","doi-asserted-by":"crossref","unstructured":"Thornton, C., et\u00a0al.: Auto-weka: combined selection and hyperparameter optimization of classification algorithms. In: Proceedings of the 19th ACM SIGKDD International Conference on Knowledge Discovery and Data Mining, pp. 847\u2013855 (2013)","DOI":"10.1145\/2487575.2487629"},{"key":"842_CR46","unstructured":"Wong, C., et\u00a0al.: Transfer learning with neural automl. In: Proceedings of the 32nd International Conference on Neural Information Processing Systems, pp. 8366\u20138375 (2018)"},{"key":"842_CR47","doi-asserted-by":"crossref","unstructured":"Wu, Z., Ramsundar, B., Feinberg, E.N., Gomes, J., Geniesse, C., Pappu, A.S., Leswing, K., Pande, V.: Moleculenet: a benchmark for molecular machine learning. Chem. Sci. 9(2), 513\u2013530 (2018)","DOI":"10.1039\/C7SC02664A"},{"key":"842_CR48","doi-asserted-by":"crossref","unstructured":"Yakovlev, A., et al.: Oracle automl: a fast and predictive automl pipeline. Proc. VLDB Endow. 13(12), 3166\u20133180 (2020)","DOI":"10.14778\/3415478.3415542"},{"key":"842_CR49","unstructured":"Yogatama, D., Mann, G.: Efficient transfer learning method for automatic hyperparameter tuning. In: Artificial Intelligence and Statistics, pp. 1077\u20131085. PMLR (2014)"},{"key":"842_CR50","doi-asserted-by":"crossref","unstructured":"Zhang, C., et\u00a0al.: An overreaction to the broken machine learning abstraction: the ease.ml vision. In: HILDA (2017)","DOI":"10.1145\/3077257.3077265"},{"key":"842_CR51","doi-asserted-by":"publisher","first-page":"409","DOI":"10.1613\/jair.1.11854","volume":"70","author":"MA Z\u00f6ller","year":"2021","unstructured":"Z\u00f6ller, M.A., Huber, M.F.: Benchmark and survey of automated machine learning frameworks. J. Artif. Intell. Res. 70, 409\u2013472 (2021)","journal-title":"J. Artif. Intell. Res."}],"container-title":["The VLDB Journal"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s00778-024-00842-3.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s00778-024-00842-3\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s00778-024-00842-3.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,4,23]],"date-time":"2024-04-23T07:07:59Z","timestamp":1713856079000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s00778-024-00842-3"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,3,15]]},"references-count":51,"journal-issue":{"issue":"3","published-print":{"date-parts":[[2024,5]]}},"alternative-id":["842"],"URL":"https:\/\/doi.org\/10.1007\/s00778-024-00842-3","relation":{},"ISSN":["1066-8888","0949-877X"],"issn-type":[{"type":"print","value":"1066-8888"},{"type":"electronic","value":"0949-877X"}],"subject":[],"published":{"date-parts":[[2024,3,15]]},"assertion":[{"value":"14 January 2023","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"1 October 2023","order":2,"name":"revised","label":"Revised","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"19 December 2023","order":3,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"15 March 2024","order":4,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}}]}}