{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,18]],"date-time":"2026-07-18T02:39:45Z","timestamp":1784342385768,"version":"3.55.0"},"reference-count":129,"publisher":"Springer Science and Business Media LLC","issue":"4","license":[{"start":{"date-parts":[[2024,6,15]],"date-time":"2024-06-15T00:00:00Z","timestamp":1718409600000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2024,6,15]],"date-time":"2024-06-15T00:00:00Z","timestamp":1718409600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"funder":[{"name":"NSER"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Empir Software Eng"],"published-print":{"date-parts":[[2024,7]]},"DOI":"10.1007\/s10664-024-10474-4","type":"journal-article","created":{"date-parts":[[2024,6,15]],"date-time":"2024-06-15T10:01:41Z","timestamp":1718445701000},"update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":15,"title":["An empirical study of challenges in machine learning asset management"],"prefix":"10.1007","volume":"29","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-0337-808X","authenticated-orcid":false,"given":"Zhimin","family":"Zhao","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Yihao","family":"Chen","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Abdul\u00a0Ali","family":"Bangash","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Bram","family":"Adams","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Ahmed\u00a0E.","family":"Hassan","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2024,6,15]]},"reference":[{"key":"10474_CR1","doi-asserted-by":"crossref","unstructured":"Agrawal N, Bolosky WJ, Douceur JR, Lorch JR (2007) A five-year study of file-system metadata. ACM Trans Storage (TOS) 3(3):9\u2013es","DOI":"10.1145\/1288783.1288788"},{"key":"10474_CR2","unstructured":"Aguilar\u00a0Melgar, L., Dao, D., Gan, S., G\u00fcrel, N.M., Hollenstein, N., Jiang, J., Karla\u0161, B., Lemmin, T., Li, T., Li, Y., et\u00a0al.: Ease. ml: a lifecycle management system for machine learning. In: Proceedings of the Annual Conference on Innovative Data Systems Research (CIDR), 2021. CIDR (2021)"},{"key":"10474_CR3","doi-asserted-by":"crossref","unstructured":"Ahmed S, Bagherzadeh M (2018) What do concurrency developers ask about?: a large-scale study using stack overflow. Proceedings of the 12th ACM\/IEEE international symposium on empirical software engineering and measurement (2018)","DOI":"10.1145\/3239235.3239524"},{"key":"10474_CR4","doi-asserted-by":"crossref","unstructured":"Alberti M, Pondenkandath V, W\u00fcrsch M, Ingold R, Liwicki M (2018) Deepdiva: a highly-functional python framework for reproducible experiments. In: 2018 16th International conference on frontiers in handwriting recognition (ICFHR). IEEE, pp 423\u2013428","DOI":"10.1109\/ICFHR-2018.2018.00080"},{"key":"10474_CR5","doi-asserted-by":"crossref","unstructured":"Amershi S, Begel A, Bird C, DeLine R, Gall H, Kamar E, Nagappan N, Nushi B, Zimmermann T (2019) Software engineering for machine learning: A case study. In: 2019 IEEE\/ACM 41st International conference on software engineering: software engineering in practice (ICSE-SEIP). IEEE, pp 291\u2013300","DOI":"10.1109\/ICSE-SEIP.2019.00042"},{"key":"10474_CR6","doi-asserted-by":"crossref","unstructured":"Bagherzadeh M, Khatchadourian R (2019) Going big: a large-scale study on what big data developers ask. In: Proceedings of the 2019 27th ACM joint meeting on european software engineering conference and symposium on the foundations of software engineering, pp 432\u2013442","DOI":"10.1145\/3338906.3338939"},{"key":"10474_CR7","unstructured":"Bahrampour S, Ramakrishnan N, Schott L, Shah M (2015) Comparative study of deep learning software frameworks. arXiv:1511.06435"},{"key":"10474_CR8","unstructured":"Baier L, J\u00f6hren F, Seebacher S (2019) Challenges in the deployment and operation of machine learning in practice. In: ECIS, vol.\u00a01"},{"key":"10474_CR9","doi-asserted-by":"crossref","unstructured":"Barde BV, Bainwad AM (2017) An overview of topic modeling methods and tools. In: 2017 International conference on intelligent computing and control systems (ICICCS). IEEE, pp 745\u2013750","DOI":"10.1109\/ICCONS.2017.8250563"},{"key":"10474_CR10","doi-asserted-by":"crossref","unstructured":"Barrak A, Eghan EE, Adams B (2021) On the co-evolution of ml pipelines and source code-empirical study of dvc projects. In: 2021 IEEE International conference on software analysis, evolution and reengineering (SANER). IEEE, pp 422\u2013433","DOI":"10.1109\/SANER50967.2021.00046"},{"key":"10474_CR11","doi-asserted-by":"crossref","first-page":"17","DOI":"10.1016\/j.entcs.2006.09.029","volume":"182","author":"M Belguidoum","year":"2007","unstructured":"Belguidoum M, Dagnat F (2007) Dependency management in software component deployment. Electron Notes Theor Comput Sci 182:17\u201332","journal-title":"Electron Notes Theor Comput Sci"},{"key":"10474_CR12","doi-asserted-by":"crossref","DOI":"10.1016\/j.knosys.2021.107489","volume":"232","author":"A Ben\u00edtez-Hidalgo","year":"2021","unstructured":"Ben\u00edtez-Hidalgo A, Barba-Gonz\u00e1lez C, Garc\u00eda-Nieto J, Guti\u00e9rrez-Moncayo P, Paneque M, Nebro AJ, del Mar Rold\u00e1n-Garc\u00eda M, Aldana-Montes JF, Navas-Delgado I (2021) Titan: A knowledge-based platform for big data workflow management. Knowledge-Based Systems 232:107489","journal-title":"Knowledge-Based Systems"},{"issue":"1","key":"10474_CR13","doi-asserted-by":"crossref","first-page":"289","DOI":"10.1111\/j.2517-6161.1995.tb02031.x","volume":"57","author":"Y Benjamini","year":"1995","unstructured":"Benjamini Y, Hochberg Y (1995) Controlling the false discovery rate: a practical and powerful approach to multiple testing. J Royal Stat Soc: Ser B (Methodol) 57(1):289\u2013300","journal-title":"J Royal Stat Soc: Ser B (Methodol)"},{"key":"10474_CR14","unstructured":"Bhattacharjee A, Barve Y, Khare S, Bao S, Gokhale A, Damiano T (2019) Stratum: A serverless framework for the lifecycle management of machine learning-based data analytics tasks. In: 2019 USENIX Conference on Operational Machine Learning (OpML 19), pp 59\u201361"},{"key":"10474_CR15","unstructured":"Bommasani R, Hudson DA, Adeli E, Altman R, Arora S, von Arx S, Bernstein MS, Bohg J, Bosselut A, Brunskill E et\u00a0al (2021) On the opportunities and risks of foundation models. arXiv:2108.07258"},{"key":"10474_CR16","doi-asserted-by":"crossref","first-page":"112","DOI":"10.1016\/j.jss.2018.09.016","volume":"146","author":"H Borges","year":"2018","unstructured":"Borges H, Valente MT (2018) What\u2019s in a github star? understanding repository starring practices in a social coding platform. J Syst Softw 146:112\u2013129","journal-title":"J Syst Softw"},{"key":"10474_CR17","doi-asserted-by":"crossref","DOI":"10.1016\/j.eswa.2022.117232","volume":"202","author":"G Bravo-Rocca","year":"2022","unstructured":"Bravo-Rocca G, Liu P, Guitart J, Dholakia A, Ellison D, Falkanger J, Hodak M (2022) Scanflow: A multi-graph framework for machine learning workflow management, supervision, and debugging. Expert Syst Appl 202:117232","journal-title":"Expert Syst Appl"},{"issue":"3","key":"10474_CR18","doi-asserted-by":"crossref","first-page":"294","DOI":"10.1177\/0049124113500475","volume":"42","author":"JL Campbell","year":"2013","unstructured":"Campbell JL, Quincy C, Osserman J, Pedersen OK (2013) Coding in-depth semistructured interviews: Problems of unitization and intercoder reliability and agreement. Sociol Methods Res 42(3):294\u2013320","journal-title":"Sociol Methods Res"},{"key":"10474_CR19","doi-asserted-by":"crossref","unstructured":"Chard R, Li Z, Chard K, Ward L, Babuji Y, Woodard A, Tuecke S, Blaiszik B, Franklin MJ, Foster I (2019) Dlhub: Model and data serving for science. In: 2019 IEEE International Parallel and Distributed Processing Symposium (IPDPS). IEEE, pp 283\u2013292","DOI":"10.1109\/IPDPS.2019.00038"},{"key":"10474_CR20","doi-asserted-by":"crossref","unstructured":"Chen Z, Cao Y, Liu Y, Wang H, Xie T, Liu X (2020) A comprehensive study on challenges in deploying deep learning based software. In: Proceedings of the 28th ACM Joint meeting on european software engineering conference and symposium on the foundations of software engineering, pp 750\u2013762","DOI":"10.1145\/3368089.3409759"},{"key":"10474_CR21","doi-asserted-by":"crossref","unstructured":"Chen A, Chow A, Davidson A, DCunha A, Ghodsi A, Hong SA, Konwinski A, Mewald C, Murching S, Nykodym T et\u00a0al (2020) Developments in mlflow: A system to accelerate the machine learning lifecycle. In: Proceedings of the fourth international workshop on data management for end-to-end machine learning, pp 1\u20134","DOI":"10.1145\/3399579.3399867"},{"key":"10474_CR22","doi-asserted-by":"crossref","unstructured":"Chen Y, Fernandes E, Adams B, Hassan AE (2023) On practitioners\u2019 concerns when adopting service mesh frameworks. Empir Softw Eng","DOI":"10.1007\/s10664-023-10348-1"},{"key":"10474_CR23","doi-asserted-by":"crossref","unstructured":"Cheng L, Li X, Bing L (2023) Is gpt-4 a good data analyst? arXiv:2305.15038","DOI":"10.18653\/v1\/2023.findings-emnlp.637"},{"key":"10474_CR24","doi-asserted-by":"crossref","unstructured":"Coelho J, Valente MT (2017) Why modern open source projects fail. In: Proceedings of the 2017 11th Joint meeting on foundations of software engineering, pp 186\u2013196","DOI":"10.1145\/3106237.3106246"},{"key":"10474_CR25","unstructured":"Cram\u00e9r H (1999) Mathematical methods of statistics, vol.\u00a043. Princeton university press"},{"key":"10474_CR26","doi-asserted-by":"crossref","unstructured":"Diamantopoulos T, Nastos DN, Symeonidis A (2023) Semantically-enriched jira issue tracking data. In: 2023 IEEE\/ACM 20th International conference on mining software repositories (MSR). IEEE, pp 218\u2013222","DOI":"10.1109\/MSR59073.2023.00039"},{"key":"10474_CR27","unstructured":"do Prado KS (2020) Kelvins: awesome-mlops: A curated list of awesome mlops tools. https:\/\/github.com\/kelvins\/awesome-mlops"},{"issue":"293","key":"10474_CR28","doi-asserted-by":"crossref","first-page":"52","DOI":"10.1080\/01621459.1961.10482090","volume":"56","author":"OJ Dunn","year":"1961","unstructured":"Dunn OJ (1961) Multiple comparisons among means. J Am Stat Assoc 56(293):52\u201364","journal-title":"J Am Stat Assoc"},{"issue":"2","key":"10474_CR29","doi-asserted-by":"crossref","first-page":"96","DOI":"10.1109\/MSEC.2022.3142338","volume":"20","author":"W Enck","year":"2022","unstructured":"Enck W, Williams L (2022) Top five challenges in software supply chain security: Observations from 30 industry and government organizations. IEEE Secur Privacy 20(2):96\u2013100","journal-title":"IEEE Secur Privacy"},{"issue":"4","key":"10474_CR30","doi-asserted-by":"crossref","first-page":"44","DOI":"10.1145\/3277539.3277541","volume":"16","author":"S Esparrachiari","year":"2018","unstructured":"Esparrachiari S, Reilly T, Rentz A (2018) Tracking and controlling microservice dependencies: Dependency management is a crucial part of system and software design. Queue 16(4):44\u201365","journal-title":"Queue"},{"key":"10474_CR31","doi-asserted-by":"crossref","DOI":"10.1016\/j.softx.2020.100551","volume":"12","author":"R Ferenc","year":"2020","unstructured":"Ferenc R, Viszkok T, Aladics T, J\u00e1sz J, Heged\u0171s P (2020) Deep-water framework: The swiss army knife of humans working with machine learning models. SoftwareX 12:100551","journal-title":"SoftwareX"},{"key":"10474_CR32","doi-asserted-by":"crossref","unstructured":"Fran\u00e7oise J, Caramiaux B, Sanchez T (2021) Marcelle: composing interactive machine learning workflows and interfaces. In: The 34th Annual ACM symposium on user interface software and technology, pp 39\u201353","DOI":"10.1145\/3472749.3474734"},{"key":"10474_CR33","unstructured":"Garcia R, Sreekanti V, Yadwadkar N, Crankshaw D, Gonzalez JE, Hellerstein JM (2018) Context: The missing piece in the machine learning lifecycle. In: KDD CMI Workshop, vol. 114, pp 1\u20134"},{"key":"10474_CR34","unstructured":"Gao C (2022) Tensorchord: awesome-llmops: An awesome curated list of best llmops tools for developers. https:\/\/github.com\/tensorchord\/Awesome-LLMOps"},{"key":"10474_CR35","doi-asserted-by":"crossref","unstructured":"Gharibi G, Walunj V, Alanazi R, Rella S, Lee Y (2019) Automated management of deep learning experiments. In: Proceedings of the 3rd International workshop on data management for end-to-end machine learning, pp 1\u20134","DOI":"10.1145\/3329486.3329495"},{"key":"10474_CR36","doi-asserted-by":"crossref","unstructured":"Gilardi F, Alizadeh M, Kubli M (2023) Chatgpt outperforms crowd-workers for text-annotation tasks. arXiv:2303.15056","DOI":"10.1073\/pnas.2305016120"},{"key":"10474_CR37","doi-asserted-by":"crossref","first-page":"111031","DOI":"10.1016\/j.jss.2021.111031","volume":"180","author":"G Giray","year":"2021","unstructured":"Giray G (2021) A software engineering perspective on engineering machine learning systems: State of the art and challenges. J Syst Softw 180:111031","journal-title":"J Syst Softw"},{"key":"10474_CR38","doi-asserted-by":"crossref","unstructured":"Goniwada SR, Goniwada SR (2022) Observability. Cloud native architecture and design: a handbook for modern day architecture and design with enterprise-grade examples pp 661\u2013676","DOI":"10.1007\/978-1-4842-7226-8_19"},{"key":"10474_CR39","doi-asserted-by":"crossref","first-page":"1789","DOI":"10.1007\/s11263-021-01453-z","volume":"129","author":"J Gou","year":"2021","unstructured":"Gou J, Yu B, Maybank SJ, Tao D (2021) Knowledge distillation: A survey. Int J Comput Vision 129:1789\u20131819","journal-title":"Int J Comput Vision"},{"key":"10474_CR40","doi-asserted-by":"crossref","unstructured":"Groeneveld D, Beltagy I, Walsh P, Bhagia A, Kinney R, Tafjord O, Jha AH, Ivison H, Magnusson I, Wang Y et\u00a0al (2024) Olmo: Accelerating the science of language models. arXiv:2402.00838","DOI":"10.18653\/v1\/2024.acl-long.841"},{"key":"10474_CR41","unstructured":"Grootendorst M (2022) Bertopic: Neural topic modeling with a class-based tf-idf procedure. arXiv:2203.05794"},{"key":"10474_CR42","doi-asserted-by":"crossref","unstructured":"Grubb P, Takang AA (2003) Software maintenance: concepts and practice. World Scientific","DOI":"10.1142\/9789812564429"},{"key":"10474_CR43","doi-asserted-by":"crossref","unstructured":"Gu H, He H, Zhou M (2023) Self-admitted library migrations in java, javascript, and python packaging ecosystems: A comparative study. In: 2023 IEEE international conference on software analysis, evolution and reengineering (SANER). IEEE, pp 627\u2013638","DOI":"10.1109\/SANER56733.2023.00064"},{"key":"10474_CR44","doi-asserted-by":"crossref","unstructured":"Hartley M, Olsson TS (2020) dtoolai: Reproducibility for deep learning. Patterns 1(5)","DOI":"10.1016\/j.patter.2020.100073"},{"key":"10474_CR45","doi-asserted-by":"crossref","unstructured":"Hastie T, Tibshirani R, Friedman JH, Friedman JH (2009) The elements of statistical learning: data mining, inference, and prediction, vol.\u00a02. Springer","DOI":"10.1007\/978-0-387-84858-7"},{"key":"10474_CR46","unstructured":"Hewage N, Meedeniya D (2022) Machine learning operations: A survey on mlops tool support. arXiv:2202.10169"},{"key":"10474_CR47","doi-asserted-by":"crossref","unstructured":"Hummer W, Muthusamy V, Rausch T, Dube P, El\u00a0Maghraoui K, Murthi A, Oum P (2019) Modelops: Cloud-based lifecycle management for reliable and trusted ai. In: 2019 IEEE International Conference on Cloud Engineering (IC2E). IEEE, pp 113\u2013120","DOI":"10.1109\/IC2E.2019.00025"},{"key":"10474_CR48","doi-asserted-by":"publisher","unstructured":"Idowu S, Str\u00fcber D, Berger T (2022) Asset management in machine learning: State-of-research and state-of-practice. ACM Comput Surv. https:\/\/doi.org\/10.1145\/3543847. Just Accepted","DOI":"10.1145\/3543847"},{"key":"10474_CR49","doi-asserted-by":"crossref","unstructured":"Idowu S, Str\u00fcber D, Berger T (2022) Emmm: A unified meta-model for tracking machine learning experiments. In: 2022 48th Euromicro conference on software engineering and advanced applications (SEAA). IEEE, pp 48\u201355","DOI":"10.1109\/SEAA56994.2022.00016"},{"key":"10474_CR50","doi-asserted-by":"crossref","first-page":"154300","DOI":"10.1109\/ACCESS.2019.2946884","volume":"7","author":"H Isah","year":"2019","unstructured":"Isah H, Abughofa T, Mahfuz S, Ajerla D, Zulkernine F, Khan S (2019) A survey of distributed data stream processing frameworks. IEEE Access 7:154300\u2013154316","journal-title":"IEEE Access"},{"key":"10474_CR51","doi-asserted-by":"crossref","unstructured":"Izquierdo JLC, Cosentino V, Cabot J (2017) An empirical study on the maturity of the eclipse modeling ecosystem. In: 2017 ACM\/IEEE 20th International Conference on Model Driven Engineering Languages and Systems (MODELS). IEEE, pp 292\u2013302","DOI":"10.1109\/MODELS.2017.19"},{"key":"10474_CR52","doi-asserted-by":"crossref","unstructured":"Jalali S, Wohlin C (2012) Systematic literature studies: database searches vs. backward snowballing. In: Proceedings of the ACM-IEEE international symposium on Empirical software engineering and measurement, pp 29\u201338","DOI":"10.1145\/2372251.2372257"},{"key":"10474_CR53","unstructured":"Jiang AQ, Sablayrolles A, Mensch A, Bamford C, Chaplot DS, Casas Ddl, Bressand F, Lengyel G, Lample G, Saulnier L et\u00a0al (2023) Mistral 7b. arXiv:2310.06825"},{"key":"10474_CR54","doi-asserted-by":"crossref","unstructured":"Jiang W, Synovic N, Hyatt M, Schorlemmer TR, Sethi R, Lu YH, Thiruvathukal GK, Davis JC (2023) An empirical study of pre-trained model reuse in the hugging face deep learning model registry. arXiv:2303.02552","DOI":"10.1109\/ICSE48619.2023.00206"},{"key":"10474_CR55","doi-asserted-by":"crossref","unstructured":"Khondhu J, Capiluppi A, Stol KJ (2013) Is it all lost? a study of inactive open source projects. In: Open source software: quality verification: 9th IFIP WG 2.13 International conference, OSS 2013, Koper-Capodistria, Slovenia, June 25-28, 2013. Proceedings 9. Springer, pp 61\u201379","DOI":"10.1007\/978-3-642-38928-3_5"},{"issue":"6","key":"10474_CR56","doi-asserted-by":"crossref","first-page":"365","DOI":"10.1002\/(SICI)1096-908X(199911\/12)11:6<365::AID-SMR200>3.0.CO;2-W","volume":"11","author":"BA Kitchenham","year":"1999","unstructured":"Kitchenham BA, Travassos GH, Von Mayrhauser A, Niessink F, Schneidewind NF, Singer J, Takada S, Vehvilainen R, Yang H (1999) Towards an ontology of software maintenance. J Softw Maintenance: Res Pract 11(6):365\u2013389","journal-title":"J Softw Maintenance: Res Pract"},{"key":"10474_CR57","unstructured":"Klaise J, Van\u00a0Looveren A, Cox C, Vacanti G, Coca A (2020) Monitoring and explainability of models in production. arXiv:2007.06299"},{"issue":"1","key":"10474_CR58","doi-asserted-by":"crossref","first-page":"14","DOI":"10.1109\/JPROC.2014.2371999","volume":"103","author":"D Kreutz","year":"2014","unstructured":"Kreutz D, Ramos FM, Verissimo PE, Rothenberg CE, Azodolmolky S, Uhlig S (2014) Software-defined networking: A comprehensive survey. Proc of the IEEE 103(1):14\u201376","journal-title":"Proc of the IEEE"},{"key":"10474_CR59","doi-asserted-by":"crossref","unstructured":"Kumar A, Boehm M, Yang J (2017) Data management in machine learning: Challenges, techniques, and systems. In: Proceedings of the 2017 ACM International conference on management of data, pp 1717\u20131722","DOI":"10.1145\/3035918.3054775"},{"key":"10474_CR60","volume-title":"Deep Reinforcement Learning Hands-On: Apply modern RL methods, with deep Q-networks, value iteration, policy gradients","author":"M Lapan","year":"2018","unstructured":"Lapan M (2018) Deep Reinforcement Learning Hands-On: Apply modern RL methods, with deep Q-networks, value iteration, policy gradients. Packt Publishing Ltd, AlphaGo Zero and more, TRPO"},{"key":"10474_CR61","unstructured":"Le VD (2023) Veml: An end-to-end machine learning lifecycle for large-scale and high-dimensional data. arXiv:2304.13037"},{"key":"10474_CR62","unstructured":"Liu A, Han X, Wang Y, Tsvetkov Y, Choi Y, Smith NA (2024) Tuning language models by proxy. arXiv:2401.08565"},{"key":"10474_CR63","doi-asserted-by":"crossref","unstructured":"Liu Y, Iter D, Xu Y, Wang S, Xu R, Zhu C (2023) Gpteval: Nlg evaluation using gpt-4 with better human alignment. arXiv:2303.16634","DOI":"10.18653\/v1\/2023.emnlp-main.153"},{"key":"10474_CR64","unstructured":"Loeliger J, McCullough M (2012) Version Control with Git: Powerful tools and techniques for collaborative software development. \" O\u2019Reilly Media, Inc.\""},{"key":"10474_CR65","unstructured":"Lu L, Arpaci-Dusseau AC, Arpaci-Dusseau RH, Lu S (2013) A study of linux file system evolution. In: 11th USENIX Conference on file and storage technologies (FAST 13), pp 31\u201344"},{"key":"10474_CR66","doi-asserted-by":"crossref","first-page":"424","DOI":"10.1016\/j.jnca.2013.10.004","volume":"41","author":"SS Manvi","year":"2014","unstructured":"Manvi SS, Shyam GK (2014) Resource management for infrastructure as a service (iaas) in cloud computing: A survey. J Netw Comput Appl 41:424\u2013440","journal-title":"J Netw Comput Appl"},{"issue":"3","key":"10474_CR67","doi-asserted-by":"crossref","first-page":"276","DOI":"10.11613\/BM.2012.031","volume":"22","author":"ML McHugh","year":"2012","unstructured":"McHugh ML (2012) Interrater reliability: the kappa statistic. Biochem Med 22(3):276\u2013282","journal-title":"Biochem Med"},{"issue":"11","key":"10474_CR68","doi-asserted-by":"crossref","first-page":"205","DOI":"10.21105\/joss.00205","volume":"2","author":"L McInnes","year":"2017","unstructured":"McInnes L, Healy J, Astels S (2017) hdbscan: Hierarchical density based clustering. J Open Source Softw 2(11):205","journal-title":"J Open Source Softw"},{"issue":"9","key":"10474_CR69","first-page":"1","volume":"14","author":"W McKinney","year":"2011","unstructured":"McKinney W et al (2011) pandas: a foundational python library for data analysis and statistics. Python high Perform Sci Comput 14(9):1\u20139","journal-title":"Python high Perform Sci Comput"},{"key":"10474_CR70","unstructured":"Melin PD (2023) Tackling version management and reproducibility in mlops"},{"key":"10474_CR71","unstructured":"Mens T, Goeminne M, Raja U, Serebrenik A (2014) Survivability of software projects in gnome\u2013a replication study. In: 7th International seminar series on advanced techniques & tools for software evolution (SATToSE), pp 79\u201382"},{"key":"10474_CR72","doi-asserted-by":"crossref","unstructured":"Miao H, Chavan A, Deshpande A (2017) Provdb: Lifecycle management of collaborative analysis workflows. In: Proceedings of the 2nd workshop on human-in-the-loop data analytics, pp 1\u20136","DOI":"10.1145\/3077257.3077267"},{"key":"10474_CR73","doi-asserted-by":"crossref","unstructured":"Miao H, Li A, Davis LS, Deshpande A (2017) Modelhub: Deep learning lifecycle management. In: 2017 IEEE 33rd International conference on data engineering (ICDE). IEEE, pp 1393\u20131394","DOI":"10.1109\/ICDE.2017.192"},{"key":"10474_CR74","doi-asserted-by":"crossref","unstructured":"Miao H, Li A, Davis LS, Deshpande A (2017) Towards unified data and lifecycle management for deep learning. In: 2017 IEEE 33rd International Conference on Data Engineering (ICDE). IEEE, pp 571\u2013582","DOI":"10.1109\/ICDE.2017.112"},{"issue":"6","key":"10474_CR75","doi-asserted-by":"crossref","first-page":"1236","DOI":"10.1093\/bib\/bbx044","volume":"19","author":"R Miotto","year":"2018","unstructured":"Miotto R, Wang F, Wang S, Jiang X, Dudley JT (2018) Deep learning for healthcare: review, opportunities and challenges. Briefings Bioinf 19(6):1236\u20131246","journal-title":"Briefings Bioinf"},{"issue":"02","key":"10474_CR76","first-page":"295","volume":"14","author":"M Moreno","year":"2020","unstructured":"Moreno M, Louren\u00e7o V, Fiorini SR, Costa P, Brand\u00e3o R, Civitarese D, Cerqueira R (2020) Managing machine learning workflow components. Int J Sem Comput 14(02):295\u2013309","journal-title":"Int J Sem Comput"},{"key":"10474_CR77","unstructured":"Moreschi S, Recupito G, Lenarduzzi V, Palomba F, Hastbacka D, Taibi D (2023) Toward end-to-end mlops tools map: A preliminary study based on a multivocal literature review. arXiv:2304.03254"},{"key":"10474_CR78","doi-asserted-by":"crossref","unstructured":"Munappy AR, Bosch J, Olsson HH, Arpteg A, Brinne B (2022) Data management for production quality deep learning models: Challenges and solutions. J Syst Softw 191:111359","DOI":"10.1016\/j.jss.2022.111359"},{"key":"10474_CR79","doi-asserted-by":"crossref","first-page":"186","DOI":"10.1016\/j.compeleceng.2015.07.021","volume":"47","author":"S Mustafa","year":"2015","unstructured":"Mustafa S, Nazir B, Hayat A, Madani SA et al (2015) Resource management in cloud computing: Taxonomy, prospects, and challenges. Comput Electr Eng 47:186\u2013203","journal-title":"Comput Electr Eng"},{"key":"10474_CR80","doi-asserted-by":"crossref","first-page":"148","DOI":"10.1016\/j.pmcj.2018.07.004","volume":"50","author":"AM Nagy","year":"2018","unstructured":"Nagy AM, Simon V (2018) Survey on traffic prediction in smart cities. Pervasive Mobile Comput 50:148\u2013163","journal-title":"Pervasive Mobile Comput"},{"key":"10474_CR81","doi-asserted-by":"crossref","unstructured":"Namaki MH, Floratou A, Psallidas F, Krishnan S, Agrawal A, Wu Y (2020) Vamsa: Tracking provenance in data science scripts. arXiv:2001.01861","DOI":"10.1145\/3394486.3403205"},{"key":"10474_CR82","doi-asserted-by":"crossref","first-page":"77","DOI":"10.1007\/s10462-018-09679-z","volume":"52","author":"G Nguyen","year":"2019","unstructured":"Nguyen G, Dlugolinsky S, Bob\u00e1k M, Tran V, L\u00f3pez Garc\u00eda \u00c1, Heredia I, Mal\u00edk P, Hluch\u1ef3 L (2019) Machine learning and deep learning frameworks and libraries for large-scale data mining: a survey. Artif Intell Rev 52:77\u2013124","journal-title":"Artif Intell Rev"},{"key":"10474_CR83","doi-asserted-by":"crossref","unstructured":"Openja M, Adams B, Khomh F (2020) Analysis of modern release engineering topics: A large-scale study using stackoverflow. In: Proceedings of the 36th International conference on software maintenance and evolution (ICSME), pp 104\u2013114","DOI":"10.1109\/ICSME46990.2020.00020"},{"issue":"6","key":"10474_CR84","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1145\/3533378","volume":"55","author":"A Paleyes","year":"2022","unstructured":"Paleyes A, Urma RG, Lawrence ND (2022) Challenges in deploying machine learning: a survey of case studies. ACM Comput Surv 55(6):1\u201329","journal-title":"ACM Comput Surv"},{"issue":"2","key":"10474_CR85","doi-asserted-by":"crossref","first-page":"40","DOI":"10.1007\/s10664-021-10095-1","volume":"27","author":"E Parra","year":"2022","unstructured":"Parra E, Alahmadi M, Ellis A, Haiduc S (2022) A comparative study and analysis of developer communications on slack and gitter. Empir Softw Eng 27(2):40","journal-title":"Empir Softw Eng"},{"key":"10474_CR86","unstructured":"Pavao A, Guyon I, Letournel AC, Bar\u00f3 X, Escalante H, Escalera S, Thomas T, Xu Z (2022) Codalab competitions: An open source platform to organize scientific challenges. Ph.D. thesis, Universit\u00e9 Paris-Saclay, FRA. (2022)"},{"key":"10474_CR87","doi-asserted-by":"crossref","unstructured":"Pearson K (1900) X. on the criterion that a given system of deviations from the probable in the case of a correlated system of variables is such that it can be reasonably supposed to have arisen from random sampling. The London, Edinburgh, and Dublin Philos Mag J Sci 50(302):157\u2013175","DOI":"10.1080\/14786440009463897"},{"key":"10474_CR88","doi-asserted-by":"crossref","unstructured":"Peili Y, Xuezhen Y, Jian Y, Lingfeng Y, Hui Z, Jimin L (2018) Deep learning model management for coronary heart disease early warning research. In: 2018 IEEE 3rd International Conference on Cloud Computing and Big Data Analysis (ICCCBDA). IEEE, pp 552\u2013557","DOI":"10.1109\/ICCCBDA.2018.8386577"},{"issue":"2","key":"10474_CR89","doi-asserted-by":"crossref","first-page":"17","DOI":"10.1145\/3299887.3299891","volume":"47","author":"N Polyzotis","year":"2018","unstructured":"Polyzotis N, Roy S, Whang SE, Zinkevich M (2018) Data lifecycle challenges in production machine learning: a survey. ACM SIGMOD Record 47(2):17\u201328","journal-title":"ACM SIGMOD Record"},{"key":"10474_CR90","doi-asserted-by":"crossref","unstructured":"Recupito G, Pecorelli F, Catolino G, Moreschini S, Di\u00a0Nucci D, Palomba F, Tamburri DA (2022) A multivocal literature review of mlops tools and features. In: 2022 48th Euromicro Conference on Software Engineering and Advanced Applications (SEAA). IEEE, pp 84\u201391","DOI":"10.1109\/SEAA56994.2022.00021"},{"key":"10474_CR91","unstructured":"Rigby PC, Barr ET, Bird C, German DM, Devanbu P (2009) Collaboration and governance with distributed version control. ACM Trans Software Engineering and Methodology, Submission number TOSEM-2009-0087 p\u00a033"},{"key":"10474_CR92","doi-asserted-by":"crossref","first-page":"364","DOI":"10.1109\/TSE.1975.6312866","volume":"4","author":"MJ Rochkind","year":"1975","unstructured":"Rochkind MJ (1975) The source code control system. IEEE Trans Softw Eng 4:364\u2013370","journal-title":"IEEE Trans Softw Eng"},{"key":"10474_CR93","doi-asserted-by":"crossref","first-page":"1192","DOI":"10.1007\/s10664-015-9379-3","volume":"21","author":"C Rosen","year":"2016","unstructured":"Rosen C, Shihab E (2016) What are mobile developers asking about? a large scale study using stack overflow. Empir Softw Eng 21:1192\u20131223","journal-title":"Empir Softw Eng"},{"issue":"19","key":"10474_CR94","doi-asserted-by":"crossref","first-page":"8861","DOI":"10.3390\/app11198861","volume":"11","author":"P Ruf","year":"2021","unstructured":"Ruf P, Madan M, Reich C, Ould-Abdeslam D (2021) Demystifying mlops and presenting a recipe for the selection of open-source tools. Appl Sci 11(19):8861","journal-title":"Appl Sci"},{"key":"10474_CR95","doi-asserted-by":"crossref","unstructured":"Sallou J, Durieux T, Panichella A (2024) Breaking the silence: the threats of using llms in software engineering. In: ACM\/IEEE 46th International conference on software engineering. ACM\/IEEE","DOI":"10.1145\/3639476.3639764"},{"key":"10474_CR96","unstructured":"Saucedo A (2018) EthicalML: awesome-production-machine-learning: A curated list of awesome open source libraries to deploy, monitor, version and scale your machine learning. https:\/\/github.com\/EthicalML\/awesome-production-machine-learning"},{"key":"10474_CR97","unstructured":"Schelter S, Biessmann F, Januschowski T, Salinas D, Seufert S, Szarvas G (2015) On challenges in machine learning model management"},{"key":"10474_CR98","unstructured":"Schelter S, B\u00f6se JH, Kirschnick J, Klein T, Seufert S (2018) Declarative metadata management: A missing piece in end-to-end machine learning"},{"key":"10474_CR99","doi-asserted-by":"crossref","unstructured":"Schick T, Sch\u00fctze H (2020) It\u2019s not just size that matters: Small language models are also few-shot learners. arXiv:2009.07118","DOI":"10.18653\/v1\/2021.naacl-main.185"},{"issue":"4","key":"10474_CR100","doi-asserted-by":"crossref","first-page":"18","DOI":"10.1145\/3582302.3582306","volume":"51","author":"M Schlegel","year":"2023","unstructured":"Schlegel M, Sattler KU (2023) Management of machine learning lifecycle artifacts: A survey. ACM SIGMOD Record 51(4):18\u201335","journal-title":"ACM SIGMOD Record"},{"key":"10474_CR101","unstructured":"Sculley D, Holt G, Golovin D, Davydov E, Phillips T, Ebner D, Chaudhary V, Young M, Crespo JF, Dennison D (2015) Hidden technical debt in machine learning systems. Advances in neural information processing systems 28"},{"issue":"2","key":"10474_CR102","doi-asserted-by":"crossref","first-page":"215","DOI":"10.1016\/j.ijinfomgt.2015.11.009","volume":"36","author":"ZA Soomro","year":"2016","unstructured":"Soomro ZA, Shah MH, Ahmed J (2016) Information security management needs more holistic approach: A literature review. Int J Inf Manag 36(2):215\u2013225","journal-title":"Int J Inf Manag"},{"key":"10474_CR103","doi-asserted-by":"crossref","unstructured":"Sorokin A, Forsyth D (2008) Utility data annotation with amazon mechanical turk. In: 2008 IEEE computer society conference on computer vision and pattern recognition workshops. IEEE, pp 1\u20138","DOI":"10.1109\/CVPRW.2008.4562953"},{"key":"10474_CR104","doi-asserted-by":"crossref","unstructured":"Squire M (2015) \"should we move to stack overflow?\" measuring the utility of social media for developer support. In: 2015 IEEE\/ACM 37th IEEE International Conference on Software Engineering, vol.\u00a02. IEEE, pp 219\u2013228","DOI":"10.1109\/ICSE.2015.150"},{"issue":"3","key":"10474_CR105","doi-asserted-by":"crossref","first-page":"479","DOI":"10.1111\/1467-9868.00346","volume":"64","author":"JD Storey","year":"2002","unstructured":"Storey JD (2002) A direct approach to false discovery rates. J Royal Stat Soc Ser B: Stat Methodol 64(3):479\u2013498","journal-title":"J Royal Stat Soc Ser B: Stat Methodol"},{"key":"10474_CR106","unstructured":"Sun C, Azari N, Turakhia C (2020) Gallery: A machine learning model management system at uber. In: EDBT, vol.\u00a020, pp 474\u2013485"},{"key":"10474_CR107","unstructured":"Sung N, Kim M, Jo H, Yang Y, Kim J, Lausen L, Kim Y, Lee G, Kwak D, Ha JW et\u00a0al (2017) Nsml: A machine learning platform that enables you to focus on your models. arXiv:1712.05902"},{"key":"10474_CR108","doi-asserted-by":"crossref","unstructured":"Syed S, Spruit M (2017) Full-text or abstract? examining topic coherence scores using latent dirichlet allocation. In: 2017 IEEE International conference on data science and advanced analytics (DSAA). IEEE, pp 165\u2013174","DOI":"10.1109\/DSAA.2017.61"},{"key":"10474_CR109","doi-asserted-by":"crossref","unstructured":"Symeonidis G, Nerantzis E, Kazakis A, Papakostas GA (2022) Mlops-definitions, tools and challenges. In: 2022 IEEE 12th Annual computing and communication workshop and conference (CCWC). IEEE, pp 0453\u20130460","DOI":"10.1109\/CCWC54503.2022.9720902"},{"key":"10474_CR110","doi-asserted-by":"crossref","unstructured":"Tao L, Cazan AP, Ibraimoski S, Moran S (2023) Code librarian: A software package recommendation system. In: 2023 IEEE\/ACM 45th International conference on software engineering: software engineering in practice (ICSE-SEIP). IEEE, pp 196\u2013198","DOI":"10.1109\/ICSE-SEIP58684.2023.00023"},{"key":"10474_CR111","unstructured":"Touvron H, Martin L, Stone K, Albert P, Almahairi A, Babaei Y, Bashlykov N, Batra S, Bhargava P, Bhosale S et\u00a0al (2023) Llama 2: Open foundation and fine-tuned chat models. arXiv:2307.09288"},{"key":"10474_CR112","doi-asserted-by":"crossref","unstructured":"Treude C, Barzilay O, Storey MA (2011) How do programmers ask and answer questions on the web?(nier track). In: Proceedings of the 33rd international conference on software engineering, pp 804\u2013807","DOI":"10.1145\/1985793.1985907"},{"key":"10474_CR113","unstructured":"Tsay J, Mummert T, Bobroff N, Braz A, Westerink P, Hirzel M (2018) Runway: machine learning model experiment management tool. In: Conference on systems and machine learning (sysML)"},{"key":"10474_CR114","doi-asserted-by":"crossref","unstructured":"Vadlamani SL, Baysal O (2020) Studying software developer expertise and contributions in stack overflow and github. In: 2020 IEEE International Conference on Software Maintenance and Evolution (ICSME). IEEE, pp 312\u2013323","DOI":"10.1109\/ICSME46990.2020.00038"},{"issue":"4","key":"10474_CR115","first-page":"16","volume":"41","author":"M Vartak","year":"2018","unstructured":"Vartak M, Madden S (2018) Modeldb: Opportunities and challenges in managing machine learning models. IEEE Data Eng Bull 41(4):16\u201325","journal-title":"IEEE Data Eng Bull"},{"key":"10474_CR116","doi-asserted-by":"crossref","unstructured":"Vasilescu B, Filkov V, Serebrenik A (2013) Stackoverflow and github: Associations between software development and crowdsourced knowledge. In: 2013 International conference on social computing. IEEE, pp 188\u2013195","DOI":"10.1109\/SocialCom.2013.35"},{"key":"10474_CR117","doi-asserted-by":"crossref","unstructured":"Venkatesh PK, Wang S, Zhang F, Zou Y, Hassan AE (2016) What do client developers concern when using web apis? an empirical study on developer forums and stack overflow. In: 2016 IEEE International Conference on Web Services (ICWS). IEEE, pp 131\u2013138","DOI":"10.1109\/ICWS.2016.25"},{"key":"10474_CR118","doi-asserted-by":"crossref","unstructured":"Wang Z, Liu K, Li J, Zhu Y, Zhang Y (2019) Various frameworks and libraries of machine learning and deep learning: a survey. Archives of computational methods in engineering pp 1\u201324","DOI":"10.1007\/s11831-018-09312-w"},{"issue":"1","key":"10474_CR119","doi-asserted-by":"crossref","first-page":"4","DOI":"10.1108\/09685220910944722","volume":"17","author":"R Werlinger","year":"2009","unstructured":"Werlinger R, Hawkey K, Beznosov K (2009) An integrated view of human, organizational, and technological challenges of it security management. Inf Manag Comput Secur 17(1):4\u201319","journal-title":"Inf Manag Comput Secur"},{"issue":"1","key":"10474_CR120","first-page":"1","volume":"4","author":"JR Wood","year":"2008","unstructured":"Wood JR, Wood LE (2008) Card sorting: current practices and beyond. J Usability Studies 4(1):1\u20136","journal-title":"J Usability Studies"},{"issue":"18","key":"10474_CR121","first-page":"59","volume":"19","author":"JM Wozniak","year":"2018","unstructured":"Wozniak JM, Jain R, Balaprakash P, Ozik J, Collier NT, Bauer J, Xia F, Brettin T, Stevens R, Mohd-Yusof J et al (2018) Candle\/supervisor: A workflow framework for machine learning applied to cancer research. BMC Bioinf 19(18):59\u201369","journal-title":"BMC Bioinf"},{"issue":"1","key":"10474_CR122","doi-asserted-by":"crossref","first-page":"27","DOI":"10.1109\/COMST.2014.2330903","volume":"17","author":"W Xia","year":"2014","unstructured":"Xia W, Wen Y, Foh CH, Niyato D, Xie H (2014) A survey on software-defined networking. IEEE Commun Surv Tutor 17(1):27\u201351","journal-title":"IEEE Commun Surv Tutor"},{"key":"10474_CR123","doi-asserted-by":"crossref","unstructured":"Xin D, Miao H, Parameswaran A, Polyzotis N (2021) Production machine learning pipelines: Empirical analysis and optimization opportunities. In: Proceedings of the 2021 international conference on management of data, pp 2639\u20132652","DOI":"10.1145\/3448016.3457566"},{"issue":"1","key":"10474_CR124","doi-asserted-by":"crossref","first-page":"114","DOI":"10.1109\/MS.2020.2975159","volume":"38","author":"M Xiu","year":"2020","unstructured":"Xiu M, Jiang ZMJ, Adams B (2020) An exploratory study of machine learning model stores. IEEE Software 38(1):114\u2013122","journal-title":"IEEE Software"},{"key":"10474_CR125","doi-asserted-by":"crossref","first-page":"910","DOI":"10.1007\/s11390-016-1672-0","volume":"31","author":"X Yang","year":"2016","unstructured":"Yang X, Lo D, Xia X, Wan Z, Sun J (2016) What security questions do developers ask? a large-scale study of stack overflow posts. J Comput Sci Technol 31:910\u2013924","journal-title":"J Comput Sci Technol"},{"key":"10474_CR126","doi-asserted-by":"crossref","first-page":"2993","DOI":"10.1007\/s10994-021-06052-0","volume":"110","author":"C Yang","year":"2021","unstructured":"Yang C, Wang W, Zhang Y, Zhang Z, Shen L, Li Y, See J (2021) Mlife: A lite framework for machine learning lifecycle initialization. Mach Learn 110:2993\u20133013","journal-title":"Mach Learn"},{"key":"10474_CR127","doi-asserted-by":"crossref","unstructured":"Yao Y, Duan J, Xu K, Cai Y, Sun E, Zhang Y (2023) A survey on large language model (llm) security and privacy: The good, the bad, and the ugly. arXiv:2312.02003","DOI":"10.1016\/j.hcc.2024.100211"},{"issue":"4","key":"10474_CR128","first-page":"39","volume":"41","author":"M Zaharia","year":"2018","unstructured":"Zaharia M, Chen A, Davidson A, Ghodsi A, Hong SA, Konwinski A, Murching S, Nykodym T, Ogilvie P, Parkhe M et al (2018) Accelerating the machine learning lifecycle with mlflow. IEEE Data Eng Bull 41(4):39\u201345","journal-title":"IEEE Data Eng Bull"},{"key":"10474_CR129","unstructured":"Zhang S, Dong L, Li X, Zhang S, Sun X, Wang S, Li J, Hu R, Zhang T, Wu F et\u00a0al (2023) Instruction tuning for large language models: A survey. arXiv:2308.10792"}],"container-title":["Empirical Software Engineering"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10664-024-10474-4.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s10664-024-10474-4\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10664-024-10474-4.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,11,21]],"date-time":"2024-11-21T22:59:21Z","timestamp":1732229961000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s10664-024-10474-4"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,6,15]]},"references-count":129,"journal-issue":{"issue":"4","published-print":{"date-parts":[[2024,7]]}},"alternative-id":["10474"],"URL":"https:\/\/doi.org\/10.1007\/s10664-024-10474-4","relation":{},"ISSN":["1382-3256","1573-7616"],"issn-type":[{"value":"1382-3256","type":"print"},{"value":"1573-7616","type":"electronic"}],"subject":[],"published":{"date-parts":[[2024,6,15]]},"assertion":[{"value":"13 March 2024","order":1,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"15 June 2024","order":2,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Declarations"}},{"value":"The authors declare that they have no conflict of interest.","order":2,"name":"Ethics","group":{"name":"EthicsHeading","label":"Conflict of interest"}}],"article-number":"98"}}