{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,8]],"date-time":"2026-07-08T23:36:35Z","timestamp":1783553795251,"version":"3.55.0"},"reference-count":73,"publisher":"Springer Science and Business Media LLC","issue":"6","license":[{"start":{"date-parts":[[2024,9,11]],"date-time":"2024-09-11T00:00:00Z","timestamp":1726012800000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2024,9,11]],"date-time":"2024-09-11T00:00:00Z","timestamp":1726012800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"funder":[{"DOI":"10.13039\/501100000038","name":"NSERC","doi-asserted-by":"crossref","award":["ALLRP561420-20"],"award-info":[{"award-number":["ALLRP561420-20"]}],"id":[{"id":"10.13039\/501100000038","id-type":"DOI","asserted-by":"crossref"}]},{"name":"Prompt","award":["157"],"award-info":[{"award-number":["157"]}]}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Empir Software Eng"],"published-print":{"date-parts":[[2024,11]]},"DOI":"10.1007\/s10664-024-10536-7","type":"journal-article","created":{"date-parts":[[2024,9,11]],"date-time":"2024-09-11T09:03:21Z","timestamp":1726045401000},"update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":10,"title":["Quality issues in machine learning software systems"],"prefix":"10.1007","volume":"29","author":[{"ORCID":"https:\/\/orcid.org\/0009-0009-9937-1509","authenticated-orcid":false,"given":"Pierre-Olivier","family":"C\u00f4t\u00e9","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Amin","family":"Nikanjam","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Rached","family":"Bouchoucha","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Ilan","family":"Basta","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Mouna","family":"Abidi","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Foutse","family":"Khomh","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2024,9,11]]},"reference":[{"key":"10536_CR1","unstructured":"(2022) Tesla behind eight-vehicle crash was in full self-driving mode, says driver. URL https:\/\/www.theguardian.com\/technology\/2022\/dec\/22\/tesla-crash-full-self-driving-mode-san-francisco"},{"key":"10536_CR2","doi-asserted-by":"crossref","unstructured":"Alahdab M, \u00c7al\u0131kl\u0131 G (2019) Empirical analysis of hidden technical debt patterns in machine learning software. In: International Conference on Product-Focused Software Process Improvement, Springer, pp 195\u2013202","DOI":"10.1007\/978-3-030-35333-9_14"},{"key":"10536_CR3","doi-asserted-by":"crossref","unstructured":"Amershi S, Begel A, Bird C, DeLine R, Gall H, Kamar E, Nagappan N, Nushi B, Zimmermann T (2019) Software engineering for machine learning: A case study. In: 2019 IEEE\/ACM 41st International Conference on Software Engineering: Software Engineering in Practice (ICSE-SEIP), IEEE, pp 291\u2013300","DOI":"10.1109\/ICSE-SEIP.2019.00042"},{"key":"10536_CR4","unstructured":"Angwin J, Larson J, Kirchner L, Mattu S (2016) Machine bias. URL https:\/\/www.propublica.org\/article\/machine-bias-risk-assessments-in-criminal-sentencing"},{"key":"10536_CR5","unstructured":"Avgeriou P, Kruchten P, Ozkaya I, Seaman C (2016) Managing technical debt in software engineering (dagstuhl seminar 16162). In: Dagstuhl reports, Schloss Dagstuhl-Leibniz-Zentrum fuer Informatik, vol\u00a06"},{"key":"10536_CR6","doi-asserted-by":"crossref","unstructured":"Azimi S, Pahl C (2020) Root cause analysis and remediation for quality and value improvement in machine learning driven information models. In: ICEIS (1), pp 656\u2013665","DOI":"10.5220\/0009783106560665"},{"key":"10536_CR7","doi-asserted-by":"crossref","unstructured":"Berglund L, Grube T, Gay G, de\u00a0Oliveira\u00a0Neto FG, Platis D (2022) Test maintenance for machine learning systems: A case study in the automotive industry. PhD thesis, Master\u2019s thesis, University of Gothenburg, 2022, available from https\u00a0..","DOI":"10.1109\/ICST57152.2023.00045"},{"key":"10536_CR8","unstructured":"Bhatt U, Xiang A, Sharma S, Weller A, Taly A, Jia Y, Ghosh J, Puri R, Moura JMF, Eckersley P (2019) Explainable machine learning in deployment. CoRR abs\/1909.06342, URL http:\/\/arxiv.org\/abs\/1909.06342, 1909.06342"},{"key":"10536_CR9","unstructured":"Black A, van Nederpelt P (2020) Dimensions of data quality (ddq). URL https:\/\/www.dama-nl.org\/wp-content\/uploads\/2020\/09\/DDQ-Dimensions-of-Data-Quality-Research-Paper-version-1.2-d.d.-3-Sept-2020.pdf"},{"key":"10536_CR10","unstructured":"Blais O (2020) Validate and monitor your machine learning models. URL https:\/\/github.com\/moovai\/model_validation_tutorial\/blob\/master\/ODSC_East_2020_Validation_Monitoring_Training.pdf"},{"key":"10536_CR11","doi-asserted-by":"crossref","unstructured":"Bogner J, Verdecchia R, Gerostathopoulos I (2021) Characterizing technical debt and antipatterns in ai-based systems: A systematic mapping study. In: 2021 IEEE\/ACM International Conference on Technical Debt (TechDebt), IEEE, pp 64\u201373","DOI":"10.1109\/TechDebt52882.2021.00016"},{"key":"10536_CR12","doi-asserted-by":"crossref","unstructured":"Breck E, Cai S, Nielsen E, Salib M, Sculley D (2017) The ml test score: A rubric for ml production readiness and technical debt reduction. In: 2017 IEEE International Conference on Big Data (Big Data), IEEE, pp 1123\u20131132","DOI":"10.1109\/BigData.2017.8258038"},{"key":"10536_CR13","unstructured":"Breck E, Polyzotis N, Roy S, Whang S, Zinkevich M (2019) Data validation for machine learning. In: MLSys"},{"key":"10536_CR14","unstructured":"Cappi C, Chapdelaine C, Gardes L, Jenn E, Lefevre B, Picard S, Soumarmon T (2021) Dataset definition standard (dds). arXiv:2101.03020"},{"key":"10536_CR15","unstructured":"Chattopadhyay A, Manupriya P, Sarkar A, Balasubramanian VN (2019) Neural network attributions: A causal perspective. In: Chaudhuri K, Salakhutdinov R (eds) Proceedings of the 36th International Conference on Machine Learning, PMLR, Proceedings of Machine Learning Research, vol\u00a097, pp 981\u2013990, URL https:\/\/proceedings.mlr.press\/v97\/chattopadhyay19a.html"},{"key":"10536_CR16","unstructured":"Dastin J (2018) Amazon scraps secret ai recruiting tool that showed bias against women. URL https:\/\/www.reuters.com\/article\/us-amazon-com-jobs-automation-insight-idUSKCN1MK08G"},{"key":"10536_CR17","doi-asserted-by":"crossref","unstructured":"Dilhara M, Ketkar A, Dig D (2021) Understanding software-2.0: A study of machine learning library usage and evolution. ACM Transactions on Software Engineering and Methodology (TOSEM) 30(4):1\u201342","DOI":"10.1145\/3453478"},{"key":"10536_CR18","doi-asserted-by":"crossref","unstructured":"Doke A, Gaikwad M (2021) Survey on automated machine learning (automl) and meta learning. In: 2021 12th International Conference on Computing Communication and Networking Technologies (ICCCNT), IEEE, pp 1\u20135","DOI":"10.1109\/ICCCNT51525.2021.9579526"},{"key":"10536_CR19","unstructured":"Feldt R, Magazinius A (2010) Validity threats in empirical software engineering research-an initial survey. In: Seke, pp 374\u2013379"},{"key":"10536_CR20","doi-asserted-by":"crossref","unstructured":"Foidl H, Felderer M (2019) Risk-based data validation in machine learning-based software systems. In: proceedings of the 3rd ACM SIGSOFT international workshop on machine learning techniques for software quality evaluation, pp 13\u201318","DOI":"10.1145\/3340482.3342743"},{"key":"10536_CR21","doi-asserted-by":"crossref","unstructured":"Fujii G, Hamada K, Ishikawa F, Masuda S, Matsuya M, Myojin T, Nishi Y, Ogawa H, Toku T, Tokumoto S, et\u00a0al. (2020) Guidelines for quality assurance of machine learning-based artificial intelligence. International journal of software engineering and knowledge engineering 30(11n12):1589\u20131606","DOI":"10.1142\/S0218194020400227"},{"key":"10536_CR22","doi-asserted-by":"crossref","unstructured":"Gholami A, Kim S, Dong Z, Yao Z, Mahoney MW, Keutzer K (2021) A survey of quantization methods for efficient neural network inference. arXiv preprint arXiv:2103.13630","DOI":"10.1201\/9781003162810-13"},{"issue":"1","key":"10536_CR23","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1007\/s10664-023-10426-4","volume":"29","author":"D Hidellaarachchi","year":"2024","unstructured":"Hidellaarachchi D, Grundy J, Hoda R, Mueller I (2024) The impact of personality on requirements engineering activities: A mixed-methods study. Empir Softw Eng 29(1):1\u201356","journal-title":"Empir Softw Eng"},{"key":"10536_CR24","doi-asserted-by":"crossref","unstructured":"Horkoff J (2019) Non-functional requirements for machine learning: Challenges and new directions. In: 2019 IEEE 27th International Requirements Engineering Conference (RE), IEEE, pp 386\u2013391","DOI":"10.1109\/RE.2019.00050"},{"key":"10536_CR25","doi-asserted-by":"publisher","unstructured":"Hove S, Anda B (2005) Experiences from conducting semi-structured interviews in empirical software engineering research. In: 11th IEEE International Software Metrics Symposium (METRICS\u201905), pp 10 pp.\u201323, https:\/\/doi.org\/10.1109\/METRICS.2005.24","DOI":"10.1109\/METRICS.2005.24"},{"key":"10536_CR26","unstructured":"Howard AG, Zhu M, Chen B, Kalenichenko D, Wang W, Weyand T, Andreetto M, Adam H (2017) Mobilenets: Efficient convolutional neural networks for mobile vision applications. arXiv preprint arXiv:1704.04861"},{"key":"10536_CR27","unstructured":"Hudgeon D, Nichol R (2020) Machine learning for business: Using amazon sagemaker and jupyter. URL https:\/\/aws.amazon.com\/sagemaker\/data-wrangler\/"},{"key":"10536_CR28","doi-asserted-by":"crossref","unstructured":"Humbatova N, Jahangirova G, Bavota G, Riccio V, Stocco A, Tonella P (2020) Taxonomy of real faults in deep learning systems. In: Proceedings of the ACM\/IEEE 42nd International Conference on Software Engineering, pp 1110\u20131121","DOI":"10.1145\/3377811.3380395"},{"key":"10536_CR29","unstructured":"Hynes N, Sculley D, Terry M (2017) The data linter: Lightweight, automated sanity checking for ml data sets. In: NIPS MLSys Workshop, vol\u00a01"},{"key":"10536_CR30","doi-asserted-by":"publisher","unstructured":"Jebnoun H, Rahman MS, Khomh F, Muse BA (2022) Clones in deep learning code: What, where, and why? Empirical Softw Engg 27(4),https:\/\/doi.org\/10.1007\/s10664-021-10099-x","DOI":"10.1007\/s10664-021-10099-x"},{"key":"10536_CR31","unstructured":"Krisher T (2022) Us report: Nearly 400 crashes of automated tech vehicles. URL https:\/\/apnews.com\/article\/self-driving-car-crash-data-ae87cadec79966a9ba56e99b4110b8d6"},{"key":"10536_CR32","unstructured":"Labbe M (2021) Energy consumption of ai poses environmental problems: Techtarget. URL https:\/\/www.techtarget.com\/searchenterpriseai\/feature\/Energy-consumption-of-AI-poses-environmental-problems"},{"key":"10536_CR33","unstructured":"Laberge G, A\u00efvodji U, Hara S (2022) Fooling shap with stealthily biased sampling. arXiv preprint arXiv:2205.15419"},{"issue":"1","key":"10536_CR34","doi-asserted-by":"publisher","first-page":"6039","DOI":"10.1038\/s41467-022-33128-9","volume":"13","author":"A Lavin","year":"2022","unstructured":"Lavin A, Gilligan-Lee CM, Visnjic A, Ganju S, Newman D, Ganguly S, Lange D, Baydin AG, Sharma A, Gibson A et al (2022) Technology readiness levels for machine learning systems. Nat Commun 13(1):6039","journal-title":"Nat Commun"},{"key":"10536_CR35","doi-asserted-by":"crossref","unstructured":"Liu Z, Zhou Z, Rekatsinas T (2022) Picket: guarding against corrupted data in tabular data during learning and inference. The VLDB Journal pp 1\u201329","DOI":"10.1007\/s00778-021-00699-w"},{"key":"10536_CR36","unstructured":"Luther D (2022) What are business metrics? 35 metrics businesses need to track. URL https:\/\/www.netsuite.com\/portal\/resource\/articles\/business-strategy\/business-metrics.shtml#:~:text=Business%20metrics%20are%20quantifiable%20measures,businesses%2C%20with%20many%20different%20processes"},{"issue":"4","key":"10536_CR37","doi-asserted-by":"publisher","first-page":"18","DOI":"10.1109\/MC.2023.3243182","volume":"56","author":"W Maalej","year":"2023","unstructured":"Maalej W, Pham YD, Chazette L (2023) Tailoring requirements engineering for responsible ai. Computer 56(4):18\u201327. https:\/\/doi.org\/10.1109\/MC.2023.3243182","journal-title":"Computer"},{"key":"10536_CR38","unstructured":"Mailach A, Siegmund N (2023) Socio-technical anti-patterns in building ml-enabled software. Tech. rep"},{"key":"10536_CR39","doi-asserted-by":"crossref","unstructured":"Marijan D, Gotlieb A, Ahuja MK (2019) Challenges of testing machine learning based systems. In: 2019 IEEE International Conference On Artificial Intelligence Testing (AITest), IEEE, pp 101\u2013102","DOI":"10.1109\/AITest.2019.00010"},{"key":"10536_CR40","unstructured":"Martinez E, Kirchner L (2021) The secret bias hidden in mortgage-approval algorithms - the markup. URL https:\/\/themarkup.org\/denied\/2021\/08\/25\/the-secret-bias-hidden-in-mortgage-approval-algorithms"},{"key":"10536_CR41","doi-asserted-by":"crossref","unstructured":"Merrick L, Taly A (2020) The explanation game: Explaining machine learning models using shapley values. In: Machine Learning and Knowledge Extraction: 4th IFIP TC 5, TC 12, WG 8.4, WG 8.9, WG 12.9 International Cross-Domain Conference, CD-MAKE 2020, Dublin, Ireland, August 25\u201328, 2020, Proceedings 4, Springer, pp 17\u201338","DOI":"10.1007\/978-3-030-57321-8_2"},{"issue":"2","key":"10536_CR42","first-page":"3","volume":"1","author":"N Nahar","year":"2022","unstructured":"Nahar N, Zhou S, Lewis G, K\u00e4stner C (2022) Collaboration challenges in building ml-enabled systems: Communication, documentation, engineering, and process. Organization 1(2):3","journal-title":"Organization"},{"key":"10536_CR43","doi-asserted-by":"crossref","unstructured":"Narayan A, Chami I, Orr L, R\u00e9 C (2022) Can foundation models wrangle your data? arXiv:2205.09911","DOI":"10.14778\/3574245.3574258"},{"key":"10536_CR44","doi-asserted-by":"crossref","unstructured":"Nikanjam A, Khomh F (2021) Design smells in deep learning programs: An empirical study. In: 2021 IEEE International Conference on Software Maintenance and Evolution (ICSME), IEEE, pp 332\u2013342","DOI":"10.1109\/ICSME52107.2021.00036"},{"key":"10536_CR45","unstructured":"Oxford Languages (2023) URL https:\/\/languages.oup.com\/"},{"issue":"6","key":"10536_CR46","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1145\/3533378","volume":"55","author":"A Paleyes","year":"2022","unstructured":"Paleyes A, Urma RG, Lawrence ND (2022) Challenges in deploying machine learning: a survey of case studies. ACM Comput Surv 55(6):1\u201329","journal-title":"ACM Comput Surv"},{"key":"10536_CR47","doi-asserted-by":"publisher","unstructured":"Polyzotis N, Roy S, Whang SE, Zinkevich M (2017) Data management challenges in production machine learning. In: Proceedings of the 2017 ACM International Conference on Management of Data, Association for Computing Machinery, New York, NY, USA, SIGMOD \u201917, p 1723-1726.https:\/\/doi.org\/10.1145\/3035918.3054782","DOI":"10.1145\/3035918.3054782"},{"issue":"2","key":"10536_CR48","doi-asserted-by":"publisher","first-page":"17","DOI":"10.1145\/3299887.3299891","volume":"47","author":"N Polyzotis","year":"2018","unstructured":"Polyzotis N, Roy S, Whang SE, Zinkevich M (2018) Data lifecycle challenges in production machine learning: a survey. ACM SIGMOD Rec 47(2):17\u201328","journal-title":"ACM SIGMOD Rec"},{"key":"10536_CR49","unstructured":"Renggli C, Rimanic L, Kolar L, , Wu W, Zhang C (2022) Ease.ml\/snoopy: Towards automatic feasibility studies for ml via quantitative understanding of \"data quality for ml\". arXiv preprint arXiv:2010.08410"},{"key":"10536_CR50","unstructured":"Rezig EK, Cao L, Simonini G, Schoemans M, Madden S, Tang N, Ouzzani M, Stonebraker M (2020) Dagger: a data (not code) debugger. In: CIDR 2020, 10th Conference on Innovative Data Systems Research, Amsterdam, The Netherlands, January 12-15, 2020, Online Proceedings"},{"key":"10536_CR51","doi-asserted-by":"crossref","unstructured":"Ribeiro MT, Singh S, Guestrin C (2016) \" why should i trust you?\" explaining the predictions of any classifier. In: Proceedings of the 22nd ACM SIGKDD international conference on knowledge discovery and data mining, pp 1135\u20131144","DOI":"10.1145\/2939672.2939778"},{"key":"10536_CR52","doi-asserted-by":"publisher","unstructured":"Rudin C (2018) Stop explaining black box machine learning models for high stakes decisions and use interpretable models instead. arXiv preprint arXiv:1811.10154https:\/\/doi.org\/10.48550\/ARXIV.1811.10154, URL https:\/\/arxiv.org\/abs\/1811.10154","DOI":"10.48550\/ARXIV.1811.10154"},{"key":"10536_CR53","doi-asserted-by":"crossref","unstructured":"Sambasivan N, Kapania S, Highfill H, Akrong D, Paritosh P, Aroyo LM (2021) \u201ceveryone wants to do the model work, not the data work\u201d: Data cascades in high-stakes ai. In: proceedings of the 2021 CHI Conference on Human Factors in Computing Systems, pp 1\u201315","DOI":"10.1145\/3411764.3445518"},{"key":"10536_CR54","unstructured":"Sato D, Wider A, Windheuser C (2019) Continuous delivery for machine learning. URL https:\/\/martinfowler.com\/articles\/cd4ml.html"},{"key":"10536_CR55","unstructured":"Schelter S, Biessmann F, Januschowski T, Salinas D, Seufert S, Szarvas G (2015) On challenges in machine learning model management. Tech. rep"},{"key":"10536_CR56","unstructured":"Sculley D, Holt G, Golovin D, Davydov E, Phillips T, Ebner D, Chaudhary V, Young M, Crespo JF, Dennison D (2015) Hidden technical debt in machine learning systems. In: Cortes C, Lawrence N, Lee D, Sugiyama M, Garnett R (eds) Advances in Neural Information Processing Systems, Curran Associates, Inc., vol\u00a028, URL https:\/\/proceedings.neurips.cc\/paper\/2015\/file\/86df7dcfd896fcaf2674f757a2463eba-Paper.pdf"},{"issue":"4","key":"10536_CR57","doi-asserted-by":"publisher","first-page":"557","DOI":"10.1109\/32.799955","volume":"25","author":"CB Seaman","year":"1999","unstructured":"Seaman CB (1999) Qualitative methods in empirical studies of software engineering. IEEE Trans Software Eng 25(4):557\u2013572","journal-title":"IEEE Trans Software Eng"},{"key":"10536_CR58","unstructured":"Serban A, Visser J (2021) An empirical study of software architecture for machine learning. arXiv:2105.12422"},{"key":"10536_CR59","unstructured":"Shankar S, Garcia R, Hellerstein JM, Parameswaran AG (2022) Operationalizing machine learning: An interview study. arXiv:2209.09125"},{"key":"10536_CR60","unstructured":"Shoeybi M, Patwary M, Puri R, LeGresley P, Casper J, Catanzaro B (2019) Megatron-lm: Training multi-billion parameter language models using model parallelism. arXiv:1909.08053"},{"issue":"2","key":"10536_CR61","doi-asserted-by":"publisher","first-page":"307","DOI":"10.1007\/s11219-021-09557-y","volume":"30","author":"J Siebert","year":"2022","unstructured":"Siebert J, Joeckel L, Heidrich J, Trendowicz A, Nakamichi K, Ohashi K, Namba I, Yamamoto R, Aoyama M (2022) Construction of a quality model for machine learning systems. Software Qual J 30(2):307\u2013335","journal-title":"Software Qual J"},{"key":"10536_CR62","doi-asserted-by":"crossref","unstructured":"Steidl M, Felderer M, Ramler R (2023) The pipeline for the continuous development of artificial intelligence models-current state of research and practice. Journal of Systems and Software p 111615","DOI":"10.1016\/j.jss.2023.111615"},{"key":"10536_CR63","unstructured":"Strauss A, Corbin J (1994) Grounded theory methodology: An overview"},{"key":"10536_CR64","unstructured":"Strauss A, Corbin J, Corbin J (1998) Basics of Qualitative Research: Techniques and Procedures for Developing Grounded Theory. SAGE Publications, URL https:\/\/books.google.ca\/books?id=wTwYUnHYsmMC"},{"issue":"2","key":"10536_CR65","doi-asserted-by":"publisher","first-page":"392","DOI":"10.3390\/make3020020","volume":"3","author":"S Studer","year":"2021","unstructured":"Studer S, Bui TB, Drescher C, Hanuschkin A, Winkler L, Peters S, M\u00fcller KR (2021) Towards crisp-ml (q): a machine learning process model with quality assurance methodology. Machine Learning and Knowledge Extraction 3(2):392\u2013413","journal-title":"Machine Learning and Knowledge Extraction"},{"key":"10536_CR66","unstructured":"Tannor P (2023) Data drift vs. concept drift. URL https:\/\/deepchecks.com\/data-drift-vs-concept-drift-what-are-the-main-differences\/"},{"key":"10536_CR67","doi-asserted-by":"crossref","unstructured":"Van Oort B, Cruz L, Aniche M, van Deursen A (2021) The prevalence of code smells in machine learning projects. In: 2021 IEEE\/ACM 1st Workshop on AI Engineering-Software Engineering for AI (WAIN), IEEE, pp 1\u20138","DOI":"10.1109\/WAIN52551.2021.00011"},{"key":"10536_CR68","doi-asserted-by":"crossref","unstructured":"Vartak M, Rahman S, Madden S, Parameswaran A, Polyzotis N (2015) Seedb: Efficient data-driven visualization recommendations to support visual analytics. In: Proceedings of the VLDB Endowment International Conference on Very Large Data Bases, NIH Public Access, vol\u00a08, p 2182","DOI":"10.14778\/2831360.2831371"},{"key":"10536_CR69","doi-asserted-by":"crossref","unstructured":"Vogelsang A, Borg M (2019) Requirements engineering for machine learning: Perspectives from data scientists. In: 2019 IEEE 27th International Requirements Engineering Conference Workshops (REW), IEEE, pp 245\u2013251","DOI":"10.1109\/REW.2019.00050"},{"key":"10536_CR70","doi-asserted-by":"crossref","unstructured":"Washizaki H, Uchida H, Khomh F, Gu\u00e9h\u00e9neuc YG (2019) Studying software engineering patterns for designing machine learning systems. In: 2019 10th International Workshop on Empirical Software Engineering in Practice (IWESEP), IEEE, pp 49\u2013495","DOI":"10.1109\/IWESEP49350.2019.00017"},{"key":"10536_CR71","unstructured":"Whang SE, Roh Y, Song H, Lee JG (2021) Data collection and quality challenges in deep learning: A data-centric ai perspective. arXiv:2112.06409"},{"issue":"2","key":"10536_CR72","doi-asserted-by":"publisher","first-page":"43","DOI":"10.1007\/s10664-022-10258-8","volume":"28","author":"N Yang","year":"2023","unstructured":"Yang N, Cuijpers P, Hendriks D, Schiffelers R, Lukkien J, Serebrenik A (2023) An interview study about the use of logs in embedded software engineering. Empir Softw Eng 28(2):43","journal-title":"Empir Softw Eng"},{"key":"10536_CR73","unstructured":"Zhang JM, Harman M, Ma L, Liu Y (2020) Machine learning testing: Survey, landscapes and horizons. IEEE Trans Softw Eng"}],"container-title":["Empirical Software Engineering"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10664-024-10536-7.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s10664-024-10536-7\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10664-024-10536-7.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,11,6]],"date-time":"2024-11-06T04:52:01Z","timestamp":1730868721000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s10664-024-10536-7"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,9,11]]},"references-count":73,"journal-issue":{"issue":"6","published-print":{"date-parts":[[2024,11]]}},"alternative-id":["10536"],"URL":"https:\/\/doi.org\/10.1007\/s10664-024-10536-7","relation":{},"ISSN":["1382-3256","1573-7616"],"issn-type":[{"value":"1382-3256","type":"print"},{"value":"1573-7616","type":"electronic"}],"subject":[],"published":{"date-parts":[[2024,9,11]]},"assertion":[{"value":"30 July 2024","order":1,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"11 September 2024","order":2,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Declarations"}},{"value":"The authors declare that they have no conflict of interest.","order":2,"name":"Ethics","group":{"name":"EthicsHeading","label":"Conflicts of interest"}}],"article-number":"149"}}