{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,16]],"date-time":"2026-04-16T03:37:56Z","timestamp":1776310676666,"version":"3.50.1"},"reference-count":105,"publisher":"Springer Science and Business Media LLC","issue":"3","license":[{"start":{"date-parts":[[2023,4,5]],"date-time":"2023-04-05T00:00:00Z","timestamp":1680652800000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2023,4,5]],"date-time":"2023-04-05T00:00:00Z","timestamp":1680652800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"funder":[{"name":"DEpendable Explainable Learning (DEEL) Project"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Empir Software Eng"],"published-print":{"date-parts":[[2023,5]]},"DOI":"10.1007\/s10664-023-10291-1","type":"journal-article","created":{"date-parts":[[2023,4,5]],"date-time":"2023-04-05T15:08:21Z","timestamp":1680707301000},"update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":21,"title":["Bugs in machine learning-based systems: a faultload benchmark"],"prefix":"10.1007","volume":"28","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-7942-4791","authenticated-orcid":false,"given":"Mohammad Mehdi","family":"Morovati","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Amin","family":"Nikanjam","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Foutse","family":"Khomh","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Zhen Ming","family":"Jiang","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2023,4,5]]},"reference":[{"key":"10291_CR1","unstructured":"Abadi M, Barham P, Chen J, Chen Z, Davis A, Dean J, Devin M, Ghemawat S, Irving G, Isard M et al (2016) Tensorflow: a system for large-scale machine learning. In: 12th {USENIX} symposium on operating systems design and implementation ({OSDI} 16). Savannah, USENIX, pp 265\u2013283"},{"key":"10291_CR2","doi-asserted-by":"crossref","unstructured":"Abidi M, Grichi M, Khomh F, Gu\u00e9h\u00e9neuc Y G (2019a) Code smells for multi-language systems. In: Proceedings of the 24th European conference on pattern languages of programs, pp 1\u201313","DOI":"10.1145\/3361149.3361161"},{"key":"10291_CR3","doi-asserted-by":"crossref","unstructured":"Abidi M, Khomh F, Gu\u00e9h\u00e9neuc Y G (2019b) Anti-patterns for multi-language systems. In: Proceedings of the 24th European conference on pattern languages of programs, pp 1\u201314","DOI":"10.1145\/3361149.3364227"},{"issue":"3","key":"10291_CR4","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1145\/3432690","volume":"30","author":"M Abidi","year":"2021","unstructured":"Abidi M, Rahman M S, Openja M, Khomh F (2021) Are multi-language design smells fault-prone? An empirical study. ACM Trans Softw Eng Methodol (TOSEM) 30(3):1\u201356","journal-title":"ACM Trans Softw Eng Methodol (TOSEM)"},{"key":"10291_CR5","unstructured":"Al-Rfou R, Alain G, Almahairi A, Angermueller C, Bahdanau D, Ballas N, Bastien F, Bayer J, Belikov A, Belopolsky A et al (2016) Theano: a python framework for fast computation of mathematical expressions. arXiv e-prints pp arXiv\u20131605"},{"key":"10291_CR6","doi-asserted-by":"crossref","unstructured":"Amershi S, Begel A, Bird C, DeLine R, Gall H, Kamar E, Nagappan N, Nushi B, Zimmermann T (2019) Software engineering for machine learning: a case study. In: 2019 IEEE\/ACM 41st international conference on software engineering: Software engineering in practice (ICSE-SEIP). IEEE, pp 291\u2013300","DOI":"10.1109\/ICSE-SEIP.2019.00042"},{"issue":"3","key":"10291_CR7","first-page":"671","volume":"104","author":"S Barocas","year":"2016","unstructured":"Barocas S, Selbst AD (2016) Big data\u2019s disparate impact. Calif Law Rev 104(3):671\u2013732. http:\/\/www.jstor.org\/stable\/24758720. Accessed 11 Jan 2022","journal-title":"Calif Law Rev"},{"key":"10291_CR8","doi-asserted-by":"crossref","unstructured":"Borg M (2021) The aiq meta-testbed: pragmatically bridging academic ai testing and industrial q needs. In: International conference on software quality. Springer, pp 66\u201377","DOI":"10.1007\/978-3-030-65854-0_6"},{"issue":"6","key":"10291_CR9","doi-asserted-by":"publisher","first-page":"35","DOI":"10.1109\/52.805471","volume":"16","author":"P Bourque","year":"1999","unstructured":"Bourque P, Dupuis R, Abran A, Moore J W, Tripp L (1999) The guide to the software engineering body of knowledge. IEEE Softw 16(6):35\u201344","journal-title":"IEEE Softw"},{"key":"10291_CR10","unstructured":"Brownlee J (2020) Use early stopping to halt the training of neural networks at the right time. https:\/\/machinelearningmastery.com\/how-to-stop-training-deep-neural-networks-at-the-right-time-using-early-stopping\/. Accessed: 2022-12-29"},{"key":"10291_CR11","unstructured":"Chollet F et al (2018) Keras: the python deep learning library. Astrophysics Source Code Library, pp ascl\u20131806"},{"key":"10291_CR12","unstructured":"Chouldechova A, Roth A (2018) The frontiers of fairness in machine learning. arXiv:1810.08810"},{"key":"10291_CR13","unstructured":"Collobert R, Bengio S, Mari\u00e9thoz J (2002) Torch: a modular machine learning software library. Tech. rep. Idiap"},{"key":"10291_CR14","unstructured":"Developer guideline documentation G (2021) Github rest api. https:\/\/developer.github.com\/v3\/. Accessed: 2021-7-27"},{"key":"10291_CR15","doi-asserted-by":"crossref","unstructured":"Dwork C (2008) Differential privacy: a survey of results. In: International conference on theory and applications of models of computation. Springer, pp 1\u201319","DOI":"10.1007\/978-3-540-79228-4_1"},{"issue":"1","key":"10291_CR16","doi-asserted-by":"publisher","first-page":"24","DOI":"10.1038\/s41591-018-0316-z","volume":"25","author":"A Esteva","year":"2019","unstructured":"Esteva A, Robicquet A, Ramsundar B, Kuleshov V, DePristo M, Chou K, Cui C, Corrado G, Thrun S, Dean J (2019) A guide to deep learning in healthcare. Nat Med 25(1):24\u201329","journal-title":"Nat Med"},{"key":"10291_CR17","doi-asserted-by":"crossref","unstructured":"Felderer M, Ramler R (2021) Quality assurance for ai-based systems: overview and challenges (introduction to interactive session). In: International conference on software quality. Springer, pp 33\u201342","DOI":"10.1007\/978-3-030-65854-0_3"},{"key":"10291_CR18","volume-title":"Software quality assurance: from theory to implementation","author":"D Galin","year":"2004","unstructured":"Galin D (2004) Software quality assurance: from theory to implementation. Pearson Education, England"},{"key":"10291_CR19","unstructured":"GitHub (2021) Github official website. https:\/\/github.com\/about. Accessed: 2021-7-27"},{"key":"10291_CR20","unstructured":"Gupta S (2021) What is the best language for machine learning? https:\/\/www.springboard.com\/blog\/data-science\/best-language-for-machine-learning. Accessed: 2021-10-06"},{"issue":"1","key":"10291_CR21","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1021\/ci0342472","volume":"44","author":"DM Hawkins","year":"2004","unstructured":"Hawkins D M (2004) The problem of overfitting. J Chem Inf Comput 44(1):1\u201312","journal-title":"J Chem Inf Comput"},{"key":"10291_CR22","unstructured":"https:\/\/github.com\/dpressel\/baseline\/commit\/4dad463 (2016). Accessed: 2021-11-01"},{"key":"10291_CR23","unstructured":"https:\/\/stackoverflow.com\/questions\/34311586 (2016). Accessed: 2021-11-01"},{"key":"10291_CR24","unstructured":"https:\/\/stackoverflow.com\/questions\/38080035 (2017). Accessed: 2021-11-01"},{"key":"10291_CR25","unstructured":"https:\/\/stackoverflow.com\/questions\/42264649 (2017). Accessed: 2021-11-01"},{"key":"10291_CR26","unstructured":"https:\/\/github.com\/suchaoxiao\/keras-frcnn_modify\/commit\/2f51f68 (2017). Accessed: 2021-11-01"},{"key":"10291_CR27","unstructured":"https:\/\/github.com\/albu\/albumentations\/commit\/fec1f3b (2018). Accessed: 2021-11-01"},{"key":"10291_CR28","unstructured":"https:\/\/github.com\/vmelan\/cifar-experiment\/commit\/561c82e (2018). Accessed: 2022-06-01"},{"key":"10291_CR29","unstructured":"https:\/\/stackoverflow.com\/questions\/53119432 (2018). Accessed: 2021-11-01"},{"key":"10291_CR30","unstructured":"https:\/\/github.com\/acflorea\/keras-playground\/commit\/d44c90c (2018). Accessed: 2022-06-01"},{"key":"10291_CR31","unstructured":"https:\/\/github.com\/keras-team\/keras-tuner\/commit\/3758611 (2018). Accessed: 2022-06-01"},{"key":"10291_CR32","unstructured":"https:\/\/github.com\/hunkim\/DeepLearningZeroToAll\/commit\/9f8fb94 (2018). Accessed: 2022-06-01"},{"key":"10291_CR33","unstructured":"https:\/\/stackoverflow.com\/questions\/44924690 (2018). Accessed: 2021-11-01"},{"key":"10291_CR34","unstructured":"https:\/\/stackoverflow.com\/questions\/58636087 (2018). Accessed: 2021-11-01"},{"key":"10291_CR35","unstructured":"https:\/\/stackoverflow.com\/questions\/50079585 (2018). Accessed: 2021-11-01"},{"key":"10291_CR36","unstructured":"https:\/\/github.com\/PhilippeNguyen\/kinopt\/commit\/fdee16f (2018). Accessed: 2021-11-01"},{"key":"10291_CR37","unstructured":"https:\/\/stackoverflow.com\/questions\/56103207 (2019). Accessed: 2021-11-01"},{"key":"10291_CR38","unstructured":"https:\/\/github.com\/vaclavcadek\/keras2pmml\/commit\/4795ec6 (2019). Accessed: 2021-11-01"},{"key":"10291_CR39","doi-asserted-by":"crossref","unstructured":"Humbatova N, Jahangirova G, Bavota G, Riccio V, Stocco A, Tonella P (2020) Taxonomy of real faults in deep learning systems. In: Proceedings of the ACM\/IEEE 42nd international conference on software engineering, pp 1110\u20131121","DOI":"10.1145\/3377811.3380395"},{"key":"10291_CR40","doi-asserted-by":"crossref","unstructured":"Huppler K (2009) The art of building a good benchmark. In: Technology conference on performance evaluation and benchmarking. Springer, pp 18\u201330","DOI":"10.1007\/978-3-642-10424-4_3"},{"key":"10291_CR41","doi-asserted-by":"publisher","unstructured":"IEEE standard for system, software, and hardware verification and validation (2017). IEEE Std 1012-2016 (Revision of IEEE Std 1012-2012\/ Incorporates IEEE Std 1012-2016\/Cor1-2017), pp 1\u2013260. https:\/\/doi.org\/10.1109\/IEEESTD.2017.8055462","DOI":"10.1109\/IEEESTD.2017.8055462"},{"key":"10291_CR42","doi-asserted-by":"publisher","unstructured":"IEEE standard glossary of software engineering terminology (1990). IEEE Std 610.12-1990, pp 1\u201384. https:\/\/doi.org\/10.1109\/IEEESTD.1990.101064","DOI":"10.1109\/IEEESTD.1990.101064"},{"key":"10291_CR43","doi-asserted-by":"publisher","unstructured":"ISO\/IEC\/IEEE international standard\u2014systems and software engineering\u2014vocabulary (2010). ISO\/IEC\/IEEE 24765:2010(E), pp 1\u2013418. https:\/\/doi.org\/10.1109\/IEEESTD.2010.5733835","DOI":"10.1109\/IEEESTD.2010.5733835"},{"key":"10291_CR44","doi-asserted-by":"crossref","unstructured":"Islam M J, Nguyen G, Pan R, Rajan H (2019) A comprehensive study on deep learning bug characteristics. In: Proceedings of the 2019 27th ACM joint meeting on European software engineering conference and symposium on the foundations of software engineering, pp 510\u2013520","DOI":"10.1145\/3338906.3338955"},{"key":"10291_CR45","doi-asserted-by":"crossref","unstructured":"Islam M J, Pan R, Nguyen G, Rajan H (2020) Repairing deep neural networks: fix patterns and challenges. In: 2020 IEEE\/ACM 42nd international conference on software engineering (ICSE). IEEE, pp 1135\u20131146","DOI":"10.1145\/3377811.3380378"},{"key":"10291_CR46","doi-asserted-by":"crossref","unstructured":"Jia L, Zhong H, Huang L (2021a) The unit test quality of deep learning libraries: a mutation analysis. In: 2021 IEEE International conference on software maintenance and evolution (ICSME). IEEE, pp 47\u201357","DOI":"10.1109\/ICSME52107.2021.00011"},{"key":"10291_CR47","doi-asserted-by":"crossref","unstructured":"Jia L, Zhong H, Wang X, Huang L, Lu X (2021b) The symptoms, causes, and repairs of bugs inside a deep learning library. J Syst Softw 177:110935","DOI":"10.1016\/j.jss.2021.110935"},{"key":"10291_CR48","doi-asserted-by":"crossref","unstructured":"Jia L, Zhong H, Wang X, Huang L, Li Z (2022) How do injected bugs affect deep learning?. In: 2022 IEEE International conference on software analysis, evolution and reengineering (SANER). IEEE, pp 793\u2013804","DOI":"10.1109\/SANER53432.2022.00097"},{"key":"10291_CR49","doi-asserted-by":"crossref","unstructured":"Jiang Y, Liu H, Niu N, Zhang L, Hu Y (2021) Extracting concise bug-fixing patches from human-written patches in version control systems. In: 2021 IEEE\/ACM 43rd international conference on software engineering (ICSE). IEEE, pp 686\u2013698","DOI":"10.1109\/ICSE43902.2021.00069"},{"key":"10291_CR50","doi-asserted-by":"crossref","unstructured":"Just R, Jalali D, Ernst M D (2014) Defects4j: a database of existing faults to enable controlled testing studies for java programs. In: Proceedings of the 2014 international symposium on software testing and analysis, pp 437\u2013440","DOI":"10.1145\/2610384.2628055"},{"key":"10291_CR51","unstructured":"Keras (2016) Keras 2.1.5. https:\/\/github.com\/keras-team\/keras\/releases\/tag\/2.1.5. Accessed: 2021-11-01"},{"key":"10291_CR52","doi-asserted-by":"crossref","unstructured":"Kim M, Kim Y, Lee E (2021) Denchmark: a bug benchmark of deep learning-related software. In: 2021 IEEE\/ACM 18th international conference on mining software repositories (MSR). IEEE, pp 540\u2013544","DOI":"10.1109\/MSR52588.2021.00070"},{"key":"10291_CR53","unstructured":"Kirk M (2014) Thoughtful machine learning: a test-driven approach. O\u2019Reilly Media, Inc."},{"key":"10291_CR54","doi-asserted-by":"crossref","unstructured":"Kistowski JV, Arnold JA, Huppler K, Lange KD, Henning JL, Cao P (2015) How to build a benchmark. In: Proceedings of the 6th ACM\/SPEC international conference on performance engineering, pp 333\u2013336","DOI":"10.1145\/2668930.2688819"},{"key":"10291_CR55","unstructured":"Krizhevsky A, Hinton G et al (2009) Learning multiple layers of features from tiny images"},{"issue":"12","key":"10291_CR56","doi-asserted-by":"publisher","first-page":"1236","DOI":"10.1109\/TSE.2015.2454513","volume":"41","author":"C Le Goues","year":"2015","unstructured":"Le Goues C, Holtschulte N, Smith E K, Brun Y, Devanbu P, Forrest S, Weimer W (2015) The manybugs and introclass benchmarks for automated repair of c programs. IEEE Trans Softw Eng 41(12):1236\u20131256","journal-title":"IEEE Trans Softw Eng"},{"issue":"11","key":"10291_CR57","doi-asserted-by":"publisher","first-page":"2278","DOI":"10.1109\/5.726791","volume":"86","author":"Y LeCun","year":"1998","unstructured":"LeCun Y, Bottou L, Bengio Y, Haffner P (1998) Gradient-based learning applied to document recognition. Proc IEEE 86(11):2278\u20132324","journal-title":"Proc IEEE"},{"key":"10291_CR58","doi-asserted-by":"crossref","unstructured":"Lenarduzzi V, Lomio F, Moreschini S, Taibi D, Tamburri D A (2021) Software quality for ai: where we are now?. In: International conference on software quality. Springer, pp 43\u201353","DOI":"10.1007\/978-3-030-65854-0_4"},{"key":"10291_CR59","doi-asserted-by":"crossref","unstructured":"Lin Z, Marinov D, Zhong H, Chen Y, Zhao J (2015) Jacontebe: a benchmark suite of real-world java concurrency bugs (t). In: 2015 30th IEEE\/ACM international conference on automated software engineering (ASE). IEEE, pp 178\u2013189","DOI":"10.1109\/ASE.2015.87"},{"issue":"3","key":"10291_CR60","doi-asserted-by":"publisher","first-page":"31","DOI":"10.1145\/3236386.3241340","volume":"16","author":"ZC Lipton","year":"2018","unstructured":"Lipton Z C (2018) The mythos of model interpretability: in machine learning, the concept of interpretability is both important and slippery. Queue 16 (3):31\u201357","journal-title":"Queue"},{"key":"10291_CR61","doi-asserted-by":"publisher","first-page":"4566","DOI":"10.1109\/ACCESS.2020.3045078","volume":"9","author":"X Liu","year":"2020","unstructured":"Liu X, Xie L, Wang Y, Zou J, Xiong J, Ying Z, Vasilakos A V (2020) Privacy and security issues in deep learning: a survey. IEEE Access 9:4566\u20134593","journal-title":"IEEE Access"},{"key":"10291_CR62","unstructured":"Lu S, Li Z, Qin F, Tan L, Zhou P, Zhou Y (2005) Bugbench: benchmarks for evaluating bug detection tools. In: Workshop on the evaluation of software defect detection tools, vol 5. Chicago"},{"key":"10291_CR63","doi-asserted-by":"crossref","unstructured":"Lyu M R (2007) Software reliability engineering: a roadmap. In: Future of software engineering (FOSE\u201907). IEEE, Minneapolis, pp 153\u2013170","DOI":"10.1109\/FOSE.2007.24"},{"key":"10291_CR64","doi-asserted-by":"crossref","unstructured":"Ma L, Juefei-Xu F, Zhang F, Sun J, Xue M, Li B, Chen C, Su T, Li L, Liu Y et al (2018) Deepgauge: multi-granularity testing criteria for deep learning systems. In: Proceedings of the 33rd ACM\/IEEE international conference on automated software engineering. Association for Computing Machinery (ACM), New York, pp 120\u2013131","DOI":"10.1145\/3238147.3238202"},{"key":"10291_CR65","doi-asserted-by":"crossref","unstructured":"Madeiral F, Urli S, Maia M, Monperrus M (2019) Bears: an extensible java bug benchmark for automatic program repair studies. In: 2019 IEEE 26th international conference on software analysis, evolution and reengineering (SANER). IEEE, pp 468\u2013478","DOI":"10.1109\/SANER.2019.8667991"},{"key":"10291_CR66","doi-asserted-by":"crossref","unstructured":"Marijan D, Gotlieb A, Ahuja M K (2019) Challenges of testing machine learning based systems. In: 2019 IEEE International conference on artificial intelligence testing (AITest). IEEE, pp 101\u2013102","DOI":"10.1109\/AITest.2019.00010"},{"key":"10291_CR67","doi-asserted-by":"crossref","unstructured":"Mart\u00ednez-Fern\u00e1ndez S, Bogner J, Franch X, Oriol M, Siebert J, Trendowicz A, Vollmer AM, Wagner S (2021) Software engineering for ai-based systems: a survey. arXiv:2105.01984","DOI":"10.1145\/3487043"},{"issue":"CSCW","key":"10291_CR68","first-page":"1","volume":"3","author":"N McDonald","year":"2019","unstructured":"McDonald N, Schoenebeck S, Forte A (2019) Reliability and inter-rater reliability in qualitative research: Norms and guidelines for cscw and hci practice. Proc ACM on Human-Comput Interact 3(CSCW):1\u201323","journal-title":"Proc ACM on Human-Comput Interact"},{"issue":"3","key":"10291_CR69","doi-asserted-by":"publisher","first-page":"276","DOI":"10.11613\/BM.2012.031","volume":"22","author":"ML McHugh","year":"2012","unstructured":"McHugh M L (2012) Interrater reliability: the kappa statistic. Biochemia Medica 22(3):276\u2013282","journal-title":"Biochemia Medica"},{"key":"10291_CR70","doi-asserted-by":"crossref","unstructured":"Nejadgholi M, Yang J (2019) A study of oracle approximations in testing deep learning libraries. In: 2019 34th IEEE\/ACM international conference on automated software engineering (ASE). IEEE, pp 785\u2013796","DOI":"10.1109\/ASE.2019.00078"},{"key":"10291_CR71","doi-asserted-by":"crossref","unstructured":"Nikanjam A, Khomh F (2021) Design smells in deep learning programs: an empirical study. In: 2021 IEEE International conference on software maintenance and evolution (ICSME), pp 332\u2013342","DOI":"10.1109\/ICSME52107.2021.00036"},{"key":"10291_CR72","doi-asserted-by":"publisher","unstructured":"Nikanjam A, Braiek H B, Morovati M M, Khomh F (2021a) Automatic fault detection for deep learning programs using graph transformations. ACM Trans Softw Eng Methodol 31(1). https:\/\/doi.org\/10.1145\/3470006","DOI":"10.1145\/3470006"},{"key":"10291_CR73","doi-asserted-by":"crossref","unstructured":"Nikanjam A, Morovati M M, Khomh F, Braiek H B (2021b) Faults in deep reinforcement learning programs: a taxonomy and a detection approach. arXiv:2101.00135","DOI":"10.1007\/s10515-021-00313-x"},{"key":"10291_CR74","unstructured":"Organisation T (2021) Torch official github repository. https:\/\/github.com\/torch\/torch7. Accessed: 2021-9-1"},{"key":"10291_CR75","unstructured":"Paszke A, Gross S, Massa F, Lerer A, Bradbury J, Chanan G, Killeen T, Lin Z, Gimelshein N, Antiga L et al (2019) Pytorch: an imperative style, high-performance deep learning library. arXiv:1912.01703"},{"key":"10291_CR76","doi-asserted-by":"crossref","unstructured":"Pei K, Cao Y, Yang J, Jana S (2017) Deepxplore: automated whitebox testing of deep learning systems. In: Proceedings of the 26th symposium on operating systems principles. Association for Computing Machinery (ACM), New York, pp 1\u201318","DOI":"10.1145\/3132747.3132785"},{"key":"10291_CR77","doi-asserted-by":"publisher","unstructured":"Pham H V, Qian S, Wang J, Lutellier T, Rosenthal J, Tan L, Yu Y, Nagappan N (2021) Problems and opportunities in training deep learning software systems: an analysis of variance. In: Proceedings of the 35th IEEE\/ACM international conference on automated software engineering, ASE \u201920. Association for Computing Machinery, New York, pp 771\u2013783. https:\/\/doi.org\/10.1145\/3324884.3416545","DOI":"10.1145\/3324884.3416545"},{"key":"10291_CR78","unstructured":"Pressman R S (2005) Software engineering: a practitioner\u2019s approach. Palgrave Macmillan"},{"issue":"8","key":"10291_CR79","doi-asserted-by":"publisher","first-page":"1397","DOI":"10.1016\/j.infsof.2013.02.009","volume":"55","author":"D Radjenovi\u0107","year":"2013","unstructured":"Radjenovi\u0107 D, Heri\u010dko M, Torkar R, \u017eivkovi\u010d A (2013) Software fault prediction metrics: a systematic literature review. Inf Softw Technol 55(8):1397\u20131418","journal-title":"Inf Softw Technol"},{"issue":"6","key":"10291_CR80","doi-asserted-by":"publisher","first-page":"5193","DOI":"10.1007\/s10664-020-09881-0","volume":"25","author":"V Riccio","year":"2020","unstructured":"Riccio V, Jahangirova G, Stocco A, Humbatova N, Weiss M, Tonella P (2020) Testing machine learning based systems: a systematic mapping. Empir Softw Eng 25(6):5193\u20135254","journal-title":"Empir Softw Eng"},{"key":"10291_CR81","unstructured":"Rice L, Wong E, Kolter Z (2020) Overfitting in adversarially robust deep learning. In: International conference on machine learning. PMLR, pp 8093\u20138104"},{"key":"10291_CR82","doi-asserted-by":"crossref","unstructured":"Rivera-Landos E, Khomh F, Nikanjam A (2021) The challenge of reproducible ml: an empirical study on the impact of bugs","DOI":"10.1109\/QRS54544.2021.00116"},{"key":"10291_CR83","unstructured":"Road vehicles\u2014safety of the intended functionality. Standard (2019). https:\/\/www.iso.org\/standard\/70939.html. Accessed 11 Jan 2022"},{"key":"10291_CR84","doi-asserted-by":"publisher","first-page":"164","DOI":"10.1016\/j.infsof.2018.03.009","volume":"99","author":"G Rodr\u00edguez-P\u00e9rez","year":"2018","unstructured":"Rodr\u00edguez-P\u00e9rez G, Robles G, Gonz\u00e1lez-Barahona JM (2018) Reproducibility and credibility in empirical software engineering: a case study based on a systematic literature review of the use of the szz algorithm. Inf Softw Technol 99:164\u2013176","journal-title":"Inf Softw Technol"},{"key":"10291_CR85","doi-asserted-by":"crossref","unstructured":"Schoop E, Huang F, Hartmann B (2021) Umlaut: debugging deep learning programs using program structure and model behavior. In: Proceedings of the 2021 CHI conference on human factors in computing systems, pp 1\u201316","DOI":"10.1145\/3411764.3445538"},{"key":"10291_CR86","first-page":"2503","volume":"28","author":"D Sculley","year":"2015","unstructured":"Sculley D, Holt G, Golovin D, Davydov E, Phillips T, Ebner D, Chaudhary V, Young M, Crespo J F, Dennison D (2015) Hidden technical debt in machine learning systems. Adv Neural Inf Process Syst 28:2503\u20132511","journal-title":"Adv Neural Inf Process Syst"},{"key":"10291_CR87","doi-asserted-by":"crossref","unstructured":"Shen Q, Ma H, Chen J, Tian Y, Cheung S C, Chen X (2021) A comprehensive study of deep learning compiler bugs. In: Proceedings of the 29th ACM joint meeting on European software engineering conference and symposium on the foundations of software engineering, pp 968\u2013980","DOI":"10.1145\/3468264.3468591"},{"key":"10291_CR88","doi-asserted-by":"publisher","unstructured":"Spadini D, Aniche M, Bacchelli A (2018) PyDriller: python framework for mining software repositories. In: Proceedings of the 2018 26th ACM joint meeting on European software engineering conference and symposium on the foundations of software engineering\u2014ESEC\/FSE 2018. ACM Press, New York, pp 908\u2013911. https:\/\/doi.org\/10.1145\/3236024.3264598","DOI":"10.1145\/3236024.3264598"},{"key":"10291_CR89","unstructured":"StackOverflow: Stack overflow annual developer survey. https:\/\/insights.stackoverflow.com\/survey\/2021 (2021). Accessed: 2022-04-01"},{"key":"10291_CR90","doi-asserted-by":"crossref","unstructured":"Tambon F, Nikanjam A, An L, Khomh F, Antoniol G (2021) Silent bugs in deep learning frameworks: an empirical study of keras and tensorflow","DOI":"10.26226\/morressier.613b5418842293c031b5b61d"},{"key":"10291_CR91","doi-asserted-by":"crossref","unstructured":"Tian Y, Pei K, Jana S, Ray B (2018) Deeptest: automated testing of deep-neural-network-driven autonomous cars. In: Proceedings of the 40th international conference on software engineering, pp 303\u2013314","DOI":"10.1145\/3180155.3180220"},{"key":"10291_CR92","doi-asserted-by":"crossref","unstructured":"Vieira M, Madeira H, Sachs K, Kounev S (2012) Resilience benchmarking. In: Resilience assessment and evaluation of computing systems. Springer, pp 283\u2013301","DOI":"10.1007\/978-3-642-29032-9_14"},{"key":"10291_CR93","unstructured":"Voskoglou C (2017) What is the best programming language for machine learning. https:\/\/towardsdatascience.com\/what-is-the-best-programming-language-for-machine-learning-a745c156d6b7. Accessed: 2021-10-06"},{"key":"10291_CR94","doi-asserted-by":"crossref","unstructured":"Wardat M, Le W, Rajan H (2021) Deeplocalize: fault localization for deep neural networks. In: 2021 IEEE\/ACM 43rd international conference on software engineering (ICSE). IEEE, pp 251\u2013262","DOI":"10.1109\/ICSE43902.2021.00034"},{"key":"10291_CR95","doi-asserted-by":"crossref","unstructured":"Wardat M, Cruz B D, Le W, Rajan H (2022) Deepdiagnosis: automatically diagnosing faults and recommending actionable fixes in deep learning programs. In: Proceedings of the 44th international conference on software engineering, pp 561\u2013572","DOI":"10.1145\/3510003.3510071"},{"key":"10291_CR96","doi-asserted-by":"crossref","unstructured":"Widyasari R, Sim S Q, Lok C, Qi H, Phan J, Tay Q, Tan C, Wee F, Tan J E, Yieh Y et al (2020) Bugsinpy: a database of existing bugs in python programs to enable controlled testing and debugging studies. In: Proceedings of the 28th ACM joint meeting on European software engineering conference and symposium on the foundations of software engineering, pp 1556\u20131560","DOI":"10.1145\/3368089.3417943"},{"key":"10291_CR97","doi-asserted-by":"publisher","first-page":"74720","DOI":"10.1109\/ACCESS.2020.2987435","volume":"8","author":"M Xue","year":"2020","unstructured":"Xue M, Yuan C, Wu H, Zhang Y, Liu W (2020) Machine learning security: threats, countermeasures, and evaluations. IEEE Access 8:74720\u201374742","journal-title":"IEEE Access"},{"key":"10291_CR98","unstructured":"Yal\u00e7\u0131n OG (2021) Top 5 deep learning frameworks to watch in 2021 and why tensorflow. https:\/\/towardsdatascience.com\/top-5-deep-learning-frameworks-to-watch-in-2021-and-why-tensorflow-98d8d6667351. Accessed: 2022-12-29"},{"key":"10291_CR99","doi-asserted-by":"crossref","unstructured":"Zerouali A, Mens T, Robles G, Gonzalez-Barahona J M (2019) On the diversity of software package popularity metrics: an empirical study of npm. In: 2019 IEEE 26th international conference on software analysis, evolution and reengineering (SANER). IEEE, pp 589\u2013593","DOI":"10.1109\/SANER.2019.8667997"},{"key":"10291_CR100","doi-asserted-by":"crossref","unstructured":"Zhang M, Zhang Y, Zhang L, Liu C, Khurshid S (2018a) Deeproad: Gan-based metamorphic testing and input validation framework for autonomous driving systems. In: 2018 33rd IEEE\/ACM international conference on automated software engineering (ASE). IEEE, pp 132\u2013142","DOI":"10.1145\/3238147.3238187"},{"key":"10291_CR101","doi-asserted-by":"crossref","unstructured":"Zhang Y, Chen Y, Cheung S C, Xiong Y, Zhang L (2018b) An empirical study on tensorflow program bugs. In: Proceedings of the 27th ACM SIGSOFT international symposium on software testing and analysis, pp 129\u2013140","DOI":"10.1145\/3213846.3213866"},{"key":"10291_CR102","unstructured":"Zhang J, Barr E T, Guedj B, Harman M, Shawe-Taylor J (2019) Perturbed model validation: a new framework to validate model relevance"},{"key":"10291_CR103","unstructured":"Zhang J M, Harman M, Ma L, Liu Y (2020) Machine learning testing: survey, landscapes and horizons. IEEE Trans Softw Eng"},{"key":"10291_CR104","unstructured":"Zhu C, Huang W R, Li H, Taylor G, Studer C, Goldstein T (2019) Transferable clean-label poisoning attacks on deep neural nets. In: International conference on machine learning. PMLR, pp 7614\u20137623"},{"key":"10291_CR105","unstructured":"Zubrow D (2009) IEEE Standard classification for software anomalies. IEEE Computer Society"}],"container-title":["Empirical Software Engineering"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10664-023-10291-1.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s10664-023-10291-1\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10664-023-10291-1.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2023,5,26]],"date-time":"2023-05-26T09:46:55Z","timestamp":1685094415000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s10664-023-10291-1"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023,4,5]]},"references-count":105,"journal-issue":{"issue":"3","published-print":{"date-parts":[[2023,5]]}},"alternative-id":["10291"],"URL":"https:\/\/doi.org\/10.1007\/s10664-023-10291-1","relation":{},"ISSN":["1382-3256","1573-7616"],"issn-type":[{"value":"1382-3256","type":"print"},{"value":"1573-7616","type":"electronic"}],"subject":[],"published":{"date-parts":[[2023,4,5]]},"assertion":[{"value":"11 January 2023","order":1,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"5 April 2023","order":2,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}}],"article-number":"62"}}