{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,8]],"date-time":"2026-07-08T23:36:40Z","timestamp":1783553800179,"version":"3.55.0"},"reference-count":105,"publisher":"Springer Science and Business Media LLC","issue":"1","license":[{"start":{"date-parts":[[2023,12,5]],"date-time":"2023-12-05T00:00:00Z","timestamp":1701734400000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2023,12,5]],"date-time":"2023-12-05T00:00:00Z","timestamp":1701734400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"funder":[{"name":"National Science and Engineering Research Council of Canada"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Empir Software Eng"],"published-print":{"date-parts":[[2024,1]]},"DOI":"10.1007\/s10664-023-10400-0","type":"journal-article","created":{"date-parts":[[2023,12,5]],"date-time":"2023-12-05T12:02:06Z","timestamp":1701777726000},"update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":31,"title":["Bug characterization in machine learning-based systems"],"prefix":"10.1007","volume":"29","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-7942-4791","authenticated-orcid":false,"given":"Mohammad Mehdi","family":"Morovati","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Amin","family":"Nikanjam","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Florian","family":"Tambon","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Foutse","family":"Khomh","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Zhen Ming","family":"Jiang","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2023,12,5]]},"reference":[{"key":"10400_CR1","unstructured":"Abadi M, Barham P, Chen J, Chen Z, Davis A, Dean J, Devin M, Ghemawat S, Irving G, Isard M et al (2016) Tensorflow: a system for large-scale machine learning. In: 12th $$\\{$$USENIX$$\\}$$ symposium on operating systems design and implementation ($$\\{$$OSDI$$\\}$$ 16). Savannah, GA, USA, USENIX, pp 265\u2013283"},{"key":"10400_CR2","unstructured":"Add typing for trainer.logger (2021) https:\/\/github.com\/Lightning-AI\/lightning\/pull\/11114"},{"issue":"11","key":"10400_CR3","doi-asserted-by":"publisher","first-page":"8484","DOI":"10.1007\/s10489-021-02348-9","volume":"51","author":"SG Aithal","year":"2021","unstructured":"Aithal SG, Rao AB, Singh S (2021) Automatic question-answer pairs generation and question similarity mechanism in question answering system. Appl Intell 51(11):8484\u20138497","journal-title":"Appl Intell"},{"key":"10400_CR4","doi-asserted-by":"crossref","unstructured":"Amershi S, Begel A, Bird C, DeLine R, Gall H, Kamar E, Nagappan N, Nushi B, Zimmermann T (2019) Software engineering for machine learning: a case study. In: 2019 IEEE\/ACM 41st International conference on software engineering: software engineering in practice (ICSE-SEIP), IEEE, pp 291\u2013300","DOI":"10.1109\/ICSE-SEIP.2019.00042"},{"key":"10400_CR5","doi-asserted-by":"publisher","unstructured":"Anvik J, Hiew L, Murphy GC (2005) Coping with an open bug repository. In: Proceedings of the 2005 OOPSLA workshop on eclipse technology exchange, ser. eclipse \u201905. New York, NY, USA: Association for Computing Machinery, pp 35\u201339. [Online]. Available: https:\/\/doi.org\/10.1145\/1117696.1117704","DOI":"10.1145\/1117696.1117704"},{"issue":"3","key":"10400_CR6","doi-asserted-by":"publisher","first-page":"219","DOI":"10.1002\/stvr.1486","volume":"24","author":"A Arcuri","year":"2014","unstructured":"Arcuri A, Briand L (2014) A hitchhiker\u2019s guide to statistical tests for assessing randomized algorithms in software engineering. Softw Test Verif Rel 24(3):219\u2013250","journal-title":"Softw Test Verif Rel"},{"key":"10400_CR7","doi-asserted-by":"crossref","unstructured":"Bennett KH, Rajlich VT (2000) Software maintenance and evolution: a roadmap. In: Proceedings of the conference on the future of software engineering, pp 73\u201387","DOI":"10.1145\/336512.336534"},{"issue":"1","key":"10400_CR8","doi-asserted-by":"publisher","first-page":"56","DOI":"10.1109\/TSE.2016.2576451","volume":"43","author":"A Bosu","year":"2016","unstructured":"Bosu A, Carver JC, Bird C, Orbeck J, Chockley C (2016) Process aspects and social dynamics of contemporary code review: insights from open source development and industrial practice at microsoft. IEEE Trans Softw Eng 43(1):56\u201375","journal-title":"IEEE Trans Softw Eng"},{"key":"10400_CR9","doi-asserted-by":"crossref","unstructured":"Bosu A, Carver JC (2014) Impact of developer reputation on code review outcomes in oss projects: an empirical investigation. In: Proceedings of the 8th ACM\/IEEE international symposium on empirical software engineering and measurement, pp 1\u201310","DOI":"10.1145\/2652524.2652544"},{"key":"10400_CR10","unstructured":"Bug fix for static pulse shapes (2018). https:\/\/github.com\/cms-sw\/cmssw\/pull\/23001"},{"key":"10400_CR11","unstructured":"Bug: mismatch value with speechbrain.nnet.pooling.statisticalpooling (2021) https:\/\/github.com\/speechbrain\/speechbrain\/issues\/1048"},{"key":"10400_CR12","doi-asserted-by":"crossref","unstructured":"Cao J, Chen B, Sun C, Hu L, Peng X (2021) Characterizing performance bugs in deep learning systems. [Online]. Available: arXiv:2112.01771","DOI":"10.1145\/3540250.3549123"},{"issue":"2","key":"10400_CR13","doi-asserted-by":"publisher","first-page":"889","DOI":"10.1007\/s10489-020-01839-5","volume":"51","author":"S Carta","year":"2021","unstructured":"Carta S, Corriga A, Ferreira A, Podda AS, Recupero DR (2021) A multi-layer and multi-ensemble stock trader using deep learning and deep reinforcement learning. Appl Intell 51(2):889\u2013905","journal-title":"Appl Intell"},{"issue":"2","key":"10400_CR14","doi-asserted-by":"publisher","first-page":"155","DOI":"10.1007\/s13198-014-0226-5","volume":"5","author":"K Chaturvedi","year":"2014","unstructured":"Chaturvedi K, Kapur P, Anand S, Singh V (2014) Predicting the complexity of code changes using entropy based measures. Int J Syst Assur Eng Manag 5(2):155\u2013164","journal-title":"Int J Syst Assur Eng Manag"},{"key":"10400_CR15","doi-asserted-by":"publisher","unstructured":"Chen Z, Cao Y, Liu Y, Wang H, Xie T, Liu X (2020) A comprehensive study on challenges in deploying deep learning based software. In: Proceedings of the 28th ACM joint meeting on european software engineering conference and symposium on the foundations of software engineering, ser. ESEC\/FSE 2020. New York, NY, USA: Association for Computing Machinery, pp 750\u2013762. [Online]. Available: https:\/\/doi.org\/10.1145\/3368089.3409759","DOI":"10.1145\/3368089.3409759"},{"key":"10400_CR16","unstructured":"Chollet F et\u00a0al (2018) Keras: the python deep learning library. Michigan, United States, pp ascl\u20131806"},{"issue":"3","key":"10400_CR17","doi-asserted-by":"publisher","first-page":"494","DOI":"10.1037\/0033-2909.114.3.494","volume":"114","author":"N Cliff","year":"1993","unstructured":"Cliff N (1993) Dominance statistics: Ordinal analyses to answer ordinal questions. Psychol Bull 114(3):494","journal-title":"Psychol Bull"},{"key":"10400_CR18","unstructured":"developer\u00a0guideline documentation G (2021) Github rest api. https:\/\/developer.github.com\/v3\/. Accessed 27 July 2021"},{"issue":"2","key":"10400_CR19","doi-asserted-by":"publisher","first-page":"463","DOI":"10.1007\/s11135-014-0003-1","volume":"49","author":"R Falotico","year":"2015","unstructured":"Falotico R, Quatto P (2015) Fleiss\u2019 kappa statistic without paradoxes. Quality & Quantity 49(2):463\u2013470","journal-title":"Quality & Quantity"},{"key":"10400_CR20","unstructured":"Fix default ckpt path when logger exists (2020). https:\/\/github.com\/Lightning-AI\/lightning\/pull\/771"},{"key":"10400_CR21","unstructured":"Fix docs for early stopping (2020). https:\/\/github.com\/Lightning-AI\/lightning\/pull\/865"},{"key":"10400_CR22","unstructured":"Fix model architecture for deployment to onnx (2023). https:\/\/github.com\/mehta-lab\/microDL\/pull\/234"},{"key":"10400_CR23","unstructured":"Galin D (2004) Software quality assurance: from theory to implementation. pearson.com: Pearson education"},{"key":"10400_CR24","unstructured":"Gensim library (2022) https:\/\/radimrehurek.com\/gensim_3.8.3\/index.html"},{"key":"10400_CR25","unstructured":"GitHub (2021) Github graphql api documentation.https:\/\/docs.github.com\/en\/graphql. Accessed 27 July 2021"},{"key":"10400_CR26","unstructured":"Github (2022) https:\/\/github.com\/"},{"key":"10400_CR27","doi-asserted-by":"crossref","unstructured":"Grubb P, Takang AA (2003) Software maintenance: concepts and practice. World Scientific","DOI":"10.1142\/9789812564429"},{"key":"10400_CR28","unstructured":"Gupta S (2021) What is the best language for machine learning? https:\/\/www.springboard.com\/blog\/data-science\/best-language-for-machine-learning. Accessed 06 Oct 2021"},{"key":"10400_CR29","doi-asserted-by":"crossref","unstructured":"Hanam Q, Brito FSdM, Mesbah A (2016) Discovering bug patterns in javascript. In: Proceedings of the 2016 24th ACM SIGSOFT international symposium on foundations of software engineering, pp 144\u2013156","DOI":"10.1145\/2950290.2950308"},{"key":"10400_CR30","unstructured":"HanLP (2021) Hanlp: han language processing. https:\/\/github.com\/hankcs\/HanLP. Accessed 01 Nov 2021"},{"key":"10400_CR31","unstructured":"Hartling L, Hamm M, Milne A, Vandermeer B, Santaguida PL, Ansari M, Tsertsvadze A, Hempel S, Shekelle P, Dryden DM (2012) Validity and inter-rater reliability testing of quality assessment instruments"},{"key":"10400_CR32","doi-asserted-by":"publisher","unstructured":"Hata H, Kula RG, Ishio T, Treude C (2021) Same file, different changes: the potential of meta-maintenance on github. In: Proceedings of the 43rd international conference on software engineering, IEEE. 3 Park Avenue, New York NY 10016-5997, USA: IEEE Press, pp 773\u2013784. [Online]. Available: https:\/\/doi.org\/10.1109\/ICSE43902.2021.00076","DOI":"10.1109\/ICSE43902.2021.00076"},{"key":"10400_CR33","doi-asserted-by":"publisher","first-page":"2585","DOI":"10.1007\/s10115-021-01605-0","volume":"63","author":"X Hu","year":"2021","unstructured":"Hu X, Chu L, Pei J, Liu W, Bian J (2021) Model complexity of deep learning: a survey. Knowl Inf Syst 63:2585\u20132619","journal-title":"Knowl Inf Syst"},{"key":"10400_CR34","doi-asserted-by":"publisher","unstructured":"Humbatova N, Jahangirova G, Bavota G, Riccio V, Stocco A, Tonella P (2020) Taxonomy of real faults in deep learning systems. In: Proceedings of the ACM\/IEEE 42nd international conference on software engineering, ser. ICSE \u201920. New York, USA: Association for Computing Machinery, pp 1110\u20131121. [Online]. Available: https:\/\/doi.org\/10.1145\/3377811.3380395","DOI":"10.1145\/3377811.3380395"},{"key":"10400_CR35","doi-asserted-by":"publisher","unstructured":"Humbatova N, Jahangirova G, Tonella P (2021) Deepcrime: mutation testing of deep learning systems based on real faults. In: Proceedings of the30th ACM SIGSOFT international symposium on software testing and analysis, ser. ISSTA 2021. New York, USA: Association for Computing Machinery, pp 67\u201378. [Online]. Available: https:\/\/doi.org\/10.1145\/3460319.3464825","DOI":"10.1145\/3460319.3464825"},{"key":"10400_CR36","unstructured":"IEEE (2010) ISO\/IEC\/IEEE International Standard - Systems and software engineering \u2013 Vocabulary. 3 Park Avenue, New York 10016-5997, USA: IEEE"},{"key":"10400_CR37","unstructured":"IEEE (2017) IEEE recommended practice on software reliability. 3 Park Avenue, New York 10016-5997, USA: IEEE"},{"key":"10400_CR38","unstructured":"Interactive learning has server error hosting on docker (2019) https:\/\/github.com\/RasaHQ\/rasa\/issues\/4142"},{"key":"10400_CR39","doi-asserted-by":"publisher","unstructured":"Islam MJ, Nguyen G, Pan R, Rajan H (2019) A comprehensive study on deep learning bug characteristics. In: Proceedings of the 2019 27th ACM joint meeting on european software engineering conference and symposium on the foundations of software engineering, ser. ESEC\/FSE 2019. New York, USA: Association for Computing Machinery, pp 510\u2013520. [Online]. Available: https:\/\/doi.org\/10.1145\/3338906.3338955","DOI":"10.1145\/3338906.3338955"},{"key":"10400_CR40","doi-asserted-by":"publisher","unstructured":"Islam MJ, Pan R, Nguyen G, Rajan H (2020) Repairing deep neural networks: fix patterns and challenges. In: Proceedings of the ACM\/IEEE 42nd International Conference on Software Engineering, ser. ICSE \u201920. New York, USA: Association for Computing Machinery, pp 1135\u20131146. [Online]. Available: https:\/\/doi.org\/10.1145\/3377811.3380378","DOI":"10.1145\/3377811.3380378"},{"key":"10400_CR41","doi-asserted-by":"publisher","first-page":"110935","DOI":"10.1016\/j.jss.2021.110935","volume":"177","author":"L Jia","year":"2021","unstructured":"Jia L, Zhong H, Wang X, Huang L, Lu X (2021) The symptoms, causes, and repairs of bugs inside a deep learning library. J Syst Softw 177:110935","journal-title":"J Syst Softw"},{"key":"10400_CR42","doi-asserted-by":"publisher","unstructured":"Joshua G, Yang F, Junjie S, Sumaya A, Yuan Chen X, Alfred Q (2020) A comprehensive study of autonomous vehicle bugs. In: Proceedings of the ACM\/IEEE 42nd international conference on software engineering, ser. ICSE\u2019 20. New York, USA: Association for Computing Machinery, pp 385\u2013396. [Online]. Available: https:\/\/doi.org\/10.1145\/3377811.3380397","DOI":"10.1145\/3377811.3380397"},{"key":"10400_CR43","doi-asserted-by":"crossref","unstructured":"Kampenes VB, Dyb\u00e5 T, Hannay JE, Sj\u00f8berg DI (2007) A systematic review of effect size in software engineering experiments. Inf Softw Technol 49(11):1073\u20131086. [Online]. Available: https:\/\/www.sciencedirect.com\/science\/article\/pii\/S0950584907000195","DOI":"10.1016\/j.infsof.2007.02.015"},{"key":"10400_CR44","unstructured":"Keras (2022) Formal documentation of keras apis. https:\/\/keras.io\/api\/models\/"},{"key":"10400_CR45","doi-asserted-by":"crossref","unstructured":"Kononenko O, Rose T, Baysal O, Godfrey M, Theisen D, De\u00a0Water B (2018) Studying pull request merges: a case study of shopify\u2019s active merchant. In: Proceedings of the 40th international conference on software engineering: software engineering in practice. 3 Park Avenue, New York 10016-5997, USA: IEEE, pp 124\u2013133","DOI":"10.1145\/3183519.3183542"},{"key":"10400_CR46","doi-asserted-by":"crossref","unstructured":"Krishna R, Agrawal A, Rahman A, Sobran A, Menzies T (2018) What is the connection between issues, bugs, and enhancements? In: 2018 IEEE\/ACM 40th International conference on software engineering: software engineering in practice track (ICSE-SEIP). 3 Park Avenue, New York 10016-5997, USA: IEEE, pp 306\u2013315","DOI":"10.1145\/3183519.3183548"},{"key":"10400_CR47","doi-asserted-by":"crossref","unstructured":"Lau K-K, di\u00a0Cola S (2017) An introduction to component-based software development. World Scientific Publishing Co Pte Ltd: World Scientific, [Online]. Available: https:\/\/www.worldscientific.com\/doi\/abs\/10.1142\/10486","DOI":"10.1142\/10486"},{"key":"10400_CR48","doi-asserted-by":"crossref","unstructured":"Lenarduzzi V, Lomio F, Moreschini S, Taibi D, Tamburri DA (2021) Software quality for ai: where we are now? In: Winkler D, Biffl S, Mendez D, Wimmer M, Bergsmann J (eds) Software quality: future perspectives on software engineering quality. Springer International Publishing, Cham, pp 43\u201353","DOI":"10.1007\/978-3-030-65854-0_4"},{"issue":"1","key":"10400_CR49","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1007\/s10489-019-01511-7","volume":"50","author":"Z Liu","year":"2020","unstructured":"Liu Z, Li D, Ge SS, Tian F (2020) Small traffic sign detection from large image. Appl Intell 50(1):1\u201313","journal-title":"Appl Intell"},{"key":"10400_CR50","doi-asserted-by":"crossref","unstructured":"Liu C, Lu J, Li G, Yuan T, Li L, Tan F, Yang J, You L, Xue J (2021) Detecting tensorflow program bugs in real-world industrial environment. In: 2021 36th IEEE\/ACM International conference on automated software engineering (ASE). 3 Park Avenue, New York 10016-5997, USA: IEEE, pp 55\u201366","DOI":"10.1109\/ASE51524.2021.9678891"},{"key":"10400_CR51","doi-asserted-by":"crossref","unstructured":"Li S, Wu Y, Liu Y, Wang D, Wen M, Tao Y, Sui Y, Liu Y (2020) An exploratory study of bugs in extended reality applications on the web. In: 2020 IEEE 31st International symposium on software reliability engineering (ISSRE). 3 Park Avenue, New York 10016\u20135997, USA: IEEE, pp 172\u2013183","DOI":"10.1109\/ISSRE5003.2020.00025"},{"key":"10400_CR52","unstructured":"Loading a checkpoint that was saved in pl $$<$$ 1.2 still breaks (2021) https:\/\/github.com\/Lightning-AI\/lightning\/issues\/7400"},{"key":"10400_CR53","unstructured":"Loading saved destvi.from_rna_model dosn\u2019t consider two anndatas used to create model (2021) https:\/\/github.com\/scverse\/scvi-tools\/issues\/1087"},{"key":"10400_CR54","doi-asserted-by":"crossref","unstructured":"Long G, Chen T (2022) On reporting performance and accuracy bugs for deep learning frameworks: an exploratory study from github. [Online]. Available: arXiv:2204.07893","DOI":"10.1145\/3530019.3530029"},{"key":"10400_CR55","doi-asserted-by":"crossref","unstructured":"Lyu MR (2007) Software reliability engineering: a roadmap. In: Future of software engineering (FOSE\u201907). 3 Park Avenue, New York 10016-5997, USA: IEEE, pp 153\u2013170","DOI":"10.1109\/FOSE.2007.24"},{"issue":"2","key":"10400_CR56","doi-asserted-by":"publisher","first-page":"545","DOI":"10.11144\/Javeriana.upsy10-2.cdcp","volume":"10","author":"G Macbeth","year":"2011","unstructured":"Macbeth G, Razumiejczyk E, Ledesma RD (2011) Cliff\u2019s delta calculator: a non-parametric effect size program for two groups of observations. Universitas Psychologica 10(2):545\u2013555","journal-title":"Universitas Psychologica"},{"key":"10400_CR57","doi-asserted-by":"crossref","unstructured":"Maddila C, Bansal C, Nagappan N (2019) Predicting pull request completion time: a case study on large scale cloud services. In: Proceedings of the 2019 27th acm joint meeting on european software engineering conference and symposium on the foundations of software engineering, pp 874\u2013882","DOI":"10.1145\/3338906.3340457"},{"key":"10400_CR58","unstructured":"Mallet library (2022) https:\/\/mimno.github.io\/Mallet\/"},{"key":"10400_CR59","doi-asserted-by":"crossref","unstructured":"Mart\u00ednez-Fern\u00e1ndez S, Bogner J, Franch X, Oriol M, Siebert J, Trendowicz A, Vollmer AM, Wagner S (2021) Software engineering for ai-based systems: a survey","DOI":"10.1145\/3487043"},{"issue":"1","key":"10400_CR60","doi-asserted-by":"publisher","first-page":"81","DOI":"10.1109\/MS.2019.2954841","volume":"37","author":"T Menzies","year":"2019","unstructured":"Menzies T (2019) The five laws of se for ai. IEEE Softw 37(1):81\u201385","journal-title":"IEEE Softw"},{"issue":"3","key":"10400_CR61","doi-asserted-by":"publisher","first-page":"62","DOI":"10.1007\/s10664-023-10291-1","volume":"28","author":"MM Morovati","year":"2023","unstructured":"Morovati MM, Nikanjam A, Khomh F, Jiang ZM (2023) Bugs in machine learning-based systems: a faultload benchmark. Empir Softw Eng 28(3):62","journal-title":"Empir Softw Eng"},{"key":"10400_CR62","doi-asserted-by":"publisher","DOI":"10.1016\/j.jss.2020.110538","volume":"163","author":"Z Ni","year":"2020","unstructured":"Ni Z, Li B, Sun X, Chen T, Tang B, Shi X (2020) Analyzing bug fix for automatic bug cause classification. J Syst Softw 163:110538","journal-title":"J Syst Softw"},{"issue":"1","key":"10400_CR63","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1145\/3470006","volume":"31","author":"A Nikanjam","year":"2021","unstructured":"Nikanjam A, Braiek HB, Morovati MM, Khomh F (2021) Automatic fault detection for deep learning programs using graph transformations. ACM Trans Softw Eng Methodol (TOSEM) 31(1):1\u201327","journal-title":"ACM Trans Softw Eng Methodol (TOSEM)"},{"issue":"1","key":"10400_CR64","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1007\/s10515-021-00313-x","volume":"29","author":"A Nikanjam","year":"2022","unstructured":"Nikanjam A, Morovati MM, Khomh F, Ben Braiek H (2022) Faults in deep reinforcement learning programs: a taxonomy and a detection approach. Autom Softw Eng 29(1):1\u201332","journal-title":"Autom Softw Eng"},{"key":"10400_CR65","unstructured":"NVIDIA (2021) Nvtabular. https:\/\/github.com\/NVIDIA-Merlin\/NVTabular. Accessed 01 Nov 2021"},{"key":"10400_CR66","volume-title":"Pytorch: an imperative style, high-performance deep learning library","author":"A Paszke","year":"2019","unstructured":"Paszke A, Gross S, Massa F, Lerer A, Bradbury J, Chanan G, Killeen T, Lin Z, Gimelshein N, Antiga L et al (2019) Pytorch: an imperative style, high-performance deep learning library. Ithaca, NY, United States"},{"key":"10400_CR67","unstructured":"Pedregosa F, Varoquaux G, Gramfort A, Michel V, Thirion B, Grisel O, Blondel M, Prettenhofer P et\u00a0al (2011) Scikit-learn: machine learning in python. J Mach Learn Res 12:2825\u20132830. [Online]. Available: http:\/\/scikit-learn.sourceforge.net"},{"key":"10400_CR68","unstructured":"Pps: fix of ispixelhit (2021) https:\/\/github.com\/cms-sw\/cmssw\/pull\/35089"},{"issue":"4","key":"10400_CR69","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1007\/s10664-021-09970-8","volume":"26","author":"S Quach","year":"2021","unstructured":"Quach S, Lamothe M, Kamei Y, Shang W (2021) An empirical study on the use of szz for identifying inducing changes of non-functional bugs. Empir Softw Eng 26(4):1\u201325","journal-title":"Empir Softw Eng"},{"key":"10400_CR70","unstructured":"RasaHQ (2021a) https:\/\/github.com\/RasaHQ\/rasa\/issues\/8541. Accessed 01 Nov 2021"},{"key":"10400_CR71","unstructured":"RasaHQ (2021b) https:\/\/github.com\/RasaHQ\/rasa\/issues\/4730. Accessed 01 Nov 2021"},{"key":"10400_CR72","unstructured":"reduce_lr_on_plateau can\u2019t find validation metrics in most recent release (2021) https:\/\/github.com\/scverse\/scvi-tools\/issues\/1112"},{"key":"10400_CR73","unstructured":"Replication package (2023) https:\/\/github.com\/ML-Bugs-2022\/Replication-Package"},{"issue":"6","key":"10400_CR74","doi-asserted-by":"publisher","first-page":"5193","DOI":"10.1007\/s10664-020-09881-0","volume":"25","author":"V Riccio","year":"2020","unstructured":"Riccio V, Jahangirova G, Stocco A, Humbatova N, Weiss M, Tonella P (2020) Testing machine learning based systems: a systematic mapping. Empir Softw Eng 25(6):5193\u20135254","journal-title":"Empir Softw Eng"},{"key":"10400_CR75","doi-asserted-by":"crossref","unstructured":"Rivera-Landos E, Khomh F, Nikanjam A (2021) The challenge of reproducible ml: an empirical study on the impact of bugs","DOI":"10.1109\/QRS54544.2021.00116"},{"key":"10400_CR76","unstructured":"Romano J, Kromrey JD, Coraggio J, Skowronek J, Devine L (2006) Exploring methods for evaluating group differences on the nsse and other surveys: are the t-test and cohen\u2019sd indices the most appropriate choices. In: Annual meeting of the Southern association for institutional research. The Pennsylvania State University, Citeseer, pp 1\u201351"},{"key":"10400_CR77","doi-asserted-by":"crossref","unstructured":"Romano A, Liu X, Kwon Y, Wang W (2021) An empirical study of bugs in webassembly compilers. In: 2021 36th IEEE\/ACM International conference on automated software engineering (ASE). 3 Park Avenue, New York NY 10016-5997, USA: IEEE, pp 42\u201354","DOI":"10.1109\/ASE51524.2021.9678776"},{"issue":"5","key":"10400_CR78","doi-asserted-by":"publisher","first-page":"1763","DOI":"10.1213\/ANE.0000000000002864","volume":"126","author":"P Schober","year":"2018","unstructured":"Schober P, Boer C, Schwarte LA (2018) Correlation coefficients: appropriate use and interpretation. Anesth Analg 126(5):1763\u20131768","journal-title":"Anesth Analg"},{"key":"10400_CR79","doi-asserted-by":"publisher","unstructured":"Schoop E, Huang F, Hartmann B (2021) Umlaut: debugging deep learning programs using program structure and model behavior. In: Proceedings of the 2021 CHI conference on human factors in computing systems, ser. CHI\u2019 21. New York, USA: Association for Computing Machinery, [Online]. Available: https:\/\/doi.org\/10.1145\/3411764.3445538","DOI":"10.1145\/3411764.3445538"},{"key":"10400_CR80","unstructured":"Sculley D, Holt G, Golovin D, Davydov E, Phillips T, Ebner D, Chaudhary V, Young M, Crespo J-F, Dennison D (2015) Hidden technical debt in machine learning systems. Adv Neural Inf Process Syst 28"},{"issue":"4","key":"10400_CR81","doi-asserted-by":"publisher","first-page":"557","DOI":"10.1109\/32.799955","volume":"25","author":"CB Seaman","year":"1999","unstructured":"Seaman CB (1999) Qualitative methods in empirical studies of software engineering. IEEE Trans Softw Eng 25(4):557\u2013572","journal-title":"IEEE Trans Softw Eng"},{"key":"10400_CR82","doi-asserted-by":"publisher","unstructured":"Shen Q, Ma H, Chen J, Tian Y, Cheung S-C, Chen X (2021) A comprehensive study of deep learning compiler bugs. In: Proceedings of the 29th ACM joint meeting on european software engineering conference and symposium on the foundations of software engineering, ser. ESEC\/FSE 2021. New York, USA: Association for Computing Machinery, pp 968\u2013980. [Online]. Available: https:\/\/doi.org\/10.1145\/3468264.3468591","DOI":"10.1145\/3468264.3468591"},{"key":"10400_CR83","unstructured":"some tests fails with pytorch 1.6 (2020) https:\/\/github.com\/speechbrain\/speechbrain\/issues\/248"},{"issue":"4","key":"10400_CR84","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1007\/s10664-022-10133-6","volume":"27","author":"A Tagra","year":"2022","unstructured":"Tagra A, Zhang H, Rajbahadur GK, Hassan AE (2022) Revisiting reopened bugs in open source software systems. Empir Softw Eng 27(4):1\u201334","journal-title":"Empir Softw Eng"},{"key":"10400_CR85","doi-asserted-by":"crossref","unstructured":"Tambon F, Nikanjam A, An L, Khomh F, Antoniol G (2021) Silent bugs in deep learning frameworks: an empirical study of keras and tensorflow","DOI":"10.26226\/morressier.613b5418842293c031b5b61d"},{"key":"10400_CR86","doi-asserted-by":"publisher","first-page":"1665","DOI":"10.1007\/s10664-013-9258-8","volume":"19","author":"L Tan","year":"2014","unstructured":"Tan L, Liu C, Li Z, Wang X, Zhou Y, Zhai C (2014) Bug characteristics in open source software. Empir Softw Eng 19:1665\u20131705","journal-title":"Empir Softw Eng"},{"key":"10400_CR87","unstructured":"Tensorflow (2021) https:\/\/github.com\/tensorflow\/models. Accessed 01 Nov 2021"},{"key":"10400_CR88","unstructured":"Tensorpack (2021) https:\/\/github.com\/tensorpack\/tensorpack. Accessed 01 Nov 2021"},{"key":"10400_CR89","unstructured":"tf.function not used for model inference (2022). https:\/\/github.com\/RasaHQ\/rasa\/issues\/10728"},{"key":"10400_CR90","unstructured":"ultralytics (2021) Yolov3.\u2019 https:\/\/github.com\/ultralytics\/yolov3. Accessed 01 Nov 2021"},{"key":"10400_CR91","unstructured":"Vlasic B, Boudette NE (2016) Self-driving tesla was involved in fatal crash, us says. The New York Times Company New York, NY, USA. [Online]. Available: https:\/\/www.nytimes.com\/2016\/07\/01\/business\/self-driving-tesla-fatal-crash-investigation.html"},{"key":"10400_CR92","unstructured":"Voskoglou C (2017) What is the best programming language for machine learning. Towards data science. [Online]. Available: https:\/\/towardsdatascience.com\/what-is-the-best-programming-language-for-machine-learning-a745c156d6b7"},{"key":"10400_CR93","first-page":"14197","volume-title":"Reliability and optimal maintenance","author":"H Wang","year":"2006","unstructured":"Wang H, Pham H et al (2006) Reliability and optimal maintenance. Springer International Publishing, Springer, p 14197"},{"key":"10400_CR94","doi-asserted-by":"crossref","unstructured":"Wardat M, Cruz BD, Le W, Rajan H (2022) Deepdiagnosis: automatically diagnosing faults and recommending actionable fixes in deep learning programs. In: Proceedings of the 44th international conference on software engineering, pp 561\u2013572","DOI":"10.1145\/3510003.3510071"},{"key":"10400_CR95","doi-asserted-by":"crossref","unstructured":"Wardat M, Le W, Rajan H (2021) Deeplocalize: fault localization for deep neural networks. In: 2021 IEEE\/ACM 43rd International conference on software engineering (ICSE). 3 Park Avenue, New York 10016-5997, USA: IEEE, pp 251\u2013262","DOI":"10.1109\/ICSE43902.2021.00034"},{"key":"10400_CR96","volume-title":"Hands-on genetic algorithms with Python: applying genetic algorithms to solve real-world deep learning and artificial intelligence problems","author":"E Wirsansky","year":"2020","unstructured":"Wirsansky E (2020) Hands-on genetic algorithms with Python: applying genetic algorithms to solve real-world deep learning and artificial intelligence problems. Packt Publishing Ltd, Packt Publishing Ltd"},{"key":"10400_CR97","unstructured":"Write complete json log after training (2019) https:\/\/github.com\/snorkel-team\/snorkel\/pull\/1445"},{"key":"10400_CR98","doi-asserted-by":"publisher","unstructured":"Yan M, Chen J, Zhang X, Tan L, Wang G, Wang Z (2021) Exposing numerical bugs in deep learning via gradient back-propagation. In: Proceedings of the 29th ACM joint meeting on european software engineering conference and symposium on the foundations of software engineering, ser. ESEC\/FSE 2021. New York, USA: Association for Computing Machinery, pp 627\u2013638. [Online]. Available: https:\/\/doi.org\/10.1145\/3468264.3468612","DOI":"10.1145\/3468264.3468612"},{"issue":"2","key":"10400_CR99","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1007\/s10664-021-10087-1","volume":"27","author":"Y Yang","year":"2022","unstructured":"Yang Y, He T, Feng Y, Liu S, Xu B (2022) Mining python fix patterns via analyzing fine-grained source code changes. Empir Softw Eng 27(2):1\u201337","journal-title":"Empir Softw Eng"},{"key":"10400_CR100","doi-asserted-by":"crossref","unstructured":"Yao Y, Xiao Z, Wang B, Viswanath B, Zheng H, Zhao BY (2017) Complexity vs. performance: empirical analysis of machine learning as a service. In: Proceedings of the 2017 internet measurement conference, pp 384\u2013397","DOI":"10.1145\/3131365.3131372"},{"issue":"01","key":"10400_CR101","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1109\/TSE.2019.2962027","volume":"48","author":"JM Zhang","year":"2022","unstructured":"Zhang JM, Harman M, Ma L, Liu Y (2022) Machine learning testing: survey, landscapes and horizons. IEEE Trans Softw Eng 48(01):1\u201336","journal-title":"IEEE Trans Softw Eng"},{"key":"10400_CR102","doi-asserted-by":"crossref","unstructured":"Zhang Y, Chen Y, Cheung S-C, Xiong Y, Zhang L (2018) An empirical study on tensorflow program bugs. In: Proceedings of the 27th ACM SIGSOFT international symposium on software testing and analysis. 3 Park Avenue, New York 10016-5997, USA: IEEE, pp 129\u2013140","DOI":"10.1145\/3213846.3213866"},{"key":"10400_CR103","doi-asserted-by":"crossref","unstructured":"Zhang T, Gao C, Ma L, Lyu M, Kim M (2019) An empirical study of common challenges in developing deep learning applications. In: 2019 IEEE 30th International symposium on software reliability engineering (ISSRE). 3 Park Avenue, New York 10016\u20135997, USA: IEEE, pp 104\u2013115","DOI":"10.1109\/ISSRE.2019.00020"},{"key":"10400_CR104","doi-asserted-by":"crossref","unstructured":"Zhang X, Zhai J, Ma S, Shen C (2021) Autotrainer: an automatic dnn training problem detection and repair system. In: 2021 IEEE\/ACM 43rd International conference on software engineering (ICSE). IEEE, pp 359\u2013371","DOI":"10.1109\/ICSE43902.2021.00043"},{"key":"10400_CR105","doi-asserted-by":"crossref","unstructured":"Zimmermann T, Nagappan N, Guo PJ, Murphy B (2012) Characterizing and predicting which bugs get reopened. In: 2012 34th International conference on software engineering (ICSE). IEEE, pp 1074\u20131083","DOI":"10.1109\/ICSE.2012.6227112"}],"container-title":["Empirical Software Engineering"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10664-023-10400-0.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s10664-023-10400-0\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10664-023-10400-0.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,4,29]],"date-time":"2024-04-29T14:15:55Z","timestamp":1714400155000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s10664-023-10400-0"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023,12,5]]},"references-count":105,"journal-issue":{"issue":"1","published-print":{"date-parts":[[2024,1]]}},"alternative-id":["10400"],"URL":"https:\/\/doi.org\/10.1007\/s10664-023-10400-0","relation":{},"ISSN":["1382-3256","1573-7616"],"issn-type":[{"value":"1382-3256","type":"print"},{"value":"1573-7616","type":"electronic"}],"subject":[],"published":{"date-parts":[[2023,12,5]]},"assertion":[{"value":"2 October 2023","order":1,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"5 December 2023","order":2,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}}],"article-number":"14"}}