{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,6,3]],"date-time":"2026-06-03T00:54:29Z","timestamp":1780448069659,"version":"3.54.1"},"reference-count":71,"publisher":"Springer Science and Business Media LLC","issue":"1","license":[{"start":{"date-parts":[[2024,11,9]],"date-time":"2024-11-09T00:00:00Z","timestamp":1731110400000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2024,11,9]],"date-time":"2024-11-09T00:00:00Z","timestamp":1731110400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Empir Software Eng"],"published-print":{"date-parts":[[2025,1]]},"DOI":"10.1007\/s10664-024-10579-w","type":"journal-article","created":{"date-parts":[[2024,11,9]],"date-time":"2024-11-09T12:21:09Z","timestamp":1731154869000},"update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":8,"title":["Towards enhancing the reproducibility of deep learning bugs: an empirical study"],"prefix":"10.1007","volume":"30","author":[{"ORCID":"https:\/\/orcid.org\/0009-0000-3584-0470","authenticated-orcid":false,"given":"Mehil B.","family":"Shah","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-3821-5990","authenticated-orcid":false,"given":"Mohammad Masudur","family":"Rahman","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-5704-4173","authenticated-orcid":false,"given":"Foutse","family":"Khomh","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2024,11,9]]},"reference":[{"issue":"2","key":"10579_CR1","doi-asserted-by":"publisher","first-page":"38","DOI":"10.3390\/risks6020038","volume":"6","author":"PM Addo","year":"2018","unstructured":"Addo PM, Guegan D, Hassani B (2018) Credit risk analysis using machine and deep learning models. Risks 6(2):38","journal-title":"Risks"},{"key":"10579_CR2","unstructured":"Agrawal R,\u00a0Srikant R (1994) Fast algorithms for mining association rules in large databases. In: Proceedings of the 20th international conference on very large data bases, ser. VLDB \u201994. San Francisco, CA, USA: Morgan Kaufmann Publishers Inc., pp 487\u2013499"},{"key":"10579_CR3","doi-asserted-by":"crossref","unstructured":"Alahmari SS, Goldgof DB, Mouton PR, Hall LO (2020) Challenges for the repeatability of deep learning models. IEEE Access 8:211 860\u2013211 868","DOI":"10.1109\/ACCESS.2020.3039833"},{"key":"10579_CR4","unstructured":"ATMA (2018) Stack Overflow. Accessed: 28 Dec 2023. Available: https:\/\/stackoverflow.com\/q\/50920908"},{"issue":"5","key":"10579_CR5","doi-asserted-by":"publisher","first-page":"111","DOI":"10.1145\/2160718.2160742","volume":"55","author":"A Aviram","year":"2012","unstructured":"Aviram A, Weng S-C, Hu S, Ford B (2012) Efficient system-enforced deterministic parallelism. Commun ACM 55(5):111\u2013119","journal-title":"Commun ACM"},{"key":"10579_CR6","unstructured":"Balaniuk R (2019) Stack Overflow. Accessed: 3 Jan 2024. Available: https:\/\/stackoverflow.com\/q\/58190114"},{"issue":"4","key":"10579_CR7","doi-asserted-by":"publisher","first-page":"122","DOI":"10.3390\/info10040122","volume":"10","author":"DS Berman","year":"2019","unstructured":"Berman DS, Buczak AL, Chavis JS, Corbett CL (2019) A survey of deep learning methods for cyber security. Information 10(4):122","journal-title":"Information"},{"key":"10579_CR8","doi-asserted-by":"crossref","unstructured":"Braiek HB,\u00a0Khomh F (2020) On testing machine learning programs. J Syst Soft 164:110542. [Online]. Available: https:\/\/www.sciencedirect.com\/science\/article\/pii\/S0164121220300248","DOI":"10.1016\/j.jss.2020.110542"},{"key":"10579_CR9","unstructured":"Breck E,\u00a0Polyzotis N,\u00a0Roy S,\u00a0Whang S,\u00a0Zinkevich M (2019) Data validation for machine learning. In MLSys"},{"key":"10579_CR10","first-page":"1040","volume":"19","author":"M Ceccato","year":"2014","unstructured":"Ceccato M, Di Penta M, Falcarin P, Ricca F, Torchiano M, Tonella P (2014) A family of experiments to assess the effectiveness and efficiency of source code obfuscation techniques. Empirical Soft Eng 19:1040\u20131074","journal-title":"Empirical Soft Eng"},{"key":"10579_CR11","doi-asserted-by":"crossref","unstructured":"Chen B, Jiang ZMJ (2022) A survey of software log instrumentation. ACM Comput Surv 54(4):1\u201334. [Online]. Available: https:\/\/dl.acm.org\/doi\/10.1145\/3448976","DOI":"10.1145\/3448976"},{"key":"10579_CR12","doi-asserted-by":"publisher","unstructured":"Chen B,\u00a0Wen M,\u00a0Shi Y,\u00a0Lin D, Rajbahadur GK, Jiang ZMJ (2022) Towards training reproducible deep learning models. In: Proceedings of the 44th international conference on software engineering, ser. ICSE \u201922. New York, NY, USA: Association for Computing Machinery, pp 2202\u20132214. [Online]. Available: https:\/\/doi.org\/10.1145\/3510003.3510163","DOI":"10.1145\/3510003.3510163"},{"key":"10579_CR13","unstructured":"Cochran WG (1977) Sampling techniques. john wiley & sons"},{"key":"10579_CR14","doi-asserted-by":"crossref","unstructured":"Croft R, Babar MA, Kholoosi MM (2023) Data quality for software vulnerability datasets. In: 2023 IEEE\/ACM 45th International conference on software engineering (ICSE). IEEE, pp 121\u2013133","DOI":"10.1109\/ICSE48619.2023.00022"},{"issue":"1","key":"10579_CR15","doi-asserted-by":"publisher","first-page":"24","DOI":"10.1038\/s41591-018-0316-z","volume":"25","author":"A Esteva","year":"2019","unstructured":"Esteva A, Robicquet A, Ramsundar B, Kuleshov V, DePristo M, Chou K, Cui C, Corrado G, Thrun S, Dean J (2019) A guide to deep learning in healthcare. Nat Med 25(1):24\u201329","journal-title":"Nat Med"},{"key":"10579_CR16","unstructured":"Exchange S (2023) All sites - stack exchange. accessed on December 12, 2023. [Online]. Available: https:\/\/stackexchange.com\/sites?view=list"},{"key":"10579_CR17","unstructured":"Geeks For Geeks (2023) Accessed: 25 December 2023. Available: https:\/\/www.geeksforgeeks.org\/best-ides-for-machine-learning\/"},{"key":"10579_CR18","doi-asserted-by":"crossref","unstructured":"Gori M,\u00a0Betti A,\u00a0Melacci S (2023) Machine Learning: A constraint-based approach. Elsevier","DOI":"10.1016\/B978-0-32-389859-1.00013-1"},{"key":"10579_CR19","unstructured":"Grosse R (2017) Lecture 15: Exploding and vanishing gradients. University Toronto Comput Sci"},{"key":"10579_CR20","doi-asserted-by":"crossref","unstructured":"Humbatova N,\u00a0Jahangirova G,\u00a0Bavota G,\u00a0Riccio V,\u00a0Stocco A,\u00a0Tonella P (2020) Taxonomy of real faults in deep learning systems. In: Proceedings of the ACM\/IEEE 42nd International conference on software engineering, ser. ICSE \u201920","DOI":"10.1145\/3377811.3380395"},{"key":"10579_CR21","doi-asserted-by":"publisher","unstructured":"Islam MJ,\u00a0Nguyen G,\u00a0Pan R,\u00a0Rajan H (2019) A comprehensive study on deep learning bug characteristics. In: Proceedings of the 2019 27th ACM joint meeting on european software engineering conference and symposium on the foundations of software engineering, ser. ESEC\/FSE 2019. New York, NY, USA: Association for Computing Machinery, pp 510\u2013520. [Online]. Available: https:\/\/doi.org\/10.1145\/3338906.3338955","DOI":"10.1145\/3338906.3338955"},{"key":"10579_CR22","doi-asserted-by":"publisher","unstructured":"Islam MJ,\u00a0Nguyen G,\u00a0Pan R,\u00a0Rajan H (2019) A comprehensive study on deep learning bug characteristics. ser. ESEC\/FSE 2019. New York, NY, USA: Association for Computing Machinery, pp 510\u2013520. [Online]. Available: https:\/\/doi.org\/10.1145\/3338906.3338955","DOI":"10.1145\/3338906.3338955"},{"key":"10579_CR23","doi-asserted-by":"publisher","unstructured":"Jafarinejad F,\u00a0Narasimhan K,\u00a0Mezini M (2021) Nerdbug: Automated bug detection in neural networks. In: Proceedings of the 1st ACM international workshop on AI and Software Testing\/Analysis, ser. AISTA 2021. New York, NY, USA: Association for Computing Machinery, pp 13\u201316. [Online]. Available: https:\/\/doi.org\/10.1145\/3464968.3468409","DOI":"10.1145\/3464968.3468409"},{"key":"10579_CR24","unstructured":"Jahan S, Shah MB, Rahman MM (2024) Towards understanding the challenges of bug localization in deep learning systems. arXiv:2402.01021"},{"key":"10579_CR25","unstructured":"Khan MAI (2024) Machine Learning Mastery. Accessed: 28 May 2024. Available: https:\/\/machinelearningmastery.com\/building-a-logistic-regression-classifier-in-pytorch\/"},{"issue":"3","key":"10579_CR26","doi-asserted-by":"publisher","first-page":"487","DOI":"10.1007\/s13347-019-00372-9","volume":"33","author":"M Krishnan","year":"2020","unstructured":"Krishnan M (2020) Against interpretability: a critical examination of the interpretability problem in machine learning. Philosophy Technol 33(3):487\u2013502","journal-title":"Philosophy Technol"},{"key":"10579_CR27","doi-asserted-by":"crossref","unstructured":"Liang Y,\u00a0Lin Y,\u00a0Song X,\u00a0Sun J,\u00a0Feng Z, Dong JS (2022) gdefects4dl: a dataset of general real-world deep learning program defects. In: Proceedings of the ACM\/IEEE 44th international conference on software engineering: companion proceedings, pp 90\u201394","DOI":"10.1145\/3510454.3516826"},{"key":"10579_CR28","doi-asserted-by":"publisher","unstructured":"Liu C,\u00a0Gao C,\u00a0Xia X,\u00a0Lo D,\u00a0Grundy J,\u00a0Yang X (2021) On the reproducibility and replicability of deep learning in software engineering. ACM Trans Softw Eng Methodol 31(1):. [Online]. Available: https:\/\/doi.org\/10.1145\/3477535","DOI":"10.1145\/3477535"},{"key":"10579_CR29","doi-asserted-by":"crossref","unstructured":"Liu B,\u00a0Hsu W,\u00a0Ma Y (1999) Mining association rules with multiple minimum supports. In: Proceedings of the fifth ACM SIGKDD international conference on Knowledge discovery and data mining, pp 337\u2013341","DOI":"10.1145\/312129.312274"},{"key":"10579_CR30","doi-asserted-by":"crossref","unstructured":"Long G,\u00a0Chen T (2022) On reporting performance and accuracy bugs for deep learning frameworks: An exploratory study from github. In: Proceedings of the 26th international conference on evaluation and assessment in software engineering, pp 90\u201399","DOI":"10.1145\/3530019.3530029"},{"key":"10579_CR31","doi-asserted-by":"publisher","unstructured":"Ma L,\u00a0Juefei-Xu F,\u00a0Zhang F,\u00a0Sun J,\u00a0Xue M,\u00a0Li B,\u00a0Chen C,\u00a0Su T,\u00a0Li L,\u00a0Liu Y,\u00a0Zhao J,\u00a0Wang Y (2018) Deepgauge: Multi-granularity testing criteria for deep learning systems. In: Proceedings of the 33rd ACM\/IEEE International conference on automated software engineering, ser. ASE \u201918. New York, NY, USA: Association for Computing Machinery, pp 120\u2013131. [Online]. Available: https:\/\/doi.org\/10.1145\/3238147.3238202","DOI":"10.1145\/3238147.3238202"},{"key":"10579_CR32","doi-asserted-by":"crossref","unstructured":"Makkouk T, Kim DJ, Chen T-HP (2022) An empirical study on performance bugs in deep learning frameworks. In: 2022 IEEE International conference on software maintenance and evolution (ICSME), pp 35\u201346","DOI":"10.1109\/ICSME55016.2022.00012"},{"issue":"3","key":"10579_CR33","doi-asserted-by":"publisher","first-page":"276","DOI":"10.11613\/BM.2012.031","volume":"22","author":"ML McHugh","year":"2012","unstructured":"McHugh ML (2012) Interrater reliability: the kappa statistic. Biochemia Med 22(3):276\u2013282","journal-title":"Biochemia Med"},{"issue":"3","key":"10579_CR34","doi-asserted-by":"publisher","first-page":"62","DOI":"10.1007\/s10664-021-10113-2","volume":"27","author":"S Mondal","year":"2022","unstructured":"Mondal S, Rahman MM, Roy CK, Schneider K (2022) The reproducibility of programming-related issues in stack overflow questions. Empirical Softw Eng 27(3):62","journal-title":"Empirical Softw Eng"},{"key":"10579_CR35","doi-asserted-by":"crossref","unstructured":"Mondal S, Rahman MM, Roy CK (2019) Can issues reported at stack overflow questions be reproduced? an exploratory study. In: 2019 IEEE\/ACM 16th International conference on mining software repositories (MSR), pp 479\u2013489","DOI":"10.1109\/MSR.2019.00074"},{"key":"10579_CR36","doi-asserted-by":"crossref","unstructured":"Mondal S, Rahman MM, Roy CK (2024) Can we identify stack overflow questions requiring code snippets? investigating the cause & effect of missing code snippets. arXiv:2402.04575","DOI":"10.1109\/SANER60148.2024.00083"},{"key":"10579_CR37","doi-asserted-by":"publisher","DOI":"10.1016\/j.jss.2024.112158","volume":"217","author":"S Mondal","year":"2024","unstructured":"Mondal S, Roy B (2024) Reproducibility of issues reported in stack overflow questions: Challenges, impact & estimation. J Syst Softw 217:112158","journal-title":"J Syst Softw"},{"key":"10579_CR38","doi-asserted-by":"publisher","unstructured":"Morovati MM,\u00a0Nikanjam A,\u00a0Khomh F, Jiang ZMJ (2023) Bugs in machine learning-based systems: A faultload benchmark. Empirical Softw Engg 28(3). [Online]. Available: https:\/\/doi.org\/10.1007\/s10664-023-10291-1","DOI":"10.1007\/s10664-023-10291-1"},{"key":"10579_CR39","unstructured":"Nagarajan P,\u00a0Warnell G,\u00a0Stone P (2018) The impact of nondeterminism on reproducibility in deep reinforcement learning"},{"issue":"3","key":"10579_CR40","doi-asserted-by":"publisher","first-page":"370","DOI":"10.2307\/2344614","volume":"135","author":"JA Nelder","year":"1972","unstructured":"Nelder JA, Wedderburn RW (1972) Generalized linear models. J Royal Statistical Soc Series A: Statistics Soc 135(3):370\u2013384","journal-title":"J Royal Statistical Soc Series A: Statistics Soc"},{"key":"10579_CR41","unstructured":"Opencv (2024). Accessed: 6 June 2024. Available: https:\/\/opencv.org\/"},{"key":"10579_CR42","unstructured":"Opinio (2024) Accessed: 7 June 2024. Available: https:\/\/surveys.dal.ca\/opinio\/admin\/folder.do"},{"key":"10579_CR43","doi-asserted-by":"publisher","unstructured":"Pei K,\u00a0Cao Y,\u00a0Yang J,\u00a0Jana S (2019) Deepxplore: Automated whitebox testing of deep learning systems. Commun ACM 62(11):137\u2013145. [Online]. Available: https:\/\/doi.org\/10.1145\/3361566","DOI":"10.1145\/3361566"},{"key":"10579_CR44","doi-asserted-by":"crossref","unstructured":"Pham HV,\u00a0Qian S,\u00a0Wang J,\u00a0Lutellier T,\u00a0Rosenthal J,\u00a0Tan L,\u00a0Yu Y,\u00a0Nagappan N (2020) Problems and opportunities in training deep learning software systems: An analysis of variance. In: Proceedings of the 35th IEEE\/ACM international conference on automated software engineering, pp 771\u2013783","DOI":"10.1145\/3324884.3416545"},{"key":"10579_CR45","doi-asserted-by":"crossref","unstructured":"Ponzanelli L,\u00a0Mocci A,\u00a0Bacchelli A,\u00a0Lanza M (2014) Understanding and classifying the quality of technical forum questions. In: 2014 14th International conference on quality software, pp 343\u2013352","DOI":"10.1109\/QSIC.2014.27"},{"key":"10579_CR46","unstructured":"PythonPillow (2024). Accessed: 6 June 2024. Available: https:\/\/python-pillow.org\/"},{"key":"10579_CR47","unstructured":"PyTorch (2024) Accessed: 3 June 2024. Available: https:\/\/pytorch.org\/docs\/1.6.0\/"},{"issue":"8","key":"10579_CR48","first-page":"9","volume":"1","author":"A Radford","year":"2019","unstructured":"Radford A, Wu J, Child R, Luan D, Amodei D, Sutskever I et al (2019) Language models are unsupervised multitask learners. OpenAI blog 1(8):9","journal-title":"OpenAI blog"},{"key":"10579_CR49","doi-asserted-by":"crossref","unstructured":"Rahman MM,\u00a0Khomh F,\u00a0Castelluccio M (2020) Why are some bugs non-reproducible? : -an empirical investigation using data fusion-. In: 2020 IEEE International conference on software maintenance and evolution (ICSME), pp 605\u2013616","DOI":"10.1109\/ICSME46990.2020.00063"},{"issue":"5","key":"10579_CR50","doi-asserted-by":"publisher","first-page":"111","DOI":"10.1007\/s10664-022-10153-2","volume":"27","author":"MM Rahman","year":"2022","unstructured":"Rahman MM, Khomh F, Castelluccio M (2022) Works for me! cannot reproduce-a large scale empirical study of non-reproducible bugs\u2019\u2019. Empirical Softw Eng 27(5):111","journal-title":"Empirical Softw Eng"},{"key":"10579_CR51","doi-asserted-by":"publisher","DOI":"10.1002\/9781118491683","volume-title":"ANOVA and ANCOVA: a GLM approach","author":"A Rutherford","year":"2011","unstructured":"Rutherford A (2011) ANOVA and ANCOVA: a GLM approach. John Wiley & Sons"},{"key":"10579_CR52","volume-title":"CUDA by example: an introduction to general-purpose GPU programming","author":"J Sanders","year":"2010","unstructured":"Sanders J, Kandrot E (2010) CUDA by example: an introduction to general-purpose GPU programming. Addison-Wesley Professional"},{"key":"10579_CR53","unstructured":"Shah M (2024) mehilshah\/bug_reproducibility_dl_bugs. accessed on January 3, 2024. [Online]. Available: https:\/\/github.com\/mehilshah\/Bug_Reproducibility_DL_Bugs"},{"key":"10579_CR54","doi-asserted-by":"publisher","first-page":"221","DOI":"10.1146\/annurev-bioeng-071516-044442","volume":"19","author":"D Shen","year":"2017","unstructured":"Shen D, Wu G, Suk H-I (2017) Deep learning in medical image analysis. Ann Rev Biomed Eng 19:221\u2013248","journal-title":"Ann Rev Biomed Eng"},{"key":"10579_CR55","doi-asserted-by":"crossref","unstructured":"Shi S,\u00a0Wang Q,\u00a0Xu P,\u00a0Chu X (2016) Benchmarking state-of-the-art deep learning software tools. In: 2016 7th International conference on cloud computing and big data (CCBD). IEEE, pp 99\u2013104","DOI":"10.1109\/CCBD.2016.029"},{"key":"10579_CR56","doi-asserted-by":"publisher","first-page":"5255","DOI":"10.1007\/s10664-020-09882-z","volume":"25","author":"M Soltani","year":"2020","unstructured":"Soltani M, Hermans F, B\u00e4ck T (2020) The significance of bug report elements. Empirical Softw Eng 25:5255\u20135294","journal-title":"Empirical Softw Eng"},{"key":"10579_CR57","unstructured":"Stack Exchange (2024) [Online]. Available: https:\/\/data.stackexchange.com\/"},{"key":"10579_CR58","doi-asserted-by":"crossref","unstructured":"Talwar D,\u00a0Guruswamy S,\u00a0Ravipati N,\u00a0Eirinaki M (2020) Evaluating validity of synthetic data in perception tasks for autonomous vehicles. In: 2020 IEEE International conference on artificial intelligence testing (AITest). IEEE, pp 73\u201380","DOI":"10.1109\/AITEST49225.2020.00018"},{"issue":"1","key":"10579_CR59","doi-asserted-by":"publisher","first-page":"10","DOI":"10.1007\/s10664-023-10389-6","volume":"29","author":"F Tambon","year":"2024","unstructured":"Tambon F, Nikanjam A, An L, Khomh F, Antoniol G (2024) Silent bugs in deep learning frameworks: an empirical study of keras and tensorflow. Empirical Softw Eng 29(1):10","journal-title":"Empirical Softw Eng"},{"key":"10579_CR60","unstructured":"Team K (2024) Keras documentation: Python & numpy utilities. Keras. Accessed: 21 Dec 2023. Available: https:\/\/keras.io\/2.16\/api\/utils\/python_utils\/#sequence-class"},{"key":"10579_CR61","doi-asserted-by":"crossref","unstructured":"Tishby N,\u00a0Zaslavsky N (2015) Deep learning and the information bottleneck principle. In: 2015 IEEE Information theory workshop (ITW), , pp 1\u20135. [Online]. Available: https:\/\/ieeexplore.ieee.org\/abstract\/document\/7133169","DOI":"10.1109\/ITW.2015.7133169"},{"key":"10579_CR62","doi-asserted-by":"crossref","unstructured":"Tiwari D,\u00a0Gupta S,\u00a0Rogers J,\u00a0Maxwell D,\u00a0Rech P,\u00a0Vazhkudai S,\u00a0Oliveira D,\u00a0Londo D,\u00a0DeBardeleben N,\u00a0Navaux P, et\u00a0al (2015) Understanding gpu errors on large-scale hpc systems and the implications for system design and operation. In: 2015 IEEE 21st International symposium on high performance computer architecture (HPCA). IEEE, pp 331\u2013342","DOI":"10.1109\/HPCA.2015.7056044"},{"key":"10579_CR63","unstructured":"Wakabayashi D (2023) Self-driving uber car kills pedestrian in arizona, where robots roam. Mar 2018, accessed on December 17, 2023. [Online]. Available: https:\/\/www.nytimes.com\/2018\/03\/19\/technology\/uber-driverless-fatality.html"},{"key":"10579_CR64","doi-asserted-by":"crossref","unstructured":"White M,\u00a0Linares-V\u00e1squez M,\u00a0Johnson P,\u00a0Bernal-C\u00e1rdenas C,\u00a0Poshyvanyk D (2015) Generating reproducible and replayable bug reports from android application crashes. In: 2015 IEEE 23rd International conference on program comprehension, pp 48\u201359","DOI":"10.1109\/ICPC.2015.14"},{"key":"10579_CR65","doi-asserted-by":"crossref","unstructured":"Yan M,\u00a0Chen J,\u00a0Zhang X,\u00a0Tan L,\u00a0Wang G,\u00a0Wang Z (2021) Exposing numerical bugs in deep learning via gradient back-propagation. In: Proceedings of the 29th ACM joint meeting on european software engineering conference and symposium on the foundations of software engineering, pp 627\u2013638","DOI":"10.1145\/3468264.3468612"},{"key":"10579_CR66","doi-asserted-by":"publisher","DOI":"10.1016\/j.infsof.2022.107004","volume":"151","author":"Y Yang","year":"2022","unstructured":"Yang Y, He T, Xia Z, Feng Y (2022) A comprehensive empirical study on bug characteristics of deep learning frameworks. Inf Softw Technol 151:107004","journal-title":"Inf Softw Technol"},{"key":"10579_CR67","doi-asserted-by":"publisher","unstructured":"Zhang Y,\u00a0Chen Y, Cheung S-C,\u00a0Xiong Y,\u00a0Zhang L (2018) An empirical study on tensorflow program bugs. In: Proceedings of the 27th ACM SIGSOFT international symposium on software testing and analysis, ser. ISSTA 2018. New York, NY, USA: Association for Computing Machinery, pp 129\u2013140. [Online]. Available: https:\/\/doi.org\/10.1145\/3213846.3213866","DOI":"10.1145\/3213846.3213866"},{"key":"10579_CR68","doi-asserted-by":"crossref","unstructured":"Zhang T,\u00a0Gao C,\u00a0Ma L,\u00a0Lyu M,\u00a0Kim M (2019) An empirical study of common challenges in developing deep learning applications. In: 2019 IEEE 30th International symposium on software reliability engineering (ISSRE). IEEE, pp 104\u2013115","DOI":"10.1109\/ISSRE.2019.00020"},{"key":"10579_CR69","unstructured":"Zhang K,\u00a0He B,\u00a0Hu J,\u00a0Wang Z,\u00a0Hua B,\u00a0Meng J,\u00a0Yang L (2018) $$\\{$$G-NET$$\\}$$: Effective $$\\{$$GPU$$\\}$$ sharing in $$\\{$$NFV$$\\}$$ systems. In: 15th USENIX Symposium on networked systems design and implementation (NSDI 18), pp 187\u2013200"},{"key":"10579_CR70","doi-asserted-by":"crossref","unstructured":"Zhang Y,\u00a0Ren L,\u00a0Chen L,\u00a0Xiong Y, Cheung S-C,\u00a0Xie T (2020) Detecting numerical bugs in neural network architectures. In: Proceedings of the 28th ACM joint meeting on european software engineering conference and symposium on the foundations of software engineering, pp 826\u2013837","DOI":"10.1145\/3368089.3409720"},{"key":"10579_CR71","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1007\/s11432-019-3018-6","volume":"64","author":"H Zhao","year":"2021","unstructured":"Zhao H, Li Y, Liu F, Xie X, Chen L (2021) State and tendency: an empirical study of deep learning question &answer topics on stack overflow. Sci China Inf Sci 64:1\u201323","journal-title":"Sci China Inf Sci"}],"container-title":["Empirical Software Engineering"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10664-024-10579-w.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s10664-024-10579-w\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10664-024-10579-w.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,1,7]],"date-time":"2025-01-07T14:14:04Z","timestamp":1736259244000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s10664-024-10579-w"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,11,9]]},"references-count":71,"journal-issue":{"issue":"1","published-print":{"date-parts":[[2025,1]]}},"alternative-id":["10579"],"URL":"https:\/\/doi.org\/10.1007\/s10664-024-10579-w","relation":{},"ISSN":["1382-3256","1573-7616"],"issn-type":[{"value":"1382-3256","type":"print"},{"value":"1573-7616","type":"electronic"}],"subject":[],"published":{"date-parts":[[2024,11,9]]},"assertion":[{"value":"21 October 2024","order":1,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"9 November 2024","order":2,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Declarations"}},{"value":"The authors declare that they have no conflict of interest.","order":2,"name":"Ethics","group":{"name":"EthicsHeading","label":"Conflict of Interest"}}],"article-number":"23"}}