{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,17]],"date-time":"2026-07-17T14:58:32Z","timestamp":1784300312209,"version":"3.55.0"},"reference-count":86,"publisher":"Springer Science and Business Media LLC","issue":"3","license":[{"start":{"date-parts":[[2025,2,13]],"date-time":"2025-02-13T00:00:00Z","timestamp":1739404800000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2025,2,13]],"date-time":"2025-02-13T00:00:00Z","timestamp":1739404800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"funder":[{"DOI":"10.13039\/501100000038","name":"Natural Sciences and Engineering Research Council of Canada","doi-asserted-by":"publisher","id":[{"id":"10.13039\/501100000038","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100020951","name":"Fonds de Recherche du Qu\u00e9bec","doi-asserted-by":"publisher","id":[{"id":"10.13039\/501100020951","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100007631","name":"Canadian Institute for Advanced Research","doi-asserted-by":"publisher","id":[{"id":"10.13039\/100007631","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100015773","name":"Consortium de Recherche et d\u2019innovation en A\u00e9rospatiale au Qu\u00e9be","doi-asserted-by":"publisher","id":[{"id":"10.13039\/100015773","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Empir Software Eng"],"published-print":{"date-parts":[[2025,5]]},"DOI":"10.1007\/s10664-025-10614-4","type":"journal-article","created":{"date-parts":[[2025,2,13]],"date-time":"2025-02-13T04:52:19Z","timestamp":1739422339000},"update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":55,"title":["Bugs in large language models generated code: an empirical study"],"prefix":"10.1007","volume":"30","author":[{"ORCID":"https:\/\/orcid.org\/0000-0001-5593-9400","authenticated-orcid":false,"given":"Florian","family":"Tambon","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Arghavan","family":"Moradi-Dakhel","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Amin","family":"Nikanjam","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Foutse","family":"Khomh","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Michel C.","family":"Desmarais","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Giuliano","family":"Antoniol","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2025,2,13]]},"reference":[{"key":"10614_CR1","unstructured":"Anthropic (2013) Model card and evaluations for claude models. https:\/\/www.anthropic.com\/index\/introducing-claude"},{"key":"10614_CR2","doi-asserted-by":"publisher","unstructured":"Asare O, Nagappan M, Asokan N (2023) Is github\u2019s copilot as bad as humans at introducing vulnerabilities in code? Empirical Softw Eng 28(6). https:\/\/doi.org\/10.1007\/s10664-023-10380-1","DOI":"10.1007\/s10664-023-10380-1"},{"issue":"6","key":"10614_CR3","doi-asserted-by":"publisher","first-page":"35","DOI":"10.1145\/3582083","volume":"20","author":"C Bird","year":"2022","unstructured":"Bird C, Ford D, Zimmermann T, Forsgren N, Kalliamvakou E, Lowdermilk T, Gazit I (2022) Taking flight with copilot: early insights and opportunities of ai-powered pair-programming tools. Queue 20(6):35\u201357","journal-title":"Queue"},{"key":"10614_CR4","doi-asserted-by":"crossref","unstructured":"Bogatinovski J, Kao O (2023) Auto-logging: AI-centred logging instrumentation. In: 2023 IEEE\/ACM 45th international conference on software engineering: new ideas and emerging results (ICSE-NIER). IEEE, pp 95\u2013100","DOI":"10.1109\/ICSE-NIER58687.2023.00023"},{"key":"10614_CR5","first-page":"1877","volume":"33","author":"T Brown","year":"2020","unstructured":"Brown T, Mann B, Ryder N, Subbiah M, Kaplan JD, Dhariwal P, Neelakantan A, Shyam P, Sastry G, Askell A et al (2020) Language models are few-shot learners. Adv Neural Inf Process Syst 33:1877\u20131901","journal-title":"Adv Neural Inf Process Syst"},{"key":"10614_CR6","unstructured":"Cater-Steel A, Toleman M, Rout T (2005) Addressing the challenges of replications of surveys in software engineering research. In: 2005 International symposium on empirical software engineering, 2005. IEEE, pp 10\u2013pp"},{"key":"10614_CR7","doi-asserted-by":"publisher","first-page":"165","DOI":"10.1016\/j.jss.2019.03.002","volume":"152","author":"G Catolino","year":"2019","unstructured":"Catolino G, Palomba F, Zaidman A, Ferrucci F (2019) Not all bugs are the same: understanding, characterizing, and classifying bug types. J Syst Softw 152:165\u2013181","journal-title":"J Syst Softw"},{"key":"10614_CR8","doi-asserted-by":"crossref","unstructured":"Chen Z, Cao Y, Liu Y, Wang H, Xie T, Liu X (2020) A comprehensive study on challenges in deploying deep learning based software. In: Proceedings of the 28th ACM joint meeting on European software engineering conference and symposium on the foundations of software engineering pp 750\u2013762","DOI":"10.1145\/3368089.3409759"},{"key":"10614_CR9","unstructured":"Chen X, Lin M, Sch\u00e4rli N, Zhou D (2023) Teaching large language models to self-debug. arXiv:2304.05128"},{"key":"10614_CR10","unstructured":"Chen M, Tworek J, Jun H, Yuan Q, de\u00a0Oliveira\u00a0Pinto HP, Kaplan J, Edwards H, Burda Y, Joseph Y, Brockman G, Ray A, Puri R, Krueger G, Petrov M, Khlaaf H, Sastry G, Mishkin P, Chan B, Gray S, Ryder N, Pavlov M, Power A, Kaiser L, Bavarian M, Winter C, Tillet P, Such FP, Cummings D, Plappert M, Chantzis F, Barnes E, Herbert-Voss A, Guss WH, Nichol A, Paino A, Tezak N, Tang J, Babuschkin I, Balaji S, Jain S, Saunders W, Hesse C, Carr AN, Leike J, Achiam J, Misra V, Morikawa E, Radford A, Knight M, Brundage M, Murati M, Mayer K, Welinder P, McGrew B, Amodei D, McCandlish S, Sutskever I, Zaremba W (2021) Evaluating large language models trained on code"},{"key":"10614_CR11","unstructured":"Chen M, Tworek J, Jun H, Yuan Q, Pinto HPdO, Kaplan J, Edwards H, Burda Y, Joseph N, Brockman G et\u00a0al (2021) Evaluating large language models trained on code. arXiv:2107.03374"},{"key":"10614_CR12","unstructured":"Chen B, Zhang F, Nguyen A, Zan D, Lin Z, Lou J-G, Chen W (2022) Codet: code generation with generated tests. arXiv:2207.10397"},{"issue":"240","key":"10614_CR13","first-page":"1","volume":"24","author":"A Chowdhery","year":"2023","unstructured":"Chowdhery A, Narang S, Devlin J, Bosma M, Mishra G, Roberts A, Barham P, Chung HW, Sutton C, Gehrmann S et al (2023) Palm: scaling language modeling with pathways. J Mach Learn Res 24(240):1\u2013113","journal-title":"J Mach Learn Res"},{"key":"10614_CR14","unstructured":"Christopoulou F, Lampouras G, Gritta M, Zhang G, Guo Y, Li Z, Zhang Q, Xiao M, Shen M, Li L et\u00a0al (2022) Pangu-coder: program synthesis with function-level language modeling. arXiv:2207.11280"},{"key":"10614_CR15","doi-asserted-by":"crossref","unstructured":"Cohen J (2013) Statistical power analysis for the behavioral sciences. Routledge","DOI":"10.4324\/9780203771587"},{"key":"10614_CR16","unstructured":"Cram\u00e9r H (1999) Mathematical methods of statistics. Princeton university press, vol\u00a026"},{"key":"10614_CR17","doi-asserted-by":"crossref","unstructured":"Du X, Liu M, Wang K, Wang H, Liu J, Chen Y, Feng J, Sha C, Peng X, Lou Y (2023) Classeval: a manually-crafted benchmark for evaluating llms on class-level code generation","DOI":"10.1145\/3597503.3639219"},{"key":"10614_CR18","doi-asserted-by":"crossref","unstructured":"Fan Z, Gao X, Mirchev M, Roychoudhury A, Tan SH (2023) Automated repair of programs from large language models. In: 2023 IEEE\/ACM 45th international conference on software engineering (ICSE). IEEE, pp 1469\u20131481","DOI":"10.1109\/ICSE48619.2023.00128"},{"key":"10614_CR19","doi-asserted-by":"crossref","unstructured":"Fischer F, B\u00f6ttinger K, Xiao H, Stransky C, Acar Y, Backes M, Fahl S (2017) Stack overflow considered harmful? the impact of copy & paste on android application security. In: IEEE symposium on security and privacy (SP) pp 121\u2013136","DOI":"10.1109\/SP.2017.31"},{"key":"10614_CR20","doi-asserted-by":"crossref","unstructured":"Fisher RA (1922) On the interpretation of $$\\chi $$ 2 from contingency tables, and the calculation of p. J R Stat Soc 85(1):87\u201394. http:\/\/www.jstor.org\/stable\/2340521","DOI":"10.2307\/2340521"},{"key":"10614_CR21","doi-asserted-by":"publisher","unstructured":"Fraser G, Arcuri A (2014) A large-scale evaluation of automated unit test generation using evosuite. ACM Trans Softw Eng Methodol 24(2). https:\/\/doi.org\/10.1145\/2685612","DOI":"10.1145\/2685612"},{"key":"10614_CR22","unstructured":"Fu Y, Liang P, Tahir A, Li Z, Shahin M, Yu J (2023) Security weaknesses of copilot generated code in github. arXiv:2310.02059"},{"key":"10614_CR23","unstructured":"gitindex (2023). https:\/\/githut.info\/"},{"key":"10614_CR24","unstructured":"Google forms (2022) https:\/\/www.google.ca\/forms\/about\/"},{"key":"10614_CR25","doi-asserted-by":"publisher","unstructured":"Guilherme V, Vincenzi A (2023) An initial investigation of chatgpt unit test generation capability. In: Proceedings of the 8th brazilian symposium on systematic and automated software testing, ser. SAST \u201923. New York, NY, USA: Association for Computing Machinery pp 15-24. https:\/\/doi.org\/10.1145\/3624032.3624035","DOI":"10.1145\/3624032.3624035"},{"issue":"4","key":"10614_CR26","doi-asserted-by":"publisher","first-page":"e1751","DOI":"10.1002\/stvr.1751","volume":"31","author":"P Gyimesi","year":"2021","unstructured":"Gyimesi P, Vancsics B, Stocco A, Mazinanian D, Besz\u00e9des \u00c1, Ferenc R, Mesbah A (2021) Bugsjs: a benchmark and taxonomy of javascript bugs. Softw Test Verif Reliab 31(4):e1751","journal-title":"Softw Test Verif Reliab"},{"key":"10614_CR27","unstructured":"Honarvar S, van\u00a0der Wilk M, Donaldson A (2023) Turbulence: systematically and automatically testing instruction-tuned large language models for code. arXiv:2312.14856"},{"key":"10614_CR28","unstructured":"Huang D, Zhang JM, Qing Y, Cui H (2024) Effibench: benchmarking the efficiency of automatically generated code"},{"key":"10614_CR29","doi-asserted-by":"crossref","unstructured":"Humbatova N, Jahangirova G, Bavota G, Riccio V, Stocco A, Tonella P (2020) Taxonomy of real faults in deep learning systems. In: Proceedings of the ACM\/IEEE 42nd international conference on software engineering pp 1110\u20131121","DOI":"10.1145\/3377811.3380395"},{"key":"10614_CR30","doi-asserted-by":"crossref","unstructured":"Imai S (2022) Is github copilot a substitute for human pair-programming? an empirical study. In: Proceedings of the ACM\/IEEE 44th International Conference on Software Engineering: Companion Proceedings pp 319\u2013321","DOI":"10.1145\/3510454.3522684"},{"key":"10614_CR31","doi-asserted-by":"crossref","unstructured":"Jesse Z, Ahmed T, Devanbu PT, Morgan E (2023) Large language models and simple, stupid bugs. In: 2023 IEEE\/ACM 20th international conference on mining software repositories (MSR). Los Alamitos, CA, USA: IEEE Computer Society, pp. 563\u2013575. https:\/\/doi.ieeecomputersociety.org\/10.1109\/MSR59073.2023.00082","DOI":"10.1109\/MSR59073.2023.00082"},{"key":"10614_CR32","unstructured":"Jiang J, Wang F, Shen J, Kim S, Kim S (2024) A survey on large language models for code generation. arXiv:2406.00515"},{"key":"10614_CR33","unstructured":"Ji Z, Ma P, Li Z, Wang S (2023) Benchmarking and explaining large language model-based code generation: a causality-centric approach"},{"key":"10614_CR34","doi-asserted-by":"publisher","unstructured":"Jin M, Shahriar S, Tufano M, Shi X, Lu S, Sundaresan N, Svyatkovskiy A (2023) Inferfix: end-to-end program repair with llms,\u201d In: Proceedings of the 31st ACM joint european software engineering conference and symposium on the foundations of software engineering, ser. ESEC\/FSE 2023. New York, NY, USA: Association for Computing Machinery, pp 1646-1656. https:\/\/doi.org\/10.1145\/3611643.3613892","DOI":"10.1145\/3611643.3613892"},{"key":"10614_CR35","doi-asserted-by":"crossref","unstructured":"Just R, Jalali D, Inozemtseva L, Ernst MD, Holmes R, Fraser G (2014) Are mutants a valid substitute for real faults in software testing?. In: Proceedings of the 22nd ACM SIGSOFT international symposium on foundations of software engineering, ser. FSE 2014. New York, NY, USA: Association for Computing Machinery pp 654-665. https:\/\/doi.org\/10.1145\/2635868.2635929","DOI":"10.1145\/2635868.2635929"},{"key":"10614_CR36","doi-asserted-by":"crossref","unstructured":"Kazemitabaar M, Hou X, Henley A, Ericson BJ, Weintrop D, Grossman T (2023) How novices use llm-based code generators to solve cs1 coding tasks in a self-paced learning environment. arXiv:2309.14049","DOI":"10.1145\/3631802.3631806"},{"key":"10614_CR37","unstructured":"Kou B, Chen S, Wang S, Ma L, Zhang T (2023) Is model attention aligned with human attention? an empirical study on large language models for code generation"},{"issue":"12","key":"10614_CR38","doi-asserted-by":"publisher","first-page":"1236","DOI":"10.1109\/TSE.2015.2454513","volume":"41","author":"C Le Goues","year":"2015","unstructured":"Le Goues C, Holtschulte N, Smith EK, Brun Y, Devanbu P, Forrest S, Weimer W (2015) The manybugs and introclass benchmarks for automated repair of c programs. IEEE Trans Softw Eng 41(12):1236\u20131256","journal-title":"IEEE Trans Softw Eng"},{"key":"10614_CR39","unstructured":"Leetcode contest (2023). https:\/\/leetcode.com\/contest\/"},{"key":"10614_CR40","doi-asserted-by":"crossref","unstructured":"Lemieux C, Inala JP, Lahiri SK, Sen S (2023) Codamosa: escaping coverage plateaus in test generation with pre-trained large language models. In: Accepted by 45th international conference on software engineering (ICSE)","DOI":"10.1109\/ICSE48619.2023.00085"},{"key":"10614_CR41","unstructured":"Li J, Li G, Li Y, Jin Z (2023) Structured chain-of-thought prompting for code generation. arXiv:2305.06599"},{"key":"10614_CR42","doi-asserted-by":"crossref","unstructured":"Liu MX, Sarkar A, Negreanu C, Zorn B, Williams J, Toronto N, Gordon AD (2023) \u201cwhat it wants me to say\u201d: bridging the abstraction gap between end-user programmers and code-generating large language models. In: Proceedings of the 2023 CHI conference on human factors in computing systems pp 1\u201331","DOI":"10.1145\/3544548.3580817"},{"key":"10614_CR43","doi-asserted-by":"crossref","unstructured":"Liu Y, Le-Cong T, Widyasari R, Tantithamthavorn C, Li L, Le X-BD, Lo D (2023) Refining chatgpt-generated code: characterizing and mitigating code quality issues","DOI":"10.1145\/3643674"},{"key":"10614_CR44","unstructured":"Liu Y, Tantithamthavorn C, Liu Y, Li L (2023) On the reliability and explainability of automated code generation approaches. arXiv:2302.09587"},{"key":"10614_CR45","unstructured":"Liu J, Xia CS, Wang Y, Zhang L (2023) Is your code generated by chatgpt really correct? rigorous evaluation of large language models for code generation. arXiv:2305.01210"},{"key":"10614_CR46","unstructured":"Liu C, Zhang SD, Jabbarvand R (2024) Codemind: a framework to challenge large language models for code reasoning"},{"key":"10614_CR47","doi-asserted-by":"crossref","unstructured":"Li Z, Wang C, Liu Z, Wang H, Chen D, Wang S, Gao C (2023) Cctest: testing and repairing code completion systems. In: 2023 IEEE\/ACM 45th international conference on software engineering (ICSE) pp 1238\u20131250","DOI":"10.1109\/ICSE48619.2023.00110"},{"key":"10614_CR48","doi-asserted-by":"crossref","unstructured":"Li J, Zhang Y, Karas Z, McMillan C, Leach K, Huang Y (2024) Do machines and humans focus on similar code? exploring explainability of large language models in code summarization. arXiv:2402.14182","DOI":"10.1145\/3643916.3644434"},{"key":"10614_CR49","unstructured":"Lu S, Guo D, Ren S, Huang J, Svyatkovskiy A, Blanco A, Clement C, Drain D, Jiang D, Tang D et\u00a0al (2021) Codexglue: a machine learning benchmark dataset for code understanding and generation. arXiv:2102.04664"},{"key":"10614_CR50","doi-asserted-by":"crossref","unstructured":"Mastropaolo A, Pascarella L, Guglielmi E, Ciniselli M, Scalabrino S, Oliveto R, Bavota G (2023) On the robustness of code generation techniques: an empirical study on github copilot. arXiv:2302.00438","DOI":"10.1109\/ICSE48619.2023.00181"},{"issue":"1","key":"10614_CR51","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1145\/3105906","volume":"51","author":"M Monperrus","year":"2018","unstructured":"Monperrus M (2018) Automatic software repair: a bibliography. ACM Comput Surv 51(1):1. https:\/\/doi.org\/10.1145\/3105906","journal-title":"ACM Comput Surv"},{"key":"10614_CR52","unstructured":"Moon S, Song Y, Chae H, Kang D, Kwon T, iunn Ong KT, won Hwang S, Yeo J (2023) Coffee: boost your code llms by fixing bugs with feedback"},{"key":"10614_CR53","doi-asserted-by":"crossref","unstructured":"Moradi Dakhel A, Majdinasab V, Nikanjam A, Khomh F, Desmarais MC, Jiang ZMJ (2023) Github copilot ai pair programmer: asset or liability. J Syst Softw 203:111734. https:\/\/www.sciencedirect.com\/science\/article\/pii\/S0164121223001292","DOI":"10.1016\/j.jss.2023.111734"},{"issue":"1","key":"10614_CR54","doi-asserted-by":"publisher","first-page":"14","DOI":"10.1007\/s10664-023-10400-0","volume":"29","author":"MM Morovati","year":"2024","unstructured":"Morovati MM, Nikanjam A, Tambon F, Khomh F, Jiang ZM (2024) Bug characterization in machine learning-based systems. Empir Softw Eng 29(1):14","journal-title":"Empir Softw Eng"},{"key":"10614_CR55","unstructured":"Muennighoff N, Liu Q, Zebaze A, Zheng Q, Hui B, Zhuo TY, Singh S, Tang X, von Werra L, Longpre S (2023) Octopack: instruction tuning code large language models. arXiv:2308.07124"},{"key":"10614_CR56","doi-asserted-by":"publisher","unstructured":"Nardone V, Muse B, Abidi M, Khomh F, Di Penta M (2023) Video game bad smells: what they are and how developers perceive them. ACM Trans Softw Eng Methodol 32(4). https:\/\/doi.org\/10.1145\/3563214","DOI":"10.1145\/3563214"},{"key":"10614_CR57","doi-asserted-by":"crossref","unstructured":"Nguyen N, Nadi S (2022) An empirical evaluation of github copilot\u2019s code suggestions. In: Proceedings of the 19th International Conference on Mining Software Repositories pp 1\u20135","DOI":"10.1145\/3524842.3528470"},{"key":"10614_CR58","unstructured":"Nijkamp E, Pang B, Hayashi H, Tu L, Wang H, Zhou Y, Savarese S, Xiong C (2022) Codegen: an open large language model for code with multi-turn program synthesis. arXiv:2203.13474"},{"key":"10614_CR59","doi-asserted-by":"crossref","unstructured":"Nikanjam A, Morovati MM, Khomh F, BenBraiek H (2021) Faults in deep reinforcement learning programs: a taxonomy and a detection approach. Autom Softw Eng 29","DOI":"10.1007\/s10515-021-00313-x"},{"key":"10614_CR60","unstructured":"Oppenheim AN (2000) Questionnaire design, interviewing and attitude measurement. Bloomsbury Publishing"},{"key":"10614_CR61","doi-asserted-by":"publisher","first-page":"286","DOI":"10.1007\/s10664-008-9077-5","volume":"14","author":"K Pan","year":"2009","unstructured":"Pan K, Kim S, Whitehead EJ (2009) Toward an understanding of bug fix patterns. Empir Softw Eng 14:286\u2013315","journal-title":"Empir Softw Eng"},{"key":"10614_CR62","doi-asserted-by":"crossref","unstructured":"Pan R, Ibrahimzada AR, Krishna R, Sankar D, Wassi LP, Merler M, Sobolev B, Pavuluri R, Sinha S, Jabbarvand R (2023) Lost in translation: a study of bugs introduced by large language models while translating code. arXiv:2308.03109v3","DOI":"10.1145\/3597503.3639226"},{"key":"10614_CR63","doi-asserted-by":"crossref","unstructured":"Sakib FA, Khan SH, Karim A (2023) Extending the frontier of chatgpt: code generation and debugging. arXiv:2307.08260","DOI":"10.1109\/ICECET61485.2024.10698405"},{"key":"10614_CR64","unstructured":"Sch\u00e4fer M, Nadi S, Eghbali A, Tip F (2023) Adaptive test generation using a large language model. arXiv:2302.06527"},{"key":"10614_CR65","unstructured":"Schulman J, Zoph B, Kim JH, Menick J, Weng J, Uribe JF, Fedus L, Metz L, Pokorny M, et al (2022) Chatgpt: optimizing language models for dialogue"},{"key":"10614_CR66","doi-asserted-by":"crossref","unstructured":"Scoccia GL (2023) Exploring early adopters\u2019 perceptions of chatgpt as a code generation tool. In: 2023 38th IEEE\/ACM international conference on automated software engineering workshops (ASEW) pp 88\u201393","DOI":"10.1109\/ASEW60602.2023.00016"},{"issue":"4","key":"10614_CR67","doi-asserted-by":"publisher","first-page":"557","DOI":"10.1109\/32.799955","volume":"25","author":"CB Seaman","year":"1999","unstructured":"Seaman CB (1999) Qualitative methods in empirical studies of software engineering. IEEE Trans Softw Eng 25(4):557\u2013572","journal-title":"IEEE Trans Softw Eng"},{"key":"10614_CR68","doi-asserted-by":"crossref","unstructured":"Spadini D, Aniche M, Bacchelli A (2018) Pydriller: python framework for mining software repositories. In: Proceedings of the 2018 26th ACM Joint meeting on european software engineering conference and symposium on the foundations of software engineering pp 908\u2013911","DOI":"10.1145\/3236024.3264598"},{"issue":"1","key":"10614_CR69","doi-asserted-by":"publisher","first-page":"10","DOI":"10.1007\/s10664-023-10389-6","volume":"29","author":"F Tambon","year":"2024","unstructured":"Tambon F, Nikanjam A, An L, Khomh F, Antoniol G (2024) Silent bugs in deep learning frameworks: an empirical study of keras and tensorflow. Empir Softw Eng 29(1):10","journal-title":"Empir Softw Eng"},{"key":"10614_CR70","unstructured":"Tan SH, Yi J, Mechtaev S, Roychoudhury A, et\u00a0al (2017) Codeflaws: a programming competition benchmark for evaluating automated program repair tools. In: 2017 IEEE\/ACM 39th international conference on software engineering companion (ICSE-C). IEEE, pp 180\u2013182"},{"key":"10614_CR71","doi-asserted-by":"publisher","first-page":"1665","DOI":"10.1007\/s10664-013-9258-8","volume":"19","author":"L Tan","year":"2014","unstructured":"Tan L, Liu C, Li Z, Wang X, Zhou Y, Zhai C (2014) Bug characteristics in open source software. Empir Softw Eng 19:1665\u20131705","journal-title":"Empir Softw Eng"},{"key":"10614_CR72","doi-asserted-by":"crossref","unstructured":"Tang Y, Liu Z, Zhou Z, Luo X (2023) Chatgpt vs sbst: a comparative assessment of unit test suite generation","DOI":"10.1109\/TSE.2024.3382365"},{"key":"10614_CR73","unstructured":"The replication package (2023). https:\/\/github.com\/FlowSs\/BugsInLLMs"},{"key":"10614_CR74","unstructured":"tiobeindex (2023). https:\/\/www.tiobe.com\/tiobe-index\/"},{"key":"10614_CR75","unstructured":"Touvron H, Martin L, Stone K, Albert P, Almahairi A, Babaei Y, Bashlykov N, Batra S, Bhargava P, Bhosale S et\u00a0al (2023) Llama 2: open foundation and fine-tuned chat models. arXiv:2307.09288"},{"key":"10614_CR76","doi-asserted-by":"crossref","unstructured":"Vaithilingam P, Zhang T, Glassman EL (2022) Expectation vs. experience: evaluating the usability of code generation tools powered by large language models. In: CHI conference on human factors in computing systems extended abstracts pp 1\u20137","DOI":"10.1145\/3491101.3519665"},{"issue":"5","key":"10614_CR77","doi-asserted-by":"publisher","first-page":"1497","DOI":"10.1109\/TSE.2020.3023664","volume":"48","author":"M Verdi","year":"2022","unstructured":"Verdi M, Sami A, Akhondali J, Khomh F, Uddin G, Motlagh AK (2022) An empirical study of c++ vulnerabilities in crowd-sourced code examples. IEEE Trans Softw Eng 48(5):1497\u20131514. https:\/\/doi.org\/10.1109\/TSE.2020.3023664","journal-title":"IEEE Trans Softw Eng"},{"key":"10614_CR78","unstructured":"Wei J, Wang X, Schuurmans D, Bosma M, ichter b, Xia F, Chi E, Le QV, Zhou D (2022) Chain-of-thought prompting elicits reasoning in large language models. In: Koyejo S, Mohamed S, Agarwal A, Belgrave D, Cho K, Oh A (eds) Advances in neural information processing systems, vol\u00a035. Curran Associates, Inc., pp 24\u00a0824\u201324\u00a0837. https:\/\/proceedings.neurips.cc\/paper_files\/paper\/2022\/file\/9d5609613524ecf4f15af0f7b31abca4-Paper-Conference.pdf"},{"key":"10614_CR79","unstructured":"Wong D, Kothig A, Lam P (2022) Exploring the verifiability of code generated by github copilot. arXiv:2209.01766"},{"key":"10614_CR80","doi-asserted-by":"crossref","unstructured":"Yu H, Shen B, Ran D, Zhang J, Zhang Q, Ma Y, Liang G, Li Y, Wang Q, Xie T (2024) Codereval: a benchmark of pragmatic code generation with generative pre-trained models. In: Proceedings of the 46th IEEE\/ACM international conference on software engineering pp 1\u201312","DOI":"10.1145\/3597503.3623316"},{"key":"10614_CR81","doi-asserted-by":"crossref","unstructured":"Yu H, Shen B, Ran D, Zhang J, Zhang Q, Ma Y, Liang G, Li Y, Xie T, Wang Q (2023) Codereval: a benchmark of pragmatic code generation with generative pre-trained models. arXiv:2302.00288v1","DOI":"10.1145\/3597503.3623316"},{"key":"10614_CR82","doi-asserted-by":"crossref","unstructured":"Zan D, Chen B, Zhang F, Lu D, Wu B, Guan B, Wang Y, Lou J-G (2022) When neural model meets nl2code: a survey. arXiv:2212.09420","DOI":"10.18653\/v1\/2023.acl-long.411"},{"key":"10614_CR83","unstructured":"Zeng W, Ren X, Su T, Wang H, Liao Y, Wang Z, Jiang X, Yang Z, Wang K, Zhang X et\u00a0al (2021) Pangu-$$\\alpha $$: large-scale autoregressive pretrained chinese language models with auto-parallel computation. arXiv:2104.12369"},{"key":"10614_CR84","doi-asserted-by":"crossref","unstructured":"Zhang T, Gao C, Ma L, Lyu M, Kim M (2019) An empirical study of common challenges in developing deep learning applications. In: 2019 IEEE 30th international symposium on software reliability engineering (ISSRE) pp 104\u2013115","DOI":"10.1109\/ISSRE.2019.00020"},{"key":"10614_CR85","doi-asserted-by":"crossref","unstructured":"Zhang T, Upadhyaya G, Reinhardt A, Rajan H, Kim M (2018) Are code examples on an online q & a forum reliable?: a study of api misuse on stack overflow,\u201d In: 2018 IEEE\/ACM 40th international conference on software engineering (ICSE) pp 886\u2013896","DOI":"10.1145\/3180155.3180260"},{"key":"10614_CR86","doi-asserted-by":"publisher","unstructured":"Zid C, Zampetti F, Antoniol G, Di\u00a0Penta M (2024) A study on the pythonic functional constructs\u2019 understandability. In: Proceedings of the IEEE\/ACM 46th international conference on software engineering, ser. ICSE \u201924. New York, NY, USA: Association for Computing Machinery. https:\/\/doi.org\/10.1145\/3597503.3639211","DOI":"10.1145\/3597503.3639211"}],"container-title":["Empirical Software Engineering"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10664-025-10614-4.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s10664-025-10614-4\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10664-025-10614-4.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,11,20]],"date-time":"2025-11-20T13:28:45Z","timestamp":1763645325000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s10664-025-10614-4"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,2,13]]},"references-count":86,"journal-issue":{"issue":"3","published-print":{"date-parts":[[2025,5]]}},"alternative-id":["10614"],"URL":"https:\/\/doi.org\/10.1007\/s10664-025-10614-4","relation":{},"ISSN":["1382-3256","1573-7616"],"issn-type":[{"value":"1382-3256","type":"print"},{"value":"1573-7616","type":"electronic"}],"subject":[],"published":{"date-parts":[[2025,2,13]]},"assertion":[{"value":"10 January 2025","order":1,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"13 February 2025","order":2,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Declarations"}},{"value":"The authors declare that they have no Conflict of interest.","order":2,"name":"Ethics","group":{"name":"EthicsHeading","label":"Conflict of interest"}}],"article-number":"65"}}