{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,1]],"date-time":"2026-04-01T04:40:24Z","timestamp":1775018424141,"version":"3.50.1"},"reference-count":68,"publisher":"Pleiades Publishing Ltd","issue":"8","license":[{"start":{"date-parts":[[2025,12,1]],"date-time":"2025-12-01T00:00:00Z","timestamp":1764547200000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2025,12,1]],"date-time":"2025-12-01T00:00:00Z","timestamp":1764547200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Program Comput Soft"],"published-print":{"date-parts":[[2025,12]]},"DOI":"10.1134\/s0361768825700446","type":"journal-article","created":{"date-parts":[[2026,2,27]],"date-time":"2026-02-27T10:50:39Z","timestamp":1772189439000},"page":"587-604","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":0,"title":["State of the Art of the Security of Code Generated by LLMs: A Multivocal Literature Review"],"prefix":"10.1134","volume":"51","author":[{"ORCID":"https:\/\/orcid.org\/0009-0000-0682-9854","authenticated-orcid":false,"given":"Leonardo Criollo","family":"Ram\u00edrez","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-4654-636X","authenticated-orcid":false,"given":"Xavier","family":"Lim\u00f3n","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-2917-2960","authenticated-orcid":false,"given":"\u00c1ngel J.","family":"S\u00e1nchez-Garc\u00eda","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-2354-2462","authenticated-orcid":false,"given":"Juan Carlos","family":"P\u00e9rez-Arriaga","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"137","published-online":{"date-parts":[[2026,2,27]]},"reference":[{"key":"3976_CR1","doi-asserted-by":"publisher","unstructured":"Ram\u00edrez, L.C., Lim\u00f3n, X., S\u00e1nchez-Garc\u00eda, \u00c1.J., and P\u00e9rez-Arriaga, J.C., State of the art of the security of code generated by LLMs: A systematic literature review, 2024 12th International Conference in Software Engineering Research and Innovation (CONISOFT), Escondido, Mexico, 2024, IEEE, 2024, pp. 331\u2013339. https:\/\/doi.org\/10.1109\/conisoft63288.2024.00050","DOI":"10.1109\/conisoft63288.2024.00050"},{"key":"3976_CR2","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-21448-6_2","volume-title":"(Research for Policy), Artificial intelligence: Definition and background, Mission AI","author":"H. Sheikh","year":"2023","unstructured":"Sheikh, H., Prins, C., and Schrijvers, E. (Research for Policy), Artificial intelligence: Definition and background, Mission AI, Cham: Springer, 2023, pp. 15\u201341. https:\/\/doi.org\/10.1007\/978-3-031-21448-6_2"},{"key":"3976_CR3","unstructured":"Dastin, J., Microsoft attracting users to its code-writing, generative AI software, Reuters, 2023. https:\/\/www.reuters.com\/technology\/microsoft-attracting-users-its-code-writing-generative-ai-software-2023-01-25\/."},{"key":"3976_CR4","doi-asserted-by":"publisher","unstructured":"Al-Kaswan, A. and Izadi, M., The (ab)use of open source code to train large language models, 2023 IEEE\/ACM 2nd International Workshop on Natural Language-Based Software Engineering (NLBSE), Melbourne, 2023, IEEE, 2023, pp. 9\u201310. https:\/\/doi.org\/10.1109\/nlbse59153.2023.00008","DOI":"10.1109\/nlbse59153.2023.00008"},{"key":"3976_CR5","doi-asserted-by":"publisher","unstructured":"Improta, C., Poisoning programs by un-repairing code: Security concerns of AI-generated code, 2023 IEEE 34th International Symposium on Software Reliability Engineering Workshops (ISSREW), Florence, 2023, IEEE, 2023, pp. 128\u2013131. https:\/\/doi.org\/10.1109\/issrew60843.2023.00060","DOI":"10.1109\/issrew60843.2023.00060"},{"key":"3976_CR6","doi-asserted-by":"publisher","unstructured":"Liang, J.T., Yang, Ch., and Myers, B.A., A large-scale survey on the usability of AI programming assistants: Successes and challenges, Proceedings of the IEEE\/ACM 46th International Conference on Software Engineering, Lisbon, 2024, Association for Computing Machinery, 2024, p. 52. https:\/\/doi.org\/10.1145\/3597503.3608128","DOI":"10.1145\/3597503.3608128"},{"key":"3976_CR7","doi-asserted-by":"publisher","unstructured":"Pearce, H., Ahmad, B., Tan, B., Dolan-Gavitt, B., and Karri, R., Asleep at the keyboard? Assessing the security of GitHub Copilot\u2019s code contributions, 2022 IEEE Symposium on Security and Privacy (SP), San Francisco, 2022, IEEE, 2022, pp. 754\u2013768. https:\/\/doi.org\/10.1109\/sp46214.2022.9833571","DOI":"10.1109\/sp46214.2022.9833571"},{"key":"3976_CR8","doi-asserted-by":"publisher","unstructured":"Siddiq, M.L., Majumder, S.H., Mim, M.R., Jajodia, S., and Santos, J.C.S., An empirical study of code smells in transformer-based code generation techniques, IEEE 22nd International Working Conference on Source Code Analysis and Manipulation (SCAM), Limassol, Cyprus, 2022, IEEE, 2022, pp. 71\u201382. https:\/\/doi.org\/10.1109\/SCAM55253.2022.00014","DOI":"10.1109\/SCAM55253.2022.00014"},{"key":"3976_CR9","doi-asserted-by":"publisher","first-page":"101","DOI":"10.1016\/j.infsof.2018.09.006","volume":"106","author":"V. Garousi","year":"2019","unstructured":"Garousi, V., Felderer, M., and M\u00e4ntyl\u00e4, M.V., Guidelines for including grey literature and conducting multivocal literature reviews in software engineering, Inf. Software Technol., 2019, vol. 106, pp. 101\u2013121. https:\/\/doi.org\/10.1016\/j.infsof.2018.09.006","journal-title":"Inf. Software Technol."},{"key":"3976_CR10","doi-asserted-by":"publisher","unstructured":"Wang, J. and Chen, Yi., A review on code generation with LLMs: Application and evaluation, 2023 IEEE International Conference on Medical Artificial Intelligence (MedAI), Beijing, 2023, IEEE, 2023, pp. 284\u2013289. https:\/\/doi.org\/10.1109\/medai59581.2023.00044","DOI":"10.1109\/medai59581.2023.00044"},{"key":"3976_CR11","doi-asserted-by":"publisher","first-page":"13061","DOI":"10.3390\/app132413061","volume":"13","author":"T. Hli\u0161","year":"2023","unstructured":"Hli\u0161, T., \u010cetina, L., Berani\u010d, T., and Pavli\u010d, L., Evaluating the usability and functionality of intelligent source code completion assistants: A comprehensive review, Appl. Sci., 2023, vol. 13, no. 24, p. 13061. https:\/\/doi.org\/10.3390\/app132413061","journal-title":"Appl. Sci."},{"key":"3976_CR12","doi-asserted-by":"publisher","unstructured":"Yao, Y., Duan, J., Xu, K., Cai, Y., Sun, Z., and Zhang, Y., A survey on large language model (LLM) security and privacy: The good, the bad, and the ugly, High-Confidence Comput., 2024, vol. 4, no. 2, p. 100211. https:\/\/doi.org\/10.1016\/j.hcc.2024.100211","DOI":"10.1016\/j.hcc.2024.100211"},{"key":"3976_CR13","doi-asserted-by":"publisher","unstructured":"Horne, D., PwnPilot: Reflections on trusting trust in the age of large language models and AI code assistants, 2023 Congress in Computer Science, Computer Engineering, amp;amp; Applied Computing (CSCE), Las Vegas, 2023, IEEE, 2023, pp. 2457\u20132464. https:\/\/doi.org\/10.1109\/csce60160.2023.00396","DOI":"10.1109\/csce60160.2023.00396"},{"key":"3976_CR14","doi-asserted-by":"publisher","unstructured":"Kaniewski, S., Holstein, D., Schmidt, F., and Heer, T., Vulnerability handling of AI-generated code\u2014Existing solutions and open challenges, 2024 Conference on AI, Science, Engineering, and Technology (AIxSET), Laguna Hills, CA, 2024, IEEE, 2024, pp. 145\u2013148. https:\/\/doi.org\/10.1109\/aixset62544.2024.00026","DOI":"10.1109\/aixset62544.2024.00026"},{"key":"3976_CR15","doi-asserted-by":"publisher","first-page":"225","DOI":"10.1145\/3722108","volume":"34","author":"C. Tony","year":"2025","unstructured":"Tony, C., D\u00edaz Ferreyra, N.E., Mutas, M., Dhif, S., and Scandariato, R., Prompting techniques for secure code generation: A systematic investigation, ACM Trans. Software Eng. Methodology, 2025, vol. 34, no. 8, p. 225. https:\/\/doi.org\/10.1145\/3722108","journal-title":"ACM Trans. Software Eng. Methodology"},{"key":"3976_CR16","doi-asserted-by":"publisher","unstructured":"Kitchenham, B.A., Budgen, D., and Brereton, P., Evidence-Based Software Engineering and Systematic Reviews, Chapman & Hall\/CRC Innovations in Software Engineering and Software Development Series, Chapman and Hall\/CRC, 2015. https:\/\/doi.org\/10.1201\/b19467","DOI":"10.1201\/b19467"},{"key":"3976_CR17","doi-asserted-by":"publisher","first-page":"625","DOI":"10.1016\/j.infsof.2010.12.010","volume":"53","author":"H. Zhang","year":"2011","unstructured":"Zhang, H., Babar, M.A., and Tell, P., Identifying relevant studies in software engineering, Inf. Software Technol., 2011, vol. 53, no. 6, pp. 625\u2013637. https:\/\/doi.org\/10.1016\/j.infsof.2010.12.010","journal-title":"Inf. Software Technol."},{"key":"3976_CR18","first-page":"a7","volume":"59","author":"J. Popay","year":"2005","unstructured":"Popay, J., Roberts, H., Sowden, A., Petticrew, M., Britten, N., Arai, L., Roen, K., and Rodgers, M., Developing guidance on the conduct of narrative synthesis in systematic reviews, J. Epidemiol. Community Health, 2005, vol. 59, suppl. 1, p. a7.","journal-title":"J. Epidemiol. Community Health"},{"key":"3976_CR19","doi-asserted-by":"publisher","unstructured":"Wohlin, C., Guidelines for snowballing in systematic literature studies and a replication in software engineering, Proceedings of the 18th International Conference on Evaluation and Assessment in Software Engineering, London, 2014, New York: Association for Computing Machinery, 2014, p. 38. https:\/\/doi.org\/10.1145\/2601248.2601268","DOI":"10.1145\/2601248.2601268"},{"key":"3976_CR20","doi-asserted-by":"publisher","unstructured":"Jesse, K., Ahmed, T., Devanbu, P.T., and Morgan, E., Large language models and simple, stupid bugs, 2023 IEEE\/ACM 20th International Conference on Mining Software Repositories (MSR), Melbourne, 2023, IEEE, 2023, pp. 563\u2013575. https:\/\/doi.org\/10.1109\/msr59073.2023.00082","DOI":"10.1109\/msr59073.2023.00082"},{"key":"3976_CR21","doi-asserted-by":"publisher","unstructured":"Storhaug, A., Li, J., and Hu, T., Efficient avoidance of vulnerabilities in auto-completed smart contract code using vulnerability-constrained decoding, 2023 IEEE 34th International Symposium on Software Reliability Engineering (ISSRE), Florence, 2023, IEEE, 2023, pp.\u00a0683\u2013693. https:\/\/doi.org\/10.1109\/issre59848.2023.00035","DOI":"10.1109\/issre59848.2023.00035"},{"key":"3976_CR22","doi-asserted-by":"publisher","first-page":"106","DOI":"10.1109\/ms.2021.3133805","volume":"39","author":"N.A. Ernst","year":"2022","unstructured":"Ernst, N.A. and Bavota, G., AI-driven development is here: Should you worry?, IEEE Software, 2022, vol. 39, no. 2, pp. 106\u2013110. https:\/\/doi.org\/10.1109\/ms.2021.3133805","journal-title":"IEEE Software"},{"key":"3976_CR23","doi-asserted-by":"publisher","unstructured":"Tony, C., Mutas, M., Ferreyra, N.E.D., and Scandariato, R., LLMSecEval: A dataset of natural language prompts for security evaluations, 2023 IEEE\/ACM 20th International Conference on Mining Software Repositories (MSR), Melbourne, 2023, IEEE, 2023, pp. 588\u2013592. https:\/\/doi.org\/10.1109\/msr59073.2023.00084","DOI":"10.1109\/msr59073.2023.00084"},{"key":"3976_CR24","doi-asserted-by":"publisher","first-page":"60","DOI":"10.1145\/3630010","volume":"33","author":"G. Yang","year":"2023","unstructured":"Yang, G., Zhou, Yu., Yang, W., Yue, T., Chen, X., and Chen, T., How important are good method names in neural code generation? A model robustness perspective, ACM Trans. Software Eng. Methodology, 2023, vol.\u00a033, no. 3, p. 60. https:\/\/doi.org\/10.1145\/3630010","journal-title":"ACM Trans. Software Eng. Methodology"},{"key":"3976_CR25","doi-asserted-by":"publisher","unstructured":"Barke, Sh., James, M.B., and Polikarpova, N., Grounded copilot: How programmers interact with code-generating models, Proc. ACM Program. Lang., 2023, vol. 7, no. oopsla1, pp. 85\u2013111. https:\/\/doi.org\/10.1145\/3586030","DOI":"10.1145\/3586030"},{"key":"3976_CR26","doi-asserted-by":"publisher","unstructured":"Perry, N., Srivastava, M., Kumar, D., and Boneh, D., Do users write more insecure code with AI assistants?, Proceedings of the 2023 ACM SIGSAC Conference on Computer and Communications Security, Copenhagen, 2023, New York: Association for Computing Machinery, 2023, pp. 2785\u20132799. https:\/\/doi.org\/10.1145\/3576915.3623157","DOI":"10.1145\/3576915.3623157"},{"key":"3976_CR27","doi-asserted-by":"publisher","unstructured":"Seo, J., Zhang, N., and Rong, Ch., Flexible and secure code deployment in federated learning using large language models: Prompt engineering to enhance malicious code detection, 2023 IEEE International Conference on Cloud Computing Technology and Science (CloudCom), Naples, 2023, IEEE, 2023, pp. 341\u2013349. https:\/\/doi.org\/10.1109\/cloudcom59040.2023.00062","DOI":"10.1109\/cloudcom59040.2023.00062"},{"key":"3976_CR28","doi-asserted-by":"publisher","unstructured":"Ren, X., Ye, X., Zhao, D., Xing, Zh., and Yang, X., From misuse to mastery: Enhancing code generation with knowledge-driven AI chaining, 2023 38th IEEE\/ACM International Conference on Automated Software Engineering (ASE), Luxembourg, 2023, IEEE, 2023, pp. 976\u2013987. https:\/\/doi.org\/10.1109\/ase56229.2023.00143","DOI":"10.1109\/ase56229.2023.00143"},{"key":"3976_CR29","doi-asserted-by":"publisher","first-page":"116","DOI":"10.1145\/3643674","volume":"33","author":"Yu. Liu","year":"2024","unstructured":"Liu, Yu., Le-Cong, T., Widyasari, R., Tantithamthavorn, Ch., Li, L., Le, X.-B.D., and Lo, D., Refining ChatGPT-generated code: Characterizing and mitigating code quality issues, ACM Trans. Software Eng. Methodology, 2024, vol. 33, no. 5, p. 116. https:\/\/doi.org\/10.1145\/3643674","journal-title":"ACM Trans. Software Eng. Methodology"},{"key":"3976_CR30","doi-asserted-by":"publisher","unstructured":"Asare, O., Nagappan, M., and Asokan, N., Is GitHub\u2019s Copilot as bad as humans at introducing vulnerabilities in code?, arXiv Preprint, 2022. https:\/\/doi.org\/10.48550\/arXiv.2204.04741","DOI":"10.48550\/arXiv.2204.04741"},{"key":"3976_CR31","doi-asserted-by":"publisher","unstructured":"Sandoval, G.A., Pearce, H., Nys, T., Karri, R., Dolan-Gavitt, B., and Garg, S., Lost at C: A user study on the security implications of large language model code assistants, arXiv Preprint, 2022. https:\/\/doi.org\/10.48550\/arXiv.2208.09727","DOI":"10.48550\/arXiv.2208.09727"},{"key":"3976_CR32","doi-asserted-by":"publisher","unstructured":"Chen, M., Tworek, J., Jun, H., Yuan, Q., Pinto, H.P.D.O., Kaplan, J., Edwards, H., Burda, Y., Joseph, N., Brockman, G., Ray, A., Puri, R., Krueger, G., Petrov, M., Khlaaf, H., Sastry, G., Mishkin, P., et al., Evaluating large language models trained on code, arXiv Preprint, 2021. https:\/\/doi.org\/10.48550\/arXiv.2107.03374","DOI":"10.48550\/arXiv.2107.03374"},{"key":"3976_CR33","doi-asserted-by":"publisher","unstructured":"Mousavi, Z., Islam, C., Moore, K., Abuadbba, A., and Babar, M.A., An investigation into misuse of java security APIs by large language models, arXiv Preprint, 2024. https:\/\/doi.org\/10.48550\/arXiv.2404.03823","DOI":"10.48550\/arXiv.2404.03823"},{"key":"3976_CR34","doi-asserted-by":"publisher","unstructured":"Elgedawy, R., Sadik, J., Dutta, S., Gautam, A., Georgiou, K., Gholamrezae, F., Ji, F., Lim, K., Liu, Q., and Ruoti, S., Ocassionally secure: A comparative analysis of code generation assistants, arXiv Preprint, 2024. https:\/\/doi.org\/10.48550\/arXiv.2402.00689","DOI":"10.48550\/arXiv.2402.00689"},{"key":"3976_CR35","doi-asserted-by":"publisher","unstructured":"Liu, Z., Tang, Y., Luo, X., Zhou, Y., and Zhang, L.F., No need to lift a finger anymore? Assessing the quality of code generation by ChatGPT, arXiv Preprint, 2023. https:\/\/doi.org\/10.48550\/arXiv.2308.04838","DOI":"10.48550\/arXiv.2308.04838"},{"key":"3976_CR36","doi-asserted-by":"publisher","unstructured":"Fu, Y., Liang, P., Tahir, A., Li, Z., Shahin, M., Yu, J., and Chen, J., Security weaknesses of copilot generated code in GitHub, arXiv Preprint, 2023. https:\/\/doi.org\/10.48550\/arXiv.2310.02059","DOI":"10.48550\/arXiv.2310.02059"},{"key":"3976_CR37","doi-asserted-by":"publisher","unstructured":"Asare, O., Nagappan, M., and Asokan, N., A user-centered security evaluation of copilot, Proceedings of the IEEE\/ACM 46th International Conference on Software Engineering, Lisbon, 2024, New York: Association for Computing Machinery, 2024, pp. 1\u201311. https:\/\/doi.org\/10.1145\/3597503.3639154","DOI":"10.1145\/3597503.3639154"},{"key":"3976_CR38","doi-asserted-by":"publisher","unstructured":"Zhong, L. and Wang, Z., Can ChatGPT replace StackOverflow? A study on robustness and reliability of large language model code generation, arXiv Preprint, 2023. https:\/\/doi.org\/10.48550\/arXiv.2308.10335","DOI":"10.48550\/arXiv.2308.10335"},{"key":"3976_CR39","doi-asserted-by":"publisher","unstructured":"Khoury, R., Avila, A.R., Brunelle, J., and Camara, B.M., How secure is code generated by ChatGPT?, arXiv Preprint, 2023. https:\/\/doi.org\/10.48550\/arXiv.2304.09655","DOI":"10.48550\/arXiv.2304.09655"},{"key":"3976_CR40","doi-asserted-by":"publisher","unstructured":"Hamer, S., D\u2019Amorim, M., and Williams, L., Just another copy and paste? Comparing the security vulnerabilities of ChatGPT generated code and StackOverflow answers, 2024 IEEE Security and Privacy Workshops (SPW), San Francisco, CA, 2024, IEEE, 2024, pp. 87\u201394. https:\/\/doi.org\/10.1109\/spw63631.2024.00014","DOI":"10.1109\/spw63631.2024.00014"},{"key":"3976_CR41","unstructured":"Jakkal, V., Introducing Microsoft Security Copilot: Empowering defenders at the speed of AI, The Official Microsoft Blog, 2023. https:\/\/blogs.microsoft.com\/blog\/2023\/03\/28\/introducing-microsoft-security-copilot-empowering-defenders-at-the-speed-of-ai\/."},{"key":"3976_CR42","doi-asserted-by":"publisher","unstructured":"Mohsin, A., Janicke, H., Wood, A., Sarker, I.H., Maglaras, L., and Janjua, N., Can we trust large language models generated code? A framework for in-context learning, security patterns, and code evaluations across diverse LLMs, arXiv Preprint, 2024. https:\/\/doi.org\/10.48550\/arXiv.2406.12513","DOI":"10.48550\/arXiv.2406.12513"},{"key":"3976_CR43","doi-asserted-by":"publisher","unstructured":"Siddiq, M.L., Roney, L., Zhang, J., and Da Silva Santos, J.C., Quality assessment of ChatGPT generated code and their use by developers, Proceedings of the 21st International Conference on Mining Software Repositories, Lisbon, 2024, New York: Association for Computing Machinery, 2024, pp. 152\u2013156. https:\/\/doi.org\/10.1145\/3643991.3645071","DOI":"10.1145\/3643991.3645071"},{"key":"3976_CR44","doi-asserted-by":"publisher","first-page":"3435","DOI":"10.1109\/tse.2024.3492204","volume":"50","author":"X. Yu","year":"2024","unstructured":"Yu, X., Liu, L., Hu, X., Keung, J.W., Liu, J., and Xia, X., Fight fire with fire: How much can we trust ChatGPT on source code-related tasks?, IEEE Trans. Software Eng., 2024, vol. 50, no. 12, pp. 3435\u20133453. https:\/\/doi.org\/10.1109\/tse.2024.3492204","journal-title":"IEEE Trans. Software Eng."},{"key":"3976_CR45","doi-asserted-by":"publisher","unstructured":"Rabbi, Md.F., Champa, A.I., Zibran, M.F., and Islam, M.R., AI writes, we analyze: The ChatGPT Python code saga, Proceedings of the 21st International Conference on Mining Software Repositories, Lisbon, 2024, New York: Association for Computing Machinery, 2024, pp. 177\u2013181. https:\/\/doi.org\/10.1145\/3643991.3645076","DOI":"10.1145\/3643991.3645076"},{"key":"3976_CR46","doi-asserted-by":"publisher","unstructured":"Majdinasab, V., Bishop, M.J., Rasheed, Sh., Moradidakhel, A., Tahir, A., and Khomh, F., Assessing the security of GitHub Copilot\u2019s generated code\u2014A targeted replication study, 2024 IEEE International Conference on Software Analysis, Evolution and Reengineering (SANER), Rovaniemi, Finland, 2024, IEEE, 2024, pp.\u00a0435\u2013444. https:\/\/doi.org\/10.1109\/saner60148.2024.00051","DOI":"10.1109\/saner60148.2024.00051"},{"key":"3976_CR47","doi-asserted-by":"publisher","unstructured":"Bruni, M., Gabrielli, F., Ghafari, M., and Kropp, M., Benchmarking prompt engineering techniques for secure code generation with GPT models, arXiv Preprint, 2025. https:\/\/doi.org\/10.48550\/arXiv.2502.06039","DOI":"10.48550\/arXiv.2502.06039"},{"key":"3976_CR48","doi-asserted-by":"publisher","unstructured":"Peng, J., Cui, L., Huang, K., Yang, J., and Ray, B., Cweval: Outcome-driven evaluation on functionality and security of LLM code generation, arXiv Preprint, 2025. https:\/\/doi.org\/10.48550\/arXiv.2501.08200","DOI":"10.48550\/arXiv.2501.08200"},{"key":"3976_CR49","doi-asserted-by":"publisher","unstructured":"Kharma, M., Choi, S., Alkhanafseh, M., and Mohaisen, D., Security and quality in LLM-generated code: A multi-language, multi-model analysis, arXiv Preprint, 2025. https:\/\/doi.org\/10.48550\/arXiv.2502.01853","DOI":"10.48550\/arXiv.2502.01853"},{"key":"3976_CR50","doi-asserted-by":"publisher","unstructured":"Sajadi, A., Le, B., Nguyen, A., Damevski, K., and Chatterjee, P., Do LLMs consider security? an empirical study on responses to programming questions, arXiv Preprint, 2025. https:\/\/doi.org\/10.48550\/arXiv.2502.14202","DOI":"10.48550\/arXiv.2502.14202"},{"key":"3976_CR51","doi-asserted-by":"publisher","unstructured":"Licorish, S.A., Bajpai, A., Arora, C., Wang, F., and Tantithamthavorn, K., Comparing human and LLM generated code: The jury is still out!, arXiv Preprint, 2025. https:\/\/doi.org\/10.48550\/arXiv.2501.16857","DOI":"10.48550\/arXiv.2501.16857"},{"key":"3976_CR52","doi-asserted-by":"publisher","unstructured":"Dunne, M., Schram, K., and Fischmeister, S., Weaknesses in LLM-generated code for embedded systems networking, 2024 IEEE 24th International Conference on Software Quality, Reliability and Security (QRS), Cambridge, 2024, IEEE, 2024, pp. 250\u2013261. https:\/\/doi.org\/10.1109\/QRS62785.2024.00033","DOI":"10.1109\/QRS62785.2024.00033"},{"key":"3976_CR53","doi-asserted-by":"publisher","unstructured":"Ambati, S.H., Ridley, N., Branca, E., and Stakhanova, N., Navigating (in)security of AI-generated code, 2024 IEEE International Conference on Cyber Security and Resilience (CSR), London, 2024, IEEE, 2024, pp. 1\u20138. https:\/\/doi.org\/10.1109\/csr61664.2024.10679468","DOI":"10.1109\/csr61664.2024.10679468"},{"key":"3976_CR54","doi-asserted-by":"publisher","unstructured":"Wang, B., Li, H., Liu, A., Yang, B., Yang, A., Zhong, Yi., Huang, W., Huang, R., Zeng, W., and Zhang, Ya., RefleXGen: The unexamined code is not worth using, ICASSP 2025\u20142025 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP), Hyderabad, India, 2025, IEEE, 2025, pp. 1\u20135. https:\/\/doi.org\/10.1109\/icassp49660.2025.10890824","DOI":"10.1109\/icassp49660.2025.10890824"},{"key":"3976_CR55","doi-asserted-by":"publisher","unstructured":"Hajipour, H., Hassler, K., Holz, T., Sch\u00f6nherr, L., and Fritz, M., Codelmsec benchmark: Systematically evaluating and finding security vulnerabilities in black-box code language models, 2024 IEEE Conference on Secure and Trustworthy Machine Learning (SaTML), Toronto, 2024, IEEE, 2024, pp. 684\u2013709. https:\/\/doi.org\/10.1109\/SaTML59370.2024.00040","DOI":"10.1109\/SaTML59370.2024.00040"},{"key":"3976_CR56","doi-asserted-by":"publisher","first-page":"419","DOI":"10.1109\/tetci.2024.3446695","volume":"9","author":"G.S. Black","year":"2025","unstructured":"Black, G.S., Rimal, B.P., and Vaidyan, V.M., Balancing security and correctness in code generation: An empirical study on commercial large language models, IEEE Trans. Emerging Top. Comput. Intell., 2025, vol. 9, no. 1, pp. 419\u2013430. https:\/\/doi.org\/10.1109\/tetci.2024.3446695","journal-title":"IEEE Trans. Emerging Top. Comput. Intell."},{"key":"3976_CR57","doi-asserted-by":"publisher","unstructured":"Li, J., Sangalay, A., Cheng, Ch., Tian, Yu., and Yang, J., Fine tuning large language model for secure code generation, Proceedings of the 2024 IEEE\/ACM First International Conference on AI Foundation Models and Software Engineering, Lisbon, 2024, New York: Association for Computing Machinery, 2024, pp. 86\u201390. https:\/\/doi.org\/10.1145\/3650105.3652299","DOI":"10.1145\/3650105.3652299"},{"key":"3976_CR58","doi-asserted-by":"publisher","unstructured":"Nazzal, M., Khalil, I., Khreishah, A., and Phan, N., PromSec: Prompt optimization for secure generation of functional source code with large language models (LLMs), Proceedings of the 2024 on ACM SIGSAC Conference on Computer and Communications Security, Salt Lake City, UT, 2024, New York: Association for Computing Machinery, 2024, pp. 2266\u20132280. https:\/\/doi.org\/10.1145\/3658644.3690298","DOI":"10.1145\/3658644.3690298"},{"key":"3976_CR59","doi-asserted-by":"publisher","unstructured":"Jamdade, M. and Liu, Yi., A pilot study on secure code generation with ChatGPT for web applications, Proceedings of the 2024 ACM Southeast Conference on ZZZ, Marietta, GA, 2024, New York: Association for Computing Machinery, 2024, pp. 229\u2013234. https:\/\/doi.org\/10.1145\/3603287.3651194","DOI":"10.1145\/3603287.3651194"},{"key":"3976_CR60","doi-asserted-by":"publisher","unstructured":"Chavan, S.B. and Mondal, Sh., Do large language models recognize Python identifier swaps in their generated code?, Companion Proceedings of the 32nd ACM International Conference on the Foundations of Software Engineering, Porto de Galinhas, Brazil, 2024, New York: Association for Computing Machinery, 2024, pp.\u00a0663\u2013664. https:\/\/doi.org\/10.1145\/3663529.3663869","DOI":"10.1145\/3663529.3663869"},{"key":"3976_CR61","doi-asserted-by":"publisher","unstructured":"Abdali, S., Anarfi, R., Barberan, C., and He, J., Securing large language models: Threats, vulnerabilities and responsible practices, arXiv Preprint, 2024. https:\/\/doi.org\/10.48550\/arXiv.2403.12503","DOI":"10.48550\/arXiv.2403.12503"},{"key":"3976_CR62","doi-asserted-by":"publisher","unstructured":"Karampatsis, R.-M. and Sutton, Ch., How often do single-statement bugs occur?, Proceedings of the 17th International Conference on Mining Software Repositories, Seoul, 2020, New York: Association for Computing Machinery, 2020, pp. 573\u2013577. https:\/\/doi.org\/10.1145\/3379597.3387491","DOI":"10.1145\/3379597.3387491"},{"key":"3976_CR63","doi-asserted-by":"publisher","unstructured":"Shekhar, S., Dubey, T., Mukherjee, K., Saxena, A., Tyagi, A., and Kotla, N., Towards optimizing the costs of LLM usage, arXiv Preprint, 2024. https:\/\/doi.org\/10.48550\/arXiv.2407.02395","DOI":"10.48550\/arXiv.2407.02395"},{"key":"3976_CR64","doi-asserted-by":"publisher","unstructured":"Wang, J., Luo, X., Cao, L., He, H., Huang, H., Xie, J., Jatowt, A., and Cai, Y., Is your ai-generated code really safe? evaluating large language models on secure code generation with codeseceval, arXiv Preprint, 2024. https:\/\/doi.org\/10.48550\/arXiv.2407.02395","DOI":"10.48550\/arXiv.2407.02395"},{"key":"3976_CR65","unstructured":"Stack Overflow developer survey 2023\u2014Survey. https:\/\/survey.stackoverflow.co\/2023\/. Cited May 6, 2024."},{"key":"3976_CR66","doi-asserted-by":"publisher","first-page":"796","DOI":"10.1134\/S0361768824700774","volume":"50","author":"Fernandez-Y Fernandez","year":"2024","unstructured":"Fernandez-Y Fernandez, C., S\u00e1nchez-Soto, E., Cisnero, J.A., and Ju\u00e1rez-Ram\u00edrez, R., Exploring the frontier of software engineering education with chatbots, Program. Comput. Software, 2024, vol. 50, no. 8, pp.\u00a0796\u2013815. https:\/\/doi.org\/10.1134\/S0361768824700774","journal-title":"Program. Comput. Software"},{"key":"3976_CR67","doi-asserted-by":"publisher","first-page":"712","DOI":"10.1134\/S0361768823080145","volume":"49","author":"R. Moguel-S\u00e1nchez","year":"2023","unstructured":"Moguel-S\u00e1nchez, R., Mart\u00ednez-Palacios, C.S., Ochar\u00e1n-Hern\u00e1ndez, J.O., Lim\u00f3n, X., and S\u00e1nchez-Garc\u00eda, A., Bots in software development: A systematic literature review and thematic analysis, Program. Comput. Software, 2023, vol. 49, no. 8, pp. 712\u2013734. https:\/\/doi.org\/10.1134\/S0361768823080145","journal-title":"Program. Comput. Software"},{"key":"3976_CR68","doi-asserted-by":"publisher","unstructured":"Xu, Z., Jain, S., and Kankanhalli, M., Hallucination is inevitable: An innate limitation of large language models, arXiv Preprint, 2024. https:\/\/doi.org\/10.48550\/arXiv.2401.11817","DOI":"10.48550\/arXiv.2401.11817"}],"container-title":["Programming and Computer Software"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1134\/S0361768825700446.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1134\/S0361768825700446","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1134\/S0361768825700446.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,4,1]],"date-time":"2026-04-01T02:52:25Z","timestamp":1775011945000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1134\/S0361768825700446"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,12]]},"references-count":68,"journal-issue":{"issue":"8","published-print":{"date-parts":[[2025,12]]}},"alternative-id":["3976"],"URL":"https:\/\/doi.org\/10.1134\/s0361768825700446","relation":{},"ISSN":["0361-7688","1608-3261"],"issn-type":[{"value":"0361-7688","type":"print"},{"value":"1608-3261","type":"electronic"}],"subject":[],"published":{"date-parts":[[2025,12]]},"assertion":[{"value":"17 August 2025","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"27 August 2025","order":2,"name":"revised","label":"Revised","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"27 August 2025","order":3,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"27 February 2026","order":4,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"The authors of this work declare that they have no conflicts of interest.","order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"CONFLICT OF INTEREST"}}]}}