{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,6,15]],"date-time":"2026-06-15T22:27:52Z","timestamp":1781562472583,"version":"3.54.5"},"reference-count":75,"publisher":"Elsevier BV","license":[{"start":{"date-parts":[[2026,8,1]],"date-time":"2026-08-01T00:00:00Z","timestamp":1785542400000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.elsevier.com\/tdm\/userlicense\/1.0\/"},{"start":{"date-parts":[[2026,8,1]],"date-time":"2026-08-01T00:00:00Z","timestamp":1785542400000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.elsevier.com\/legal\/tdmrep-license"},{"start":{"date-parts":[[2026,4,4]],"date-time":"2026-04-04T00:00:00Z","timestamp":1775260800000},"content-version":"vor","delay-in-days":0,"URL":"http:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"funder":[{"DOI":"10.13039\/501100001858","name":"Sweden's Innovation Agency","doi-asserted-by":"publisher","award":["2023-00546"],"award-info":[{"award-number":["2023-00546"]}],"id":[{"id":"10.13039\/501100001858","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["elsevier.com","sciencedirect.com"],"crossmark-restriction":true},"short-container-title":["Journal of Systems and Software"],"published-print":{"date-parts":[[2026,8]]},"DOI":"10.1016\/j.jss.2026.112885","type":"journal-article","created":{"date-parts":[[2026,4,4]],"date-time":"2026-04-04T12:49:58Z","timestamp":1775306998000},"page":"112885","update-policy":"https:\/\/doi.org\/10.1016\/elsevier_cm_policy","source":"Crossref","is-referenced-by-count":3,"special_numbering":"C","title":["Quality assurance of LLM-generated code: Addressing non-functional quality characteristics"],"prefix":"10.1016","volume":"238","author":[{"ORCID":"https:\/\/orcid.org\/0009-0004-4854-6229","authenticated-orcid":false,"given":"Xin","family":"Sun","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-1675-6884","authenticated-orcid":false,"given":"Daniel","family":"St\u00e5hl","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-3052-5604","authenticated-orcid":false,"given":"Kristian","family":"Sandahl","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-5241-0026","authenticated-orcid":false,"given":"Christoph","family":"Kessler","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"78","reference":[{"issue":"6","key":"10.1016\/j.jss.2026.112885_b1","doi-asserted-by":"crossref","first-page":"129","DOI":"10.1007\/s10664-023-10380-1","article-title":"Is GitHub\u2019s copilot as bad as humans at introducing vulnerabilities in code?","volume":"28","author":"Asare","year":"2023","journal-title":"Empir. Softw. Eng."},{"key":"10.1016\/j.jss.2026.112885_b2","series-title":"Bolt \u2013 Your AI-powered development agent","author":"Bolt","year":"2024"},{"issue":"6","key":"10.1016\/j.jss.2026.112885_b3","doi-asserted-by":"crossref","first-page":"128","DOI":"10.1007\/s10664-023-10381-0","article-title":"Developers talking about code quality","volume":"28","author":"B\u00f6rstler","year":"2023","journal-title":"Empir. Softw. Eng."},{"key":"10.1016\/j.jss.2026.112885_b4","doi-asserted-by":"crossref","DOI":"10.1145\/3715007","article-title":"An empirical study on challenges for LLM application developers","author":"Chen","year":"2025","journal-title":"ACM Trans. Softw. Eng. Methodol."},{"key":"10.1016\/j.jss.2026.112885_b5","series-title":"Evaluating large language models trained on code","author":"Chen","year":"2021"},{"key":"10.1016\/j.jss.2026.112885_b6","series-title":"Proceedings of the 28th International Conference on Evaluation and Assessment in Software Engineering","first-page":"79","article-title":"A performance study of LLM-generated code on Leetcode","author":"Coignion","year":"2024"},{"key":"10.1016\/j.jss.2026.112885_b7","series-title":"Cursor: The AI coding assistant","author":"Cursor","year":"2024"},{"key":"10.1016\/j.jss.2026.112885_b8","series-title":"Proceedings of the 29th International Conference on Evaluation and Assessment in Software Engineering","first-page":"181","article-title":"Do prompt patterns affect code quality? A first empirical assessment of ChatGPT-generated code","author":"Della Porta","year":"2025"},{"key":"10.1016\/j.jss.2026.112885_b9","series-title":"Nocode-bench: A benchmark for evaluating natural language-driven feature addition","author":"Deng","year":"2025"},{"key":"10.1016\/j.jss.2026.112885_b10","series-title":"Proceedings of the 46th IEEE\/ACM International Conference on Software Engineering","first-page":"81:1","article-title":"Evaluating large language models in class-level code generation","author":"Du","year":"2024"},{"key":"10.1016\/j.jss.2026.112885_b11","series-title":"Ocassionally secure: A comparative analysis of code generation assistants","author":"Elgedawy","year":"2024"},{"key":"10.1016\/j.jss.2026.112885_b12","series-title":"2024 34th International Conference on Computer Theory and Applications","first-page":"200","article-title":"Quality of AI-generated vs. Human-generated code","author":"Eltabakh","year":"2024"},{"key":"10.1016\/j.jss.2026.112885_b13","doi-asserted-by":"crossref","DOI":"10.1145\/3716848","article-title":"Security weaknesses of copilot-generated code in GitHub projects: An empirical study","author":"Fu","year":"2025","journal-title":"ACM Trans. Softw. Eng. Methodol."},{"key":"10.1016\/j.jss.2026.112885_b14","series-title":"GitHub copilot: AI code completion tool","author":"GitHub","year":"2025"},{"key":"10.1016\/j.jss.2026.112885_b15","series-title":"Theory and Practice of Model Transformations","first-page":"183","article-title":"Code generation by model transformation","author":"Hemel","year":"2008"},{"key":"10.1016\/j.jss.2026.112885_b16","first-page":"84482","article-title":"EffiLearner: Enhancing efficiency of generated code via self-optimization","volume":"37","author":"Huang","year":"2024","journal-title":"Adv. Neural Inf. Process. Syst."},{"key":"10.1016\/j.jss.2026.112885_b17","series-title":"Advances in Neural Information Processing Systems 38: Annual Conference on Neural Information Processing Systems 2024","article-title":"EffiBench: Benchmarking the efficiency of automatically generated code","author":"Huang","year":"2024"},{"key":"10.1016\/j.jss.2026.112885_b18","series-title":"Qwen2.5-coder technical report","author":"Hui","year":"2024"},{"key":"10.1016\/j.jss.2026.112885_b19","series-title":"ISO\/IEC 25010:2023 Systems and software engineering \u2014 Systems and software Quality Requirements and Evaluation (SQuaRE) \u2014 Product quality model","author":"International Organization for Standardization","year":"2023"},{"key":"10.1016\/j.jss.2026.112885_b20","series-title":"Mistral 7B","author":"Jiang","year":"2023"},{"key":"10.1016\/j.jss.2026.112885_b21","series-title":"The Twelfth International Conference on Learning Representations","article-title":"SWE-bench: Can language models resolve real-world Github issues?","author":"Jimenez","year":"2024"},{"key":"10.1016\/j.jss.2026.112885_b22","series-title":"Findings of the Association for Computational Linguistics","first-page":"11561","article-title":"Revisiting the impact of pursuing modularity for code generation","author":"Kang","year":"2024"},{"issue":"8","key":"10.1016\/j.jss.2026.112885_b23","doi-asserted-by":"crossref","first-page":"2381","DOI":"10.1109\/TSE.2025.3587794","article-title":"The impact of prompt programming on function-level code generation","volume":"51","author":"Khojah","year":"2025","journal-title":"IEEE Trans. Softw. Eng."},{"key":"10.1016\/j.jss.2026.112885_b24","series-title":"IEEE International Conference on Systems, Man, and Cybernetics","first-page":"2445","article-title":"How secure is code generated by ChatGPT?","author":"Khoury","year":"2023"},{"key":"10.1016\/j.jss.2026.112885_b25","series-title":"Proceedings of the 2024 on ACM SIGSAC Conference on Computer and Communications Security","first-page":"2726","article-title":"Using AI assistants in software development: A qualitative study on security practices and concerns","author":"Klemmer","year":"2024"},{"key":"10.1016\/j.jss.2026.112885_b26","series-title":"ICLR2024","article-title":"Codechain: Towards modular code generation through chain of self-revisions with rep- resentative sub-modules","author":"Le","year":"2024"},{"key":"10.1016\/j.jss.2026.112885_b27","article-title":"StarCoder: may the source be with you!","volume":"2023","author":"Li","year":"2023","journal-title":"Trans. Mach. Learn. Res."},{"issue":"6624","key":"10.1016\/j.jss.2026.112885_b28","doi-asserted-by":"crossref","first-page":"1092","DOI":"10.1126\/science.abq1158","article-title":"Competition-level code generation with AlphaCode","volume":"378","author":"Li","year":"2022","journal-title":"Science"},{"key":"10.1016\/j.jss.2026.112885_b29","series-title":"Proceedings of the 63rd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)","first-page":"17160","article-title":"FEA-bench: A benchmark for evaluating repository-level code generation for feature implementation","author":"Li","year":"2025"},{"key":"10.1016\/j.jss.2026.112885_b30","series-title":"Comparing human and LLM generated code: The jury is still out!","author":"Licorish","year":"2025"},{"issue":"5","key":"10.1016\/j.jss.2026.112885_b31","doi-asserted-by":"crossref","first-page":"116:1","DOI":"10.1145\/3643674","article-title":"Refining ChatGPT-Generated Code: Characterizing and Mitigating Code Quality Issues","volume":"33","author":"Liu","year":"2024","journal-title":"ACM Trans. Softw. Eng. Methodol."},{"issue":"6","key":"10.1016\/j.jss.2026.112885_b32","doi-asserted-by":"crossref","first-page":"1548","DOI":"10.1109\/TSE.2024.3392499","article-title":"No need to lift a finger anymore? Assessing the quality of code generation by ChatGPT","volume":"50","author":"Liu","year":"2024","journal-title":"IEEE Trans. Softw. Eng."},{"key":"10.1016\/j.jss.2026.112885_b33","series-title":"Proceedings of the Genetic and Evolutionary Computation Conference","first-page":"1146","article-title":"Fully autonomous programming with large language models","author":"Liventsev","year":"2023"},{"key":"10.1016\/j.jss.2026.112885_b34","series-title":"Build apps with an AI engineer","author":"Lovable","year":"2024"},{"key":"10.1016\/j.jss.2026.112885_b35","series-title":"The Twelfth International Conference on Learning Representations","article-title":"WizardCoder: Empowering code large language models with evol-instruct","author":"Luo","year":"2024"},{"issue":"3","key":"10.1016\/j.jss.2026.112885_b36","doi-asserted-by":"crossref","first-page":"151","DOI":"10.1145\/362566.362568","article-title":"Toward automatic program synthesis","volume":"14","author":"Manna","year":"1971","journal-title":"Commun. ACM"},{"key":"10.1016\/j.jss.2026.112885_b37","series-title":"CWE Top 25 Most Dangerous Software Weaknesses","author":"MITRE","year":"2024"},{"key":"10.1016\/j.jss.2026.112885_b38","doi-asserted-by":"crossref","DOI":"10.1016\/j.jss.2023.111734","article-title":"GitHub Copilot AI pair programmer: Asset or Liability?","volume":"203","author":"Moradi Dakhel","year":"2023","journal-title":"J. Syst. Softw."},{"key":"10.1016\/j.jss.2026.112885_b39","series-title":"Proceedings of the 21st International Conference on Mining Software Repositories","first-page":"147","article-title":"Write me this Code: An Analysis of ChatGPT Quality for Producing Source Code","author":"Moratis","year":"2024"},{"issue":"5","key":"10.1016\/j.jss.2026.112885_b40","doi-asserted-by":"crossref","first-page":"73","DOI":"10.1109\/MSEC.2024.3355713","article-title":"AI code generators for security: Friend or foe?","volume":"22","author":"Natella","year":"2024","journal-title":"IEEE Secur. Priv."},{"key":"10.1016\/j.jss.2026.112885_b41","series-title":"19th IEEE\/ACM International Conference on Mining Software Repositories","first-page":"1","article-title":"An empirical evaluation of GitHub copilot\u2019s code suggestions","author":"Nguyen","year":"2022"},{"key":"10.1016\/j.jss.2026.112885_b42","series-title":"Proceedings of the 2024 IEEE\/ACM First International Conference on AI Foundation Models and Software Engineering","first-page":"103","article-title":"On evaluating the efficiency of source code generated by LLMs","author":"Niu","year":"2024"},{"key":"10.1016\/j.jss.2026.112885_b43","article-title":"An Empirical Study of the Non-determinism of ChatGPT in Code Generation","author":"Ouyang","year":"2024","journal-title":"ACM Trans. Softw. Eng. Methodol."},{"key":"10.1016\/j.jss.2026.112885_b44","article-title":"The PRISMA 2020 statement: An updated guideline for reporting systematic reviews","volume":"372","author":"Page","year":"2021","journal-title":"BMJ (Clinical Res. Ed.)"},{"key":"10.1016\/j.jss.2026.112885_b45","series-title":"2022 IEEE Symposium on Security and Privacy","first-page":"754","article-title":"Asleep at the Keyboard? Assessing the Security of GitHub Copilot\u2019s Code Contributions","author":"Pearce","year":"2022"},{"key":"10.1016\/j.jss.2026.112885_b46","series-title":"Proceedings of the 2023 ACM SIGSAC Conference on Computer and Communications Security","first-page":"2785","article-title":"Do users write more insecure code with AI assistants?","author":"Perry","year":"2023"},{"key":"10.1016\/j.jss.2026.112885_b47","series-title":"Proceedings of the 38th IEEE\/ACM International Conference on Automated Software Engineering","first-page":"976","article-title":"From Misuse to Mastery: Enhancing Code Generation with Knowledge-Driven AI Chaining","author":"Ren","year":"2024"},{"key":"10.1016\/j.jss.2026.112885_b48","series-title":"Code llama: Open foundation models for code","author":"Rozi\u00e8re","year":"2023"},{"key":"10.1016\/j.jss.2026.112885_b49","doi-asserted-by":"crossref","first-page":"72303","DOI":"10.1109\/ACCESS.2024.3403858","article-title":"Methodology for Code Synthesis Evaluation of LLMs Presented by a Case Study of ChatGPT and Copilot","volume":"12","author":"S\u00e1godi","year":"2024","journal-title":"IEEE Access"},{"key":"10.1016\/j.jss.2026.112885_b50","series-title":"2024 International Conference on Electrical, Computer and Energy Technologies","first-page":"1","article-title":"Extending the Frontier of ChatGPT: Code Generation and Debugging","author":"Sakib","year":"2024"},{"key":"10.1016\/j.jss.2026.112885_b51","series-title":"2024 2nd International Conference on Foundation and Large Language Models","first-page":"407","article-title":"Evaluating Large Language Models for Code Generation: Assessing Accuracy, Quality, and Performance","author":"Shehab","year":"2024"},{"key":"10.1016\/j.jss.2026.112885_b52","series-title":"The Twelfth International Conference on Learning Representations","article-title":"Learning performance-improving code edits","author":"Shypula","year":"2024"},{"key":"10.1016\/j.jss.2026.112885_b53","series-title":"Proceedings - 2024 39th ACM\/IEEE International Conference on Automated Software Engineering Workshops","first-page":"54","article-title":"Sallm: Security Assessment of Generated Code","author":"Siddiq","year":"2024"},{"key":"10.1016\/j.jss.2026.112885_b54","series-title":"2022 IEEE 22nd International Working Conference on Source Code Analysis and Manipulation","first-page":"71","article-title":"An Empirical Study of Code Smells in Transformer-based Code Generation Techniques","author":"Siddiq","year":"2022"},{"key":"10.1016\/j.jss.2026.112885_b55","series-title":"Proceedings - 2024 IEEE\/ACM 21st International Conference on Mining Software Repositories","first-page":"152","article-title":"Quality Assessment of ChatGPT Generated Code and their Use by Developers","author":"Siddiq","year":"2024"},{"key":"10.1016\/j.jss.2026.112885_b56","first-page":"29","article-title":"SecurityEval dataset: mining vulnerability examples to evaluate machine learning-based code generation techniques","author":"Siddiq","year":"2022"},{"key":"10.1016\/j.jss.2026.112885_b57","series-title":"NoFunEval: Funny How Code LMs Falter on Requirements Beyond Functional Correctness","author":"Singhal","year":"2024"},{"key":"10.1016\/j.jss.2026.112885_b58","series-title":"IEEE\/ACM Second International Conference on AI Foundation Models and Software Engineering","first-page":"49","article-title":"AI-powered, but power-hungry? Energy efficiency of LLM-generated code","author":"Solovyeva","year":"2025"},{"key":"10.1016\/j.jss.2026.112885_b59","series-title":"Time Horizons: How Far Can We See in AI?","author":"The AI Digest","year":"2024"},{"key":"10.1016\/j.jss.2026.112885_b60","doi-asserted-by":"crossref","DOI":"10.1145\/3722108","article-title":"Prompting Techniques for Secure Code Generation: A Systematic Investigation","author":"Tony","year":"2025","journal-title":"ACM Trans. Softw. Eng. Methodol."},{"key":"10.1016\/j.jss.2026.112885_b61","series-title":"2023 IEEE\/ACM 20th International Conference on Mining Software Repositories","first-page":"588","article-title":"LLMSecEval: A Dataset of Natural Language Prompts for Security Evaluations","author":"Tony","year":"2023"},{"key":"10.1016\/j.jss.2026.112885_b62","series-title":"LLM4CODE@ICSE","first-page":"30","article-title":"Learn to code sustainably: An empirical study on green code generation","author":"Vartziotis","year":"2024"},{"key":"10.1016\/j.jss.2026.112885_b63","series-title":"Proceedings of the 2024 Conference on Empirical Methods in Natural Language Processing","first-page":"15362","article-title":"ECCO: can we improve model-generated code efficiency without sacrificing functional correctness?","author":"Waghjale","year":"2024"},{"key":"10.1016\/j.jss.2026.112885_b64","series-title":"2023 IEEE International Conference on Medical Artificial Intelligence","first-page":"284","article-title":"A review on code generation with LLMs: Application and evaluation","author":"Wang","year":"2023"},{"key":"10.1016\/j.jss.2026.112885_b65","article-title":"CodeUltraFeedback: An LLM-as-a-Judge Dataset for Aligning Large Language Models to Coding Preferences","author":"Weyssow","year":"2025","journal-title":"ACM Trans. Softw. Eng. Methodol."},{"key":"10.1016\/j.jss.2026.112885_b66","series-title":"Windsurf Editor: The AI-native IDE","author":"Windsurf","year":"2024"},{"key":"10.1016\/j.jss.2026.112885_b67","series-title":"The Thirty-Eighth Annual Conference on Neural Information Processing Systems","article-title":"SWE-agent: Agent-computer interfaces enable automated software engineering","author":"Yang","year":"2024"},{"key":"10.1016\/j.jss.2026.112885_b68","series-title":"Robustness, security, privacy, explainability, efficiency, and usability of large language models for code","author":"Yang","year":"2024"},{"key":"10.1016\/j.jss.2026.112885_b69","series-title":"Evaluating the code quality of AI-assisted code generation tools: An empirical study on GitHub copilot, amazon CodeWhisperer, and ChatGPT","author":"Yetistiren","year":"2023"},{"key":"10.1016\/j.jss.2026.112885_b70","series-title":"Proceedings of the 62nd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)","first-page":"13643","article-title":"CodeAgent: Enhancing code generation with tool-integrated agent systems for real-world repo-level coding challenges","author":"Zhang","year":"2024"},{"issue":"12","key":"10.1016\/j.jss.2026.112885_b71","doi-asserted-by":"crossref","first-page":"3507","DOI":"10.1109\/TSE.2025.3619281","article-title":"Towards Secure Code Generation With LLMs: A Study on Common Weakness Enumeration","volume":"51","author":"Zhao","year":"2025","journal-title":"IEEE Trans. Softw. Eng."},{"key":"10.1016\/j.jss.2026.112885_b72","series-title":"Beyond correctness: Benchmarking multi-dimensional code generation for large language models","author":"Zheng","year":"2024"},{"issue":"2","key":"10.1016\/j.jss.2026.112885_b73","doi-asserted-by":"crossref","first-page":"50","DOI":"10.1007\/s10664-024-10602-0","article-title":"Towards an understanding of large language models in software engineering tasks","volume":"30","author":"Zheng","year":"2024","journal-title":"Empir. Softw. Eng."},{"issue":"19","key":"10.1016\/j.jss.2026.112885_b74","first-page":"21841","article-title":"Can LLM Replace Stack Overflow? A Study on Robustness and Reliability of Large Language Model Code Generation","volume":"38","author":"Zhong","year":"2024","journal-title":"Proc. the AAAI Conf. Artif. Intell."},{"key":"10.1016\/j.jss.2026.112885_b75","series-title":"Findings of the Association for Computational Linguistics","first-page":"851","article-title":"Debug like a human: A large language model debugger via verifying runtime execution step by step","author":"Zhong","year":"2024"}],"container-title":["Journal of Systems and Software"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/api.elsevier.com\/content\/article\/PII:S0164121226001184?httpAccept=text\/xml","content-type":"text\/xml","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/api.elsevier.com\/content\/article\/PII:S0164121226001184?httpAccept=text\/plain","content-type":"text\/plain","content-version":"vor","intended-application":"text-mining"}],"deposited":{"date-parts":[[2026,5,26]],"date-time":"2026-05-26T20:29:19Z","timestamp":1779827359000},"score":1,"resource":{"primary":{"URL":"https:\/\/linkinghub.elsevier.com\/retrieve\/pii\/S0164121226001184"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026,8]]},"references-count":75,"alternative-id":["S0164121226001184"],"URL":"https:\/\/doi.org\/10.1016\/j.jss.2026.112885","relation":{},"ISSN":["0164-1212"],"issn-type":[{"value":"0164-1212","type":"print"}],"subject":[],"published":{"date-parts":[[2026,8]]},"assertion":[{"value":"Elsevier","name":"publisher","label":"This article is maintained by"},{"value":"Quality assurance of LLM-generated code: Addressing non-functional quality characteristics","name":"articletitle","label":"Article Title"},{"value":"Journal of Systems and Software","name":"journaltitle","label":"Journal Title"},{"value":"https:\/\/doi.org\/10.1016\/j.jss.2026.112885","name":"articlelink","label":"CrossRef DOI link to publisher maintained version"},{"value":"article","name":"content_type","label":"Content Type"},{"value":"\u00a9 2026 The Authors. Published by Elsevier Inc.","name":"copyright","label":"Copyright"}],"article-number":"112885"}}