{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,3,30]],"date-time":"2026-03-30T15:08:17Z","timestamp":1774883297933,"version":"3.50.1"},"reference-count":99,"publisher":"Springer Science and Business Media LLC","issue":"3","license":[{"start":{"date-parts":[[2026,1,19]],"date-time":"2026-01-19T00:00:00Z","timestamp":1768780800000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2026,1,19]],"date-time":"2026-01-19T00:00:00Z","timestamp":1768780800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Empir Software Eng"],"published-print":{"date-parts":[[2026,5]]},"DOI":"10.1007\/s10664-025-10776-1","type":"journal-article","created":{"date-parts":[[2026,1,19]],"date-time":"2026-01-19T05:27:04Z","timestamp":1768800424000},"update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":0,"title":["Performance analysis of AI-generated code: A case study of Copilot, Copilot Chat, CodeLlaMa, and DeepSeek-Coder models"],"prefix":"10.1007","volume":"31","author":[{"given":"Shuang","family":"Li","sequence":"first","affiliation":[]},{"given":"Yuntao","family":"Cheng","sequence":"additional","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0001-7410-9146","authenticated-orcid":false,"given":"Jinfu","family":"Chen","sequence":"additional","affiliation":[]},{"given":"Jifeng","family":"Xuan","sequence":"additional","affiliation":[]},{"given":"Sen","family":"He","sequence":"additional","affiliation":[]},{"given":"Weiyi","family":"Shang","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2026,1,19]]},"reference":[{"key":"10776_CR1","doi-asserted-by":"crossref","unstructured":"Ahmad WU, Chakraborty S, Ray B, Chang K (2021) Unified pre-training for program understanding and generation. In: Proceedings of the 2021 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, NAACL-HLT 2021, Online, June 6-11, 2021, pages 2655\u20132668. Association for Computational Linguistics","DOI":"10.18653\/v1\/2021.naacl-main.211"},{"key":"10776_CR2","unstructured":"Austin J, Odena A, Nye M, Bosma M, Michalewski H, Dohan D, Jiang E, Cai C, Terry M, Le Q, et\u00a0al (2021) Program synthesis with large language models. arXiv:2108.07732"},{"issue":"OOPSLA1","key":"10776_CR3","doi-asserted-by":"publisher","first-page":"85","DOI":"10.1145\/3586030","volume":"7","author":"S Barke","year":"2023","unstructured":"Barke S, James MB, Polikarpova N (2023) Grounded copilot: How programmers interact with code-generating models. Proc ACM Program Lang 7(OOPSLA1):85\u2013111","journal-title":"Proc ACM Program Lang"},{"key":"10776_CR4","doi-asserted-by":"crossref","unstructured":"Cappendijk T, de\u00a0Reus P, Oprescu A (2025) An exploration of prompting llms to generate energy-efficient code. In: 9th IEEE\/ACM International Workshop on Green and Sustainable Software, GREENS@ICSE 2025, Ottawa, ON, Canada, April 29, 2025, pages 31\u201338. IEEE","DOI":"10.1109\/GREENS66463.2025.00010"},{"key":"10776_CR5","doi-asserted-by":"crossref","unstructured":"Chae H, Kwon T, Moon S, Song Y, Kang D, Ong KT, Kwak B, Bae S, Hwang S, Yeo J (2024) Coffee-gym: An environment for evaluating and improving natural language feedback on erroneous code. In Y.\u00a0Al-Onaizan, M.\u00a0Bansal, and Y.\u00a0Chen, editors, Proceedings of the 2024 Conference on Empirical Methods in Natural Language Processing, EMNLP 2024, Miami, FL, USA, November 12-16, 2024, pages 22503\u201322524. Association for Computational Linguistics","DOI":"10.18653\/v1\/2024.emnlp-main.1254"},{"issue":"5","key":"10776_CR6","doi-asserted-by":"publisher","first-page":"1529","DOI":"10.1109\/TSE.2020.3023955","volume":"48","author":"J Chen","year":"2022","unstructured":"Chen J, Shang W, Shihab E (2022) Perfjit: Test-level just-in-time prediction for performance regression introducing commits. IEEE Trans Software Eng 48(5):1529\u20131544","journal-title":"IEEE Trans Software Eng"},{"key":"10776_CR7","doi-asserted-by":"crossref","unstructured":"Chen J, Ding Z, Tang Y, Sayagh M, Li H, Adams B, Shang W (2023) Iopv: On inconsistent option performance variations. In: Proceedings of the 31st ACM Joint European Software Engineering Conference and Symposium on the Foundations of Software Engineering, ESEC\/FSE 2023, San Francisco, CA, USA, December 3-9, 2023, pages 845\u2013857. ACM","DOI":"10.1145\/3611643.3616319"},{"key":"10776_CR8","unstructured":"Chen M, Tworek J, Jun H, Yuan Q, Pinto H.P dO, Kaplan J, Edwards H, Burda Y, Joseph N, Brockman G et\u00a0al (2021) Evaluating large language models trained on code. arXiv:2107.03374"},{"key":"10776_CR9","doi-asserted-by":"crossref","unstructured":"Chen T, Shang W, Jiang ZM, Hassan AE, Nasser MN, Flora P (2014) Detecting performance anti-patterns for applications developed using object-relational mapping. In: 36th International Conference on Software Engineering, ICSE \u201914, Hyderabad, India - May 31 - June 07, 2014, pages 1001\u20131012. ACM","DOI":"10.1145\/2568225.2568259"},{"key":"10776_CR10","unstructured":"Claude Code (2025) Claude code: The ai coding assistant. https:\/\/docs.anthropic.com\/en\/docs\/agents-and-tools\/claude-code\/overview. Accessed:2025-02-27"},{"key":"10776_CR11","doi-asserted-by":"crossref","unstructured":"Coignion T, Quinton C, Rouvoy R (2024) A performance study of llm-generated code on leetcode. In: Proceedings of the 28th International Conference on Evaluation and Assessment in Software Engineering, pages 79\u201389","DOI":"10.1145\/3661167.3661221"},{"key":"10776_CR12","doi-asserted-by":"crossref","unstructured":"Cursaru V, Duits L, Milligan J, Ural D, Sanchez BR, Stoico V, Malavolta I (2024) A controlled experiment on the energy efficiency of the source code generated by code llama. CoRR, abs\/2405.03616","DOI":"10.1007\/978-3-031-70245-7_12"},{"key":"10776_CR13","unstructured":"Cursor (2025) Cursor - The AI Code Editor. https:\/\/www.cursor.com\/en. Accessed: 2025-02-27"},{"key":"10776_CR14","doi-asserted-by":"crossref","unstructured":"Ding Z, Chen J, Shang W (2020) Towards the use of the readily available tests from the release pipeline as performance tests: are we there yet? In: ICSE \u201920: 42nd International Conference on Software Engineering, Seoul, South Korea, 27 June - 19 July, 2020, pages 1435\u20131446. ACM","DOI":"10.1145\/3377811.3380351"},{"key":"10776_CR15","unstructured":"Drain D, Hu C, Wu C, Breslav M, Sundaresan N (2021) Generating code with the help of retrieved template functions and stack overflow answers. CoRR, abs\/2104.05310"},{"key":"10776_CR16","first-page":"10","volume-title":"Advances in Neural Information Processing Systems 38: Annual Conference on Neural Information Processing Systems 2024, NeurIPS 2024","author":"M Du","year":"2024","unstructured":"Du M, Luu AT, Ji B, Liu Q (2024) Ng S (2024) Mercury: A code efficiency benchmark for code large language models. In: Globersons A, Mackey L, Belgrave D, Fan A, Paquet U, Tomczak JM, Zhang C (eds) Advances in Neural Information Processing Systems 38: Annual Conference on Neural Information Processing Systems 2024, NeurIPS 2024. BC, Canada, December, Vancouver, pp 10\u201315"},{"key":"10776_CR17","doi-asserted-by":"crossref","unstructured":"Feng Z, Guo D, Tang D, Duan N, Feng X, Gong M, Shou L, Qin B, Liu T, Jiang D, Zhou M (2020) Codebert: A pre-trained model for programming and natural languages. In: Findings of the Association for Computational Linguistics: EMNLP 2020, Online Event, 16-20 November 2020, volume EMNLP 2020 of Findings of ACL, pages 1536\u20131547. Association for Computational Linguistics","DOI":"10.18653\/v1\/2020.findings-emnlp.139"},{"key":"10776_CR18","doi-asserted-by":"crossref","unstructured":"Fu Y, Liang P, Li Z, Shahin M, Yu J, Chen J (2025) Security weaknesses of copilot-generated code in github projects: An empirical study. ACM Transactions on Software Engineering and Methodology","DOI":"10.1145\/3716848"},{"key":"10776_CR19","unstructured":"Gao S, Gao C, Gu W, Lyu MR (2024) Search-based llms for code optimization. CoRR, abs\/2408.12159"},{"key":"10776_CR20","unstructured":"Garg S, Moghaddam RZ, Sundaresan N (2023) Rapgen: An approach for fixing code inefficiencies in zero-shot. arXiv:2306.17077"},{"key":"10776_CR21","unstructured":"GitHub Docs (2025a) Asking GitHub copilot questions in your IDE. https:\/\/docs.github.com\/en\/copilot\/how-tos\/use-chat\/use-chat-in-ide. Accessed: 2025-09-01"},{"key":"10776_CR22","unstructured":"GitHub Docs (2025b) Github language support. https:\/\/docs.github.com\/en\/get-started\/learning-about-github\/github-language-support. Accessed: 2025-02-25"},{"key":"10776_CR23","unstructured":"GitHub, Inc. (2025) Quickstart for GitHub Copilot. https:\/\/docs.github.com\/en\/copilot\/get-started\/quickstart?tool=vscode. Accessed: 2025-09-01"},{"key":"10776_CR24","doi-asserted-by":"crossref","unstructured":"Guo D, Tang D, Duan N, Zhou M, Yin J (2019) Coupling retrieval and meta-learning for context-dependent semantic parsing. In: Proceedings of the 57th Conference of the Association for Computational Linguistics, ACL 2019, Florence, Italy, July 28- August 2, 2019, Volume 1: Long Papers, pages 855\u2013866. Association for Computational Linguistics","DOI":"10.18653\/v1\/P19-1082"},{"key":"10776_CR25","unstructured":"Guo D, Ren S, Lu S, Feng Z, Tang D, Liu S, Zhou L, Duan N, Svyatkovskiy A, Fu S, Tufano M, Deng SK, Clement CB, Drain D, Sundaresan N, Yin J, Jiang D, Zhou M (2021) Graphcodebert: Pre-training code representations with data flow. In: 9th International Conference on Learning Representations, ICLR 2021, Virtual Event, Austria, May 3-7, 2021. OpenReview.net"},{"key":"10776_CR26","unstructured":"Guo D, Zhu Q, Yang D, Xie Z, Dong K, Zhang W, Chen G, Bi X, Wu Y, Li YK, Luo F, Xiong Y, Liang W (2024) Deepseek-coder: When the large language model meets programming - the rise of code intelligence. CoRR, abs\/2401.14196"},{"key":"10776_CR27","unstructured":"Hao Y, Li G, Liu Y, Miao X, Zong H, Jiang S, Liu Y, Wei H (2022) Aixbench: A code generation benchmark dataset. arXiv:2206.13179"},{"key":"10776_CR28","doi-asserted-by":"crossref","unstructured":"Hayati SA, Olivier R, Avvaru P, Yin P, Tomasic A, Neubig G (2018) Retrieval-based neural code generation. In: Proceedings of the 2018 Conference on Empirical Methods in Natural Language Processing, Brussels, Belgium, October 31 - November 4, 2018, pages 925\u2013930. Association for Computational Linguistics","DOI":"10.18653\/v1\/D18-1111"},{"key":"10776_CR29","unstructured":"He P, Wang S, Chen T-H (2025) Codepromptzip: Code-specific prompt compression for retrieval-augmented generation in coding tasks with lms. arXiv:2502.14925"},{"key":"10776_CR30","unstructured":"Hou W, Ji Z (2024) A systematic evaluation of large language models for generating programming code. arXiv:2403.00894"},{"key":"10776_CR31","unstructured":"Huang D, Zeng G, Dai J, Luo M, Weng H, Qing Y, Cui H, Guo Z, Zhang JM (2024a) Effi-code: Unleashing code efficiency in language models. CoRR, abs\/2410.10209"},{"key":"10776_CR32","first-page":"10","volume-title":"Advances in Neural Information Processing Systems 38: Annual Conference on Neural Information Processing Systems 2024, NeurIPS 2024","author":"D Huang","year":"2024","unstructured":"Huang D, Qing Y, Shang W, Cui H (2024) Zhang J (2024b) Effibench: Benchmarking the efficiency of automatically generated code. In: Globersons A, Mackey L, Belgrave D, Fan A, Paquet U, Tomczak JM, Zhang C (eds) Advances in Neural Information Processing Systems 38: Annual Conference on Neural Information Processing Systems 2024, NeurIPS 2024. BC, Canada, December, Vancouver, pp 10\u201315"},{"key":"10776_CR33","first-page":"10","volume-title":"Advances in Neural Information Processing Systems 38: Annual Conference on Neural Information Processing Systems 2024, NeurIPS 2024","author":"D Huang","year":"2024","unstructured":"Huang D, Dai J, Weng H, Wu P, Qing Y, Cui H, Guo Z (2024) Zhang J (2024c) Effilearner: Enhancing efficiency of generated code via self-optimization. In: Globersons A, Mackey L, Belgrave D, Fan A, Paquet U, Tomczak JM, Zhang C (eds) Advances in Neural Information Processing Systems 38: Annual Conference on Neural Information Processing Systems 2024, NeurIPS 2024. BC, Canada, December, Vancouver, pp 10\u201315"},{"key":"10776_CR34","unstructured":"Huang Y, Li Y, Wu W, Zhang J, Lyu MR (2023) Do not give away my secrets: Uncovering the privacy issue of neural code completion tools. CoRR, abs\/2309.07639"},{"key":"10776_CR35","doi-asserted-by":"crossref","unstructured":"Islam MA, Jonnala DV, Rekhi R, Pokharel P, Cilamkoti S, Imran A, Kosar T, Turkkan BO (2025) Evaluating the energy-efficiency of the code generated by llms. CoRR, abs\/2505.20324","DOI":"10.1109\/FLLM67465.2025.11391172"},{"key":"10776_CR36","doi-asserted-by":"crossref","unstructured":"Iyer S, Konstas I, Cheung A, Zettlemoyer L (2018) Mapping language to code in programmatic context. In: Proceedings of the 2018 Conference on Empirical Methods in Natural Language Processing, Brussels, Belgium, October 31 - November 4, 2018, pages 1643\u20131652. Association for Computational Linguistics","DOI":"10.18653\/v1\/D18-1192"},{"issue":"4","key":"10776_CR37","doi-asserted-by":"publisher","first-page":"1704","DOI":"10.1109\/TSE.2022.3188005","volume":"49","author":"M Jangali","year":"2023","unstructured":"Jangali M, Tang Y, Alexandersson N, Leitner P, Yang J, Shang W (2023) Automated generation and evaluation of JMH microbenchmark suites from unit tests. IEEE Trans Software Eng 49(4):1704\u20131725","journal-title":"IEEE Trans Software Eng"},{"key":"10776_CR38","unstructured":"JetBrains s.r.o (2024) About Qodana. https:\/\/www.jetbrains.com\/help\/qodana\/about-qodana.html. Accessed: 2024-12-30"},{"key":"10776_CR39","doi-asserted-by":"crossref","unstructured":"Khoury R, Avila AR, Brunelle J, Camara BM (2023) How secure is code generated by chatgpt? In: IEEE International Conference on Systems, Man, and Cybernetics, SMC 2023, Honolulu, Oahu, HI, USA, October 1-4, 2023, pages 2445\u20132451. IEEE","DOI":"10.1109\/SMC53992.2023.10394237"},{"key":"10776_CR40","doi-asserted-by":"crossref","unstructured":"Laaber C, Leitner P (2018) An evaluation of open-source software microbenchmark suites for continuous performance assessment. In: Proceedings of the 15th International Conference on Mining Software Repositories, MSR 2018, Gothenburg, Sweden, May 28-29, 2018, pages 119\u2013130. ACM","DOI":"10.1145\/3196398.3196407"},{"key":"10776_CR41","doi-asserted-by":"crossref","unstructured":"Lertbanjongngam, S., Chinthanet, B., Ishio, T., Kula, R.\u00a0G., Leelaprute, P., Manaskasemsak, B., Rungsawang, A., and Matsumoto, K. (2022). An empirical evaluation of competitive programming AI: A case study of alphacode. In: 16th IEEE International Workshop on Software Clones, IWSC 2022, Limassol, Cyprus, October 2, 2022, pages 10\u201315. IEEE","DOI":"10.1109\/IWSC55060.2022.00010"},{"key":"10776_CR42","doi-asserted-by":"crossref","unstructured":"Li B, Sun Z, Huang T, Zhang H, Wan Y, Li G, Jin Z, Lyu C (2024a) Ircoco: Immediate rewards-guided deep reinforcement learning for code completion. Proc ACM Softw Eng 1(FSE):182\u2013203","DOI":"10.1145\/3643735"},{"key":"10776_CR43","doi-asserted-by":"crossref","unstructured":"Li S, Cheng Y, Chen J, Xuan J, He S, Shang W (2024b) Assessing the performance of ai-generated code: A case study on github copilot. In: 35th IEEE International Symposium on Software Reliability Engineering, ISSRE 2024, Tsukuba, Japan, October 28-31, 2024, pages 216\u2013227. IEEE","DOI":"10.1109\/ISSRE62328.2024.00030"},{"key":"10776_CR44","doi-asserted-by":"crossref","unstructured":"Li Y, Choi DH, Chung J, Kushman N, Schrittwieser J, Leblond R, Eccles T, Keeling J, Gimeno F, Lago AD, Hubert T, Choy P, de\u00a0Masson\u00a0d\u2019Autume C, Babuschkin I, Chen X, Huang P, Welbl J, Gowal S, Cherepanov A, Molloy J, Mankowitz DJ, Robson ES, Kohli P, de\u00a0Freitas N, Kavukcuoglu K, Vinyals O (2022) Competition-level code generation with alphacode. CoRR, abs\/2203.07814","DOI":"10.1126\/science.abq1158"},{"key":"10776_CR45","doi-asserted-by":"crossref","unstructured":"Ling W, Blunsom P, Grefenstette E, Hermann KM, Kocisk\u00fd T, Wang F, Senior AW (2016) Latent predictor networks for code generation. In: Proceedings of the 54th Annual Meeting of the Association for Computational Linguistics, ACL 2016, August 7-12, 2016, Berlin, Germany, Volume 1: Long Papers. The Association for Computer Linguistics","DOI":"10.18653\/v1\/P16-1057"},{"key":"10776_CR46","unstructured":"Liu J, Xie S, Wang J, Wei Y, Ding Y, Zhang L (2024a) Evaluating language models for efficient code generation. CoRR, abs\/2408.06450"},{"key":"10776_CR47","doi-asserted-by":"crossref","unstructured":"Liu,J. Xia CS, Wang Y, Zhang L (2024b) Is your code generated by chatgpt really correct? rigorous evaluation of large language models for code generation. Advances in Neural Information Processing Systems, 36","DOI":"10.52202\/075280-0943"},{"key":"10776_CR48","unstructured":"Luo W, Keung JW, Yang B, Ye H, Le Goues C, Bissyand\u00e9 TF, Tian H, Le B (2024a) When fine-tuning llms meets data privacy: An empirical study of federated learning in llm-based program repair. CoRR, abs\/2412.01072"},{"key":"10776_CR49","unstructured":"Luo Z, Xu C, Zhao P, Sun Q, Geng X, Hu W, Tao C, Ma J, Lin Q, Jiang D (2024b) Wizardcoder: Empowering code large language models with evol-instruct. In: The Twelfth International Conference on Learning Representations, ICLR 2024, Vienna, Austria, May 7-11, 2024. OpenReview.net"},{"key":"10776_CR50","unstructured":"Mayer L, Heumann C, A\u00dfenmacher M (2024) Can opensource beat chatgpt? - A comparative study of large language models for text-to-code generation. CoRR, abs\/2409.04164"},{"issue":"3","key":"10776_CR51","doi-asserted-by":"publisher","first-page":"276","DOI":"10.11613\/BM.2012.031","volume":"22","author":"ML McHugh","year":"2012","unstructured":"McHugh ML (2012) Interrater reliability: the kappa statistic. Biochemia medica 22(3):276\u2013282","journal-title":"Biochemia medica"},{"key":"10776_CR52","unstructured":"Microsoft (2025) Visual Studio Code - Code Editing. https:\/\/code.visualstudio.com\/. Accessed: 2025-09-01"},{"key":"10776_CR53","doi-asserted-by":"crossref","unstructured":"Nguyen N, Nadi S (2022) An empirical evaluation of github copilot\u2019s code suggestions. In: 19th IEEE\/ACM International Conference on Mining Software Repositories, MSR 2022, Pittsburgh, PA, USA, May 23-24, 2022, pages 1\u20135. ACM","DOI":"10.1145\/3524842.3528470"},{"key":"10776_CR54","unstructured":"Nijkamp E, Pang B, Hayashi H, Tu L, Wang H, Zhou Y, Savarese S, Xiong C (2023) Codegen: An open large language model for code with multi-turn program synthesis. In: The Eleventh International Conference on Learning Representations, ICLR 2023, Kigali, Rwanda, May 1-5, 2023. OpenReview.net"},{"key":"10776_CR55","unstructured":"Niu L, Mirza MS, Maradni Z, P\u00f6pper C (2023) Codexleaks: Privacy leaks from code generation language models in github copilot. In: 32nd USENIX Security Symposium, USENIX Security 2023, Anaheim, CA, USA, August 9-11, 2023, pages 2133\u20132150. USENIX Association"},{"key":"10776_CR56","doi-asserted-by":"crossref","unstructured":"Parvez MR, Ahmad WU, Chakraborty S, Ray B, Chang K (2021) Retrieval augmented code generation and summarization. In: Findings of the Association for Computational Linguistics: EMNLP 2021, Virtual Event \/ Punta Cana, Dominican Republic, 16-20 November, 2021, pages 2719\u20132734. Association for Computational Linguistics","DOI":"10.18653\/v1\/2021.findings-emnlp.232"},{"key":"10776_CR57","doi-asserted-by":"crossref","unstructured":"Pearce H, Ahmad B, Tan B, Dolan-Gavitt B, Karri R (2022) Asleep at the keyboard? assessing the security of github copilot\u2019s code contributions. In: 43rd IEEE Symposium on Security and Privacy, SP 2022, San Francisco, CA, USA, May 22-26, 2022, pages 754\u2013768. IEEE","DOI":"10.1109\/SP46214.2022.9833571"},{"key":"10776_CR58","doi-asserted-by":"crossref","unstructured":"Peng Y, Gotmare AD, Lyu MR, Xiong C, Savarese S, Sahoo D (2025) Perfcodegen: Improving performance of LLM generated code with execution feedback. In: IEEE\/ACM Second International Conference on AI Foundation Models and Software Engineering, Forge@ICSE 2025, Ottawa, ON, Canada, April 27-28, 2025, pages 1\u201313. IEEE","DOI":"10.1109\/Forge66646.2025.00008"},{"key":"10776_CR59","unstructured":"PMD (2024) PMD: An extensible cross-language static code analyzer. https:\/\/pmd.github.io\/. Accessed: 2024-04-25"},{"key":"10776_CR60","unstructured":"Psutil Documentation Team (2025). Psutil documentation \u2014 psutil 7.1.0 documentation. https:\/\/psutil.readthedocs.io\/en\/latest\/. Accessed: 2025-09-01"},{"key":"10776_CR61","unstructured":"Python Software Foundation (2025a) psutil 7.0.0. https:\/\/pypi.org\/project\/psutil\/. Accessed: 2025-09-01"},{"key":"10776_CR62","unstructured":"Python Software Foundation (2025b) The Python Profilers. https:\/\/docs.python.org\/3\/library\/profile.html. Accessed: 2025-09-01"},{"key":"10776_CR63","unstructured":"Python Software Foundation (2025c) tracemalloc \u2013 Trace memory allocations. https:\/\/docs.python.org\/3\/library\/tracemalloc.html. Accessed: 2025-09-01"},{"key":"10776_CR64","doi-asserted-by":"crossref","unstructured":"Rabinovich M, Stern M, Klein D (2017) Abstract syntax networks for code generation and semantic parsing. In: Proceedings of the 55th Annual Meeting of the Association for Computational Linguistics, ACL 2017, Vancouver, Canada, July 30 - August 4, Volume 1: Long Papers, pages 1139\u20131149. Association for Computational Linguistics","DOI":"10.18653\/v1\/P17-1105"},{"issue":"4","key":"10776_CR65","first-page":"32","volume":"39","author":"DG Reichelt","year":"2019","unstructured":"Reichelt DG, K\u00fchne S, Hasselbring W (2019) On the validity of performance antipatterns at code level. Softwaretechnik-Trends 39(4):32\u201334","journal-title":"Softwaretechnik-Trends"},{"key":"10776_CR66","doi-asserted-by":"crossref","unstructured":"Reichelt DG, K\u00fchne S, Hasselbring W (2019b) Peass: A tool for identifying performance changes at code level. In: 34th IEEE\/ACM International Conference on Automated Software Engineering, ASE 2019, San Diego, CA, USA, November 11-15, 2019, pages 1146\u20131149. IEEE","DOI":"10.1109\/ASE.2019.00123"},{"key":"10776_CR67","unstructured":"Rozi\u00e8re B, Gehring J, Gloeckle F, Sootla S, Gat I, Tan XE, Adi Y, Liu J, Remez T, Rapin J, Kozhevnikov A, Evtimov I, Bitton J, Bhatt M, Canton-Ferrer C, Grattafiori A, Xiong W, D\u00e9fossez A, Copet J, Azhar F, Touvron H, Martin L, Usunier N, Scialom T, Synnaeve G (2023) Code llama: Open foundation models for code. CoRR, abs\/2308.12950"},{"key":"10776_CR68","first-page":"11045","volume-title":"Findings of the Association for Computational Linguistics: EMNLP 2024, Miami, Florida, USA, November 12\u201316, 2024","author":"M Seo","year":"2024","unstructured":"Seo M, Baek J, Hwang SJ (2024) Rethinking code refinement: Learning to judge code efficiency. In: Al-Onaizan Y, Bansal M, Chen Y (eds) Findings of the Association for Computational Linguistics: EMNLP 2024, Miami, Florida, USA, November 12\u201316, 2024. Association for Computational Linguistics, pp 11045\u201311056"},{"issue":"5","key":"10776_CR69","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1145\/3708525","volume":"34","author":"J Shi","year":"2025","unstructured":"Shi J, Yang Z, Lo D (2025) Efficient and green large language models for software engineering: Literature review, vision, and the road ahead. ACM Trans. Softw. Eng. Methodol. 34(5):1\u201322","journal-title":"ACM Trans. Softw. Eng. Methodol."},{"key":"10776_CR70","unstructured":"Shypula A, Madaan A, Zeng Y, Alon U, Gardner JR, Yang Y, Hashemi M, Neubig G, Ranganathan P, Bastani O, Yazdanbakhsh A (2024) Learning performance-improving code edits. In: The Twelfth International Conference on Learning Representations, ICLR 2024, Vienna, Austria, May 7-11, 2024. OpenReview.net"},{"key":"10776_CR71","doi-asserted-by":"crossref","unstructured":"Sikand S, Mehra R, Sharma VS, Kaulgud V, Podder S, Burden AP (2024) Do generative AI tools ensure green code? an investigative study. In: Proceedings of the 2nd International Workshop on Responsible AI Engineering, RAIE 2024, Lisbon, Portugal, 16 April 2024, pages 52\u201355. ACM","DOI":"10.1145\/3643691.3648588"},{"key":"10776_CR72","doi-asserted-by":"crossref","unstructured":"Sobania D, Briesch M, Rothlauf F (2022) Choose your programming copilot: a comparison of the program synthesis performance of github copilot and genetic programming. In: GECCO \u201922: Genetic and Evolutionary Computation Conference, Boston, Massachusetts, USA, July 9 - 13, 2022, pages 1019\u20131027. ACM","DOI":"10.1145\/3512290.3528700"},{"key":"10776_CR73","unstructured":"SonarQube (2024) SonarQube: A code quality management platform. https:\/\/www.sonarqube.org\/. Accessed: 2024-04-28"},{"key":"10776_CR74","unstructured":"SpotBugs (2024) SpotBugs: Find bugs in Java Programs. https:\/\/spotbugs.github.io\/. Accessed: 2024-04-25"},{"key":"10776_CR75","doi-asserted-by":"crossref","unstructured":"Sun Z, Zhu Q, Xiong Y, Sun Y, Mou L, Zhang L (2020) Treegen: A tree-based transformer architecture for code generation. In: The Thirty-Fourth AAAI Conference on Artificial Intelligence, AAAI 2020, The Thirty-Second Innovative Applications of Artificial Intelligence Conference, IAAI 2020, The Tenth AAAI Symposium on Educational Advances in Artificial Intelligence, EAAI 2020, New York, NY, USA, February 7-12, 2020, pages 8984\u20138991. AAAI Press","DOI":"10.1609\/aaai.v34i05.6430"},{"key":"10776_CR76","unstructured":"Tanenbaum AS (2015) Modern Operating Systems. Pearson, Boston, fourth edition edition"},{"key":"10776_CR77","doi-asserted-by":"publisher","DOI":"10.1016\/j.jss.2023.111904","volume":"210","author":"I van Dinten","year":"2024","unstructured":"van Dinten I, Derakhshanfar P, Panichella A, Zaidman A (2024) The slow and the furious? performance antipattern detection in cyber-physical systems. J Syst Softw 210:111904","journal-title":"J Syst Softw"},{"key":"10776_CR78","doi-asserted-by":"crossref","unstructured":"Vartziotis T, Dellatolas I, Dasoulas G, Schmidt M, Schneider F, Hoffmann T, Kotsopoulos S, Keckeisen M (2024) Learn to code sustainably: An empirical study on green code generation. In: LLM4CODE@ICSE, pages 30\u201337","DOI":"10.1145\/3643795.3648394"},{"key":"10776_CR79","doi-asserted-by":"crossref","unstructured":"Venkatesh APS, Sabu S, Mir AM, Reis S, Bodden E (2024) The emergence of large language models in static analysis: A first look through micro-benchmarks. In: D.\u00a0Lo, X.\u00a0Xia, M.\u00a0D. Penta, and X.\u00a0Hu, editors, Proceedings of the 2024 IEEE\/ACM First International Conference on AI Foundation Models and Software Engineering, FORGE 2024, Lisbon, Portugal, 14 April 2024, pages 35\u201339. ACM","DOI":"10.1145\/3650105.3652288"},{"key":"10776_CR80","unstructured":"Visual Studio Code Docs (2025) Github copilot in vs code settings reference. https:\/\/code.visualstudio.com\/docs\/copilot\/reference\/copilot-settings. Accessed: 2025-09-01"},{"key":"10776_CR81","doi-asserted-by":"crossref","unstructured":"Waghjale S, Veerendranath V, Wang Z, Fried D (2024) ECCO: can we improve model-generated code efficiency without sacrificing functional correctness? In: Y.\u00a0Al-Onaizan, M.\u00a0Bansal, and Y.\u00a0Chen, editors, Proceedings of the 2024 Conference on Empirical Methods in Natural Language Processing, EMNLP 2024, Miami, FL, USA, November 12-16, 2024, pages 15362\u201315376. Association for Computational Linguistics","DOI":"10.18653\/v1\/2024.emnlp-main.859"},{"key":"10776_CR82","doi-asserted-by":"crossref","unstructured":"Wang Y, Wang W, Joty SR, Hoi SCH (2021) Codet5: Identifier-aware unified pre-trained encoder-decoder models for code understanding and generation. In: Proceedings of the 2021 Conference on Empirical Methods in Natural Language Processing, EMNLP 2021, Virtual Event \/ Punta Cana, Dominican Republic, 7-11 November, 2021, pages 8696\u20138708. Association for Computational Linguistics","DOI":"10.18653\/v1\/2021.emnlp-main.685"},{"key":"10776_CR83","doi-asserted-by":"crossref","unstructured":"Wang Z, Zhou Z, Song D, Huang Y, Chen S, Ma L, Zhang T (2025) Towards understanding the characteristics of code generation errors made by large language models","DOI":"10.1109\/ICSE55347.2025.00180"},{"key":"10776_CR84","unstructured":"Wei B, Li G, Xia X, Fu Z, Jin Z (2019) Code generation as a dual task of code summarization. In: Advances in Neural Information Processing Systems 32: Annual Conference on Neural Information Processing Systems 2019, NeurIPS 2019, December 8-14, 2019, Vancouver, BC, Canada, pages 6559\u20136569"},{"key":"10776_CR85","unstructured":"Wei J, Wang X, Schuurmans D, Bosma M, Ichter B, Xia F, Chi EH, Le QV, Zhou D (2022) Chain-of-thought prompting elicits reasoning in large language models. In S.\u00a0Koyejo, S.\u00a0Mohamed, A.\u00a0Agarwal, D.\u00a0Belgrave, K.\u00a0Cho, and A.\u00a0Oh, editors, Advances in Neural Information Processing Systems 35: Annual Conference on Neural Information Processing Systems 2022, NeurIPS 2022, New Orleans, LA, USA, November 28 - December 9, 2022"},{"key":"10776_CR86","first-page":"10","volume-title":"Advances in Neural Information Processing Systems 38: Annual Conference on Neural Information Processing Systems 2024, NeurIPS 2024","author":"Y Wei","year":"2024","unstructured":"Wei Y, Cassano F, Liu J, Ding Y, Jain N, Mueller Z, de Vries H, von Werra L, Guha A (2024) Zhang L (2024) Selfcodealign: Self-alignment for code generation. In: Globersons A, Mackey L, Belgrave D, Fan A, Paquet U, Tomczak JM, Zhang C (eds) Advances in Neural Information Processing Systems 38: Annual Conference on Neural Information Processing Systems 2024, NeurIPS 2024. BC, Canada, December, Vancouver, pp 10\u201315"},{"key":"10776_CR87","doi-asserted-by":"crossref","unstructured":"Weng L, Hu Y, Huang P, Nieh J, Yang J (2023) Effective performance issue diagnosis with value-assisted cost profiling. In: Proceedings of the Eighteenth European Conference on Computer Systems, EuroSys 2023, Rome, Italy, May 8-12, 2023, pages 1\u201317. ACM","DOI":"10.1145\/3552326.3587444"},{"key":"10776_CR88","unstructured":"White J, Fu Q, Hays S, Sandborn M, Olea C, Gilbert H, Elnashar A, Spencer-Smith J, Schmidt DC (2023) A prompt pattern catalog to enhance prompt engineering with chatgpt. CoRR, abs\/2302.11382"},{"key":"10776_CR89","doi-asserted-by":"crossref","unstructured":"Yan D, Xu G, Rountev A (2012) Uncovering performance problems in java applications with reference propagation profiling. In M.\u00a0Glinz, G.\u00a0C. Murphy, and M.\u00a0Pezz\u00e8, editors, 34th International Conference on Software Engineering, ICSE 2012, June 2-9, 2012, Zurich, Switzerland, pages 134\u2013144. IEEE Computer Society","DOI":"10.1109\/ICSE.2012.6227199"},{"key":"10776_CR90","unstructured":"Yang C, Kang HJ, Shi J, Lo D (2024) Acecode: A reinforcement learning framework for aligning code efficiency and correctness in code language models. CoRR, abs\/2412.17264"},{"key":"10776_CR91","unstructured":"Yang Z, Zhao Z, Wang C, Shi J, Kim D, Han D, Lo D (2023) Gotcha! this model uses my code! evaluating membership leakage risks in code models. CoRR, abs\/2310.01166"},{"key":"10776_CR92","doi-asserted-by":"crossref","unstructured":"Yetistiren B, Ozsoy I, Tuzun E (2022) Assessing the quality of github copilot\u2019s code generation. In: Proceedings of the 18th International Conference on Predictive Models and Data Analytics in Software Engineering, PROMISE 2022, Singapore, Singapore, 17 November 2022, pages 62\u201371. ACM","DOI":"10.1145\/3558489.3559072"},{"key":"10776_CR93","unstructured":"Yetistiren B, \u00d6zsoy I, Ayerdem M, T\u00fcz\u00fcn E (2023) Evaluating the code quality of ai-assisted code generation tools: An empirical study on github copilot, amazon codewhisperer, and chatgpt. CoRR, abs\/2304.10778"},{"key":"10776_CR94","doi-asserted-by":"crossref","unstructured":"Yin P, Neubig G (2017) A syntactic neural model for general-purpose code generation. In: Proceedings of the 55th Annual Meeting of the Association for Computational Linguistics, ACL 2017, Vancouver, Canada, July 30 - August 4, Volume 1: Long Papers, pages 440\u2013450. Association for Computational Linguistics","DOI":"10.18653\/v1\/P17-1041"},{"key":"10776_CR95","doi-asserted-by":"crossref","unstructured":"Zan D, Chen B, Zhang F, Lu D, Wu B, Guan B, Wang Y, Lou J-G (2022) Large language models meet nl2code: A survey. preprint arXiv:2212.09420","DOI":"10.18653\/v1\/2023.acl-long.411"},{"issue":"6","key":"10776_CR96","doi-asserted-by":"publisher","first-page":"3394","DOI":"10.1007\/s10664-019-09687-9","volume":"24","author":"Y Zeng","year":"2019","unstructured":"Zeng Y, Chen J, Shang W, Chen T-HP (2019) Studying the characteristics of logging practices in mobile apps: a case study on F-Droid. Empir Softw Eng 24(6):3394\u20133434","journal-title":"Empir Softw Eng"},{"key":"10776_CR97","doi-asserted-by":"crossref","unstructured":"Zhang B, Du T, Tong J, Zhang X, Chow K, Cheng S, Wang X, Yin J (2024) Seccoder: Towards generalizable and robust secure code generation. In: Y.\u00a0Al-Onaizan, M.\u00a0Bansal, and Y.\u00a0Chen, editors, Proceedings of the 2024 Conference on Empirical Methods in Natural Language Processing, EMNLP 2024, Miami, FL, USA, November 12-16, 2024, pages 14557\u201314571. Association for Computational Linguistics","DOI":"10.18653\/v1\/2024.emnlp-main.806"},{"key":"10776_CR98","unstructured":"Zheng Z, Ning K, Wang Y, Zhang J, Zheng D, Ye M, Chen J (2023) A survey of large language models for code: Evolution, benchmarking, and future trends. arXiv:2311.10372"},{"key":"10776_CR99","unstructured":"Zhou S, Alon U, Xu FF, Jiang Z, Neubig G (2023) Docprompting: Generating code by retrieving the docs. In: The Eleventh International Conference on Learning Representations, ICLR 2023, Kigali, Rwanda, May 1-5, 2023. OpenReview.net"}],"container-title":["Empirical Software Engineering"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10664-025-10776-1.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s10664-025-10776-1","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10664-025-10776-1.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,3,30]],"date-time":"2026-03-30T14:36:41Z","timestamp":1774881401000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s10664-025-10776-1"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026,1,19]]},"references-count":99,"journal-issue":{"issue":"3","published-print":{"date-parts":[[2026,5]]}},"alternative-id":["10776"],"URL":"https:\/\/doi.org\/10.1007\/s10664-025-10776-1","relation":{},"ISSN":["1382-3256","1573-7616"],"issn-type":[{"value":"1382-3256","type":"print"},{"value":"1573-7616","type":"electronic"}],"subject":[],"published":{"date-parts":[[2026,1,19]]},"assertion":[{"value":"5 March 2025","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"11 November 2025","order":2,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"19 January 2026","order":3,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Declarations"}},{"value":"Not applicable.","order":2,"name":"Ethics","group":{"name":"EthicsHeading","label":"Ethical Approval"}},{"value":"Not applicable.","order":3,"name":"Ethics","group":{"name":"EthicsHeading","label":"Informed Consent"}},{"value":"The authors declare no potential conflict of interest.","order":4,"name":"Ethics","group":{"name":"EthicsHeading","label":"Conflict of Interest"}},{"value":"Not applicable.","order":5,"name":"Ethics","group":{"name":"EthicsHeading","label":"Clinical Trial Number"}}],"article-number":"62"}}