{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,8]],"date-time":"2026-04-08T16:24:02Z","timestamp":1775665442201,"version":"3.50.1"},"publisher-location":"New York, NY, USA","reference-count":43,"publisher":"ACM","license":[{"start":{"date-parts":[[2024,6,18]],"date-time":"2024-06-18T00:00:00Z","timestamp":1718668800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"name":"DISTILLER","award":["ANR-21-CE25-0022"],"award-info":[{"award-number":["ANR-21-CE25-0022"]}]},{"name":"KOALA","award":["ANR-19-CE25-0003-01"],"award-info":[{"award-number":["ANR-19-CE25-0003-01"]}]},{"name":"CARECloud","award":["ANR-23-PECL-0003"],"award-info":[{"award-number":["ANR-23-PECL-0003"]}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2024,6,18]]},"DOI":"10.1145\/3661167.3661221","type":"proceedings-article","created":{"date-parts":[[2024,6,14]],"date-time":"2024-06-14T12:24:25Z","timestamp":1718367865000},"page":"79-89","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":60,"title":["A Performance Study of LLM-Generated Code on Leetcode"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0009-0003-2525-3637","authenticated-orcid":false,"given":"Tristan","family":"Coignion","sequence":"first","affiliation":[{"name":"CRIStAL, Universit\u00e9 de Lille - Inria, France"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-3203-6107","authenticated-orcid":false,"given":"Cl\u00e9ment","family":"Quinton","sequence":"additional","affiliation":[{"name":"CRIStAL, Universit\u00e9 de Lille - Inria, France"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-1771-8791","authenticated-orcid":false,"given":"Romain","family":"Rouvoy","sequence":"additional","affiliation":[{"name":"CRIStAL, Universit\u00e9 de Lille - Inria, France"}]}],"member":"320","published-online":{"date-parts":[[2024,6,18]]},"reference":[{"key":"e_1_3_2_1_1_1","unstructured":"2022. Codeparrot\/Codeparrot \u00b7 Hugging Face. https:\/\/huggingface.co\/codeparrot\/codeparrot."},{"key":"e_1_3_2_1_2_1","first-page":"42","article-title":"The Impact of Source Code in Software on Power Consumption","volume":"14","author":"Acar Hayri","year":"2016","unstructured":"Hayri Acar, G\u00fclfem\u00a0I Alptekin, Jean-Patrick Gelas, and Parisa Ghodous. 2016. The Impact of Source Code in Software on Power Consumption. International Journal of Electronic Business Management 14 (2016), 42\u201352.","journal-title":"International Journal of Electronic Business Management"},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","unstructured":"Hojjat Aghakhani 2023. TrojanPuzzle: Covertly Poisoning Code-Suggestion Models. (2023). https:\/\/doi.org\/10.48550\/ARXIV.2301.02344","DOI":"10.48550\/ARXIV.2301.02344"},{"key":"e_1_3_2_1_4_1","unstructured":"Loubna\u00a0Ben Allal 2023. SantaCoder: Don\u2019t Reach for the Stars!arxiv:2301.03988\u00a0[cs]"},{"key":"e_1_3_2_1_5_1","unstructured":"Jacob Austin 2021. Program Synthesis with Large Language Models. arxiv:2108.07732\u00a0[cs]"},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"crossref","unstructured":"Daniel Balouek 2013. Adding Virtualization Capabilities to the Grid\u20195000 Testbed. In Cloud Computing and Services Science. Communications in Computer and Information Science Vol.\u00a0367. 3\u201320.","DOI":"10.1007\/978-3-319-04519-1_1"},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"publisher","DOI":"10.1145\/3586030"},{"key":"e_1_3_2_1_8_1","unstructured":"S\u00e9bastien Bubeck 2023. Sparks of Artificial General Intelligence: Early Experiments with GPT-4. arxiv:2303.12712\u00a0[cs]"},{"key":"e_1_3_2_1_9_1","unstructured":"Binghong Chen and othersy. 2022. Learning to Improve Code Efficiency. arxiv:2208.05297\u00a0[cs]"},{"key":"e_1_3_2_1_10_1","unstructured":"Mark Chen 2021. Evaluating Large Language Models Trained on Code. arxiv:2107.03374\u00a0[cs]"},{"key":"e_1_3_2_1_11_1","volume-title":"Supersonic: Learning to Generate Source Code Optimizations in C\/C++. arxiv:2309.14846\u00a0[cs]","author":"Chen Zimin","year":"2023","unstructured":"Zimin Chen, Sen Fang, and Martin Monperrus. 2023. Supersonic: Learning to Generate Source Code Optimizations in C\/C++. arxiv:2309.14846\u00a0[cs]"},{"key":"e_1_3_2_1_12_1","unstructured":"Fenia Christopoulou 2022. PanGu-Coder: Program Synthesis with Function-Level Language Modeling. arxiv:2207.11280\u00a0[cs]"},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","unstructured":"Jean-Baptiste D\u00f6derlein Mathieu Acher Djamel\u00a0Eddine Khelladi and Benoit Combemale. 2023. Piloting Copilot and Codex: Hot Temperature Cold Prompts or Black Magic?https:\/\/doi.org\/10.2139\/ssrn.4496380","DOI":"10.2139\/ssrn.4496380"},{"key":"e_1_3_2_1_14_1","unstructured":"Daniel Fried Armen Aghajanyan Jessy Lin Sida Wang Eric Wallace Freda Shi Ruiqi Zhong Wen-tau Yih Luke Zettlemoyer and Mike Lewis. 2022. InCoder: A Generative Model for Code Infilling and Synthesis. arxiv:2204.05999\u00a0[cs]"},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.1145\/3540250.3549096"},{"key":"e_1_3_2_1_16_1","unstructured":"Spandan Garg 2023. RAPGen: An Approach for Fixing Code Inefficiencies in Zero-Shot. arxiv:2306.17077\u00a0[cs]"},{"key":"e_1_3_2_1_17_1","volume-title":"Proceedings of the Neural Information Processing Systems Track on Datasets and Benchmarks 1 (Dec.","author":"Hendrycks Dan","year":"2021","unstructured":"Dan Hendrycks 2021. Measuring Coding Challenge Competence With APPS. Proceedings of the Neural Information Processing Systems Track on Datasets and Benchmarks 1 (Dec. 2021)."},{"key":"e_1_3_2_1_18_1","unstructured":"Ari Holtzman Jan Buys Li Du Maxwell Forbes and Yejin Choi. 2020. The Curious Case of Neural Text Degeneration. arxiv:1904.09751\u00a0[cs]"},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"crossref","unstructured":"Alon Jacovi Avi Caciularu Omer Goldman and Yoav Goldberg. 2023. Stop Uploading Test Data in Plain Text: Practical Strategies for Mitigating Data Contamination by Evaluation Benchmarks. arxiv:2305.10160\u00a0[cs]","DOI":"10.18653\/v1\/2023.emnlp-main.308"},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.1109\/MSR59073.2023.00082"},{"key":"e_1_3_2_1_21_1","volume-title":"StarCoder: May the Source Be with You!","author":"Raymond Li","year":"2023","unstructured":"Raymond Li 2023. StarCoder: May the Source Be with You! (2023). arxiv:2305.06161\u00a0[cs.CL]"},{"key":"e_1_3_2_1_22_1","volume-title":"Is your code generated by chatgpt really correct? rigorous evaluation of large language models for code generation. Advances in Neural Information Processing Systems 36","author":"Liu Jiawei","year":"2024","unstructured":"Jiawei Liu, Chunqiu\u00a0Steven Xia, Yuyao Wang, and Lingming Zhang. 2024. Is your code generated by chatgpt really correct? rigorous evaluation of large language models for code generation. Advances in Neural Information Processing Systems 36 (2024)."},{"key":"e_1_3_2_1_23_1","unstructured":"Shuai Lu 2021. CodeXGLUE: A Machine Learning Benchmark Dataset for Code Understanding and Generation. arxiv:2102.04664\u00a0[cs]"},{"key":"e_1_3_2_1_24_1","unstructured":"Ziyang Luo 2023. WizardCoder: Empowering Code Large Language Models with Evol-Instruct. arxiv:2306.08568\u00a0[cs]"},{"key":"e_1_3_2_1_25_1","unstructured":"Aman Madaan 2023. Learning Performance-Improving Code Edits."},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.1145\/3524842.3528470"},{"key":"e_1_3_2_1_27_1","unstructured":"Erik Nijkamp 2022. CodeGen: An Open Large Language Model for Code with Multi-Turn Program Synthesis. arxiv:2203.13474\u00a0[cs]"},{"key":"e_1_3_2_1_28_1","unstructured":"OpenAI. 2023. GPT-4 Technical Report. arxiv:2303.08774\u00a0[cs]"},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.1109\/SP46214.2022.9833571"},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"publisher","DOI":"10.1145\/3576915.3623157"},{"key":"e_1_3_2_1_31_1","volume-title":"Proceedings of the Neural Information Processing Systems Track on Datasets and Benchmarks 1 (Dec.","author":"Puri Ruchir","year":"2021","unstructured":"Ruchir Puri 2021. CodeNet: A Large-Scale AI for Code Dataset for Learning a Diversity of Coding Tasks. Proceedings of the Neural Information Processing Systems Track on Datasets and Benchmarks 1 (Dec. 2021)."},{"key":"e_1_3_2_1_32_1","doi-asserted-by":"publisher","unstructured":"Baptiste Rozi\u00e8re 2023. Code Llama: Open Foundation Models for Code. https:\/\/doi.org\/10.48550\/arXiv.2308.12950 arxiv:2308.12950\u00a0[cs]","DOI":"10.48550\/arXiv.2308.12950"},{"key":"e_1_3_2_1_33_1","volume-title":"32nd USENIX Security Symposium (USENIX Security 23)","author":"Sandoval Gustavo","year":"2023","unstructured":"Gustavo Sandoval 2023. Lost at C: A User Study on the Security Implications of Large Language Model Code Assistants. In 32nd USENIX Security Symposium (USENIX Security 23). 2205\u20132222."},{"key":"e_1_3_2_1_34_1","doi-asserted-by":"publisher","DOI":"10.1145\/3491101.3519665"},{"key":"e_1_3_2_1_35_1","doi-asserted-by":"publisher","unstructured":"Helena Vasconcelos 2023. Generation Probabilities Are Not Enough: Exploring the Effectiveness of Uncertainty Highlighting in AI-Powered Code Completions. (2023). https:\/\/doi.org\/10.48550\/ARXIV.2302.07248","DOI":"10.48550\/ARXIV.2302.07248"},{"key":"e_1_3_2_1_36_1","doi-asserted-by":"publisher","DOI":"10.1109\/ESEM.2017.39"},{"key":"e_1_3_2_1_37_1","doi-asserted-by":"publisher","unstructured":"Shiqi Wang 2022. ReCode: Robustness Evaluation of Code Generation Models. https:\/\/doi.org\/10.48550\/ARXIV.2212.10264","DOI":"10.48550\/ARXIV.2212.10264"},{"key":"e_1_3_2_1_38_1","doi-asserted-by":"crossref","unstructured":"Yue Wang 2021. CodeT5: Identifier-aware Unified Pre-trained Encoder-Decoder Models for Code Understanding and Generation. arxiv:2109.00859\u00a0[cs]","DOI":"10.18653\/v1\/2021.emnlp-main.685"},{"key":"e_1_3_2_1_39_1","unstructured":"Jason Wei 2023. Chain-of-Thought Prompting Elicits Reasoning in Large Language Models. arxiv:2201.11903\u00a0[cs]"},{"key":"e_1_3_2_1_40_1","doi-asserted-by":"publisher","DOI":"10.1145\/3520312.3534862"},{"key":"e_1_3_2_1_41_1","doi-asserted-by":"publisher","DOI":"10.1145\/3510454.3516866"},{"key":"e_1_3_2_1_42_1","doi-asserted-by":"publisher","DOI":"10.1145\/3558489.3559072"},{"key":"e_1_3_2_1_43_1","unstructured":"Hao Yu 2023. CoderEval: A Benchmark of Pragmatic Code Generation with Generative Pre-trained Models. arxiv:2302.00288\u00a0[cs]"}],"event":{"name":"EASE 2024: 28th International Conference on Evaluation and Assessment in Software Engineering","location":"Salerno Italy","acronym":"EASE 2024"},"container-title":["Proceedings of the 28th International Conference on Evaluation and Assessment in Software Engineering"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3661167.3661221","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3661167.3661221","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,8,22]],"date-time":"2025-08-22T11:15:48Z","timestamp":1755861348000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3661167.3661221"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,6,18]]},"references-count":43,"alternative-id":["10.1145\/3661167.3661221","10.1145\/3661167"],"URL":"https:\/\/doi.org\/10.1145\/3661167.3661221","relation":{},"subject":[],"published":{"date-parts":[[2024,6,18]]},"assertion":[{"value":"2024-06-18","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}