{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,3]],"date-time":"2026-07-03T16:31:11Z","timestamp":1783096271832,"version":"3.54.6"},"publisher-location":"New York, NY, USA","reference-count":65,"publisher":"ACM","license":[{"start":{"date-parts":[[2024,10,27]],"date-time":"2024-10-27T00:00:00Z","timestamp":1729987200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"name":"National Science Foundation of China","award":["61932021"],"award-info":[{"award-number":["61932021"]}]},{"name":"Hong Kong Research Grant Council\/General Research Fund","award":["16207120"],"award-info":[{"award-number":["16207120"]}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2024,10,27]]},"DOI":"10.1145\/3691620.3695470","type":"proceedings-article","created":{"date-parts":[[2024,10,18]],"date-time":"2024-10-18T15:39:19Z","timestamp":1729265959000},"page":"870-882","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":13,"title":["JavaBench: A Benchmark of Object-Oriented Code Generation for Evaluating Large Language Models"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-4892-6294","authenticated-orcid":false,"given":"Jialun","family":"Cao","sequence":"first","affiliation":[{"name":"The Hong Kong University of Science and Technology, Hong Kong, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0005-2555-5728","authenticated-orcid":false,"given":"Zhiyong","family":"Chen","sequence":"additional","affiliation":[{"name":"Nanjing University, Nanjing, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-6126-303X","authenticated-orcid":false,"given":"Jiarong","family":"Wu","sequence":"additional","affiliation":[{"name":"The Hong Kong University of Science and Technology, Hong Kong, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-3508-7172","authenticated-orcid":false,"given":"Shing-Chi","family":"Cheung","sequence":"additional","affiliation":[{"name":"The Hong Kong University of Science and Technology, Hong Kong, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-6299-4704","authenticated-orcid":false,"given":"Chang","family":"Xu","sequence":"additional","affiliation":[{"name":"Nanjing University, Nanjing, China"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2024,10,27]]},"reference":[{"key":"e_1_3_2_1_1_1","unstructured":"2023. bigcode\/starcoderbase. https:\/\/huggingface.co\/bigcode\/starcoderbase."},{"key":"e_1_3_2_1_2_1","unstructured":"2023. CoderEval\/CoderEval. https:\/\/github.com\/CoderEval\/CoderEval\/commits\/main\/CoderEval4Python.json."},{"key":"e_1_3_2_1_3_1","unstructured":"2023. Deepseek-33b. https:\/\/huggingface.co\/deepseek-ai\/deepseek-coder-33b-instruct."},{"key":"e_1_3_2_1_4_1","unstructured":"2023. Deepseek-6.7b. https:\/\/huggingface.co\/deepseek-ai\/deepseek-coder-6.7b-instruct."},{"key":"e_1_3_2_1_5_1","unstructured":"2023. GPT-3.5-turbo Model Availability. https:\/\/learn.microsoft.com\/en-us\/azure\/ai-services\/openai\/concepts\/models#gpt-35-turbo-model-availability."},{"key":"e_1_3_2_1_6_1","unstructured":"2023. GPT-4 and GPT-4-turbo Preview. https:\/\/learn.microsoft.com\/en-us\/azure\/ai-services\/openai\/concepts\/models#gpt-4-and-gpt-4-turbo-preview."},{"key":"e_1_3_2_1_7_1","unstructured":"2023. Phind\/Phind-CodeLlama-34B-v2. https:\/\/huggingface.co\/Phind\/Phind-CodeLlama-34B-v2\/commit\/29c3be6006297754f344ba05678c038b0b77f6c0."},{"key":"e_1_3_2_1_8_1","unstructured":"2023. Phind\/Phind-CodeLlama-34B-v2. https:\/\/huggingface.co\/Phind\/Phind-CodeLlama-34B-v2."},{"key":"e_1_3_2_1_9_1","unstructured":"2023. WizardLM\/WizardCoder. https:\/\/huggingface.co\/WizardLM\/WizardCoder-15B-V1.0."},{"key":"e_1_3_2_1_10_1","unstructured":"2024. GitHub 2.0 Pull Requests Ranking over Programming Languages. https:\/\/madnight.github.io\/githut\/#\/pull{_}requests\/2024\/1."},{"key":"e_1_3_2_1_11_1","volume-title":"Dagstuhl Seminar Proceedings. Schloss Dagstuhl-Leibniz-Zentrum f\u00fcr Informatik.","author":"Alshahwan Nadia","year":"2010","unstructured":"Nadia Alshahwan, Yue Jia, Kiran Lakhotia, Gordon Fraser, David Shuler, and Paolo Tonella. 2010. AUTOMOCK: automated synthesis of a mock environment for test case generation. In Dagstuhl Seminar Proceedings. Schloss Dagstuhl-Leibniz-Zentrum f\u00fcr Informatik."},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/N19-1245"},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1145\/2642937.2642986"},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICST.2017.19"},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","unstructured":"Ben Athiwaratkun Sanjay Krishna Gouda Zijian Wang Xiaopeng Li Yuchen Tian Ming Tan Wasi Uddin Ahmad Shiqi Wang Qing Sun Mingyue Shang Sujan Kumar Gonugondla Hantian Ding Varun Kumar Nathan Fulton Arash Farahani Siddhartha Jain Robert Giaquinto Haifeng Qian Murali Krishna Ramanathan Ramesh Nallapati Baishakhi Ray Parminder Bhatia Sudipta Sengupta Dan Roth and Bing Xiang. 2022. Multi-lingual Evaluation of Code Generation Models. (2022). 10.48550\/ARXIV.2210.14868","DOI":"10.48550\/ARXIV.2210.14868"},{"key":"e_1_3_2_1_16_1","unstructured":"Jacob Austin Augustus Odena Maxwell Nye Maarten Bosma Henryk Michalewski David Dohan Ellen Jiang Carrie Cai Michael Terry Quoc Le et al. 2021. Program Synthesis with Large Language Models. arXiv preprint arXiv:2108.07732 (2021)."},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-46002-9_23"},{"key":"e_1_3_2_1_18_1","volume-title":"Beyond the Imitation Game: Quantifying and extrapolating the capabilities of language models. Transactions on Machine Learning Research","author":"BIG","year":"2023","unstructured":"BIG bench authors. 2023. Beyond the Imitation Game: Quantifying and extrapolating the capabilities of language models. Transactions on Machine Learning Research (2023). https:\/\/openreview.net\/forum?id=uyTL5Bvosj"},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","DOI":"10.1037\/h0043308"},{"key":"e_1_3_2_1_20_1","volume-title":"Concerned with Data Contamination? Assessing Countermeasures in Code Language Model. arXiv preprint arXiv:2403.16898","author":"Cao Jialun","year":"2024","unstructured":"Jialun Cao, Wuqi Zhang, and Shing-Chi Cheung. 2024. Concerned with Data Contamination? Assessing Countermeasures in Code Language Model. arXiv preprint arXiv:2403.16898 (2024)."},{"key":"e_1_3_2_1_21_1","volume-title":"Molly Q Feldman, et al.","author":"Cassano Federico","year":"2022","unstructured":"Federico Cassano, John Gouwar, Daniel Nguyen, Sydney Nguyen, Luna Phipps-Costin, Donald Pinckney, Ming-Ho Yee, Yangtian Zi, Carolyn Jane Anderson, Molly Q Feldman, et al. 2022. A scalable and extensible approach to benchmarking nl2code for 18 programming languages. arXiv preprint arXiv:2208.08227 (2022)."},{"key":"e_1_3_2_1_22_1","volume-title":"Training and evaluating a jupyter notebook data science assistant. arXiv preprint arXiv:2201.12901","author":"Chandel Shubham","year":"2022","unstructured":"Shubham Chandel, Colin B Clement, Guillermo Serrato, and Neel Sundaresan. 2022. Training and evaluating a jupyter notebook data science assistant. arXiv preprint arXiv:2201.12901 (2022)."},{"key":"e_1_3_2_1_23_1","unstructured":"Mark Chen Jerry Tworek Heewoo Jun Qiming Yuan Henrique Ponde de Oliveira Pinto Jared Kaplan Harri Edwards Yuri Burda Nicholas Joseph Greg Brockman Alex Ray Raul Puri Gretchen Krueger Michael Petrov Heidy Khlaaf Girish Sastry Pamela Mishkin Brooke Chan Scott Gray Nick Ryder Mikhail Pavlov Alethea Power Lukasz Kaiser Mohammad Bavarian Clemens Winter Philippe Tillet Felipe Petroski Such Dave Cummings Matthias Plappert Fotios Chantzis Elizabeth Barnes Ariel Herbert-Voss William Hebgen Guss Alex Nichol Alex Paino Nikolas Tezak Jie Tang Igor Babuschkin Suchir Balaji Shantanu Jain William Saunders Christopher Hesse Andrew N. Carr Jan Leike Josh Achiam Vedant Misra Evan Morikawa Alec Radford Matthew Knight Miles Brundage Mira Murati Katie Mayer Peter Welinder Bob McGrew Dario Amodei Sam McCandlish Ilya Sutskever and Wojciech Zaremba. 2021. Evaluating Large Language Models Trained on Code. (2021). arXiv:cs.LG\/2107.03374"},{"key":"e_1_3_2_1_24_1","unstructured":"Wei Cheng Yuhan Wu and Wei Hu. 2024. Dataflow-Guided Retrieval Augmentation for Repository-Level Code Completion. http:\/\/arxiv.org\/abs\/2405.19782 arXiv:2405.19782 [cs]."},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.1109\/2.303623"},{"key":"e_1_3_2_1_26_1","volume-title":"DeepSeek LLM: Scaling Open-Source Language Models with Longtermism. arXiv preprint arXiv:2401.02954","author":"AI.","year":"2024","unstructured":"DeepSeek-AI. 2024. DeepSeek LLM: Scaling Open-Source Language Models with Longtermism. arXiv preprint arXiv:2401.02954 (2024). https:\/\/github.com\/deepseek-ai\/DeepSeek-LLM"},{"key":"e_1_3_2_1_27_1","volume-title":"Hantian Ding, Ming Tan, Nihal Jain, Murali Krishna Ramanathan, Ramesh Nallapati, Parminder Bhatia, Dan Roth, and Bing Xiang.","author":"Ding Yangruibo","year":"2023","unstructured":"Yangruibo Ding, Zijian Wang, Wasi Uddin Ahmad, Hantian Ding, Ming Tan, Nihal Jain, Murali Krishna Ramanathan, Ramesh Nallapati, Parminder Bhatia, Dan Roth, and Bing Xiang. 2023. CrossCodeEval: A Diverse and Multilingual Benchmark for Cross-File Code Completion. In Thirty-seventh Conference on Neural Information Processing Systems Datasets and Benchmarks Track. https:\/\/arxiv.org\/pdf\/2310.11248.pdf"},{"key":"e_1_3_2_1_28_1","unstructured":"Xueying Du Mingwei Liu Kaixin Wang Hanlin Wang Junwei Liu Yixuan Chen Jiayi Feng Chaofeng Sha Xin Peng and Yiling Lou. 2023. ClassEval: A Manually-Crafted Benchmark for Evaluating LLMs on Class-level Code Generation. arXiv:cs.CL\/2308.01861"},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.1145\/3009837.3009851"},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.findings-emnlp.139"},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"publisher","DOI":"10.1145\/1808266.1808273"},{"key":"e_1_3_2_1_32_1","first-page":"1","article-title":"Code structure-guided transformer for source code summarization","volume":"32","author":"Gao Shuzheng","year":"2023","unstructured":"Shuzheng Gao, Cuiyun Gao, Yulan He, Jichuan Zeng, Lunyiu Nie, Xin Xia, and Michael Lyu. 2023. Code structure-guided transformer for source code summarization. ACM Transactions on Software Engineering and Methodology 32, 1 (2023), 1--32.","journal-title":"ACM Transactions on Software Engineering and Methodology"},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"publisher","DOI":"10.48550\/ARXIV.2308.08493"},{"key":"e_1_3_2_1_34_1","volume-title":"Cruxeval: A benchmark for code reasoning, understanding and execution. arXiv preprint arXiv:2401.03065","author":"Gu Alex","year":"2024","unstructured":"Alex Gu, Baptiste Rozi\u00e8re, Hugh Leather, Armando Solar-Lezama, Gabriel Synnaeve, and Sida I Wang. 2024. Cruxeval: A benchmark for code reasoning, understanding and execution. arXiv preprint arXiv:2401.03065 (2024)."},{"key":"e_1_3_2_1_35_1","doi-asserted-by":"publisher","DOI":"10.1145\/1926385.1926423"},{"key":"e_1_3_2_1_36_1","volume-title":"Measuring Coding Challenge Competence With APPS. NeurIPS","author":"Hendrycks Dan","year":"2021","unstructured":"Dan Hendrycks, Steven Basart, Saurav Kadavath, Mantas Mazeika, Akul Arora, Ethan Guo, Collin Burns, Samir Puranik, Horace He, Dawn Song, and Jacob Steinhardt. 2021. Measuring Coding Challenge Competence With APPS. NeurIPS (2021)."},{"key":"e_1_3_2_1_37_1","volume-title":"The Curious Case of Neural Text Degeneration. In 8th International Conference on Learning Representations, ICLR 2020","author":"Holtzman Ari","year":"2020","unstructured":"Ari Holtzman, Jan Buys, Li Du, Maxwell Forbes, and Yejin Choi. 2020. The Curious Case of Neural Text Degeneration. In 8th International Conference on Learning Representations, ICLR 2020, Addis Ababa, Ethiopia, April 26--30, 2020. OpenReview.net. https:\/\/openreview.net\/forum?id=rygGQyrFvH"},{"key":"e_1_3_2_1_38_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D18-1192"},{"key":"e_1_3_2_1_39_1","volume-title":"Proceedings of the 40th International Conference on Machine Learning (Proceedings of Machine Learning Research)","author":"Lai Yuhang","year":"1831","unstructured":"Yuhang Lai, Chengxi Li, Yiming Wang, Tianyi Zhang, Ruiqi Zhong, Luke Zettle-moyer, Wen-Tau Yih, Daniel Fried, Sida Wang, and Tao Yu. 2023. DS-1000: A Natural and Reliable Benchmark for Data Science Code Generation. In Proceedings of the 40th International Conference on Machine Learning (Proceedings of Machine Learning Research), Andreas Krause, Emma Brunskill, Kyunghyun Cho, Barbara Engelhardt, Sivan Sabato, and Jonathan Scarlett (Eds.), Vol. 202. PMLR, 18319--18345."},{"key":"e_1_3_2_1_40_1","doi-asserted-by":"crossref","unstructured":"Jia Li Ge Li Yunfei Zhao Yongmin Li Huanyu Liu Hao Zhu Lecheng Wang Kaibo Liu Zheng Fang Lanshen Wang Jiazheng Ding Xuanming Zhang Yuqi Zhu Yihong Dong Zhi Jin Binhua Li Fei Huang and Yongbin Li. 2024. DevEval: A Manually-Annotated Code Generation Benchmark Aligned with Real-World Code Repositories. http:\/\/arxiv.org\/abs\/2405.19856 arXiv:2405.19856 [cs].","DOI":"10.18653\/v1\/2024.findings-acl.214"},{"key":"e_1_3_2_1_41_1","unstructured":"Raymond Li Loubna Ben Allal Yangtian Zi Niklas Muennighoff Denis Kocetkov Chenghao Mou Marc Marone Christopher Akiki Jia Li Jenny Chim Qian Liu Evgenii Zheltonozhskii Terry Yue Zhuo Thomas Wang Olivier Dehaene Mishig Davaadorj Joel Lamy-Poirier Jo\u00e3o Monteiro Oleh Shliazhko Nicolas Gontier Nicholas Meade Armel Zebaze Ming-Ho Yee Logesh Kumar Umapathi Jian Zhu Benjamin Lipkin Muhtasham Oblokulov Zhiruo Wang Rudra Murthy Jason Stillerman Siva Sankalp Patel Dmitry Abulkhanov Marco Zocca Manan Dey Zhihan Zhang Nour Fahmy Urvashi Bhattacharyya Wenhao Yu Swayam Singh Sasha Luccioni Paulo Villegas Maxim Kunakov Fedor Zhdanov Manuel Romero Tony Lee Nadav Timor Jennifer Ding Claire Schlesinger Hailey Schoelkopf Jan Ebert Tri Dao Mayank Mishra Alex Gu Jennifer Robinson Carolyn Jane Anderson Brendan Dolan-Gavitt Danish Contractor Siva Reddy Daniel Fried Dzmitry Bahdanau Yacine Jernite Carlos Mu\u00f1oz Ferrandis Sean Hughes Thomas Wolf Arjun Guha Leandro von Werra and Harm de Vries. 2023. StarCoder: may the source be with you! (2023). arXiv:cs.CL\/2305.06161"},{"key":"e_1_3_2_1_42_1","volume-title":"Rigorous Evaluation of Large Language Models for Code Generation. In Thirty-seventh Conference on Neural Information Processing Systems. https:\/\/openreview.net\/forum?id=1qvx610Cu7","author":"Liu Jiawei","year":"2023","unstructured":"Jiawei Liu, Chunqiu Steven Xia, Yuyao Wang, and Lingming Zhang. 2023. Is Your Code Generated by ChatGPT Really Correct? Rigorous Evaluation of Large Language Models for Code Generation. In Thirty-seventh Conference on Neural Information Processing Systems. https:\/\/openreview.net\/forum?id=1qvx610Cu7"},{"key":"e_1_3_2_1_43_1","doi-asserted-by":"publisher","DOI":"10.1162\/tacl_a_00638"},{"key":"e_1_3_2_1_44_1","doi-asserted-by":"publisher","DOI":"10.1145\/3560815"},{"key":"e_1_3_2_1_45_1","volume-title":"Wizardcoder: Empowering code large language models with evol-instruct. arXiv preprint arXiv:2306.08568","author":"Luo Ziyang","year":"2023","unstructured":"Ziyang Luo, Can Xu, Pu Zhao, Qingfeng Sun, Xiubo Geng, Wenxiang Hu, Chongyang Tao, Jing Ma, Qingwei Lin, and Daxin Jiang. 2023. Wizardcoder: Empowering code large language models with evol-instruct. arXiv preprint arXiv:2306.08568 (2023)."},{"key":"e_1_3_2_1_46_1","volume-title":"Codegen: An open large language model for code with multi-turn program synthesis. arXiv preprint arXiv:2203.13474","author":"Nijkamp Erik","year":"2022","unstructured":"Erik Nijkamp, Bo Pang, Hiroaki Hayashi, Lifu Tu, Huan Wang, Yingbo Zhou, Silvio Savarese, and Caiming Xiong. 2022. Codegen: An open large language model for code with multi-turn program synthesis. arXiv preprint arXiv:2203.13474 (2022)."},{"key":"e_1_3_2_1_47_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSM.1992.242525"},{"key":"e_1_3_2_1_48_1","volume-title":"LLM is Like a Box of Chocolates: the Non-determinism of ChatGPT in Code Generation. arXiv preprint arXiv:2308.02828","author":"Ouyang Shuyin","year":"2023","unstructured":"Shuyin Ouyang, Jie M Zhang, Mark Harman, and Meng Wang. 2023. LLM is Like a Box of Chocolates: the Non-determinism of ChatGPT in Code Generation. arXiv preprint arXiv:2308.02828 (2023)."},{"key":"e_1_3_2_1_49_1","doi-asserted-by":"publisher","DOI":"10.3115\/1073083.1073135"},{"key":"e_1_3_2_1_50_1","volume-title":"Yossi Adi, Jingyu Liu, Tal Remez, J\u00e9r\u00e9my Rapin, et al.","author":"Roziere Baptiste","year":"2023","unstructured":"Baptiste Roziere, Jonas Gehring, Fabian Gloeckle, Sten Sootla, Itai Gat, Xiaoqing Ellen Tan, Yossi Adi, Jingyu Liu, Tal Remez, J\u00e9r\u00e9my Rapin, et al. 2023. Code llama: Open foundation models for code. arXiv preprint arXiv:2308.12950 (2023)."},{"key":"e_1_3_2_1_51_1","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2023.3334955"},{"key":"e_1_3_2_1_52_1","doi-asserted-by":"publisher","DOI":"10.1145\/3290386"},{"key":"e_1_3_2_1_53_1","volume-title":"Repository-Level Prompt Generation for Large Language Models of Code. In International Conference on Machine Learning, ICML 2023","volume":"202","author":"Shrivastava Disha","year":"2023","unstructured":"Disha Shrivastava, Hugo Larochelle, and Daniel Tarlow. 2023. Repository-Level Prompt Generation for Large Language Models of Code. In International Conference on Machine Learning, ICML 2023, 23--29 July 2023, Honolulu, Hawaii, USA (Proceedings of Machine Learning Research), Andreas Krause, Emma Brunskill, Kyunghyun Cho, Barbara Engelhardt, Sivan Sabato, and Jonathan Scarlett (Eds.), Vol. 202. PMLR, 31693--31715. https:\/\/proceedings.mlr.press\/v202\/shrivastava23a.html"},{"key":"e_1_3_2_1_54_1","volume-title":"OOP: Object-Oriented Programming Evaluation Benchmark for Large Language Models. arXiv preprint arXiv:2401.06628","author":"Wang Shuai","year":"2024","unstructured":"Shuai Wang, Liang Ding, Li Shen, Yong Luo, Bo Du, and Dacheng Tao. 2024. OOP: Object-Oriented Programming Evaluation Benchmark for Large Language Models. arXiv preprint arXiv:2401.06628 (2024)."},{"key":"e_1_3_2_1_55_1","doi-asserted-by":"publisher","DOI":"10.18653\/V1\/2021.EMNLP-MAIN.685"},{"key":"e_1_3_2_1_56_1","volume-title":"Execution-based evaluation for open-domain code generation. arXiv preprint arXiv:2212.10481","author":"Wang Zhiruo","year":"2022","unstructured":"Zhiruo Wang, Shuyan Zhou, Daniel Fried, and Graham Neubig. 2022. Execution-based evaluation for open-domain code generation. arXiv preprint arXiv:2212.10481 (2022)."},{"key":"e_1_3_2_1_57_1","doi-asserted-by":"publisher","DOI":"10.1145\/3196398.3196408"},{"key":"e_1_3_2_1_58_1","volume-title":"CoderEval: A Benchmark of Pragmatic Code Generation with Generative Pre-trained Models. arXiv preprint arXiv:2302.00288","author":"Yu Hao","year":"2023","unstructured":"Hao Yu, Bo Shen, Dezhi Ran, Jiaxin Zhang, Qi Zhang, Yuchi Ma, Guangtai Liang, Ying Li, Tao Xie, and Qianxiang Wang. 2023. CoderEval: A Benchmark of Pragmatic Code Generation with Generative Pre-trained Models. arXiv preprint arXiv:2302.00288 (2023)."},{"key":"e_1_3_2_1_59_1","volume-title":"When language model meets private library. arXiv preprint arXiv:2210.17236","author":"Zan Daoguang","year":"2022","unstructured":"Daoguang Zan, Bei Chen, Zeqi Lin, Bei Guan, Yongji Wang, and Jian-Guang Lou. 2022. When language model meets private library. arXiv preprint arXiv:2210.17236 (2022)."},{"key":"e_1_3_2_1_60_1","volume-title":"CERT: continual pre-training on sketches for library-oriented code generation. arXiv preprint arXiv:2206.06888","author":"Zan Daoguang","year":"2022","unstructured":"Daoguang Zan, Bei Chen, Dejian Yang, Zeqi Lin, Minsu Kim, Bei Guan, Yongji Wang, Weizhu Chen, and Jian-Guang Lou. 2022. CERT: continual pre-training on sketches for library-oriented code generation. arXiv preprint arXiv:2206.06888 (2022)."},{"key":"e_1_3_2_1_61_1","volume-title":"Large language models meet nl2code: A survey. arXiv preprint arXiv:2212.09420","author":"Zan Daoguang","year":"2022","unstructured":"Daoguang Zan, Bei Chen, Fengji Zhang, Dianjie Lu, Bingchao Wu, Bei Guan, Yongji Wang, and Jian-Guang Lou. 2022. Large language models meet nl2code: A survey. arXiv preprint arXiv:2212.09420 (2022)."},{"key":"e_1_3_2_1_62_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2023.emnlp-main.151"},{"key":"e_1_3_2_1_63_1","doi-asserted-by":"publisher","DOI":"10.1145\/3580305.3599790"},{"key":"e_1_3_2_1_64_1","volume-title":"A survey of large language models for code: Evolution, benchmarking, and future trends. arXiv preprint arXiv:2311.10372","author":"Zheng Zibin","year":"2023","unstructured":"Zibin Zheng, Kaiwen Ning, Yanlin Wang, Jingwen Zhang, Dewu Zheng, Mingxi Ye, and Jiachi Chen. 2023. A survey of large language models for code: Evolution, benchmarking, and future trends. arXiv preprint arXiv:2311.10372 (2023)."},{"key":"e_1_3_2_1_65_1","doi-asserted-by":"publisher","DOI":"10.1145\/3324884.3416539"}],"event":{"name":"ASE '24: 39th IEEE\/ACM International Conference on Automated Software Engineering","location":"Sacramento CA USA","acronym":"ASE '24","sponsor":["SIGAI ACM Special Interest Group on Artificial Intelligence","SIGSOFT ACM Special Interest Group on Software Engineering","IEEE CS"]},"container-title":["Proceedings of the 39th IEEE\/ACM International Conference on Automated Software Engineering"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3691620.3695470","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3691620.3695470","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,19]],"date-time":"2025-06-19T00:06:19Z","timestamp":1750291579000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3691620.3695470"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,10,27]]},"references-count":65,"alternative-id":["10.1145\/3691620.3695470","10.1145\/3691620"],"URL":"https:\/\/doi.org\/10.1145\/3691620.3695470","relation":{},"subject":[],"published":{"date-parts":[[2024,10,27]]},"assertion":[{"value":"2024-10-27","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}