{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,20]],"date-time":"2026-07-20T14:14:49Z","timestamp":1784556889532,"version":"3.55.0"},"publisher-location":"New York, NY, USA","reference-count":75,"publisher":"ACM","license":[{"start":{"date-parts":[[2024,4,12]],"date-time":"2024-04-12T00:00:00Z","timestamp":1712880000000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"name":"National Key R&D Program of China","award":["2021ZD0112903"],"award-info":[{"award-number":["2021ZD0112903"]}]},{"name":"the National Natural Science Foundation of China","award":["No. 62302099"],"award-info":[{"award-number":["No. 62302099"]}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2024,4,12]]},"DOI":"10.1145\/3597503.3639219","type":"proceedings-article","created":{"date-parts":[[2024,4,12]],"date-time":"2024-04-12T16:43:26Z","timestamp":1712940206000},"page":"1-13","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":145,"title":["Evaluating Large Language Models in Class-Level Code Generation"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0009-0005-0004-9183","authenticated-orcid":false,"given":"Xueying","family":"Du","sequence":"first","affiliation":[{"name":"Fudan University, Shanghai, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-3462-997X","authenticated-orcid":false,"given":"Mingwei","family":"Liu","sequence":"additional","affiliation":[{"name":"Fudan University, Shanghai, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0002-6414-2174","authenticated-orcid":false,"given":"Kaixin","family":"Wang","sequence":"additional","affiliation":[{"name":"Fudan University, Shanghai, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0008-2374-1246","authenticated-orcid":false,"given":"Hanlin","family":"Wang","sequence":"additional","affiliation":[{"name":"Fudan University, Shanghai, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0005-7299-3851","authenticated-orcid":false,"given":"Junwei","family":"Liu","sequence":"additional","affiliation":[{"name":"Fudan University, Shanghai, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0006-8804-1828","authenticated-orcid":false,"given":"Yixuan","family":"Chen","sequence":"additional","affiliation":[{"name":"Fudan University, Shanghai, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0000-0258-030X","authenticated-orcid":false,"given":"Jiayi","family":"Feng","sequence":"additional","affiliation":[{"name":"Fudan University, Shanghai, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0004-4195-0122","authenticated-orcid":false,"given":"Chaofeng","family":"Sha","sequence":"additional","affiliation":[{"name":"Fudan University, Shanghai, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-3376-2581","authenticated-orcid":false,"given":"Xin","family":"Peng","sequence":"additional","affiliation":[{"name":"Fudan University, Shanghai, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-4066-3365","authenticated-orcid":false,"given":"Yiling","family":"Lou","sequence":"additional","affiliation":[{"name":"Fudan University, Shanghai, China"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2024,4,12]]},"reference":[{"key":"e_1_3_2_1_1_1","unstructured":"[n. d.]. ClassEval on GitHub. https:\/\/github.com\/FudanSELab\/ClassEval"},{"key":"e_1_3_2_1_2_1","unstructured":"[n. d.]. ClassEval on Hugging Face. https:\/\/huggingface.co\/datasets\/FudanSELab\/ClassEval"},{"key":"e_1_3_2_1_3_1","unstructured":"[n.d.]. Code generation datasets in Huggingface. https:\/\/hf.co\/datasets?other=code-generation"},{"key":"e_1_3_2_1_4_1","unstructured":"[n. d.]. Coverage Library. https:\/\/pypi.org\/project\/coverage"},{"key":"e_1_3_2_1_5_1","unstructured":"[n. d.]. mutmut. https:\/\/mutmut.readthedocs.io\/en\/latest\/"},{"key":"e_1_3_2_1_6_1","unstructured":"[n. d.]. OpenAI API interface. https:\/\/platform.openai.com\/docs\/api-reference"},{"key":"e_1_3_2_1_7_1","unstructured":"[n. d.]. PyPI. https:\/\/pypi.org\/search"},{"key":"e_1_3_2_1_8_1","unstructured":"[n.d.]. Unittest Framework. https:\/\/pypi.org\/project\/unitest"},{"key":"e_1_3_2_1_9_1","unstructured":"2021. Dense-6.7B. https:\/\/huggingface.co\/KoboldAI\/fairseq-dense-6.7B-Shinen"},{"key":"e_1_3_2_1_10_1","unstructured":"2023. Instruct-CodeGen. https:\/\/huggingface.co\/sahil2801\/instruct-codegen-16B"},{"key":"e_1_3_2_1_11_1","unstructured":"2023. Instruct-StarCoder. https:\/\/huggingface.co\/GeorgiaTechResearchInstitute\/starcoder-gpteacher-code-instruct"},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.1145\/3551349.3559555"},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.48550\/arXiv.2301.03988"},{"key":"e_1_3_2_1_14_1","volume-title":"Multilingual Evaluation of Code Generation Models. In The Eleventh International Conference on Learning Representations, ICLR 2023","author":"Athiwaratkun Ben","year":"2023","unstructured":"Ben Athiwaratkun, Sanjay Krishna Gouda, and Zijian Wang et al. 2023. Multilingual Evaluation of Code Generation Models. In The Eleventh International Conference on Learning Representations, ICLR 2023, Kigali, Rwanda, May 1--5, 2023. OpenReview.net. https:\/\/openreview.net\/pdf?id=Bo7eeXm6An8"},{"key":"e_1_3_2_1_15_1","volume-title":"Program Synthesis with Large Language Models. CoRR abs\/2108.07732","author":"Austin Jacob","year":"2021","unstructured":"Jacob Austin, Augustus Odena, Maxwell I. Nye, Maarten Bosma, Henryk Michalewski, David Dohan, Ellen Jiang, Carrie J. Cai, Michael Terry, Quoc V. Le, and Charles Sutton. 2021. Program Synthesis with Large Language Models. CoRR abs\/2108.07732 (2021). arXiv:2108.07732 https:\/\/arxiv.org\/abs\/2108.07732"},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.48550\/arXiv.2306.04181"},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.48550\/arXiv.2302.04023"},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1145\/1159733.1159787"},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","DOI":"10.48550\/arXiv.2303.16421"},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.48550\/arXiv.2307.03109"},{"key":"e_1_3_2_1_21_1","unstructured":"Mark Chen Jerry Tworek and Heewoo Jun et al. 2021. Evaluating Large Language Models Trained on Code. CoRR abs\/2107.03374 (2021). arXiv:2107.03374 https:\/\/arxiv.org\/abs\/2107.03374"},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.1109\/TSP.2015.2469645"},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.fertnstert.2023.05.151"},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"publisher","DOI":"10.48550\/arXiv.2207.11280"},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.48550\/arXiv.2307.02486"},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.48550\/arXiv.2312.10448"},{"key":"e_1_3_2_1_27_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2022.acl-long.26"},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"publisher","DOI":"10.48550\/arXiv.2307.02046"},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2023.acl-short.101"},{"key":"e_1_3_2_1_30_1","volume-title":"InCoder: A Generative Model for Code Infilling and Synthesis. In The Eleventh International Conference on Learning Representations, ICLR 2023","author":"Fried Daniel","year":"2023","unstructured":"Daniel Fried, Armen Aghajanyan, Jessy Lin, Sida Wang, Eric Wallace, Freda Shi, Ruiqi Zhong, Scott Yih, Luke Zettlemoyer, and Mike Lewis. 2023. InCoder: A Generative Model for Code Infilling and Synthesis. In The Eleventh International Conference on Learning Representations, ICLR 2023, Kigali, Rwanda, May 1--5, 2023. OpenReview.net."},{"key":"e_1_3_2_1_31_1","first-page":"12098","article-title":"Long-Coder","volume":"202","author":"Guo Daya","year":"2023","unstructured":"Daya Guo, Canwen Xu, Nan Duan, Jian Yin, and Julian J. McAuley. 2023. Long-Coder: A Long-Range Pre-trained Language Model for Code Completion. 202 (2023), 12098--12107. https:\/\/proceedings.mlr.press\/v202\/guo23j.html","journal-title":"A Long-Range Pre-trained Language Model for Code Completion."},{"key":"e_1_3_2_1_32_1","volume-title":"Proceedings of the Neural Information Processing Systems Track on Datasets and Benchmarks 1, NeurIPS Datasets and Benchmarks 2021","author":"Hendrycks Dan","year":"2021","unstructured":"Dan Hendrycks, Steven Basart, Saurav Kadavath, Mantas Mazeika, Akul Arora, Ethan Guo, Collin Burns, Samir Puranik, Horace He, Dawn Song, and Jacob Steinhardt. 2021. Measuring Coding Challenge Competence With APPS. In Proceedings of the Neural Information Processing Systems Track on Datasets and Benchmarks 1, NeurIPS Datasets and Benchmarks 2021, December 2021, virtual. https:\/\/datasets-benchmarks-proceedings.neurips.cc\/paper\/2021\/hash\/c24cd76e1ce41366a4bbe8a49b02a028-Abstract-round2.html"},{"key":"e_1_3_2_1_33_1","volume-title":"The Curious Case of Neural Text Degeneration. In 8th International Conference on Learning Representations, ICLR 2020","author":"Holtzman Ari","year":"2020","unstructured":"Ari Holtzman, Jan Buys, Li Du, Maxwell Forbes, and Yejin Choi. 2020. The Curious Case of Neural Text Degeneration. In 8th International Conference on Learning Representations, ICLR 2020, Addis Ababa, Ethiopia, April 26--30, 2020. OpenReview.net. https:\/\/openreview.net\/forum?id=rygGQyrFvH"},{"key":"e_1_3_2_1_34_1","doi-asserted-by":"publisher","DOI":"10.48550\/arXiv.2302.14045"},{"key":"e_1_3_2_1_35_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D18-1192"},{"key":"e_1_3_2_1_36_1","doi-asserted-by":"publisher","DOI":"10.48550\/arXiv.2302.05020"},{"key":"e_1_3_2_1_37_1","doi-asserted-by":"publisher","unstructured":"Matthew Jin Syed Shahriar Michele Tufano Xin Shi Shuai Lu Neel Sundaresan and Alexey Svyatkovskiy. 2023. InferFix: End-to-End Program Repair with LLMs. (2023) 1646--1656. 10.1145\/3611643.3613892","DOI":"10.1145\/3611643.3613892"},{"key":"e_1_3_2_1_38_1","doi-asserted-by":"publisher","DOI":"10.48550\/ARXIV.2304.02195"},{"key":"e_1_3_2_1_39_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE48619.2023.00194"},{"key":"e_1_3_2_1_40_1","doi-asserted-by":"publisher","DOI":"10.48550\/arXiv.2211.11501"},{"key":"e_1_3_2_1_41_1","doi-asserted-by":"publisher","DOI":"10.48550\/arXiv.2305.06599"},{"key":"e_1_3_2_1_42_1","doi-asserted-by":"publisher","DOI":"10.48550\/arXiv.2305.06161"},{"key":"e_1_3_2_1_43_1","doi-asserted-by":"publisher","unstructured":"Yujia Li David H. Choi Junyoung Chung Nate Kushman Julian Schrittwieser Remi Leblond Tom Eccles James Keeling Felix Gimeno Agustin Dal Lago Thomas Hubert Peter Choy Cyprien de Masson d'Autume Igor Babuschkin Xinyun Chen Po-Sen Huang Johannes Welbl Sven Gowal Alexey Cherepanov James Molloy Daniel J. Mankowitz Esme Sutherland Robson Pushmeet Kohli Nando de Freitas Koray Kavukcuoglu and Oriol Vinyals. 2022. Competition-Level Code Generation with AlphaCode. CoRR abs\/2203.07814 (2022). arXiv:2203.07814 10.48550\/arXiv.2203.07814","DOI":"10.48550\/arXiv.2203.07814"},{"key":"e_1_3_2_1_44_1","doi-asserted-by":"publisher","DOI":"10.48550\/arXiv.2305.01210"},{"key":"e_1_3_2_1_45_1","doi-asserted-by":"publisher","DOI":"10.1145\/3468264.3468618"},{"key":"e_1_3_2_1_46_1","doi-asserted-by":"publisher","DOI":"10.1109\/ASE56229.2023.00159"},{"key":"e_1_3_2_1_47_1","doi-asserted-by":"publisher","DOI":"10.48550\/arXiv.2307.03172"},{"key":"e_1_3_2_1_48_1","doi-asserted-by":"publisher","DOI":"10.48550\/arXiv.2305.08291"},{"key":"e_1_3_2_1_49_1","doi-asserted-by":"publisher","DOI":"10.48550\/arXiv.2306.08568"},{"key":"e_1_3_2_1_50_1","doi-asserted-by":"publisher","DOI":"10.1109\/2.161279"},{"key":"e_1_3_2_1_51_1","doi-asserted-by":"publisher","DOI":"10.1016\/J.AMC.2023.128461"},{"key":"e_1_3_2_1_52_1","doi-asserted-by":"publisher","DOI":"10.48550\/arXiv.2203.13474"},{"key":"e_1_3_2_1_54_1","volume-title":"Synchromesh: Reliable code generation from pre-trained language models.","author":"Poesia Gabriel","year":"2022","unstructured":"Gabriel Poesia, Oleksandr Polozov, Vu Le, Ashish Tiwari, Gustavo Soares, Christopher Meek, and Sumit Gulwani. 2022. Synchromesh: Reliable code generation from pre-trained language models. (2022). arXiv:2201.11227 [cs.LG]"},{"key":"e_1_3_2_1_55_1","unstructured":"Alec Radford Jeffrey Wu Rewon Child David Luan Dario Amodei Ilya Sutskever et al. 2019. Language models are unsupervised multitask learners. OpenAI blog 1 8 (2019) 9."},{"key":"e_1_3_2_1_56_1","doi-asserted-by":"publisher","DOI":"10.48550\/ARXIV.2307.14936"},{"key":"e_1_3_2_1_57_1","first-page":"354","article-title":"Python-the fastest growing programming language","volume":"4","author":"Srinath KR","year":"2017","unstructured":"KR Srinath. 2017. Python-the fastest growing programming language. International Research Journal of Engineering and Technology 4, 12 (2017), 354--357.","journal-title":"International Research Journal of Engineering and Technology"},{"key":"e_1_3_2_1_58_1","doi-asserted-by":"publisher","DOI":"10.48550\/ARXIV.2308.03314"},{"key":"e_1_3_2_1_59_1","doi-asserted-by":"publisher","unstructured":"Hugo Touvron Thibaut Lavril and Gautier Izacard et al. 2023. LLaMA: Open and Efficient Foundation Language Models. CoRR abs\/2302.13971 (2023). arXiv:2302.13971 10.48550\/arXiv.2302.13971","DOI":"10.48550\/arXiv.2302.13971"},{"key":"e_1_3_2_1_60_1","doi-asserted-by":"publisher","DOI":"10.48550\/ARXIV.2307.04346"},{"key":"e_1_3_2_1_61_1","doi-asserted-by":"publisher","DOI":"10.48550\/ARXIV.2311.04448"},{"key":"e_1_3_2_1_62_1","unstructured":"Chong Wang Jian Zhang Yebo Feng Tianlin Li Weisong Sun Yang Liu and Xin Peng. 2024. Teaching Code LLMs to Use Autocompletion Tools in Repository-Level Code Generation. arXiv:2401.06391 [cs.SE]"},{"key":"e_1_3_2_1_63_1","doi-asserted-by":"publisher","DOI":"10.48550\/ARXIV.2301.13246"},{"key":"e_1_3_2_1_64_1","doi-asserted-by":"publisher","DOI":"10.1145\/3368089.3409731"},{"key":"e_1_3_2_1_65_1","doi-asserted-by":"publisher","DOI":"10.1145\/3520312.3534862"},{"key":"e_1_3_2_1_66_1","doi-asserted-by":"publisher","DOI":"10.1145\/3196398.3196408"},{"key":"e_1_3_2_1_67_1","doi-asserted-by":"publisher","DOI":"10.48550\/arXiv.2302.00288"},{"key":"e_1_3_2_1_68_1","doi-asserted-by":"publisher","DOI":"10.48550\/ARXIV.2309.13574"},{"key":"e_1_3_2_1_69_1","doi-asserted-by":"publisher","DOI":"10.48550\/ARXIV.2305.04207"},{"key":"e_1_3_2_1_70_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2023.acl-long.411"},{"key":"e_1_3_2_1_71_1","volume-title":"The Eleventh International Conference on Learning Representations, ICLR 2023","author":"Zeng Aohan","year":"2023","unstructured":"Aohan Zeng, Xiao Liu, and Zhengxiao Du et al. 2023. GLM-130B: An Open Bilingual Pre-trained Model. In The Eleventh International Conference on Learning Representations, ICLR 2023, Kigali, Rwanda, May 1--5, 2023. OpenReview.net. https:\/\/openreview.net\/pdf?id=-Aw0rrrPUF"},{"key":"e_1_3_2_1_72_1","doi-asserted-by":"publisher","DOI":"10.1145\/3533767.3534390"},{"key":"e_1_3_2_1_73_1","doi-asserted-by":"publisher","DOI":"10.48550\/ARXIV.2308.12697"},{"key":"e_1_3_2_1_74_1","doi-asserted-by":"crossref","unstructured":"Fengji Zhang Bei Chen Yue Zhang Jacky Keung Jin Liu Daoguang Zan Yi Mao Jian-Guang Lou and Weizhu Chen. 2023. RepoCoder: Repository-Level Code Completion Through Iterative Retrieval and Generation. (2023) 2471--2484. https:\/\/aclanthology.org\/2023.emnlp-main.151","DOI":"10.18653\/v1\/2023.emnlp-main.151"},{"key":"e_1_3_2_1_75_1","doi-asserted-by":"publisher","DOI":"10.48550\/arXiv.2306.05685"},{"key":"e_1_3_2_1_76_1","doi-asserted-by":"publisher","DOI":"10.48550\/arXiv.2303.17568"}],"event":{"name":"ICSE '24: IEEE\/ACM 46th International Conference on Software Engineering","location":"Lisbon Portugal","acronym":"ICSE '24","sponsor":["SIGSOFT ACM Special Interest Group on Software Engineering","IEEE CS","Faculty of Engineering of University of Porto"]},"container-title":["Proceedings of the IEEE\/ACM 46th International Conference on Software Engineering"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3597503.3639219","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3597503.3639219","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,18]],"date-time":"2025-06-18T22:49:13Z","timestamp":1750286953000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3597503.3639219"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,4,12]]},"references-count":75,"alternative-id":["10.1145\/3597503.3639219","10.1145\/3597503"],"URL":"https:\/\/doi.org\/10.1145\/3597503.3639219","relation":{},"subject":[],"published":{"date-parts":[[2024,4,12]]},"assertion":[{"value":"2024-04-12","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}