{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,24]],"date-time":"2026-07-24T16:57:23Z","timestamp":1784912243435,"version":"3.55.0"},"publisher-location":"New York, NY, USA","reference-count":91,"publisher":"ACM","license":[{"start":{"date-parts":[[2024,4,12]],"date-time":"2024-04-12T00:00:00Z","timestamp":1712880000000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"name":"RGC GRF","award":["16214723"],"award-info":[{"award-number":["16214723"]}]},{"name":"RGC GRF","award":["RMGS24EG03"],"award-info":[{"award-number":["RMGS24EG03"]}]},{"name":"RGC GRF","award":["RMGS24CR01"],"award-info":[{"award-number":["RMGS24CR01"]}]},{"DOI":"10.13039\/501100003453","name":"Natural Science Foundation of Guangdong Province","doi-asserted-by":"publisher","award":["2023A1515011959"],"award-info":[{"award-number":["2023A1515011959"]}],"id":[{"id":"10.13039\/501100003453","id-type":"DOI","asserted-by":"publisher"}]},{"name":"Shenzhen Basic Research","award":["JCYJ20220531095214031"],"award-info":[{"award-number":["JCYJ20220531095214031"]}]},{"name":"National Research Foundation, Singapore, and the Cyber Security Agency under its National Cybersecurity R&D Programme","award":["NCRP25-P04-TAICeN"],"award-info":[{"award-number":["NCRP25-P04-TAICeN"]}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2024,4,12]]},"DOI":"10.1145\/3597503.3639091","type":"proceedings-article","created":{"date-parts":[[2024,4,12]],"date-time":"2024-04-12T16:43:26Z","timestamp":1712940206000},"page":"1-13","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":13,"title":["On Extracting Specialized Code Abilities from Large Language Models: A Feasibility Study"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-9897-4086","authenticated-orcid":false,"given":"Zongjie","family":"Li","sequence":"first","affiliation":[{"name":"Hong Kong University of Science and Technology, Hong Kong, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-3935-7328","authenticated-orcid":false,"given":"Chaozheng","family":"Wang","sequence":"additional","affiliation":[{"name":"The Chinese University of Hong Kong, Hong Kong, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-7680-2817","authenticated-orcid":false,"given":"Pingchuan","family":"Ma","sequence":"additional","affiliation":[{"name":"Hong Kong University of Science and Technology, Hong Kong, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0006-3232-0115","authenticated-orcid":false,"given":"Chaowei","family":"Liu","sequence":"additional","affiliation":[{"name":"National University of Singapore, Singapore, Singapore"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-0866-0308","authenticated-orcid":false,"given":"Shuai","family":"Wang","sequence":"additional","affiliation":[{"name":"Hong Kong University of Science and Technology, Hong Kong, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-3752-0718","authenticated-orcid":false,"given":"Daoyuan","family":"Wu","sequence":"additional","affiliation":[{"name":"Nanyang Technological University, Singapore, Singapore"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-4774-2434","authenticated-orcid":false,"given":"Cuiyun","family":"Gao","sequence":"additional","affiliation":[{"name":"Harbin Institute of Technology, Shenzhen, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-7300-9215","authenticated-orcid":false,"given":"Yang","family":"Liu","sequence":"additional","affiliation":[{"name":"Nanyang Technological University, Singapore, Singapore"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2024,4,12]]},"reference":[{"key":"e_1_3_2_1_1_1","unstructured":"[n.d.]. Artifacts. https:\/\/sites.google.com\/view\/saellcm\/main."},{"key":"e_1_3_2_1_2_1","unstructured":"[n.d.]. billing for AI21 LAB. https:\/\/studio.ai21.com\/pricing."},{"key":"e_1_3_2_1_3_1","unstructured":"[n. d.]. billing for GitHub Copilot. https:\/\/docs.github.com\/en\/billing\/managing-billing-for-github-copilot\/about-billing-for-github-copilot."},{"key":"e_1_3_2_1_4_1","unstructured":"[n.d.]. CCTest's codebase. https:\/\/sites.google.com\/view\/cctest-info\/main-page."},{"key":"e_1_3_2_1_5_1","unstructured":"[n.d.]. chatgpt. https:\/\/chat.openai.com\/chat."},{"key":"e_1_3_2_1_6_1","unstructured":"[n.d.]. claude2. https:\/\/www.anthropic.com\/index\/claude-2."},{"key":"e_1_3_2_1_7_1","unstructured":"[n.d.]. Copilot. https:\/\/github.com\/features\/copilot."},{"key":"e_1_3_2_1_8_1","unstructured":"[n. d.]. davinci. https:\/\/platform.openai.com\/docs\/models\/gpt-3-5."},{"key":"e_1_3_2_1_9_1","unstructured":"[n.d.]. Google Bard steal. https:\/\/www.theinformation.com\/articles\/alphabets-google-and-deepmind-pause-grudges-join-forces-to-chase-openai."},{"key":"e_1_3_2_1_10_1","unstructured":"[n. d.]. gpt4. https:\/\/cdn.openai.com\/papers\/gpt-4-system-card.pdf."},{"key":"e_1_3_2_1_11_1","unstructured":"[n.d.]. leetcode. https:\/\/github.com\/qiyuangong\/leetcode."},{"key":"e_1_3_2_1_12_1","unstructured":"[n.d.]. Openai. https:\/\/openai.com\/product."},{"key":"e_1_3_2_1_13_1","unstructured":"[n. d.]. OpenAI Watermarking prototype. https:\/\/scottaaronson.blog\/?p=6823."},{"key":"e_1_3_2_1_14_1","unstructured":"[n. d.]. treesitter. https:\/\/tree-sitter.github.io\/tree-sitter\/."},{"key":"e_1_3_2_1_15_1","volume-title":"Fixing Hardware Security Bugs with Large Language Models. arXiv preprint arXiv:2302.01215","author":"Ahmad Baleegh","year":"2023","unstructured":"Baleegh Ahmad, Shailja Thakur, Benjamin Tan, Ramesh Karri, and Hammond Pearce. 2023. Fixing Hardware Security Bugs with Large Language Models. arXiv preprint arXiv:2302.01215 (2023)."},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2021.naacl-main.211"},{"key":"e_1_3_2_1_17_1","unstructured":"Yuvanesh Anand Zach Nussbaum Brandon Duderstadt Benjamin Schmidt and Andriy Mulyar. 2023. GPT4All: Training an Assistant-style Chatbot with Large Scale Data Distillation from GPT-3.5-Turbo. https:\/\/github.com\/nomic-ai\/gpt4all."},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE.2013.6606617"},{"key":"e_1_3_2_1_19_1","volume-title":"5th International Conference on Learning Representations, ICLR 2017, Toulon, France, April 24--26, 2017, Conference Track Proceedings. OpenReview.net.","author":"Balog Matej","year":"2017","unstructured":"Matej Balog, Alexander L. Gaunt, Marc Brockschmidt, Sebastian Nowozin, and Daniel Tarlow. 2017. DeepCoder: Learning to Write Programs. In 5th International Conference on Learning Representations, ICLR 2017, Toulon, France, April 24--26, 2017, Conference Track Proceedings. OpenReview.net."},{"key":"e_1_3_2_1_20_1","volume-title":"GPT Takes the Bar Exam. arXiv preprint arXiv:2212.14402","author":"Daniel Martin Katz Michael","year":"2022","unstructured":"Michael Bommarito II and Daniel Martin Katz. 2022. GPT Takes the Bar Exam. arXiv preprint arXiv:2212.14402 (2022)."},{"key":"e_1_3_2_1_21_1","unstructured":"Tom Brown Benjamin Mann Nick Ryder Melanie Subbiah Jared D Kaplan Prafulla Dhariwal Arvind Neelakantan Pranav Shyam Girish Sastry Amanda Askell et al. 2020. Language models are few-shot learners. Advances in neural information processing systems 33 (2020) 1877--1901."},{"key":"e_1_3_2_1_22_1","volume-title":"Yuanzhi Li, Scott Lundberg, et al.","author":"Bubeck S\u00ebbastien","year":"2023","unstructured":"S\u00ebbastien Bubeck, Varun Chandrasekaran, Ronen Eldan, Johannes Gehrke, Eric Horvitz, Ece Kamar, Peter Lee, Yin Tat Lee, Yuanzhi Li, Scott Lundberg, et al. 2023. Sparks of artificial general intelligence: Early experiments with gpt-4. arXiv preprint arXiv:2303.12712 (2023)."},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.5555\/3489212.3489286"},{"key":"e_1_3_2_1_24_1","volume-title":"31st USENIX Security Symposium (USENIX Security 22)","author":"Chen Ju","year":"2022","unstructured":"Ju Chen, Wookhyun Han, Mingjun Yin, Haochen Zeng, Chengyu Song, Byoungyoung Lee, Heng Yin, and InsikShin. 2022. {SYMSAN}: Time and Space Efficient Concolic Execution via Dynamic Data-flow Analysis. In 31st USENIX Security Symposium (USENIX Security 22). 2531--2548."},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2022.naacl-main.260"},{"key":"e_1_3_2_1_26_1","volume-title":"Jared Kaplan, Harri Edwards, Yuri Burda, Nicholas Joseph, Greg Brockman, et al.","author":"Chen Mark","year":"2021","unstructured":"Mark Chen, Jerry Tworek, Heewoo Jun, Qiming Yuan, Henrique Ponde de Oliveira Pinto, Jared Kaplan, Harri Edwards, Yuri Burda, Nicholas Joseph, Greg Brockman, et al. 2021. Evaluating large language models trained on code. arXiv preprint arXiv:2107.03374 (2021)."},{"key":"e_1_3_2_1_27_1","volume-title":"Xing","author":"Chiang Wei-Lin","year":"2023","unstructured":"Wei-Lin Chiang, Zhuohan Li, Zi Lin, Ying Sheng, Zhanghao Wu, Hao Zhang, Lianmin Zheng, Siyuan Zhuang, Yonghao Zhuang, Joseph E. Gonzalez, Ion Stoica, and Eric P. Xing. 2023. Vicuna: An Open-Source Chatbot Impressing GPT-4 with 90%* ChatGPT Quality. https:\/\/vicuna.lmsys.org"},{"key":"e_1_3_2_1_28_1","volume-title":"Charles Sutton, Sebastian Gehrmann, et al.","author":"Chowdhery Aakanksha","year":"2022","unstructured":"Aakanksha Chowdhery, Sharan Narang, Jacob Devlin, Maarten Bosma, Gaurav Mishra, Adam Roberts, Paul Barham, Hyung Won Chung, Charles Sutton, Sebastian Gehrmann, et al. 2022. Palm: Scaling language modeling with pathways. arXiv preprint arXiv:2204.02311 (2022)."},{"key":"e_1_3_2_1_29_1","volume-title":"CodeBERT: A Pre-Trained Model for Programming and Natural Languages. In Findings of the Association for Computational Linguistics: EMNLP 2020, Online Event, 16--20 November 2020 (Findings of ACL","author":"Feng Zhangyin","year":"2020","unstructured":"Zhangyin Feng, Daya Guo, Duyu Tang, Nan Duan, Xiaocheng Feng, Ming Gong, Linjun Shou, Bing Qin, Ting Liu, Daxin Jiang, and Ming Zhou. 2020. CodeBERT: A Pre-Trained Model for Programming and Natural Languages. In Findings of the Association for Computational Linguistics: EMNLP 2020, Online Event, 16--20 November 2020 (Findings of ACL, Vol. EMNLP 2020), Trevor Cohn, Yulan He, and Yang Liu (Eds.). Association for Computational Linguistics."},{"key":"e_1_3_2_1_30_1","volume-title":"Specializing Smaller Language Models towards Multi-Step Reasoning. arXiv preprint arXiv:2301.12726","author":"Fu Yao","year":"2023","unstructured":"Yao Fu, Hao Peng, Litu Ou, Ashish Sabharwal, and Tushar Khot. 2023. Specializing Smaller Language Models towards Multi-Step Reasoning. arXiv preprint arXiv:2301.12726 (2023)."},{"key":"e_1_3_2_1_31_1","volume-title":"The Pile: An 800GB Dataset of Diverse Text for Language Modeling. arXiv preprint arXiv: Arxiv-2101.00027","author":"Gao Leo","year":"2020","unstructured":"Leo Gao, Stella Biderman, Sid Black, Laurence Golding, Travis Hoppe, Charles Foster, Jason Phang, Horace He, Anish Thite, Noa Nabeshima, Shawn Presser, and Connor Leahy. 2020. The Pile: An 800GB Dataset of Diverse Text for Language Modeling. arXiv preprint arXiv: Arxiv-2101.00027 (2020)."},{"key":"e_1_3_2_1_32_1","volume-title":"Explaining and harnessing adversarial examples. arXiv preprint arXiv:1412.6572","author":"Goodfellow Ian J","year":"2014","unstructured":"Ian J Goodfellow, Jonathon Shlens, and Christian Szegedy. 2014. Explaining and harnessing adversarial examples. arXiv preprint arXiv:1412.6572 (2014)."},{"key":"e_1_3_2_1_33_1","volume-title":"Dynamic testing. Psychological bulletin 124, 1","author":"Grigorenko Elena L","year":"1998","unstructured":"Elena L Grigorenko and Robert J Sternberg. 1998. Dynamic testing. Psychological bulletin 124, 1 (1998), 75."},{"key":"e_1_3_2_1_34_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2021.naacl-main.161"},{"key":"e_1_3_2_1_35_1","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v36i10.21321"},{"key":"e_1_3_2_1_36_1","volume-title":"CATER: Intellectual Property Protection on Text Generation APIs via Conditional Watermarks. arXiv preprint arXiv:2209.08773","author":"He Xuanli","year":"2022","unstructured":"Xuanli He, Qiongkai Xu, Yi Zeng, Lingjuan Lyu, Fangzhao Wu, Jiwei Li, and Ruoxi Jia. 2022. CATER: Intellectual Property Protection on Text Generation APIs via Conditional Watermarks. arXiv preprint arXiv:2209.08773 (2022)."},{"key":"e_1_3_2_1_37_1","volume-title":"Large Language Models Are Reasoning Teachers. arXiv preprint arXiv:2212.10071","author":"Ho Namgyu","year":"2022","unstructured":"Namgyu Ho, Laura Schmid, and Se-Young Yun. 2022. Large Language Models Are Reasoning Teachers. arXiv preprint arXiv:2212.10071 (2022)."},{"key":"e_1_3_2_1_38_1","volume-title":"Codesearchnet challenge: Evaluating the state of semantic code search. arXiv preprint arXiv:1909.09436","author":"Husain Hamel","year":"2019","unstructured":"Hamel Husain, Ho-Hsiang Wu, Tiferet Gazit, Miltiadis Allamanis, and Marc Brockschmidt. 2019. Codesearchnet challenge: Evaluating the state of semantic code search. arXiv preprint arXiv:1909.09436 (2019)."},{"key":"e_1_3_2_1_39_1","doi-asserted-by":"publisher","DOI":"10.5555\/3489212.3489288"},{"key":"e_1_3_2_1_40_1","volume-title":"Codeattack: Code-based adversarial attacks for pre-trained programming language models. arXiv preprint arXiv:2206.00052","author":"Jha Akshita","year":"2022","unstructured":"Akshita Jha and Chandan K Reddy. 2022. Codeattack: Code-based adversarial attacks for pre-trained programming language models. arXiv preprint arXiv:2206.00052 (2022)."},{"key":"e_1_3_2_1_41_1","volume-title":"Machel Reid, Yutaka Matsuo, and Yusuke Iwasawa.","author":"Kojima Takeshi","year":"2022","unstructured":"Takeshi Kojima, Shixiang Shane Gu, Machel Reid, Yutaka Matsuo, and Yusuke Iwasawa. 2022. Large language models are zero-shot reasoners. arXiv preprint arXiv:2205.11916 (2022)."},{"key":"e_1_3_2_1_42_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2022.findings-emnlp.38"},{"key":"e_1_3_2_1_43_1","volume-title":"Yangtian Zi, Niklas Muennighoff, Denis Kocetkov, Chenghao Mou, Marc Marone, Christopher Akiki, Jia Li, Jenny Chim, et al.","author":"Li Raymond","year":"2023","unstructured":"Raymond Li, Loubna Ben Allal, Yangtian Zi, Niklas Muennighoff, Denis Kocetkov, Chenghao Mou, Marc Marone, Christopher Akiki, Jia Li, Jenny Chim, et al. 2023. StarCoder: may the source be with you! arXiv preprint arXiv:2305.06161 (2023)."},{"key":"e_1_3_2_1_44_1","unstructured":"Shiyang Li Jianshu Chen Yelong Shen Zhiyu Chen Xinlu Zhang Zekun Li Hong Wang Jing Qian Baolin Peng Yi Mao et al. 2022. Explanations from large language models make small reasoners better. arXiv preprint arXiv:2210.06726 (2022)."},{"key":"e_1_3_2_1_45_1","doi-asserted-by":"crossref","unstructured":"Yujia Li David Choi Junyoung Chung Nate Kushman Julian Schrittwieser R\u00e9mi Leblond Tom Eccles James Keeling Felix Gimeno Agustin Dal Lago Thomas Hubert Peter Choy Cyprien de Masson d'Autume Igor Babuschkin Xinyun Chen Po-Sen Huang Johannes Welbl Sven Gowal Alexey Cherepanov James Molloy Daniel J. Mankowitz Esme Sutherland Robson Pushmeet Kohli Nando de Freitas Koray Kavukcuoglu and Oriol Vinyals. 2022. Competition-level code generation with AlphaCode. Science 378 6624 (2022).","DOI":"10.1126\/science.abq1158"},{"key":"e_1_3_2_1_46_1","volume-title":"On the advance of making language models better reasoners. arXiv preprint arXiv:2206.02336","author":"Li Yifei","year":"2022","unstructured":"Yifei Li, Zeqi Lin, Shizhuo Zhang, Qiang Fu, Bei Chen, Jian-Guang Lou, and Weizhu Chen. 2022. On the advance of making language models better reasoners. arXiv preprint arXiv:2206.02336 (2022)."},{"key":"e_1_3_2_1_47_1","doi-asserted-by":"publisher","DOI":"10.1145\/3510003.3510217"},{"key":"e_1_3_2_1_48_1","volume-title":"CCTEST: Testing and Repairing Code Completion Systems. In 45th IEEE\/ACM International Conference on Software Engineering, ICSE 2023","author":"Li Zongjie","year":"2023","unstructured":"Zongjie Li, Chaozheng Wang, Zhibo Liu, Haoxuan Wang, Dong Chen, Shuai Wang, and Cuiyun Gao. 2023. CCTEST: Testing and Repairing Code Completion Systems. In 45th IEEE\/ACM International Conference on Software Engineering, ICSE 2023, Melbourne, Australia, May 14--20, 2023. IEEE, 1238--1250."},{"key":"e_1_3_2_1_49_1","doi-asserted-by":"publisher","DOI":"10.1145\/3576915.3623120"},{"key":"e_1_3_2_1_50_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2022.deelio-1.10"},{"key":"e_1_3_2_1_51_1","unstructured":"Pengfei Liu Weizhe Yuan Jinlan Fu Zhengbao Jiang Hiroaki Hayashi and Graham Neubig. 2023. Pre-train Prompt and Predict: A Systematic Survey of Prompting Methods in Natural Language Processing. ACM Comput. Surv. (2023)."},{"key":"e_1_3_2_1_52_1","volume-title":"Shengyu Fu, and Shujie Liu.","author":"Lu Shuai","year":"2021","unstructured":"Shuai Lu, Daya Guo, Shuo Ren, Junjie Huang, Alexey Svyatkovskiy, Ambrosio Blanco, Colin B. Clement, Dawn Drain, Daxin Jiang, Duyu Tang, Ge Li, Lidong Zhou, Linjun Shou, Long Zhou, Michele Tufano, Ming Gong, Ming Zhou, Nan Duan, Neel Sundaresan, Shao Kun Deng, Shengyu Fu, and Shujie Liu. 2021. CodeXGLUE: A Machine Learning Benchmark Dataset for Code Understanding and Generation. In NeurIPS Datasets and Benchmarks 2021, virtual, Joaquin Vanschoren and Sai-Kit Yeung (Eds.)."},{"key":"e_1_3_2_1_53_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2022.acl-long.556"},{"key":"e_1_3_2_1_54_1","volume-title":"WizardCoder: Empowering Code Large Language Models with Evol-Instruct. CoRR abs\/2306.08568","author":"Luo Ziyang","year":"2023","unstructured":"Ziyang Luo, Can Xu, Pu Zhao, Qingfeng Sun, Xiubo Geng, Wenxiang Hu, Chongyang Tao, Jing Ma, Qingwei Lin, and Daxin Jiang. 2023. WizardCoder: Empowering Code Large Language Models with Evol-Instruct. CoRR abs\/2306.08568 (2023)."},{"key":"e_1_3_2_1_55_1","volume-title":"Hassan","author":"McIntosh Shane","year":"2014","unstructured":"Shane McIntosh, Yasutaka Kamei, Bram Adams, and Ahmed E. Hassan. 2014. The Impact of Code Review Coverage and Code Review Participation on Software Quality: A Case Study of the Qt, VTK, and ITK Projects (MSR 2014)."},{"key":"e_1_3_2_1_56_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE48619.2023.00180"},{"key":"e_1_3_2_1_57_1","unstructured":"OpenAI. 2023. Our approach to AI safety. https:\/\/openai.com\/blog\/our-approach-to-ai-safety"},{"key":"e_1_3_2_1_58_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2019.00509"},{"key":"e_1_3_2_1_59_1","volume-title":"Proceedings of the 29th USENIX Conference on Security Symposium. 181--198","author":"Poeplau Sebastian","year":"2020","unstructured":"Sebastian Poeplau and Aurelien Francillon. 2020. Symbolic execution with SymCC: Don't interpret, compile!. In Proceedings of the 29th USENIX Conference on Security Symposium. 181--198."},{"key":"e_1_3_2_1_60_1","volume-title":"Liu","author":"Raffel Colin","year":"2020","unstructured":"Colin Raffel, Noam Shazeer, Adam Roberts, Katherine Lee, Sharan Narang, Michael Matena, Yanqi Zhou, Wei Li, and Peter J. Liu. 2020. Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer. J. Mach. Learn. Res. 21 (2020)."},{"key":"e_1_3_2_1_61_1","volume-title":"Codebleu: a method for automatic evaluation of code synthesis. arXiv preprint arXiv:2009.10297","author":"Ren Shuo","year":"2020","unstructured":"Shuo Ren, Daya Guo, Shuai Lu, Long Zhou, Shujie Liu, Duyu Tang, Neel Sundaresan, Ming Zhou, Ambrosio Blanco, and Shuai Ma. 2020. Codebleu: a method for automatic evaluation of code synthesis. arXiv preprint arXiv:2009.10297 (2020)."},{"key":"e_1_3_2_1_62_1","volume-title":"Unsupervised translation of programming languages. Advances in Neural Information Processing Systems 33","author":"Roziere Baptiste","year":"2020","unstructured":"Baptiste Roziere, Marie-Anne Lachaux, Lowik Chanussot, and Guillaume Lample. 2020. Unsupervised translation of programming languages. Advances in Neural Information Processing Systems 33 (2020)."},{"key":"e_1_3_2_1_63_1","volume-title":"Arun Raja, et al.","author":"Sanh Victor","year":"2022","unstructured":"Victor Sanh, Albert Webson, Colin Raffel, Stephen H Bach, Lintang Sutawika, Zaid Alyafeai, Antoine Chaffin, Arnaud Stiegler, Teven Le Scao, Arun Raja, et al. 2022. Multitask Prompted Training Enables Zero-Shot Task Generalization. (2022)."},{"key":"e_1_3_2_1_64_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.emnlp-main.346"},{"key":"e_1_3_2_1_65_1","volume-title":"Primers or Reminders? The Effects of Existing Review Comments on Code Review (ICSE '20)","author":"Spadini Davide","unstructured":"Davide Spadini, G\u00fcl \u00c7alikli, and Alberto Bacchelli. 2020. Primers or Reminders? The Effects of Existing Review Comments on Code Review (ICSE '20). Association for Computing Machinery, New York, NY, USA."},{"key":"e_1_3_2_1_66_1","volume-title":"Hashimoto","author":"Taori Rohan","year":"2023","unstructured":"Rohan Taori, Ishaan Gulrajani, Tianyi Zhang, Yann Dubois, Xuechen Li, Carlos Guestrin, Percy Liang, and Tatsunori B. Hashimoto. 2023. Stanford Alpaca: An Instruction-following LLaMA model. https:\/\/github.com\/tatsu-lab\/stanford_alpaca."},{"key":"e_1_3_2_1_67_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.emnlp-main.446"},{"key":"e_1_3_2_1_68_1","volume-title":"Towards Understanding Chain-of-Thought Prompting: An Empirical Study of What Matters. arXiv preprint arXiv:2212.10001","author":"Wang Boshi","year":"2022","unstructured":"Boshi Wang, Sewon Min, Xiang Deng, Jiaming Shen, You Wu, Luke Zettlemoyer, and Huan Sun. 2022. Towards Understanding Chain-of-Thought Prompting: An Empirical Study of What Matters. arXiv preprint arXiv:2212.10001 (2022)."},{"key":"e_1_3_2_1_69_1","doi-asserted-by":"publisher","DOI":"10.1145\/3540250.3549113"},{"key":"e_1_3_2_1_70_1","unstructured":"Jindong Wang Xixu Hu Wenxin Hou Hao Chen Runkai Zheng Yidong Wang Linyi Yang Haojun Huang Wei Ye Xiubo Geng et al. 2023. On the Robustness of ChatGPT: An Adversarial and Out-of-distribution Perspective. arXiv preprint arXiv:2302.12095 (2023)."},{"key":"e_1_3_2_1_71_1","volume-title":"Chi, and Denny Zhou","author":"Wang Xuezhi","year":"2022","unstructured":"Xuezhi Wang, Jason Wei, Dale Schuurmans, Quoc Le, Ed Chi, and Denny Zhou. 2022. Rationale-augmented ensembles in language models. arXiv preprint arXiv:2207.00747 (2022)."},{"key":"e_1_3_2_1_72_1","volume-title":"Self-Consistency Improves Chain of Thought Reasoning in Language Models. In The Eleventh International Conference on Learning Representations, ICLR 2023","author":"Wang Xuezhi","year":"2023","unstructured":"Xuezhi Wang, Jason Wei, Dale Schuurmans, Quoc V. Le, Ed H. Chi, Sharan Narang, Aakanksha Chowdhery, and Denny Zhou. 2023. Self-Consistency Improves Chain of Thought Reasoning in Language Models. In The Eleventh International Conference on Learning Representations, ICLR 2023, Kigali, Rwanda, May 1--5, 2023."},{"key":"e_1_3_2_1_73_1","volume-title":"David Wadden","author":"Wang Yizhong","year":"2023","unstructured":"Yizhong Wang, Hamish Ivison, Pradeep Dasigi, Jack Hessel, Tushar Khot, Khyathi Raghavi Chandu, David Wadden, Kelsey MacMillan, Noah ASmith, Iz Beltagy, et al. 2023. How Far Can Camels Go? Exploring the State of Instruction Tuning on Open Resources. arXiv preprint arXiv:2306.04751 (2023)."},{"key":"e_1_3_2_1_74_1","volume-title":"Proceedings of the 2021 Conference on Empirical Methods in Natural Language Processing, EMNLP 2021, Virtual Event \/ Punta Cana, Dominican Republic, 7--11","author":"Wang Yue","year":"2021","unstructured":"Yue Wang, Weishi Wang, Shafiq R. Joty, and Steven C. H. Hoi. 2021. CodeT5: Identifier-aware Unified Pre-trained Encoder-Decoder Models for Code Understanding and Generation. In Proceedings of the 2021 Conference on Empirical Methods in Natural Language Processing, EMNLP 2021, Virtual Event \/ Punta Cana, Dominican Republic, 7--11 November, 2021, Marie-Francine Moens, Xuanjing Huang, Lucia Specia, and Scott Wen-tau Yih (Eds.). Association for Computational Linguistics."},{"key":"e_1_3_2_1_75_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2022.naacl-main.167"},{"key":"e_1_3_2_1_76_1","volume-title":"Code generation as a dual task of code summarization. Advances in neural information processing systems 32","author":"Wei Bolin","year":"2019","unstructured":"Bolin Wei, Ge Li, Xin Xia, Zhiyi Fu, and Zhi Jin. 2019. Code generation as a dual task of code summarization. Advances in neural information processing systems 32 (2019)."},{"key":"e_1_3_2_1_77_1","unstructured":"Jason Wei Yi Tay Rishi Bommasani Colin Raffel Barret Zoph Sebastian Borgeaud Dani Yogatama Maarten Bosma Denny Zhou Donald Metzler et al. 2022. Emergent abilities of large language models. arXiv preprint arXiv:2206.07682 (2022)."},{"key":"e_1_3_2_1_78_1","volume-title":"Chi, Quoc Le, and Denny Zhou","author":"Wei Jason","year":"2022","unstructured":"Jason Wei, Xuezhi Wang, Dale Schuurmans, Maarten Bosma, Ed Chi, Quoc Le, and Denny Zhou. 2022. Chain of thought prompting elicits reasoning in large language models. arXiv preprint arXiv:2201.11903 (2022)."},{"key":"e_1_3_2_1_79_1","volume-title":"Chatgpt prompt patterns for improving code quality, refactoring, requirements elicitation, and software design. arXiv preprint arXiv:2303.07839","author":"White Jules","year":"2023","unstructured":"Jules White, Sam Hays, Quchen Fu, Jesse Spencer-Smith, and Douglas C Schmidt. 2023. Chatgpt prompt patterns for improving code quality, refactoring, requirements elicitation, and software design. arXiv preprint arXiv:2303.07839 (2023)."},{"key":"e_1_3_2_1_80_1","volume-title":"Conversational automated program repair. arXiv preprint arXiv:2301.13246","author":"Xia Chunqiu Steven","year":"2023","unstructured":"Chunqiu Steven Xia and Lingming Zhang. 2023. Conversational automated program repair. arXiv preprint arXiv:2301.13246 (2023)."},{"key":"e_1_3_2_1_81_1","volume-title":"Beyond model extraction: Imitation attack for black-box nlp apis. arXiv preprint arXiv:2108.13873","author":"Xu Qiongkai","year":"2021","unstructured":"Qiongkai Xu, Xuanli He, Lingjuan Lyu, Lizhen Qu, and Gholamreza Haffari. 2021. Beyond model extraction: Imitation attack for black-box nlp apis. arXiv preprint arXiv:2108.13873 (2021)."},{"key":"e_1_3_2_1_82_1","volume-title":"Student Surpasses Teacher: Imitation Attack for Black-Box NLP APIs. arXiv preprint arXiv:2108.13873","author":"Xu Qiongkai","year":"2021","unstructured":"Qiongkai Xu, Xuanli He, Lingjuan Lyu, Lizhen Qu, and Gholamreza Haffari. 2021. Student Surpasses Teacher: Imitation Attack for Black-Box NLP APIs. arXiv preprint arXiv:2108.13873 (2021)."},{"key":"e_1_3_2_1_83_1","volume-title":"How Important are Good Method Names in Neural Code Generation? A Model Robustness Perspective. arXiv preprint arXiv:2211.15844","author":"Yang Guang","year":"2022","unstructured":"Guang Yang, Yu Zhou, Wenhua Yang, Tao Yue, Xiang Chen, and Taolue Chen. 2022. How Important are Good Method Names in Neural Code Generation? A Model Robustness Perspective. arXiv preprint arXiv:2211.15844 (2022)."},{"key":"e_1_3_2_1_84_1","doi-asserted-by":"publisher","DOI":"10.1145\/3196398.3196408"},{"key":"e_1_3_2_1_85_1","unstructured":"Honggang Yu Kaichen Yang Teng Zhang Yun-Yun Tsai Tsung-Yi Ho and Yier Jin. 2020. CloudLeak: Large-Scale Deep Learning Models Stealing Through Adversarial Examples.. In NDSS."},{"key":"e_1_3_2_1_86_1","unstructured":"Lianmin Zheng Wei-Lin Chiang Ying Sheng Siyuan Zhuang Zhanghao Wu Yonghao Zhuang Zi Lin Zhuohan Li Dacheng Li Eric Xing et al. 2023. Judging LLM-as-a-judge with MT-Bench and Chatbot Arena. arXiv preprint arXiv:2306.05685 (2023)."},{"key":"e_1_3_2_1_87_1","doi-asserted-by":"crossref","unstructured":"Qinkai Zheng Xiao Xia Xu Zou Yuxiao Dong Shan Wang Yufei Xue Zihan Wang Lei Shen Andi Wang Yang Li et al. 2023. CodeGeeX: A Pre-Trained Model for Code Generation with Multilingual Evaluations on HumanEval-X. arXiv preprint arXiv:2303.17568 (2023).","DOI":"10.1145\/3580305.3599790"},{"key":"e_1_3_2_1_88_1","volume-title":"Xiaoning Du, and Yang Liu.","author":"Zhou Yaqin","year":"2019","unstructured":"Yaqin Zhou, Shangqing Liu, Jing Kai Siow, Xiaoning Du, and Yang Liu. 2019. Devign: Effective Vulnerability Identification by Learning Comprehensive Program Semantics via Graph Neural Networks. In NeurIPS 2019, December 8--14, 2019, Vancouver, BC, Canada, Hanna M. Wallach, Hugo Larochelle, Alina Beygelzimer, Florence d'Alch\u00e9-Buc, Emily B. Fox, and Roman Garnett (Eds.)."},{"key":"e_1_3_2_1_89_1","volume-title":"XLCoST: A Benchmark Dataset for Cross-lingual Code Intelligence. arXiv preprint arXiv:2206.08474","author":"Zhu Ming","year":"2022","unstructured":"Ming Zhu, Aneesh Jain, Karthik Suresh, Roshan Ravindran, Sindhu Tipirneni, and Chandan K Reddy. 2022. XLCoST: A Benchmark Dataset for Cross-lingual Code Intelligence. arXiv preprint arXiv:2206.08474 (2022)."},{"key":"e_1_3_2_1_90_1","volume-title":"USA","author":"Ziegler Albert","year":"2022","unstructured":"Albert Ziegler, Eirini Kalliamvakou, X. Alice Li, Andrew Rice, Devon Rifkin, Shawn Simister, Ganesh Sittampalam, and Edward Aftandilian. 2022. Productivity assessment of neural code completion. In MAPS@PLDI 2022: 6th ACM SIGPLAN International Symposium on Machine Programming, San Diego, CA, USA, 13 June 2022, Swarat Chaudhuri and Charles Sutton (Eds.)."},{"key":"e_1_3_2_1_91_1","unstructured":"Andy Zou Zifan Wang J. Zico Kolter and Matt Fredrikson. 2023. Universal and Transferable Adversarial Attacks on Aligned Language Models. arXiv:2307.15043 [cs.CL]"}],"event":{"name":"ICSE '24: IEEE\/ACM 46th International Conference on Software Engineering","location":"Lisbon Portugal","acronym":"ICSE '24","sponsor":["SIGSOFT ACM Special Interest Group on Software Engineering","IEEE CS","Faculty of Engineering of University of Porto"]},"container-title":["Proceedings of the IEEE\/ACM 46th International Conference on Software Engineering"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3597503.3639091","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3597503.3639091","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,18]],"date-time":"2025-06-18T22:49:11Z","timestamp":1750286951000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3597503.3639091"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,4,12]]},"references-count":91,"alternative-id":["10.1145\/3597503.3639091","10.1145\/3597503"],"URL":"https:\/\/doi.org\/10.1145\/3597503.3639091","relation":{},"subject":[],"published":{"date-parts":[[2024,4,12]]},"assertion":[{"value":"2024-04-12","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}