{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,1,13]],"date-time":"2026-01-13T14:43:25Z","timestamp":1768315405863,"version":"3.49.0"},"publisher-location":"New York, NY, USA","reference-count":38,"publisher":"ACM","license":[{"start":{"date-parts":[[2024,10,27]],"date-time":"2024-10-27T00:00:00Z","timestamp":1729987200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2024,10,27]]},"DOI":"10.1145\/3691620.3695297","type":"proceedings-article","created":{"date-parts":[[2024,10,18]],"date-time":"2024-10-18T15:39:19Z","timestamp":1729265959000},"page":"2268-2272","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":3,"title":["Attacks and Defenses for Large Language Models on Coding Tasks"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0009-0003-0012-1320","authenticated-orcid":false,"given":"Chi","family":"Zhang","sequence":"first","affiliation":[{"name":"Electrical and Computer Engineering, CyLab, Carnegie Mellon University, Pittsburgh, Pennsylvania, USA"}]},{"ORCID":"https:\/\/orcid.org\/0009-0004-6233-2120","authenticated-orcid":false,"given":"Zifan","family":"Wang","sequence":"additional","affiliation":[{"name":"Center for AI Safety, San Francisco, California, USA"}]},{"ORCID":"https:\/\/orcid.org\/0009-0003-2412-9634","authenticated-orcid":false,"given":"Ruoshi","family":"Zhao","sequence":"additional","affiliation":[{"name":"Independent Researcher, Piscataway, New Jersey, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-6267-6995","authenticated-orcid":false,"given":"Ravi","family":"Mangal","sequence":"additional","affiliation":[{"name":"Department of Computer Science, Colorado State University, Fort Collins, Colorado, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-1820-1698","authenticated-orcid":false,"given":"Matt","family":"Fredrikson","sequence":"additional","affiliation":[{"name":"School of Computer Science, CyLab, Carnegie Mellon University, Pittsburgh, Pennsylvania, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-8160-349X","authenticated-orcid":false,"given":"Limin","family":"Jia","sequence":"additional","affiliation":[{"name":"Electrical and Computer Engineering, CyLab, Carnegie Mellon University, Pittsburgh, Pennsylvania, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-5579-6961","authenticated-orcid":false,"given":"Corina","family":"Pasareanu","sequence":"additional","affiliation":[{"name":"Electrical and Computer Engineering, CyLab, Carnegie Mellon University, Moffett Field, California, USA"}]}],"member":"320","published-online":{"date-parts":[[2024,10,27]]},"reference":[{"key":"e_1_3_2_1_1_1","volume-title":"International Conference on Learning Representations. https:\/\/openreview.net\/forum?id=BJOFETxR-","author":"Allamanis Miltiadis","year":"2018","unstructured":"Miltiadis Allamanis, Marc Brockschmidt, and Mahmoud Khademi. 2018. Learning to Represent Programs with Graphs. In International Conference on Learning Representations. https:\/\/openreview.net\/forum?id=BJOFETxR-"},{"key":"e_1_3_2_1_2_1","volume-title":"International conference on machine learning. PMLR","author":"Allamanis Miltiadis","year":"2016","unstructured":"Miltiadis Allamanis, Hao Peng, and Charles Sutton. 2016. A convolutional attention network for extreme summarization of source code. In International conference on machine learning. PMLR, 2091--2100."},{"key":"e_1_3_2_1_3_1","volume-title":"International Conference on Learning Representations. https:\/\/openreview.net\/forum?id=H1gKYo09tX","author":"Alon Uri","year":"2019","unstructured":"Uri Alon, Omer Levy, and Eran Yahav. 2019. code2seq: Generating Sequences from Structured Representations of Code. In International Conference on Learning Representations. https:\/\/openreview.net\/forum?id=H1gKYo09tX"},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.1145\/3290353"},{"key":"e_1_3_2_1_5_1","unstructured":"Anthropic. 2023. Model Card and Evaluations for Claude Models. https:\/\/efficient-manatee.files.svdcdn.com\/production\/images\/Model-Card-Claude-2.pdf?dm=1689034733"},{"key":"e_1_3_2_1_6_1","volume-title":"Program Synthesis with Large Language Models. CoRR abs\/2108.07732","author":"Austin Jacob","year":"2021","unstructured":"Jacob Austin, Augustus Odena, Maxwell I. Nye, Maarten Bosma, Henryk Michalewski, David Dohan, Ellen Jiang, Carrie J. Cai, Michael Terry, Quoc V. Le, and Charles Sutton. 2021. Program Synthesis with Large Language Models. CoRR abs\/2108.07732 (2021). arXiv:2108.07732 https:\/\/arxiv.org\/abs\/2108.07732"},{"key":"e_1_3_2_1_7_1","unstructured":"Pavol Bielik and Martin Vechev. 2020. Adversarial Robustness for Code. arXiv:2002.04694 [cs.LG]"},{"key":"e_1_3_2_1_8_1","unstructured":"Tom Brown Benjamin Mann Nick Ryder Melanie Subbiah Jared D Kaplan Prafulla Dhariwal Arvind Neelakantan Pranav Shyam Girish Sastry Amanda Askell et al. 2020. Language models are few-shot learners. Advances in neural information processing systems 33 (2020) 1877--1901."},{"key":"e_1_3_2_1_9_1","unstructured":"Mark Chen Jerry Tworek Heewoo Jun Qiming Yuan Henrique Pond\u00e9 de Oliveira Pinto Jared Kaplan Harrison Edwards Yuri Burda Nicholas Joseph Greg Brockman Alex Ray Raul Puri Gretchen Krueger Michael Petrov Heidy Khlaaf Girish Sastry Pamela Mishkin Brooke Chan Scott Gray Nick Ryder Mikhail Pavlov Alethea Power Lukasz Kaiser Mohammad Bavarian Clemens Winter Philippe Tillet Felipe Petroski Such Dave Cummings Matthias Plappert Fotios Chantzis Elizabeth Barnes Ariel Herbert-Voss William Hebgen Guss Alex Nichol Alex Paino Nikolas Tezak Jie Tang Igor Babuschkin Suchir Balaji Shantanu Jain William Saunders Christopher Hesse Andrew N. Carr Jan Leike Joshua Achiam Vedant Misra Evan Morikawa Alec Radford Matthew Knight Miles Brundage Mira Murati Katie Mayer Peter Welinder Bob McGrew Dario Amodei Sam McCandlish Ilya Sutskever and Wojciech Zaremba. 2021. Evaluating Large Language Models Trained on Code. CoRR abs\/2107.03374 (2021). arXiv:2107.03374 https:\/\/arxiv.org\/abs\/2107.03374"},{"key":"e_1_3_2_1_10_1","volume-title":"8th International Conference on Learning Representations, ICLR 2020","author":"Dinella Elizabeth","year":"2020","unstructured":"Elizabeth Dinella, Hanjun Dai, Ziyang Li, Mayur Naik, Le Song, and Ke Wang. 2020. Hoppity: Learning Graph Transformations to Detect and Fix Bugs in Programs. In 8th International Conference on Learning Representations, ICLR 2020, Addis Ababa, Ethiopia, April 26--30, 2020. OpenReview.net."},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.1145\/3591227"},{"key":"e_1_3_2_1_12_1","volume-title":"3rd International Conference on Learning Representations, ICLR","author":"Goodfellow Ian J.","year":"2015","unstructured":"Ian J. Goodfellow, Jonathon Shlens, and Christian Szegedy. 2015. Explaining and Harnessing Adversarial Examples. In 3rd International Conference on Learning Representations, ICLR 2015, San Diego, CA, USA, May 7--9, 2015, Conference Track Proceedings, Yoshua Bengio and Yann LeCun (Eds.). http:\/\/arxiv.org\/abs\/1412.6572"},{"key":"e_1_3_2_1_13_1","volume-title":"A Survey on Transferability of Adversarial Examples across Deep Neural Networks. arXiv preprint arXiv:2310.17626","author":"Gu Jindong","year":"2023","unstructured":"Jindong Gu, Xiaojun Jia, Pau de Jorge, Wenqain Yu, Xinwei Liu, Avery Ma, Yuan Xun, Anjun Hu, Ashkan Khakzar, Zhijiang Li, Xiaochun Cao, and Philip Torr. 2023. A Survey on Transferability of Adversarial Examples across Deep Neural Networks. arXiv preprint arXiv:2310.17626 (2023)."},{"key":"e_1_3_2_1_14_1","volume-title":"Baseline defenses for adversarial attacks against aligned language models. arXiv preprint arXiv:2309.00614","author":"Jain Neel","year":"2023","unstructured":"Neel Jain, Avi Schwarzschild, Yuxin Wen, Gowthami Somepalli, John Kirchenbauer, Ping-yeh Chiang, Micah Goldblum, Aniruddha Saha, Jonas Geiping, and Tom Goldstein. 2023. Baseline defenses for adversarial attacks against aligned language models. arXiv preprint arXiv:2309.00614 (2023)."},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v37i12.26739"},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.1145\/3571730"},{"key":"e_1_3_2_1_17_1","volume-title":"Certifying llm safety against adversarial prompting. arXiv preprint arXiv:2309.02705","author":"Kumar Aounon","year":"2023","unstructured":"Aounon Kumar, Chirag Agarwal, Suraj Srinivas, Soheil Feizi, and Hima Lakkaraju. 2023. Certifying llm safety against adversarial prompting. arXiv preprint arXiv:2309.02705 (2023)."},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","unstructured":"Yujia Li David Choi Junyoung Chung Nate Kushman Julian Schrittwieser R\u00e9mi Leblond Tom Eccles James Keeling Felix Gimeno Agustin Dal Lago Thomas Hubert Peter Choy Cyprien de Masson d'Autume Igor Babuschkin Xinyun Chen Po-Sen Huang Johannes Welbl Sven Gowal Alexey Cherepanov James Molloy Daniel J. Mankowitz Esme Sutherland Robson Pushmeet Kohli Nando de Freitas Koray Kavukcuoglu and Oriol Vinyals. 2022. Competition-level code generation with AlphaCode. Science 378 6624 (2022) 1092--1097. arXiv:https:\/\/www.science.org\/doi\/pdf\/10.1126\/science.abq1158 10.1126\/science.abq1158","DOI":"10.1126\/science.abq1158"},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","DOI":"10.1145\/3510003.3510181"},{"key":"e_1_3_2_1_20_1","volume-title":"Rigorous Evaluation of Large Language Models for Code Generation. In Thirty-seventh Conference on Neural Information Processing Systems. https:\/\/openreview.net\/forum?id=1qvx610Cu7","author":"Liu Jiawei","year":"2023","unstructured":"Jiawei Liu, Chunqiu Steven Xia, Yuyao Wang, and LINGMING ZHANG. 2023. Is Your Code Generated by ChatGPT Really Correct? Rigorous Evaluation of Large Language Models for Code Generation. In Thirty-seventh Conference on Neural Information Processing Systems. https:\/\/openreview.net\/forum?id=1qvx610Cu7"},{"key":"e_1_3_2_1_21_1","unstructured":"Aleksander Madry Aleksandar Makelov Ludwig Schmidt Dimitris Tsipras and Adrian Vladu. 2019. Towards Deep Learning Models Resistant to Adversarial Attacks. arXiv:1706.06083 [stat.ML]"},{"key":"e_1_3_2_1_23_1","volume-title":"Proceedings of the 40th International Conference on Machine Learning (Proceedings of Machine Learning Research","volume":"27520","author":"Pei Kexin","year":"2023","unstructured":"Kexin Pei, David Bieber, Kensen Shi, Charles Sutton, and Pengcheng Yin. 2023. Can Large Language Models Reason about Program Invariants?. In Proceedings of the 40th International Conference on Machine Learning (Proceedings of Machine Learning Research, Vol. 202), Andreas Krause, Emma Brunskill, Kyunghyun Cho, Barbara Engelhardt, Sivan Sabato, and Jonathan Scarlett (Eds.). PMLR, 27496--27520. https:\/\/proceedings.mlr.press\/v202\/pei23a.html"},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"publisher","DOI":"10.1145\/3276517"},{"key":"e_1_3_2_1_25_1","volume-title":"Chenguang Zhu, and Michael Zeng.","author":"Pryzant Reid","year":"2023","unstructured":"Reid Pryzant, Dan Iter, Jerry Li, Yin Tat Lee, Chenguang Zhu, and Michael Zeng. 2023. Automatic prompt optimization with\" gradient descent\" and beam search. arXiv preprint arXiv:2305.03495 (2023)."},{"key":"e_1_3_2_1_26_1","volume-title":"Semantic robustness of models of source code. arXiv preprint arXiv:2002.03043","author":"Ramakrishnan Goutham","year":"2020","unstructured":"Goutham Ramakrishnan, Jordan Henkel, Zi Wang, Aws Albarghouthi, Somesh Jha, and Thomas Reps. 2020. Semantic robustness of models of source code. arXiv preprint arXiv:2002.03043 (2020)."},{"key":"e_1_3_2_1_27_1","volume-title":"SmoothLLM: Defending Large Language Models Against Jailbreaking Attacks. arXiv preprint arXiv:2310.03684","author":"Robey Alexander","year":"2023","unstructured":"Alexander Robey, Eric Wong, Hamed Hassani, and George J Pappas. 2023. SmoothLLM: Defending Large Language Models Against Jailbreaking Attacks. arXiv preprint arXiv:2310.03684 (2023)."},{"key":"e_1_3_2_1_28_1","unstructured":"Baptiste Rozi\u00e8re Jonas Gehring Fabian Gloeckle Sten Sootla Itai Gat Xiaoqing Ellen Tan Yossi Adi Jingyu Liu Tal Remez J\u00e9r\u00e9my Rapin Artyom Kozhevnikov Ivan Evtimov Joanna Bitton Manish Bhatt Cristian Canton Ferrer Aaron Grattafiori Wenhan Xiong Alexandre D\u00e9fossez Jade Copet Faisal Azhar Hugo Touvron Louis Martin Nicolas Usunier Thomas Scialom and Gabriel Synnaeve. 2023. Code Llama: Open Foundation Models for Code. arXiv:2308.12950 [cs.CL]"},{"key":"e_1_3_2_1_29_1","volume-title":"Characterizing and evaluating in-the-wild jailbreak prompts on large language models. arXiv preprint arXiv:2308.03825","author":"Shen Xinyue","year":"2023","unstructured":"Xinyue Shen, Zeyuan Chen, Michael Backes, Yun Shen, and Yang Zhang. 2023. \" do anything now\": Characterizing and evaluating in-the-wild jailbreak prompts on large language models. arXiv preprint arXiv:2308.03825 (2023)."},{"key":"e_1_3_2_1_30_1","volume-title":"International Conference on Learning Representations. https:\/\/openreview.net\/forum?id=PH5PH9ZO_4","author":"Srikant Shashank","year":"2021","unstructured":"Shashank Srikant, Sijia Liu, Tamara Mitrovska, Shiyu Chang, Quanfu Fan, Gaoyuan Zhang, and Una-May O'Reilly. 2021. Generating Adversarial Computer Programs using Optimized Obfuscations. In International Conference on Learning Representations. https:\/\/openreview.net\/forum?id=PH5PH9ZO_4"},{"key":"e_1_3_2_1_31_1","volume-title":"Sequence to sequence learning with neural networks. Advances in neural information processing systems 27","author":"Sutskever Ilya","year":"2014","unstructured":"Ilya Sutskever, Oriol Vinyals, and Quoc V Le. 2014. Sequence to sequence learning with neural networks. Advances in neural information processing systems 27 (2014)."},{"key":"e_1_3_2_1_32_1","unstructured":"Christian Szegedy Wojciech Zaremba Ilya Sutskever Joan Bruna Dumitru Erhan Ian Goodfellow and Rob Fergus. 2014. Intriguing properties of neural networks. arXiv:1312.6199 [cs.CV]"},{"key":"e_1_3_2_1_33_1","volume-title":"International Conference on Learning Representations. https:\/\/openreview.net\/forum?id=ByloJ20qtm","author":"Vasic Marko","year":"2019","unstructured":"Marko Vasic, Aditya Kanade, Petros Maniatis, David Bieber, and Rishabh singh. 2019. Neural Program Repair by Jointly Learning to Localize and Repair. In International Conference on Learning Representations. https:\/\/openreview.net\/forum?id=ByloJ20qtm"},{"key":"e_1_3_2_1_34_1","volume-title":"Dan Roth, and Bing Xiang.","author":"Wang Shiqi","year":"2022","unstructured":"Shiqi Wang, Zheng Li, Haifeng Qian, Chenghao Yang, Zijian Wang, Mingyue Shang, Varun Kumar, Samson Tan, Baishakhi Ray, Parminder Bhatia, Ramesh Nallapati, Murali Krishna Ramanathan, Dan Roth, and Bing Xiang. 2022. ReCode: Robustness Evaluation of Code Generation Models. arXiv:2212.10264 [cs.LG]"},{"key":"e_1_3_2_1_35_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE48619.2023.00129"},{"key":"e_1_3_2_1_36_1","doi-asserted-by":"publisher","DOI":"10.1145\/3428230"},{"key":"e_1_3_2_1_37_1","volume-title":"The Eleventh International Conference on Learning Representations. https:\/\/openreview.net\/forum?id=92gvk82DE-","author":"Zhou Yongchao","year":"2023","unstructured":"Yongchao Zhou, Andrei Ioan Muresanu, Ziwen Han, Keiran Paster, Silviu Pitis, Harris Chan, and Jimmy Ba. 2023. Large Language Models are Human-Level Prompt Engineers. In The Eleventh International Conference on Learning Representations. https:\/\/openreview.net\/forum?id=92gvk82DE-"},{"key":"e_1_3_2_1_38_1","doi-asserted-by":"publisher","DOI":"10.48550\/arXiv.2307.15043"},{"key":"e_1_3_2_1_39_1","unstructured":"Andy Zou Zifan Wang J. Zico Kolter and Matt Fredrikson. 2023. Universal and Transferable Adversarial Attacks on Aligned Language Models. arXiv:2307.15043 [cs.CL]"}],"event":{"name":"ASE '24: 39th IEEE\/ACM International Conference on Automated Software Engineering","location":"Sacramento CA USA","acronym":"ASE '24","sponsor":["SIGAI ACM Special Interest Group on Artificial Intelligence","SIGSOFT ACM Special Interest Group on Software Engineering","IEEE CS"]},"container-title":["Proceedings of the 39th IEEE\/ACM International Conference on Automated Software Engineering"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3691620.3695297","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3691620.3695297","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,19]],"date-time":"2025-06-19T00:04:07Z","timestamp":1750291447000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3691620.3695297"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,10,27]]},"references-count":38,"alternative-id":["10.1145\/3691620.3695297","10.1145\/3691620"],"URL":"https:\/\/doi.org\/10.1145\/3691620.3695297","relation":{},"subject":[],"published":{"date-parts":[[2024,10,27]]},"assertion":[{"value":"2024-10-27","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}