{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,17]],"date-time":"2026-07-17T15:00:22Z","timestamp":1784300422644,"version":"3.55.0"},"publisher-location":"New York, NY, USA","reference-count":79,"publisher":"ACM","license":[{"start":{"date-parts":[[2023,11,15]],"date-time":"2023-11-15T00:00:00Z","timestamp":1700006400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2023,11,15]]},"DOI":"10.1145\/3576915.3623175","type":"proceedings-article","created":{"date-parts":[[2023,11,21]],"date-time":"2023-11-21T12:35:13Z","timestamp":1700570113000},"page":"1865-1879","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":119,"title":["Large Language Models for Code: Security Hardening and Adversarial Testing"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0009-0003-9858-5640","authenticated-orcid":false,"given":"Jingxuan","family":"He","sequence":"first","affiliation":[{"name":"ETH Zurich, Zurich, Switzerland"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-0054-9568","authenticated-orcid":false,"given":"Martin","family":"Vechev","sequence":"additional","affiliation":[{"name":"ETH Zurich, Zurich, Switzerland"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2023,11,21]]},"reference":[{"key":"e_1_3_2_1_1_1","unstructured":"2022. 2022 CWE Top 25 Most Dangerous Software Weaknesses. https:\/\/cwe. mitre.org\/data\/definitions\/1387.html"},{"key":"e_1_3_2_1_2_1","unstructured":"2023. AI Assistant for software developers | Tabnine. https:\/\/www.tabnine.com"},{"key":"e_1_3_2_1_3_1","unstructured":"2023. AI Code Generator - Amazon CodeWhisperer - AWS. https:\/\/aws.amazon. com\/codewhisperer"},{"key":"e_1_3_2_1_4_1","unstructured":"2023. ChatGPT. https:\/\/openai.com\/blog\/chatgpt"},{"key":"e_1_3_2_1_5_1","unstructured":"2023. Codeium. https:\/\/codeium.com"},{"key":"e_1_3_2_1_6_1","unstructured":"2023. CodeQL - GitHub. https:\/\/codeql.github.com"},{"key":"e_1_3_2_1_7_1","unstructured":"2023. fgets - cppreference.com. https:\/\/en.cppreference.com\/w\/c\/io\/fgets"},{"key":"e_1_3_2_1_8_1","unstructured":"2023. Ghostwriter - Code faster with AI. https:\/\/replit.com\/site\/ghostwriter"},{"key":"e_1_3_2_1_9_1","unstructured":"2023. GitHub Copilot - Your AI pair programmer. https:\/\/github.com\/features\/ copilot"},{"key":"e_1_3_2_1_10_1","unstructured":"2023. malloc - cppreference.com. https:\/\/en.cppreference.com\/w\/c\/memory\/ malloc"},{"key":"e_1_3_2_1_11_1","unstructured":"2023. MarkupSafe \u00b7 PyPI. https:\/\/pypi.org\/project\/MarkupSafe"},{"key":"e_1_3_2_1_12_1","unstructured":"2023. Models - Hugging Face. https:\/\/huggingface.co\/models"},{"key":"e_1_3_2_1_13_1","unstructured":"2023. safe_join - Flask API. https:\/\/tedboy.github.io\/flask\/generated\/flask.safe_ join.html"},{"key":"e_1_3_2_1_14_1","unstructured":"2023. The diff-match-patch Library. https:\/\/github.com\/google\/diff-match-patch"},{"key":"e_1_3_2_1_15_1","unstructured":"2023. Wikipedia - Common Weakness Enumeration. https:\/\/en.wikipedia.org\/ wiki\/Common_Weakness_Enumeration"},{"key":"e_1_3_2_1_16_1","unstructured":"2023. Wikipedia - Kullback?Leibler Divergence. https:\/\/en.wikipedia.org\/wiki\/ Kullback%E2%80%93Leibler_divergence"},{"key":"e_1_3_2_1_17_1","volume-title":"Niklas Muennighoff, Mayank Mishra, Alex Gu, Manan Dey, et al.","author":"Allal Loubna Ben","year":"2023","unstructured":"Loubna Ben Allal, Raymond Li, Denis Kocetkov, Chenghao Mou, Christopher Akiki, Carlos Mu n oz Ferrandis, Niklas Muennighoff, Mayank Mishra, Alex Gu, Manan Dey, et al. 2023. SantaCoder: Don't Reach for the Stars! CoRR, Vol. abs\/2301.03988 (2023). https:\/\/arxiv.org\/abs\/2301.03988"},{"key":"e_1_3_2_1_18_1","volume-title":"Program Synthesis with Large Language Models. CoRR","author":"Austin Jacob","year":"2021","unstructured":"Jacob Austin, Augustus Odena, Maxwell I. Nye, Maarten Bosma, Henryk Michalewski, David Dohan, Ellen Jiang, Carrie J. Cai, Michael Terry, Quoc V. Le, and Charles Sutton. 2021. Program Synthesis with Large Language Models. CoRR, Vol. abs\/2108.07732 (2021). https:\/\/arxiv.org\/abs\/2108.07732"},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","DOI":"10.1109\/SP46214.2022.9833659"},{"key":"e_1_3_2_1_20_1","volume-title":"Efficient Training of Language Models to Fill in the Middle. CoRR","author":"Bavarian Mohammad","year":"2022","unstructured":"Mohammad Bavarian, Heewoo Jun, Nikolas Tezak, John Schulman, Christine McLeavey, Jerry Tworek, and Mark Chen. 2022. Efficient Training of Language Models to Fill in the Middle. CoRR, Vol. abs\/2207.14255 (2022). https:\/\/arxiv.org\/abs\/2207.14255"},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","unstructured":"Guru Prasad Bhandari Amara Naseer and Leon Moonen. 2021. CVEfixes: Automated Collection of Vulnerabilities and Their Fixes from Open-source Software. In PROMISE. https:\/\/doi.org\/10.1145\/3475960.3475985","DOI":"10.1145\/3475960.3475985"},{"key":"e_1_3_2_1_22_1","unstructured":"Tom B. Brown Benjamin Mann Nick Ryder Melanie Subbiah Jared Kaplan Prafulla Dhariwal Arvind Neelakantan Pranav Shyam Girish Sastry Amanda Askell et al. 2020. Language Models are Few-Shot Learners. In NeurIPS. https:\/\/proceedings.neurips.cc\/paper\/2020\/hash\/1457c0d6bfcb4967418bfb8ac142f64a-Abstract.html"},{"key":"e_1_3_2_1_23_1","volume-title":"Molly Q Feldman, Arjun Guha, Michael Greenberg, and Abhinav Jangda.","author":"Cassano Federico","year":"2022","unstructured":"Federico Cassano, John Gouwar, Daniel Nguyen, Sydney Nguyen, Luna Phipps-Costin, Donald Pinckney, Ming-Ho Yee, Yangtian Zi, Carolyn Jane Anderson, Molly Q Feldman, Arjun Guha, Michael Greenberg, and Abhinav Jangda. 2022. MultiPL-E: A Scalable and Extensible Approach to Benchmarking Neural Code Generation. CoRR, Vol. abs\/2208.08227 (2022). https:\/\/arxiv.org\/abs\/2208.08227"},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2021.3087402"},{"key":"e_1_3_2_1_25_1","volume-title":"Jared Kaplan, Harrison Edwards, Yuri Burda, Nicholas Joseph, Greg Brockman, et al.","author":"Chen Mark","year":"2021","unstructured":"Mark Chen, Jerry Tworek, Heewoo Jun, Qiming Yuan, Henrique Ponde de Oliveira Pinto, Jared Kaplan, Harrison Edwards, Yuri Burda, Nicholas Joseph, Greg Brockman, et al. 2021. Evaluating Large Language Models Trained on Code. CoRR, Vol. abs\/2107.03374 (2021). https:\/\/arxiv.org\/abs\/2107.03374"},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2022.3147265"},{"key":"e_1_3_2_1_27_1","volume-title":"Charles Sutton, Sebastian Gehrmann, et al.","author":"Chowdhery Aakanksha","year":"2022","unstructured":"Aakanksha Chowdhery, Sharan Narang, Jacob Devlin, Maarten Bosma, Gaurav Mishra, Adam Roberts, Paul Barham, Hyung Won Chung, Charles Sutton, Sebastian Gehrmann, et al. 2022. PaLM: Scaling Language Modeling with Pathways. CoRR, Vol. abs\/2204.02311 (2022). https:\/\/arxiv.org\/abs\/2204.02311"},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE48619.2023.00022"},{"key":"e_1_3_2_1_29_1","unstructured":"Sumanth Dathathri Andrea Madotto Janice Lan Jane Hung Eric Frank Piero Molino Jason Yosinski and Rosanne Liu. 2020. Plug and Play Language Models: A Simple Approach to Controlled Text Generation. In ICLR. https:\/\/openreview.net\/forum?id=H1edEyBKDS"},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/n19--1423"},{"key":"e_1_3_2_1_31_1","unstructured":"Thomas Dohmke. 2023. GitHub Copilot X: the AI-powered Developer Experience. https:\/\/github.blog\/2023-03-22-github-copilot-x-the-ai-powered-developer-experience"},{"key":"e_1_3_2_1_32_1","doi-asserted-by":"publisher","DOI":"10.1109\/SP.2016.15"},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"publisher","DOI":"10.1145\/3379597.3387501"},{"key":"e_1_3_2_1_34_1","unstructured":"Daniel Fried Armen Aghajanyan Jessy Lin Sida Wang Eric Wallace Freda Shi Ruiqi Zhong Wen-tau Yih Luke Zettlemoyer and Mike Lewis. 2023. InCoder: A Generative Model for Code Infilling and Synthesis. In ICLR. https:\/\/arxiv.org\/abs\/2204.05999"},{"key":"e_1_3_2_1_35_1","doi-asserted-by":"publisher","unstructured":"Luca Gazzola Daniela Micucci and Leonardo Mariani. 2018. Automatic Software Repair: a Survey. In ICSE. https:\/\/doi.org\/10.1145\/3180155.3182526","DOI":"10.1145\/3180155.3182526"},{"key":"e_1_3_2_1_36_1","doi-asserted-by":"publisher","DOI":"10.1109\/MS.2021.3072577"},{"key":"e_1_3_2_1_37_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2021.acl-long.381"},{"key":"e_1_3_2_1_38_1","unstructured":"Jingxuan He Luca Beurer-Kellner and Martin Vechev. 2022. On Distribution Shift in Learning-based Bug Detectors. In ICML. https:\/\/proceedings.mlr.press\/v162\/he22a.html"},{"key":"e_1_3_2_1_39_1","doi-asserted-by":"publisher","DOI":"10.1162\/neco.1997.9.8.1735"},{"key":"e_1_3_2_1_40_1","doi-asserted-by":"publisher","DOI":"10.1162\/coli_a_00426"},{"key":"e_1_3_2_1_41_1","unstructured":"Eirini Kalliamvakou. 2022. Research: Quantifying GitHub Copilot's Impact on Developer Productivity and Happiness. https:\/\/github.blog\/2022-09-07-research-quantifying-github-copilots-impact-on-developer-productivity-and-happiness"},{"key":"e_1_3_2_1_42_1","volume-title":"CTRL: a Conditional Transformer Language Model for Controllable Generation. CoRR","author":"Keskar Nitish Shirish","year":"2019","unstructured":"Nitish Shirish Keskar, Bryan McCann, Lav R. Varshney, Caiming Xiong, and Richard Socher. 2019. CTRL: a Conditional Transformer Language Model for Controllable Generation. CoRR, Vol. abs\/1909.05858 (2019). http:\/\/arxiv.org\/abs\/1909.05858"},{"key":"e_1_3_2_1_43_1","volume-title":"How Secure is Code Generated by ChatGPT? CoRR","author":"Khoury Rapha\u00ebl","year":"2023","unstructured":"Rapha\u00ebl Khoury, Anderson R. Avila, Jacob Brunelle, and Baba Mamadou Camara. 2023. How Secure is Code Generated by ChatGPT? CoRR, Vol. abs\/2304.09655 (2023). https:\/\/arxiv.org\/abs\/2304.09655"},{"key":"e_1_3_2_1_44_1","volume-title":"Marvin Zhang, Akshay Balsubramani, Weihua Hu, Michihiro Yasunaga, Richard Lanas Phillips, Irena Gao, et al.","author":"Koh Pang Wei","year":"2021","unstructured":"Pang Wei Koh, Shiori Sagawa, Henrik Marklund, Sang Michael Xie, Marvin Zhang, Akshay Balsubramani, Weihua Hu, Michihiro Yasunaga, Richard Lanas Phillips, Irena Gao, et al. 2021. WILDS: A Benchmark of in-the-Wild Distribution Shifts. In ICML. http:\/\/proceedings.mlr.press\/v139\/koh21a.html"},{"key":"e_1_3_2_1_45_1","unstructured":"Tomasz Korbak Hady Elsahar Germ\u00e1n Kruszewski and Marc Dymetman. 2022. Controlling Conditional Language Models without Catastrophic Forgetting. In ICML Kamalika Chaudhuri Stefanie Jegelka Le Song Csaba Szepesv\u00e1ri Gang Niu and Sivan Sabato (Eds.). https:\/\/proceedings.mlr.press\/v162\/korbak22a.html"},{"key":"e_1_3_2_1_46_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2021.findings-emnlp.424"},{"key":"e_1_3_2_1_47_1","volume-title":"Silvio Savarese, and Steven Chu-Hong Hoi.","author":"Le Hung","year":"2022","unstructured":"Hung Le, Yue Wang, Akhilesh Deepak Gotmare, Silvio Savarese, and Steven Chu-Hong Hoi. 2022. CodeRL: Mastering Code Generation through Pretrained Models and Deep Reinforcement Learning. In NeurIPS. http:\/\/papers.nips.cc\/paper_files\/paper\/2022\/hash\/8636419dea1aa9fbd25fc4248e702da4-Abstract-Conference.html"},{"key":"e_1_3_2_1_48_1","doi-asserted-by":"publisher","unstructured":"Brian Lester Rami Al-Rfou and Noah Constant. 2021. The Power of Scale for Parameter-Efficient Prompt Tuning. In EMNLP. https:\/\/doi.org\/10.18653\/v1\/2021.emnlp-main.243","DOI":"10.18653\/v1\/2021.emnlp-main.243"},{"key":"e_1_3_2_1_49_1","doi-asserted-by":"publisher","unstructured":"Xiang Lisa Li and Percy Liang. 2021. Prefix-Tuning: Optimizing Continuous Prompts for Generation. In ACL\/IJCNLP Chengqing Zong Fei Xia Wenjie Li and Roberto Navigli (Eds.). https:\/\/doi.org\/10.18653\/v1\/2021.acl-long.353","DOI":"10.18653\/v1\/2021.acl-long.353"},{"key":"e_1_3_2_1_50_1","volume-title":"Tom Eccles, James Keeling, Felix Gimeno, Agustin Dal Lago, et al.","author":"Li Yujia","year":"2022","unstructured":"Yujia Li, David H. Choi, Junyoung Chung, Nate Kushman, Julian Schrittwieser, R\u00e9 mi Leblond, Tom Eccles, James Keeling, Felix Gimeno, Agustin Dal Lago, et al. 2022a. Competition-Level Code Generation with AlphaCode. CoRR, Vol. abs\/2203.07814 (2022). https:\/\/arxiv.org\/abs\/2203.07814"},{"key":"e_1_3_2_1_51_1","doi-asserted-by":"publisher","DOI":"10.1109\/TDSC.2021.3051525"},{"key":"e_1_3_2_1_52_1","volume-title":"NDSS","author":"Li Zhen","unstructured":"Zhen Li, Deqing Zou, Shouhuai Xu, Xinyu Ou, Hai Jin, Sujuan Wang, Zhijun Deng, and Yuyi Zhong. 2018. VulDeePecker: A Deep Learning-Based System for Vulnerability Detection. In NDSS. http:\/\/wp.internetsociety.org\/ndss\/wp-content\/uploads\/sites\/25\/2018\/02\/ndss2018_03A-2_Li_paper.pdf"},{"key":"e_1_3_2_1_53_1","doi-asserted-by":"publisher","DOI":"10.1109\/JPROC.2020.2993293"},{"key":"e_1_3_2_1_54_1","volume-title":"abs\/2103.10385","author":"Liu Xiao","year":"2021","unstructured":"Xiao Liu, Yanan Zheng, Zhengxiao Du, Ming Ding, Yujie Qian, Zhilin Yang, and Jie Tang. 2021. GPT Understands, Too. CoRR, Vol. abs\/2103.10385 (2021). https:\/\/arxiv.org\/abs\/2103.10385"},{"key":"e_1_3_2_1_55_1","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2019.2946563"},{"key":"e_1_3_2_1_56_1","unstructured":"Erik Nijkamp Bo Pang Hiroaki Hayashi Lifu Tu Huan Wang Yingbo Zhou Silvio Savarese and Caiming Xiong. 2023. CodeGen: An Open Large Language Model for Code with Multi-Turn Program Synthesis. In ICLR. https:\/\/arxiv.org\/abs\/2203.13474"},{"key":"e_1_3_2_1_57_1","doi-asserted-by":"publisher","unstructured":"Georgios Nikitopoulos Konstantina Dritsa Panos Louridas and Dimitris Mitropoulos. 2021. CrossVul: a Cross-language Vulnerability Dataset with Commit Data. In ESEC\/FSE. https:\/\/doi.org\/10.1145\/3468264.3473122","DOI":"10.1145\/3468264.3473122"},{"key":"e_1_3_2_1_58_1","doi-asserted-by":"publisher","unstructured":"Yu Nong Yuzhe Ou Michael Pradel Feng Chen and Haipeng Cai. 2022. Generating Realistic Vulnerabilities via Neural Code Editing: an Empirical Study. In ESEC\/FSE. https:\/\/doi.org\/10.1145\/3540250.3549128","DOI":"10.1145\/3540250.3549128"},{"key":"e_1_3_2_1_59_1","doi-asserted-by":"publisher","DOI":"10.1109\/SP46214.2022.9833571"},{"key":"e_1_3_2_1_60_1","volume-title":"Examining Zero-Shot Vulnerability Repair with Large Language Models","author":"Pearce Hammond","year":"2023","unstructured":"Hammond Pearce, Benjamin Tan, Baleegh Ahmad, Ramesh Karri, and Brendan Dolan-Gavitt. 2023. Examining Zero-Shot Vulnerability Repair with Large Language Models. In IEEE S&P. https:\/\/doi.ieeecomputersociety.org\/10.1109\/SP46215.2023.00001"},{"key":"e_1_3_2_1_61_1","doi-asserted-by":"publisher","unstructured":"Jing Qian Li Dong Yelong Shen Furu Wei and Weizhu Chen. 2022. Controllable Natural Language Generation with Contrastive Prefixes. In Findings of ACL. https:\/\/doi.org\/10.18653\/v1\/2022.findings-acl.229","DOI":"10.18653\/v1\/2022.findings-acl.229"},{"key":"e_1_3_2_1_62_1","doi-asserted-by":"publisher","unstructured":"Guanghui Qin and Jason Eisner. 2021. Learning How to Ask: Querying LMs with Mixtures of Soft Prompts. In NAACL. https:\/\/doi.org\/10.18653\/v1\/2021.naacl-main.410","DOI":"10.18653\/v1\/2021.naacl-main.410"},{"key":"e_1_3_2_1_63_1","unstructured":"Alec Radford Jeff Wu Rewon Child David Luan Dario Amodei and Ilya Sutskever. 2019. Language Models are Unsupervised Multitask Learners. (2019). https:\/\/d4mucfpksywv.cloudfront.net\/better-language-models\/language-models.pdf"},{"key":"e_1_3_2_1_64_1","volume-title":"A Ground-truth Dataset of Real Security Patches. CoRR","author":"Reis Sofia","year":"2021","unstructured":"Sofia Reis and Rui Abreu. 2021. A Ground-truth Dataset of Real Security Patches. CoRR, Vol. abs\/2110.09635 (2021). https:\/\/arxiv.org\/abs\/2110.09635"},{"key":"e_1_3_2_1_65_1","unstructured":"Gustavo Sandoval Hammond Pearce Teo Nys Ramesh Karri Siddharth Garg and Brendan Dolan-Gavitt. 2023. Lost at C: A User Study on the Security Implications of Large Language Model Code Assistants. In USENIX Security. https:\/\/www.usenix.org\/conference\/usenixsecurity23\/presentation\/sandoval"},{"key":"e_1_3_2_1_66_1","unstructured":"Roei Schuster Congzheng Song Eran Tromer and Vitaly Shmatikov. 2021. You Autocomplete Me: Poisoning Vulnerabilities in Neural Code Completion. In USENIX Security. https:\/\/www.usenix.org\/conference\/usenixsecurity21\/presentation\/schuster"},{"key":"e_1_3_2_1_67_1","doi-asserted-by":"publisher","DOI":"10.1145\/3549035.3561184"},{"key":"e_1_3_2_1_68_1","unstructured":"John Smith. 2023. StarCoder: May the source be with you! https:\/\/drive.google.com\/file\/d\/1cN-b9GnWtHzQRoE7M7gAEyivY0kl4BYs\/view?usp=sharing."},{"key":"e_1_3_2_1_69_1","doi-asserted-by":"publisher","unstructured":"Justin Smith Brittany Johnson Emerson R. Murphy-Hill Bill Chu and Heather Richter Lipford. 2015. Questions Developers Ask While Diagnosing Potential Security Vulnerabilities with Static Analysis. In ESEC\/FSE. https:\/\/doi.org\/10.1145\/2786805.2786812","DOI":"10.1145\/2786805.2786812"},{"key":"e_1_3_2_1_70_1","doi-asserted-by":"publisher","unstructured":"Zhensu Sun Xiaoning Du Fu Song Mingze Ni and Li Li. 2022. CoProtector: Protect Open-Source Code against Unauthorized Training Usage with Data Poisoning. In WWW. https:\/\/doi.org\/10.1145\/3485447.3512225","DOI":"10.1145\/3485447.3512225"},{"key":"e_1_3_2_1_71_1","unstructured":"Maxim Tabachnyk and Stoyan Nikolov. 2022. ML-Enhanced Code Completion Improves Developer Productivity. https:\/\/ai.googleblog.com\/2022\/07\/ml-enhanced-code-completion-improves.html"},{"key":"e_1_3_2_1_72_1","doi-asserted-by":"publisher","DOI":"10.1145\/3491101.3519665"},{"key":"e_1_3_2_1_73_1","unstructured":"Ashish Vaswani Noam Shazeer Niki Parmar Jakob Uszkoreit Llion Jones Aidan N. Gomez Lukasz Kaiser and Illia Polosukhin. 2017. Attention is All you Need. In NeurIPS. https:\/\/proceedings.neurips.cc\/paper\/2017\/hash\/3f5ee243547dee91fbd053c1c4a845aa-Abstract.html"},{"key":"e_1_3_2_1_74_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2021.emnlp-main.685"},{"key":"e_1_3_2_1_75_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.infsof.2021.106809"},{"key":"e_1_3_2_1_76_1","doi-asserted-by":"publisher","unstructured":"Frank F. Xu Uri Alon Graham Neubig and Vincent Josua Hellendoorn. 2022. A Systematic Evaluation of Large Language Models of Code. In MAPS@PLDI. https:\/\/doi.org\/10.1145\/3520312.3534862","DOI":"10.1145\/3520312.3534862"},{"key":"e_1_3_2_1_77_1","volume-title":"FIXREVERTER: A Realistic Bug Injection Methodology for Benchmarking Fuzz Testing. In USENIX Security. https:\/\/www.usenix.org\/conference\/usenixsecurity22\/presentation\/zhang-zenong","author":"Zhang Zenong","year":"2022","unstructured":"Zenong Zhang, Zach Patterson, Michael Hicks, and Shiyi Wei. 2022. FIXREVERTER: A Realistic Bug Injection Methodology for Benchmarking Fuzz Testing. In USENIX Security. https:\/\/www.usenix.org\/conference\/usenixsecurity22\/presentation\/zhang-zenong"},{"key":"e_1_3_2_1_78_1","unstructured":"Shuyin Zhao. 2023. GitHub Copilot Now Has a Better AI Model and New Capabilities. https:\/\/github.blog\/2023-02-14-github-copilot-now-has-a-better-ai-model-and-new-capabilities"},{"key":"e_1_3_2_1_79_1","volume-title":"Xiaoning Du, and Yang Liu.","author":"Zhou Yaqin","year":"2019","unstructured":"Yaqin Zhou, Shangqing Liu, Jing Kai Siow, Xiaoning Du, and Yang Liu. 2019. Devign: Effective Vulnerability Identification by Learning Comprehensive Program Semantics via Graph Neural Networks. In NeurIPS. https:\/\/proceedings.neurips.cc\/paper\/2019\/hash\/49265d2447bc3bbfe9e76306ce40a31f-Abstract.html"}],"event":{"name":"CCS '23: ACM SIGSAC Conference on Computer and Communications Security","location":"Copenhagen Denmark","acronym":"CCS '23","sponsor":["SIGSAC ACM Special Interest Group on Security, Audit, and Control"]},"container-title":["Proceedings of the 2023 ACM SIGSAC Conference on Computer and Communications Security"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3576915.3623175","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3576915.3623175","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,8,21]],"date-time":"2025-08-21T01:49:11Z","timestamp":1755740951000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3576915.3623175"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023,11,15]]},"references-count":79,"alternative-id":["10.1145\/3576915.3623175","10.1145\/3576915"],"URL":"https:\/\/doi.org\/10.1145\/3576915.3623175","relation":{},"subject":[],"published":{"date-parts":[[2023,11,15]]},"assertion":[{"value":"2023-11-21","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}