{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,2,24]],"date-time":"2026-02-24T18:08:53Z","timestamp":1771956533280,"version":"3.50.1"},"reference-count":40,"publisher":"Springer Science and Business Media LLC","issue":"5","license":[{"start":{"date-parts":[[2025,8,11]],"date-time":"2025-08-11T00:00:00Z","timestamp":1754870400000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2025,8,11]],"date-time":"2025-08-11T00:00:00Z","timestamp":1754870400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Empir Software Eng"],"published-print":{"date-parts":[[2025,9]]},"DOI":"10.1007\/s10664-025-10693-3","type":"journal-article","created":{"date-parts":[[2025,8,11]],"date-time":"2025-08-11T12:54:37Z","timestamp":1754916877000},"update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":1,"title":["Adversarial attack classification and robustness testing for large language models for code"],"prefix":"10.1007","volume":"30","author":[{"ORCID":"https:\/\/orcid.org\/0009-0004-2102-1756","authenticated-orcid":false,"given":"Yang","family":"Liu","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Armstrong","family":"Foundjem","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Foutse","family":"Khomh","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Heng","family":"Li","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2025,8,11]]},"reference":[{"key":"10693_CR1","unstructured":"Adversarial attack classification and robustness testing for code generation models. Empir Softw Eng 38. https:\/\/www.dropbox.com\/scl\/fo\/1zmopkdwx2s4o73j6lr1o\/ALiXzdU8vI03OMepNYBVbBg?rlkey=6uax4qc1tqzx5trl0nso41p2c&st=768p2bqo &dl=0"},{"key":"10693_CR2","unstructured":"Alon U, Brody S, Levy O, Yahav E (2018) code2seq: Generating sequences from structured representations of code. arXiv preprint arXiv:1808.01400"},{"key":"10693_CR3","doi-asserted-by":"crossref","unstructured":"Alon U, Zilberstein M, Levy O, Yahav E (2019) code2vec: Learning distributed representations of code. Proceedings of the ACM on Programming Languages 3(POPL):1","DOI":"10.1145\/3290353"},{"key":"10693_CR4","unstructured":"Anand M, Kayal P, Singh M (2021) On adversarial robustness of synthetic code generation, arXiv preprint arXiv:2106.11629"},{"key":"10693_CR5","unstructured":"Austin J, Odena A, Nye M, Bosma M, Michalewski H, Dohan D, Jiang E, Cai C, Terry M, Le Q et\u00a0al (2021) Program synthesis with large language models. arXiv preprint arXiv:2108.07732"},{"key":"10693_CR6","doi-asserted-by":"crossref","unstructured":"Batouta ZI, Dehbi R, Talea M, Hajoui O (2016) In 2016 4th IEEE international colloquium on information science and technology (CiSt), IEEE, pp 200\u2013205","DOI":"10.1109\/CIST.2016.7805042"},{"key":"10693_CR7","unstructured":"Bielik P, Vechev M (2020) In international conference on machine learning, PMLR, pp 896\u2013907"},{"key":"10693_CR8","unstructured":"Cao J, Chen Z, Wu J, Cheung S, Xu C (2024) Can ai beat undergraduates in entry-level java assignments? benchmarking large language models on javabench, arXiv e-prints pp. arXiv\u20132406"},{"key":"10693_CR9","doi-asserted-by":"crossref","unstructured":"Carlini N, Wagner D (2017) In 2017 IEEE symposium on security and privacy (sp), IEEE, pp 39\u201357","DOI":"10.1109\/SP.2017.49"},{"key":"10693_CR10","unstructured":"Chen M, Tworek J, Jun H, Yuan Q, Pinto HPO, Kaplan J, Edwards H, Burda Y, Joseph N, Brockman G et\u00a0al (2021) Evaluating large language models trained on code. arXiv preprint arXiv:2107.03374"},{"key":"10693_CR11","unstructured":"De\u00a0la Rosa J, Fern\u00e1ndez A (2022) In IberLEF@ SEPLN"},{"key":"10693_CR12","unstructured":"Ernst MD (2017) In 2nd summit on advances in programming languages (SNAPL), (Schloss-Dagstuhl-Leibniz Zentrum f\u00fcr Informatik)"},{"key":"10693_CR13","doi-asserted-by":"crossref","unstructured":"Feng Z, Guo D, Tang D, Duan N, Feng X, Gong M, Shou L, Qin B, Liu T, Jiang D et\u00a0al (2020) Codebert: A pre-trained model for programming and natural languages. arXiv preprint arXiv:2002.08155","DOI":"10.18653\/v1\/2020.findings-emnlp.139"},{"key":"10693_CR14","unstructured":"Fried D, Aghajanyan A, Lin J, Wang S, Wallace E, Shi F, Zhong R, Yih W, Zettlemoyer L, Lewis M (2022) Incoder: A generative model for code infilling and synthesis. arXiv preprint arXiv:2204.05999"},{"key":"10693_CR15","doi-asserted-by":"crossref","unstructured":"Fronza I, Hellas A, Ihantola P, Mikkonen T (2020) In Software quality: Quality intelligence in software and systems engineering: 12th international conference, SWQD 2020, Vienna, Austria, January 14\u201317, 2020, Proceedings 12, Springer, pp 121\u2013133","DOI":"10.1007\/978-3-030-35510-4_8"},{"key":"10693_CR16","doi-asserted-by":"crossref","unstructured":"Garg S, Ramakrishnan G (2020) Bae: Bert-based adversarial examples for text classification. arXiv preprint arXiv:2004.01970","DOI":"10.18653\/v1\/2020.emnlp-main.498"},{"key":"10693_CR17","unstructured":"Guo D, Ren S, Lu S, Feng Z, Tang D, Liu S, Zhou L, Duan N, Svyatkovskiy A, Fu S et\u00a0al (2020) Graphcodebert: Pre-training code representations with data flow, arXiv preprint arXiv:2009.08366"},{"key":"10693_CR18","unstructured":"Hendrycks D, Basart S, Kadavath S, Mazeika M, Arora A, Guo E, Burns C, Puranik S, He H, Song D et\u00a0al (2021) Measuring coding challenge competence with apps (2021). arXiv preprint arXiv:2105.09938"},{"key":"10693_CR19","unstructured":"Huang W, Ma X, Qin H, Zheng X, Lv C, Chen H, Luo J, Qi X, Liu X, Magno M (2024) How good are low-bit quantized llama3 models? an empirical study, arXiv preprint arXiv:2404.14047"},{"issue":"1","key":"10693_CR20","doi-asserted-by":"publisher","first-page":"7","DOI":"10.1007\/s10664-024-10569-y","volume":"30","author":"C Improta","year":"2025","unstructured":"Improta C, Liguori P, Natella R, Cukic B, Cotroneo D (2025) Enhancing robustness of ai offensive code generators via data augmentation. Empir Softw Eng 30(1):7","journal-title":"Empir Softw Eng"},{"key":"10693_CR21","unstructured":"Jha A, Reddy CK (2023) in Proceedings of the AAAI conference on artificial intelligence, vol\u00a037, pp 14,892\u201314,900"},{"key":"10693_CR22","doi-asserted-by":"crossref","unstructured":"Jia R, Liang P (2017) Adversarial examples for evaluating reading comprehension systems. arXiv preprint arXiv:1707.07328","DOI":"10.18653\/v1\/D17-1215"},{"key":"10693_CR23","unstructured":"Juristo N, Moreno AM (2013) Basics of software engineering experimentation, Springer Sci Bus Media"},{"key":"10693_CR24","unstructured":"Kumar SS, Adam\u00a0Lones M, Maarek M, Zantout H (2024) In Proceedings of the 1st international workshop on large language models for code, pp88\u201393"},{"key":"10693_CR25","doi-asserted-by":"crossref","unstructured":"Le T, Lee J, Yen K, Hu Y, Lee D (2022) Perturbations in the wild: Leveraging human-written text perturbations for realistic adversarial attack and defense. arXiv preprint arXiv:2203.10346","DOI":"10.18653\/v1\/2022.findings-acl.232"},{"issue":"1","key":"10693_CR26","doi-asserted-by":"publisher","first-page":"355","DOI":"10.1137\/20M1321620","volume":"60","author":"Y Lin","year":"2022","unstructured":"Lin Y, Ren Z, Touzi N, Yang J (2022) Random horizon principal-agent problems. SIAM J Control Optim 60(1):355","journal-title":"SIAM J Control Optim"},{"key":"10693_CR27","doi-asserted-by":"crossref","unstructured":"Mastropaolo A, Pascarella L, Guglielmi E, Ciniselli M, Scalabrino S, Oliveto R, Bavota G (2023) In 2023 IEEE\/ACM 45th international conference on software engineering (ICSE), IEEE, pp 2149\u20132160","DOI":"10.1109\/ICSE48619.2023.00181"},{"key":"10693_CR28","unstructured":"Metzen JH, Genewein T, Fischer V, Bischoff B (2017) On detecting adversarial perturbations. arXiv preprint arXiv:1702.04267 (2017)"},{"key":"10693_CR29","unstructured":"Nijkamp E, Pang B, Hayashi H, Tu L, Wang H, Zhou Y, Savarese S, Xiong C (2022) Codegen: An open large language model for code with multi-turn program synthesis. arXiv preprint arXiv:2203.13474"},{"key":"10693_CR30","doi-asserted-by":"crossref","unstructured":"Peng J, Cui L, Huang K, Yang J, Ray B (2025) Cweval: Outcome-driven evaluation on functionality and security of llm code generation, arXiv preprint arXiv:2501.08200","DOI":"10.1109\/LLM4Code66737.2025.00009"},{"issue":"1","key":"10693_CR31","first-page":"5485","volume":"21","author":"C Raffel","year":"2020","unstructured":"Raffel C, Shazeer N, Roberts A, Lee K, Narang S, Matena M, Zhou Y, Li W, Liu PJ (2020) Exploring the limits of transfer learning with a unified text-to-text transformer. J Mach Learn Res 21(1):5485","journal-title":"J Mach Learn Res"},{"key":"10693_CR32","doi-asserted-by":"publisher","DOI":"10.1016\/j.cose.2023.103622","volume":"137","author":"S Rizvi","year":"2024","unstructured":"Rizvi S, Williams I (2024) Analyzing transparency and malicious insiders prevention for cloud computing environment. Comput Secur 137:103622","journal-title":"Comput Secur"},{"key":"10693_CR33","unstructured":"Roziere B, Gehring J, Gloeckle F, Sootla S, Gat I, Tan XE, Adi Y, Liu J, Remez T, Rapin J et\u00a0al (2023) Code llama: Open foundation models for code. arXiv preprint arXiv:2308.12950"},{"key":"10693_CR34","unstructured":"Szegedy C, Zaremba W, Sutskever I, Bruna J, Erhan D, Goodfellow I, Fergus R (2013) Intriguing properties of neural networks. arXiv preprint arXiv:1312.6199"},{"key":"10693_CR35","doi-asserted-by":"crossref","unstructured":"Valeev A, Garaev R, Lomshakov V, Piontkovskaya I, Ivanov V, Adewuyi I (2025) Yabloco: Yet another benchmark for long context code generation, arXiv preprint arXiv:2505.04406","DOI":"10.1109\/LLM4Code66737.2025.00016"},{"key":"10693_CR36","doi-asserted-by":"crossref","unstructured":"Wang S, Li Z, Qian H, Yang C, Wang Z, Shang M, Kumar V, Tan S, Ray B, Bhatia P et\u00a0al (2022) Recode: Robustness evaluation of code generation models. arXiv preprint arXiv:2212.10264","DOI":"10.18653\/v1\/2023.acl-long.773"},{"key":"10693_CR37","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-29044-2","volume-title":"Experimentation in software engineering, vol236","author":"C Wohlin","year":"2012","unstructured":"Wohlin C, Runeson P, H\u00f6st M, Ohlsson MC, Regnell B, Wessl\u00e9n A et al (2012) Experimentation in software engineering, vol236. Springer"},{"key":"10693_CR38","doi-asserted-by":"crossref","unstructured":"Yang Z, Shi J, He J, Lo D (2022) In Proceedings of the 44th international conference on software engineering, pp 1482\u20131493","DOI":"10.1145\/3510003.3510146"},{"issue":"9","key":"10693_CR39","doi-asserted-by":"publisher","first-page":"2805","DOI":"10.1109\/TNNLS.2018.2886017","volume":"30","author":"X Yuan","year":"2019","unstructured":"Yuan X, He P, Zhu Q, Li X (2019) Adversarial examples: Attacks and defenses for deep learning. IEEE Trans Neural Netw Learn Syst 30(9):2805","journal-title":"IEEE Trans Neural Netw Learn Syst"},{"key":"10693_CR40","doi-asserted-by":"crossref","unstructured":"Zeng G, Qi F, Zhou Q, Zhang T, Ma Z, Hou B, Zang Y, Liu Z, Sun M (2020) Openattack: An open-source textual adversarial attack toolkit. arXiv preprint arXiv:2009.09191","DOI":"10.18653\/v1\/2021.acl-demo.43"}],"container-title":["Empirical Software Engineering"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10664-025-10693-3.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s10664-025-10693-3\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10664-025-10693-3.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,9,13]],"date-time":"2025-09-13T08:54:26Z","timestamp":1757753666000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s10664-025-10693-3"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,8,11]]},"references-count":40,"journal-issue":{"issue":"5","published-print":{"date-parts":[[2025,9]]}},"alternative-id":["10693"],"URL":"https:\/\/doi.org\/10.1007\/s10664-025-10693-3","relation":{},"ISSN":["1382-3256","1573-7616"],"issn-type":[{"value":"1382-3256","type":"print"},{"value":"1573-7616","type":"electronic"}],"subject":[],"published":{"date-parts":[[2025,8,11]]},"assertion":[{"value":"19 June 2025","order":1,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"11 August 2025","order":2,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Declarations"}},{"value":"This study did not involve any human participants or animals and therefore did not require ethical approval.","order":2,"name":"Ethics","group":{"name":"EthicsHeading","label":"Ethical Approval"}},{"value":"Written informed consent was obtained from all individual participants included in the study.","order":3,"name":"Ethics","group":{"name":"EthicsHeading","label":"Informed Consent"}},{"value":"The authors declared that they have no conflict of interest.","order":4,"name":"Ethics","group":{"name":"EthicsHeading","label":"Conflict of interests\/Competing interests"}},{"value":"not applicable.","order":5,"name":"Ethics","group":{"name":"EthicsHeading","label":"Clinical Trial Number"}}],"article-number":"154"}}