{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,9]],"date-time":"2026-10-09T06:52:58Z","timestamp":1791528778881,"version":"4.3.4"},"reference-count":85,"publisher":"Springer Science and Business Media LLC","issue":"1","license":[{"start":{"date-parts":[[2024,10,19]],"date-time":"2024-10-19T00:00:00Z","timestamp":1729296000000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2024,10,19]],"date-time":"2024-10-19T00:00:00Z","timestamp":1729296000000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Empir Software Eng"],"published-print":{"date-parts":[[2025,1]]},"DOI":"10.1007\/s10664-024-10569-y","type":"journal-article","created":{"date-parts":[[2024,10,19]],"date-time":"2024-10-19T11:02:30Z","timestamp":1729335750000},"update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":10,"title":["Enhancing robustness of AI offensive code generators via data augmentation"],"prefix":"10.1007","volume":"30","author":[{"ORCID":"https:\/\/orcid.org\/0000-0001-5300-2358","authenticated-orcid":false,"given":"Cristina","family":"Improta","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-5579-1696","authenticated-orcid":false,"given":"Pietro","family":"Liguori","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-1084-4824","authenticated-orcid":false,"given":"Roberto","family":"Natella","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-7130-9054","authenticated-orcid":false,"given":"Bojan","family":"Cukic","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-7103-592X","authenticated-orcid":false,"given":"Domenico","family":"Cotroneo","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2024,10,19]]},"reference":[{"key":"10569_CR1","doi-asserted-by":"crossref","unstructured":"Ahmed T, Pai KS, Devanbu P, Barr E (2024) Automatic semantic augmentation of language model prompts (for code summarization). In: Proceedings of the IEEE\/ACM 46th international conference on software engineering, pp 1\u201313","DOI":"10.1145\/3597503.3639183"},{"key":"10569_CR2","unstructured":"Akbik A, Blythe D, Vollgraf R (2018) Contextual string embeddings for sequence labeling. In: Proceedings of the 27th international conference on computational linguistics, COLING 2018, Santa Fe, New Mexico, USA, August 20-26, 2018, pp 1638\u20131649. Association for Computational Linguistics. https:\/\/aclanthology.org\/C18-1139\/"},{"key":"10569_CR3","doi-asserted-by":"publisher","unstructured":"Alzantot M, Sharma Y, Elgohary A, Ho B, Srivastava MB, Chang K (2018) Generating natural language adversarial examples. In: Riloff E,\u00a0Chiang D,\u00a0Hockenmaier J,\u00a0Tsujii J (Eds.) Proceedings of the 2018 conference on empirical methods in natural language processing, Brussels, Belgium, October 31 - November 4, 2018, pp 2890\u20132896. Association for Computational Linguistics. https:\/\/doi.org\/10.18653\/v1\/d18-1316","DOI":"10.18653\/v1\/d18-1316"},{"issue":"2","key":"10569_CR4","doi-asserted-by":"publisher","first-page":"74","DOI":"10.1145\/2560217.2560219","volume":"57","author":"T Avgerinos","year":"2014","unstructured":"Avgerinos T, Cha SK, Rebert A, Schwartz EJ, Woo M, Brumley D (2014) Automatic exploit generation. Commun ACM 57(2):74\u201384","journal-title":"Commun ACM"},{"key":"10569_CR5","unstructured":"Bahdanau D, Cho K, Bengio Y (2015) Neural machine translation by jointly learning to align and translate. In: 3rd International conference on learning representations, ICLR 2015, San Diego, CA, USA, May 7-9, 2015, Conference Track Proceedings. http:\/\/arxiv.org\/abs\/1409.0473"},{"key":"10569_CR6","unstructured":"Belinkov Y, Bisk Y (2018) Synthetic and natural noise both break neural machine translation. In: 6th International conference on learning representations, ICLR 2018, Vancouver, BC, Canada, April 30 - May 3, 2018, Conference Track Proceedings. OpenReview.net. https:\/\/openreview.net\/forum?id=BJ8vJebC-"},{"key":"10569_CR7","doi-asserted-by":"crossref","unstructured":"Bird S (2006) Nltk: the natural language toolkit. In: Proceedings of the COLING\/ACL 2006 interactive presentation sessions, pp 69\u201372","DOI":"10.3115\/1225403.1225421"},{"key":"10569_CR8","doi-asserted-by":"crossref","unstructured":"Botacin M (2023) Gpthreats-3: is automatic malware generation a threat? In: 2023 IEEE Security and privacy workshops (SPW), IEEE, pp 238\u2013254","DOI":"10.1109\/SPW59333.2023.00027"},{"key":"10569_CR9","doi-asserted-by":"publisher","unstructured":"Boucher N, Shumailov I, Anderson R, Papernot N (2022) Bad characters: imperceptible nlp attacks. In: 2022 IEEE Symposium on security and privacy (SP), pp 1987\u20132004. https:\/\/doi.org\/10.1109\/SP46214.2022.9833641","DOI":"10.1109\/SP46214.2022.9833641"},{"key":"10569_CR10","doi-asserted-by":"crossref","unstructured":"Cheng M, Yi J, Chen PY, Zhang H, Hsieh CJ (2020) Seq2sick: evaluating the robustness of sequence-to-sequence models with adversarial examples. In: Proceedings of the AAAI conference on artificial intelligence, vol.\u00a034, pp 3601\u20133608","DOI":"10.1609\/aaai.v34i04.5767"},{"issue":"2","key":"10569_CR11","doi-asserted-by":"publisher","first-page":"251","DOI":"10.1016\/j.system.2003.11.008","volume":"32","author":"TM Chung","year":"2004","unstructured":"Chung TM, Nation P (2004) Identifying technical vocabulary. System 32(2):251\u2013263","journal-title":"System"},{"key":"10569_CR12","doi-asserted-by":"publisher","unstructured":"Cotroneo D, Foggia A, Improta C, Liguori P, Natella R (2024) Automating the correctness assessment of ai-generated code for security contexts. J Syst Soft 216, 112:113. https:\/\/doi.org\/10.1016\/j.jss.2024.112113. https:\/\/www.sciencedirect.com\/science\/article\/pii\/S0164121224001584","DOI":"10.1016\/j.jss.2024.112113"},{"key":"10569_CR13","unstructured":"Exploit-db (2023) Exploit Database Shellcodes. https:\/\/www.exploit-db.com\/shellcodes?platform=linux_x86\/"},{"key":"10569_CR14","doi-asserted-by":"publisher","unstructured":"Feng SY, Gangal V, Wei J, Chandar S, Vosoughi S, Mitamura T, Hovy E (2021) A survey of data augmentation approaches for NLP. In:\u00a0Zong C,\u00a0Xia F,\u00a0Li W,\u00a0Navigli R (eds.) Findings of the association for Computational Linguistics: ACL-IJCNLP 2021, pp 968\u2013988. Association for Computational Linguistics, Online. https:\/\/doi.org\/10.18653\/v1\/2021.findings-acl.84. https:\/\/aclanthology.org\/2021.findings-acl.84","DOI":"10.18653\/v1\/2021.findings-acl.84"},{"key":"10569_CR15","doi-asserted-by":"publisher","unstructured":"Feng Z, Guo D, Tang D, Duan N, Feng X, Gong M, Shou L, Qin B, Liu T, Jiang D, Zhou M (2020) Codebert: A pre-trained model for programming and natural languages. In: Findings of the association for computational linguistics: EMNLP 2020, online event, 16-20 November 2020, Findings of ACL, vol EMNLP 2020, pp 1536\u20131547. Association for computational linguistics. https:\/\/doi.org\/10.18653\/v1\/2020.findings-emnlp.139. https:\/\/doi.org\/10.18653\/v1\/2020.findings-emnlp.139","DOI":"10.18653\/v1\/2020.findings-emnlp.139"},{"key":"10569_CR16","doi-asserted-by":"crossref","unstructured":"Foster J (2005) Sockets, Shellcode, Porting, and Coding: Reverse Engineering Exploits and Tool Coding for Security Professionals. Elsevier Sci. https:\/\/books.google.it\/books?id=ZNI5dvBSfZoC","DOI":"10.1016\/B978-159749005-4\/50016-X"},{"key":"10569_CR17","doi-asserted-by":"crossref","unstructured":"Gao F, Zhu J, Wu L, Xia Y, Qin T, Cheng X, Zhou W, Liu TY (2019) Soft contextual data augmentation for neural machine translation. In: Proceedings of the 57th annual meeting of the association for computational linguistics, pp 5539\u20135544","DOI":"10.18653\/v1\/P19-1555"},{"key":"10569_CR18","doi-asserted-by":"publisher","unstructured":"Gao S, Zhang H, Gao C, Wang C (2023) Keeping pace with ever-increasing data: towards continual learning of code intelligence models. In: 45th IEEE\/ACM International conference on software engineering, ICSE 2023, Melbourne, Australia, May 14-20, 2023, pp 30\u201342. IEEE (2023). https:\/\/doi.org\/10.1109\/ICSE48619.2023.00015. https:\/\/doi.org\/10.1109\/ICSE48619.2023.00015","DOI":"10.1109\/ICSE48619.2023.00015"},{"key":"10569_CR19","doi-asserted-by":"crossref","unstructured":"Gupta M, Akiri C, Aryal K, Parker E, Praharaj L (2023) From chatgpt to threatgpt: impact of generative ai in cybersecurity and privacy. IEEE Access","DOI":"10.1109\/ACCESS.2023.3300381"},{"key":"10569_CR20","unstructured":"Han L, Smeaton A, Jones G (2021) Translation quality assessment: a brief survey on manual and automatic methods. In: Proceedings for the first workshop on modelling translation: translatology in the digital age, pp 15\u201333. Association for Computational Linguistics, online. https:\/\/aclanthology.org\/2021.motra-1.3"},{"key":"10569_CR21","unstructured":"Hauser J, Meng Z, Pascual D, Wattenhofer R (2021) BERT is robust! A case against synonym-based adversarial examples in text classification. arXiv:2109.07403"},{"key":"10569_CR22","unstructured":"Heigold G, Varanasi S, Neumann G, van Genabith J (2018) How robust are character-based word embeddings in tagging and MT against wrod scramlbing or randdm nouse? In: Proceedings of the 13th conference of the association for machine translation in the Americas (Volume 1: Research Track), pp 68\u201380. Association for Machine Translation in the Americas, Boston, MA. https:\/\/aclanthology.org\/W18-1807"},{"key":"10569_CR23","doi-asserted-by":"crossref","unstructured":"Henkel J, Ramakrishnan G, Wang Z, Albarghouthi A, Jha S, Reps T (2022) Semantic robustness of models of source code. In: 2022 IEEE International conference on software analysis, evolution and reengineering (SANER), IEEE, pp 526\u2013537","DOI":"10.1109\/SANER53432.2022.00070"},{"key":"10569_CR24","doi-asserted-by":"crossref","unstructured":"Huang S, Li Z, Qu L, Pan L (2021) On robustness of neural semantic parsers. In: Proceedings of the 16th conference of the European chapter of the association for computational linguistics: Main Volume, EACL 2021, Online, April 19 - 23, 2021, pp 3333\u20133342. Association for Computational Linguistics. https:\/\/aclanthology.org\/2021.eacl-main.292\/","DOI":"10.18653\/v1\/2021.eacl-main.292"},{"key":"10569_CR25","doi-asserted-by":"crossref","unstructured":"Jha A, Reddy CK (2022) Codeattack: code-based adversarial attacks for pre-trained programming language models. arXiv:2206.00052","DOI":"10.1609\/aaai.v37i12.26739"},{"key":"10569_CR26","doi-asserted-by":"crossref","unstructured":"Jin D, Jin Z, Zhou JT, Szolovits P (2020) Is bert really robust? a strong baseline for natural language attack on text classification and entailment. In: Proceedings of the AAAI conference on artificial intelligence, vol.\u00a034, pp 8018\u20138025","DOI":"10.1609\/aaai.v34i05.6311"},{"key":"10569_CR27","unstructured":"Kachru BB, Kachru Y, Nelson CL (2019) The handbook of world Englishes, vol\u00a048. John Wiley & Sons"},{"issue":"5","key":"10569_CR28","doi-asserted-by":"publisher","first-page":"439","DOI":"10.1016\/j.crad.2017.11.015","volume":"73","author":"D Kim","year":"2018","unstructured":"Kim D, MacKinnon T (2018) Artificial intelligence in fracture detection: transfer learning from deep convolutional neural networks. Clinical Radiol 73(5):439\u2013445","journal-title":"Clinical Radiol"},{"key":"10569_CR29","unstructured":"Kingma DP, Ba J (2015) Adam: A method for stochastic optimization. In: 3rd International conference on learning representations, ICLR 2015, San Diego, CA, USA, May 7-9, 2015, Conference Track Proceedings. arXiv:1412.6980"},{"key":"10569_CR30","doi-asserted-by":"crossref","unstructured":"Li J, Ji S, Du T, Li B, Wang T (2019) Textbugger: generating adversarial text against real-world applications. In: 26th Annual network and distributed system security symposium, NDSS 2019, San Diego, California, USA, February 24-27, 2019. The Internet Society. https:\/\/www.ndss-symposium.org\/ndss-paper\/textbugger-generating-adversarial-text-against-real-world-applications\/","DOI":"10.14722\/ndss.2019.23138"},{"key":"10569_CR31","doi-asserted-by":"crossref","unstructured":"Li Z, Wang X, Aw A, Chng ES, Li H (2018) Named-entity tagging and domain adaptation for better customized translation. In: Proceedings of the seventh named entities workshop, pp 41\u201346","DOI":"10.18653\/v1\/W18-2407"},{"key":"10569_CR32","doi-asserted-by":"publisher","unstructured":"Li Z, Xu J, Zeng J, Li L, Zheng X, Zhang Q, Chang KW, Hsieh CJ (2021) Searching for an effective defender: benchmarking defense against adversarial word substitution. In: Proceedings of the 2021 conference on empirical methods in natural language processing, pp 3137\u20133147. Association for Computational Linguistics, Online and Punta Cana, Dominican Republic. https:\/\/doi.org\/10.18653\/v1\/2021.emnlp-main.251. https:\/\/aclanthology.org\/2021.emnlp-main.251","DOI":"10.18653\/v1\/2021.emnlp-main.251"},{"key":"10569_CR33","doi-asserted-by":"publisher","unstructured":"Liguori P, Al-Hossami E, Cotroneo D, Natella R, Cukic B, Shaikh S (2021) Shellcode_IA32: a dataset for automatic shellcode generation. In: Proceedings of the 1st workshop on natural language processing for programming (NLP4Prog 2021), pp 58\u201364. Association for Computational Linguistics, Online. https:\/\/doi.org\/10.18653\/v1\/2021.nlp4prog-1.7. https:\/\/aclanthology.org\/2021.nlp4prog-1.7","DOI":"10.18653\/v1\/2021.nlp4prog-1.7"},{"key":"10569_CR34","doi-asserted-by":"publisher","unstructured":"Liguori P, Al-Hossami E, Cotroneo D, Natella R, Cukic B, Shaikh S (2022) Can we generate shellcodes via natural language? an empirical study. Automated Softw Eng 29(1):30. https:\/\/doi.org\/10.1007\/s10515-022-00331-3","DOI":"10.1007\/s10515-022-00331-3"},{"key":"10569_CR35","doi-asserted-by":"publisher","unstructured":"Liguori P, Al-Hossami E, Orbinato V, Natella R, Shaikh S, Cotroneo D, Cukic B (2021) Evil: exploiting software via natural language. In: 2021 IEEE 32nd International symposium on software reliability engineering (ISSRE), pp 321\u2013332. https:\/\/doi.org\/10.1109\/ISSRE52982.2021.00042","DOI":"10.1109\/ISSRE52982.2021.00042"},{"key":"10569_CR36","doi-asserted-by":"crossref","unstructured":"Liguori P, Improta C, Natella R, Cukic B, Cotroneo D (2023) Who evaluates the evaluators? on automatic metrics for assessing ai-based offensive code generators. Expert Syst Appl 225:120,073","DOI":"10.1016\/j.eswa.2023.120073"},{"key":"10569_CR37","doi-asserted-by":"crossref","unstructured":"Liu D, Lei L (2019) Technical vocabulary. In: The Routledge handbook of vocabulary studies, Routledge, pp 111\u2013124","DOI":"10.4324\/9780429291586-8"},{"key":"10569_CR38","unstructured":"Liu Y, Ott M, Goyal N, Du J, Joshi M, Chen D, Levy O, Lewis M, Zettlemoyer L, Stoyanov V (209) Roberta: a robustly optimized BERT pretraining approach. arXiv:1907.11692 (2019)"},{"key":"10569_CR39","doi-asserted-by":"crossref","unstructured":"Lutellier T, Pham HV, Pang L, Li Y, Wei M, Tan L (2020) Coconut: combining context-aware neural translation models using ensemble for program repair. In: Proceedings of the 29th ACM SIGSOFT international symposium on software testing and analysis, pp 101\u2013114","DOI":"10.1145\/3395363.3397369"},{"key":"10569_CR40","doi-asserted-by":"publisher","unstructured":"Mashhadi E, Hemmati H (2021) Applying codebert for automated program repair of java simple bugs. In: 18th IEEE\/ACM international conference on mining software repositories, MSR 2021, Madrid, Spain, IEEE, May 17-19, 2021, pp 505\u2013509. https:\/\/doi.org\/10.1109\/MSR52588.2021.00063","DOI":"10.1109\/MSR52588.2021.00063"},{"key":"10569_CR41","doi-asserted-by":"publisher","unstructured":"Mastropaolo A, Pascarella L, Guglielmi E, Ciniselli M, Scalabrino S, Oliveto R, Bavota G (2023) On the robustness of code generation techniques: An empirical study on github copilot. In: 45th IEEE\/ACM International conference on software engineering, ICSE 2023, Melbourne, Australia, IEEE, May 14-20, 2023, pp 2149\u20132160. https:\/\/doi.org\/10.1109\/ICSE48619.2023.00181","DOI":"10.1109\/ICSE48619.2023.00181"},{"key":"10569_CR42","doi-asserted-by":"crossref","unstructured":"Mastropaolo A, Scalabrino S, Cooper N, Palacio DN, Poshyvanyk D, Oliveto R, Bavota G (2021) Studying the usage of text-to-text transfer transformer to support code-related tasks. In: 2021 IEEE\/ACM 43rd International conference on software engineering (ICSE), IEEE, pp 336\u2013347","DOI":"10.1109\/ICSE43902.2021.00041"},{"key":"10569_CR43","unstructured":"Mathai A, Khare S, Tamilselvam S, Mani S (2020) Adversarial black-box attacks on text classifiers using multi-objective genetic optimization guided by deep networks. arXiv:2011.03901"},{"key":"10569_CR44","unstructured":"Megahed H (2018) Penetration Testing with Shellcode: detect, exploit, and secure network-level and operating system vulnerabilities. Packt Publishing"},{"key":"10569_CR45","doi-asserted-by":"publisher","unstructured":"Michel P, Li X, Neubig G, Pino JM (2019) On evaluation of adversarial perturbations for sequence-to-sequence models. In: Proceedings of the 2019 conference of the North American chapter of the association for computational linguistics: human language technologies, NAACL-HLT 2019, Minneapolis, MN, USA, June 2-7, 2019, Volume 1 (Long and Short Papers), pp 3103\u20133114. Association for Computational Linguistics. https:\/\/doi.org\/10.18653\/v1\/n19-1314","DOI":"10.18653\/v1\/n19-1314"},{"key":"10569_CR46","unstructured":"Modrzejewski M, Exel M, Buschbeck B, Ha TL, Waibel A (2020) Incorporating external annotation to improve named entity translation in nmt. In: Proceedings of the 22nd annual conference of the european association for machine translation, pp 45\u201351"},{"key":"10569_CR47","doi-asserted-by":"crossref","unstructured":"Morris JX, Lifland E, Lanchantin J, Ji Y, Qi Y (2020) Reevaluating adversarial examples in natural language. arXiv:2004.14174","DOI":"10.18653\/v1\/2020.findings-emnlp.341"},{"key":"10569_CR48","doi-asserted-by":"publisher","unstructured":"Morris JX, Lifland E, Yoo JY, Grigsby J, Jin D, Qi Y (2020) Textattack: a framework for adversarial attacks, data augmentation, and adversarial training in NLP. In: Proceedings of the 2020 conference on empirical methods in natural language processing: system demonstrations, EMNLP 2020 - Demos, Online, November 16-20, 2020, pp 119\u2013126. Association for Computational Linguistics (2020). https:\/\/doi.org\/10.18653\/v1\/2020.emnlp-demos.16. https:\/\/doi.org\/10.18653\/v1\/2020.emnlp-demos.16","DOI":"10.18653\/v1\/2020.emnlp-demos.16"},{"key":"10569_CR49","doi-asserted-by":"crossref","unstructured":"Mrk\u0161i\u0107 N, \u00d3 S\u00e9aghdha D, Thomson B, Ga\u0161i\u0107 M, Rojas-Barahona L, Su PH, Vandyke D, Wen TH, Young S (2016) Counter-fitting word vectors to linguistic constraints. In: Proceedings of HLT-NAACL","DOI":"10.18653\/v1\/N16-1018"},{"key":"10569_CR50","unstructured":"Neubig G, Sperber M, Wang X, Felix M, Matthews A, Padmanabhan S, Qi Y, Sachan DS, Arthur P, Godard P, Hewitt J, Riad R, Wang L (2018) XNMT: The eXtensible Neural Machine Translation Toolkit. In: Conference of the Association for Machine Translation in the Americas (AMTA) Open Source Software Showcase. Boston, USA. https:\/\/arxiv.org\/pdf\/1803.00188.pdf"},{"key":"10569_CR51","first-page":"10018","volume":"33","author":"XP Nguyen","year":"2020","unstructured":"Nguyen XP, Joty S, Wu K, Aw AT (2020) Data diversification: a simple strategy for neural machine translation. Adv Neural Inf Process Syst 33:10018\u201310029","journal-title":"Adv Neural Inf Process Syst"},{"key":"10569_CR52","doi-asserted-by":"crossref","unstructured":"Pa\u00a0Pa YM, Tanizaki S, Kou T, Van\u00a0Eeten M, Yoshioka K, Matsumoto T (2023) An attacker\u2019s dream? exploring the capabilities of chatgpt for developing malware. In: Proceedings of the 16th cyber security experimentation and test workshop, pp 10\u201318","DOI":"10.1145\/3607505.3607513"},{"key":"10569_CR53","doi-asserted-by":"crossref","unstructured":"Panichella S, Aponte J, Di\u00a0Penta M, Marcus A, Canfora G (2012) Mining source code descriptions from developer communications. In: 2012 20th IEEE International conference on program comprehension (ICPC), IEEE, pp 63\u201372","DOI":"10.1109\/ICPC.2012.6240510"},{"key":"10569_CR54","unstructured":"Python: tokenize (2023). https:\/\/docs.python.org\/3\/library\/tokenize.html"},{"key":"10569_CR55","unstructured":"Raffel C, Shazeer N, Roberts A, Lee K, Narang S, Matena M, Zhou Y, Li W, Liu PJ (2020) Exploring the limits of transfer learning with a unified text-to-text transformer. J Mach Learn Res 21(140):1\u2013140:67. http:\/\/jmlr.org\/papers\/v21\/20-074.html"},{"key":"10569_CR56","unstructured":"Reimers N (2022) Sentencetransformers documentation. https:\/\/www.sbert.net\/"},{"key":"10569_CR57","doi-asserted-by":"publisher","unstructured":"Reimers N, Gurevych I (2019) Sentence-bert: sentence embeddings using siamese bert-networks. In:\u00a0Inui K,\u00a0Jiang J,\u00a0Ng V,\u00a0Wan X (eds.) Proceedings of the 2019 conference on empirical methods in natural language processing and the 9th international joint conference on natural language processing, EMNLP-IJCNLP 2019, Hong Kong, China, November 3-7, 2019, pp 3980\u20133990. Association for Computational Linguistics (2019). https:\/\/doi.org\/10.18653\/v1\/D19-1410","DOI":"10.18653\/v1\/D19-1410"},{"key":"10569_CR58","doi-asserted-by":"publisher","unstructured":"Reimers N, Gurevych I (2020) Making monolingual sentence embeddings multilingual using knowledge distillation. In:\u00a0Webber B,\u00a0Cohn T,\u00a0He Y,\u00a0Liu Y (eds.) Proceedings of the 2020 conference on empirical methods in natural language processing, EMNLP 2020, Online, November 16-20, 2020, pp 4512\u20134525. Association for Computational Linguistics (2020). https:\/\/doi.org\/10.18653\/v1\/2020.emnlp-main.365. https:\/\/doi.org\/10.18653\/v1\/2020.emnlp-main.365","DOI":"10.18653\/v1\/2020.emnlp-main.365"},{"key":"10569_CR59","doi-asserted-by":"crossref","unstructured":"Ruan X, Yu Y, Ma W, Cai B (2023) Prompt learning for developing software exploits. In: Proceedings of the 14th Asia-Pacific symposium on internetware, pp 154\u2013164","DOI":"10.1145\/3609437.3609445"},{"key":"10569_CR60","unstructured":"Shell-storm (2022) Shellcodes database for study cases. http:\/\/shell-storm.org\/shellcode\/"},{"key":"10569_CR61","doi-asserted-by":"publisher","unstructured":"Silva C, Ribeiro B (2003) The importance of stop word removal on recall values in text categorization. In: Proceedings of the international joint conference on neural networks, 2003., vol\u00a03, pp 1661\u20131666 vol 3. https:\/\/doi.org\/10.1109\/IJCNN.2003.1223656","DOI":"10.1109\/IJCNN.2003.1223656"},{"key":"10569_CR62","unstructured":"spaCy (2023) Industrial-Strength Natural Language Processing. https:\/\/spacy.io\/"},{"key":"10569_CR63","doi-asserted-by":"crossref","unstructured":"Tufano M, Pantiuchina J, Watson C, Bavota G, Poshyvanyk D (2019) On learning meaningful code changes via neural machine translation. In: 2019 IEEE\/ACM 41st International Conference on Software Engineering (ICSE), IEEE, pp 25\u201336","DOI":"10.1109\/ICSE.2019.00021"},{"key":"10569_CR64","unstructured":"Tychonievich L (2014) Parts of Programming Speech. https:\/\/luthert.web.illinois.edu\/blog\/posts\/519.html"},{"key":"10569_CR65","unstructured":"Vaswani A, Shazeer N, Parmar N, Uszkoreit J, Jones L, Gomez AN, Kaiser \u0141, Polosukhin I (2017) Attention is all you need. In: Advances in neural information processing systems, pp 5998\u20136008"},{"key":"10569_CR66","doi-asserted-by":"publisher","unstructured":"Wang S, Li Z, Qian H, Yang C, Wang Z, Shang M, Kumar V, Tan S, Ray B, Bhatia P, Nallapati R, Ramanathan MK, Roth D, Xiang B (2023) ReCode: robustness evaluation of code generation models. In:\u00a0Rogers A,\u00a0Boyd-Graber J,\u00a0Okazaki N (eds.) pp 13,818\u201313,843. Association for Computational Linguistics, Toronto, Canada (2023). https:\/\/doi.org\/10.18653\/v1\/2023.acl-long.773. https:\/\/aclanthology.org\/2023.acl-long.773","DOI":"10.18653\/v1\/2023.acl-long.773"},{"key":"10569_CR67","doi-asserted-by":"crossref","unstructured":"Wang X, Pham H, Dai Z, Neubig G (2018) Switchout: an efficient data augmentation algorithm for neural machine translation. In: Proceedings of the 2018 Conference on empirical methods in natural language processing, pp 856\u2013861","DOI":"10.18653\/v1\/D18-1100"},{"key":"10569_CR68","doi-asserted-by":"crossref","unstructured":"Wang Y, Le H, Gotmare AD, Bui ND, Li J, Hoi SC (2023) Codet5+: open code large language models for code understanding and generation. arXiv preprint arXiv:2305.07922","DOI":"10.18653\/v1\/2023.emnlp-main.68"},{"key":"10569_CR69","doi-asserted-by":"crossref","unstructured":"Wang Z (2013) Study on the importance of cultural context analysis in machine translation. In: Proceedings of the eighth international conference on bio-inspired computing: theories and applications (BIC-TA), 2013, Springer, pp 29\u201335","DOI":"10.1007\/978-3-642-37502-6_4"},{"key":"10569_CR70","doi-asserted-by":"crossref","unstructured":"Wei J, Zou K (2019) Eda: Easy data augmentation techniques for boosting performance on text classification tasks. In: Proceedings of the 2019 conference on empirical methods in natural language processing and the 9th international joint conference on Natural Language Processing (EMNLP-IJCNLP), pp 6382\u20136388","DOI":"10.18653\/v1\/D19-1670"},{"key":"10569_CR71","unstructured":"Wu W, Arendt D, Volkova S (2020) Evaluating neural machine comprehension model robustness to noisy inputs and adversarial attacks. arXiv:2005.00190"},{"issue":"3","key":"10569_CR72","doi-asserted-by":"publisher","first-page":"917","DOI":"10.1007\/s10994-021-06070-y","volume":"111","author":"X Wu","year":"2022","unstructured":"Wu X, Xia Y, Zhu J, Wu L, Xie S, Qin T (2022) A study of bert for context-aware neural machine translation. Mach Learn 111(3):917\u2013935","journal-title":"Mach Learn"},{"key":"10569_CR73","doi-asserted-by":"crossref","unstructured":"Xu D, Chen K, Lin M, Lin C, Wang X (2023) Autopwn: artifact-assisted heap exploit generation for ctf pwn competitions. IEEE Trans Inf Forensics Sec","DOI":"10.1109\/TIFS.2023.3322319"},{"key":"10569_CR74","doi-asserted-by":"crossref","unstructured":"Yang G, Chen X, Zhou Y, Yu C (2022) Dualsc: automatic generation and summarization of shellcode via transformer and dual learning. In: 2022 IEEE International conference on software analysis, evolution and reengineering (SANER), IEEE, pp 361\u2013372","DOI":"10.1109\/SANER53432.2022.00052"},{"key":"10569_CR75","doi-asserted-by":"crossref","unstructured":"Yang G, Zhou Y, Chen X, Zhang X, Han T, Chen T (2023) Exploitgen: template-augmented exploit code generation based on codebert. J Syst Softw 197:111,577","DOI":"10.1016\/j.jss.2022.111577"},{"key":"10569_CR76","doi-asserted-by":"crossref","unstructured":"Yang Z, Shi J, He J, Lo D (2022) Natural attack for pre-trained models of code. In: Proceedings of the 44th international conference on software engineering, pp 1482\u20131493","DOI":"10.1145\/3510003.3510146"},{"key":"10569_CR77","doi-asserted-by":"publisher","unstructured":"Yefet N, Alon U, Yahav E (2020) Adversarial examples for models of code. Proc. ACM Program Lang 4(OOPSLA):162:1\u2013162:30. https:\/\/doi.org\/10.1145\/3428230","DOI":"10.1145\/3428230"},{"key":"10569_CR78","doi-asserted-by":"publisher","unstructured":"Yoo JY, Qi Y (2021) Towards improving adversarial training of NLP models. In: Findings of the association for computational linguistics: EMNLP 2021, pp 945\u2013956. Association for Computational Linguistics, Punta Cana, Dominican Republic. https:\/\/doi.org\/10.18653\/v1\/2021.findings-emnlp.81. https:\/\/aclanthology.org\/2021.findings-emnlp.81","DOI":"10.18653\/v1\/2021.findings-emnlp.81"},{"key":"10569_CR79","doi-asserted-by":"crossref","unstructured":"Yu S, Wang T, Wang J (2022) Data augmentation by program transformation. J Syst Softw 190, 111,304","DOI":"10.1016\/j.jss.2022.111304"},{"key":"10569_CR80","doi-asserted-by":"crossref","unstructured":"Zeng Z, Tan H, Zhang H, Li J, Zhang Y, Zhang L (2022) An extensive study on pre-trained models for program understanding and generation. In: Proceedings of the 31st ACM SIGSOFT international symposium on software testing and analysis, pp 39\u201351","DOI":"10.1145\/3533767.3534390"},{"key":"10569_CR81","doi-asserted-by":"crossref","unstructured":"Zhang X, Zhou Y, Han T, Chen T (2020) Training deep code comment generation models via data augmentation. In: Proceedings of the 12th Asia-Pacific symposium on internetware, pp 185\u2013188","DOI":"10.1145\/3457913.3457937"},{"key":"10569_CR82","unstructured":"Zhang Y, Wang X, Xi Z, Xia H, Gui T, Zhang Q, Huang X (2024) Rocoins: enhancing robustness of large language models through code-style instructions. arXiv:2402.16431"},{"key":"10569_CR83","doi-asserted-by":"publisher","unstructured":"Zhou Y, Zhang X, Shen J, Han T, Chen T, Gall HC (2022) Adversarial robustness of deep code comment generation. ACM Trans Softw Eng Methodol 31(4):60:1\u201360:30. https:\/\/doi.org\/10.1145\/3501256","DOI":"10.1145\/3501256"},{"key":"10569_CR84","doi-asserted-by":"publisher","unstructured":"Zhu R, Zhang C (2023) How robust is a large pre-trained language model for code generationf a case on attacking gpt2. In: 2023 IEEE International conference on software analysis, evolution and reengineering (SANER), pp 708\u2013712. IEEE Computer Society, Los Alamitos, CA, USA. https:\/\/doi.org\/10.1109\/SANER56733.2023.00076. https:\/\/doi.ieeecomputersociety.org\/10.1109\/SANER56733.2023.00076","DOI":"10.1109\/SANER56733.2023.00076"},{"key":"10569_CR85","doi-asserted-by":"publisher","unstructured":"Zhuo TY, Li Z, Huang Y, Shiri F, Wang W, Haffari G, Li YF (2023) On robustness of prompt-based semantic parsing with large pre-trained language model: an empirical study on codex. In: Proceedings of the 17th conference of the european chapter of the association for computational linguistics, pp 1090\u20131102. Association for Computational Linguistics, Dubrovnik, Croatia. https:\/\/doi.org\/10.18653\/v1\/2023.eacl-main.77. https:\/\/aclanthology.org\/2023.eacl-main.77","DOI":"10.18653\/v1\/2023.eacl-main.77"}],"container-title":["Empirical Software Engineering"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10664-024-10569-y.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s10664-024-10569-y\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10664-024-10569-y.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,1,7]],"date-time":"2025-01-07T14:08:53Z","timestamp":1736258933000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s10664-024-10569-y"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,10,19]]},"references-count":85,"journal-issue":{"issue":"1","published-print":{"date-parts":[[2025,1]]}},"alternative-id":["10569"],"URL":"https:\/\/doi.org\/10.1007\/s10664-024-10569-y","relation":{},"ISSN":["1382-3256","1573-7616"],"issn-type":[{"value":"1382-3256","type":"print"},{"value":"1573-7616","type":"electronic"}],"subject":[],"published":{"date-parts":[[2024,10,19]]},"assertion":[{"value":"10 October 2024","order":1,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"19 October 2024","order":2,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Declarations"}},{"value":"The authors have no conflicts of interest to declare that are relevant to the content of this article. All authors certify that they have no affiliations with or involvement in any organization or entity with any financial interest or non-financial interest in the subject matter or materials discussed in this manuscript. The authors have no financial or proprietary interests in any material discussed in this article.","order":2,"name":"Ethics","group":{"name":"EthicsHeading","label":"Conflicts of Interests"}}],"article-number":"7"}}