{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,17]],"date-time":"2026-07-17T15:42:19Z","timestamp":1784302939322,"version":"3.55.0"},"reference-count":45,"publisher":"Springer Science and Business Media LLC","issue":"6","license":[{"start":{"date-parts":[[2023,9,23]],"date-time":"2023-09-23T00:00:00Z","timestamp":1695427200000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2023,9,23]],"date-time":"2023-09-23T00:00:00Z","timestamp":1695427200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"funder":[{"name":"WHJIL"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Empir Software Eng"],"published-print":{"date-parts":[[2023,11]]},"DOI":"10.1007\/s10664-023-10380-1","type":"journal-article","created":{"date-parts":[[2023,9,23]],"date-time":"2023-09-23T07:01:41Z","timestamp":1695452501000},"update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":103,"title":["Is GitHub\u2019s Copilot as bad as humans at introducing vulnerabilities in code?"],"prefix":"10.1007","volume":"28","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-6755-605X","authenticated-orcid":false,"given":"Owura","family":"Asare","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Meiyappan","family":"Nagappan","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"N.","family":"Asokan","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2023,9,23]]},"reference":[{"key":"10380_CR1","doi-asserted-by":"crossref","unstructured":"Asare, O., M.\u00a0Nagappan, and N.\u00a0Asokan. 2022. Is GitHub\u2019s Copilot as Bad as Humans at Introducing Vulnerabilities in Code? _eprint: 2204.04741","DOI":"10.1007\/s10664-023-10380-1"},{"key":"10380_CR2","doi-asserted-by":"crossref","unstructured":"Barke, S., M.B. James, and N.\u00a0Polikarpova. 2022, August. Grounded Copilot: How Programmers Interact with Code-Generating Models. arXiv:2206.15000","DOI":"10.1145\/3586030"},{"key":"10380_CR3","unstructured":"Bengio, Y., R.\u00a0Ducharme, and P.\u00a0Vincent 2000.A Neural Probabilistic Language Model. In Advances in Neural Information Processing Systems,Volume 13. MIT Press"},{"key":"10380_CR4","doi-asserted-by":"crossref","unstructured":"Bielik, P., V.\u00a0Raychev, and M.\u00a0Vechev 2016.PHOG: probabilistic model for code. In International Conference on Machine Learning, pp. 2933\u20132942. PMLR","DOI":"10.1145\/2983990.2984041"},{"key":"10380_CR5","unstructured":"Brown, T.B., B.\u00a0Mann, N.\u00a0Ryder, M.\u00a0Subbiah, J.\u00a0Kaplan, P.\u00a0Dhariwal,A.\u00a0Neelakantan, P.\u00a0Shyam, G.\u00a0Sastry, A.\u00a0Askell, S.\u00a0Agarwal, A.\u00a0Herbert-Voss,G.\u00a0Krueger, T.\u00a0Henighan, R.\u00a0Child, A.\u00a0Ramesh, D.M. Ziegler, J.\u00a0Wu, C.\u00a0Winter,C.\u00a0Hesse, M.\u00a0Chen, E.\u00a0Sigler, M.\u00a0Litwin, S.\u00a0Gray, B.\u00a0Chess, J.\u00a0Clark,C.\u00a0Berner, S.\u00a0McCandlish, A.\u00a0Radford, I.\u00a0Sutskever, and D.\u00a0Amodei. 2020,July.Language Models are Few-Shot Learners. arXiv:2005.14165 [cs]"},{"issue":"9","key":"10380_CR6","doi-asserted-by":"publisher","first-page":"3280","DOI":"10.1109\/TSE.2021.3087402","volume":"48","author":"S Chakraborty","year":"2022","unstructured":"Chakraborty S, Krishna R, Ding Y, Ray B (2022) Learning Based Vulnerability Detection: Are We There Yet? IEEE Transactions on Software Engineering 48(9):3280\u20133296. https:\/\/doi.org\/10.1109\/TSE.2021.3087402","journal-title":"IEEE Transactions on Software Engineering"},{"key":"10380_CR7","doi-asserted-by":"crossref","unstructured":"Chen, D. and C.\u00a0Manning 2014, October. A Fast and Accurate Dependency Parser using Neural Networks. In Proceedings of the 2014 Conference on Empirical Methods in Natural Language Processing (EMNLP), Doha, Qatar, pp. 740\u2013750. Association for Computational Linguistics","DOI":"10.3115\/v1\/D14-1082"},{"key":"10380_CR8","unstructured":"Chen, M., J.\u00a0Tworek, H.\u00a0Jun, Q.\u00a0Yuan, H.P.d.O. Pinto, J.\u00a0Kaplan, H.\u00a0Edwards,Y.\u00a0Burda, N.\u00a0Joseph, G.\u00a0Brockman, A.\u00a0Ray, R.\u00a0Puri, G.\u00a0Krueger, M.\u00a0Petrov,H.\u00a0Khlaaf, G.\u00a0Sastry, P.\u00a0Mishkin, B.\u00a0Chan, S.\u00a0Gray, N.\u00a0Ryder, M.\u00a0Pavlov,A.\u00a0Power, L.\u00a0Kaiser, M.\u00a0Bavarian, C.\u00a0Winter, P.\u00a0Tillet, F.P. Such,D.\u00a0Cummings, M.\u00a0Plappert, F.\u00a0Chantzis, E.\u00a0Barnes, A.\u00a0Herbert-Voss, W.H. Guss,A.\u00a0Nichol, A.\u00a0Paino, N.\u00a0Tezak, J.\u00a0Tang, I.\u00a0Babuschkin, S.\u00a0Balaji, S.\u00a0Jain,W.\u00a0Saunders, C.\u00a0Hesse, A.N. Carr, J.\u00a0Leike, J.\u00a0Achiam, V.\u00a0Misra, E.\u00a0Morikawa,A.\u00a0Radford, M.\u00a0Knight, M.\u00a0Brundage, M.\u00a0Murati, K.\u00a0Mayer, P.\u00a0Welinder,B.\u00a0McGrew, D.\u00a0Amodei, S.\u00a0McCandlish, I.\u00a0Sutskever, and W.\u00a0Zaremba. 2021,July.Evaluating Large Language Models Trained on Code. arXiv:2107.03374 [cs]"},{"key":"10380_CR9","doi-asserted-by":"crossref","unstructured":"Ciniselli, M., L.\u00a0Pascarella, and G.\u00a0Bavota. 2022, April.To What Extent do Deep Learning-based Code Recommenders Generate Predictions by Cloning Code from the Training Set? arXiv:2204.06894","DOI":"10.1145\/3524842.3528440"},{"key":"10380_CR10","unstructured":"Dakhel, A.M., V.\u00a0Majdinasab, A.\u00a0Nikanjam, F.\u00a0Khomh, M.C. Desmarais, Z.\u00a0Ming,and Jiang. 2022, June. GitHub Copilot AI pair programmer: Asset or Liability? arXiv:2206.15331"},{"key":"10380_CR11","unstructured":"Desai, A. and A.\u00a0Deo. 2022. Introducing Amazon CodeWhisperer, the ML-powered coding companion"},{"key":"10380_CR12","unstructured":"Devlin, J., M.W. Chang, K.\u00a0Lee, and K.\u00a0Toutanova. 2019, May. BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding. arXiv:1810.04805"},{"key":"10380_CR13","unstructured":"Dohmke, T. 2022, June.GitHub Copilot is generally available to all developers"},{"key":"10380_CR14","doi-asserted-by":"crossref","unstructured":"Fan, J., Y.\u00a0Li, S.\u00a0Wang, and T.N. Nguyen 2020, June.A C\/C++ Code Vulnerability Dataset with Code Changes and CVE Summaries. In Proceedings of the 17th International Conference on Mining Software Repositories, Seoul Republic of Korea, pp. 508\u2013512.ACM","DOI":"10.1145\/3379597.3387501"},{"key":"10380_CR15","doi-asserted-by":"crossref","unstructured":"Feng, Z., D.\u00a0Guo, D.\u00a0Tang, N.\u00a0Duan, X.\u00a0Feng, M.\u00a0Gong, L.\u00a0Shou, B.\u00a0Qin, T.\u00a0Liu,D.\u00a0Jiang, and M.\u00a0Zhou. 2020, September.CodeBERT: A Pre-Trained Model for Programming and Natural Languages.arXiv:2002.08155","DOI":"10.18653\/v1\/2020.findings-emnlp.139"},{"key":"10380_CR16","doi-asserted-by":"publisher","unstructured":"Galassi, A., M.\u00a0Lippi, and P.\u00a0Torroni. 2021, October. Natural Language Processing.IEEE Transactions on Neural Networks and Learning Systems 32(10): 4291\u20134308. https:\/\/doi.org\/10.1109\/TNNLS.2020.3019893","DOI":"10.1109\/TNNLS.2020.3019893"},{"key":"10380_CR17","unstructured":"GitHub Inc. 2019.CodeQL"},{"key":"10380_CR18","unstructured":"GitHub Inc. 2021.GitHub Copilot Your AI pair programmer"},{"key":"10380_CR19","unstructured":"Hardmeier, C. 2016, December.A Neural Model for Part-of-Speech Tagging in Historical Texts.In Proceedings of COLING 2016, the 26th International Conference on Computational Linguistics: Technical Papers, Osaka, Japan, pp.922\u2013931. The COLING 2016 Organizing Committee"},{"key":"10380_CR20","doi-asserted-by":"crossref","unstructured":"Hellendoorn, V.J. and P.\u00a0Devanbu 2017, August.Are deep neural networks the best choice for modeling source code? In Proceedings of the 2017 11th Joint Meeting on Foundations of Software Engineering, Paderborn Germany, pp.763\u2013773. ACM","DOI":"10.1145\/3106237.3106290"},{"key":"10380_CR21","doi-asserted-by":"crossref","unstructured":"Hindle, A., E.T. Barr, Z.\u00a0Su, M.\u00a0Gabel, and P.\u00a0Devanbu 2012.On the Naturalness of Software.In Proceedings of the 34th International Conference on Software Engineering, ICSE \u201912, pp.837\u2013847. IEEE Press. event-place: Zurich, Switzerland","DOI":"10.1109\/ICSE.2012.6227135"},{"issue":"8","key":"10380_CR22","doi-asserted-by":"publisher","first-page":"1735","DOI":"10.1162\/neco.1997.9.8.1735","volume":"9","author":"S Hochreiter","year":"1997","unstructured":"Hochreiter S, Schmidhuber J (1997) November. Long Short-Term Memory. Neural Computation 9(8):1735\u20131780. https:\/\/doi.org\/10.1162\/neco.1997.9.8.1735","journal-title":"Neural Computation"},{"key":"10380_CR23","doi-asserted-by":"crossref","unstructured":"Jiang, N., T.\u00a0Lutellier, and L.\u00a0Tan 2021, May. CURE: Code-Aware Neural Machine Translation for Automatic Program Repair. In 2021 IEEE\/ACM 43rd International Conference on Software Engineering (ICSE), pp.1161\u20131173.ISSN: 1558-1225","DOI":"10.1109\/ICSE43902.2021.00107"},{"key":"10380_CR24","doi-asserted-by":"publisher","unstructured":"Le, T.H.M., H.\u00a0Chen, and M.A. Babar. 2020, June.Deep Learning for Source Code Modeling and Generation:Models, Applications, and Challenges. ACM Comput. Surv.\u00a053(3)https:\/\/doi.org\/10.1162\/neco.10.1145\/3383458","DOI":"10.1162\/neco.10.1145\/3383458"},{"key":"10380_CR25","doi-asserted-by":"crossref","unstructured":"Li, Y., D.\u00a0Choi, J.\u00a0Chung, N.\u00a0Kushman, J.\u00a0Schrittwieser, R Leblond, T.\u00a0Eccles,J.\u00a0Keeling, F.\u00a0Gimeno, A.D. Lago, T.\u00a0Hubert, P.\u00a0Choy, C.d.M. d\u2019Autume,I.\u00a0Babuschkin, X.\u00a0Chen, P.S. Huang, J.\u00a0Welbl, S.\u00a0Gowal, A.\u00a0Cherepanov,J.\u00a0Molloy, D.J. Mankowitz, E.S. Robson, P.\u00a0Kohli, N.\u00a0de\u00a0Freitas,K.\u00a0Kavukcuoglu, and O.\u00a0Vinyals. 2022.Competition-Level Code Generation with AlphaCode","DOI":"10.1126\/science.abq1158"},{"key":"10380_CR26","unstructured":"Lu, S., D.\u00a0Guo, S.\u00a0Ren, J.\u00a0Huang, A.\u00a0Svyatkovskiy, A.\u00a0Blanco, C.\u00a0Clement,D.\u00a0Drain, D.\u00a0Jiang, D.\u00a0Tang, G.\u00a0Li, L.\u00a0Zhou, L.\u00a0Shou, L.\u00a0Zhou, M.\u00a0Tufano,M.\u00a0Gong, M.\u00a0Zhou, N.\u00a0Duan, N.\u00a0Sundaresan, S.K. Deng, S.\u00a0Fu, and S.\u00a0Liu. 2021,March.CodeXGLUE: A Machine Learning Benchmark Dataset for Code Understanding and Generation. arXiv:2102.04664"},{"key":"10380_CR27","doi-asserted-by":"crossref","unstructured":"Nguyen, N. and S.\u00a0Nadi 2022.Empirical Evaluation of GitHub Copilot\u2019s Code Suggestions.In 2022 IEEE\/ACM 19th International Conference on Mining Software Repositories (MSR), pp.1\u20135","DOI":"10.1145\/3524842.3528470"},{"key":"10380_CR28","unstructured":"Nijkamp, E., B.\u00a0Pang, H.\u00a0Hayashi, L.\u00a0Tu, H.\u00a0Wang, Y.\u00a0Zhou, S.\u00a0Savarese, and C.\u00a0Xiong. 2022.CodeGen: An Open Large Language Model for Code with Multi-Turn Program Synthesis.arXiv preprint"},{"key":"10380_CR29","doi-asserted-by":"crossref","unstructured":"Pearce, H., B.\u00a0Ahmad, B.\u00a0Tan, B.\u00a0Dolan-Gavitt, and R.\u00a0Karri 2022, May.Asleep at the Keyboard? Assessing the Security of GitHub Copilot Code Contributions. In 2022 IEEE Symposium on Security and Privacy (SP), pp.754\u2013768.ISSN: 2375-1207","DOI":"10.1109\/SP46214.2022.9833571"},{"key":"10380_CR30","doi-asserted-by":"crossref","unstructured":"Pearce, H., B.\u00a0Tan, B.\u00a0Ahmad, R.\u00a0Karri, and B.\u00a0Dolan-Gavitt 2023, May.Examining Zero-Shot Vulnerability Repair with Large Language Models.In 2023 2023 IEEE Symposium on Security and Privacy(SP) (SP), Los Alamitos, CA, USA, pp.1\u201318. IEEE Computer Society","DOI":"10.1109\/SP46215.2023.10179324"},{"key":"10380_CR31","doi-asserted-by":"crossref","unstructured":"Prenner, J., H.\u00a0Babii, and R.\u00a0Robbes 2022, May. Can OpenAI\u2019s Codex Fix Bugs?: An evaluation on QuixBugs.2022 IEEE\/ACM International Workshop on Automated Program Repair (APR), Los Alamitos, CA, USA, pp.69\u201375. IEEE Computer Society","DOI":"10.1145\/3524459.3527351"},{"key":"10380_CR32","doi-asserted-by":"crossref","unstructured":"Raychev, V., M.\u00a0Vechev, and E.\u00a0Yahav 2014, June.Code completion with statistical language models.In Proceedings of the 35th ACM SIGPLAN Conference on Programming Language Design and Implementation, Edinburgh United Kingdom, pp.419\u2013428. ACM","DOI":"10.1145\/2594291.2594321"},{"key":"10380_CR33","doi-asserted-by":"crossref","unstructured":"Sobania, D., M.\u00a0Briesch, and F.\u00a0Rothlauf 2022, July.Choose your programming copilot: a comparison of the program synthesis performance of github copilot and genetic programming.In Proceedings of the Genetic and Evolutionary Computation Conference, Boston Massachusetts, pp.1019\u20131027. ACM","DOI":"10.1145\/3512290.3528700"},{"key":"10380_CR34","doi-asserted-by":"crossref","unstructured":"Svyatkovskiy, A., S.K. Deng, S.\u00a0Fu, and N.\u00a0Sundaresan 2020, November.IntelliCode compose: code generation using transformer.In Proceedings of the 28th ACM Joint Meeting on European Software Engineering Conference and Symposium on the Foundations of Software Engineering, Virtual Event USA, pp.1433\u20131443. ACM","DOI":"10.1145\/3368089.3417058"},{"key":"10380_CR35","unstructured":"Synopsys 2022.Source Security and Risk Analysis Report. Technical report, Synopsys Inc"},{"key":"10380_CR36","unstructured":"Tabnine. 2022.Code Faster with AI Completions"},{"key":"10380_CR37","doi-asserted-by":"crossref","unstructured":"Vaithilingam, P., T.\u00a0Zhang, and E.L. Glassman 2022, April. Expectation vs. Experience: Evaluating the Usability of Code Generation Tools Powered by Large Language Models.In CHI Conference on Human Factors in Computing Systems Extended Abstracts, New Orleans LA USA, pp.1\u20137. ACM","DOI":"10.1145\/3491101.3519665"},{"key":"10380_CR38","unstructured":"Vaswani, A., N.\u00a0Shazeer, N.\u00a0Parmar, J.\u00a0Uszkoreit, L.\u00a0Jones, A.N. Gomez, \u0141 Kaiser, and I.\u00a0Polosukhin 2017.Attention is All You Need.In Proceedings of the 31st International Conference on Neural Information Processing Systems, NIPS\u201917, Red Hook, NY, USA,pp.6000\u20136010. Curran Associates Inc.event-place: Long Beach, California, USA"},{"key":"10380_CR39","doi-asserted-by":"crossref","unstructured":"Xu, F.F., U.\u00a0Alon, G.\u00a0Neubig, and V.J. Hellendoorn 2022, June. A systematic evaluation of large language models of code.In Proceedings of the 6th ACM SIGPLAN International Symposium on Machine Programming, San Diego CA USA, pp.1\u201310. ACM","DOI":"10.1145\/3520312.3534862"},{"key":"10380_CR40","doi-asserted-by":"crossref","unstructured":"Yan, W. and Y.\u00a0Li. 2022, April.WhyGen: Explaining ML-powered Code Generation by Referring to Training Examples. arXiv:2204.07940","DOI":"10.1145\/3510454.3516866"},{"key":"10380_CR41","doi-asserted-by":"crossref","unstructured":"Yin, J., X.\u00a0Jiang, Z.\u00a0Lu, L.\u00a0Shang, H.\u00a0Li, and X.\u00a0Li 2016. Neural Generative Question Answering.In Proceedings of the Twenty-Fifth International Joint Conference on Artificial Intelligence, IJCAI\u201916, pp.2972\u20132978. AAAI Press.event-place: New York, New York, USA","DOI":"10.18653\/v1\/W16-0106"},{"key":"10380_CR42","doi-asserted-by":"crossref","unstructured":"Yin, P. and G.\u00a0Neubig. 2017, April. A Syntactic Neural Model for General-Purpose Code Generation.arXiv:1704.01696","DOI":"10.18653\/v1\/P17-1041"},{"key":"10380_CR43","unstructured":"Zhang, J., J.\u00a0Cambronero, S.\u00a0Gulwani, V.\u00a0Le, R.\u00a0Piskac, G.\u00a0Soares, and G.\u00a0Verbruggen. 2022.Repairing Bugs in Python Assignments Using Large Language Models"},{"key":"10380_CR44","doi-asserted-by":"publisher","first-page":"371","DOI":"10.1162\/tacl_a_00105","volume":"4","author":"J Zhou","year":"2016","unstructured":"Zhou J, Cao Y, Wang X, Li P, Xu W (2016) Deep Recurrent Models with Fast-Forward Connections for Neural Machine Translation. Transactions of the Association for Computational Linguistics 4:371\u2013383. https:\/\/doi.org\/10.1162\/tacl_a_00105","journal-title":"Transactions of the Association for Computational Linguistics"},{"key":"10380_CR45","doi-asserted-by":"crossref","unstructured":"Ziegler, A., E.\u00a0Kalliamvakou, X.A. Li, A.\u00a0Rice, D.\u00a0Rifkin, S.\u00a0Simister,G.\u00a0Sittampalam, and E.\u00a0Aftandilian 2022, June.Productivity assessment of neural code completion.In Proceedings of the 6th ACM SIGPLAN International Symposium on Machine Programming, San Diego CA USA, pp.21\u201329. ACM","DOI":"10.1145\/3520312.3534864"}],"container-title":["Empirical Software Engineering"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10664-023-10380-1.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s10664-023-10380-1\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10664-023-10380-1.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2023,11,29]],"date-time":"2023-11-29T12:08:43Z","timestamp":1701259723000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s10664-023-10380-1"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023,9,23]]},"references-count":45,"journal-issue":{"issue":"6","published-print":{"date-parts":[[2023,11]]}},"alternative-id":["10380"],"URL":"https:\/\/doi.org\/10.1007\/s10664-023-10380-1","relation":{},"ISSN":["1382-3256","1573-7616"],"issn-type":[{"value":"1382-3256","type":"print"},{"value":"1573-7616","type":"electronic"}],"subject":[],"published":{"date-parts":[[2023,9,23]]},"assertion":[{"value":"9 August 2023","order":1,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"23 September 2023","order":2,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}}],"article-number":"129"}}