{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,21]],"date-time":"2026-04-21T14:29:09Z","timestamp":1776781749456,"version":"3.51.2"},"publisher-location":"New York, NY, USA","reference-count":30,"publisher":"ACM","license":[{"start":{"date-parts":[[2024,4,15]],"date-time":"2024-04-15T00:00:00Z","timestamp":1713139200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"name":"Nederlandse Organisatie voor Wetenschappelijk Onderzoek","award":["VI.C.182.032"],"award-info":[{"award-number":["VI.C.182.032"]}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2024,4,15]]},"DOI":"10.1145\/3644032.3644443","type":"proceedings-article","created":{"date-parts":[[2024,6,10]],"date-time":"2024-06-10T15:00:25Z","timestamp":1718031625000},"page":"45-55","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":17,"title":["Using GitHub Copilot for Test Generation in Python: An Empirical Study"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0009-0003-5143-2612","authenticated-orcid":false,"given":"Khalid","family":"El Haji","sequence":"first","affiliation":[{"name":"Delft University of Technology, Delft, Netherlands"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-7623-1970","authenticated-orcid":false,"given":"Carolin","family":"Brandt","sequence":"additional","affiliation":[{"name":"Delft University of Technology, Delft, Netherlands"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-2413-3935","authenticated-orcid":false,"given":"Andy","family":"Zaidman","sequence":"additional","affiliation":[{"name":"Delft University of Technology, Delft, Netherlands"}]}],"member":"320","published-online":{"date-parts":[[2024,6,10]]},"reference":[{"key":"e_1_3_2_1_1_1","volume-title":"Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics. Association for Computational Linguistics, Online, 4998--5007","author":"Ahmad Wasi","year":"2020","unstructured":"Wasi Ahmad, Saikat Chakraborty, Baishakhi Ray, and Kai-Wei Chang. 2020. A Transformer-based Approach for Source Code Summarization. In Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics. Association for Computational Linguistics, Online, 4998--5007."},{"key":"e_1_3_2_1_2_1","volume-title":"Examining Readability and Visual Inspection of GitHub Copilot. In 37th IEEE\/ACM International Conference on Automated Software Engineering. 1--5.","author":"Madi Naser Al","year":"2022","unstructured":"Naser Al Madi. 2022. How Readable is Model-generated Code? Examining Readability and Visual Inspection of GitHub Copilot. In 37th IEEE\/ACM International Conference on Automated Software Engineering. 1--5."},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.jss.2013.02.061"},{"key":"e_1_3_2_1_4_1","first-page":"4","article-title":"An Experience Report on Applying Software Testing Academic Results in Industry: We Need Usable Automated Test Generation","volume":"23","author":"Arcuri Andrea","year":"2018","unstructured":"Andrea Arcuri. 2018. An Experience Report on Applying Software Testing Academic Results in Industry: We Need Usable Automated Test Generation. Empirical Softw. Engg. 23, 4 (aug 2018), 1959--1981.","journal-title":"Empirical Softw. Engg."},{"key":"e_1_3_2_1_5_1","volume-title":"Pre-Trained Language Models on Code. ArXiv abs\/2206.01335","author":"Barei\u00df Patrick","year":"2022","unstructured":"Patrick Barei\u00df, Beatriz Souza, Marcelo d'Amorim, and Michael Pradel. 2022. Code Generation Tools (Almost) for Free? A Study of Few-Shot, Pre-Trained Language Models on Code. ArXiv abs\/2206.01335 (2022)."},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2014.2372785"},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"publisher","DOI":"10.1145\/2786805.2786843"},{"key":"e_1_3_2_1_8_1","volume-title":"Jared Kaplan, Harri Edwards, Yuri Burda, Nicholas Joseph, Greg Brockman, et al.","author":"Chen Mark","year":"2021","unstructured":"Mark Chen, Jerry Tworek, Heewoo Jun, Qiming Yuan, Henrique Ponde de Oliveira Pinto, Jared Kaplan, Harri Edwards, Yuri Burda, Nicholas Joseph, Greg Brockman, et al. 2021. Evaluating Large Language Models Trained on Code. arXiv preprint arXiv:2107.03374 (2021)."},{"key":"e_1_3_2_1_9_1","volume-title":"How Robust is GPT-3.5 to Predecessors? A Comprehensive Study on Language Understanding Tasks. arXiv preprint arXiv:2303.00293","author":"Chen Xuanting","year":"2023","unstructured":"Xuanting Chen, Junjie Ye, Can Zu, Nuo Xu, Rui Zheng, Minlong Peng, Jie Zhou, Tao Gui, Qi Zhang, and Xuanjing Huang. 2023. How Robust is GPT-3.5 to Predecessors? A Comprehensive Study on Language Understanding Tasks. arXiv preprint arXiv:2303.00293 (2023)."},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.1145\/2786805.2786838"},{"key":"e_1_3_2_1_11_1","volume-title":"International Symposium on Software Reliability Engineering (ISSRE). IEEE, 201--211","author":"Daka Ermira","year":"2014","unstructured":"Ermira Daka and Gordon Fraser. 2014. A Survey on Unit Testing Practices and Problems. In International Symposium on Software Reliability Engineering (ISSRE). IEEE, 201--211."},{"key":"e_1_3_2_1_12_1","volume-title":"Codebert: A pre-trained model for programming and natural languages. arXiv preprint arXiv:2002.08155","author":"Feng Zhangyin","year":"2020","unstructured":"Zhangyin Feng, Daya Guo, Duyu Tang, Nan Duan, Xiaocheng Feng, Ming Gong, Linjun Shou, Bing Qin, Ting Liu, Daxin Jiang, et al. 2020. Codebert: A pre-trained model for programming and natural languages. arXiv preprint arXiv:2002.08155 (2020)."},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1145\/2025113.2025179"},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.1145\/2699688"},{"key":"e_1_3_2_1_15_1","volume-title":"An Empirical Investigation on the Readability of Manual and Generated Test Cases. In 2018 IEEE\/ACM 26th International Conference on Program Comprehension (ICPC). 348--3483","author":"Grano Giovanni","year":"2018","unstructured":"Giovanni Grano, Simone Scalabrino, Harald C. Gall, and Rocco Oliveto. 2018. An Empirical Investigation on the Readability of Manual and Generated Test Cases. In 2018 IEEE\/ACM 26th International Conference on Program Comprehension (ICPC). 348--3483."},{"key":"e_1_3_2_1_16_1","volume-title":"Empirical Study on Test Generation Using GitHub Copilot. Master's thesis","author":"Haji Khalid El","unstructured":"Khalid El Haji. 2023. Empirical Study on Test Generation Using GitHub Copilot. Master's thesis. Delft University of Technology."},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","unstructured":"Khalid El Haji. 2023. Empirical Study on Test Generation Using GitHub Copilot --- Replication Package. 10.5281\/zenodo.8025746","DOI":"10.5281\/zenodo.8025746"},{"key":"e_1_3_2_1_18_1","volume-title":"Preventing Verbatim Memorization in Language Models Gives a False Sense of Privacy. arXiv preprint arXiv:2210.17546","author":"Ippolito Daphne","year":"2022","unstructured":"Daphne Ippolito, Florian Tram\u00e8r, Milad Nasr, Chiyuan Zhang, Matthew Jagielski, Katherine Lee, Christopher A Choquette-Choo, and Nicholas Carlini. 2022. Preventing Verbatim Memorization in Language Models Gives a False Sense of Privacy. arXiv preprint arXiv:2210.17546 (2022)."},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","DOI":"10.1145\/3571730"},{"key":"e_1_3_2_1_20_1","volume-title":"Proceedings of the International Conference on Mining Software Repositories (MSR). ACM, 1--5.","author":"Nguyen Nhan","year":"2022","unstructured":"Nhan Nguyen and Sarah Nadi. 2022. An Empirical Evaluation of GitHub Copilot's Code Suggestions. In Proceedings of the International Conference on Mining Software Repositories (MSR). ACM, 1--5."},{"key":"e_1_3_2_1_21_1","volume-title":"Codegen: An open large language model for code with multi-turn program synthesis. arXiv preprint arXiv:2203.13474","author":"Nijkamp Erik","year":"2022","unstructured":"Erik Nijkamp, Bo Pang, Hiroaki Hayashi, Lifu Tu, Huan Wang, Yingbo Zhou, Silvio Savarese, and Caiming Xiong. 2022. Codegen: An open large language model for code with multi-turn program synthesis. arXiv preprint arXiv:2203.13474 (2022)."},{"key":"e_1_3_2_1_22_1","volume-title":"Ernst","author":"Pacheco Carlos","year":"2007","unstructured":"Carlos Pacheco and Michael D. Ernst. 2007. Randoop: Feedback-Directed Random Testing for Java. In Companion to the 22nd ACM SIGPLAN Conference on Object-Oriented Programming Systems and Applications Companion (OOPSLA-Companion). ACM, 815--816."},{"key":"e_1_3_2_1_23_1","volume-title":"Adaptive Test Generation Using a Large Language Model. arXiv preprint arXiv:2302.06527","author":"Sch\u00e4fer Max","year":"2023","unstructured":"Max Sch\u00e4fer, Sarah Nadi, Aryaz Eghbali, and Frank Tip. 2023. Adaptive Test Generation Using a Large Language Model. arXiv preprint arXiv:2302.06527 (2023)."},{"key":"e_1_3_2_1_24_1","volume-title":"Proceedings of the International Conference on Mining Software Repositories (MSR). IEEE, 121--125","author":"Serra Domenico","year":"2019","unstructured":"Domenico Serra, Giovanni Grano, Fabio Palomba, Filomena Ferrucci, Harald C. Gall, and Alberto Bacchelli. 2019. On the Effectiveness of Manual and Automatic Unit Test Generation: Ten Years Later. In Proceedings of the International Conference on Mining Software Repositories (MSR). IEEE, 121--125."},{"key":"e_1_3_2_1_25_1","volume-title":"Noshin Ulfat, Fahmid Al Rifat, and Vinicius Carvalho Lopes.","author":"Siddiq Mohammed Latif","year":"2023","unstructured":"Mohammed Latif Siddiq, Joanna Santos, Ridwanul Hasan Tanvir, Noshin Ulfat, Fahmid Al Rifat, and Vinicius Carvalho Lopes. 2023. Exploring the Effectiveness of Large Language Models in Generating Unit Tests. arXiv preprint arXiv:2305.00418 (2023)."},{"key":"e_1_3_2_1_26_1","volume-title":"Attention is all you need. Advances in neural information processing systems 30","author":"Vaswani Ashish","year":"2017","unstructured":"Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, Llion Jones, Aidan N Gomez, \u0141ukasz Kaiser, and Illia Polosukhin. 2017. Attention is all you need. Advances in neural information processing systems 30 (2017)."},{"key":"e_1_3_2_1_27_1","volume-title":"Proceedings of the 2021 Conference on Empirical Methods in Natural Language Processing. Association for Computational Linguistics, 8696--8708","author":"Wang Yue","unstructured":"Yue Wang, Weishi Wang, Shafiq Joty, and Steven C.H. Hoi. 2021. CodeT5: Identifier-aware Unified Pre-trained Encoder-Decoder Models for Code Understanding and Generation. In Proceedings of the 2021 Conference on Empirical Methods in Natural Language Processing. Association for Computational Linguistics, 8696--8708."},{"key":"e_1_3_2_1_28_1","volume-title":"ChatUniTest: a ChatGPT-based automated unit test generation tool. arXiv preprint arXiv:2305.04764","author":"Xie Zhuokui","year":"2023","unstructured":"Zhuokui Xie, Yinghao Chen, Chen Zhi, Shuiguang Deng, and Jianwei Yin. 2023. ChatUniTest: a ChatGPT-based automated unit test generation tool. arXiv preprint arXiv:2305.04764 (2023)."},{"key":"e_1_3_2_1_29_1","volume-title":"No More Manual Tests? Evaluating and Improving ChatGPT for Unit Test Generation. arXiv preprint arXiv:2305.04207","author":"Yuan Zhiqiang","year":"2023","unstructured":"Zhiqiang Yuan, Yiling Lou, Mingwei Liu, Shiji Ding, Kaixin Wang, Yixuan Chen, and Xin Peng. 2023. No More Manual Tests? Evaluating and Improving ChatGPT for Unit Test Generation. arXiv preprint arXiv:2305.04207 (2023)."},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"publisher","DOI":"10.1145\/3520312.3534864"}],"event":{"name":"AST '24: 5th ACM\/IEEE International Conference on Automation of Software Test (AST 2024)","location":"Lisbon Portugal","acronym":"AST '24","sponsor":["SIGSOFT ACM Special Interest Group on Software Engineering","IEEE TCSE"]},"container-title":["Proceedings of the 5th ACM\/IEEE International Conference on Automation of Software Test (AST 2024)"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3644032.3644443","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3644032.3644443","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,18]],"date-time":"2025-06-18T23:56:57Z","timestamp":1750291017000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3644032.3644443"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,4,15]]},"references-count":30,"alternative-id":["10.1145\/3644032.3644443","10.1145\/3644032"],"URL":"https:\/\/doi.org\/10.1145\/3644032.3644443","relation":{},"subject":[],"published":{"date-parts":[[2024,4,15]]},"assertion":[{"value":"2024-06-10","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}