{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,9]],"date-time":"2026-04-09T14:36:48Z","timestamp":1775745408443,"version":"3.50.1"},"publisher-location":"New York, NY, USA","reference-count":52,"publisher":"ACM","license":[{"start":{"date-parts":[[2024,7,10]],"date-time":"2024-07-10T00:00:00Z","timestamp":1720569600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2024,7,10]]},"DOI":"10.1145\/3663529.3663839","type":"proceedings-article","created":{"date-parts":[[2024,7,10]],"date-time":"2024-07-10T19:43:13Z","timestamp":1720640593000},"page":"185-196","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":79,"title":["Automated Unit Test Improvement using Large Language Models at Meta"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0009-0009-4763-0396","authenticated-orcid":false,"given":"Nadia","family":"Alshahwan","sequence":"first","affiliation":[{"name":"Meta Platforms, Menlo Park, USA"}]},{"ORCID":"https:\/\/orcid.org\/0009-0005-0311-7890","authenticated-orcid":false,"given":"Jubin","family":"Chheda","sequence":"additional","affiliation":[{"name":"Meta Platforms, Menlo Park, USA"}]},{"ORCID":"https:\/\/orcid.org\/0009-0005-5824-5179","authenticated-orcid":false,"given":"Anastasia","family":"Finogenova","sequence":"additional","affiliation":[{"name":"Meta Platforms, Menlo Park, USA"}]},{"ORCID":"https:\/\/orcid.org\/0009-0003-0197-6806","authenticated-orcid":false,"given":"Beliz","family":"Gokkaya","sequence":"additional","affiliation":[{"name":"Meta Platforms, Menlo Park, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-5864-4488","authenticated-orcid":false,"given":"Mark","family":"Harman","sequence":"additional","affiliation":[{"name":"Meta Platforms, Menlo Park, USA \/ University College London, London, United Kingdom"}]},{"ORCID":"https:\/\/orcid.org\/0009-0008-9359-0949","authenticated-orcid":false,"given":"Inna","family":"Harper","sequence":"additional","affiliation":[{"name":"Meta Platforms, Menlo Park, USA"}]},{"ORCID":"https:\/\/orcid.org\/0009-0001-5311-762X","authenticated-orcid":false,"given":"Alexandru","family":"Marginean","sequence":"additional","affiliation":[{"name":"Meta Platforms, Menlo Park, USA"}]},{"ORCID":"https:\/\/orcid.org\/0009-0007-4204-5185","authenticated-orcid":false,"given":"Shubho","family":"Sengupta","sequence":"additional","affiliation":[{"name":"Meta Platforms, Menlo Park, USA"}]},{"ORCID":"https:\/\/orcid.org\/0009-0009-8825-6986","authenticated-orcid":false,"given":"Eddy","family":"Wang","sequence":"additional","affiliation":[{"name":"Meta Platforms, Menlo Park, USA"}]}],"member":"320","published-online":{"date-parts":[[2024,7,10]]},"reference":[{"key":"e_1_3_2_1_1_1","volume-title":"Toward the Next Generation of Recommender Systems: A Survey of the State-of-the-Art and Possible Extensions","author":"Tuzhilin Adomavicius","year":"2005","unstructured":"Adomavicius and Tuzhilin. 2005. Toward the Next Generation of Recommender Systems: A Survey of the State-of-the-Art and Possible Extensions. IEEE Transactions on Knowledge and Data Engineering, 17 (2005)."},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE-SEIP52600.2021.00023"},{"key":"e_1_3_2_1_3_1","volume-title":"Premkumar Devanbu, and Earl T. Barr.","author":"Ahmed Toufique","year":"2023","unstructured":"Toufique Ahmed, Kunal Suresh Pai, Premkumar Devanbu, and Earl T. Barr. 2023. Improving Few-Shot Prompts with Relevant Static Analysis Products. arXiv:2304.06815"},{"key":"e_1_3_2_1_4_1","volume-title":"Deploying Search Based Software Engineering with Sapienz at Facebook (keynote paper). In 10^th International Symposium on Search Based Software Engineering (SSBSE","author":"Alshahwan Nadia","year":"2018","unstructured":"Nadia Alshahwan, Xinbo Gao, Mark Harman, Yue Jia, Ke Mao, Alexander Mols, Taijin Tei, and Ilya Zorin. 2018. Deploying Search Based Software Engineering with Sapienz at Facebook (keynote paper). In 10^th International Symposium on Search Based Software Engineering (SSBSE 2018). Montpellier, France. 3\u201345. Springer LNCS 11036"},{"key":"e_1_3_2_1_5_1","volume-title":"Software Testing Research Challenges: An Industrial Perspective. In 2023 IEEE Conference on Software Testing, Verification and Validation (ICST","author":"Alshahwan Nadia","year":"2023","unstructured":"Nadia Alshahwan, Mark Harman, and Alexandru Marginean. 2023. Software Testing Research Challenges: An Industrial Perspective. In 2023 IEEE Conference on Software Testing, Verification and Validation (ICST 2023). 1\u201310."},{"key":"e_1_3_2_1_6_1","volume-title":"ICSE workshop on Interoperability and Robustness of Neural Software Engineering (InteNSE). To appear","author":"Alshahwan Nadia","year":"2024","unstructured":"Nadia Alshahwan, Mark Harman, Alexandru Marginean, Shubho Sengupta, and Eddy Wang. 2024. Assured LLM-Based Software Engineering (keynote paper). In 2^nd. ICSE workshop on Interoperability and Robustness of Neural Software Engineering (InteNSE). To appear"},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2014.2372785"},{"key":"e_1_3_2_1_8_1","volume-title":"2021 IEEE\/ACM 43rd International Conference on Software Engineering: Software Engineering in Practice (ICSE-SEIP). 268\u2013277","author":"Beller Moritz","year":"2021","unstructured":"Moritz Beller, Chu-Pan Wong, Johannes Bader, Andrew Scott, Mateusz Machalica, Satish Chandra, and Erik Meijer. 2021. What it would take to use mutation testing in industry\u2014a study at Facebook. In 2021 IEEE\/ACM 43rd International Conference on Software Engineering: Software Engineering in Practice (ICSE-SEIP). 268\u2013277."},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE.2017.61"},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.1145\/3510003.3510194"},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.1007\/s11704-019-8208-z"},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE-FoSE59343.2023.00008"},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1145\/2685612"},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.1007\/s11219-013-9208-0"},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.1145\/3318162"},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"crossref","unstructured":"Mark Harman. 2010. Why Source Code Analysis and Manipulation Will Always Be Important (Keynote Paper). In 10^th IEEE International Working Conference on Source Code Analysis and Manipulation. Timisoara Romania.","DOI":"10.1109\/SCAM.2010.28"},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.1016\/S0950-5849(01)00189-6"},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1145\/2379776.2379787"},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"crossref","unstructured":"Mark Harman and Peter O\u2019Hearn. 2018. From Start-ups to Scale-ups: Opportunities and Open Problems for Static and Dynamic Program Analysis (keynote paper). In 18^th IEEE International Working Conference on Source Code Analysis and Manipulation (SCAM 2018). Madrid Spain. 1\u201323.","DOI":"10.1109\/SCAM.2018.00009"},{"key":"e_1_3_2_1_20_1","unstructured":"Will Douglas Heaven. 2023. ChatGPT is going to change education not destroy it. MIT Technology review April."},{"key":"e_1_3_2_1_21_1","unstructured":"Jie Hu Qian Zhang and Heng Yin. 2023. Augmenting Greybox Fuzzing with Generative AI. arXiv:2306.06782"},{"key":"e_1_3_2_1_22_1","unstructured":"Kai Huang Zhengzi Xu Su Yang Hongyu Sun Xuejun Li Zheng Yan and Yuqing Zhang. 2023. A Survey on Automated Program Repair Techniques. arxiv:2303.18184."},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2010.62"},{"key":"e_1_3_2_1_24_1","volume-title":"Inferfix: End-to-end program repair with LLMs. arXiv preprint arXiv:2303.07263.","author":"Jin Matthew","year":"2023","unstructured":"Matthew Jin, Syed Shahriar, Michele Tufano, Xin Shi, Shuai Lu, Neel Sundaresan, and Alexey Svyatkovskiy. 2023. Inferfix: End-to-end program repair with LLMs. arXiv preprint arXiv:2303.07263."},{"key":"e_1_3_2_1_25_1","article-title":"Optimising Existing Software with Genetic Programming","volume":"19","author":"Langdon William B.","year":"2015","unstructured":"William B. Langdon and Mark Harman. 2015. Optimising Existing Software with Genetic Programming. IEEE Transactions on Evolutionary Computation (TEVC), 19, 1 (2015), Feb, 118\u2013135.","journal-title":"IEEE Transactions on Evolutionary Computation (TEVC)"},{"key":"e_1_3_2_1_26_1","volume-title":"Shuvendu K Lahiri, and Siddhartha Sen.","author":"Lemieux Caroline","year":"2023","unstructured":"Caroline Lemieux, Jeevana Priya Inala, Shuvendu K Lahiri, and Siddhartha Sen. 2023. CODAMOSA: Escaping Coverage Plateaus in Test Generation with Pre-trained Large Language Models."},{"key":"e_1_3_2_1_27_1","doi-asserted-by":"publisher","DOI":"10.1145\/3611643.3613078"},{"key":"e_1_3_2_1_28_1","volume-title":"An empirical analysis of flaky tests. In 22^nd International Symposium on Foundations of Software Engineering (FSE","author":"Luo Qingzhou","year":"2014","unstructured":"Qingzhou Luo, Farah Hariri, Lamyaa Eloussi, and Darko Marinov. 2014. An empirical analysis of flaky tests. In 22^nd International Symposium on Foundations of Software Engineering (FSE 2014), Shing-Chi Cheung, Alessandro Orso, and Margaret-Anne Storey (Eds.). ACM, Hong Kong, China. 643\u2013653. isbn:978-1-4503-3056-5"},{"key":"e_1_3_2_1_29_1","unstructured":"Wei Ma Shangqing Liu Wenhan Wang Qiang Hu Ye Liu Cen Zhang Liming Nie and Yang Liu. 2023. The Scope of ChatGPT in Software Engineering: A Thorough Investigation. arXiv:2305.12138"},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"publisher","DOI":"10.1145\/3545945.3569785"},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE-SEIP.2019.00039"},{"key":"e_1_3_2_1_32_1","unstructured":"Mark Chen et al.. 2021. Evaluating Large Language Models Trained on Code. arXiv:2107.03374"},{"key":"e_1_3_2_1_33_1","unstructured":"Laura Meckler and Pranshu Verma. 2022. Teachers are on alert for inevitable cheating after release of ChatGPT. The Washington post December."},{"key":"e_1_3_2_1_34_1","volume-title":"Seung-won Hwang, and Jinyoung Yeo.","author":"Moon Seungjun","year":"2023","unstructured":"Seungjun Moon, Yongho Song, Hyungjoo Chae, Dongjin Kang, Taeyoon Kwon, Kai Tzu-iunn Ong, Seung-won Hwang, and Jinyoung Yeo. 2023. Coffee: Boost Your Code LLMs by Fixing Bugs with Feedback. arXiv preprint arXiv:2311.07215."},{"key":"e_1_3_2_1_35_1","unstructured":"Arghavan Moradi Dakhel Amin Nikanjam Vahid Majdinasab Foutse Khomh and Michel C Desmarais. 2023. Effective Test Generation Using Pre-trained Large Language Models and Mutation Testing. arXiv e-prints arXiv\u20132308."},{"key":"e_1_3_2_1_36_1","unstructured":"Ruba Mutasim Gabriel Synnaeve David Pichardie and Baptiste Rozi\u00e8re. 2023. Leveraging Static Analysis for Bug Repair. arxiv:2304.10379."},{"key":"e_1_3_2_1_37_1","volume-title":"Raymond J. Mooney, and Milos Gligoric.","author":"Nie Pengyu","year":"2023","unstructured":"Pengyu Nie, Rahul Banerjee, Junyi Jessy Li, Raymond J. Mooney, and Milos Gligoric. 2023. Learning Deep Semantics for Test Completion. arXiv:2302.10166"},{"key":"e_1_3_2_1_38_1","unstructured":"David Noever and Kevin Williams. 2023. Chatbots As Fluent Polyglots: Revisiting Breakthrough Code Snippets. arXiv:2301.03373"},{"key":"e_1_3_2_1_39_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.jgo.2021.12.017"},{"key":"e_1_3_2_1_40_1","doi-asserted-by":"publisher","DOI":"10.1109\/TEVC.2017.2693219"},{"key":"e_1_3_2_1_41_1","unstructured":"Baptiste Rozi\u00e8re Jonas Gehring Fabian Gloeckle Sten Sootla Itai Gat Xiaoqing Ellen Tan Yossi Adi Jingyu Liu Tal Remez J\u00e9r\u00e9my Rapin Artyom Kozhevnikov Ivan Evtimov Joanna Bitton Manish Bhatt Cristian Canton Ferrer Aaron Grattafiori Wenhan Xiong Alexandre D\u00e9fossez Jade Copet Faisal Azhar Hugo Touvron Louis Martin Nicolas Usunier Thomas Scialom and Gabriel Synnaeve. 2023. Code Llama: Open Foundation Models for Code. arXiv:2308.12950"},{"key":"e_1_3_2_1_42_1","doi-asserted-by":"publisher","DOI":"10.1145\/3501385.3543957"},{"key":"e_1_3_2_1_43_1","unstructured":"Max Sch\u00e4fer Sarah Nadi Aryaz Eghbali and Frank Tip. 2023. Adaptive Test Generation Using a Large Language Model. arXiv:2302.06527"},{"key":"e_1_3_2_1_44_1","volume-title":"Noshin Ulfat, Fahmid Al Rifat, and Vinicius Carvalho Lopes.","author":"Siddiq Mohammed Latif","year":"2023","unstructured":"Mohammed Latif Siddiq, Joanna C. S. Santos, Ridwanul Hasan Tanvir, Noshin Ulfat, Fahmid Al Rifat, and Vinicius Carvalho Lopes. 2023. Exploring the Effectiveness of Large Language Models in Generating Unit Tests. arXiv:2305.00418"},{"key":"e_1_3_2_1_45_1","unstructured":"Jeffrey Svajlenko and Chanchal K Roy. 2020. A Survey on the Evaluation of Clone Detection Performance and Benchmarking. arXiv preprint arXiv:2006.15682."},{"key":"e_1_3_2_1_46_1","doi-asserted-by":"publisher","DOI":"10.1109\/TR.2021.3101318"},{"key":"e_1_3_2_1_47_1","unstructured":"Hugo Touvron Thibaut Lavril Gautier Izacard Xavier Martinet Marie-Anne Lachaux Timoth\u00e9e Lacroix Baptiste Rozi\u00e8re Naman Goyal Eric Hambro Faisal Azhar Aurelien Rodriguez Armand Joulin Edouard Grave and Guillaume Lample. 2023. LLaMA: Open and Efficient Foundation Language Models. arXiv:2302.13971"},{"key":"e_1_3_2_1_48_1","volume-title":"40th International Conference on Software Engineering, Software Engineering in Practice track (ICSE 2018 SEIP track). 1\u201310","author":"Urli Simon","year":"2018","unstructured":"Simon Urli, Zhongxing Yu, Lionel Seinturier, and Martin Monperrus. 2018. How to Design a Program Repair Bot? Insights from the Repairnator Project. In 40th International Conference on Software Engineering, Software Engineering in Practice track (ICSE 2018 SEIP track). 1\u201310."},{"key":"e_1_3_2_1_49_1","unstructured":"Junjie Wang Yuchao Huang Chunyang Chen Zhe Liu Song Wang and Qing Wang. 2023. Software Testing with Large Language Model: Survey Landscape and Vision. arXiv:2307.07221"},{"key":"e_1_3_2_1_50_1","volume-title":"Michael Pradel, and Lingming Zhang.","author":"Xia Chunqiu Steven","year":"2023","unstructured":"Chunqiu Steven Xia, Matteo Paltenghi, Jia Le Tian, Michael Pradel, and Lingming Zhang. 2023. Universal Fuzzing via Large Language Models. arXiv preprint arXiv:2308.04748."},{"key":"e_1_3_2_1_51_1","doi-asserted-by":"publisher","DOI":"10.1002\/stv.430"},{"key":"e_1_3_2_1_52_1","unstructured":"Zhiqiang Yuan Yiling Lou Mingwei Liu Shiji Ding Kaixin Wang Yixuan Chen and Xin Peng. 2023. No More Manual Tests? Evaluating and Improving ChatGPT for Unit Test Generation. arXiv:2305.04207"}],"event":{"name":"FSE '24: 32nd ACM International Conference on the Foundations of Software Engineering","location":"Porto de Galinhas Brazil","acronym":"FSE '24","sponsor":["SIGSOFT ACM Special Interest Group on Software Engineering"]},"container-title":["Companion Proceedings of the 32nd ACM International Conference on the Foundations of Software Engineering"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3663529.3663839","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3663529.3663839","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,18]],"date-time":"2025-06-18T23:44:21Z","timestamp":1750290261000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3663529.3663839"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,7,10]]},"references-count":52,"alternative-id":["10.1145\/3663529.3663839","10.1145\/3663529"],"URL":"https:\/\/doi.org\/10.1145\/3663529.3663839","relation":{},"subject":[],"published":{"date-parts":[[2024,7,10]]},"assertion":[{"value":"2024-07-10","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}