{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,11]],"date-time":"2026-04-11T06:18:27Z","timestamp":1775888307163,"version":"3.50.1"},"publisher-location":"New York, NY, USA","reference-count":63,"publisher":"ACM","content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2025,6,23]]},"DOI":"10.1145\/3696630.3728544","type":"proceedings-article","created":{"date-parts":[[2025,7,28]],"date-time":"2025-07-28T19:08:09Z","timestamp":1753729689000},"page":"180-191","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":4,"title":["Mutation-Guided LLM-based Test Generation at Meta"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-5864-4488","authenticated-orcid":false,"given":"Mark","family":"Harman","sequence":"first","affiliation":[{"name":"Meta platforms and UCL, london, United Kingdom"}]},{"ORCID":"https:\/\/orcid.org\/0009-0000-2014-8932","authenticated-orcid":false,"given":"Jillian","family":"Ritchey","sequence":"additional","affiliation":[{"name":"Meta platforms, san francisco, USA"}]},{"ORCID":"https:\/\/orcid.org\/0009-0008-9359-0949","authenticated-orcid":false,"given":"Inna","family":"Harper","sequence":"additional","affiliation":[{"name":"Meta platforms, london, United Kingdom"}]},{"ORCID":"https:\/\/orcid.org\/0009-0007-4204-5185","authenticated-orcid":false,"given":"Shubho","family":"Sengupta","sequence":"additional","affiliation":[{"name":"Meta platforms, san francisco, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-3956-9184","authenticated-orcid":false,"given":"Ke","family":"Mao","sequence":"additional","affiliation":[{"name":"Meta platforms, london, United Kingdom"}]},{"ORCID":"https:\/\/orcid.org\/0009-0008-5347-5321","authenticated-orcid":false,"given":"Abhishek","family":"Gulati","sequence":"additional","affiliation":[{"name":"Meta platforms, san francisco, USA"}]},{"ORCID":"https:\/\/orcid.org\/0009-0003-6819-0568","authenticated-orcid":false,"given":"Christopher","family":"Foster","sequence":"additional","affiliation":[{"name":"Meta platforms, san francisco, USA"}]},{"ORCID":"https:\/\/orcid.org\/0009-0005-9727-4562","authenticated-orcid":false,"given":"Herv\u00e9","family":"Robert","sequence":"additional","affiliation":[{"name":"Meta platforms, san francisco, USA"}]}],"member":"320","published-online":{"date-parts":[[2025,7,28]]},"reference":[{"key":"e_1_3_2_1_1_1","volume-title":"International Conference on Software Engineering (ICSE) Software Engineering in Practice (SEIP) track","author":"Ahlgren John","year":"2021","unstructured":"John Ahlgren, Maria Eugenia Berezin, Kinga Bojarczuk, Elena Dulskyte, Inna Dvortsova, Johann George, Natalija Gucevska, Mark Harman, Maria Lomeli, Erik Meijer, Silvia Sapora, and Justin Spahr-Summers. Testing web enabled simulation at scale using metamorphic testing. In International Conference on Software Engineering (ICSE) Software Engineering in Practice (SEIP) track, Virtual, 2021."},{"key":"e_1_3_2_1_2_1","volume-title":"25th International Conference on Evaluation and Assessment in Software Engineering (EASE 2021","author":"Ahlgren John","year":"2021","unstructured":"John Ahlgren, Kinga Bojarczuk, Sophia Drossopoulou, Inna Dvortsova, Johann George, Natalija Gucevska, Mark Harman, Maria Lomeli, Simon Lucas, Erik Meijer, Steve Omohundro, Rubmary Rojas, Silvia Sapora, Jie M. Zhang, and Norm Zhou. Facebook's cyber-cyber and cyber-physical digital twins (keynote paper). In 25th International Conference on Evaluation and Assessment in Software Engineering (EASE 2021), Virtual, June 2021."},{"key":"e_1_3_2_1_3_1","volume-title":"ACM International Conference on the Foundations of Software Engineering (FSE 2024)","author":"Alshahwan Nadia","year":"2024","unstructured":"Nadia Alshahwan, Jubin Chheda, Anastasia Finegenova, Mark Harman, Alexandru Marginean, Shubho Sengupta, and Eddy Wang. Automated unit test improvement using Large Language Models at Meta. In ACM International Conference on the Foundations of Software Engineering (FSE 2024), July 2024."},{"key":"e_1_3_2_1_4_1","first-page":"3","volume-title":"Proceedings of the 28th ACM SIGSOFT International Symposium on Software Testing and Analysis (ISSTA 2019","author":"Alshahwan Nadia","year":"2019","unstructured":"Nadia Alshahwan, Andrea Ciancone, Mark Harman, Yue Jia, Ke Mao, Alexandru Marginean, Alexander Mols, Hila Peleg, Federica Sarro, and Ilya Zorin. Some challenges for software testing research (keynote paper). In Dongmei Zhang and Anders M\u00f8ller, editors, Proceedings of the 28th ACM SIGSOFT International Symposium on Software Testing and Analysis (ISSTA 2019), Beijing, China, July 15\u201319, 2019, pages 1\u20133. ACM, 2019."},{"key":"e_1_3_2_1_5_1","first-page":"45","volume-title":"10th International Symposium on Search Based Software Engineering (SSBSE 2018","author":"Alshahwan Nadia","year":"2018","unstructured":"Nadia Alshahwan, Xinbo Gao, Mark Harman, Yue Jia, Ke Mao, Alexander Mols, Taijin Tei, and Ilya Zorin. Deploying search based software engineering with Sapienz at Facebook (keynote paper). In 10th International Symposium on Search Based Software Engineering (SSBSE 2018), pages 3\u201345, Montpellier, France, September 8th-10th 2018. Springer LNCS 11036."},{"key":"e_1_3_2_1_6_1","first-page":"10","volume-title":"2023 IEEE Conference on Software Testing, Verification and Validation (ICST","author":"Alshahwan Nadia","year":"2023","unstructured":"Nadia Alshahwan, Mark Harman, and Alexandru Marginean. Software testing research challenges: An industrial perspective (keynote paper). In 2023 IEEE Conference on Software Testing, Verification and Validation (ICST 2023), pages 1\u201310. IEEE, 2023."},{"key":"e_1_3_2_1_7_1","volume-title":"2nd ICSE workshop on Interoperability and Robustness of Neural Software Engineering (InteNSE)","author":"Alshahwan Nadia","year":"2024","unstructured":"Nadia Alshahwan, Mark Harman, Alexandru Marginean, Shubho Sengupta, and Eddy Wang. Assured llm-based software engineering (keynote paper). In 2nd ICSE workshop on Interoperability and Robustness of Neural Software Engineering (InteNSE), April 2024."},{"key":"e_1_3_2_1_8_1","volume-title":"Foundations of Software Engineering (FSE 2024)","author":"Alshahwan Nadia","year":"2024","unstructured":"Nadia Alshahwan, Mark Harman, Alexandru Marginean, and Eddy Wang. Observation-based unit test generation at meta. In Foundations of Software Engineering (FSE 2024), 2024."},{"key":"e_1_3_2_1_9_1","first-page":"583","volume-title":"36th International Conference on Software Engineering (ICSE 2014","author":"Androutsopoulos Kelly","year":"2014","unstructured":"Kelly Androutsopoulos, David Clark, Haitao Dan, Mark Harman, and Robert Hierons. An analysis of the relationship between conditional entropy and failed error propagation in software testing. In 36th International Conference on Software Engineering (ICSE 2014), pages 573\u2013583, Hyderabad, India, June 2014."},{"key":"e_1_3_2_1_10_1","first-page":"317","volume-title":"22nd ACM SIGSOFT International Symposium on the Foundations of Software Engineering (FSE 2014","author":"Barr Earl T.","year":"2014","unstructured":"Earl T. Barr, Yuriy Brun, Premkumar Devanbu, Mark Harman, and Federica Sarro. The plastic surgery hypothesis. In 22nd ACM SIGSOFT International Symposium on the Foundations of Software Engineering (FSE 2014), pages 306\u2013317, Hong Kong, China, November 2014."},{"issue":"5","key":"e_1_3_2_1_11_1","doi-asserted-by":"crossref","first-page":"507","DOI":"10.1109\/TSE.2014.2372785","article-title":"The oracle problem in software testing: A survey","volume":"41","author":"Barr Earl T.","year":"2015","unstructured":"Earl T. Barr, Mark Harman, Phil McMinn, Muzammil Shahbaz, and Shin Yoo. The oracle problem in software testing: A survey. IEEE Transactions on Software Engineering, 41(5):507\u2013525, May 2015.","journal-title":"IEEE Transactions on Software Engineering"},{"key":"e_1_3_2_1_12_1","first-page":"277","volume-title":"2021 IEEE\/ACM 43rd International Conference on Software Engineering: Software Engineering in Practice (ICSE-SEIP)","author":"Beller Moritz","unstructured":"Moritz Beller, Chu-Pan Wong, Johannes Bader, Andrew Scott, Mateusz Machalica, Satish Chandra, and Erik Meijer. What it would take to use mutation testing in industry\u2014a study at Facebook. In 2021 IEEE\/ACM 43rd International Conference on Software Engineering: Software Engineering in Practice (ICSE-SEIP), pages 268\u2013277. IEEE, 2021."},{"key":"e_1_3_2_1_13_1","volume-title":"1st ACM International Conference on AI-powered Software (AIware 2024)","author":"Brown Adam","year":"2024","unstructured":"Adam Brown, Sarah D'Angelo, Ambar Murillo, Ciera Jaspan, and Collin Green. Identifying the factors that influence trust in ai code completion. In 1st ACM International Conference on AI-powered Software (AIware 2024), July 2024."},{"key":"e_1_3_2_1_14_1","first-page":"159","volume-title":"International Symposium on Search Based Software Engineering","author":"Brownlee Alexander","unstructured":"Alexander Brownlee, James Callan, Karine Even-Mendoza, Alina Geiger, Carol Hanna, Justyna Petke, Federica Sarro, and Dominik Sobania. Enhancing genetic improvement mutations using large language models. In International Symposium on Search Based Software Engineering, pages 153\u2013159. Springer, 2023."},{"key":"e_1_3_2_1_15_1","first-page":"11","volume-title":"NASA Formal Methods - 7th International Symposium","author":"Calcagno C.","year":"2015","unstructured":"C. Calcagno, D. Distefano, J. Dubreil, D. Gabi, P. Hooimeijer, M. Luca, P. W. O'Hearn, I. Papakonstantinou, J. Purbrick, and D. Rodriguez. Moving fast with software verification. In NASA Formal Methods - 7th International Symposium, pages 3\u201311, 2015."},{"key":"e_1_3_2_1_16_1","volume-title":"Workshop on Experimental Software Engineering (ESELAW'14)","author":"Carlos Francisco","year":"2014","unstructured":"Francisco Carlos, Mike Papadakis, Vin\u00edcius Durelli, and Eduardo M\u00e1rcio Delamaro. Test data generation techniques for mutation testing: A systematic mapping. In Workshop on Experimental Software Engineering (ESELAW'14), 2014."},{"key":"e_1_3_2_1_17_1","first-page":"608","volume-title":"Proceedings of the 39th International Conference on Software Engineering, ICSE 2017","author":"Chekam Thierry Titcheu","year":"2017","unstructured":"Thierry Titcheu Chekam, Mike Papadakis, Yves Le Traon, and Mark Harman. An empirical study on mutation, statement and branch coverage fault revelation that avoids the unreliable clean program assumption. In Proceedings of the 39th International Conference on Software Engineering, ICSE 2017, Buenos Aires, Argentina, May 20\u201328, 2017, pages 597\u2013608, 2017."},{"key":"e_1_3_2_1_18_1","first-page":"576","volume-title":"Companion Proceedings of the 32nd ACM International Conference on the Foundations of Software Engineering","author":"Chen Yinghao","year":"2024","unstructured":"Yinghao Chen, Zehao Hu, Chen Zhi, Junxiao Han, Shuiguang Deng, and Jianwei Yin. Chatunitest: A framework for LLM-based test generation. In Companion Proceedings of the 32nd ACM International Conference on the Foundations of Software Engineering, pages 572\u2013576, 2024."},{"key":"e_1_3_2_1_19_1","first-page":"452","volume-title":"Proceedings of the 25th international symposium on software testing and analysis","author":"Coles Henry","year":"2016","unstructured":"Henry Coles, Thomas Laurent, Christopher Henard, Mike Papadakis, and Anthony Ventresque. Pit: a practical mutation testing tool for Java. In Proceedings of the 25th international symposium on software testing and analysis, pages 449\u2013452, 2016."},{"key":"e_1_3_2_1_20_1","first-page":"31","article-title":"Hints on test data selection: Help for the practical programmer","volume":"11","author":"DeMillo Richard A.","year":"1978","unstructured":"Richard A. DeMillo, Richard J. Lipton, and Frederick G. Sayward. Hints on test data selection: Help for the practical programmer. IEEE Computer, 11:31\u201341, 1978.","journal-title":"IEEE Computer"},{"issue":"7","key":"e_1_3_2_1_21_1","doi-asserted-by":"crossref","first-page":"665","DOI":"10.1109\/TSE.2003.1214329","article-title":"An empirical investigation of the influence of a type of side effects on program comprehension","volume":"29","author":"Dolado Jos\u00e9 Javier","year":"2003","unstructured":"Jos\u00e9 Javier Dolado, Mark Harman, Mari Carmen Otero, and Lin Hu. An empirical investigation of the influence of a type of side effects on program comprehension. IEEE Transactions on Software Engineering, 29(7):665\u2013670, 2003.","journal-title":"IEEE Transactions on Software Engineering"},{"key":"e_1_3_2_1_22_1","volume-title":"ICSE Future of Software Engineering (FoSE 2023)","author":"Fan Angela","year":"2023","unstructured":"Angela Fan, Beliz Gokkaya, Mitya Lyubarskiy, Mark Harman, Shubho Sengupta, Shin Yoo, and Jie Zhang. Large Language Models for Software Engineering: Survey and open problems. In ICSE Future of Software Engineering (FoSE 2023), 2023."},{"key":"e_1_3_2_1_23_1","first-page":"156","volume-title":"FSE","author":"Gabel Mark","year":"2010","unstructured":"Mark Gabel and Zhendong Su. A study of the uniqueness of source code. In FSE, pages 147\u2013156, 2010."},{"key":"e_1_3_2_1_24_1","volume-title":"A survey on LLM-as-a-judge. arXiv preprint arXiv:2411.15594","author":"Gu Jiawei","year":"2024","unstructured":"Jiawei Gu, Xuhui Jiang, Zhichao Shi, Hexiang Tan, Xuehao Zhai, Chengjin Xu, Wei Li, Yinghan Shen, Shengjie Ma, Honghao Liu, et al. A survey on LLM-as-a-judge. arXiv preprint arXiv:2411.15594, 2024."},{"key":"e_1_3_2_1_25_1","first-page":"222","volume-title":"8th European Software Engineering Conference and the ACM SIGSOFT Symposium on the Foundations of Software Engineering (ESEC\/FSE '11)","author":"Harman Mark","year":"2011","unstructured":"Mark Harman, Yue Jia, and William B. Langdon. Strong higher order mutation-based test data generation. In 8th European Software Engineering Conference and the ACM SIGSOFT Symposium on the Foundations of Software Engineering (ESEC\/FSE '11), pages 212\u2013222, New York, NY, USA, September 5th - 9th 2011. ACM."},{"key":"e_1_3_2_1_26_1","first-page":"23","volume-title":"18th IEEE International Working Conference on Source Code Analysis and Manipulation (SCAM 2018","author":"Harman Mark","year":"2018","unstructured":"Mark Harman and Peter O'Hearn. From start-ups to scale-ups: Opportunities and open problems for static and dynamic program analysis (keynote paper). In 18th IEEE International Working Conference on Source Code Analysis and Manipulation (SCAM 2018), pages 1\u201323, Madrid, Spain, September 23rd-24th 2018."},{"key":"e_1_3_2_1_27_1","first-page":"930","volume-title":"36th International Conference on Software Engineering (ICSE 2014","author":"Harman Mark","year":"2014","unstructured":"Mark Harman, Xiangjuan Yao, and Yue Jia. A study of equivalent and stubborn mutation operators using human analysis of equivalence. In 36th International Conference on Software Engineering (ICSE 2014), pages 919\u2013930, Hyderabad, India, June 2014."},{"key":"e_1_3_2_1_28_1","volume-title":"International Conference on Software Engineering (ICSE 2012","author":"Hindle Abram","year":"2012","unstructured":"Abram Hindle, Earl Barr, Zhendong Su, Prem Devanbu, and Mark Gabel. On the naturalness of software. In International Conference on Software Engineering (ICSE 2012), Zurich, Switzerland, 2012."},{"issue":"5","key":"e_1_3_2_1_29_1","doi-asserted-by":"crossref","first-page":"6","DOI":"10.1109\/MS.1985.231754","article-title":"Theory and practice of functional testing","volume":"2","author":"Howden William E.","year":"1985","unstructured":"William E. Howden. Theory and practice of functional testing. IEEE Software, 2(5):6\u201317, September 1985.","journal-title":"IEEE Software"},{"key":"e_1_3_2_1_30_1","first-page":"81","volume-title":"Proceedings of the 30th ACM Joint European Software Engineering Conference and Symposium on the Foundations of Software Engineering","author":"Ibrahimzada Ali Reza","year":"2022","unstructured":"Ali Reza Ibrahimzada, Yigit Varli, Dilara Tekinoglu, and Reyhaneh Jabbarvand. Perfect is the enemy of test oracle. In Proceedings of the 30th ACM Joint European Software Engineering Conference and Symposium on the Foundations of Software Engineering, pages 70\u201381, 2022."},{"key":"e_1_3_2_1_31_1","volume-title":"Compiler Construction","author":"Italiano Davide","year":"2025","unstructured":"Davide Italiano and Chris Cummins. Finding missed code size optimizations in compilers using LLMs. In Compiler Construction, 2025. To appear."},{"key":"e_1_3_2_1_32_1","volume-title":"A review of key likert scale development advances: 1995\u20132019. Frontiers in psychology, 12(637547)","author":"Jebb Andrew T","year":"2021","unstructured":"Andrew T Jebb, Vincent Ng, and Louis Tay. A review of key likert scale development advances: 1995\u20132019. Frontiers in psychology, 12(637547), 2021."},{"key":"e_1_3_2_1_33_1","first-page":"98","volume-title":"3rd Testing Academia and Industry Conference - Practice and Research Techniques (TAIC PART'08)","author":"Jia Yue","year":"2008","unstructured":"Yue Jia and Mark Harman. Milu: A customizable, runtime-optimized higher order mutation testing tool for the full C language. In 3rd Testing Academia and Industry Conference - Practice and Research Techniques (TAIC PART'08), pages 94\u201398, Windsor, UK, August 2008."},{"issue":"10","key":"e_1_3_2_1_34_1","doi-asserted-by":"crossref","first-page":"1379","DOI":"10.1016\/j.infsof.2009.04.016","article-title":"Higher order mutation testing","volume":"51","author":"Jia Yue","year":"2009","unstructured":"Yue Jia and Mark Harman. Higher order mutation testing. Journal of Information and Software Technology, 51(10):1379\u20131393, 2009.","journal-title":"Journal of Information and Software Technology"},{"issue":"5","key":"e_1_3_2_1_35_1","doi-asserted-by":"crossref","first-page":"649","DOI":"10.1109\/TSE.2010.62","article-title":"An analysis and survey of the development of mutation testing","volume":"37","author":"Jia Yue","year":"2011","unstructured":"Yue Jia and Mark Harman. An analysis and survey of the development of mutation testing. IEEE Transactions on Software Engineering, 37(5):649 \u2013 678, September-October 2011.","journal-title":"IEEE Transactions on Software Engineering"},{"key":"e_1_3_2_1_36_1","first-page":"10","volume-title":"Proceedings of the 12th ACM\/IEEE International Symposium on Empirical Software Engineering and Measurement, ESEM 2018","author":"Jimenez Matthieu","year":"2018","unstructured":"Matthieu Jimenez, Thierry Titcheu Chekam, Maxime Cordy, Mike Papadakis, Marinos Kintis, Yves Le Traon, and Mark Harman. Are mutants really natural?: a study on how \"naturalness\" helps mutant selection. In Markku Oivo, Daniel M\u00e9ndez Fern\u00e1ndez, and Audris Mockus, editors, Proceedings of the 12th ACM\/IEEE International Symposium on Empirical Software Engineering and Measurement, ESEM 2018, Oulu, Finland, October 11\u201312, 2018, pages 3:1\u20133:10. ACM, 2018."},{"key":"e_1_3_2_1_37_1","first-page":"436","volume-title":"Proceedings of the 2014 international symposium on software testing and analysis","author":"Just Ren\u00e9","year":"2014","unstructured":"Ren\u00e9 Just. The major mutation framework: Efficient and scalable mutation analysis for java. In Proceedings of the 2014 international symposium on software testing and analysis, pages 433\u2013436, 2014."},{"key":"e_1_3_2_1_38_1","first-page":"159","volume-title":"Proceedings of the IEEE\/ACM 3rd International Conference on AI Engineering-Software Engineering for AI","author":"Li Ziyu","year":"2024","unstructured":"Ziyu Li and Donghwan Shin. Mutation-based consistency testing for evaluating the code understanding capability of LLMs. In Proceedings of the IEEE\/ACM 3rd International Conference on AI Engineering-Software Engineering for AI, pages 150\u2013159, 2024."},{"key":"e_1_3_2_1_39_1","volume-title":"LLM-Powered test case generation for detecting tricky bugs. arXiv preprint arXiv:2404.10304","author":"Liu Kaibo","year":"2024","unstructured":"Kaibo Liu, Yiyang Liu, Zhenpeng Chen, Jie M Zhang, Yudong Han, Yun Ma, Ge Li, and Gang Huang. LLM-Powered test case generation for detecting tricky bugs. arXiv preprint arXiv:2404.10304, 2024."},{"issue":"1","key":"e_1_3_2_1_40_1","doi-asserted-by":"crossref","first-page":"23","DOI":"10.1109\/TSE.2013.44","article-title":"Overcoming the equivalent mutant problem: A systematic literature review and a comparative experiment of second order mutation","volume":"40","author":"Madeyski Lech","year":"2013","unstructured":"Lech Madeyski, Wojciech Orzeszyna, Richard Torkar, and Mariusz Jozala. Overcoming the equivalent mutant problem: A systematic literature review and a comparative experiment of second order mutation. IEEE Transactions on Software Engineering, 40(1):23\u201342, 2013.","journal-title":"IEEE Transactions on Software Engineering"},{"key":"e_1_3_2_1_41_1","first-page":"105","volume-title":"International Symposium on Software Testing and Analysis (ISSTA 2016","author":"Mao Ke","year":"2016","unstructured":"Ke Mao, Mark Harman, and Yue Jia. Sapienz: Multi-objective automated testing for Android applications. In International Symposium on Software Testing and Analysis (ISSTA 2016), pages 94\u2013105, 2016."},{"key":"e_1_3_2_1_42_1","volume-title":"July","author":"Introducing Llama","year":"2024","unstructured":"Meta. Introducing Llama 3.1: Our most capable models to date, July 2024."},{"key":"e_1_3_2_1_43_1","first-page":"2071","volume-title":"Proceedings of the 31st ACM Joint European Software Engineering Conference and Symposium on the Foundations of Software Engineering","author":"Ojdanic Milos","year":"2023","unstructured":"Milos Ojdanic, Mike Papadakis, and Mark Harman. Keeping mutation test suites consistent and relevant with long-standing mutants. In Proceedings of the 31st ACM Joint European Software Engineering Conference and Symposium on the Foundations of Software Engineering, pages 2067\u20132071, 2023."},{"key":"e_1_3_2_1_44_1","doi-asserted-by":"crossref","first-page":"113","DOI":"10.1016\/B978-0-12-800160-8.00003-6","article-title":"Automated test oracles: State of the art, taxonomies, and trends","volume":"95","author":"Oliveira Rafael AP","year":"2014","unstructured":"Rafael AP Oliveira, Upulee Kanewala, and Paulo A Nardi. Automated test oracles: State of the art, taxonomies, and trends. Advances in computers, 95:113\u2013199, 2014.","journal-title":"Advances in computers"},{"key":"e_1_3_2_1_45_1","first-page":"946","volume-title":"37th International Conference on Software Engineering (ICSE 2015","author":"Papadakis Mike","year":"2015","unstructured":"Mike Papadakis, Yue Jia, Mark Harman, and Yves Le Traon. Trivial compiler equivalence: A large scale empirical study of a simple, fast and effective equivalent mutant detection technique. In 37th International Conference on Software Engineering (ICSE 2015), pages 936\u2013946, Florence, Italy, 2015."},{"key":"e_1_3_2_1_46_1","doi-asserted-by":"crossref","first-page":"275","DOI":"10.1016\/bs.adcom.2018.03.015","article-title":"Mutation testing advances: An analysis and survey","volume":"112","author":"Papadakis Mike","year":"2019","unstructured":"Mike Papadakis, Marinos Kintis, Jie Zhang, Yue Jia, Yves Le Traon, and Mark Harman. Mutation testing advances: An analysis and survey. Advances in Computers, 112:275\u2013378, 2019.","journal-title":"Advances in Computers"},{"issue":"3","key":"e_1_3_2_1_47_1","doi-asserted-by":"crossref","first-page":"415","DOI":"10.1109\/TEVC.2017.2693219","article-title":"Genetic improvement of software: a comprehensive survey","volume":"22","author":"Petke Justyna","year":"2018","unstructured":"Justyna Petke, Saemundur O. Haraldsson, Mark Harman, William B. Langdon, David R. White, and John R. Woodward. Genetic improvement of software: a comprehensive survey. IEEE Transactions on Evolutionary Computation, 22(3):415\u2013432, June 2018.","journal-title":"IEEE Transactions on Evolutionary Computation"},{"key":"e_1_3_2_1_48_1","first-page":"171","volume-title":"Proceedings of the 40th international conference on software engineering: Software engineering in practice","author":"Petrovi\u0107 Goran","year":"2018","unstructured":"Goran Petrovi\u0107 and Marko Ivankovi\u0107. State of mutation testing at Google. In Proceedings of the 40th international conference on software engineering: Software engineering in practice, pages 163\u2013171, 2018."},{"issue":"10","key":"e_1_3_2_1_49_1","doi-asserted-by":"crossref","first-page":"3900","DOI":"10.1109\/TSE.2021.3107634","article-title":"Practical mutation testing at scale: A view from Google","volume":"48","author":"Petrovi\u0107 Goran","year":"2021","unstructured":"Goran Petrovi\u0107, Marko Ivankovi\u0107, Gordon Fraser, and Ren\u00e9 Just. Practical mutation testing at scale: A view from Google. IEEE Transactions on Software Engineering, 48(10):3900\u20133912, 2021.","journal-title":"IEEE Transactions on Software Engineering"},{"key":"e_1_3_2_1_50_1","first-page":"53","volume-title":"2018 IEEE International Conference on Software Testing, Verification and Validation Workshops (ICSTW)","author":"Petrovic Goran","unstructured":"Goran Petrovic, Marko Ivankovic, Bob Kurtz, Paul Ammann, and Ren\u00e9 Just. An industrial application of mutation testing: Lessons, challenges, and research directions. In 2018 IEEE International Conference on Software Testing, Verification and Validation Workshops (ICSTW), pages 47\u201353. IEEE, 2018."},{"key":"e_1_3_2_1_51_1","first-page":"173","volume-title":"2022 IEEE Conference on Software Testing, Verification and Validation (ICST)","author":"Richter Cedric","unstructured":"Cedric Richter and Heike Wehrheim. Learning realistic mutations: Bug creation for neural bug detectors. In 2022 IEEE Conference on Software Testing, Verification and Validation (ICST), pages 162\u2013173. IEEE, 2022."},{"key":"e_1_3_2_1_52_1","volume-title":"Proceedings of the ACM on Conference on Foundations of Software Engineering, 1(FSE):951\u2013971","author":"Ryan Gabriel","year":"2024","unstructured":"Gabriel Ryan, Siddhartha Jain, Mingyue Shang, Shiqi Wang, Xiaofei Ma, Murali Krishna Ramanathan, and Baishakhi Ray. Code-aware prompting: A study of coverage-guided test generation in regression setting using LLM. Proceedings of the ACM on Conference on Foundations of Software Engineering, 1(FSE):951\u2013971, 2024."},{"key":"e_1_3_2_1_53_1","article-title":"An empirical evaluation of using large language models for automated unit test generation","author":"Sch\u00e4fer Max","year":"2023","unstructured":"Max Sch\u00e4fer, Sarah Nadi, Aryaz Eghbali, and Frank Tip. An empirical evaluation of using large language models for automated unit test generation. IEEE Transactions on Software Engineering, 2023.","journal-title":"IEEE Transactions on Software Engineering"},{"key":"e_1_3_2_1_54_1","first-page":"298","volume-title":"7th joint meeting of the European Software Engineering Conference and the ACM SIGSOFT International Symposium on Foundations of Software Engineering (ESEC\/FSE","author":"Schuler David","year":"2009","unstructured":"David Schuler and Andreas Zeller. Javalanche: efficient mutation testing for Java. In 7th joint meeting of the European Software Engineering Conference and the ACM SIGSOFT International Symposium on Foundations of Software Engineering (ESEC\/FSE 2009), pages 297\u2013298, 2009."},{"key":"e_1_3_2_1_55_1","volume-title":"June","author":"Speed Richard","year":"2023","unstructured":"Richard Speed. Github: 30% of copilot coding suggestions are accepted, June 2023."},{"key":"e_1_3_2_1_56_1","first-page":"1745","volume-title":"Proceedings of the 33rd ACM SIGSOFT International Symposium on Software Testing and Analysis","author":"Tian Zhao","year":"2024","unstructured":"Zhao Tian, Honglin Shu, Dong Wang, Xuejie Cao, Yasutaka Kamei, and Junjie Chen. Large language models for equivalent mutant detection: How far are we? In Proceedings of the 33rd ACM SIGSOFT International Symposium on Software Testing and Analysis, pages 1733\u20131745, 2024."},{"key":"e_1_3_2_1_57_1","volume-title":"LLMorpheus: Mutation testing using large language models. arXiv preprint arXiv:2404.09952","author":"Tip Frank","year":"2024","unstructured":"Frank Tip, Jonathan Bell, and Max Sch\u00e4fer. LLMorpheus: Mutation testing using large language models. arXiv preprint arXiv:2404.09952, 2024."},{"key":"e_1_3_2_1_58_1","first-page":"133","volume-title":"45th IEEE\/ACM International Conference on Software Engineering: Software Engineering in Practice, SEIP@ICSE 2023","author":"Tuli Shreshth","year":"2023","unstructured":"Shreshth Tuli, Kinga Bojarczuk, Natalija Gucevska, Mark Harman, Xiao-Yu Wang, and Graham Wright. Simulation-driven automated end-to-end test and oracle inference. In 45th IEEE\/ACM International Conference on Software Engineering: Software Engineering in Practice, SEIP@ICSE 2023, Melbourne, Australia, May 14\u201320, 2023, pages 122\u2013133. IEEE, 2023."},{"key":"e_1_3_2_1_59_1","first-page":"12","volume-title":"2021 IEEE International Conference on Software Testing, Verification and Validation Workshops (ICSTW)","author":"van Hijfte Lars","unstructured":"Lars van Hijfte and Ana Oprescu. Mutantbench: an equivalent mutant problem comparison framework. In 2021 IEEE International Conference on Software Testing, Verification and Validation Workshops (ICSTW), pages 7\u201312. IEEE, 2021."},{"key":"e_1_3_2_1_60_1","volume-title":"An exploratory study on using large language models xfor mutation testing. arXiv preprint arXiv:2406.09843","author":"Wang Bo","year":"2024","unstructured":"Bo Wang, Mingda Chen, Youfang Lin, Mike Papadakis, and Jie M Zhang. An exploratory study on using large language models xfor mutation testing. arXiv preprint arXiv:2406.09843, 2024."},{"key":"e_1_3_2_1_61_1","volume-title":"Software testing with large language model: Survey, landscape, and vision","author":"Wang Junjie","year":"2023","unstructured":"Junjie Wang, Yuchao Huang, Chunyang Chen, Zhe Liu, Song Wang, and Qing Wang. Software testing with large language model: Survey, landscape, and vision, 2023. arXiv:2307.07221."},{"key":"e_1_3_2_1_62_1","first-page":"712","volume-title":"ACM\/IEEE International Conference on Automated Software Engineering (ASE'14)","author":"Zhang Jie","year":"2014","unstructured":"Jie Zhang, Junjie Chen, Dan Hao, Yingfei Xiong, Bing Xie, Lu Zhang, and Hong Mei. Search-based inference of polynomial metamorphic relations. In Ivica Crnkovic, Marsha Chechik, and Paul Gruenbacher, editors, ACM\/IEEE International Conference on Automated Software Engineering (ASE'14), pages 701\u2013712, Vasteras, Sweden, September 15\u201319 2014."},{"key":"e_1_3_2_1_63_1","volume-title":"Judging LLM-as-a-judge with MT-bench and chatbot arena. Advances in Neural Information Processing Systems (NeurIPS","author":"Zheng Lianmin","year":"2023","unstructured":"Lianmin Zheng, Wei-Lin Chiang, Ying Sheng, Siyuan Zhuang, Zhanghao Wu, Yonghao Zhuang, Zi Lin, Zhuohan Li, Dacheng Li, Eric Xing, et al. Judging LLM-as-a-judge with MT-bench and chatbot arena. Advances in Neural Information Processing Systems (NeurIPS 2023), 36:46595\u201346623, 2023."}],"event":{"name":"FSE Companion '25: 33rd ACM International Conference on the Foundations of Software Engineering","location":"Clarion Hotel Trondheim Trondheim Norway","acronym":"FSE Companion '25","sponsor":["SIGSOFT ACM Special Interest Group on Software Engineering"]},"container-title":["Proceedings of the 33rd ACM International Conference on the Foundations of Software Engineering"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3696630.3728544","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,7,28]],"date-time":"2025-07-28T19:15:24Z","timestamp":1753730124000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3696630.3728544"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,6,23]]},"references-count":63,"alternative-id":["10.1145\/3696630.3728544","10.1145\/3696630"],"URL":"https:\/\/doi.org\/10.1145\/3696630.3728544","relation":{},"subject":[],"published":{"date-parts":[[2025,6,23]]},"assertion":[{"value":"2025-07-28","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}