{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,2]],"date-time":"2026-10-02T10:39:46Z","timestamp":1790937586764,"version":"4.1.0"},"publisher-location":"New York, NY, USA","reference-count":47,"publisher":"ACM","license":[{"start":{"date-parts":[[2024,7,10]],"date-time":"2024-07-10T00:00:00Z","timestamp":1720569600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2024,7,10]]},"DOI":"10.1145\/3663529.3663838","type":"proceedings-article","created":{"date-parts":[[2024,7,10]],"date-time":"2024-07-10T15:43:13Z","timestamp":1720626193000},"page":"173-184","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":11,"title":["Observation-Based Unit Test Generation at Meta"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0009-0009-4763-0396","authenticated-orcid":false,"given":"Nadia","family":"Alshahwan","sequence":"first","affiliation":[{"name":"Meta Platforms, Menlo Park, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-5864-4488","authenticated-orcid":false,"given":"Mark","family":"Harman","sequence":"additional","affiliation":[{"name":"Meta Platforms, Menlo Park, USA \/ University College London, London, United Kingdom"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0001-5311-762X","authenticated-orcid":false,"given":"Alexandru","family":"Marginean","sequence":"additional","affiliation":[{"name":"Meta Platforms, Menlo Park, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0002-4320-070X","authenticated-orcid":false,"given":"Rotem","family":"Tal","sequence":"additional","affiliation":[{"name":"Meta Platforms, Menlo Park, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0009-8825-6986","authenticated-orcid":false,"given":"Eddy","family":"Wang","sequence":"additional","affiliation":[{"name":"Meta Platforms, Menlo Park, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2024,7,10]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE-SEIP52600.2021.00023"},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.1145\/3463274.3463275"},{"key":"e_1_3_2_1_3_1","volume-title":"Foundations of Software Engineering (FSE","author":"Alshahwan Nadia","year":"2024","unstructured":"Nadia Alshahwan, Jubin Chheda, Anastasia Finegenova, Mark Harman, Inna Harper, Alexandru Marginean, Shubho Sengupta, and Eddy Wang. 2024. Automated Unit Test Improvement using Large Language Models at Meta. In Foundations of Software Engineering (FSE 2024). Submitted"},{"key":"e_1_3_2_1_4_1","volume-title":"Deploying Search Based Software Engineering with Sapienz at Facebook (keynote paper). In 10^th International Symposium on Search Based Software Engineering (SSBSE","author":"Alshahwan Nadia","year":"2018","unstructured":"Nadia Alshahwan, Xinbo Gao, Mark Harman, Yue Jia, Ke Mao, Alexander Mols, Taijin Tei, and Ilya Zorin. 2018. Deploying Search Based Software Engineering with Sapienz at Facebook (keynote paper). In 10^th International Symposium on Search Based Software Engineering (SSBSE 2018). Montpellier, France. 3\u201345. Springer LNCS 11036"},{"key":"e_1_3_2_1_5_1","volume-title":"Software Testing Research Challenges: An Industrial Perspective. In 2023 IEEE Conference on Software Testing, Verification and Validation (ICST","author":"Alshahwan Nadia","year":"2023","unstructured":"Nadia Alshahwan, Mark Harman, and Alexandru Marginean. 2023. Software Testing Research Challenges: An Industrial Perspective. In 2023 IEEE Conference on Software Testing, Verification and Validation (ICST 2023). 1\u201310."},{"key":"e_1_3_2_1_6_1","volume-title":"ICSE workshop on Interoperability and Robustness of Neural Software Engineering (InteNSE). To appear","author":"Alshahwan Nadia","year":"2024","unstructured":"Nadia Alshahwan, Mark Harman, Alexandru Marginean, Shubho Sengupta, and Eddy Wang. 2024. Assured LLM-Based Software Engineering (keynote paper). In 2^nd. ICSE workshop on Interoperability and Robustness of Neural Software Engineering (InteNSE). To appear"},{"key":"e_1_3_2_1_7_1","volume-title":"Dagstuhl Seminar Proceedings.","author":"Alshahwan Nadia","year":"2010","unstructured":"Nadia Alshahwan, Yue Jia, Kiran Lakhotia, Gordon Fraser, David Shuler, and Paolo Tonella. 2010. AUTOMOCK: automated synthesis of a mock environment for test case generation. In Dagstuhl Seminar Proceedings."},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.jss.2013.02.061"},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2014.2372785"},{"key":"e_1_3_2_1_10_1","volume-title":"ORBS: Language-Independent Program Slicing. In 22^nd ACM SIGSOFT International Symposium on the Foundations of Software Engineering (FSE","author":"Binkley David","year":"2014","unstructured":"David Binkley, Nicolas Gold, Mark Harman, Syed Islam, Jens Krinke, and Shin Yoo. 2014. ORBS: Language-Independent Program Slicing. In 22^nd ACM SIGSOFT International Symposium on the Foundations of Software Engineering (FSE 2014). Hong Kong, China. 109\u2013120."},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.jss.2021.111041"},{"key":"e_1_3_2_1_12_1","volume-title":"Testing of and with cyber-cyber digital twins. In 7^th International workshop on metamorphic testing (MET","author":"Bojarczuk Kinga","year":"2022","unstructured":"Kinga Bojarczuk and Mark Harman. 2022. Testing of and with cyber-cyber digital twins. In 7^th International workshop on metamorphic testing (MET 2022). Pittsburgh, PA, USA. Keynote talk given jointly by Kinga Bojarczuk and Mark Harman"},{"key":"e_1_3_2_1_13_1","volume-title":"International Conference on Reliable Software. ACM","author":"Boyer Robert S.","unstructured":"Robert S. Boyer, Bernard Elspas, and Karl N. Levitt. 1975. SELECT \u2013 a Formal System for Testing and Debugging Programs by Symbolic Execution. In International Conference on Reliable Software. ACM, New York, NY, USA. 234\u2013245."},{"key":"e_1_3_2_1_14_1","first-page":"3675","volume-title":"Targeted Program Transformations for Symbolic Execution. In 10th Joint Meeting on Foundations of Software Engineering (ESEC\/FSE). 906\u2013909","author":"Cadar Cristian","year":"2015","unstructured":"Cristian Cadar. 2015. Targeted Program Transformations for Symbolic Execution. In 10th Joint Meeting on Foundations of Software Engineering (ESEC\/FSE). 906\u2013909. isbn:978-1-4503-3675-8"},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.1145\/2408776.2408795"},{"key":"e_1_3_2_1_16_1","volume-title":"d.]. Open-sourcing Facebook Infer: Identify bugs before you ship. code.facebook.com blog post","author":"Calcagno Cristiano","year":"2015","unstructured":"Cristiano Calcagno, Dino Distefano, and Peter O\u2019Hearn. [n. d.]. Open-sourcing Facebook Infer: Identify bugs before you ship. code.facebook.com blog post, 11 June 2015"},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1145\/3143561","article-title":"Metamorphic testing: a review of challenges and opportunities","volume":"51","author":"Chen Tsong Yueh","year":"2018","unstructured":"Tsong Yueh Chen, Fei-Ching Kuo, Huai Liu, Pak-Lok Poon, Dave Towey, T. H. Tse, and Zhi Quan Zhou. 2018. Metamorphic testing: a review of challenges and opportunities. ACM Computing Surveys (CSUR), 51, 1 (2018), January, 4:1\u20134:27. issn:1557-7341","journal-title":"ACM Computing Surveys (CSUR)"},{"key":"e_1_3_2_1_18_1","volume-title":"Proceedings of the 4th International Workshop on Metamorphic Testing, MET@ICSE 2019","author":"Donaldson Alastair F.","year":"2019","unstructured":"Alastair F. Donaldson. 2019. Metamorphic testing of Android graphics drivers. In Proceedings of the 4th International Workshop on Metamorphic Testing, MET@ICSE 2019, Montreal, QC, Canada, May 26, 2019, Xiaoyuan Xie, Pak-Lok Poon, and Laura L. Pullum (Eds.). IEEE \/ ACM, 1. isbn:978-1-7281-2235-9"},{"key":"e_1_3_2_1_19_1","volume-title":"Automated Testing as Production Simulation: Research Opportunities and Challenges. In 37^th IEEE\/ACM International Conference on Automated Software Engineering (ASE","author":"Dvortsova Inna","year":"2022","unstructured":"Inna Dvortsova and Mark Harman. 2022. Automated Testing as Production Simulation: Research Opportunities and Challenges. In 37^th IEEE\/ACM International Conference on Automated Software Engineering (ASE 2022. Michigan, USA. Keynote talk given jointly by Inna Dvortsova and Mark Harman"},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2008.103"},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.1109\/32.908957"},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.1145\/3510003.3510175"},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.1007\/s00521-011-0568-8"},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"publisher","DOI":"10.1145\/2338965.2336762"},{"key":"e_1_3_2_1_25_1","volume-title":"Making the Case for MORTO: Multi Objective Regression Test Optimization (invited position paper). In 1^st International Workshop on Regression Testing (Regression","author":"Harman Mark","year":"2011","unstructured":"Mark Harman. 2011. Making the Case for MORTO: Multi Objective Regression Test Optimization (invited position paper). In 1^st International Workshop on Regression Testing (Regression 2011). Berlin, Germany."},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.1145\/3524459.3527353"},{"key":"e_1_3_2_1_27_1","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2004.1265732"},{"key":"e_1_3_2_1_28_1","volume-title":"open problems and challenges for search based software testing (keynote Paper). In 8^th IEEE International Conference on Software Testing, Verification and Validation (ICST","author":"Harman Mark","year":"2015","unstructured":"Mark Harman, Yue Jia, and Yuanyuan Zhang. 2015. Achievements, open problems and challenges for search based software testing (keynote Paper). In 8^th IEEE International Conference on Software Testing, Verification and Validation (ICST 2015). Graz, Austria."},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.1145\/1831708.1831729"},{"key":"e_1_3_2_1_30_1","volume-title":"2019 IEEE\/ACM 41st International Conference on Software Engineering: Companion Proceedings (ICSE-Companion). 248\u2013249","author":"Kampmann Alexander","year":"2019","unstructured":"Alexander Kampmann and Andreas Zeller. 2019. Carving parameterized unit tests. In 2019 IEEE\/ACM 41st International Conference on Software Engineering: Companion Proceedings (ICSE-Companion). 248\u2013249."},{"key":"e_1_3_2_1_31_1","volume-title":"Manuel Egele, Edward J. Schwartz, and Maverick Woo.","author":"Man\u00e8s Valentin J. M.","year":"2018","unstructured":"Valentin J. M. Man\u00e8s, HyungSeok Han, Choongwoo Han, Sang Kil Cha, Manuel Egele, Edward J. Schwartz, and Maverick Woo. 2018. The Art, Science, and Engineering of Fuzzing: A Survey. CoRR, abs\/1812.00140 (2018), arxiv:1812.00140. arxiv:1812.00140"},{"key":"e_1_3_2_1_32_1","volume-title":"Sapienz: Multi-objective Automated Testing for Android Applications. In International Symposium on Software Testing and Analysis (ISSTA","author":"Mao Ke","year":"2016","unstructured":"Ke Mao, Mark Harman, and Yue Jia. 2016. Sapienz: Multi-objective Automated Testing for Android Applications. In International Symposium on Software Testing and Analysis (ISSTA 2016). 94\u2013105."},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"publisher","DOI":"10.1002\/stvr.294"},{"key":"e_1_3_2_1_34_1","volume-title":"Companion to the 22nd ACM SIGPLAN conference on Object-oriented programming systems and applications companion. 815\u2013816.","author":"Pacheco Carlos","unstructured":"Carlos Pacheco and Michael D Ernst. 2007. Randoop: feedback-directed random testing for Java. In Companion to the 22nd ACM SIGPLAN conference on Object-oriented programming systems and applications companion. 815\u2013816."},{"key":"e_1_3_2_1_35_1","doi-asserted-by":"publisher","DOI":"10.1145\/996821.996838"},{"key":"e_1_3_2_1_36_1","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2016.2532875"},{"key":"e_1_3_2_1_37_1","volume-title":"2017 IEEE\/ACM 14th International Conference on Mining Software Repositories (MSR). 402\u2013412","author":"Spadini Davide","year":"2017","unstructured":"Davide Spadini, Maur\u00edcio Aniche, Magiel Bruntink, and Alberto Bacchelli. 2017. To mock or not to mock? An empirical study on mocking practices. In 2017 IEEE\/ACM 14th International Conference on Mining Software Repositories (MSR). 402\u2013412."},{"key":"e_1_3_2_1_38_1","volume-title":"Proceedings of the 2000 ACM SIGSOFT international symposium on Software Testing and Analysis (ISSTA","author":"Steven John","year":"2000","unstructured":"John Steven, Pravir Chandra, Bob Fleck, and Andy Podgurski. 2000. jRapture: A capture\/replay tool for observation-based testing. In Proceedings of the 2000 ACM SIGSOFT international symposium on Software Testing and Analysis (ISSTA 2000). 158\u2013167."},{"key":"e_1_3_2_1_39_1","doi-asserted-by":"publisher","DOI":"10.1109\/AST.2015.19"},{"key":"e_1_3_2_1_40_1","unstructured":"Deepika Tiwari. 2024. Private correspondence concerning experience with the Pankti observation based testing system. Deepika Tiwari graciously gave consent to include this information in the public domain"},{"key":"e_1_3_2_1_41_1","unstructured":"Deepika Tiwari Martin Monperrus and Benoit Baudry. 2023. Mimicking Production Behavior with Generated Mocks. arxiv:2208.01321."},{"key":"e_1_3_2_1_42_1","doi-asserted-by":"publisher","DOI":"10.1109\/TR.2021.3101318"},{"key":"e_1_3_2_1_43_1","volume-title":"Simulation-Driven Automated End-to-End Test and Oracle Inference. In 45th IEEE\/ACM International Conference on Software Engineering: Software Engineering in Practice, SEIP@ICSE 2023","author":"Tuli Shreshth","year":"2023","unstructured":"Shreshth Tuli, Kinga Bojarczuk, Natalija Gucevska, Mark Harman, Xiao-Yu Wang, and Graham Wright. 2023. Simulation-Driven Automated End-to-End Test and Oracle Inference. In 45th IEEE\/ACM International Conference on Software Engineering: Software Engineering in Practice, SEIP@ICSE 2023, Melbourne, Australia, May 14-20, 2023. IEEE, 122\u2013133."},{"key":"e_1_3_2_1_44_1","doi-asserted-by":"publisher","DOI":"10.1109\/ISSREW.2017.46"},{"key":"e_1_3_2_1_45_1","doi-asserted-by":"crossref","unstructured":"Rahulkrishna Yandrapally Saurabh Sinha Rachel Tzoref-Brill and Ali Mesbah. 2023. Carving UI Tests to Generate API Tests and API Specification. arXiv preprint arXiv:2305.14692.","DOI":"10.1109\/ICSE48619.2023.00167"},{"key":"e_1_3_2_1_46_1","doi-asserted-by":"publisher","DOI":"10.1002\/stv.430"},{"key":"e_1_3_2_1_47_1","doi-asserted-by":"publisher","DOI":"10.1145\/2642937.2642994"}],"event":{"name":"FSE '24: 32nd ACM International Conference on the Foundations of Software Engineering","location":"Porto de Galinhas Brazil","acronym":"FSE '24","sponsor":["SIGSOFT ACM Special Interest Group on Software Engineering"]},"container-title":["Companion Proceedings of the 32nd ACM International Conference on the Foundations of Software Engineering"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3663529.3663838","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3663529.3663838","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,18]],"date-time":"2025-06-18T19:44:21Z","timestamp":1750275861000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3663529.3663838"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,7,10]]},"references-count":47,"alternative-id":["10.1145\/3663529.3663838","10.1145\/3663529"],"URL":"https:\/\/doi.org\/10.1145\/3663529.3663838","relation":{},"subject":[],"published":{"date-parts":[[2024,7,10]]},"assertion":[{"value":"2024-07-10","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}