{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,6,19]],"date-time":"2025-06-19T05:00:29Z","timestamp":1750309229245,"version":"3.41.0"},"publisher-location":"New York, NY, USA","reference-count":70,"publisher":"ACM","license":[{"start":{"date-parts":[[2024,7,10]],"date-time":"2024-07-10T00:00:00Z","timestamp":1720569600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2024,7,10]]},"DOI":"10.1145\/3663529.3663779","type":"proceedings-article","created":{"date-parts":[[2024,7,10]],"date-time":"2024-07-10T19:43:13Z","timestamp":1720640593000},"page":"467-471","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":1,"title":["Testing Learning-Enabled Cyber-Physical Systems with Large-Language Models: A Formal Approach"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-2572-2355","authenticated-orcid":false,"given":"Xi","family":"Zheng","sequence":"first","affiliation":[{"name":"Macquarie University, Sydney, Australia"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-1309-8425","authenticated-orcid":false,"given":"Aloysius K.","family":"Mok","sequence":"additional","affiliation":[{"name":"University of Texas at Austin, Austin, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-3267-0776","authenticated-orcid":false,"given":"Ruzica","family":"Piskac","sequence":"additional","affiliation":[{"name":"Yale University, New Heaven, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-9863-1270","authenticated-orcid":false,"given":"Yong Jae","family":"Lee","sequence":"additional","affiliation":[{"name":"University of Wisconsin-Madison, Madison, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-9994-9931","authenticated-orcid":false,"given":"Bhaskar","family":"Krishnamachari","sequence":"additional","affiliation":[{"name":"University of Southern California, Los Angeles, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-1938-9947","authenticated-orcid":false,"given":"Dakai","family":"Zhu","sequence":"additional","affiliation":[{"name":"University of Texas at San Antonio, San Antonio, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-5282-0658","authenticated-orcid":false,"given":"Oleg","family":"Sokolsky","sequence":"additional","affiliation":[{"name":"University of Pennsylvania, Philadelphia, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-2672-1132","authenticated-orcid":false,"given":"Insup","family":"Lee","sequence":"additional","affiliation":[{"name":"University of Pennsylvania, Philadelphia, USA"}]}],"member":"320","published-online":{"date-parts":[[2024,7,10]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.eswa.2021.115091"},{"key":"e_1_3_2_1_2_1","unstructured":"2023. AmazonPrimeAir. https:\/\/shorturl.at\/otyU3"},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","unstructured":"Saleema Amershi. 2019. Software engineering for machine learning: A case study. In ICSE-SEIP. 291\u2013300. https:\/\/doi.org\/10.1109\/ICSE-SEIP.2019.00042 10.1109\/ICSE-SEIP.2019.00042","DOI":"10.1109\/ICSE-SEIP.2019.00042"},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.1016\/0890-5401(87)90052-6"},{"key":"e_1_3_2_1_5_1","volume-title":"S-taliro: A tool for temporal logic falsification for hybrid systems. In TACAS. 254\u2013257.","author":"Annpureddy Yashwanth","year":"2011","unstructured":"Yashwanth Annpureddy. 2011. S-taliro: A tool for temporal logic falsification for hybrid systems. In TACAS. 254\u2013257."},{"key":"e_1_3_2_1_6_1","volume-title":"Modbat: A model-based API tester for event-driven systems. In HVC. 112\u2013128.","author":"Artho Cyrille Valentin","year":"2013","unstructured":"Cyrille Valentin Artho. 2013. Modbat: A model-based API tester for event-driven systems. In HVC. 112\u2013128."},{"key":"e_1_3_2_1_7_1","unstructured":"ASAM. 2021. ASAM OpenSCENARIO: User Guide. https:\/\/shorturl.at\/epuP0"},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.ic.2022.104957"},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1109\/TC.1972.5009015"},{"key":"e_1_3_2_1_10_1","volume-title":"A unified view of piecewise linear neural network verification. NeurIPS, 31","author":"Bunel Rudy R","year":"2018","unstructured":"Rudy R Bunel. 2018. A unified view of piecewise linear neural network verification. NeurIPS, 31 (2018)."},{"key":"e_1_3_2_1_11_1","first-page":"210","article-title":"From \u2018shallow\u2019to \u2018deep\u2019policing","volume":"26","author":"Button Mark","year":"2016","unstructured":"Mark Button and Graham Brooks. 2016. From \u2018shallow\u2019to \u2018deep\u2019policing. Policing and Society, 26, 2 (2016), 210\u2013229.","journal-title":"Policing and Society"},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","unstructured":"Feiyang Cai and Xenofon Koutsoukos. 2020. Real-time out-of-distribution detection in learning-enabled cyber-physical systems. In ICCPS. 174\u2013183. https:\/\/doi.org\/10.1109\/ICCPS48487.2020.00024 10.1109\/ICCPS48487.2020.00024","DOI":"10.1109\/ICCPS48487.2020.00024"},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"crossref","unstructured":"Dian Chen and Philipp Kr\u00e4henb\u00fchl. 2022. Learning from all vehicles. In CVPR.","DOI":"10.1109\/CVPR52688.2022.01671"},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","unstructured":"Siddhartha R Dalal. 1999. Model-based testing in practice. In ICSE. 285\u2013294. https:\/\/doi.org\/10.1145\/302405.302640 10.1145\/302405.302640","DOI":"10.1145\/302405.302640"},{"key":"e_1_3_2_1_15_1","unstructured":"2023. Da Vinci Surgical System. http:\/\/www.intuitivesurgical.com\/"},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","unstructured":"Yao Deng. 2020. An analysis of adversarial attacks and defenses on autonomous driving models. In PerCom. 1\u201310. https:\/\/doi.org\/10.1109\/PerCom45495.2020.9127389 10.1109\/PerCom45495.2020.9127389","DOI":"10.1109\/PerCom45495.2020.9127389"},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","unstructured":"Yao Deng. 2022. A declarative metamorphic testing framework for autonomous driving. TSE https:\/\/doi.org\/10.1109\/TSE.2022.3206427 10.1109\/TSE.2022.3206427","DOI":"10.1109\/TSE.2022.3206427"},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","unstructured":"Yao Deng. 2022. Scenario-based test reduction and prioritization for multi-module autonomous driving systems. In FSE. 82\u201393. https:\/\/doi.org\/10.1145\/3540250.3549152 10.1145\/3540250.3549152","DOI":"10.1145\/3540250.3549152"},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","unstructured":"Yinlin Deng. 2023. Large Language Models are Zero-Shot Fuzzers: Fuzzing Deep-Learning Libraries via Large Language Models. In ISSTA. 423\u2013435. https:\/\/doi.org\/10.1145\/3597926.3598067 10.1145\/3597926.3598067","DOI":"10.1145\/3597926.3598067"},{"key":"e_1_3_2_1_20_1","volume-title":"TARGET: Traffic Rule-based Test Generation for Autonomous Driving Systems. arXiv preprint arXiv:2305.06018.","author":"Deng Yao","year":"2023","unstructured":"Yao Deng. 2023. TARGET: Traffic Rule-based Test Generation for Autonomous Driving Systems. arXiv preprint arXiv:2305.06018."},{"key":"e_1_3_2_1_21_1","volume-title":"RMT: Rule-based Metamorphic Testing for Autonomous Driving Models. arXiv preprint arXiv:2012.10672.","author":"Deng Yao","year":"2020","unstructured":"Yao Deng and Xi Zheng. 2020. RMT: Rule-based Metamorphic Testing for Autonomous Driving Models. arXiv preprint arXiv:2012.10672."},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"crossref","unstructured":"Naqqash Dilshad. 2020. Applications and challenges in video surveillance via drone: A brief survey. In ICTC. 728\u2013732.","DOI":"10.1109\/ICTC49870.2020.9289536"},{"key":"e_1_3_2_1_23_1","volume-title":"Conference on robot learning. 1\u201316","author":"Dosovitskiy Alexey","year":"2017","unstructured":"Alexey Dosovitskiy. 2017. CARLA: An open urban driving simulator. In Conference on robot learning. 1\u201316."},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"crossref","unstructured":"Souradeep Dutta. 2019. Sherlock-a tool for verification of neural network feedback systems: demo abstract. In HSCC. 262\u2013263.","DOI":"10.1145\/3302504.3313351"},{"key":"e_1_3_2_1_25_1","volume-title":"Volkswagen: Robot kills worker installing it. https:\/\/t.ly\/aK_qb","author":"DW.","year":"2015","unstructured":"DW. 2015. Volkswagen: Robot kills worker installing it. https:\/\/t.ly\/aK_qb"},{"key":"e_1_3_2_1_26_1","volume-title":"Scenic: A Language for Scenario Specification and Scene Generation. In PLDI. ACM, 63\u201378. isbn:9781450367127","author":"Fremont Daniel J.","year":"2019","unstructured":"Daniel J. Fremont. 2019. Scenic: A Language for Scenario Specification and Scene Generation. In PLDI. ACM, 63\u201378. isbn:9781450367127"},{"key":"e_1_3_2_1_27_1","volume-title":"Scenic: A language for scenario specification and data generation. Machine Learning, 1\u201345.","author":"Fremont Daniel J","year":"2022","unstructured":"Daniel J Fremont. 2022. Scenic: A language for scenario specification and data generation. Machine Learning, 1\u201345."},{"key":"e_1_3_2_1_28_1","unstructured":"2023. Google Wing. http:\/\/www.wing.com\/"},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","unstructured":"Fitash Ul Haq. 2023. Many-objective reinforcement learning for online testing of dnn-enabled systems. In ICSE. 1814\u20131826. https:\/\/doi.org\/10.1109\/ICSE48619.2023.00155 10.1109\/ICSE48619.2023.00155","DOI":"10.1109\/ICSE48619.2023.00155"},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"publisher","unstructured":"Fitash Ul Haq Donghwan Shin and Lionel Briand. 2022. Efficient online testing for DNN-enabled systems using surrogate-assisted and many-objective optimization. In ICSE. 811\u2013822. https:\/\/doi.org\/10.1145\/3510003.3510188 10.1145\/3510003.3510188","DOI":"10.1145\/3510003.3510188"},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"crossref","first-page":"29","DOI":"10.1038\/s41368-023-00239-y","article-title":"ChatGPT for shaping the future of dentistry: the potential of multi-modal large language model","volume":"15","author":"Huang Hanyao","year":"2023","unstructured":"Hanyao Huang. 2023. ChatGPT for shaping the future of dentistry: the potential of multi-modal large language model. International Journal of Oral Science, 15, 1 (2023), 29.","journal-title":"International Journal of Oral Science"},{"key":"e_1_3_2_1_32_1","volume-title":"Telex: Passive stl learning using only positive examples. In RV. 208\u2013224.","author":"Jha Susmit","year":"2017","unstructured":"Susmit Jha. 2017. Telex: Passive stl learning using only positive examples. In RV. 208\u2013224."},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"crossref","unstructured":"Austin Jones Zhaodan Kong and Calin Belta. 2014. Anomaly detection in cyber-physical systems: A formal methods approach. In CDC. 848\u2013853.","DOI":"10.1109\/CDC.2014.7039487"},{"key":"e_1_3_2_1_34_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.lindif.2023.102274"},{"key":"e_1_3_2_1_35_1","doi-asserted-by":"crossref","unstructured":"Shinpei Kato. 2018. Autoware on board: Enabling autonomous vehicles with embedded systems. In ICCPS. 287\u2013296.","DOI":"10.1109\/ICCPS.2018.00035"},{"key":"e_1_3_2_1_36_1","volume-title":"Reluplex: An efficient SMT solver for verifying deep neural networks. In CAV. 97\u2013117.","author":"Katz Guy","year":"2017","unstructured":"Guy Katz. 2017. Reluplex: An efficient SMT solver for verifying deep neural networks. In CAV. 97\u2013117."},{"key":"e_1_3_2_1_37_1","doi-asserted-by":"publisher","unstructured":"Willibald Krenn. 2015. Momut:: UML model-based mutation testing for UML. In ICST. 1\u20138. https:\/\/doi.org\/10.1109\/ICST.2015.7102627 10.1109\/ICST.2015.7102627","DOI":"10.1109\/ICST.2015.7102627"},{"key":"e_1_3_2_1_38_1","doi-asserted-by":"crossref","unstructured":"Martin Leucker. 2006. Learning meets verification. In FMCO. 127\u2013151.","DOI":"10.1007\/978-3-540-74792-5_6"},{"key":"e_1_3_2_1_39_1","doi-asserted-by":"publisher","DOI":"10.1109\/ISSRE5003.2020.00012"},{"key":"e_1_3_2_1_40_1","unstructured":"Haotian Liu Chunyuan Li Qingyang Wu and Yong Jae Lee. 2023. Visual instruction tuning. NeurIPS."},{"key":"e_1_3_2_1_41_1","doi-asserted-by":"publisher","unstructured":"Guannan Lou. 2022. Testing of autonomous driving systems: where are we and where should we go? In FSE. 31\u201343. https:\/\/doi.org\/10.1145\/3540250.3549111 10.1145\/3540250.3549111","DOI":"10.1145\/3540250.3549111"},{"key":"e_1_3_2_1_42_1","unstructured":"2023. Mako Robotic-Arm. https:\/\/t.ly\/VLEwP"},{"key":"e_1_3_2_1_43_1","unstructured":"2023. Mazor Robotics. http:\/\/www.medtronic.com\/"},{"key":"e_1_3_2_1_44_1","doi-asserted-by":"publisher","unstructured":"Ruijie Meng. 2022. Linear-time temporal logic guided greybox fuzzing. In ICSE. 1343\u20131355. https:\/\/doi.org\/10.1145\/3510003.3510082 10.1145\/3510003.3510082","DOI":"10.1145\/3510003.3510082"},{"key":"e_1_3_2_1_45_1","doi-asserted-by":"publisher","unstructured":"Daniel Neider and Ivan Gavran. 2018. Learning linear temporal properties. In FMCAD. 1\u201310. https:\/\/doi.org\/10.23919\/FMCAD.2018.8603016 10.23919\/FMCAD.2018.8603016","DOI":"10.23919\/FMCAD.2018.8603016"},{"key":"e_1_3_2_1_46_1","doi-asserted-by":"crossref","unstructured":"Aditya Prakash. 2021. Multi-modal fusion transformer for end-to-end autonomous driving. In CVPR. 7077\u20137087.","DOI":"10.1109\/CVPR46437.2021.00700"},{"key":"e_1_3_2_1_47_1","unstructured":"Associated Press. 2022. Nearly 400 car crashes in 11 months involved automated tech companies tell regulators. https:\/\/t.ly\/UTb2e"},{"key":"e_1_3_2_1_48_1","doi-asserted-by":"crossref","unstructured":"Rodrigo Queiroz. 2019. GeoScenario: An Open DSL for Autonomous Driving Scenario Representation. In IV. 287\u2013294.","DOI":"10.1109\/IVS.2019.8814107"},{"key":"e_1_3_2_1_49_1","volume-title":"Model-based testing","author":"Schieferdecker Ina","year":"2012","unstructured":"Ina Schieferdecker and Andreas Hoffmann. 2012. Model-based testing. IEEE software, 29, 1 (2012), 14\u201318."},{"key":"e_1_3_2_1_50_1","unstructured":"Sanjit A Seshia. 2017. Compositional verification without compositional specification for learning-based systems. UC Berkeley 1\u20138."},{"key":"e_1_3_2_1_51_1","doi-asserted-by":"crossref","unstructured":"Shital Shah. 2017. AirSim: High-Fidelity Visual and Physical Simulation for Autonomous Vehicles. In Field and Service Robotics. arxiv:arXiv:1705.05065. arxiv:1705.05065","DOI":"10.1007\/978-3-319-67361-5_40"},{"key":"e_1_3_2_1_52_1","unstructured":"Shai Shalev-Shwartz. 2017. On a formal model of safe and scalable self-driving cars. arXiv preprint arXiv:1708.06374."},{"key":"e_1_3_2_1_53_1","doi-asserted-by":"publisher","DOI":"10.4103\/0970-1591.174781"},{"volume-title":"Software engineering (ed.)","author":"Sommerville Ian","key":"e_1_3_2_1_54_1","unstructured":"Ian Sommerville. 2011. Software engineering (ed.). America: Pearson Education Inc."},{"key":"e_1_3_2_1_55_1","doi-asserted-by":"crossref","unstructured":"Bernhard Steffen Falk Howar and Maik Merten. 2011. Introduction to active automata learning from a practical perspective. SFM 256\u2013296.","DOI":"10.1007\/978-3-642-21455-4_8"},{"key":"e_1_3_2_1_56_1","unstructured":"2023. Tesla. http:\/\/www.tesla.com\/autopilot\/"},{"key":"e_1_3_2_1_57_1","doi-asserted-by":"publisher","unstructured":"Haoxiang Tian. 2022. MOSAT: finding safety violations of autonomous driving systems using multi-objective genetic algorithm. In FSE. 94\u2013106. https:\/\/doi.org\/10.1145\/3540250.3549100 10.1145\/3540250.3549100","DOI":"10.1145\/3540250.3549100"},{"key":"e_1_3_2_1_58_1","doi-asserted-by":"publisher","DOI":"10.1145\/3180155.3180220"},{"key":"e_1_3_2_1_59_1","doi-asserted-by":"crossref","unstructured":"Hoang-Dung Tran. 2020. Verification of deep convolutional neural networks using imagestars. In CAV. 18\u201342.","DOI":"10.1007\/978-3-030-53288-8_2"},{"volume-title":"https:\/\/shorturl.at\/mSV59","key":"e_1_3_2_1_60_1","unstructured":"2023. Uber. https:\/\/shorturl.at\/mSV59"},{"key":"e_1_3_2_1_61_1","unstructured":"The Verge. 2022. Food delivery drone. https:\/\/t.ly\/ATsPu"},{"key":"e_1_3_2_1_62_1","unstructured":"Junjie Wang. 2023. Software Testing with Large Language Model: Survey Landscape and Vision. arXiv preprint arXiv:2307.07221."},{"key":"e_1_3_2_1_63_1","unstructured":"2023. Waymo. http:\/\/www.waymo.com\/"},{"key":"e_1_3_2_1_64_1","unstructured":"Cerdic Wei Kit Wong. 2022. American fuzzy lop (AFL) fuzzer."},{"key":"e_1_3_2_1_65_1","doi-asserted-by":"crossref","unstructured":"Eleni Zapridou. 2020. Runtime verification of autonomous driving systems in CARLA. In RV. 172\u2013183.","DOI":"10.1007\/978-3-030-60508-7_9"},{"key":"e_1_3_2_1_66_1","unstructured":"Cen Zhang. 2023. Understanding Large Language Model Based Fuzz Driver Generation. arXiv preprint arXiv:2307.12469."},{"key":"e_1_3_2_1_67_1","doi-asserted-by":"publisher","DOI":"10.1109\/IROS47612.2022.9981775"},{"key":"e_1_3_2_1_68_1","doi-asserted-by":"publisher","unstructured":"Ziyuan Zhong. 2022. Neural network guided evolutionary fuzzing for finding traffic violations of autonomous vehicles. TSE https:\/\/doi.org\/10.1109\/TSE.2022.3195640 10.1109\/TSE.2022.3195640","DOI":"10.1109\/TSE.2022.3195640"},{"key":"e_1_3_2_1_69_1","doi-asserted-by":"publisher","unstructured":"Yuan Zhou. 2023. Specification-based Autonomous Driving System Testing. TSE https:\/\/doi.org\/10.1109\/TSE.2023.3254142 10.1109\/TSE.2023.3254142","DOI":"10.1109\/TSE.2023.3254142"},{"key":"e_1_3_2_1_70_1","unstructured":"2023. Zipline. http:\/\/www.flyzipline.com\/"}],"event":{"name":"FSE '24: 32nd ACM International Conference on the Foundations of Software Engineering","sponsor":["SIGSOFT ACM Special Interest Group on Software Engineering"],"location":"Porto de Galinhas Brazil","acronym":"FSE '24"},"container-title":["Companion Proceedings of the 32nd ACM International Conference on the Foundations of Software Engineering"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3663529.3663779","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3663529.3663779","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,18]],"date-time":"2025-06-18T23:44:20Z","timestamp":1750290260000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3663529.3663779"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,7,10]]},"references-count":70,"alternative-id":["10.1145\/3663529.3663779","10.1145\/3663529"],"URL":"https:\/\/doi.org\/10.1145\/3663529.3663779","relation":{},"subject":[],"published":{"date-parts":[[2024,7,10]]},"assertion":[{"value":"2024-07-10","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}