{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,10]],"date-time":"2026-10-10T16:08:08Z","timestamp":1791648488793,"version":"4.3.4"},"publisher-location":"New York, NY, USA","reference-count":87,"publisher":"ACM","license":[{"start":{"date-parts":[[2024,2,6]],"date-time":"2024-02-06T00:00:00Z","timestamp":1707177600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"name":"Australian Research Council - Discovery Projects","award":["DP210100041"],"award-info":[{"award-number":["DP210100041"]}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2024,2,6]]},"DOI":"10.1145\/3597503.3623314","type":"proceedings-article","created":{"date-parts":[[2024,2,6]],"date-time":"2024-02-06T20:53:16Z","timestamp":1707252796000},"page":"1-12","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":16,"title":["Towards Reliable AI: Adequacy Metrics for Ensuring the Quality of System-level Testing of Autonomous Vehicles"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-2572-0250","authenticated-orcid":false,"given":"Neelofar","family":"Neelofar","sequence":"first","affiliation":[{"name":"Faculty of Information Technology, Monash University, Melbourne, Victoria, Australia"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-1716-690X","authenticated-orcid":false,"given":"Aldeida","family":"Aleti","sequence":"additional","affiliation":[{"name":"Faculty of Information Technology, Monash University, Melbourne, Victoria, Australia"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2024,2,6]]},"reference":[{"key":"e_1_3_2_1_1_1","unstructured":"Mathworks polyarea. https:\/\/www.mathworks.com\/help\/matlab\/ref\/polyarea.html."},{"key":"e_1_3_2_1_2_1","unstructured":"Mathworks polygons. https:\/\/au.mathworks.com\/help\/map\/create-and-display-polygons.html."},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1145\/3238147.3238192"},{"key":"e_1_3_2_1_4_1","volume-title":"Principal component analysis","author":"Abdi Herv\u00e9","year":"2010","unstructured":"Herv\u00e9 Abdi and Lynne J Williams. Principal component analysis. Wiley interdisciplinary reviews: computational statistics, 2(4):433--459, 2010."},{"key":"e_1_3_2_1_5_1","volume-title":"Black-box testing of deep neural networks through test case diversity. arXiv preprint arXiv:2112.12591","author":"Aghababaeyan Zohreh","year":"2021","unstructured":"Zohreh Aghababaeyan, Manel Abdellatif, Lionel Briand, and Mojtaba Bagherzadeh. Black-box testing of deep neural networks through test case diversity. arXiv preprint arXiv:2112.12591, 2021."},{"key":"e_1_3_2_1_6_1","volume-title":"Qi Alfred Chen, and Joshua Garcia. scenorita: Generating less-redundant, safety-critical and motion sickness-inducing scenarios for autonomous vehicles. arXiv preprint arXiv:2112.09725","author":"Almanee Sumaya","year":"2021","unstructured":"Sumaya Almanee, Xiafa Wu, Yuqi Huai, Qi Alfred Chen, and Joshua Garcia. scenorita: Generating less-redundant, safety-critical and motion sickness-inducing scenarios for autonomous vehicles. arXiv preprint arXiv:2112.09725, 2021."},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"publisher","DOI":"10.1080\/10635150590905984"},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICCIMA.2007.328"},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICST49551.2021.00042"},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.1109\/18.681318"},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2017.2731766"},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.1109\/SANER53432.2022.00030"},{"key":"e_1_3_2_1_13_1","volume-title":"Automated test cases prioritization for self-driving cars in virtual environments. arXiv preprint arXiv:2107.09614","author":"Birchler Christian","year":"2021","unstructured":"Christian Birchler, Sajad Khatiri, Pouria Derakhshanfar, Sebastiano Panichella, and Annibale Panichella. Automated test cases prioritization for self-driving cars in virtual environments. arXiv preprint arXiv:2107.09614, 2021."},{"key":"e_1_3_2_1_14_1","volume-title":"Proceedings of the ACM on Measurement and Analysis of Computing Systems","author":"Birchler Christian","year":"2022","unstructured":"Christian Birchler, Sajad Khatiri, Pouria Derakhshanfar, Sebastiano Panichella, and Annibale Panichella. Single and multi-objective test cases prioritization for self-driving cars in virtual environments. Proceedings of the ACM on Measurement and Analysis of Computing Systems, 2022."},{"key":"e_1_3_2_1_15_1","volume-title":"Stads: Software testing as species discovery. ACM Transactions on Software Engineering and Methodology (TOSEM), 27(2):1--52","author":"B\u00f6hme Marcel","year":"2018","unstructured":"Marcel B\u00f6hme. Stads: Software testing as species discovery. ACM Transactions on Software Engineering and Methodology (TOSEM), 27(2):1--52, 2018."},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.ins.2017.11.051"},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.1002\/stvr.413"},{"key":"e_1_3_2_1_18_1","volume-title":"Deep neural network test coverage: How far are we? arXiv preprint arXiv:2010.04946","author":"Chen Junjie","year":"2020","unstructured":"Junjie Chen, Ming Yan, Zan Wang, Yuning Kang, and Zhuo Wu. Deep neural network test coverage: How far are we? arXiv preprint arXiv:2010.04946, 2020."},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","DOI":"10.1109\/TIT.2005.844059"},{"key":"e_1_3_2_1_20_1","volume-title":"Normalized compression distance of multisets with applications","author":"Cohen Andrew R","year":"2014","unstructured":"Andrew R Cohen and Paul MB Vit\u00e1nyi. Normalized compression distance of multisets with applications. IEEE transactions on pattern analysis and machine intelligence, 37(8):1602--1614, 2014."},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.3390\/e20020099"},{"key":"e_1_3_2_1_22_1","volume-title":"Looking for natural patterns in data: Part 1. density-based approach. Chemometrics and Intelligent laboratory systems, 56(2):83--92","author":"Daszykowski Michal","year":"2001","unstructured":"Michal Daszykowski, Beata Walczak, and DL Massart. Looking for natural patterns in data: Part 1. density-based approach. Chemometrics and Intelligent laboratory systems, 56(2):83--92, 2001."},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.1037\/met0000079"},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"publisher","DOI":"10.1109\/PerCom45495.2020.9127389"},{"key":"e_1_3_2_1_25_1","volume-title":"Jin Song Dong, and Dai Ting. There is limited correlation between coverage and robustness for deep neural networks. arXiv preprint arXiv:1911.05904","author":"Dong Yizhen","year":"2019","unstructured":"Yizhen Dong, Peixin Zhang, Jingyi Wang, Shuang Liu, Jun Sun, Jianye Hao, Xinyu Wang, Li Wang, Jin Song Dong, and Dai Ting. There is limited correlation between coverage and robustness for deep neural networks. arXiv preprint arXiv:1911.05904, 2019."},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICECCS51672.2020.00016"},{"key":"e_1_3_2_1_27_1","doi-asserted-by":"publisher","DOI":"10.1109\/AITEST52744.2021.00030"},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"publisher","DOI":"10.1109\/TIT.1983.1056714"},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICST.2016.33"},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSTW.2008.36"},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"publisher","DOI":"10.1145\/3395363.3397357"},{"key":"e_1_3_2_1_32_1","doi-asserted-by":"publisher","DOI":"10.1145\/3293882.3330566"},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE-Companion.2019.00030"},{"key":"e_1_3_2_1_34_1","doi-asserted-by":"publisher","DOI":"10.1145\/3377811.3380391"},{"key":"e_1_3_2_1_35_1","doi-asserted-by":"publisher","DOI":"10.1145\/3236024.3264835"},{"key":"e_1_3_2_1_36_1","doi-asserted-by":"publisher","DOI":"10.1109\/ITSC.2019.8917326"},{"key":"e_1_3_2_1_37_1","doi-asserted-by":"publisher","DOI":"10.1145\/2884781.2884791"},{"key":"e_1_3_2_1_38_1","volume-title":"Application story of odd as part of safety assurance. https:\/\/www.asam.net\/index.php?eID=dumpFile&t=f&f=4303&token=3135965e578e5bb92a01725cd37823c3979da158","author":"Kaiser Bernhard","year":"2021","unstructured":"Bernhard Kaiser. Application story of odd as part of safety assurance. https:\/\/www.asam.net\/index.php?eID=dumpFile&t=f&f=4303&token=3135965e578e5bb92a01725cd37823c3979da158, 2021."},{"key":"e_1_3_2_1_39_1","doi-asserted-by":"publisher","DOI":"10.1145\/3230711"},{"key":"e_1_3_2_1_40_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE.2019.00108"},{"key":"e_1_3_2_1_41_1","doi-asserted-by":"publisher","DOI":"10.1016\/S0164-1212(03)00217-6"},{"key":"e_1_3_2_1_42_1","doi-asserted-by":"publisher","DOI":"10.1093\/bioinformatics\/bti806"},{"key":"e_1_3_2_1_43_1","volume-title":"Three approaches to the quantitative definition ofinformation'. Problems of information transmission, 1(1):1--7","author":"Kolmogorov Andrei N","year":"1965","unstructured":"Andrei N Kolmogorov. Three approaches to the quantitative definition ofinformation'. Problems of information transmission, 1(1):1--7, 1965."},{"key":"e_1_3_2_1_44_1","doi-asserted-by":"publisher","DOI":"10.1561\/9781601986290"},{"key":"e_1_3_2_1_45_1","volume-title":"ACM Transactions on Software Engineering and Methodology","author":"Laurent Thomas","year":"2022","unstructured":"Thomas Laurent, Stefan Klikovits, Paolo Arcaini, Fuyuki Ishikawa, and Anthony Ventresque. Parameter coverage for testing of autonomous driving systems under uncertainty. ACM Transactions on Software Engineering and Methodology, 2022."},{"key":"e_1_3_2_1_46_1","doi-asserted-by":"publisher","DOI":"10.1109\/ISSRE5003.2020.00012"},{"key":"e_1_3_2_1_47_1","doi-asserted-by":"publisher","DOI":"10.1109\/TIT.2004.838101"},{"key":"e_1_3_2_1_48_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE-NIER.2019.00031"},{"key":"e_1_3_2_1_49_1","article-title":"Learning configurations of operating environment of autonomous vehicles to maximize their collisions","author":"Lu Chengjie","year":"2022","unstructured":"Chengjie Lu, Yize Shi, Huihui Zhang, Man Zhang, Tiexin Wang, Tao Yue, and Shaukat Ali. Learning configurations of operating environment of autonomous vehicles to maximize their collisions. IEEE Transactions on Software Engineering, 2022.","journal-title":"IEEE Transactions on Software Engineering"},{"key":"e_1_3_2_1_50_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-88106-1_4"},{"key":"e_1_3_2_1_51_1","doi-asserted-by":"publisher","DOI":"10.1145\/3238147.3238202"},{"key":"e_1_3_2_1_52_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.cub.2021.07.049"},{"key":"e_1_3_2_1_53_1","doi-asserted-by":"publisher","DOI":"10.1109\/MODELS.2019.00-12"},{"key":"e_1_3_2_1_54_1","doi-asserted-by":"publisher","DOI":"10.1017\/CBO9780511809071"},{"key":"e_1_3_2_1_55_1","volume-title":"Anna Maria Vollmer, and Stefan Wagner. Software engineering for ai-based systems: a survey. ACM Transactions on Software Engineering and Methodology (TOSEM), 31(2):1--59","author":"Mart\u00ednez-Fern\u00e1ndez Silverio","year":"2022","unstructured":"Silverio Mart\u00ednez-Fern\u00e1ndez, Justus Bogner, Xavier Franch, Marc Oriol, Julien Siebert, Adam Trendowicz, Anna Maria Vollmer, and Stefan Wagner. Software engineering for ai-based systems: a survey. ACM Transactions on Software Engineering and Methodology (TOSEM), 31(2):1--59, 2022."},{"key":"e_1_3_2_1_56_1","volume-title":"Markus Bohlin, and Bj\u00f6rn Lisper. Machine learning testing in an adas case study using simulation-integrated bio-inspired search-based testing. arXiv preprint arXiv:2203.12026","author":"Moghadam Mahshid Helali","year":"2022","unstructured":"Mahshid Helali Moghadam, Markus Borg, Mehrdad Saadatmand, Seyed Jalaleddin Mousavirad, Markus Bohlin, and Bj\u00f6rn Lisper. Machine learning testing in an adas case study using simulation-integrated bio-inspired search-based testing. arXiv preprint arXiv:2203.12026, 2022."},{"key":"e_1_3_2_1_57_1","volume-title":"Illuminating search spaces by mapping elites. arXiv preprint arXiv:1504.04909","author":"Mouret Jean-Baptiste","year":"2015","unstructured":"Jean-Baptiste Mouret and Jeff Clune. Illuminating search spaces by mapping elites. arXiv preprint arXiv:1504.04909, 2015."},{"key":"e_1_3_2_1_58_1","doi-asserted-by":"publisher","DOI":"10.1145\/3323933.3324060"},{"key":"e_1_3_2_1_59_1","doi-asserted-by":"publisher","DOI":"10.1162\/evco_a_00194"},{"key":"e_1_3_2_1_60_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10994-017-5629-5"},{"key":"e_1_3_2_1_61_1","unstructured":"Mario A Mu\u00f1oz Tao Yan Matheus R Leal Kate A Smith-Miles Ana C Lorena Gisele L Pappa and Rodrigues Romulo M. An instance space analysis of regression problems. ACM Transactions on Knowledge Discovery from Data."},{"key":"e_1_3_2_1_62_1","article-title":"Instance space analysis of search-based software testing","author":"Smith-Miles Kate","year":"2022","unstructured":"Neelofar, Kate Smith-Miles, Mario Andr\u00e9s Mu\u00f1oz, and Aldeida Aleti. Instance space analysis of search-based software testing. IEEE Transactions on Software Engineering, 2022.","journal-title":"IEEE Transactions on Software Engineering"},{"key":"e_1_3_2_1_63_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.infsof.2006.02.001"},{"key":"e_1_3_2_1_64_1","doi-asserted-by":"publisher","DOI":"10.1145\/3132747.3132785"},{"key":"e_1_3_2_1_65_1","doi-asserted-by":"publisher","DOI":"10.1109\/TVCG.2010.216"},{"key":"e_1_3_2_1_66_1","doi-asserted-by":"publisher","DOI":"10.1109\/AITEST52744.2021.00035"},{"key":"e_1_3_2_1_67_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10664-020-09881-0"},{"key":"e_1_3_2_1_68_1","doi-asserted-by":"publisher","DOI":"10.1145\/3368089.3409730"},{"key":"e_1_3_2_1_69_1","doi-asserted-by":"publisher","DOI":"10.1109\/ITSC45102.2020.9294422"},{"key":"e_1_3_2_1_70_1","volume-title":"International Journal of Engineering and Technology (IJET), 5(3):2701--2704","author":"Saranya C","year":"2013","unstructured":"C Saranya and G Manikandan. A study on normalization techniques for privacy preserving data mining. International Journal of Engineering and Technology (IJET), 5(3):2701--2704, 2013."},{"key":"e_1_3_2_1_71_1","volume-title":"Hans Peter Kriegel, and Xiaowei Xu. Dbscan revisited, revisited: why and how you should (still) use dbscan. ACM Transactions on Database Systems (TODS), 42(3):1--21","author":"Schubert Erich","year":"2017","unstructured":"Erich Schubert, J\u00f6rg Sander, Martin Ester, Hans Peter Kriegel, and Xiaowei Xu. Dbscan revisited, revisited: why and how you should (still) use dbscan. ACM Transactions on Database Systems (TODS), 42(3):1--21, 2017."},{"key":"e_1_3_2_1_72_1","doi-asserted-by":"publisher","DOI":"10.1109\/SMC52423.2021.9658812"},{"key":"e_1_3_2_1_73_1","volume-title":"On a formal model of safe and scalable self-driving cars. arXiv preprint arXiv:1708.06374","author":"Shalev-Shwartz Shai","year":"2017","unstructured":"Shai Shalev-Shwartz, Shaked Shammah, and Amnon Shashua. On a formal model of safe and scalable self-driving cars. arXiv preprint arXiv:1708.06374, 2017."},{"key":"e_1_3_2_1_74_1","doi-asserted-by":"publisher","DOI":"10.4271\/2021-01-1011"},{"key":"e_1_3_2_1_75_1","doi-asserted-by":"publisher","DOI":"10.1145\/3551349.3556968"},{"key":"e_1_3_2_1_76_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10664-023-10306-x"},{"key":"e_1_3_2_1_77_1","doi-asserted-by":"publisher","DOI":"10.1109\/IJCNN52387.2021.9533987"},{"key":"e_1_3_2_1_78_1","volume-title":"Structural test coverage criteria for deep neural networks. ACM Transactions on Embedded Computing Systems (TECS), 18(5s):1--23","author":"Sun Youcheng","year":"2019","unstructured":"Youcheng Sun, Xiaowei Huang, Daniel Kroening, James Sharp, Matthew Hill, and Rob Ashmore. Structural test coverage criteria for deep neural networks. ACM Transactions on Embedded Computing Systems (TECS), 18(5s):1--23, 2019."},{"key":"e_1_3_2_1_79_1","doi-asserted-by":"publisher","DOI":"10.1109\/IV48863.2021.9575536"},{"key":"e_1_3_2_1_80_1","doi-asserted-by":"publisher","DOI":"10.1145\/3540250.3549100"},{"key":"e_1_3_2_1_81_1","doi-asserted-by":"publisher","DOI":"10.1145\/3180155.3180220"},{"key":"e_1_3_2_1_82_1","doi-asserted-by":"publisher","DOI":"10.1145\/3293882.3330579"},{"key":"e_1_3_2_1_83_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.aap.2022.106776"},{"key":"e_1_3_2_1_84_1","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2019.2962027"},{"key":"e_1_3_2_1_85_1","doi-asserted-by":"publisher","DOI":"10.1145\/3238147.3238187"},{"key":"e_1_3_2_1_86_1","volume-title":"Fairness testing of deep image classification with adequacy metrics. arXiv preprint arXiv:2111.08856","author":"Zhang Peixin","year":"2021","unstructured":"Peixin Zhang, Jingyi Wang, Jun Sun, and Xinyu Wang. Fairness testing of deep image classification with adequacy metrics. arXiv preprint arXiv:2111.08856, 2021."},{"key":"e_1_3_2_1_87_1","doi-asserted-by":"publisher","DOI":"10.1145\/3460319.3464811"}],"event":{"name":"ICSE '24: IEEE\/ACM 46th International Conference on Software Engineering","location":"Lisbon Portugal","acronym":"ICSE '24","sponsor":["SIGSOFT ACM Special Interest Group on Software Engineering","IEEE CS","Faculty of Engineering of University of Porto"]},"container-title":["Proceedings of the IEEE\/ACM 46th International Conference on Software Engineering"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3597503.3623314","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3597503.3623314","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T17:48:45Z","timestamp":1750182525000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3597503.3623314"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,2,6]]},"references-count":87,"alternative-id":["10.1145\/3597503.3623314","10.1145\/3597503"],"URL":"https:\/\/doi.org\/10.1145\/3597503.3623314","relation":{},"subject":[],"published":{"date-parts":[[2024,2,6]]},"assertion":[{"value":"2024-02-06","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}