{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,3]],"date-time":"2026-07-03T16:40:51Z","timestamp":1783096851079,"version":"3.54.6"},"publisher-location":"New York, NY, USA","reference-count":111,"publisher":"ACM","license":[{"start":{"date-parts":[[2022,11,7]],"date-time":"2022-11-07T00:00:00Z","timestamp":1667779200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"name":"Australian Research Council Linkage Project","award":["LP190100676"],"award-info":[{"award-number":["LP190100676"]}]},{"name":"Australian Research Council Discovery Project","award":["DP210102447"],"award-info":[{"award-number":["DP210102447"]}]},{"name":"DATA6","award":["C020996"],"award-info":[{"award-number":["C020996"]}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2022,11,7]]},"DOI":"10.1145\/3540250.3549111","type":"proceedings-article","created":{"date-parts":[[2022,11,9]],"date-time":"2022-11-09T20:46:22Z","timestamp":1668026782000},"page":"31-43","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":76,"title":["Testing of autonomous driving systems: where are we and where should we go?"],"prefix":"10.1145","author":[{"given":"Guannan","family":"Lou","sequence":"first","affiliation":[{"name":"Macquarie University, Australia"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Yao","family":"Deng","sequence":"additional","affiliation":[{"name":"Macquarie University, Australia"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Xi","family":"Zheng","sequence":"additional","affiliation":[{"name":"Macquarie University, Australia"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Mengshi","family":"Zhang","sequence":"additional","affiliation":[{"name":"Meta, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Tianyi","family":"Zhang","sequence":"additional","affiliation":[{"name":"Purdue University, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2022,11,9]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"publisher","DOI":"10.1145\/3180155.3180160"},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.1145\/3238147.3238192"},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"crossref","unstructured":"M. Althoff M. Koschi and S. Manzinger. 2017. CommonRoad: Composable benchmarks for motion planning on roads. In IV. 719\u2013726. M. Althoff M. Koschi and S. Manzinger. 2017. CommonRoad: Composable benchmarks for motion planning on roads. In IV. 719\u2013726.","DOI":"10.1109\/IVS.2017.7995802"},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"crossref","unstructured":"S. Amershi A. Begel C. Bird R. DeLine H. Gall E. Kamar N. Nagappan B. Nushi and T. Zimmermann. 2019. Software engineering for machine learning: A case study. In ICSE. 291\u2013300. S. Amershi A. Begel C. Bird R. DeLine H. Gall E. Kamar N. Nagappan B. Nushi and T. Zimmermann. 2019. Software engineering for machine learning: A case study. In ICSE. 291\u2013300.","DOI":"10.1109\/ICSE-SEIP.2019.00042"},{"key":"e_1_3_2_1_5_1","unstructured":"ApolloAuto. 2021. Apollo. https:\/\/bit.ly\/2E3vWyo ApolloAuto. 2021. Apollo. https:\/\/bit.ly\/2E3vWyo"},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"crossref","unstructured":"Paolo Arcaini Xiao-Yi Zhang and Fuyuki Ishikawa. 2021. Targeting patterns of driving characteristics in testing autonomous driving systems. In ICST. 295\u2013305. Paolo Arcaini Xiao-Yi Zhang and Fuyuki Ishikawa. 2021. Targeting patterns of driving characteristics in testing autonomous driving systems. In ICST. 295\u2013305.","DOI":"10.1109\/ICST49551.2021.00042"},{"key":"e_1_3_2_1_7_1","unstructured":"ASAM. 2021. ASAM OpenSCENARIO. https:\/\/bit.ly\/3ya34Rm ASAM. 2021. ASAM OpenSCENARIO. https:\/\/bit.ly\/3ya34Rm"},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"crossref","unstructured":"Autoware-AI. 2020. autoware.ai. https:\/\/bit.ly\/3gZ1gBS Autoware-AI. 2020. autoware.ai. https:\/\/bit.ly\/3gZ1gBS","DOI":"10.1016\/j.aiopen.2021.04.001"},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.patcog.2020.107612"},{"key":"e_1_3_2_1_10_1","volume-title":"Lanelets: Efficient map representation for autonomous driving. In IV. 420\u2013425.","author":"Bender P.","year":"2014","unstructured":"P. Bender , J. Ziegler , and C. Stiller . 2014 . Lanelets: Efficient map representation for autonomous driving. In IV. 420\u2013425. P. Bender, J. Ziegler, and C. Stiller. 2014. Lanelets: Efficient map representation for autonomous driving. In IV. 420\u2013425."},{"key":"e_1_3_2_1_11_1","volume-title":"CARSIM: Car-following model for simulation of traffic in normal and stop-and-go conditions. Transportation research record, 1194","author":"Benekohal R.","year":"1988","unstructured":"R. Benekohal and J. Treiterer . 1988 . CARSIM: Car-following model for simulation of traffic in normal and stop-and-go conditions. Transportation research record, 1194 (1988), 99\u2013111. R. Benekohal and J. Treiterer. 1988. CARSIM: Car-following model for simulation of traffic in normal and stop-and-go conditions. Transportation research record, 1194 (1988), 99\u2013111."},{"key":"e_1_3_2_1_12_1","unstructured":"The National Transportation Safety Board. 2019. Preliminary Report Highway Hwy18mh010. https:\/\/bit.ly\/2N0SHuj The National Transportation Safety Board. 2019. Preliminary Report Highway Hwy18mh010. https:\/\/bit.ly\/2N0SHuj"},{"key":"e_1_3_2_1_13_1","unstructured":"M. Bojarski P. Yeres A. Choromanska K. Choromanski B. Firner Lawrence D. Jackel and U. Muller. 2017. Explaining How a Deep Neural Network Trained with End-to-End Learning Steers a Car. CoRR abs\/1704.07911 (2017). M. Bojarski P. Yeres A. Choromanska K. Choromanski B. Firner Lawrence D. Jackel and U. Muller. 2017. Explaining How a Deep Neural Network Trained with End-to-End Learning Steers a Car. CoRR abs\/1704.07911 (2017)."},{"key":"e_1_3_2_1_14_1","volume-title":"Shiva Nejati, Fran\u00e7ois-Xavier Jegeden, and Donghwan Shin.","author":"Borg Markus","year":"2021","unstructured":"Markus Borg , Raja Ben Abdessalem , Shiva Nejati, Fran\u00e7ois-Xavier Jegeden, and Donghwan Shin. 2021 . Digital twins are not monozygotic\u2013cross-replicating adas testing in two industry-grade automotive simulators. In ICST. 383\u2013393. Markus Borg, Raja Ben Abdessalem, Shiva Nejati, Fran\u00e7ois-Xavier Jegeden, and Donghwan Shin. 2021. Digital twins are not monozygotic\u2013cross-replicating adas testing in two industry-grade automotive simulators. In ICST. 383\u2013393."},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"crossref","unstructured":"M. Borg C. Englund K. Wnuk B. Duran C. Levandowski S. Gao Y. Tan H. Kaijser H. L\u00f6nn and J. T\u00f6rnqvist. 2018. Safely entering the deep: A review of verification and validation for machine learning and a challenge elicitation in the automotive industry. arXiv preprint arXiv:1812.05389. M. Borg C. Englund K. Wnuk B. Duran C. Levandowski S. Gao Y. Tan H. Kaijser H. L\u00f6nn and J. T\u00f6rnqvist. 2018. Safely entering the deep: A review of verification and validation for machine learning and a challenge elicitation in the automotive industry. arXiv preprint arXiv:1812.05389.","DOI":"10.2991\/jase.d.190131.001"},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"crossref","unstructured":"V. Braun and V. Clarke. 2006. Using thematic analysis in psychology. Qualitative research in psychology 3 2 (2006) 77\u2013101. V. Braun and V. Clarke. 2006. Using thematic analysis in psychology. Qualitative research in psychology 3 2 (2006) 77\u2013101.","DOI":"10.1191\/1478088706qp063oa"},{"key":"e_1_3_2_1_17_1","volume-title":"Conference on Software for Citical Systems. 66\u201376","author":"Butler R.","unstructured":"R. Butler and G. Finelli . 1991. The infeasibility of experimental quantification of life-critical software reliability . In Conference on Software for Citical Systems. 66\u201376 . R. Butler and G. Finelli. 1991. The infeasibility of experimental quantification of life-critical software reliability. In Conference on Software for Citical Systems. 66\u201376."},{"key":"e_1_3_2_1_18_1","unstructured":"Alessandro Cal\u00f2 Paolo Arcaini Shaukat Ali Florian Hauer and Fuyuki Ishikawa. 2020. Generating avoidable collision scenarios for testing autonomous driving systems. In ICST. 375\u2013386. Alessandro Cal\u00f2 Paolo Arcaini Shaukat Ali Florian Hauer and Fuyuki Ishikawa. 2020. Generating avoidable collision scenarios for testing autonomous driving systems. In ICST. 375\u2013386."},{"key":"e_1_3_2_1_19_1","volume-title":"Carla: Open-Source Simulator for Autonomous Driving Research. https:\/\/bit.ly\/3qE26qA","year":"2021","unstructured":"Carla. 2021 . Carla: Open-Source Simulator for Autonomous Driving Research. https:\/\/bit.ly\/3qE26qA Carla. 2021. Carla: Open-Source Simulator for Autonomous Driving Research. https:\/\/bit.ly\/3qE26qA"},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSTW52544.2021.00022"},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"crossref","unstructured":"L. Chen S. Fidler A. Yuille and R. Urtasun. 2014. Beat the mturkers: Automatic image labeling from weak 3d supervision. In CVPR. 3198\u20133205. L. Chen S. Fidler A. Yuille and R. Urtasun. 2014. Beat the mturkers: Automatic image labeling from weak 3d supervision. In CVPR. 3198\u20133205.","DOI":"10.1109\/CVPR.2014.409"},{"key":"e_1_3_2_1_22_1","volume-title":"International Conference on Management of Data. 2201\u20132206","author":"Chu X.","unstructured":"X. Chu , I. Ilyas , S. Krishnan , and J. Wang . 2016. Data cleaning: Overview and emerging challenges . In International Conference on Management of Data. 2201\u20132206 . X. Chu, I. Ilyas, S. Krishnan, and J. Wang. 2016. Data cleaning: Overview and emerging challenges. In International Conference on Management of Data. 2201\u20132206."},{"key":"e_1_3_2_1_23_1","volume-title":"Penumbra: Automatically identifying failure-relevant inputs using dynamic tainting. In ISSTA. 249\u2013260.","author":"Clause J.","year":"2009","unstructured":"J. Clause and A. Orso . 2009 . Penumbra: Automatically identifying failure-relevant inputs using dynamic tainting. In ISSTA. 249\u2013260. J. Clause and A. Orso. 2009. Penumbra: Automatically identifying failure-relevant inputs using dynamic tainting. In ISSTA. 249\u2013260."},{"key":"e_1_3_2_1_24_1","volume-title":"ai","year":"2021","unstructured":"comma. ai . 2021 . OpenPilot . https:\/\/bit.ly\/3w099fI comma. ai. 2021. OpenPilot. https:\/\/bit.ly\/3w099fI"},{"key":"e_1_3_2_1_25_1","first-page":"1","article-title":"Taxonomy and definitions for terms related to on-road motor vehicle automated driving systems","volume":"3016","author":"SAE On-Road Automated Vehicle Standards Committee","year":"2014","unstructured":"SAE On-Road Automated Vehicle Standards Committee . 2014 . Taxonomy and definitions for terms related to on-road motor vehicle automated driving systems . SAE Standard J , 3016 (2014), 1 \u2013 16 . SAE On-Road Automated Vehicle Standards Committee. 2014. Taxonomy and definitions for terms related to on-road motor vehicle automated driving systems. SAE Standard J, 3016 (2014), 1\u201316.","journal-title":"SAE Standard J"},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"crossref","unstructured":"M. Cordts M. Omran S. Ramos T. Rehfeld M. Enzweiler R. Benenson U. Franke S. Roth and B. Schiele. 2016. The Cityscapes Dataset for Semantic Urban Scene Understanding. In CVPR. M. Cordts M. Omran S. Ramos T. Rehfeld M. Enzweiler R. Benenson U. Franke S. Roth and B. Schiele. 2016. The Cityscapes Dataset for Semantic Urban Scene Understanding. In CVPR.","DOI":"10.1109\/CVPR.2016.350"},{"key":"e_1_3_2_1_27_1","unstructured":"DeepDrive. 2020. DeepDrive. https:\/\/bit.ly\/2OTsheJ DeepDrive. 2020. DeepDrive. https:\/\/bit.ly\/2OTsheJ"},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"publisher","DOI":"10.1109\/PerCom45495.2020.9127389"},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"crossref","unstructured":"S. Dey and S. Lee. 2021. Multilayered review of safety approaches for machine learning-based systems in the days of AI. Journal of Systems and Software 176 (2021) jun 110941. S. Dey and S. Lee. 2021. Multilayered review of safety approaches for machine learning-based systems in the days of AI. Journal of Systems and Software 176 (2021) jun 110941.","DOI":"10.1016\/j.jss.2021.110941"},{"key":"e_1_3_2_1_30_1","volume-title":"Towards Verified Safety-critical Autonomous Driving Scenario with ADSML. In 2021 IEEE 45th Annual Computers, Software, and Applications Conference (COMPSAC). 1333\u20131338","author":"Du Dehui","unstructured":"Dehui Du , Jiena Chen , Mingzhuo Zhang , and Mingjun Ma. 2021. Towards Verified Safety-critical Autonomous Driving Scenario with ADSML. In 2021 IEEE 45th Annual Computers, Software, and Applications Conference (COMPSAC). 1333\u20131338 . Dehui Du, Jiena Chen, Mingzhuo Zhang, and Mingjun Ma. 2021. Towards Verified Safety-critical Autonomous Driving Scenario with ADSML. In 2021 IEEE 45th Annual Computers, Software, and Applications Conference (COMPSAC). 1333\u20131338."},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.aap.2017.11.001"},{"key":"e_1_3_2_1_32_1","doi-asserted-by":"crossref","unstructured":"Y. Feng Q. Shi X. Gao J. Wan C. Fang and Z. Chen. 2020. DeepGini: prioritizing massive tests to enhance the robustness of deep neural networks. In ISSTA. ACM 177\u2013188. Y. Feng Q. Shi X. Gao J. Wan C. Fang and Z. Chen. 2020. DeepGini: prioritizing massive tests to enhance the robustness of deep neural networks. In ISSTA. ACM 177\u2013188.","DOI":"10.1145\/3395363.3397357"},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"crossref","unstructured":"D. Fremont T. Dreossi S. Ghosh X. Yue A. Sangiovanni-Vincentelli and S. Seshia. 2019. Scenic: a language for scenario specification and scene generation. In PLDI. 63\u201378. D. Fremont T. Dreossi S. Ghosh X. Yue A. Sangiovanni-Vincentelli and S. Seshia. 2019. Scenic: a language for scenario specification and scene generation. In PLDI. 63\u201378.","DOI":"10.1145\/3314221.3314633"},{"key":"e_1_3_2_1_34_1","doi-asserted-by":"crossref","unstructured":"Jonas Fritzsch Tobias Schmid and Stefan Wagner. 2021. Experiences from Large-Scale Model Checking: Verifying a Vehicle Control System with NuSMV. In ICST. 372\u2013382. Jonas Fritzsch Tobias Schmid and Stefan Wagner. 2021. Experiences from Large-Scale Model Checking: Verifying a Vehicle Control System with NuSMV. In ICST. 372\u2013382.","DOI":"10.1109\/ICST49551.2021.00049"},{"key":"e_1_3_2_1_35_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE-Companion.2019.00119"},{"key":"e_1_3_2_1_36_1","doi-asserted-by":"publisher","DOI":"10.1145\/3338906.3338942"},{"key":"e_1_3_2_1_37_1","doi-asserted-by":"publisher","DOI":"10.1145\/3293882.3330566"},{"key":"e_1_3_2_1_38_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE-Companion.2019.00030"},{"key":"e_1_3_2_1_39_1","doi-asserted-by":"crossref","unstructured":"J. Garcia Y. Feng J. Shen S. Almanee Y. Xia Chen and Q. Alfred. 2020. A comprehensive study of autonomous vehicle bugs. In ICSE. 385\u2013396. J. Garcia Y. Feng J. Shen S. Almanee Y. Xia Chen and Q. Alfred. 2020. A comprehensive study of autonomous vehicle bugs. In ICSE. 385\u2013396.","DOI":"10.1145\/3377811.3380397"},{"key":"e_1_3_2_1_40_1","volume-title":"Google sibling waymo launches fully autonomous ride-hailing service. The Guardian, 7","author":"Gibbs S.","year":"2017","unstructured":"S. Gibbs . 2017. Google sibling waymo launches fully autonomous ride-hailing service. The Guardian, 7 ( 2017 ). S. Gibbs. 2017. Google sibling waymo launches fully autonomous ride-hailing service. The Guardian, 7 (2017)."},{"key":"e_1_3_2_1_41_1","doi-asserted-by":"publisher","DOI":"10.1109\/ASE.2019.00013"},{"key":"e_1_3_2_1_42_1","volume-title":"E. Soremekun, and A. Zeller.","author":"Gopinath R.","year":"2020","unstructured":"R. Gopinath , A. Kampmann , er, N. Havrikov , E. Soremekun, and A. Zeller. 2020 . Abstracting failure-inducing inputs. In ISSTA. 237\u2013248. R. Gopinath, A. Kampmann, er, N. Havrikov, E. Soremekun, and A. Zeller. 2020. Abstracting failure-inducing inputs. In ISSTA. 237\u2013248."},{"key":"e_1_3_2_1_43_1","first-page":"1","article-title":"Data quality considerations for big data and machine learning: Going beyond data cleaning and transformations","volume":"10","author":"Gudivada V.","year":"2017","unstructured":"V. Gudivada , A. Apon , and J. Ding . 2017 . Data quality considerations for big data and machine learning: Going beyond data cleaning and transformations . International Journal on Advances in Software , 10 , 1 (2017), 1 \u2013 20 . V. Gudivada, A. Apon, and J. Ding. 2017. Data quality considerations for big data and machine learning: Going beyond data cleaning and transformations. International Journal on Advances in Software, 10, 1 (2017), 1\u201320.","journal-title":"International Journal on Advances in Software"},{"key":"e_1_3_2_1_44_1","volume-title":"S. Tetali, T. Condie, T. Millstein, and M. Kim.","author":"Gulzar M.","year":"2016","unstructured":"M. Gulzar , Interl, M. i, S. Yoo , S. Tetali, T. Condie, T. Millstein, and M. Kim. 2016 . Bigdebug : Debugging primitives for interactive big data processing in spark. In ICSE. 784\u2013795. M. Gulzar, Interl, M. i, S. Yoo, S. Tetali, T. Condie, T. Millstein, and M. Kim. 2016. Bigdebug: Debugging primitives for interactive big data processing in spark. In ICSE. 784\u2013795."},{"key":"e_1_3_2_1_45_1","volume-title":"Metamorphic Fuzz Testing of Autonomous Vehicles. In ICSE Workshop. ACM, 380\u2013385","author":"Han J.","unstructured":"J. Han and Z. Zhou . 2020 . Metamorphic Fuzz Testing of Autonomous Vehicles. In ICSE Workshop. ACM, 380\u2013385 . J. Han and Z. Zhou. 2020. Metamorphic Fuzz Testing of Autonomous Vehicles. In ICSE Workshop. ACM, 380\u2013385."},{"key":"e_1_3_2_1_46_1","unstructured":"Fitash Ul Haq Donghwan Shin Shiva Nejati and Lionel C Briand. 2020. Comparing offline and online testing of deep neural networks: An autonomous car case study. In ICST. 85\u201395. Fitash Ul Haq Donghwan Shin Shiva Nejati and Lionel C Briand. 2020. Comparing offline and online testing of deep neural networks: An autonomous car case study. In ICST. 85\u201395."},{"key":"e_1_3_2_1_47_1","unstructured":"Arthur Frederick Hasler. 2022. 60 000 Drivers Now Have Tesla Full Self Driving (FSD)\u2014What It Is & How To Get It. https:\/\/bit.ly\/3t1zIST Arthur Frederick Hasler. 2022. 60 000 Drivers Now Have Tesla Full Self Driving (FSD)\u2014What It Is & How To Get It. https:\/\/bit.ly\/3t1zIST"},{"key":"e_1_3_2_1_48_1","volume-title":"Disclosing the Fragility Problem of Virtual Safety Testing for Autonomous Driving Systems. In 2021 IEEE International Symposium on Software Reliability Engineering Workshops (ISSREW). 387\u2013392","author":"Hu Zhisheng","year":"2021","unstructured":"Zhisheng Hu , Shengjian Guo , Zhenyu Zhong , and Kang Li . 2021 . Disclosing the Fragility Problem of Virtual Safety Testing for Autonomous Driving Systems. In 2021 IEEE International Symposium on Software Reliability Engineering Workshops (ISSREW). 387\u2013392 . Zhisheng Hu, Shengjian Guo, Zhenyu Zhong, and Kang Li. 2021. Disclosing the Fragility Problem of Virtual Safety Testing for Autonomous Driving Systems. In 2021 IEEE International Symposium on Software Reliability Engineering Workshops (ISSREW). 387\u2013392."},{"key":"e_1_3_2_1_49_1","doi-asserted-by":"crossref","unstructured":"W. Huang K. Wang Y. Lv and F. Zhu. 2016. Autonomous vehicles testing methods review. In ITSC. 163\u2013168. W. Huang K. Wang Y. Lv and F. Zhu. 2016. Autonomous vehicles testing methods review. In ITSC. 163\u2013168.","DOI":"10.1109\/ITSC.2016.7795548"},{"key":"e_1_3_2_1_50_1","doi-asserted-by":"crossref","unstructured":"Fuyuki Ishikawa. 2020. Testing and Debugging Autonomous Driving: Experiences with Path Planner and Future Challenges. In 2020 IEEE International Symposium on Software Reliability Engineering Workshops (ISSREW). xxxiii\u2013xxxiv. Fuyuki Ishikawa. 2020. Testing and Debugging Autonomous Driving: Experiences with Path Planner and Future Challenges. In 2020 IEEE International Symposium on Software Reliability Engineering Workshops (ISSREW). xxxiii\u2013xxxiv.","DOI":"10.1109\/ISSREW51248.2020.00015"},{"key":"e_1_3_2_1_51_1","doi-asserted-by":"crossref","unstructured":"Gunel Jahangirova Andrea Stocco and Paolo Tonella. 2021. Quality metrics and oracles for autonomous vehicles testing. In ICST. 194\u2013204. Gunel Jahangirova Andrea Stocco and Paolo Tonella. 2021. Quality metrics and oracles for autonomous vehicles testing. In ICST. 194\u2013204.","DOI":"10.1109\/ICST49551.2021.00030"},{"key":"e_1_3_2_1_52_1","volume-title":"Mixed methods research: A research paradigm whose time has come. Educational researcher, 33, 7","author":"Burke Johnson R","year":"2004","unstructured":"R Burke Johnson and Anthony J Onwuegbuzie . 2004. Mixed methods research: A research paradigm whose time has come. Educational researcher, 33, 7 ( 2004 ), 14\u201326. R Burke Johnson and Anthony J Onwuegbuzie. 2004. Mixed methods research: A research paradigm whose time has come. Educational researcher, 33, 7 (2004), 14\u201326."},{"key":"e_1_3_2_1_53_1","volume-title":"Machine Learning and Artificial Intelligence","author":"Joshi A.","unstructured":"A. Joshi . 2020. Amazon\u2019s machine learning toolkit: Sagemaker . In Machine Learning and Artificial Intelligence . Springer , 233\u2013243. A. Joshi. 2020. Amazon\u2019s machine learning toolkit: Sagemaker. In Machine Learning and Artificial Intelligence. Springer, 233\u2013243."},{"key":"e_1_3_2_1_54_1","doi-asserted-by":"crossref","unstructured":"S. Kato S. Tokunaga Y. Maruyama S. Maeda M. Hirabayashi Y. Kitsukawa A. Monrroy T. Ando Y. Fujii and T. Azumi. 2018. Autoware on board: Enabling autonomous vehicles with embedded systems. In ICCPS. 287\u2013296. S. Kato S. Tokunaga Y. Maruyama S. Maeda M. Hirabayashi Y. Kitsukawa A. Monrroy T. Ando Y. Fujii and T. Azumi. 2018. Autoware on board: Enabling autonomous vehicles with embedded systems. In ICCPS. 287\u2013296.","DOI":"10.1109\/ICCPS.2018.00035"},{"key":"e_1_3_2_1_55_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.patcog.2017.05.020"},{"key":"e_1_3_2_1_56_1","doi-asserted-by":"publisher","DOI":"10.1109\/TMM.2019.2895511"},{"key":"e_1_3_2_1_57_1","unstructured":"Staffs Keele. 2007. Guidelines for performing systematic literature reviews in software engineering. Technical report ver. 2.3 ebse technical report. ebse. Staffs Keele. 2007. Guidelines for performing systematic literature reviews in software engineering. Technical report ver. 2.3 ebse technical report. ebse."},{"key":"e_1_3_2_1_58_1","doi-asserted-by":"publisher","DOI":"10.1145\/3368089.3417065"},{"key":"e_1_3_2_1_59_1","doi-asserted-by":"crossref","unstructured":"Christian King Lennart Ries Christopher Kober Christoph Wohlfahrt and Eric Sax. 2019. Automated function assessment in driving scenarios. In ICST. 414\u2013419. Christian King Lennart Ries Christopher Kober Christoph Wohlfahrt and Eric Sax. 2019. Automated function assessment in driving scenarios. In ICST. 414\u2013419.","DOI":"10.1109\/ICST.2019.00050"},{"key":"e_1_3_2_1_60_1","doi-asserted-by":"publisher","DOI":"10.1109\/ISSREW.2018.00-20"},{"key":"e_1_3_2_1_61_1","doi-asserted-by":"crossref","unstructured":"F. Kluck M. Zimmermann F. Wotawa and M. Nica. 2019. Genetic Algorithm-Based Test Parameter Optimization for ADAS System Testing. In QRS. IEEE. F. Kluck M. Zimmermann F. Wotawa and M. Nica. 2019. Genetic Algorithm-Based Test Parameter Optimization for ADAS System Testing. In QRS. IEEE.","DOI":"10.1109\/QRS.2019.00058"},{"key":"e_1_3_2_1_62_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE-C.2017.67"},{"key":"e_1_3_2_1_63_1","doi-asserted-by":"publisher","DOI":"10.4271\/2016-01-0128"},{"key":"e_1_3_2_1_64_1","volume-title":"Ailivesim: An extensible virtual environment for training autonomous vehicles. In 2019 IEEE 43rd annual computer software and applications conference (COMPSAC). 1, 479\u2013488.","author":"Leudet J\u00e9r\u00f4me","year":"2019","unstructured":"J\u00e9r\u00f4me Leudet , Fran\u00e7ois Christophe , Tommi Mikkonen , and Tomi M\u00e4nnist\u00f6 . 2019 . Ailivesim: An extensible virtual environment for training autonomous vehicles. In 2019 IEEE 43rd annual computer software and applications conference (COMPSAC). 1, 479\u2013488. J\u00e9r\u00f4me Leudet, Fran\u00e7ois Christophe, Tommi Mikkonen, and Tomi M\u00e4nnist\u00f6. 2019. Ailivesim: An extensible virtual environment for training autonomous vehicles. In 2019 IEEE 43rd annual computer software and applications conference (COMPSAC). 1, 479\u2013488."},{"key":"e_1_3_2_1_65_1","unstructured":"lgsvl. 2020. simulator. https:\/\/bit.ly\/3dBDif1 lgsvl. 2020. simulator. https:\/\/bit.ly\/3dBDif1"},{"key":"e_1_3_2_1_66_1","volume-title":"Zbigniew Kalbarczyk, and Ravishankar Iyer.","author":"Li Guanpeng","year":"2020","unstructured":"Guanpeng Li , Yiran Li , Saurabh Jha , Timothy Tsai , Michael Sullivan , Siva Kumar Sastry Hari , Zbigniew Kalbarczyk, and Ravishankar Iyer. 2020 . AV-FUZZER: Finding safety violations in autonomous driving systems. In ISSRE. 25\u201336. Guanpeng Li, Yiran Li, Saurabh Jha, Timothy Tsai, Michael Sullivan, Siva Kumar Sastry Hari, Zbigniew Kalbarczyk, and Ravishankar Iyer. 2020. AV-FUZZER: Finding safety violations in autonomous driving systems. In ISSRE. 25\u201336."},{"key":"e_1_3_2_1_67_1","volume-title":"Ontology-based test generation for automated and autonomous driving functions. Information and software technology, 117","author":"Li Yihao","year":"2020","unstructured":"Yihao Li , Jianbo Tao , and Franz Wotawa . 2020. Ontology-based test generation for automated and autonomous driving functions. Information and software technology, 117 ( 2020 ), 106200. Yihao Li, Jianbo Tao, and Franz Wotawa. 2020. Ontology-based test generation for automated and autonomous driving functions. Information and software technology, 117 (2020), 106200."},{"key":"e_1_3_2_1_68_1","volume-title":"PEACEPACT: Prioritizing Examples to Accelerate Perturbation-Based Adversary Generation for DNN Classification Testing","author":"Li Z.","year":"2020","unstructured":"Z. Li , L. Zhang , J. Yan , J. Zhang , Z. Zhang , and T. H. Tse . 2020 . PEACEPACT: Prioritizing Examples to Accelerate Perturbation-Based Adversary Generation for DNN Classification Testing . In QRS. IEEE. Z. Li, L. Zhang, J. Yan, J. Zhang, Z. Zhang, and T. H. Tse. 2020. PEACEPACT: Prioritizing Examples to Accelerate Perturbation-Based Adversary Generation for DNN Classification Testing. In QRS. IEEE."},{"key":"e_1_3_2_1_69_1","volume-title":"An Analysis of Testing Scenarios for Automated Driving Systems. In 2021 IEEE International Conference on Software Analysis, Evolution and Reengineering (SANER). 622\u2013629","author":"Liu Siyuan","year":"2021","unstructured":"Siyuan Liu and Luiz Fernando Capretz . 2021 . An Analysis of Testing Scenarios for Automated Driving Systems. In 2021 IEEE International Conference on Software Analysis, Evolution and Reengineering (SANER). 622\u2013629 . Siyuan Liu and Luiz Fernando Capretz. 2021. An Analysis of Testing Scenarios for Automated Driving Systems. In 2021 IEEE International Conference on Software Analysis, Evolution and Reengineering (SANER). 622\u2013629."},{"key":"e_1_3_2_1_70_1","volume-title":"Search-Based Selection and Prioritization of Test Scenarios for Autonomous Driving Systems. In International Symposium on Search Based Software Engineering. 41\u201355","author":"Lu Chengjie","year":"2021","unstructured":"Chengjie Lu , Huihui Zhang , Tao Yue , and Shaukat Ali . 2021 . Search-Based Selection and Prioritization of Test Scenarios for Autonomous Driving Systems. In International Symposium on Search Based Software Engineering. 41\u201355 . Chengjie Lu, Huihui Zhang, Tao Yue, and Shaukat Ali. 2021. Search-Based Selection and Prioritization of Test Scenarios for Autonomous Driving Systems. In International Symposium on Search Based Software Engineering. 41\u201355."},{"key":"e_1_3_2_1_71_1","doi-asserted-by":"publisher","DOI":"10.1109\/ASE51524.2021.9678883"},{"key":"e_1_3_2_1_72_1","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1145\/3417330","article-title":"Test selection for deep learning systems","volume":"30","author":"Ma W.","year":"2021","unstructured":"W. Ma , M. Papadakis , A. Tsakmalis , M. Cordy , and Y. Traon . 2021 . Test selection for deep learning systems . ACM Transactions on Software Engineering and Methodology (TOSEM) , 30 , 2 (2021), 1 \u2013 22 . W. Ma, M. Papadakis, A. Tsakmalis, M. Cordy, and Y. Traon. 2021. Test selection for deep learning systems. ACM Transactions on Software Engineering and Methodology (TOSEM), 30, 2 (2021), 1\u201322.","journal-title":"ACM Transactions on Software Engineering and Methodology (TOSEM)"},{"key":"e_1_3_2_1_73_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSTW.2017.55"},{"key":"e_1_3_2_1_74_1","unstructured":"microsoft. 2021. AirSim. https:\/\/bit.ly\/3qREI8Q microsoft. 2021. AirSim. https:\/\/bit.ly\/3qREI8Q"},{"key":"e_1_3_2_1_75_1","unstructured":"microsoft. 2021. UVoTT. https:\/\/bit.ly\/3rEWxJO microsoft. 2021. UVoTT. https:\/\/bit.ly\/3rEWxJO"},{"key":"e_1_3_2_1_76_1","doi-asserted-by":"publisher","DOI":"10.1145\/1134285.1134307"},{"key":"e_1_3_2_1_77_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.jss.2017.10.031"},{"key":"e_1_3_2_1_78_1","unstructured":"C. Northcutt A. Athalye and J. Mueller. 2021. Pervasive label errors in test sets destabilize machine learning benchmarks. arXiv preprint arXiv:2103.14749. C. Northcutt A. Athalye and J. Mueller. 2021. Pervasive label errors in test sets destabilize machine learning benchmarks. arXiv preprint arXiv:2103.14749."},{"key":"e_1_3_2_1_79_1","unstructured":"California Department of Motor Vehicles. 2019. Autonomous Vehicle Collision Reports - California DMV. https:\/\/bit.ly\/3cPUcGC California Department of Motor Vehicles. 2019. Autonomous Vehicle Collision Reports - California DMV. https:\/\/bit.ly\/3cPUcGC"},{"key":"e_1_3_2_1_80_1","unstructured":"California Department of Motor Vehicles. 2021. Disengagement Report. https:\/\/bit.ly\/2NmXA1c California Department of Motor Vehicles. 2021. Disengagement Report. https:\/\/bit.ly\/2NmXA1c"},{"key":"e_1_3_2_1_81_1","volume-title":"ler","author":"Paolacci G.","year":"2014","unstructured":"G. Paolacci , Ch, and J. ler . 2014 . Inside the Turk : Understanding Mechanical Turk as a participant pool. Current directions in psychological science, 23, 3 (2014), 184\u2013188. G. Paolacci, Ch, and J. ler. 2014. Inside the Turk: Understanding Mechanical Turk as a participant pool. Current directions in psychological science, 23, 3 (2014), 184\u2013188."},{"key":"e_1_3_2_1_82_1","doi-asserted-by":"crossref","unstructured":"Z. Peng J. Yang T. H. Chen and L. Ma. 2020. A first look at the integration of machine learning models in complex autonomous driving systems: a case study on Apollo. In FSE. ACM 1240\u20131250. Z. Peng J. Yang T. H. Chen and L. Ma. 2020. A first look at the integration of machine learning models in complex autonomous driving systems: a case study on Apollo. In FSE. ACM 1240\u20131250.","DOI":"10.1145\/3368089.3417063"},{"key":"e_1_3_2_1_83_1","doi-asserted-by":"crossref","unstructured":"R. Queiroz T. Berger and K. Czarnecki. 2019. GeoScenario: An open DSL for autonomous driving scenario representation. In IV. 287\u2013294. R. Queiroz T. Berger and K. Czarnecki. 2019. GeoScenario: An open DSL for autonomous driving scenario representation. In IV. 287\u2013294.","DOI":"10.1109\/IVS.2019.8814107"},{"key":"e_1_3_2_1_84_1","first-page":"3","article-title":"Data cleaning: Problems and current approaches","volume":"23","author":"Rahm E.","year":"2000","unstructured":"E. Rahm and H. Do . 2000 . Data cleaning: Problems and current approaches . IEEE Data Eng. Bull. , 23 , 4 (2000), 3 \u2013 13 . E. Rahm and H. Do. 2000. Data cleaning: Problems and current approaches. IEEE Data Eng. Bull., 23, 4 (2000), 3\u201313.","journal-title":"IEEE Data Eng. Bull."},{"key":"e_1_3_2_1_85_1","doi-asserted-by":"publisher","DOI":"10.1109\/TKDE.2019.2946162"},{"key":"e_1_3_2_1_86_1","doi-asserted-by":"crossref","unstructured":"Rick Salay Matt Angus and Krzysztof Czarnecki. 2019. A safety analysis method for perceptual components in automated driving. In ISSRE. 24\u201334. Rick Salay Matt Angus and Krzysztof Czarnecki. 2019. A safety analysis method for perceptual components in automated driving. In ISSRE. 24\u201334.","DOI":"10.1109\/ISSRE.2019.00013"},{"key":"e_1_3_2_1_87_1","volume-title":"Labelbox: The best way to create and manage training data. software, LabelBox","author":"Sharma M.","year":"2019","unstructured":"M. Sharma , D. Rasmuson , B. Rieger , and D. Kjelkerud . 2019 . Labelbox: The best way to create and manage training data. software, LabelBox . Inc , https:\/\/bit.ly\/2TBLzYW. M. Sharma, D. Rasmuson, B. Rieger, and D. Kjelkerud. 2019. Labelbox: The best way to create and manage training data. software, LabelBox. Inc, https:\/\/bit.ly\/2TBLzYW."},{"key":"e_1_3_2_1_88_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-1-84800-044-5"},{"key":"e_1_3_2_1_89_1","doi-asserted-by":"publisher","DOI":"10.1145\/3180155.3180236"},{"key":"e_1_3_2_1_90_1","doi-asserted-by":"crossref","unstructured":"P. Sun H. Kretzschmar X. Dotiwalla A. Chouard V. Patnaik P. Tsui J. Guo Y. Zhou Y. Chai and B. Caine. 2020. Scalability in perception for autonomous driving: Waymo open dataset. In CVPR. 2446\u20132454. P. Sun H. Kretzschmar X. Dotiwalla A. Chouard V. Patnaik P. Tsui J. Guo Y. Zhou Y. Chai and B. Caine. 2020. Scalability in perception for autonomous driving: Waymo open dataset. In CVPR. 2446\u20132454.","DOI":"10.1109\/CVPR42600.2020.00252"},{"key":"e_1_3_2_1_91_1","volume-title":"Asia-Pacific Software Engineering Conference. 348\u2013357","author":"Sun X.","unstructured":"X. Sun , T. Zhou , G. Li , J. Hu , H. Yang , and B. Li . 2017. An empirical study on real bugs for machine learning programs . In Asia-Pacific Software Engineering Conference. 348\u2013357 . X. Sun, T. Zhou, G. Li, J. Hu, H. Yang, and B. Li. 2017. An empirical study on real bugs for machine learning programs. In Asia-Pacific Software Engineering Conference. 348\u2013357."},{"key":"e_1_3_2_1_92_1","doi-asserted-by":"publisher","DOI":"10.1109\/ASE51524.2021.9678735"},{"key":"e_1_3_2_1_93_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSTW.2019.00058"},{"key":"e_1_3_2_1_94_1","doi-asserted-by":"crossref","unstructured":"F. Thung S. Wang D. Lo and L. Jiang. 2012. An empirical study of bugs in machine learning systems. In ISSRE. 271\u2013280. F. Thung S. Wang D. Lo and L. Jiang. 2012. An empirical study of bugs in machine learning systems. In ISSRE. 271\u2013280.","DOI":"10.1109\/ISSRE.2012.22"},{"key":"e_1_3_2_1_95_1","doi-asserted-by":"publisher","DOI":"10.1145\/3180155.3180220"},{"key":"e_1_3_2_1_96_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE-Companion52605.2021.00048"},{"key":"e_1_3_2_1_97_1","unstructured":"A. Viera and J. Garrett. 2005. Understanding interobserver agreement: the kappa statistic. Fam med 37 5 (2005) 360\u2013363. A. Viera and J. Garrett. 2005. Understanding interobserver agreement: the kappa statistic. Fam med 37 5 (2005) 360\u2013363."},{"key":"e_1_3_2_1_98_1","doi-asserted-by":"crossref","unstructured":"P. Wang X. Huang X. Cheng D. Zhou Q. Geng and R. Yang. 2019. The apolloscape open dataset for autonomous driving and its application. IEEE transactions on pattern analysis and machine intelligence. P. Wang X. Huang X. Cheng D. Zhou Q. Geng and R. Yang. 2019. The apolloscape open dataset for autonomous driving and its application. IEEE transactions on pattern analysis and machine intelligence.","DOI":"10.1109\/TPAMI.2019.2926463"},{"key":"e_1_3_2_1_99_1","doi-asserted-by":"crossref","unstructured":"Z. Wang H. You J. Chen Y. Zhang X. Dong and W. Zhang. 2021. Prioritizing Test Inputs for Deep Neural Networks via Mutation Analysis. In ICSE. 397\u2013409. Z. Wang H. You J. Chen Y. Zhang X. Dong and W. Zhang. 2021. Prioritizing Test Inputs for Deep Neural Networks via Mutation Analysis. In ICSE. 397\u2013409.","DOI":"10.1109\/ICSE43902.2021.00046"},{"key":"e_1_3_2_1_100_1","doi-asserted-by":"crossref","unstructured":"C. Wohlin P. Runeson M. Hst M. Ohlsson B. Regnell and A. Wessln. 2012. Experimentation in Software Engineering. Springer Publishing Company Incorporated. isbn:3642290434 C. Wohlin P. Runeson M. Hst M. Ohlsson B. Regnell and A. Wessln. 2012. Experimentation in Software Engineering. Springer Publishing Company Incorporated. isbn:3642290434","DOI":"10.1007\/978-3-642-29044-2"},{"key":"e_1_3_2_1_101_1","doi-asserted-by":"publisher","DOI":"10.1109\/ISSREW.2017.46"},{"key":"e_1_3_2_1_102_1","doi-asserted-by":"crossref","unstructured":"B. Wu W. Chen P. Sun W. Liu B. Ghanem and S. Lyu. 2018. Tagging like humans: Diverse and distinct image annotation. In CVPR. 7967\u20137975. B. Wu W. Chen P. Sun W. Liu B. Ghanem and S. Lyu. 2018. Tagging like humans: Diverse and distinct image annotation. In CVPR. 7967\u20137975.","DOI":"10.1109\/CVPR.2018.00831"},{"key":"e_1_3_2_1_103_1","unstructured":"W. Xiang P. Musau A. Wild D. M. Lopez N. Hamilton X. Yang J. Rosenfeld and T. Johnson. 2018. Verification for machine learning autonomy and neural networks survey. arXiv preprint arXiv:1810.01989. W. Xiang P. Musau A. Wild D. M. Lopez N. Hamilton X. Yang J. Rosenfeld and T. Johnson. 2018. Verification for machine learning autonomy and neural networks survey. arXiv preprint arXiv:1810.01989."},{"key":"e_1_3_2_1_104_1","doi-asserted-by":"publisher","DOI":"10.1109\/32.988498"},{"key":"e_1_3_2_1_105_1","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2019.2962027"},{"key":"e_1_3_2_1_106_1","doi-asserted-by":"publisher","DOI":"10.1145\/3238147.3238187"},{"key":"e_1_3_2_1_107_1","doi-asserted-by":"crossref","unstructured":"T. Zhang C. Gao L. Ma M. Lyu and M. Kim. 2019. An empirical study of common challenges in developing deep learning applications. In ISSRE. 104\u2013115. T. Zhang C. Gao L. Ma M. Lyu and M. Kim. 2019. An empirical study of common challenges in developing deep learning applications. In ISSRE. 104\u2013115.","DOI":"10.1109\/ISSRE.2019.00020"},{"key":"e_1_3_2_1_108_1","doi-asserted-by":"publisher","DOI":"10.1109\/QRS-C51114.2020.00048"},{"key":"e_1_3_2_1_109_1","doi-asserted-by":"crossref","unstructured":"Y. Zhang Y. Chen S. Cheung Y. Xiong and L. Zhang. 2018. An empirical study on TensorFlow program bugs. In ISSTA. 129\u2013140. Y. Zhang Y. Chen S. Cheung Y. Xiong and L. Zhang. 2018. An empirical study on TensorFlow program bugs. In ISSTA. 129\u2013140.","DOI":"10.1145\/3213846.3213866"},{"key":"e_1_3_2_1_110_1","doi-asserted-by":"crossref","unstructured":"Xingyu Zhao Valentin Robu David Flynn Kizito Salako and Lorenzo Strigini. 2019. Assessing the safety and reliability of autonomous vehicles from road testing. In ISSRE. 13\u201323. Xingyu Zhao Valentin Robu David Flynn Kizito Salako and Lorenzo Strigini. 2019. Assessing the safety and reliability of autonomous vehicles from road testing. In ISSRE. 13\u201323.","DOI":"10.1109\/ISSRE.2019.00012"},{"key":"e_1_3_2_1_111_1","doi-asserted-by":"publisher","DOI":"10.1145\/3377811.3380422"}],"event":{"name":"ESEC\/FSE '22: 30th ACM Joint European Software Engineering Conference and Symposium on the Foundations of Software Engineering","location":"Singapore Singapore","acronym":"ESEC\/FSE '22","sponsor":["SIGSOFT ACM Special Interest Group on Software Engineering","NUS NUS"]},"container-title":["Proceedings of the 30th ACM Joint European Software Engineering Conference and Symposium on the Foundations of Software Engineering"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3540250.3549111","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3540250.3549111","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T17:51:02Z","timestamp":1750182662000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3540250.3549111"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2022,11,7]]},"references-count":111,"alternative-id":["10.1145\/3540250.3549111","10.1145\/3540250"],"URL":"https:\/\/doi.org\/10.1145\/3540250.3549111","relation":{},"subject":[],"published":{"date-parts":[[2022,11,7]]},"assertion":[{"value":"2022-11-09","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}