{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,21]],"date-time":"2026-07-21T04:17:06Z","timestamp":1784607426876,"version":"3.55.0"},"reference-count":47,"publisher":"Springer Science and Business Media LLC","issue":"2","license":[{"start":{"date-parts":[[2024,2,21]],"date-time":"2024-02-21T00:00:00Z","timestamp":1708473600000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2024,2,21]],"date-time":"2024-02-21T00:00:00Z","timestamp":1708473600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"funder":[{"DOI":"10.13039\/501100000038","name":"NSERC","doi-asserted-by":"crossref","award":["RGPIN-2020-03992"],"award-info":[{"award-number":["RGPIN-2020-03992"]}],"id":[{"id":"10.13039\/501100000038","id-type":"DOI","asserted-by":"crossref"}]}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Empir Software Eng"],"published-print":{"date-parts":[[2024,3]]},"DOI":"10.1007\/s10664-023-10433-5","type":"journal-article","created":{"date-parts":[[2024,2,21]],"date-time":"2024-02-21T10:02:17Z","timestamp":1708509737000},"update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":18,"title":["Evaluating the impact of flaky simulators on testing autonomous driving systems"],"prefix":"10.1007","volume":"29","author":[{"given":"Mohammad Hossein","family":"Amini","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Shervin","family":"Naseri","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-0281-8231","authenticated-orcid":false,"given":"Shiva","family":"Nejati","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2024,2,21]]},"reference":[{"key":"10433_CR1","unstructured":"(2016) Udacity self-driving challenge 2. https:\/\/github.com\/udacity\/self-driving-car\/tree\/master\/challenges\/challenge-2, Accessed 11 Oct 2019"},{"key":"10433_CR2","unstructured":"(2022a) Foundations. https:\/\/carla.readthedocs.io\/en\/latest\/foundations\/. Accessed 15 Nov 2022"},{"key":"10433_CR3","unstructured":"(2022b) Quick start. https:\/\/carla.readthedocs.io\/en\/latest\/start_quickstart\/. Accessed 15 Nov 2022"},{"key":"10433_CR4","unstructured":"(2022) Raquel Urtasun\u2019s tech company develops self-driving vehicle simulator. https:\/\/www.thestar.com\/business\/2022\/02\/09\/raquel-urtasuns-tech-company-develops-self-driving-vehicle-simulator.html. Accessed May 2022"},{"key":"10433_CR5","unstructured":"(2023) BeamNG.tech Website. https:\/\/beamng.tech. Accessed 3 Mar 2023"},{"key":"10433_CR6","unstructured":"(2023) Carla Challenge. https:\/\/carla.readthedocs.io\/en\/latest\/adv_traffic_manager\/, Accessed 1 Feb 2023"},{"key":"10433_CR7","unstructured":"(2023) Github repo for cyber-physical systems testing tool competition. https:\/\/github.com\/sbft-cps-tool-competition\/cps-tool-competition, Accessed 10 Apr 2023"},{"key":"10433_CR8","unstructured":"(2023) Github repo for svl simulator: an autonomous vehicle simulator. https:\/\/github.com\/lgsvl\/simulator, Accessed 10 Apr 2023"},{"key":"10433_CR9","unstructured":"(2023) Github repo for the paper. https:\/\/github.com\/anonoymous9423013\/anonymous_paper\/. Accessed 10 Apr 2023"},{"key":"10433_CR10","unstructured":"(2023) Github repo for transfuser: imitation with transformer-based sensor fusion for autonomous driving. https:\/\/github.com\/autonomousvision\/transfuser. Accessed 10 Apr 2023"},{"key":"10433_CR11","unstructured":"(2023) Online supplementary material for the paper. https:\/\/github.com\/anonoymous9423013\/anonymous_paper\/tree\/main\/supplementary_materials. Accessed 26 Apr 2023"},{"key":"10433_CR12","doi-asserted-by":"crossref","unstructured":"Abdessalem RB, Nejati S, Briand LC, Stifter T (2018) Testing vision-based control systems using learnable evolutionary algorithms. In: 2018 IEEE\/ACM 40th international conference on software engineering (ICSE), IEEE, pp 1016\u20131026","DOI":"10.1145\/3180155.3180160"},{"key":"10433_CR13","doi-asserted-by":"crossref","unstructured":"Afzal A, Katz DS, Le\u00a0Goues C, Timperley CS (2021) Simulation for robotics test automation: developer perspectives. In: 2021 14th IEEE conference on software testing, verification and validation (ICST), pp 263\u2013274","DOI":"10.1109\/ICST49551.2021.00036"},{"key":"10433_CR14","doi-asserted-by":"crossref","unstructured":"Ahlgren J, Bojarczuk K, Drossopoulou S, Dvortsova I, George J, Gucevska N, Harman M, Lomeli M, Lucas SM, Meijer E, et\u00a0al (2021) Facebook\u2019s cyber\u2013cyber and cyber\u2013physical digital twins. In: Evaluation and assessment in software engineering, pp 1\u20139","DOI":"10.1145\/3463274.3463275"},{"key":"10433_CR15","doi-asserted-by":"crossref","unstructured":"Alshammari A, Morris C, Hilton M, Bell J (2021) Flakeflagger: predicting flakiness without rerunning tests. In: 43rd IEEE\/ACM international conference on software engineering: companion proceedings, ICSE Companion 2021, Madrid, Spain, May 25-28, 2021, IEEE, p 187","DOI":"10.1109\/ICSE-Companion52605.2021.00081"},{"key":"10433_CR16","unstructured":"Bell J, Legunsen O, Hilton M, Eloussi L, Yung T, Marinov D (2018) Deflaker: automatically detecting flaky tests. In: 2018 IEEE\/ACM 40th international conference on software engineering (ICSE), pp 433\u2013444"},{"issue":"3","key":"10433_CR17","doi-asserted-by":"publisher","first-page":"71","DOI":"10.1007\/s10664-023-10286-y","volume":"28","author":"C Birchler","year":"2023","unstructured":"Birchler C, Khatiri S, Bosshard B, Gambi A, Panichella S (2023) Machine learning-based test selection for simulation-based testing of self-driving cars software. Empir Softw Eng 28(3):71","journal-title":"Empir Softw Eng"},{"key":"10433_CR18","doi-asserted-by":"crossref","unstructured":"Borg M, Abdessalem RB, Nejati S, Jegeden F, Shin D (2021) Digital twins are not monozygotic - cross-replicating ADAS testing in two industry-grade automotive simulators. In: 14th IEEE conference on software testing, verification and validation, ICST 2021, Porto de Galinhas, Brazil, April 12-16, 2021, IEEE, pp 383\u2013393","DOI":"10.1109\/ICST49551.2021.00050"},{"key":"10433_CR19","volume-title":"Elementary Statistics for the Social Sciences: Study Guide","author":"JA Capon","year":"1991","unstructured":"Capon JA (1991) Elementary Statistics for the Social Sciences: Study Guide. Wadsworth Publishing Company, Belmont, CA, USA"},{"key":"10433_CR20","doi-asserted-by":"publisher","first-page":"321","DOI":"10.1613\/jair.953","volume":"16","author":"NV Chawla","year":"2002","unstructured":"Chawla NV, Bowyer KW, Hall LO, Kegelmeyer WP (2002) Smote: synthetic minority over-sampling technique. J Artif Intell Res 16:321\u2013357","journal-title":"J Artif Intell Res"},{"key":"10433_CR21","unstructured":"Dosovitskiy A, Ros G, Codevilla F, Lopez A, Koltun V (2017) CARLA: an open urban driving simulator. In: Proceedings of the 1st annual conference on robot learning, pp 1\u201316"},{"key":"10433_CR22","doi-asserted-by":"publisher","unstructured":"Dutta S, Shi A, Choudhary R, Zhang Z, Jain A, Misailovic S (2020) Detecting flaky tests in probabilistic and machine learning applications. In: Proceedings of the 29th ACM SIGSOFT international symposium on software testing and analysis, association for computing machinery, New York, USA, ISSTA 2020, pp 211\u2013224, https:\/\/doi.org\/10.1145\/3395363.3397366","DOI":"10.1145\/3395363.3397366"},{"key":"10433_CR23","doi-asserted-by":"crossref","unstructured":"Gog I, Kalra S, Schafhalter P, Wright MA, Gonzalez JE, Stoica I (2021) Pylot: a modular platform for exploring latency-accuracy tradeoffs in autonomous vehicles. In: 2021 IEEE international conference on robotics and automation (ICRA), IEEE, pp 8806\u20138813","DOI":"10.1109\/ICRA48506.2021.9561747"},{"key":"10433_CR24","doi-asserted-by":"crossref","unstructured":"Goutte C, Gaussier E (2005) A probabilistic interpretation of precision, recall and f-score, with implication for evaluation. In: Losada DE, Fern\u00e1ndez-Luna JM (eds) Advances in Information Retrieval. Springer, Berlin, Heidelberg, pp 345\u2013359","DOI":"10.1007\/978-3-540-31865-1_25"},{"key":"10433_CR25","unstructured":"Hagan MT, Demuth HB, Beale M (1997) Neural network design. PWS Publishing Co"},{"key":"10433_CR26","doi-asserted-by":"crossref","unstructured":"Haq FU, Shin D, Nejati S, Briand LC (2020) Comparing offline and online testing of deep neural networks: An autonomous car case study. In: 13th IEEE international conference on software testing, validation and verification, ICST 2020, Porto, Portugal, October 24-28, 2020, IEEE, pp 85\u201395","DOI":"10.1109\/ICST46399.2020.00019"},{"issue":"5","key":"10433_CR27","doi-asserted-by":"publisher","first-page":"90","DOI":"10.1007\/s10664-021-09982-4","volume":"26","author":"FU Haq","year":"2021","unstructured":"Haq FU, Shin D, Nejati S, Briand LC (2021) Can offline testing of deep neural networks replace their online testing? Empir Softw Eng 26(5):90","journal-title":"Empir Softw Eng"},{"key":"10433_CR28","doi-asserted-by":"publisher","unstructured":"Haq FU, Shin D, Briand L (2022) Efficient online testing for dnn-enabled systems using surrogate-assisted and many-objective optimization. In: 2022 IEEE\/ACM 44th international conference on software engineering (ICSE), pp 811\u2013822, https:\/\/doi.org\/10.1145\/3510003.3510188","DOI":"10.1145\/3510003.3510188"},{"key":"10433_CR29","unstructured":"Haq FU, Shin D, Briand LC (2023) Many-objective reinforcement learning for online testing of dnn-enabled systems. In: 45th IEEE\/ACM international conference on software engineering, ICSE 2023, Melbourne, Australia, May 14-20, 2023, IEEE, pp 1814\u20131826"},{"issue":"2","key":"10433_CR30","doi-asserted-by":"publisher","first-page":"226","DOI":"10.1109\/TSE.2009.71","volume":"36","author":"M Harman","year":"2010","unstructured":"Harman M, McMinn P (2010) A theoretical and empirical study of search-based testing: local, global, and hybrid search. IEEE Trans Softw Eng 36(2):226\u2013247. https:\/\/doi.org\/10.1109\/TSE.2009.71","journal-title":"IEEE Trans Softw Eng"},{"key":"10433_CR31","doi-asserted-by":"crossref","unstructured":"Herzig K, Nagappan N (2015) Empirically detecting false test alarms using association rules. In: 2015 IEEE\/ACM 37th IEEE international conference on software engineering, vol\u00a02, pp 39\u201348","DOI":"10.1109\/ICSE.2015.133"},{"key":"10433_CR32","unstructured":"Luke S (2013) Essentials of Metaheuristics, 2nd edn. Lulu, available for free at http:\/\/cs.gmu.edu\/~sean\/book\/metaheuristics\/"},{"key":"10433_CR33","doi-asserted-by":"crossref","unstructured":"Luo Q, Hariri F, Eloussi L, Marinov D (2014) An empirical analysis of flaky tests. In: Cheung S, Orso A, Storey MD (eds) Proceedings of the 22nd ACM SIGSOFT international symposium on foundations of software engineering, (FSE-22), Hong Kong, China, November 16 - 22, 2014, ACM, pp 643\u2013653","DOI":"10.1145\/2635868.2635920"},{"key":"10433_CR34","doi-asserted-by":"crossref","unstructured":"Matinnejad R, Nejati S, Briand LC (2017) Automated testing of hybrid simulink\/stateflow controllers: industrial case studies. In: Bodden E, Sch\u00e4fer W, van Deursen A, Zisman A (eds) Proceedings of the 2017 11th joint meeting on foundations of software engineering, ESEC\/FSE 2017, Paderborn, Germany, September 4-8, 2017, ACM, pp 938\u2013943","DOI":"10.1145\/3106237.3117770"},{"key":"10433_CR35","unstructured":"Micco J (2018) Advances in continuous integration testing at google"},{"key":"10433_CR36","doi-asserted-by":"crossref","unstructured":"Nguyen V, Huber S, Gambi A (2021) Salvo: automated generation of diversified tests for self-driving cars from existing maps. In: 2021 IEEE international conference on artificial intelligence testing (AITest), pp 128\u2013135","DOI":"10.1109\/AITEST52744.2021.00033"},{"key":"10433_CR37","doi-asserted-by":"publisher","unstructured":"Parry O, Kapfhammer GM, Hilton M, McMinn P (2021) A survey of flaky tests. ACM Trans Softw Eng Methodol 31(1), https:\/\/doi.org\/10.1145\/3476105","DOI":"10.1145\/3476105"},{"key":"10433_CR38","doi-asserted-by":"crossref","unstructured":"Paydar S, Azamnouri A (2019) An experimental study on flakiness and fragility of randoop regression test suites. In: Fundamentals of software engineering","DOI":"10.1007\/978-3-030-31517-7_8"},{"key":"10433_CR39","doi-asserted-by":"crossref","unstructured":"Riccio V, Tonella P (2023) When and why test generators for deep learning produce invalid inputs: an empirical study. In: 45th IEEE\/ACM international conference on software engineering, ICSE 2023, Melbourne, Australia, May 14-20, 2023, IEEE, pp 1161\u20131173","DOI":"10.1109\/ICSE48619.2023.00104"},{"key":"10433_CR40","doi-asserted-by":"crossref","unstructured":"Samak CV, Samak TV, Kandhasamy S (2020) Control strategies for autonomous vehicles. arXiv:2011.08729","DOI":"10.1201\/9781003048381-3"},{"key":"10433_CR41","doi-asserted-by":"crossref","unstructured":"Shi A, Gyori A, Legunsen O, Marinov D (2016) Detecting assumptions on deterministic implementations of non-deterministic specifications. In: 2016 IEEE international conference on software testing, verification and validation (ICST), pp 80\u201390","DOI":"10.1109\/ICST.2016.40"},{"key":"10433_CR42","doi-asserted-by":"publisher","unstructured":"Ulbrich S, Menzel T, Reschka A, Schuldt F, Maurer M (2015) Defining and substantiating the terms scene, situation, and scenario for automated driving. In: 2015 IEEE 18th international conference on intelligent transportation systems, pp 982\u2013988, https:\/\/doi.org\/10.1109\/ITSC.2015.164","DOI":"10.1109\/ITSC.2015.164"},{"issue":"2","key":"10433_CR43","first-page":"101","volume":"25","author":"A Vargha","year":"2000","unstructured":"Vargha A, Delaney HD (2000) A critique and improvement of the cl common language effect size statistics of mcgraw and wong. J Educ Behav Stat 25(2):101\u2013132","journal-title":"J Educ Behav Stat"},{"key":"10433_CR44","volume-title":"Data mining: practical machine learning tools and techniques","author":"IH Witten","year":"2011","unstructured":"Witten IH, Frank E, Hall MA (2011) Data mining: practical machine learning tools and techniques, 3rd edn. Morgan Kaufmann Series in Data Management Systems, Morgan Kaufmann, Amsterdam","edition":"3"},{"key":"10433_CR45","unstructured":"Zeller A, Gopinath R, B\u00f6hme M, Fraser G, Holler C (2023) Code coverage. In: The Fuzzing Book, CISPA Helmholtz Center for Information Security, https:\/\/www.fuzzingbook.org\/html\/Coverage.html, retrieved 2023-01-07 13:54:15+01:00"},{"issue":"4","key":"10433_CR46","doi-asserted-by":"publisher","first-page":"1860","DOI":"10.1109\/TSE.2022.3195640","volume":"49","author":"Z Zhong","year":"2023","unstructured":"Zhong Z, Kaiser G, Ray B (2023) Neural network guided evolutionary fuzzing for finding traffic violations of autonomous vehicles. IEEE Trans Softw Eng 49(4):1860\u20131875. https:\/\/doi.org\/10.1109\/TSE.2022.3195640","journal-title":"IEEE Trans Softw Eng"},{"key":"10433_CR47","unstructured":"Zohdinasab T, Riccio V, Gambi A, Tonella P (2023) Deephyperion: Exploring the feature space of deep learning-based systems through illumination search. In: Engels G, Hebig R, Tichy M (eds) Software Engineering 2023, Fachtagung des GI-Fachbereichs Softwaretechnik, 20.-24. Februar 2023, Paderborn, Gesellschaft f\u00fcr Informatik e.V., LNI, vol P-332, pp 131\u2013132"}],"container-title":["Empirical Software Engineering"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10664-023-10433-5.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s10664-023-10433-5\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10664-023-10433-5.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,3,23]],"date-time":"2024-03-23T02:20:34Z","timestamp":1711160434000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s10664-023-10433-5"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,2,21]]},"references-count":47,"journal-issue":{"issue":"2","published-print":{"date-parts":[[2024,3]]}},"alternative-id":["10433"],"URL":"https:\/\/doi.org\/10.1007\/s10664-023-10433-5","relation":{},"ISSN":["1382-3256","1573-7616"],"issn-type":[{"value":"1382-3256","type":"print"},{"value":"1573-7616","type":"electronic"}],"subject":[],"published":{"date-parts":[[2024,2,21]]},"assertion":[{"value":"30 November 2023","order":1,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"21 February 2024","order":2,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Declarations"}},{"value":"The authors declared that they have no conflict of interest.","order":2,"name":"Ethics","group":{"name":"EthicsHeading","label":"COI Statement"}}],"article-number":"47"}}