{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,1,16]],"date-time":"2026-01-16T00:34:40Z","timestamp":1768523680412,"version":"3.49.0"},"reference-count":42,"publisher":"Springer Science and Business Media LLC","issue":"1","license":[{"start":{"date-parts":[[2024,11,5]],"date-time":"2024-11-05T00:00:00Z","timestamp":1730764800000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by-nc-nd\/4.0"},{"start":{"date-parts":[[2024,11,5]],"date-time":"2024-11-05T00:00:00Z","timestamp":1730764800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by-nc-nd\/4.0"}],"funder":[{"DOI":"10.13039\/100010663","name":"H2020 European Research Council","doi-asserted-by":"publisher","award":["787703"],"award-info":[{"award-number":["787703"]}],"id":[{"id":"10.13039\/100010663","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100010663","name":"H2020 European Research Council","doi-asserted-by":"publisher","award":["871342"],"award-info":[{"award-number":["871342"]}],"id":[{"id":"10.13039\/100010663","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Empir Software Eng"],"published-print":{"date-parts":[[2025,1]]},"DOI":"10.1007\/s10664-024-10562-5","type":"journal-article","created":{"date-parts":[[2024,11,5]],"date-time":"2024-11-05T06:05:57Z","timestamp":1730786757000},"update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":3,"title":["Reinforcement learning for online testing of autonomous driving systems: a replication and extension study"],"prefix":"10.1007","volume":"30","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-3767-4036","authenticated-orcid":false,"given":"Luca","family":"Giamattei","sequence":"first","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0002-7825-3409","authenticated-orcid":false,"given":"Matteo","family":"Biagiola","sequence":"additional","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0003-2449-1724","authenticated-orcid":false,"given":"Roberto","family":"Pietrantuono","sequence":"additional","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0002-8747-3446","authenticated-orcid":false,"given":"Stefano","family":"Russo","sequence":"additional","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0003-3088-0339","authenticated-orcid":false,"given":"Paolo","family":"Tonella","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2024,11,5]]},"reference":[{"key":"10562_CR1","unstructured":"Apollo B (2017) https:\/\/github.com\/ApolloAuto\/apollo"},{"key":"10562_CR2","doi-asserted-by":"crossref","unstructured":"Bellemare M, Veness J, Bowling M (2012) Investigating contingency awareness using atari 2600 games. In: Proceedings of the AAAI Conference on artificial intelligence, 26(1):864\u2013871. AAAI Press, Palo Alto, CA, USA","DOI":"10.1609\/aaai.v26i1.8321"},{"key":"10562_CR3","doi-asserted-by":"publisher","unstructured":"Abdessalem RB, Nejati S, Briand LC, Stifter T (2018) Testing vision-based control systems using learnable evolutionary algorithms. In: 2018 IEEE\/ACM 40th International Conference on Software Engineering (ICSE), pp 1016\u20131026. IEEE. https:\/\/doi.org\/10.1145\/3180155.3180160","DOI":"10.1145\/3180155.3180160"},{"key":"10562_CR4","doi-asserted-by":"crossref","unstructured":"Cal\u00f2 A, Arcaini P, Ali S, Hauer F, Ishikawa F (2020) Generating avoidable collision scenarios for testing autonomous driving systems. In: IEEE 13th International conference on software testing, validation and verification (ICST), IEEE, pp 375\u2013386","DOI":"10.1109\/ICST46399.2020.00045"},{"key":"10562_CR5","doi-asserted-by":"publisher","unstructured":"Corso A, Du P, Driggs-Campbell K, Kochenderfer MJ (2019) Adaptive stress testing with reward augmentation for autonomous vehicle validation. In: Intelligent transportation systems conference (ITSC), IEEE, pp 163\u2013168. https:\/\/doi.org\/10.1109\/ITSC.2019.8917242","DOI":"10.1109\/ITSC.2019.8917242"},{"issue":"3","key":"10562_CR6","doi-asserted-by":"publisher","first-page":"501","DOI":"10.1007\/s10664-012-9227-7","volume":"19","author":"FQB da Silva","year":"2014","unstructured":"da Silva FQB, Suassuna M, Fran\u00e7a ACC, Grubb AM, Gouveia TB, Monteiro CVF, dos Santos IE (2014) Replication of empirical studies in software engineering research: a systematic mapping study. Empirical Softw Eng 19(3):501\u2013557. https:\/\/doi.org\/10.1007\/s10664-012-9227-7","journal-title":"Empirical Softw Eng"},{"issue":"1","key":"10562_CR7","doi-asserted-by":"publisher","first-page":"3","DOI":"10.1007\/s10664-022-10243-1","volume":"28","author":"D Dell\u2019Anna","year":"2022","unstructured":"Dell\u2019Anna D, Aydemir FB, Dalpiaz F (2022) Evaluating classifiers in se research: the ecser pipeline and two replication studies. Empirical Softw Eng 28(1):3. https:\/\/doi.org\/10.1007\/s10664-022-10243-1","journal-title":"Empirical Softw Eng"},{"key":"10562_CR8","unstructured":"Dosovitskiy A, Ros G, Codevilla F, Lopez A, Koltun V (2017) CARLA: an open urban driving simulator. In: 1st Annual conference on robot learning, proceedings of machine learning research, vol\u00a078, pp 1\u201316. JMLR, Cambridge, MA, USA"},{"issue":"3","key":"10562_CR9","doi-asserted-by":"publisher","first-page":"241","DOI":"10.1080\/00401706.1964.10490181","volume":"6","author":"OJ Dunn","year":"1964","unstructured":"Dunn OJ (1964) Multiple comparisons using rank sums. Technometrics 6(3):241\u2013252. https:\/\/doi.org\/10.1080\/00401706.1964.10490181","journal-title":"Technometrics"},{"issue":"9","key":"10562_CR10","doi-asserted-by":"publisher","first-page":"e0184952","DOI":"10.1371\/journal.pone.0184952","volume":"12","author":"FM Favar\u00f2","year":"2017","unstructured":"Favar\u00f2 FM, Nader N, Eurich SO, Tripp M, Varadaraju N (2017) Examining accident reports involving autonomous vehicles in California. PLoS one 12(9):e0184952","journal-title":"PLoS one"},{"issue":"200","key":"10562_CR11","doi-asserted-by":"publisher","first-page":"675","DOI":"10.1080\/01621459.1937.10503522","volume":"32","author":"M Friedman","year":"1937","unstructured":"Friedman M (1937) The use of ranks to avoid the assumption of normality implicit in the analysis of variance. J Am Stat Assoc 32(200):675\u2013701. https:\/\/doi.org\/10.1080\/01621459.1937.10503522","journal-title":"J Am Stat Assoc"},{"key":"10562_CR12","doi-asserted-by":"publisher","unstructured":"Gambi A, M\u00fcller M, Fraser G (2019) Automatically testing self-driving cars with search-based procedural content generation. In: Proceedings of the 28th ACM SIGSOFT international symposium on software testing and analysis (ISSTA), ACM, pp 318\u2013328. https:\/\/doi.org\/10.1145\/3293882.3330566","DOI":"10.1145\/3293882.3330566"},{"key":"10562_CR13","doi-asserted-by":"publisher","unstructured":"Haq FU, Shin D, Briand, L (2022) Efficient online testing for DNN-enabled systems using surrogate-assisted and many-objective optimization. In: 44th International conference on software engineering (ICSE), ACM, pp 811\u2013822. https:\/\/doi.org\/10.1145\/3510003.3510188","DOI":"10.1145\/3510003.3510188"},{"key":"10562_CR14","doi-asserted-by":"publisher","unstructured":"Haq FU, Shin D, Briand L (2023) Many-objective reinforcement learning for online testing of DNN-enabled systems. In: IEEE\/ACM 45th International conference on software engineering (ICSE), IEEE, pp 1814\u20131826. https:\/\/doi.org\/10.1109\/ICSE48619.2023.00155","DOI":"10.1109\/ICSE48619.2023.00155"},{"issue":"1","key":"10562_CR15","doi-asserted-by":"publisher","first-page":"26","DOI":"10.1007\/s10458-022-09552-y","volume":"36","author":"CF Hayes","year":"2022","unstructured":"Hayes CF, R\u0103dulescu R, Bargiacchi E, K\u00e4llstr\u00f6m J, Macfarlane M, Reymond M, Verstraeten T, Zintgraf LM, Dazeley R, Heintz F, Howley E, Irissappane AA, Mannion P, Now\u00e9 A, Ramos G, Restelli M, Vamplew P, Roijers DM (2022) A practical guide to multi-objective reinforcement learning and planning. Autonomous Agents Multi-Agent Syst 36(1):26. https:\/\/doi.org\/10.1007\/s10458-022-09552-y","journal-title":"Autonomous Agents Multi-Agent Syst"},{"key":"10562_CR16","doi-asserted-by":"publisher","unstructured":"Klischat M, Althoff M (2019) Generating critical test scenarios for automated vehicles with evolutionary algorithms. In: IEEE Intelligent vehicles symposium (IV), IEEE, pp 2352\u20132358. https:\/\/doi.org\/10.1109\/IVS.2019.8814230","DOI":"10.1109\/IVS.2019.8814230"},{"key":"10562_CR17","doi-asserted-by":"publisher","unstructured":"Koren M, Alsaif S, Lee R, Kochenderfer MJ (2018) Adaptive stress testing for autonomous vehicles. In: Intelligent vehicles symposium, IEEE, pp 1\u20137. https:\/\/doi.org\/10.1109\/IVS.2018.8500400","DOI":"10.1109\/IVS.2018.8500400"},{"key":"10562_CR18","unstructured":"Leaderboard CAD (2020) CARLA leaderboard. https:\/\/leaderboard.carla.org\/leaderboard\/. Accessed: October 7, 2024"},{"key":"10562_CR19","doi-asserted-by":"publisher","unstructured":"Lee R, Kochenderfer MJ, Mengshoel OJ, Brat GP, Owen MP (2015) Adaptive stress testing of airborne collision avoidance systems. In: 34th Digital avionics systems conference, IEEE\/AIAA, pp 6C2\u20131\u20136C2\u201313. https:\/\/doi.org\/10.1109\/DASC.2015.7311450","DOI":"10.1109\/DASC.2015.7311450"},{"key":"10562_CR20","unstructured":"Leurent E (2018) An environment for autonomous driving decision-making. https:\/\/github.com\/eleurent\/highway-env"},{"key":"10562_CR21","unstructured":"Leurent E (2018) A survey of state-action representations for autonomous driving. https:\/\/hal.science\/hal-01908175"},{"key":"10562_CR22","doi-asserted-by":"publisher","unstructured":"Li G, Li Y, Jha S, Tsai T, Sullivan M, Hari SKS, Kalbarczyk Z, Iyer R (2020) Av-fuzzer: Finding safety violations in autonomous driving systems. In: IEEE 31st International symposium on software reliability engineering (ISSRE), IEEE, pp 25\u201336. https:\/\/doi.org\/10.1109\/ISSRE5003.2020.00012","DOI":"10.1109\/ISSRE5003.2020.00012"},{"key":"10562_CR23","doi-asserted-by":"publisher","first-page":"217","DOI":"10.1080\/00031305.1993.10475983","volume":"47","author":"RM Lindsay","year":"1993","unstructured":"Lindsay RM, Ehrenberg ASC (1993) The design of replicated studies. Am Stat 47:217\u2013228. https:\/\/doi.org\/10.1080\/00031305.1993.10475983","journal-title":"Am Stat"},{"issue":"1","key":"10562_CR24","doi-asserted-by":"publisher","first-page":"384","DOI":"10.1109\/TSE.2022.3150788","volume":"49","author":"C Lu","year":"2023","unstructured":"Lu C, Shi Y, Zhang H, Zhang M, Wang T, Yue T, Ali S (2023) Learning configurations of operating environment of autonomous vehicles to maximize their collisions. IEEE Trans Soft Eng 49(1):384\u2013402. https:\/\/doi.org\/10.1109\/TSE.2022.3150788","journal-title":"IEEE Trans Soft Eng"},{"issue":"3","key":"10562_CR25","doi-asserted-by":"publisher","first-page":"65","DOI":"10.1007\/s10664-022-10117-6","volume":"27","author":"M Maes-Bermejo","year":"2022","unstructured":"Maes-Bermejo M, Gallego M, Gort\u00e1zar F, Robles G, Gonzalez-Barahona JM (2022) Revisiting the building of past snapshots \u2013 a replication and reproduction study. Empirical Softw Eng 27(3):65. https:\/\/doi.org\/10.1007\/s10664-022-10117-6","journal-title":"Empirical Softw Eng"},{"key":"10562_CR26","unstructured":"Majumdar R, Mathur AS, Pirron M, Stegner L, Zufferey, D (2019) Paracosm: a language and tool for testing autonomous driving systems. arXiv:1902.01084"},{"issue":"7540","key":"10562_CR27","doi-asserted-by":"publisher","first-page":"529","DOI":"10.1038\/nature14236","volume":"518","author":"V Mnih","year":"2015","unstructured":"Mnih V, Kavukcuoglu K, Silver D, Rusu AA, Veness J, Bellemare MG, Graves A, Riedmiller M, Fidjeland AK, Ostrovski G et al (2015) Human-level control through deep reinforcement learning. Nature 518(7540):529\u2013533","journal-title":"Nature"},{"issue":"4","key":"10562_CR28","doi-asserted-by":"publisher","first-page":"77","DOI":"10.1007\/s10664-021-09960-w","volume":"26","author":"N Novielli","year":"2021","unstructured":"Novielli N, Calefato F, Lanubile F, Serebrenik A (2021) Assessment of off-the-shelf SE-specific sentiment analysis tools: an extended replication study. Empirical Softw Eng 26(4):77. https:\/\/doi.org\/10.1007\/s10664-021-09960-w","journal-title":"Empirical Softw Eng"},{"key":"10562_CR29","doi-asserted-by":"publisher","unstructured":"Prakash, A., Chitta, K., Geiger, A (2021) Multi-modal fusion transformer for end-to-end autonomous driving. In: IEEE\/CVF Conference on computer vision and pattern recognition (CVPR), IEEE, pp 7073\u20137083. https:\/\/doi.org\/10.1109\/CVPR46437.2021.00700","DOI":"10.1109\/CVPR46437.2021.00700"},{"key":"10562_CR30","doi-asserted-by":"publisher","unstructured":"Riccio V, Tonella P (2020) Model-based exploration of the frontier of behaviours for deep learning system testing. In: Proceedings of the 28th ACM joint meeting on european software engineering conference and symposium on the foundations of software engineering (ESEC\/FSE), ACM, pp 876\u2013888. https:\/\/doi.org\/10.1145\/3368089.3409730","DOI":"10.1145\/3368089.3409730"},{"key":"10562_CR31","doi-asserted-by":"publisher","unstructured":"Rong G, Shin BH, Tabatabaee H, Lu Q, Lemke S, Mo\u017eeiko M, Boise E, Uhm G, Gerow M, Mehta S, Agafonov E, Kim TH, Sterner E, Ushiroda K, Reyes M, Zelenkovsky D, Kim S (2020) Lgsvl simulator: a high fidelity simulator for autonomous driving. In: 23rd International conference on intelligent transportation systems (ITSC), IEEE, pp 1\u20136. https:\/\/doi.org\/10.1109\/ITSC45102.2020.9294422","DOI":"10.1109\/ITSC45102.2020.9294422"},{"key":"10562_CR32","doi-asserted-by":"publisher","unstructured":"Sharif A, Marijan D (2022) Adversarial deep reinforcement learning for improving the robustness of multi-agent autonomous driving policies. In: 29th Asia-Pacific software engineering conference (APSEC), IEEE, pp 61\u201370. (2022). https:\/\/doi.org\/10.1109\/APSEC57359.2022.00018","DOI":"10.1109\/APSEC57359.2022.00018"},{"issue":"2","key":"10562_CR33","doi-asserted-by":"publisher","first-page":"211","DOI":"10.1007\/s10664-008-9060-1","volume":"13","author":"FJ Shull","year":"2008","unstructured":"Shull FJ, Carver JC, Vegas S, Juristo N (2008) The role of replications in empirical software engineering. Empirical Softw Eng 13(2):211\u2013218. https:\/\/doi.org\/10.1007\/s10664-008-9060-1","journal-title":"Empirical Softw Eng"},{"issue":"3","key":"10562_CR34","doi-asserted-by":"publisher","first-page":"73","DOI":"10.1007\/s10664-023-10306-x","volume":"28","author":"A Stocco","year":"2023","unstructured":"Stocco A, Pulfer B, Tonella P (2023) Model vs system level testing of autonomous driving systems: a replication and extension study. Empirical Softw Eng 28(3):73. https:\/\/doi.org\/10.1007\/s10664-023-10306-x","journal-title":"Empirical Softw Eng"},{"key":"10562_CR35","volume-title":"Barto AG (2018) Reinforcement Learning: An Introduction","author":"RS Sutton","year":"2018","unstructured":"Sutton RS (2018) Barto AG (2018) Reinforcement Learning: An Introduction. A Bradford Book, Cambridge, MA, USA"},{"issue":"5","key":"10562_CR36","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1145\/3579642","volume":"32","author":"S Tang","year":"2023","unstructured":"Tang S, Zhang Z, Zhang Y, Zhou J, Guo Y, Liu S, Guo S, Xue Y et al (2023) A survey on automated driving system testing: landscapes and trends. ACM Trans Softw Eng Methodol 32(5):1\u201362","journal-title":"ACM Trans Softw Eng Methodol"},{"key":"10562_CR37","unstructured":"US\u00a0Department\u00a0of Transportation NHTSA (2022) Summary report: standing general order on crash reporting for automated driving systems. https:\/\/www.nhtsa.gov\/sites\/nhtsa.gov\/files\/2022-06\/ADS-SGO-Report-June-2022.pdf"},{"key":"10562_CR38","doi-asserted-by":"crossref","unstructured":"Tuncali CE, Fainekos G, Ito H, Kapinski J (2018) Simulation-based adversarial test generation for autonomous vehicles with machine learning components. arXiv:1804.06760","DOI":"10.1109\/IVS.2018.8500421"},{"issue":"4","key":"10562_CR39","doi-asserted-by":"publisher","first-page":"58","DOI":"10.1007\/s10664-021-09947-7","volume":"26","author":"A Vescan","year":"2021","unstructured":"Vescan A, Pintea A, Linsbauer L, Egyed A (2021) Genetic programming for feature model synthesis: a replication study. Empirical Softw Eng 26(4):58. https:\/\/doi.org\/10.1007\/s10664-021-09947-7","journal-title":"Empirical Softw Eng"},{"key":"10562_CR40","unstructured":"Watkins CJCH (1989) Learning from delayed rewards. Ph.D. thesis, King\u2019s College, Cambridge, UK (1989)"},{"key":"10562_CR41","doi-asserted-by":"publisher","unstructured":"Wilcoxon F (1992) Individual Comparisons by Ranking Methods, pp 196\u2013202. Springer New York (1992). https:\/\/doi.org\/10.1007\/978-1-4612-4380-9_16","DOI":"10.1007\/978-1-4612-4380-9_16"},{"key":"10562_CR42","doi-asserted-by":"publisher","unstructured":"Zohdinasab T, Riccio V, Gambi A, Tonella P (2021) DeepHyperion: exploring the feature space of deep learning-based systems through illumination search. In: Proceedings of the 30th ACM SIGSOFT international symposium on software testing and analysis (ISSTA), ACM, pp 79\u201390.https:\/\/doi.org\/10.1145\/3460319.3464811","DOI":"10.1145\/3460319.3464811"}],"container-title":["Empirical Software Engineering"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10664-024-10562-5.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s10664-024-10562-5\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10664-024-10562-5.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,2,22]],"date-time":"2025-02-22T14:09:54Z","timestamp":1740233394000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s10664-024-10562-5"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,11,5]]},"references-count":42,"journal-issue":{"issue":"1","published-print":{"date-parts":[[2025,1]]}},"alternative-id":["10562"],"URL":"https:\/\/doi.org\/10.1007\/s10664-024-10562-5","relation":{},"ISSN":["1382-3256","1573-7616"],"issn-type":[{"value":"1382-3256","type":"print"},{"value":"1573-7616","type":"electronic"}],"subject":[],"published":{"date-parts":[[2024,11,5]]},"assertion":[{"value":"4 October 2024","order":1,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"5 November 2024","order":2,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}}],"article-number":"19"}}