{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,2,7]],"date-time":"2026-02-07T09:51:26Z","timestamp":1770457886635,"version":"3.49.0"},"publisher-location":"Singapore","reference-count":162,"publisher":"Springer Nature Singapore","isbn-type":[{"value":"9789819517855","type":"print"},{"value":"9789819517862","type":"electronic"}],"license":[{"start":{"date-parts":[[2026,1,1]],"date-time":"2026-01-01T00:00:00Z","timestamp":1767225600000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2026,1,1]],"date-time":"2026-01-01T00:00:00Z","timestamp":1767225600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2026]]},"DOI":"10.1007\/978-981-95-1786-2_4","type":"book-chapter","created":{"date-parts":[[2026,2,6]],"date-time":"2026-02-06T16:13:05Z","timestamp":1770394385000},"page":"45-81","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":0,"title":["DevOps Testing for Cyber-Physical Systems"],"prefix":"10.1007","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-1412-1546","authenticated-orcid":false,"given":"Mattia","family":"Fazzini","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-0132-6497","authenticated-orcid":false,"given":"Alessio","family":"Gambi","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-6229-8231","authenticated-orcid":false,"given":"Vincenzo","family":"Riccio","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-7395-3588","authenticated-orcid":false,"given":"Annibale","family":"Panichella","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-4212-7029","authenticated-orcid":false,"given":"Stefan","family":"Klikovits","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2026,2,7]]},"reference":[{"key":"4_CR1","unstructured":"IEEE 2888.1-2023 IEEE Standard for Specification of Sensor Interface for Cyber and Physical Worlds (2023)"},{"key":"4_CR2","doi-asserted-by":"crossref","unstructured":"S.\u00a0Abbaspour\u00a0Asadollah, R.\u00a0Inam, H.\u00a0Hansson, A survey on testing for cyber physical system, in Proceedings of IFIP WG 6.1 International Conference on Testing Software and Systems, vol. 9447 (Springer, 2015), pp. 194\u2013207","DOI":"10.1007\/978-3-319-25945-1_12"},{"key":"4_CR3","doi-asserted-by":"crossref","unstructured":"R.B. Abdessalem, S.\u00a0Nejati, L.C. Briand, T.\u00a0Stifter, Testing vision-based control systems using learnable evolutionary algorithms, in International Conference on Software Engineering (ACM, 2018), pp. 1016\u20131026","DOI":"10.1145\/3180155.3180160"},{"key":"4_CR4","doi-asserted-by":"crossref","unstructured":"R.B. Abdessalem, A. Panichella, S. Nejati, L.C. Briand, T. Stifter, Testing autonomous cars for feature interaction failures using many-objective search, in Proceedings of the 33rd ACM\/IEEE International Conference on Automated Software Engineering, ASE 2018, Montpellier, France, September 3-7, 2018. ed. by M. Huchard, C. K\u00e4stner, G. Fraser (ACM, 2018), pp. 143\u2013154","DOI":"10.1145\/3238147.3238192"},{"key":"4_CR5","doi-asserted-by":"crossref","unstructured":"H.B. Adeyemo, R.\u00a0Bahsoon, P.\u00a0Ti\u00f1o, Surrogate-based digital twin for predictive fault modelling and testing of cyber physical systems, in 2022 IEEE\/ACM International Conference on Big Data Computing, Applications and Technologies (BDCAT) (IEEE, 2022), pp. 166\u2013169","DOI":"10.1109\/BDCAT56447.2022.00028"},{"key":"4_CR6","unstructured":"D.\u00a0Albright, P.\u00a0Brannan, C.\u00a0Walrond, Did Stuxnet take out 1,000 centrifuges at the Natanz enrichment plant? (Institute for Science and International Security, 2010)"},{"issue":"1","key":"4_CR7","doi-asserted-by":"publisher","first-page":"193","DOI":"10.1007\/s11219-023-09639-z","volume":"32","author":"I Aldalur","year":"2024","unstructured":"I. Aldalur, A. Arrieta, A. Agirre, G. Sagardui, M. Arratibel, A microservice-based framework for multi-level testing of cyber-physical systems. Softw. Qual. J. 32(1), 193\u2013223 (2024)","journal-title":"Softw. Qual. J."},{"issue":"2","key":"4_CR8","doi-asserted-by":"publisher","first-page":"47","DOI":"10.1007\/s10664-023-10433-5","volume":"29","author":"MH Amini","year":"2024","unstructured":"M.H. Amini, S. Naseri, S. Nejati, Evaluating the impact of flaky simulators on testing autonomous driving systems. Empir. Softw. Eng. 29(2), 47 (2024)","journal-title":"Empir. Softw. Eng."},{"key":"4_CR9","doi-asserted-by":"crossref","unstructured":"P.\u00a0Ammann, J.\u00a0Offutt, Introduction to Software Testing (Cambridge University Press, 2017)","DOI":"10.1017\/9781316771273"},{"key":"4_CR10","doi-asserted-by":"crossref","unstructured":"P.\u00a0Arcaini, A.\u00a0Cetinkaya, Detour at the icst 2025 tool competition\u2014self-driving car testing track, in 2025 IEEE Conference on Software Testing, Verification and Validation (ICST) (2015), pp. 805\u2013806","DOI":"10.1109\/ICST62969.2025.10988989"},{"key":"4_CR11","doi-asserted-by":"crossref","unstructured":"A.\u00a0Arrieta, J.\u00a0Ayerdi, M.\u00a0Illarramendi, A.\u00a0Agirre, G.\u00a0Sagardui, M.\u00a0Arratibel, Using machine learning to build test oracles: an industrial case study on elevators dispatching algorithms, in 2021 IEEE\/ACM International Conference on Automation of Software Test (AST) (IEEE, 2021), pp. 30\u201339","DOI":"10.1109\/AST52587.2021.00012"},{"key":"4_CR12","doi-asserted-by":"crossref","unstructured":"A.\u00a0Arrieta, M.\u00a0Otaegi, L.\u00a0Han, G.\u00a0Sagardui, S.\u00a0Ali, M.\u00a0Arratibel, Automating test oracle generation in devops for industrial elevators, in 2022 IEEE International Conference on Software Analysis, Evolution and Reengineering (SANER) (IEEE, 2022), pp. 284\u2013288","DOI":"10.1109\/SANER53432.2022.00044"},{"key":"4_CR13","doi-asserted-by":"crossref","unstructured":"P.\u00a0Aryan, S.\u00a0Khatiri, Nn-sdctest at the icst 2025 tool competition-self-driving car testing track, in 2025 IEEE Conference on Software Testing, Verification and Validation (ICST) (IEEE, 2025), pp. 813\u2013814","DOI":"10.1109\/ICST62969.2025.10988985"},{"key":"4_CR14","doi-asserted-by":"crossref","unstructured":"J.\u00a0Ayerdi, S.\u00a0Segura, A.\u00a0Arrieta, G.\u00a0Sagardui, M.\u00a0Arratibel, Qos-aware metamorphic testing: An elevation case study, in 2020 IEEE 31st International Symposium on Software Reliability Engineering (ISSRE) (IEEE, 2020), pp. 104\u2013114","DOI":"10.1109\/ISSRE5003.2020.00019"},{"key":"4_CR15","doi-asserted-by":"crossref","unstructured":"M.\u00a0Bacic, On hardware-in-the-loop simulation, in Proceedings of the 44th IEEE Conference on Decision and Control (IEEE, 2005), pp. 3194\u20133198","DOI":"10.1109\/CDC.2005.1582653"},{"issue":"2","key":"4_CR16","doi-asserted-by":"publisher","first-page":"221","DOI":"10.3390\/sym16020221","volume":"16","author":"A Barbie","year":"2024","unstructured":"A. Barbie, W. Hasselbring, M. Hansen, Digital twin prototypes for supporting automated integration testing of smart farming applications. Symmetry 16(2), 221 (2024)","journal-title":"Symmetry"},{"key":"4_CR17","doi-asserted-by":"crossref","unstructured":"L.\u00a0Baresi, D.Y.X. Hu, A.\u00a0Stocco, P.\u00a0Tonella, Efficient domain augmentation for autonomous driving testing using diffusion models, in 2025 IEEE\/ACM 47th International Conference on Software Engineering (ICSE) (IEEE Computer Society, Los Alamitos, CA, USA, 2025), pp. 743\u2013743","DOI":"10.1109\/ICSE55347.2025.00206"},{"key":"4_CR18","doi-asserted-by":"crossref","unstructured":"A.\u00a0Bartlett, C.\u00a0Liem, A.\u00a0Panichella, Drvn at the icst 2025 tool competition\u2013self-driving car testing track, in 2025 IEEE Conference on Software Testing, Verification and Validation (ICST) (IEEE, 2025), pp. 807\u2013808","DOI":"10.1109\/ICST62969.2025.10988997"},{"key":"4_CR19","doi-asserted-by":"crossref","unstructured":"E.\u00a0Bartocci, L.\u00a0Mariani, D.\u00a0Ni\u010dkovi\u0107, D.\u00a0Yadav, Signal feature coverage and testing for CPS Dataflow Models. ACM Trans. Softw. Eng. Methodol. (2025)","DOI":"10.1145\/3714467"},{"key":"4_CR20","unstructured":"A.\u00a0Bayha, F.\u00a0Gr\u00fcneis, B.\u00a0Sch\u00e4tz, Model-based software in-the-loop-test of autonomous systems, in Proceedings of the 2012 Symposium on Theory of Modeling and Simulation-DEVS Integrative M &S Symposium (Citeseer, 2012), pp. 1\u20136"},{"key":"4_CR21","unstructured":"B.\u00a0Beizer. Software System Testing and Quality Assurance (Van Nostrand Reinhold Co., 1984)"},{"key":"4_CR22","doi-asserted-by":"crossref","unstructured":"R.\u00a0Ben\u00a0Abdessalem, S.\u00a0Nejati, L.C. Briand, T.\u00a0Stifter, Testing advanced driver assistance systems using multi-objective search and neural networks, in Proceedings of the 31st IEEE\/ACM International Conference on Automated Software Engineering (2016), pp. 63\u201374","DOI":"10.1145\/2970276.2970311"},{"issue":"4","key":"4_CR23","doi-asserted-by":"publisher","first-page":"72","DOI":"10.1007\/s10664-024-10458-4","volume":"29","author":"M Biagiola","year":"2024","unstructured":"M. Biagiola, A. Stocco, V. Riccio, P. Tonella, Two is better than one: digital siblings to improve autonomous driving testing. Empir. Softw. Eng. 29(4), 72 (2024)","journal-title":"Empir. Softw. Eng."},{"issue":"8","key":"4_CR24","doi-asserted-by":"publisher","first-page":"2040","DOI":"10.1109\/TSE.2024.3420816","volume":"50","author":"M Biagiola","year":"2024","unstructured":"M. Biagiola, P. Tonella, Boundary state generation for testing and improvement of autonomous driving systems. IEEE Trans. Softw. Eng. 50(8), 2040\u20132053 (2024)","journal-title":"IEEE Trans. Softw. Eng."},{"key":"4_CR25","doi-asserted-by":"crossref","unstructured":"C.\u00a0Birchler, N.\u00a0Ganz, S.\u00a0Khatiri, A.\u00a0Gambi, S.\u00a0Panichella, Cost-effective simulation-based test selection in self-driving cars software with SDC-Scissor, in IEEE International Conference on Software Analysis, Evolution and Reengineering, SANER 2022, Honolulu, HI, USA, March 15-18, 2022 (IEEE, 2022), pp. 164\u2013168","DOI":"10.1109\/SANER53432.2022.00030"},{"key":"4_CR26","doi-asserted-by":"publisher","DOI":"10.1016\/j.scico.2023.102926","volume":"226","author":"C Birchler","year":"2023","unstructured":"C. Birchler, N. Ganz, S. Khatiri, A. Gambi, S. Panichella, Cost-effective simulation-based test selection in self-driving cars software. Sci. Comput. Program. 226, 102926 (2023)","journal-title":"Sci. Comput. Program."},{"issue":"3","key":"4_CR27","doi-asserted-by":"publisher","first-page":"71","DOI":"10.1007\/s10664-023-10286-y","volume":"28","author":"C Birchler","year":"2023","unstructured":"C. Birchler, S. Khatiri, B. Bosshard, A. Gambi, S. Panichella, Machine learning-based test selection for simulation-based testing of self-driving cars software. Empir. Softw. Eng. 28(3), 71 (2023)","journal-title":"Empir. Softw. Eng."},{"key":"4_CR28","doi-asserted-by":"crossref","unstructured":"C.\u00a0Birchler, S.\u00a0Khatiri, P.\u00a0Derakhshanfar, S.\u00a0Panichella, A.\u00a0Panichella, Single and multi-objective test cases prioritization for self-driving cars in virtual environments. ACM Trans. Softw. Eng. Methodol. 32(2), 28:1\u201328:30 (2023)","DOI":"10.1145\/3533818"},{"key":"4_CR29","doi-asserted-by":"crossref","unstructured":"C.\u00a0Birchler, S.\u00a0Khatiri, P.\u00a0Rani, T.\u00a0Kehrer, S.\u00a0Panichella, A roadmap for simulation-based testing of autonomous cyber-physical systems: challenges and future direction (2025)","DOI":"10.1145\/3711906"},{"key":"4_CR30","doi-asserted-by":"crossref","unstructured":"C.\u00a0Birchler, T.K. Mohammed, P.\u00a0Rani, T.\u00a0Nechita, T.\u00a0Kehrer, S.\u00a0Panichella, How does simulation-based testing for self-driving cars match human perception? in ACM International Conference on the Foundations of Software Engineering (2024)","DOI":"10.1145\/3643768"},{"key":"4_CR31","doi-asserted-by":"crossref","unstructured":"L.\u00a0Birkemeyer, J.\u00a0Fuchs, A.\u00a0Gambi, I.\u00a0Schaefer, SOTIF-Compliant scenario generation using semi-concrete scenarios and parameter sampling, in 26th IEEE International Conference on Intelligent Transportation Systems, ITSC 2023, Bilbao, Spain, September 24-28, 2023 (IEEE, 2023), pp. 2139\u20132144","DOI":"10.1109\/ITSC57777.2023.10422637"},{"key":"4_CR32","doi-asserted-by":"crossref","unstructured":"M.\u00a0B\u00f6ck, S.\u00a0Habchi, M.\u00a0Nayrolles, J.\u00a0Cito, Performance prediction from source code is task and domain specific, in 31st IEEE\/ACM International Conference on Program Comprehension, ICPC 2023, Melbourne, Australia, May 15-16, 2023 (IEEE, 2023), pp. 35\u201342","DOI":"10.1109\/ICPC58990.2023.00015"},{"key":"4_CR33","doi-asserted-by":"crossref","unstructured":"M.\u00a0Borg, R.B. Abdessalem, S.\u00a0Nejati, F.\u00a0Jegeden, D.\u00a0Shin, Digital twins are not monozygotic\u2014cross-replicating ADAS testing in two industry-grade automotive simulators, in 14th IEEE Conference on Software Testing, Verification and Validation, ICST 2021, Porto de Galinhas, Brazil, April 12-16, 2021 (IEEE, 2021), pp. 383\u2013393","DOI":"10.1109\/ICST49551.2021.00050"},{"issue":"1","key":"4_CR34","doi-asserted-by":"publisher","first-page":"73","DOI":"10.1016\/j.trc.2002.10.002","volume":"12","author":"D Bullock","year":"2004","unstructured":"D. Bullock, B. Johnson, R.B. Wells, M. Kyte, Z. Li, Hardware-in-the-loop simulation. Transp. Res. Part C: Emerg. Technol. 12(1), 73\u201389 (2004)","journal-title":"Transp. Res. Part C: Emerg. Technol."},{"key":"4_CR35","doi-asserted-by":"crossref","unstructured":"E.\u00a0Castellano, S.\u00a0Klikovits, A.\u00a0Cetinkaya, P.\u00a0Arcaini, FreneticV at the SBST 2022 tool competition, in 2022 IEEE\/ACM 15th International Workshop on Search-Based Software Testing (SBST) (2022), pp. 47\u201348","DOI":"10.1145\/3526072.3527532"},{"issue":"03","key":"4_CR36","first-page":"1750012","volume":"02","author":"H Chen","year":"2017","unstructured":"H. Chen, Applications of cyber-physical system: a literature review. J. Ind. Integr. Manag. 02(03), 1750012 (2017)","journal-title":"J. Ind. Integr. Manag."},{"key":"4_CR37","doi-asserted-by":"crossref","unstructured":"C.-H. Cheng, C.-H. Huang, H.\u00a0Yasuoka, Quantitative projection coverage for testing ML-enabled autonomous systems, in Automated Technology for Verification and Analysis: 16th International Symposium, ATVA 2018, Los Angeles, CA, USA, October 7-10, 2018, Proceedings 16 (Springer, 2018), pp. 126\u2013142","DOI":"10.1007\/978-3-030-01090-4_8"},{"key":"4_CR38","doi-asserted-by":"crossref","unstructured":"B.\u00a0Combemale, M.\u00a0Wimmer, Towards a model-based devops for cyber-physical systems, in International Workshop on Software Engineering Aspects of Continuous Development and New Paradigms of Software Production and Deployment (Springer, 2019), pp. 84\u201394","DOI":"10.1007\/978-3-030-39306-9_6"},{"issue":"10","key":"4_CR39","doi-asserted-by":"publisher","first-page":"3913","DOI":"10.1109\/TSE.2021.3107680","volume":"48","author":"O Cornejo","year":"2021","unstructured":"O. Cornejo, F. Pastore, L.C. Briand, Mutation analysis for cyber-physical systems: scalable solutions and results in the space domain. IEEE Trans. Softw. Eng. 48(10), 3913\u20133939 (2021)","journal-title":"IEEE Trans. Softw. Eng."},{"key":"4_CR40","unstructured":"L.F. Cranor, A Framework for Reasoning About the Human in the Loop (2008)"},{"key":"4_CR41","doi-asserted-by":"crossref","unstructured":"E.\u00a0Cruciani, B.\u00a0Miranda, R.\u00a0Verdecchia, A.\u00a0Bertolino, Scalable approaches for test suite reduction, in 2019 IEEE\/ACM 41st International Conference on Software Engineering (ICSE) (2019), pp. 419\u2013429","DOI":"10.1109\/ICSE.2019.00055"},{"key":"4_CR42","doi-asserted-by":"crossref","unstructured":"A. Deljouyi, A. Zaidman, Generating understandable unit tests through end-to-end test scenario carving, in 23rd IEEE International Working Conference on Source Code Analysis and Manipulation, SCAM 2023, Bogot\u00e1, Colombia, October 2-3, 2023. ed. by L. Moonen, C.D. Newman, A. Gorla (IEEE, 2023), pp. 107\u2013118","DOI":"10.1109\/SCAM59687.2023.00021"},{"key":"4_CR43","doi-asserted-by":"crossref","unstructured":"S.\u00a0Demers, P.\u00a0Gopalakrishnan, L.\u00a0Kant, A generic solution to software-in-the-loop, in MILCOM 2007\u2014IEEE Military Communications Conference (2007), pp. 1\u20136","DOI":"10.1109\/MILCOM.2007.4455268"},{"key":"4_CR44","doi-asserted-by":"crossref","unstructured":"A.\u00a0Di\u00a0Sorbo, F.\u00a0Zampetti, A.\u00a0Visaggio, M.\u00a0Di\u00a0Penta, S.\u00a0Panichella, Automated identification and qualitative characterization of safety concerns reported in uav software platforms. ACM Trans. Softw. Eng. Methodol. 32(3) (2023)","DOI":"10.1145\/3564821"},{"key":"4_CR45","doi-asserted-by":"crossref","unstructured":"J.\u00a0Dobaj, A.\u00a0Riel, T.\u00a0Krug, M.\u00a0Seidl, G.\u00a0Macher, M.\u00a0Egretzberger, Towards digital twin-enabled devops for cps providing architecture-based service adaptation and verification at runtime, in Proceedings of the 17th Symposium on Software Engineering for Adaptive and Self-Managing Systems, SEAMS \u201922 (Association for Computing Machinery, New York, NY, USA, 2022), pp. 132\u2013143","DOI":"10.1145\/3524844.3528057"},{"key":"4_CR46","doi-asserted-by":"crossref","unstructured":"S.\u00a0Dola, M.B. Dwyer, M.L. Soffa, Distribution-aware testing of neural networks using generative models, in 43rd IEEE\/ACM International Conference on Software Engineering, ICSE 2021, Madrid, Spain, 22-30 May 2021 (IEEE, 2021), pp. 226\u2013237","DOI":"10.1109\/ICSE43902.2021.00032"},{"key":"4_CR47","doi-asserted-by":"crossref","unstructured":"A. Donz\u00e9, T. Ferr\u00e8re, O. Maler, Efficient robust monitoring for STL, in Computer Aided Verification\u201425th International Conference, CAV 2013, Saint Petersburg, Russia, July 13-19, 2013. Proceedings. ed. by N. Sharygina, H. Veith. Volume 8044 of Lecture Notes in Computer Science. (Springer, 2013), pp. 264\u2013279","DOI":"10.1007\/978-3-642-39799-8_19"},{"key":"4_CR48","unstructured":"eSol, Overcoming major challenges of continuous integration (ci) in embedded software development, 2023. [Online; accessed 20-May-2025]"},{"issue":"4","key":"4_CR49","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1145\/3569935","volume":"32","author":"H Fahmy","year":"2023","unstructured":"H. Fahmy, F. Pastore, L. Briand, T. Stifter, Simulator-based explanation and debugging of hazard-triggering events in DNN-based safety-critical systems. ACM Trans. Softw. Eng. Methodol. 32(4), 1\u201347 (2023)","journal-title":"ACM Trans. Softw. Eng. Methodol."},{"key":"4_CR50","unstructured":"A.\u00a0for Standardization\u00a0of Automation and M.\u00a0Systems. Vector informatik. https:\/\/www.asam.net\/members\/detail\/vector-informatik. Accessed 30 May 2025"},{"issue":"2","key":"4_CR51","doi-asserted-by":"publisher","first-page":"136","DOI":"10.1002\/sdr.4260030205","volume":"3","author":"JW Forrester","year":"1987","unstructured":"J.W. Forrester, Lessons from system dynamics modeling. Syst. Dyn. Rev. 3(2), 136\u2013149 (1987)","journal-title":"Syst. Dyn. Rev."},{"key":"4_CR52","unstructured":"T.E. Foundation, The Eclipse software defined vehicle (SDV) working group. https:\/\/sdv.eclipse.org\/what-is-sdv\/. Accessed 22 May 2025"},{"key":"4_CR53","doi-asserted-by":"crossref","unstructured":"A.\u00a0Gambi, P.\u00a0Arcaini, D.\u00a0Nickovic, Generation of critical interactive scenarios for trajectory planning, in IEEE Intelligent Vehicles Symposium, IV 2025 (IEEE, 2025) to appear","DOI":"10.1109\/IV64158.2025.11097787"},{"key":"4_CR54","doi-asserted-by":"crossref","unstructured":"A.\u00a0Gambi, J.\u00a0Bell, A.\u00a0Zeller, Practical test dependency detection, in 11th IEEE International Conference on Software Testing, Verification and Validation, ICST 2018, V\u00e4ster\u00e5s, Sweden, April 9-13, 2018 (IEEE Computer Society, 2018), pp. 1\u201311","DOI":"10.1109\/ICST.2018.00011"},{"key":"4_CR55","doi-asserted-by":"crossref","unstructured":"A.\u00a0Gambi, A.\u00a0Gorla, A.\u00a0Zeller, O!Snap: cost-efficient testing in the cloud, in 2017 IEEE International Conference on Software Testing, Verification and Validation, ICST 2017, Tokyo, Japan, March 13-17, 2017 (IEEE Computer Society, 2017), pp. 454\u2013459","DOI":"10.1109\/ICST.2017.51"},{"key":"4_CR56","doi-asserted-by":"crossref","unstructured":"A.\u00a0Gambi, H.\u00a0Gouni, D.\u00a0Berreiter, V.\u00a0Tymofyeyev, M.\u00a0Fazzini, Action-based test carving for Android apps, in IEEE International Conference on Software Testing, Verification and Validation, ICST 2023\u2014Workshops, Dublin, Ireland, April 16-20, 2023 (IEEE, 2023), pp. 107\u2013116","DOI":"10.1109\/ICSTW58534.2023.00032"},{"key":"4_CR57","doi-asserted-by":"crossref","unstructured":"A. Gambi, T. Huynh, G. Fraser, Generating effective test cases for self-driving cars from police reports, in Proceedings of the ACM Joint Meeting on European Software Engineering Conference and Symposium on the Foundations of Software Engineering, ESEC\/SIGSOFT FSE 2019, Tallinn, Estonia, August 26-30, 2019. ed. by M. Dumas, D. Pfahl, S. Apel, A. Russo (ACM, 2019), pp. 257\u2013267","DOI":"10.1145\/3338906.3338942"},{"key":"4_CR58","doi-asserted-by":"crossref","unstructured":"A. Gambi, M. M\u00fcller, G. Fraser, Automatically testing self-driving cars with search-based procedural content generation, in Proceedings of the 28th ACM SIGSOFT International Symposium on Software Testing and Analysis, ISSTA 2019, Beijing, China, July 15-19, 2019. ed. by D. Zhang, A. M\u00f8ller (ACM, 2019), pp. 318\u2013328","DOI":"10.1145\/3293882.3330566"},{"key":"4_CR59","doi-asserted-by":"crossref","unstructured":"A.\u00a0Gambi, V.\u00a0Nguyen, J.\u00a0Ahmed, G.\u00a0Fraser, Generating critical driving scenarios from accident sketches, in IEEE International Conference On Artificial Intelligence Testing, AITest 2022, Newark, CA, USA, August 15-18, 2022 (IEEE, 2022), pp. 95\u2013102","DOI":"10.1109\/AITest55621.2022.00022"},{"issue":"11","key":"4_CR60","doi-asserted-by":"publisher","DOI":"10.1002\/smr.2465","volume":"34","author":"A Gartziandia","year":"2022","unstructured":"A. Gartziandia, A. Arrieta, J. Ayerdi, M. Illarramendi, A. Agirre, G. Sagardui, M. Arratibel, Machine learning-based test oracles for performance testing of cyber-physical systems: an industrial case study on elevators dispatching algorithms. J. Softw. Evol. Process 34(11), e2465 (2022)","journal-title":"J. Softw. Evol. Process"},{"key":"4_CR61","unstructured":"D.\u00a0Ghobari, M.H. Amini, D.Q. Tran, S.\u00a0Park, S.\u00a0Nejati, M.\u00a0Sabetzadeh, Test input validation for vision-based DL systems: An active learning approach. CoRR (2015). arxiv:abs\/2501.01606"},{"key":"4_CR62","doi-asserted-by":"publisher","DOI":"10.1016\/j.jss.2023.111906","volume":"208","author":"L Giamattei","year":"2024","unstructured":"L. Giamattei, A. Guerriero, R. Pietrantuono, S. Russo, I. Malavolta, T. Islam, M. D\u00eenga, A. Koziolek, S. Singh, M. Armbruster et al., Monitoring tools for devops and microservices: a systematic grey literature review. J. Syst. Softw. 208, 111906 (2024)","journal-title":"J. Syst. Softw."},{"key":"4_CR63","doi-asserted-by":"crossref","unstructured":"I.\u00a0Graessler, D.\u00a0Wiechel, D.\u00a0Roesmann, H.\u00a0Thiele, V-model based development of cyber-physical systems and cyber-physical production systems. Procedia CIRP 100, 253\u2013258 (2021); 31st CIRP Design Conference 2021 (CIRP Design 2021)","DOI":"10.1016\/j.procir.2021.05.119"},{"key":"4_CR64","doi-asserted-by":"crossref","unstructured":"A.\u00a0Guerriero, R.\u00a0Pietrantuono, S.\u00a0Russo, Operation is the hardest teacher: estimating DNN accuracy looking for mispredictions, in 2021 IEEE\/ACM 43rd International Conference on Software Engineering (ICSE) (IEEE, 2021), pp. 348\u2013358","DOI":"10.1109\/ICSE43902.2021.00042"},{"key":"4_CR65","doi-asserted-by":"crossref","unstructured":"A.\u00a0Guerriero, R.\u00a0Pietrantuono, S.\u00a0Russo, DeepSample: DNN sampling-based testing for operational accuracy assessment, in Proceedings of the IEEE\/ACM 46th International Conference on Software Engineering (2024), pp. 1\u201312","DOI":"10.1145\/3597503.3639584"},{"key":"4_CR66","doi-asserted-by":"crossref","unstructured":"A.\u00a0G\u00fcll\u00fc, F.\u00a0A. Shah, D.\u00a0Pfahl, Its4sdc at the icst 2025 tool competition-self-driving car testing track, in 2025 IEEE Conference on Software Testing, Verification and Validation (ICST) (IEEE, 2025), pp. 809\u2013810","DOI":"10.1109\/ICST62969.2025.10988983"},{"key":"4_CR67","doi-asserted-by":"crossref","unstructured":"C. Hildebrandt, M. von Stein, S.G. Elbaum, PhysCov: physical test coverage for autonomous vehicles, in Proceedings of the 32nd ACM SIGSOFT International Symposium on Software Testing and Analysis, ISSTA 2023, Seattle, WA, USA, July 17-21, 2023. ed. by R. Just, G. Fraser (ACM, 2023), pp. 449\u2013461","DOI":"10.1145\/3597926.3598069"},{"issue":"6","key":"4_CR68","doi-asserted-by":"publisher","first-page":"1802","DOI":"10.1109\/JIOT.2017.2703172","volume":"4","author":"A Humayed","year":"2017","unstructured":"A. Humayed, J. Lin, F. Li, B. Luo, Cyber-physical systems security-a survey. IEEE Internet Things J. 4(6), 1802\u20131831 (2017)","journal-title":"IEEE Internet Things J."},{"key":"4_CR69","doi-asserted-by":"crossref","unstructured":"N. Humbatova, G. Jahangirova, G. Bavota, V. Riccio, A. Stocco, P. Tonella, Taxonomy of real faults in deep learning systems, in ICSE \u201920: 42nd International Conference on Software Engineering, Seoul, South Korea, 27 June - 19 July, 2020. ed. by G. Rothermel, D. Bae (ACM, 2020), pp. 1110\u20131121","DOI":"10.1145\/3377811.3380395"},{"key":"4_CR70","doi-asserted-by":"crossref","unstructured":"N.\u00a0Humbatova, G.\u00a0Jahangirova, P.\u00a0Tonella, Deepcrime: mutation testing of deep learning systems based on real faults, in Proceedings of the 30th ACM SIGSOFT International Symposium on Software Testing and Analysis (2021), pp. 67\u201378","DOI":"10.1145\/3460319.3464825"},{"key":"4_CR71","doi-asserted-by":"publisher","DOI":"10.1016\/j.infsof.2022.106936","volume":"149","author":"D Humeniuk","year":"2022","unstructured":"D. Humeniuk, F. Khomh, G. Antoniol, A search-based framework for automatic generation of testing environments for cyber-physical systems. Inf. Softw. Technol. 149, 106936 (2022)","journal-title":"Inf. Softw. Technol."},{"key":"4_CR72","doi-asserted-by":"crossref","unstructured":"G.\u00a0Jahangirova, A.\u00a0Stocco, P.\u00a0Tonella, Quality metrics and oracles for autonomous vehicles testing, in 2021 14th IEEE Conference on Software Testing, Verification and Validation (ICST) (IEEE, 2021), pp. 194\u2013204","DOI":"10.1109\/ICST49551.2021.00030"},{"key":"4_CR73","doi-asserted-by":"crossref","unstructured":"G.\u00a0Jahangirova, P.\u00a0Tonella, An empirical evaluation of mutation operators for deep learning systems, in 2020 IEEE 13th International Conference on Software Testing, Validation and Verification (ICST) (IEEE, 2020), pp. 74\u201384","DOI":"10.1109\/ICST46399.2020.00018"},{"key":"4_CR74","doi-asserted-by":"crossref","unstructured":"S.\u00a0Jeong, Y.\u00a0Kwak, W.J. Lee, Software-in-the-loop simulation for early-stage testing of autosar software component, in 2016 Eighth International Conference on Ubiquitous and Future Networks (ICUFN) (IEEE, 2016), pp. 59\u201363","DOI":"10.1109\/ICUFN.2016.7536980"},{"key":"4_CR75","unstructured":"A. Johansson, S. Paulsson, Microservice integration testing with hardware-in-the-loop in ci\/cd pipelines (Chalmers University of Technology, Thesis, 2024)"},{"key":"4_CR76","first-page":"182","volume":"94","author":"N Kalra","year":"2016","unstructured":"N. Kalra, S.M. Paddock, Driving to safety: how many miles of driving would it take to demonstrate autonomous vehicle reliability? Transp. Res. Part A: Policy Practice 94, 182\u2013193 (2016)","journal-title":"Transp. Res. Part A: Policy Practice"},{"issue":"8","key":"4_CR77","doi-asserted-by":"publisher","first-page":"4950","DOI":"10.1109\/TSMC.2021.3106228","volume":"52","author":"Z Kazemi","year":"2021","unstructured":"Z. Kazemi, A.A. Safavi, M.M. Arefi, F. Naseri, Finite-time secure dynamic state estimation for cyber-physical systems under unknown inputs and sensor attacks. IEEE Trans. Syst. Man Cybern. Syst. 52(8), 4950\u20134959 (2021)","journal-title":"IEEE Trans. Syst. Man Cybern. Syst."},{"key":"4_CR78","doi-asserted-by":"crossref","unstructured":"S.\u00a0Khatiri, F.M. Amin, S.\u00a0Panichella, P.\u00a0Tonella, When uncertainty leads to unsafety: empirical insights into the role of uncertainty in unmanned aerial vehicle safety. Empir. Softw. Eng. (2025)","DOI":"10.1007\/s10664-025-10697-z"},{"key":"4_CR79","doi-asserted-by":"crossref","unstructured":"S.\u00a0Khatiri, S.\u00a0Panichella, P.\u00a0Tonella, Simulation-based test case generation for unmanned aerial vehicles in the neighborhood of real flights, in International Conference on Software Testing, Verification and Validation (IEEE, 2023), pp. 281\u2013292","DOI":"10.1109\/ICST57152.2023.00034"},{"key":"4_CR80","doi-asserted-by":"crossref","unstructured":"S.\u00a0Khatiri, P.\u00a0Saurabh, T.\u00a0Zimmermann, C.\u00a0Munasinghe, C.\u00a0Birchler, S.\u00a0Panichella, SBFT tool competition 2024\u2014cps-uav test case generation track, in IEEE\/ACM International Workshop on Search-Based and Fuzz Testing, SBFT@ICSE 2024 (2024)","DOI":"10.1145\/3643659.3643931"},{"key":"4_CR81","unstructured":"G.\u00a0Kim, P.\u00a0Debois, J.\u00a0Willis, J.\u00a0Humble, The DevOps Handbook: How to Create World-Class Agility, Reliability, and Security in Technology Organizations (IT Revolution Press, 2016)"},{"key":"4_CR82","doi-asserted-by":"crossref","unstructured":"S. Klikovits, A. Gambi, D. Dhungana, R. Rabiser, Leveraging software product lines for testing autonomous vehicles, in Proceedings of the 18th International Working Conference on Variability Modelling of Software-Intensive Systems, VaMoS 2024, Bern, Switzerland, February 7-9, 2024. ed. by T. Kehrer, M. Huchard, L. Teixeira, C. Birchler (ACM, 2024), pp. 56\u201360","DOI":"10.1145\/3634713.3634720"},{"key":"4_CR83","doi-asserted-by":"crossref","unstructured":"S. Klikovits, F. Ishikawa, T. Kobayashi, Formal modelling, in Safety Assurance under Uncertainties: From Software to Cyber-Physical\/Machine Learning Systems, Chapter 8. ed. by I. Hasuo, F. Ishikawa (CRC Press, 2025)","DOI":"10.1201\/9781003093350-8"},{"key":"4_CR84","doi-asserted-by":"publisher","DOI":"10.1016\/j.asoc.2023.110631","volume":"146","author":"S Klikovits","year":"2023","unstructured":"S. Klikovits, C.H. Thanh, A. Cetinkaya, P. Arcaini, Trust your neighbours: Handling noise in multi-objective optimisation using kNN-averaging. Appl. Soft Comput. 146, 110631 (2023)","journal-title":"Appl. Soft Comput."},{"key":"4_CR85","doi-asserted-by":"crossref","unstructured":"S. Klikovits, C.H. Thanh, A. Cetinkaya, P. Arcaini, Trust your neighbours: Handling noise in multi-objective optimisation using kNN-Averaging (GECCO\u201924 Hot off the Press), in Proceedings of the Genetic and Evolutionary Computation Conference Companion, GECCO 2024, Melbourne, VIC, Australia, July 14-18, 2024. ed. by X. Li, J. Handl (ACM, 2024), pp. 39\u201340","DOI":"10.1145\/3638530.3664075"},{"key":"4_CR86","doi-asserted-by":"crossref","unstructured":"T. Kobayashi, S. Klikovits, R. Bohrer, A. Yamada, I. Hasuo, Theorem proving at work, in Safety Assurance under Uncertainties: From Software to Cyber-Physical\/Machine Learning Systems, Chapter 9. ed. by I. Hasuo, F. Ishikawa (CRC Press, 2025)","DOI":"10.1201\/9781003093350-9"},{"issue":"11","key":"4_CR87","doi-asserted-by":"publisher","first-page":"1016","DOI":"10.1016\/j.ifacol.2018.08.474","volume":"51","author":"W Kritzinger","year":"2018","unstructured":"W. Kritzinger, M. Karner, G. Traar, J. Henjes, W. Sihn, Digital twin in manufacturing: a categorical literature review and classification. Ifac-PapersOnline 51(11), 1016\u20131022 (2018)","journal-title":"Ifac-PapersOnline"},{"key":"4_CR88","doi-asserted-by":"crossref","unstructured":"S.C. Lambertenghi, H.\u00a0Leonhard, A.\u00a0Stocco, Benchmarking image perturbations for testing automated driving assistance systems, in 2025 IEEE Conference on Software Testing, Verification and Validation (ICST) (IEEE, 2025), pp. 150\u2013161","DOI":"10.1109\/ICST62969.2025.10988980"},{"key":"4_CR89","doi-asserted-by":"crossref","unstructured":"T.\u00a0Laurent, S.\u00a0Klikovits, P.\u00a0Arcaini, F.\u00a0Ishikawa, A.\u00a0Ventresque, Parameter coverage for testing of autonomous driving systems under uncertainty. ACM Trans. Softw. Eng. Methodol. 32(3), 58:1\u201358:31 (2023)","DOI":"10.1145\/3550270"},{"key":"4_CR90","first-page":"42","volume":"12","author":"JA Ledin","year":"1999","unstructured":"J.A. Ledin, Hardware-in-the-loop simulation. Embed. Syst. Program. 12, 42\u201362 (1999)","journal-title":"Embed. Syst. Program."},{"key":"4_CR91","doi-asserted-by":"crossref","unstructured":"S.-Y. Lee, J.\u00a0Andert, D.\u00a0Neumann, C.\u00a0Querel, T.\u00a0Scheel, S.\u00a0Aktas, M.\u00a0Miccio, J.\u00a0Schaub, M.\u00a0Koetter, M.\u00a0Ehrly, Hardware-in-the-loop-based virtual calibration approach to meet real driving emissions requirements. SAE Int. J. Engines 11(2018-01-0869), 1479\u20131504 (2018)","DOI":"10.4271\/2018-01-0869"},{"key":"4_CR92","doi-asserted-by":"crossref","unstructured":"A.\u00a0Li, J.\u00a0Wang, S.\u00a0Baruah, B.\u00a0Sinopoli, N.\u00a0Zhang, An empirical study of performance interference: timing violation patterns and impacts, in 2024 IEEE 30th Real-Time and Embedded Technology and Applications Symposium (RTAS) (IEEE, 2024), pp. 320\u2013333","DOI":"10.1109\/RTAS61025.2024.00033"},{"key":"4_CR93","unstructured":"W.\u00a0Li, D.\u00a0Sadigh, S.S. Sastry, and S.\u00a0A. Seshia, Synthesis for human-in-the-loop control systems, in Tools and Algorithms for the Construction and Analysis of Systems: 20th International Conference, TACAS 2014, Held as Part of the European Joint Conferences on Theory and Practice of Software, ETAPS 2014, Grenoble, France, April 5-13, 2014. Proceedings 20 (Springer, 2014), pp. 470\u2013484"},{"issue":"2","key":"4_CR94","doi-asserted-by":"publisher","first-page":"919","DOI":"10.1109\/TIE.2007.892253","volume":"54","author":"B Lu","year":"2007","unstructured":"B. Lu, X. Wu, H. Figueroa, A. Monti, A low-cost real-time hardware-in-the-loop testing approach of power electronics controls. IEEE Trans. Ind. Electron. 54(2), 919\u2013931 (2007)","journal-title":"IEEE Trans. Ind. Electron."},{"key":"4_CR95","doi-asserted-by":"crossref","unstructured":"Q. Luo, F. Hariri, L. Eloussi, D. Marinov, An empirical analysis of flaky tests, in Proceedings of the 22nd ACM SIGSOFT International Symposium on Foundations of Software Engineering, (FSE-22), Hong Kong, China, November 16-22, 2014. ed. by S. Cheung, A. Orso, M.D. Storey (ACM, 2014), pp. 643\u2013653","DOI":"10.1145\/2635868.2635920"},{"issue":"4","key":"4_CR96","doi-asserted-by":"publisher","first-page":"3378","DOI":"10.1109\/TDSC.2022.3194089","volume":"20","author":"RR Maiti","year":"2022","unstructured":"R.R. Maiti, C.H. Yoong, V.R. Palleti, A. Silva, C.M. Poskitt, Mitigating adversarial attacks on data-driven invariant checkers for cyber-physical systems. IEEE Trans. Dependable Secure Comput. 20(4), 3378\u20133391 (2022)","journal-title":"IEEE Trans. Dependable Secure Comput."},{"key":"4_CR97","doi-asserted-by":"crossref","unstructured":"F.M. Malik, S.M. Khatiri, Certifail at the icst 2025 tool competition\u2014self-driving car testing track, in 2025 IEEE Conference on Software Testing, Verification and Validation (ICST) (2025), pp. 811\u2013812","DOI":"10.1109\/ICST62969.2025.10988970"},{"key":"4_CR98","doi-asserted-by":"crossref","unstructured":"G. Mallardi, F. Calefato, L. Quaranta, F. Lanubile, An mlops approach for deploying machine learning models in healthcare systems, in IEEE International Conference on Bioinformatics and Biomedicine, BIBM 2024, Lisbon, Portugal, December 3-6, 2024. ed. by M. Cannataro, H.J. Zheng, L. Gao, J. Cheng, J.L. de\u00a0Miranda, E. Zumpano, X. Hu, Y. Cho, T. Park (IEEE, 2024), pp. 6832\u20136837","DOI":"10.1109\/BIBM62325.2024.10822603"},{"key":"4_CR99","doi-asserted-by":"crossref","unstructured":"C.\u00a0Mandrioli, S.Y. Shin, D.\u00a0Bianculli, L.\u00a0Briand, Testing cps with design assumptions-based metamorphic relations and genetic programming. IEEE Trans. Softw. Eng. (2025)","DOI":"10.1109\/TSE.2025.3563121"},{"key":"4_CR100","doi-asserted-by":"crossref","unstructured":"T.\u00a0M\u00e5rtensson, D.\u00a0St\u00e5hl, J.\u00a0Bosch, Continuous integration applied to software-intensive embedded systems\u2013problems and experiences, in Product-Focused Software Process Improvement: 17th International Conference, PROFES 2016, Trondheim, Norway, November 22-24, 2016, Proceedings 17 (Springer, 2016), pp. 448\u2013457","DOI":"10.1007\/978-3-319-49094-6_30"},{"key":"4_CR101","doi-asserted-by":"crossref","unstructured":"M.\u00a0Maryam, M.\u00a0Biagiola, A.\u00a0Stocco, V.\u00a0Riccio, Benchmarking generative AI models for deep learning test input generation, in 2025 IEEE Conference on Software Testing, Verification and Validation (ICST) (IEEE, 2025), pp. 174\u2013185","DOI":"10.1109\/ICST62969.2025.10989043"},{"key":"4_CR102","doi-asserted-by":"crossref","unstructured":"C.\u00a0Menghi, S.\u00a0Nejati, L.\u00a0Briand, Y.I. Parache, Approximation-refinement testing of compute-intensive cyber-physical models: an approach based on system identification, in Proceedings of the ACM\/IEEE 42nd International Conference on Software Engineering, ICSE \u201920 (Association for Computing Machinery, New York, NY, USA, 2020), pp. 372\u2013384","DOI":"10.1145\/3377811.3380370"},{"key":"4_CR103","doi-asserted-by":"crossref","unstructured":"J.\u00a0Mertens, S.\u00a0Klikovits, F.\u00a0Bordeleau, J.\u00a0Denil, \u00d8.\u00a0Haugen, Continuous evolution of digital twins using the DarTwin notation. Softw. Syst. Model. (2024)","DOI":"10.1007\/s10270-024-01216-7"},{"key":"4_CR104","doi-asserted-by":"crossref","unstructured":"M.\u00a0Mossige, A.\u00a0Gotlieb, H.\u00a0Spieker, H.\u00a0Meling, M.\u00a0Carlsson, Time-aware test case execution scheduling for cyber-physical systems, in Principles and Practice of Constraint Programming: 23rd International Conference, CP 2017, Melbourne, VIC, Australia, August 28\u2013September 1, 2017, Proceedings 23 (Springer, 2017), pp. 387\u2013404","DOI":"10.1007\/978-3-319-66158-2_25"},{"key":"4_CR105","doi-asserted-by":"crossref","unstructured":"P.\u00a0Munk, M.\u00a0Schweizer, Devops and safety? safeops! towards ensuring safety in feature-driven development with frequent releases, in International Conference on Computer Safety, Reliability, and Security (Springer, Berlin, 2022), pp. 145\u2013157","DOI":"10.1007\/978-3-031-14862-0_11"},{"key":"4_CR106","doi-asserted-by":"crossref","unstructured":"S.\u00a0Nabi, M.\u00a0Balike, J.\u00a0Allen, K.\u00a0Rzemien, An overview of hardware-in-the-loop testing systems at Visteon (2004)","DOI":"10.4271\/2004-01-1240"},{"issue":"4","key":"4_CR107","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1145\/3640335","volume":"33","author":"N Neelofar","year":"2024","unstructured":"N. Neelofar, A. Aleti, Identifying and explaining safety-critical scenarios for autonomous vehicles via key features. ACM Trans. Softw. Eng. Methodol. 33(4), 1\u201332 (2024)","journal-title":"ACM Trans. Softw. Eng. Methodol."},{"key":"4_CR108","doi-asserted-by":"crossref","unstructured":"N.\u00a0Neelofar, A.\u00a0Aleti, Towards reliable ai: adequacy metrics for ensuring the quality of system-level testing of autonomous vehicles, in Proceedings of the 46th IEEE\/ACM International Conference on Software Engineering (2024), pp. 1\u201312","DOI":"10.1145\/3597503.3623314"},{"key":"4_CR109","doi-asserted-by":"crossref","unstructured":"E.\u00a0Nesterini, E.\u00a0Bartocci, A.\u00a0Gambi, D.\u00a0Nickovic, S.A. Seshia, H.\u00a0Torfah, Mining specifications for predictive safety monitoring, in Proceedings of the ACM\/IEEE 16th International Conference on Cyber-Physical Systems, ICCPS 2025, (with CPS-IoT Week 2025), Irvine, CA, USA, May 6-9, 2025 (ACM, 2025), pp. 6:1\u20136:11","DOI":"10.1145\/3716550.3722021"},{"key":"4_CR110","doi-asserted-by":"crossref","unstructured":"A.\u00a0Ngo, M.P. Bauer, M.\u00a0Resch, A multi-layered approach for measuring the simulation-to-reality gap of radar perception for autonomous driving, in 24th IEEE International Intelligent Transportation Systems Conference, ITSC 2021, Indianapolis, IN, USA, September 19-22, 2021 (IEEE, 2021), pp. 4008\u20134014","DOI":"10.1109\/ITSC48978.2021.9564521"},{"key":"4_CR111","doi-asserted-by":"crossref","unstructured":"V.\u00a0Nguyen, S.\u00a0Huber, A.\u00a0Gambi, SALVO: automated generation of diversified tests for self-driving cars from existing maps, in International Conference on Artificial Intelligence Testing (IEEE, 2021), pp. 128\u2013135","DOI":"10.1109\/AITEST52744.2021.00033"},{"key":"4_CR112","doi-asserted-by":"crossref","unstructured":"F.\u00a0Niu, W.K.G. Assun\u00e7\u00e3o, L.\u00a0Huang, C.\u00a0Mayr-Dorn, J.\u00a0Ge, B.\u00a0Luo, A.\u00a0Egyed, RAT: a refactoring-aware traceability model for bug localization. In 45th IEEE\/ACM International Conference on Software Engineering, ICSE 2023, Melbourne, Australia, May 14-20, 2023 (IEEE, 2023), pp. 196\u2013207","DOI":"10.1109\/ICSE48619.2023.00028"},{"key":"4_CR113","doi-asserted-by":"crossref","unstructured":"C.\u00a0Papamichail, D.\u00a0Stark, A.\u00a0Gambi, Scenario-based testing with beamng.tech (hands-on training), in IEEE Conference on Software Testing, Verification and Validation, ICST 2025, Napoli, Italy, March 31 - April 4, 2025 (IEEE, 2025), pp. 767\u2013768","DOI":"10.1109\/ICST62969.2025.10989037"},{"key":"4_CR114","unstructured":"Parasoft. Guide to CI\/CD for Medical Device Software DevOps (2022)"},{"key":"4_CR115","doi-asserted-by":"crossref","unstructured":"K.\u00a0Pei, Y.\u00a0Cao, J.\u00a0Yang, S.\u00a0Jana, Deepxplore: Automated whitebox testing of deep learning systems, in proceedings of the 26th Symposium on Operating Systems Principles (2017), pp. 1\u201318","DOI":"10.1145\/3132747.3132785"},{"key":"4_CR116","doi-asserted-by":"crossref","unstructured":"M.\u00a0Pezze, M.\u00a0Young, Testing object-oriented software, in Proceedings. 26th International Conference on Software Engineering (IEEE, 2004), pp. 739\u2013740","DOI":"10.1109\/ICSE.2004.1317513"},{"key":"4_CR117","doi-asserted-by":"crossref","unstructured":"A.R. Plummer, Model-in-the-loop testing. Proc Inst Mech Eng, Part I: J Syst Control Eng 220(3), 183\u2013199 (2006)","DOI":"10.1243\/09596518JSCE207"},{"key":"4_CR118","doi-asserted-by":"crossref","unstructured":"P.\u00a0Polack, F.\u00a0Altch\u00e9, B.\u00a0d\u2019Andr\u00e9a Novel, A.\u00a0de\u00a0La\u00a0Fortelle, The kinematic bicycle model: a consistent model for planning feasible trajectories for autonomous vehicles? in 2017 IEEE Intelligent Vehicles Symposium (IV) (2017), pp. 812\u2013818","DOI":"10.1109\/IVS.2017.7995816"},{"key":"4_CR119","doi-asserted-by":"crossref","unstructured":"P.\u00a0Polack, F.\u00a0Altch\u00e9, B.\u00a0d\u2019Andr\u00e9a Novel, A.\u00a0de\u00a0La\u00a0Fortelle, The kinematic bicycle model: a consistent model for planning feasible trajectories for autonomous vehicles? in 2017 IEEE intelligent vehicles symposium (IV) (IEEE, 2017), pp. 812\u2013818","DOI":"10.1109\/IVS.2017.7995816"},{"key":"4_CR120","doi-asserted-by":"publisher","first-page":"5193","DOI":"10.1007\/s10664-020-09881-0","volume":"25","author":"V Riccio","year":"2020","unstructured":"V. Riccio, G. Jahangirova, A. Stocco, N. Humbatova, M. Weiss, P. Tonella, Testing machine learning based systems: a systematic mapping. Empir. Softw. Eng. 25, 5193\u20135254 (2020)","journal-title":"Empir. Softw. Eng."},{"key":"4_CR121","doi-asserted-by":"crossref","unstructured":"V. Riccio, P. Tonella, Model-based exploration of the frontier of behaviours for deep learning system testing, in ESEC\/FSE \u201920: 28th ACM Joint European Software Engineering Conference and Symposium on the Foundations of Software Engineering, Virtual Event, USA, November 8-13, 2020. ed. by P. Devanbu, M.B. Cohen, T. Zimmermann (ACM, 2020), pp. 876\u2013888","DOI":"10.1145\/3368089.3409730"},{"key":"4_CR122","doi-asserted-by":"crossref","unstructured":"V.\u00a0Riccio, P.\u00a0Tonella, When and why test generators for deep learning produce invalid inputs: an empirical study. In 45th IEEE\/ACM International Conference on Software Engineering, ICSE 2023, Melbourne, Australia, May 14-20, 2023 (IEEE, 2023), pp. 1161\u20131173","DOI":"10.1109\/ICSE48619.2023.00104"},{"key":"4_CR123","doi-asserted-by":"crossref","unstructured":"L.\u00a0Rothrock, S.\u00a0Narayanan, Human-in-the-Loop Simulations (Springer, 2011)","DOI":"10.1007\/978-0-85729-883-6"},{"issue":"1","key":"4_CR124","doi-asserted-by":"publisher","DOI":"10.1002\/stvr.1794","volume":"32","author":"Z Sadri-Moshkenani","year":"2022","unstructured":"Z. Sadri-Moshkenani, J. Bradley, G. Rothermel, Survey on test case generation, selection and prioritization for cyber-physical systems. Softw. Test. Verification and Reliab. 32(1), e1794 (2022)","journal-title":"Softw. Test. Verification and Reliab."},{"key":"4_CR125","doi-asserted-by":"crossref","unstructured":"F.\u00a0Scheuer, A.\u00a0Gambi, P.\u00a0Arcaini, STRETCH: generating challenging scenarios for testing collision avoidance systems, in IEEE Intelligent Vehicles Symposium, IV 2023, Anchorage, AK, USA, June 4-7, 2023 (IEEE, 2023), pp. 1\u20136","DOI":"10.1109\/IV55152.2023.10186634"},{"key":"4_CR126","doi-asserted-by":"publisher","DOI":"10.1016\/j.jss.2020.110813","volume":"171","author":"SY Shin","year":"2021","unstructured":"S.Y. Shin, K. Chaouch, S. Nejati, M. Sabetzadeh, L.C. Briand, F. Zimmer, Uncertainty-aware specification and analysis for hardware-in-the-loop testing of cyber-physical systems. J. Syst. Softw. 171, 110813 (2021)","journal-title":"J. Syst. Softw."},{"key":"4_CR127","unstructured":"L.\u00a0Sorokin, M.\u00a0Biagiola, A.\u00a0Stocco, Simulator ensembles for trustworthy autonomous driving testing (2025). arXiv:2503.08936"},{"key":"4_CR128","doi-asserted-by":"crossref","unstructured":"M.\u00a0Stadler, M.\u00a0Vierhauser, A.\u00a0Garmendia, M.\u00a0Wimmer, J.\u00a0Cleland-Huang, Flexible model-driven runtime monitoring support for cyber-physical systems, in Proceedings of the ACM\/IEEE 44th International Conference on Software Engineering: Companion Proceedings, ICSE \u201922 (Association for Computing Machinery, New York, NY, USA, 2022), pp. 350\u2013351","DOI":"10.1145\/3510454.3528647"},{"key":"4_CR129","doi-asserted-by":"crossref","unstructured":"B.\u00a0Steininger, C.\u00a0Papamichail, D.\u00a0Stark, D.\u00a0Nickovic, A.\u00a0Gambi, Automatically generating content for testing autonomous vehicles from user descriptions, in Proceedings of the 2025 ACM\/IEEE 47th International Conference on Software Engineering: New Ideas and Emerging Results, NIER@ICSE 2025 (2025) to appear","DOI":"10.1109\/ICSE-NIER66352.2025.00021"},{"issue":"4","key":"4_CR130","doi-asserted-by":"publisher","first-page":"1928","DOI":"10.1109\/TSE.2022.3202311","volume":"49","author":"A Stocco","year":"2023","unstructured":"A. Stocco, B. Pulfer, P. Tonella, Mind the gap! A study on the transferability of virtual versus physical-world testing of autonomous driving systems. IEEE Trans. Softw. Eng. 49(4), 1928\u20131940 (2023)","journal-title":"IEEE Trans. Softw. Eng."},{"key":"4_CR131","doi-asserted-by":"crossref","unstructured":"A.\u00a0Stocco, P.\u00a0Tonella, Towards anomaly detectors that learn continuously, in 2020 IEEE international symposium on software reliability engineering workshops (ISSREW) (IEEE, 2020), pp. 201\u2013208","DOI":"10.1109\/ISSREW51248.2020.00073"},{"issue":"10","key":"4_CR132","doi-asserted-by":"publisher","DOI":"10.1002\/smr.2386","volume":"34","author":"A Stocco","year":"2022","unstructured":"A. Stocco, P. Tonella, Confidence-driven weighted retraining for predicting safety-critical failures in autonomous driving systems. J. Softw. Evol. Process 34(10), e2386 (2022)","journal-title":"J. Softw. Evol. Process"},{"key":"4_CR133","doi-asserted-by":"crossref","unstructured":"A. Stocco, M. Weiss, M. Calzana, P. Tonella, Misbehaviour prediction for autonomous driving systems, in Proceedings of the ACM\/IEEE 42nd International Conference on Software Engineering. ed. by G. Rothermel, D. Bae (ACM, 2020), pp. 359\u2013371","DOI":"10.1145\/3377811.3380353"},{"key":"4_CR134","unstructured":"Swissinfo, swiss post drone crashes in zurich\u2014again (2019)"},{"key":"4_CR135","doi-asserted-by":"crossref","unstructured":"V. Terragni, P. Salza, F. Ferrucci, A container-based infrastructure for fuzzy-driven root causing of flaky tests, in ICSE-NIER 2020: 42nd International Conference on Software Engineering, New Ideas and Emerging Results, Seoul, South Korea, 27 June-19 July, 2020. ed. by G. Rothermel, D. Bae (ACM, 2020), pp. 69\u201372","DOI":"10.1145\/3377816.3381742"},{"key":"4_CR136","doi-asserted-by":"crossref","unstructured":"G.\u00a0Tibba, C.\u00a0Malz, C.\u00a0Stoermer, N.\u00a0Nagarajan, L.\u00a0Zhang, S.\u00a0Chakraborty, Testing automotive embedded systems under X-in-the-loop setups, in 2016 IEEE\/ACM International Conference on Computer-Aided Design (ICCAD) (IEEE, 2016), pp. 1\u20138","DOI":"10.1145\/2966986.2980076"},{"key":"4_CR137","unstructured":"T.N.Y. Times, Are there problems with the boeing 737 max? a second deadly crash raises new questions (2019)"},{"key":"4_CR138","doi-asserted-by":"crossref","unstructured":"P.\u00a0Tonella, The road toward dependable ai based systems, in 2023 IEEE\/ACM 45th International Conference on Software Engineering (ICSE) (IEEE, 2023), p. 2","DOI":"10.1109\/ICSE48619.2023.00011"},{"issue":"2","key":"4_CR139","doi-asserted-by":"publisher","first-page":"110","DOI":"10.1109\/MS.2018.2875989","volume":"36","author":"C Trubiani","year":"2019","unstructured":"C. Trubiani, P. Jamshidi, J. Cito, W. Shang, Z.M. Jiang, M. Borg, Performance issues? Hey DevOps, mind the uncertainty. IEEE Softw. 36(2), 110\u2013117 (2019)","journal-title":"IEEE Softw."},{"key":"4_CR140","unstructured":"University\u00a0of Luxembourg. Addressing the challenges of devops for cyber-physical systems, 2023. [Online; accessed 20-May-2025]"},{"key":"4_CR141","doi-asserted-by":"crossref","unstructured":"P.\u00a0Valle, A.\u00a0Arrieta, L.\u00a0Han, S.\u00a0Ali, T.\u00a0Yue, Defining and generating multi-level and uncertainty-wise test oracles for cyber-physical systems. Softw. Syst. Model. 1\u201326 (2025)","DOI":"10.1007\/s10270-025-01271-8"},{"issue":"4","key":"4_CR142","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1007\/s10664-025-10670-w","volume":"30","author":"P Valle","year":"2025","unstructured":"P. Valle, V. Riccio, A. Arrieta, P. Tonella, M. Arratibel, An industrial experience report on applying search-based boundary input generation to cyber-physical systems. Empir. Softw. Eng. 30(4), 1\u201334 (2025)","journal-title":"Empir. Softw. Eng."},{"key":"4_CR143","doi-asserted-by":"publisher","DOI":"10.1016\/j.jss.2023.111904","volume":"210","author":"I van Dinten","year":"2024","unstructured":"I. van Dinten, P. Derakhshanfar, A. Panichella, A. Zaidman, The slow and the furious? performance antipattern detection in cyber-physical systems. J. Syst. Softw. 210, 111904 (2024)","journal-title":"J. Syst. Softw."},{"issue":"12","key":"4_CR144","doi-asserted-by":"publisher","first-page":"1295","DOI":"10.1177\/0037549714553229","volume":"90","author":"T Veps\u00e4l\u00e4inen","year":"2014","unstructured":"T. Veps\u00e4l\u00e4inen, S. Kuikka, Integrating model-in-the-loop simulations to model-driven development in industrial control. Simulation 90(12), 1295\u20131311 (2014)","journal-title":"Simulation"},{"key":"4_CR145","doi-asserted-by":"crossref","unstructured":"S.\u00a0Vost, S.\u00a0Wagner, Keeping continuous deliveries safe, in 2017 IEEE\/ACM 39th International Conference on Software Engineering Companion (ICSE-C) (2017), pp. 259\u2013261","DOI":"10.1109\/ICSE-C.2017.135"},{"key":"4_CR146","doi-asserted-by":"crossref","unstructured":"F.\u00a0Warg, H.\u00a0Blom, J.\u00a0Borg, R.\u00a0Johansson, Continuous deployment for dependable systems with continuous assurance cases, in 2019 IEEE International Symposium on Software Reliability Engineering Workshops (ISSREW) (IEEE, 2019), pp. 318\u2013325","DOI":"10.1109\/ISSREW.2019.00091"},{"key":"4_CR147","doi-asserted-by":"crossref","unstructured":"H.\u00a0Weber, J.\u00a0Bock, J.\u00a0Klimke, C.\u00a0Roesener, J.\u00a0Hiller, R.\u00a0Krajewski, A.\u00a0Zlocki, L.E., A framework for definition of logical scenarios for safety assurance of automated driving. Traffic Injury Prev. 20(sup1), S65\u2013S70 (2019)","DOI":"10.1080\/15389588.2019.1630827"},{"key":"4_CR148","doi-asserted-by":"crossref","unstructured":"T.\u00a0Woodlief, F.\u00a0Toledo, S.G. Elbaum, M.B. Dwyer, S3C: spatial semantic scene coverage for autonomous vehicles, in Proceedings of the 46th IEEE\/ACM International Conference on Software Engineering, ICSE 2024, Lisbon, Portugal, April 14-20, 2024 (ACM, 2024), pp. 141:1\u2013141:13","DOI":"10.1145\/3597503.3639178"},{"key":"4_CR149","doi-asserted-by":"publisher","DOI":"10.1016\/j.jss.2022.111425","volume":"192","author":"F Zampetti","year":"2022","unstructured":"F. Zampetti, R. Kapur, M. Di Penta, S. Panichella, An empirical characterization of software bugs in open-source cyber\u2013physical systems. J. Syst. Softw. 192, 111425 (2022)","journal-title":"J. Syst. Softw."},{"key":"4_CR150","doi-asserted-by":"crossref","unstructured":"F.\u00a0Zampetti, V.\u00a0Nardone, M.\u00a0Di\u00a0Penta, Problems and solutions in applying continuous integration and delivery to 20 open-source cyber-physical systems, in Proceedings of the 19th International Conference on Mining Software Repositories, MSR \u201922 (Association for Computing Machinery, New York, NY, USA, 2022), pp. 646\u2013657","DOI":"10.1145\/3524842.3527948"},{"issue":"3","key":"4_CR151","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1145\/3571854","volume":"32","author":"F Zampetti","year":"2023","unstructured":"F. Zampetti, D. Tamburri, S. Panichella, A. Panichella, G. Canfora, M. Di Penta, Continuous integration and delivery practices for cyber-physical systems: an interview-based study. ACM Trans. Softw. Eng. Methodol. 32(3), 1\u201344 (2023)","journal-title":"ACM Trans. Softw. Eng. Methodol."},{"key":"4_CR152","doi-asserted-by":"crossref","unstructured":"F.\u00a0Zampetti, D.A. Tamburri, S.\u00a0Panichella, A.\u00a0Panichella, G.\u00a0Canfora, M.D. Penta, Continuous integration and delivery practices for cyber-physical systems: an interview-based study. ACM Trans. Softw. Eng. Methodol. 32(3), 73:1\u201373:44 (2023)","DOI":"10.1145\/3571854"},{"key":"4_CR153","doi-asserted-by":"crossref","unstructured":"M.\u00a0Zeller, Towards continuous safety assessment in context of devops, in Computer Safety, Reliability, and Security. SAFECOMP 2021 Workshops: DECSoS, MAPSOD, DepDevOps, USDAI, and WAISE, York, UK, September 7, 2021, Proceedings 40 (Springer, 2021), pp. 145\u2013157","DOI":"10.1007\/978-3-030-83906-2_11"},{"key":"4_CR154","doi-asserted-by":"crossref","unstructured":"M.\u00a0Zhang, Y.\u00a0Zhang, L.\u00a0Zhang, C.\u00a0Liu, S.\u00a0Khurshid, Deeproad: Gan-based metamorphic testing and input validation framework for autonomous driving systems, in Proceedings of the 33rd ACM\/IEEE international conference on automated software engineering (2018), pp. 132\u2013142","DOI":"10.1145\/3238147.3238187"},{"issue":"4","key":"4_CR155","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1145\/3063382","volume":"16","author":"X Zheng","year":"2017","unstructured":"X. Zheng, C. Julien, H. Chen, R. Podorozhny, F. Cassez, Real-time simulation support for runtime verification of cyber-physical systems. ACM Trans. Embed. Comput. Syst. (TECS) 16(4), 1\u201324 (2017)","journal-title":"ACM Trans. Embed. Comput. Syst. (TECS)"},{"key":"4_CR156","doi-asserted-by":"publisher","first-page":"52179","DOI":"10.1109\/ACCESS.2018.2869834","volume":"6","author":"X Zhou","year":"2018","unstructured":"X. Zhou, X. Gou, T. Huang, S. Yang, Review on testing of cyber physical systems: methods and testbeds. IEEE Access 6, 52179\u201352194 (2018)","journal-title":"IEEE Access"},{"key":"4_CR157","doi-asserted-by":"crossref","unstructured":"T. Zohdinasab, V. Riccio, A. Gambi, P. Tonella, DeepHyperion: exploring the feature space of deep learning-based systems through illumination search, in ISSTA \u201921: 30th ACM SIGSOFT International Symposium on Software Testing and Analysis, Virtual Event, Denmark, July 11-17, 2021. ed. by C. Cadar, X. Zhang (ACM, 2021), pp. 79\u201390","DOI":"10.1145\/3460319.3464811"},{"key":"4_CR158","unstructured":"T. Zohdinasab, V. Riccio, A. Gambi, P. Tonella, Deephyperion: exploring the feature space of deep learning-based systems through illumination search, in Software Engineering 2023, Fachtagung des GI-Fachbereichs Softwaretechnik, 20.-24. Februar 2023, Paderborn. ed. by G. Engels, R. Hebig, M. Tichy. Volume P-332 of LNI. (Gesellschaft f\u00fcr Informatik e.V., 2023), pp. 131\u2013132"},{"key":"4_CR159","doi-asserted-by":"crossref","unstructured":"T.\u00a0Zohdinasab, V.\u00a0Riccio, A.\u00a0Gambi, P.\u00a0Tonella, Efficient and effective feature space exploration for testing deep learning systems. ACM Trans. Softw. Eng. Methodol. 32(2), 49:1\u201349:38 (2023)","DOI":"10.1145\/3544792"},{"key":"4_CR160","doi-asserted-by":"crossref","unstructured":"T. Zohdinasab, V. Riccio, P. Tonella, DeepAtash: focused test generation for deep learning systems, in Proceedings of the 32nd ACM SIGSOFT International Symposium on Software Testing and Analysis, ISSTA 2023, Seattle, WA, USA, July 17-21, 2023. ed. by R. Just, G. Fraser (ACM, 2023), pp. 954\u2013966","DOI":"10.1145\/3597926.3598109"},{"key":"4_CR161","doi-asserted-by":"crossref","unstructured":"T.\u00a0Zohdinasab, V.\u00a0Riccio, P.\u00a0Tonella, An empirical study on low-and high-level explanations of deep learning misbehaviours, in 2023 ACM\/IEEE International Symposium on Empirical Software Engineering and Measurement (ESEM) (IEEE, 2023), pp. 1\u201311","DOI":"10.1109\/ESEM56168.2023.10304866"},{"issue":"6","key":"4_CR162","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1145\/3664605","volume":"33","author":"T Zohdinasab","year":"2024","unstructured":"T. Zohdinasab, V. Riccio, P. Tonella, Focused test generation for autonomous driving systems. ACM Trans. Softw. Eng. Methodol. 33(6), 1\u201332 (2024)","journal-title":"ACM Trans. Softw. Eng. Methodol."}],"container-title":["Roadmap for DevOps in Cyber-Physical Systems"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-981-95-1786-2_4","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,2,6]],"date-time":"2026-02-06T16:13:53Z","timestamp":1770394433000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-981-95-1786-2_4"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026]]},"ISBN":["9789819517855","9789819517862"],"references-count":162,"URL":"https:\/\/doi.org\/10.1007\/978-981-95-1786-2_4","relation":{},"subject":[],"published":{"date-parts":[[2026]]},"assertion":[{"value":"7 February 2026","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}}]}}