{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,3,12]],"date-time":"2026-03-12T14:55:42Z","timestamp":1773327342553,"version":"3.50.1"},"publisher-location":"Cham","reference-count":46,"publisher":"Springer Nature Switzerland","isbn-type":[{"value":"9783031197581","type":"print"},{"value":"9783031197598","type":"electronic"}],"license":[{"start":{"date-parts":[[2022,1,1]],"date-time":"2022-01-01T00:00:00Z","timestamp":1640995200000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"},{"start":{"date-parts":[[2022,1,1]],"date-time":"2022-01-01T00:00:00Z","timestamp":1640995200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2022]]},"DOI":"10.1007\/978-3-031-19759-8_24","type":"book-chapter","created":{"date-parts":[[2022,10,19]],"date-time":"2022-10-19T09:07:32Z","timestamp":1666170452000},"page":"393-405","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":11,"title":["Formal Methods Meet Machine Learning (F3ML)"],"prefix":"10.1007","author":[{"given":"Kim","family":"Larsen","sequence":"first","affiliation":[]},{"given":"Axel","family":"Legay","sequence":"additional","affiliation":[]},{"given":"Gerrit","family":"Nolte","sequence":"additional","affiliation":[]},{"given":"Maximilian","family":"Schl\u00fcter","sequence":"additional","affiliation":[]},{"given":"Marielle","family":"Stoelinga","sequence":"additional","affiliation":[]},{"given":"Bernhard","family":"Steffen","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2022,10,17]]},"reference":[{"issue":"4","key":"24_CR1","doi-asserted-by":"publisher","first-page":"319","DOI":"10.1007\/BF00116828","volume":"2","author":"D Angluin","year":"1988","unstructured":"Angluin, D.: Queries and concept learning. Mach. Learn. 2(4), 319\u2013342 (1988)","journal-title":"Mach. Learn."},{"key":"24_CR2","unstructured":"Athalye, A., Engstrom, L., Ilyas, A., Kwok, K.: Synthesizing robust adversarial examples. In: Dy, J., Krause, A. (eds.) Proceedings of the 35th International Conference on Machine Learning, Proceedings of Machine Learning Research, vol. 80, pp. 284\u2013293. PMLR (2018). https:\/\/proceedings.mlr.press\/v80\/athalye18b.html"},{"issue":"2","key":"24_CR3","doi-asserted-by":"publisher","first-page":"171","DOI":"10.1023\/A:1008699807402","volume":"10","author":"RI Bahar","year":"1997","unstructured":"Bahar, R.I., et al.: Algebric decision diagrams and their applications. Formal Methods Syst. Des. 10(2), 171\u2013206 (1997)","journal-title":"Formal Methods Syst. Des."},{"key":"24_CR4","doi-asserted-by":"publisher","first-page":"305","DOI":"10.1007\/978-3-319-10575-8_11","volume-title":"Handbook of Model Checking","author":"C Barrett","year":"2018","unstructured":"Barrett, C., Tinelli, C.: Satisfiability modulo theories. In: Clarke, E., Henzinger, T., Veith, H., Bloem, R. (eds.) Handbook of Model Checking, pp. 305\u2013343. Springer, Cham (2018). https:\/\/doi.org\/10.1007\/978-3-319-10575-8_11"},{"issue":"6","key":"24_CR5","doi-asserted-by":"publisher","first-page":"234","DOI":"10.1145\/390016.808445","volume":"10","author":"RS Boyer","year":"1975","unstructured":"Boyer, R.S., Elspas, B., Levitt, K.N.: Select-a formal system for testing and debugging programs by symbolic execution. ACM SigPlan Not. 10(6), 234\u2013245 (1975)","journal-title":"ACM SigPlan Not."},{"key":"24_CR6","first-page":"1877","volume":"33","author":"T Brown","year":"2020","unstructured":"Brown, T., et al.: Language models are few-shot learners. Adv. Neural. Inf. Process. Syst. 33, 1877\u20131901 (2020)","journal-title":"Adv. Neural. Inf. Process. Syst."},{"key":"24_CR7","unstructured":"Chung, J., Gulcehre, C., Cho, K., Bengio, Y.: Empirical evaluation of gated recurrent neural networks on sequence modeling. arXiv preprint arXiv:1412.3555 (2014)"},{"key":"24_CR8","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"54","DOI":"10.1007\/BFb0058022","volume-title":"Foundations of Software Technology and Theoretical Computer Science","author":"EM Clarke","year":"1997","unstructured":"Clarke, E.M.: Model checking. In: Ramesh, S., Sivakumar, G. (eds.) FSTTCS 1997. LNCS, vol. 1346, pp. 54\u201356. Springer, Heidelberg (1997). https:\/\/doi.org\/10.1007\/BFb0058022"},{"key":"24_CR9","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"285","DOI":"10.1007\/978-3-030-30985-5_18","volume-title":"From Software Engineering to Formal Methods and Tools, and Back","author":"M Cordy","year":"2019","unstructured":"Cordy, M., et al.: A decade of featured transition systems. In: ter Beek, M.H., Fantechi, A., Semini, L. (eds.) From Software Engineering to Formal Methods and Tools, and Back. LNCS, vol. 11865, pp. 285\u2013312. Springer, Cham (2019). https:\/\/doi.org\/10.1007\/978-3-030-30985-5_18"},{"key":"24_CR10","unstructured":"Doran, D., Schulz, S., Besold, T.R.: What does explainable AI really mean? A new conceptualization of perspectives. arXiv preprint arXiv:1710.00794 (2017)"},{"key":"24_CR11","doi-asserted-by":"crossref","unstructured":"Do\u0161ilovi\u0107, F.K., Br\u010di\u0107, M., Hlupi\u0107, N.: Explainable artificial intelligence: a survey. In: 2018 41st International Convention on Information and Communication Technology, Electronics and Microelectronics (MIPRO), pp. 0210\u20130215. IEEE (2018)","DOI":"10.23919\/MIPRO.2018.8400040"},{"key":"24_CR12","unstructured":"Dosovitskiy, A., et al.: An image is worth 16x16 words: transformers for image recognition at scale. arXiv preprint arXiv:2010.11929 (2020)"},{"key":"24_CR13","doi-asserted-by":"crossref","unstructured":"Everett, G.D., McLeod Jr., R.: Software testing. Testing Across the Entire (2007)","DOI":"10.1002\/9780470146354"},{"key":"24_CR14","unstructured":"Goodfellow, I.J., Shlens, J., Szegedy, C.: Explaining and harnessing adversarial examples. arXiv preprint arXiv:1412.6572 (2014)"},{"key":"24_CR15","doi-asserted-by":"publisher","DOI":"10.1007\/s10009-021-00635-x","author":"F Gossen","year":"2021","unstructured":"Gossen, F., Steffen, B.: Algebraic aggregation of random forests: towards explainability and rapid evaluation. Int. J. Softw. Tools Technol. Transf. (2021). https:\/\/doi.org\/10.1007\/s10009-021-00635-x","journal-title":"Int. J. Softw. Tools Technol. Transf."},{"key":"24_CR16","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"96","DOI":"10.1007\/978-3-030-50086-3_6","volume-title":"Formal Techniques for Distributed Objects, Components, and Systems","author":"TP Gros","year":"2020","unstructured":"Gros, T.P., Hermanns, H., Hoffmann, J., Klauck, M., Steinmetz, M.: Deep statistical model checking. In: Gotsman, A., Sokolova, A. (eds.) FORTE 2020. LNCS, vol. 12136, pp. 96\u2013114. Springer, Cham (2020). https:\/\/doi.org\/10.1007\/978-3-030-50086-3_6"},{"key":"24_CR17","doi-asserted-by":"publisher","unstructured":"Guidotti, R., Monreale, A., Ruggieri, S., Turini, F., Giannotti, F., Pedreschi, D.: A survey of methods for explaining black box models. ACM Comput. Surv. 51(5) (2018). https:\/\/doi.org\/10.1145\/3236009","DOI":"10.1145\/3236009"},{"key":"24_CR18","doi-asserted-by":"crossref","unstructured":"Hartmanns, A., Klauck, M.: The modest state of learning, sampling, and verifying strategies. In: Margaria, T., Steffen, B. (eds.) ISoLA 2022. LNCS, vol. 13703, pp. 406\u2013432. Springer, Cham (2022)","DOI":"10.1007\/978-3-031-19759-8_25"},{"issue":"8","key":"24_CR19","doi-asserted-by":"publisher","first-page":"1735","DOI":"10.1162\/neco.1997.9.8.1735","volume":"9","author":"S Hochreiter","year":"1997","unstructured":"Hochreiter, S., Schmidhuber, J.: Long short-term memory. Neural Comput. 9(8), 1735\u20131780 (1997)","journal-title":"Neural Comput."},{"key":"24_CR20","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"163","DOI":"10.1007\/978-3-319-47677-3_11","volume-title":"Dependable Software Engineering: Theories, Tools, and Applications","author":"C Jegourel","year":"2016","unstructured":"Jegourel, C., Larsen, K.G., Legay, A., Miku\u010dionis, M., Poulsen, D.B., Sedwards, S.: Importance sampling for stochastic timed automata. In: Fr\u00e4nzle, M., Kapur, D., Zhan, N. (eds.) SETTA 2016. LNCS, vol. 9984, pp. 163\u2013178. Springer, Cham (2016). https:\/\/doi.org\/10.1007\/978-3-319-47677-3_11"},{"key":"24_CR21","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"576","DOI":"10.1007\/978-3-642-39799-8_38","volume-title":"Computer Aided Verification","author":"C Jegourel","year":"2013","unstructured":"Jegourel, C., Legay, A., Sedwards, S.: Importance splitting for statistical model checking rare properties. In: Sharygina, N., Veith, H. (eds.) CAV 2013. LNCS, vol. 8044, pp. 576\u2013591. Springer, Heidelberg (2013). https:\/\/doi.org\/10.1007\/978-3-642-39799-8_38"},{"key":"24_CR22","doi-asserted-by":"crossref","unstructured":"J\u00fcngermann, F., Kret\u00ednsk\u00fd, J., Weininger, M.: Algebraically explainable controllers: decision trees and support vector machines join forces. Int. J. Softw. Tools Technol. Transf. (2022, to appear)","DOI":"10.1007\/s10009-023-00716-z"},{"key":"24_CR23","doi-asserted-by":"publisher","first-page":"237","DOI":"10.1613\/jair.301","volume":"4","author":"LP Kaelbling","year":"1996","unstructured":"Kaelbling, L.P., Littman, M.L., Moore, A.W.: Reinforcement learning: a survey. J. Artif. Intell. Res. 4, 237\u2013285 (1996)","journal-title":"J. Artif. Intell. Res."},{"key":"24_CR24","doi-asserted-by":"crossref","unstructured":"Khmelnitsky, I., et al.: Analysis of recurrent neural networks via property-directed verification of surrogate models. Int. J. Softw. Tools Technol. Transf. (2022, to appear)","DOI":"10.1007\/s10009-022-00684-w"},{"issue":"7","key":"24_CR25","doi-asserted-by":"publisher","first-page":"385","DOI":"10.1145\/360248.360252","volume":"19","author":"JC King","year":"1976","unstructured":"King, J.C.: Symbolic execution and program testing. Commun. ACM 19(7), 385\u2013394 (1976)","journal-title":"Commun. ACM"},{"key":"24_CR26","series-title":"Lecture Notes in Networks and Systems","doi-asserted-by":"publisher","first-page":"261","DOI":"10.1007\/978-3-030-12388-8_19","volume-title":"Advances in Information and Communication","author":"P Kohli","year":"2020","unstructured":"Kohli, P., Chadha, A.: Enabling pedestrian safety using computer vision techniques: a case study of the 2018 Uber Inc. self-driving car crash. In: Arai, K., Bhatia, R. (eds.) FICC 2019. LNNS, vol. 69, pp. 261\u2013279. Springer, Cham (2020). https:\/\/doi.org\/10.1007\/978-3-030-12388-8_19"},{"key":"24_CR27","doi-asserted-by":"crossref","unstructured":"Larsen, K.G., Legay, A., Miku\u010dionis, M., Poulse, D.B.: Importance splitting in uppaal. In: Margaria, T., Steffen, B. (eds.) ISoLA 2022. LNCS, vol. 13703, pp. 433\u2013447. Springer, Cham (2022)","DOI":"10.1007\/978-3-031-19759-8_26"},{"key":"24_CR28","doi-asserted-by":"crossref","unstructured":"Lazreg, S., Cordy, M., Legay, A.: Verification of variability-intensive stochastic systems with statistical model checking. In: Margaria, T., Steffen, B. (eds.) ISoLA 2022. LNCS, vol. 13703, pp. 448\u2013471. Springer, Cham (2022)","DOI":"10.1007\/978-3-031-19759-8_27"},{"key":"24_CR29","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"122","DOI":"10.1007\/978-3-642-16612-9_11","volume-title":"Runtime Verification","author":"A Legay","year":"2010","unstructured":"Legay, A., Delahaye, B., Bensalem, S.: Statistical model checking: an overview. In: Barringer, H., et al. (eds.) RV 2010. LNCS, vol. 6418, pp. 122\u2013135. Springer, Heidelberg (2010). https:\/\/doi.org\/10.1007\/978-3-642-16612-9_11"},{"key":"24_CR30","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"478","DOI":"10.1007\/978-3-319-91908-9_23","volume-title":"Computing and Software Science","author":"A Legay","year":"2019","unstructured":"Legay, A., Lukina, A., Traonouez, L.M., Yang, J., Smolka, S.A., Grosu, R.: Statistical model checking. In: Steffen, B., Woeginger, G. (eds.) Computing and Software Science. LNCS, vol. 10000, pp. 478\u2013504. Springer, Cham (2019). https:\/\/doi.org\/10.1007\/978-3-319-91908-9_23"},{"issue":"6","key":"24_CR31","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1145\/3457607","volume":"54","author":"N Mehrabi","year":"2021","unstructured":"Mehrabi, N., Morstatter, F., Saxena, N., Lerman, K., Galstyan, A.: A survey on bias and fairness in machine learning. ACM Comput. Surv. (CSUR) 54(6), 1\u201335 (2021)","journal-title":"ACM Comput. Surv. (CSUR)"},{"key":"24_CR32","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"330","DOI":"10.1007\/3-540-48294-6_22","volume-title":"Static Analysis","author":"M M\u00fcller-Olm","year":"1999","unstructured":"M\u00fcller-Olm, M., Schmidt, D., Steffen, B.: Model-checking. In: Cortesi, A., Fil\u00e9, G. (eds.) SAS 1999. LNCS, vol. 1694, pp. 330\u2013354. Springer, Heidelberg (1999). https:\/\/doi.org\/10.1007\/3-540-48294-6_22"},{"key":"24_CR33","unstructured":"Murthy, S.K.: Automatic construction of decision trees from data: a multi-disciplinary survey. Data Min. Knowl. Discov. 2(4), 345\u2013389 (1998)"},{"key":"24_CR34","doi-asserted-by":"crossref","unstructured":"Murtovi, A., Bainczyk, A., Nolte, G., Schl\u00fcter, M., Bernhard, S.: Forest gump: a tool for veriification and explanation. Int. J. Softw. Tools Technol. Transf. (2022, to appear)","DOI":"10.1007\/978-3-030-99527-0_17"},{"issue":"12","key":"24_CR35","doi-asserted-by":"publisher","first-page":"1565","DOI":"10.1038\/nbt1206-1565","volume":"24","author":"WS Noble","year":"2006","unstructured":"Noble, W.S.: What is a support vector machine? Nat. Biotechnol. 24(12), 1565\u20131567 (2006)","journal-title":"Nat. Biotechnol."},{"key":"24_CR36","doi-asserted-by":"crossref","unstructured":"Rao, Q., Frtunikj, J.: Deep learning for self-driving cars: chances and challenges. In: Proceedings of the 1st International Workshop on Software Engineering for AI in Autonomous Systems, pp. 35\u201338 (2018)","DOI":"10.1145\/3194085.3194087"},{"key":"24_CR37","doi-asserted-by":"publisher","unstructured":"Rodrigues, G.N., et al.: Modeling and verification for probabilistic properties in software product lines. In: 16th IEEE International Symposium on High Assurance Systems Engineering, HASE 2015, Daytona Beach, FL, USA, 8\u201310, January 2015, pp. 173\u2013180. IEEE Computer Society (2015). https:\/\/doi.org\/10.1109\/HASE.2015.34","DOI":"10.1109\/HASE.2015.34"},{"key":"24_CR38","doi-asserted-by":"crossref","unstructured":"Schl\u00fcter, M., Nolte, G., Steffen, B.: Towards rigorous understanding of neural networks via semantics preserving transformation. Int. J. Softw. Tools Technol. Transf. (2022, to appear)","DOI":"10.1007\/s10009-023-00700-7"},{"key":"24_CR39","doi-asserted-by":"crossref","unstructured":"Silver, D., et al.: Mastering the game of go without human knowledge. Nature 550(7676), 354\u2013359 (2017)","DOI":"10.1038\/nature24270"},{"key":"24_CR40","unstructured":"Simonyan, K., Zisserman, A.: Very deep convolutional networks for large-scale image recognition. arXiv preprint arXiv:1409.1556 (2014)"},{"key":"24_CR41","unstructured":"Sutton, R.S., Barto, A.G.: Reinforcement Learning: An Introduction. MIT Press, Cambridge (2018)"},{"key":"24_CR42","unstructured":"Szegedy, C., et al.: Intriguing properties of neural networks. arXiv preprint arXiv:1312.6199 (2013)"},{"key":"24_CR43","doi-asserted-by":"crossref","unstructured":"Usman, M., Sun, Y., Gopinath, D., Dange, R., Manolache, L., Pasareanu, C.: An overview of structural coverage metrics for testing neural networks. Int. J. Softw. Tools Technol. Transf. (2022, to appear)","DOI":"10.1007\/s10009-022-00683-x"},{"key":"24_CR44","unstructured":"Vardi, M.Y., Wolper, P.: An automata-theoretic approach to automatic program verification. In: 1st Symposium in Logic in Computer Science (LICS). IEEE Computer Society (1986)"},{"issue":"4","key":"24_CR45","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1145\/1592434.1592436","volume":"41","author":"J Woodcock","year":"2009","unstructured":"Woodcock, J., Larsen, P.G., Bicarregui, J., Fitzgerald, J.: Formal methods: practice and experience. ACM Comput. Surv. (CSUR) 41(4), 1\u201336 (2009)","journal-title":"ACM Comput. Surv. (CSUR)"},{"key":"24_CR46","series-title":"Lecture Notes in Computer Science (Lecture Notes in Artificial Intelligence)","doi-asserted-by":"publisher","first-page":"563","DOI":"10.1007\/978-3-030-32236-6_51","volume-title":"Natural Language Processing and Chinese Computing","author":"F Xu","year":"2019","unstructured":"Xu, F., Uszkoreit, H., Du, Y., Fan, W., Zhao, D., Zhu, J.: Explainable AI: a brief survey on history, research areas, approaches and challenges. In: Tang, J., Kan, M.-Y., Zhao, D., Li, S., Zan, H. (eds.) NLPCC 2019. LNCS (LNAI), vol. 11839, pp. 563\u2013574. Springer, Cham (2019). https:\/\/doi.org\/10.1007\/978-3-030-32236-6_51"}],"container-title":["Lecture Notes in Computer Science","Leveraging Applications of Formal Methods, Verification and Validation. Adaptation and Learning"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-031-19759-8_24","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2023,11,29]],"date-time":"2023-11-29T03:45:06Z","timestamp":1701229506000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-3-031-19759-8_24"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2022]]},"ISBN":["9783031197581","9783031197598"],"references-count":46,"URL":"https:\/\/doi.org\/10.1007\/978-3-031-19759-8_24","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"value":"0302-9743","type":"print"},{"value":"1611-3349","type":"electronic"}],"subject":[],"published":{"date-parts":[[2022]]},"assertion":[{"value":"17 October 2022","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"ISoLA","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"International Symposium on Leveraging Applications of Formal Methods","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Rhodes","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Greece","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2022","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"22 October 2022","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"30 October 2022","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"11","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"isola2022","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"http:\/\/www.isola-conference.org\/","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}}]}}