{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,3,12]],"date-time":"2026-03-12T15:32:02Z","timestamp":1773329522849,"version":"3.50.1"},"publisher-location":"Cham","reference-count":77,"publisher":"Springer Nature Switzerland","isbn-type":[{"value":"9783031460012","type":"print"},{"value":"9783031460029","type":"electronic"}],"license":[{"start":{"date-parts":[[2023,12,14]],"date-time":"2023-12-14T00:00:00Z","timestamp":1702512000000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2023,12,14]],"date-time":"2023-12-14T00:00:00Z","timestamp":1702512000000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2024]]},"DOI":"10.1007\/978-3-031-46002-9_4","type":"book-chapter","created":{"date-parts":[[2023,12,13]],"date-time":"2023-12-13T16:02:36Z","timestamp":1702483356000},"page":"55-76","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":8,"title":["What, Indeed, is an\u00a0Achievable Provable Guarantee for\u00a0Learning-Enabled Safety-Critical Systems"],"prefix":"10.1007","author":[{"given":"Saddek","family":"Bensalem","sequence":"first","affiliation":[]},{"given":"Chih-Hong","family":"Cheng","sequence":"additional","affiliation":[]},{"given":"Wei","family":"Huang","sequence":"additional","affiliation":[]},{"given":"Xiaowei","family":"Huang","sequence":"additional","affiliation":[]},{"given":"Changshun","family":"Wu","sequence":"additional","affiliation":[]},{"given":"Xingyu","family":"Zhao","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2023,12,14]]},"reference":[{"key":"4_CR1","unstructured":"Kulstad, M., Carlin, L.: Leibniz\u2019s philosophy of mind (1997)"},{"issue":"37","key":"4_CR2","doi-asserted-by":"publisher","first-page":"eaay7120","DOI":"10.1126\/scirobotics.aay7120","volume":"4","author":"D Gunning","year":"2019","unstructured":"Gunning, D., Stefik, M., Choi, J., Miller, T., Stumpf, S., Yang, G.-Z.: Xai-explainable artificial intelligence. Sci. Rob. 4(37), eaay7120 (2019)","journal-title":"Sci. Rob."},{"issue":"1","key":"4_CR3","doi-asserted-by":"publisher","first-page":"1096","DOI":"10.1038\/s41467-019-08987-4","volume":"10","author":"S Lapuschkin","year":"2019","unstructured":"Lapuschkin, S., W\u00e4ldchen, S., Binder, A., Montavon, G., Samek, W., M\u00fcller, K.-R.: Unmasking clever hans predictors and assessing what machines really learn. Nat. Commun. 10(1), 1096 (2019)","journal-title":"Nat. Commun."},{"issue":"1","key":"4_CR4","doi-asserted-by":"publisher","DOI":"10.1002\/widm.1391","volume":"11","author":"R Confalonieri","year":"2021","unstructured":"Confalonieri, R., Coba, L., Wagner, B., Besold, T.R.: A historical perspective of explainable artificial intelligence. Wiley Interdisc. Rev. Data Min. Knowl. Disc. 11(1), e1391 (2021)","journal-title":"Wiley Interdisc. Rev. Data Min. Knowl. Disc."},{"key":"4_CR5","doi-asserted-by":"crossref","unstructured":"Do\u0161ilovi\u0107, F.K., Br\u010di\u0107, M., Hlupi\u0107, N., Explainable artificial intelligence: a survey. In: 41st International Convention on Information and Communication Technology, Electronics and Microelectronics (MIPRO), vol. 2018, pp. 0210\u20130215. IEEE (2018)","DOI":"10.23919\/MIPRO.2018.8400040"},{"key":"4_CR6","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"3","DOI":"10.1007\/978-3-319-63387-9_1","volume-title":"Computer Aided Verification","author":"X Huang","year":"2017","unstructured":"Huang, X., Kwiatkowska, M., Wang, S., Wu, M.: Safety verification of deep neural networks. In: Majumdar, R., Kun\u010dak, V. (eds.) CAV 2017. LNCS, vol. 10426, pp. 3\u201329. Springer, Cham (2017). https:\/\/doi.org\/10.1007\/978-3-319-63387-9_1"},{"key":"4_CR7","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"432","DOI":"10.1007\/978-3-030-25540-4_25","volume-title":"Computer Aided Verification","author":"T Dreossi","year":"2019","unstructured":"Dreossi, T., et al.: VerifAI: a toolkit for the formal design and analysis of artificial intelligence-based systems. In: Dillig, I., Tasiran, S. (eds.) CAV 2019. LNCS, vol. 11561, pp. 432\u2013442. Springer, Cham (2019). https:\/\/doi.org\/10.1007\/978-3-030-25540-4_25"},{"key":"4_CR8","doi-asserted-by":"publisher","first-page":"298","DOI":"10.1016\/j.tcs.2019.05.046","volume":"807","author":"M Wu","year":"2020","unstructured":"Wu, M., Wicker, M., Ruan, W., Huang, X., Kwiatkowska, M.: A game-based approximate verification of deep neural networks with provable guarantees. Theor. Comput. Sci. 807, 298\u2013329 (2020)","journal-title":"Theor. Comput. Sci."},{"issue":"3\u20134","key":"4_CR9","doi-asserted-by":"publisher","first-page":"244","DOI":"10.1561\/2400000035","volume":"4","author":"C Liu","year":"2021","unstructured":"Liu, C., et al.: Algorithms for verifying deep neural networks. Found. Trends\u00ae Optim. 4(3\u20134), 244\u2013404 (2021)","journal-title":"Found. Trends\u00ae Optim."},{"issue":"7","key":"4_CR10","doi-asserted-by":"publisher","first-page":"46","DOI":"10.1145\/3503914","volume":"65","author":"SA Seshia","year":"2022","unstructured":"Seshia, S.A., Sadigh, D., Sastry, S.S.: Toward verified artificial intelligence. Commun. ACM 65(7), 46\u201355 (2022)","journal-title":"Commun. ACM"},{"key":"4_CR11","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"79","DOI":"10.1007\/978-3-030-86362-3_7","volume-title":"Artificial Neural Networks and Machine Learning \u2013 ICANN 2021","author":"C Huang","year":"2021","unstructured":"Huang, C., Hu, Z., Huang, X., Pei, K.: Statistical certification of acceptable robustness for neural networks. In: Farka\u0161, I., Masulli, P., Otte, S., Wermter, S. (eds.) ICANN 2021. LNCS, vol. 12891, pp. 79\u201390. Springer, Cham (2021). https:\/\/doi.org\/10.1007\/978-3-030-86362-3_7"},{"key":"4_CR12","doi-asserted-by":"publisher","first-page":"154","DOI":"10.1007\/978-3-031-26409-2_10","volume-title":"Joint European Conference on Machine Learning and Knowledge Discovery in Databases","author":"T Zhang","year":"2022","unstructured":"Zhang, T., Ruan, W., Fieldsend, J.E.: Proa: a probabilistic robustness assessment against functional perturbations. In: Amini, M.R., Canu, S., Fischer, A., Guns, T., Kralj Novak, P., Tsoumakas, G. (eds.) ECML PKDD 2022. LNCS, vol. 13715, pp. 154\u2013170. Springer, Heidelberg (2022). https:\/\/doi.org\/10.1007\/978-3-031-26409-2_10"},{"key":"4_CR13","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"458","DOI":"10.1007\/978-3-319-99229-7_39","volume-title":"Computer Safety, Reliability, and Security","author":"S Shafaei","year":"2018","unstructured":"Shafaei, S., Kugele, S., Osman, M.H., Knoll, A.: Uncertainty in machine learning: a safety perspective on autonomous driving. In: Gallina, B., Skavhaug, A., Schoitsch, E., Bitsch, F. (eds.) SAFECOMP 2018. LNCS, vol. 11094, pp. 458\u2013464. Springer, Cham (2018). https:\/\/doi.org\/10.1007\/978-3-319-99229-7_39"},{"key":"4_CR14","unstructured":"Gawlikowski, J., et al.: A survey of uncertainty in deep neural networks. arXiv preprint arXiv:2107.03342 (2021)"},{"key":"4_CR15","doi-asserted-by":"publisher","first-page":"457","DOI":"10.1007\/s10994-021-05946-3","volume":"110","author":"E H\u00fcllermeier","year":"2021","unstructured":"H\u00fcllermeier, E., Waegeman, W.: Aleatoric and epistemic uncertainty in machine learning: an introduction to concepts and methods. Mach. Learn. 110, 457\u2013506 (2021)","journal-title":"Mach. Learn."},{"key":"4_CR16","unstructured":"Gruber, C., Schenk, P.O., Schierholz, M., Kreuter, F., Kauermann, G.: Sources of uncertainty in machine learning - a statisticians\u2019 view. arXiv:2305.16703 (2023)"},{"key":"4_CR17","doi-asserted-by":"crossref","unstructured":"Cheng, C.-H., N\u00fchrenberg, G., Yasuoka, H.: Runtime monitoring neuron activation patterns. In: Design, Automation & Test in Europe Conference & Exhibition (DATE), vol. 2019, pp. 300\u2013303. IEEE (2019)","DOI":"10.23919\/DATE.2019.8714971"},{"key":"4_CR18","unstructured":"Henzinger, T.A., Lukina, A., Schilling, C.: Outside the box: abstraction-based monitoring of neural networks. In: ECAI 2020, pp. 2433\u20132440. IOS Press (2020)"},{"key":"4_CR19","doi-asserted-by":"crossref","unstructured":"Cheng, C.-H.: Provably-robust runtime monitoring of neuron activation patterns. In: Design, Automation & Test in Europe Conference & Exhibition (DATE), vol. 2021, pp. 1310\u20131313. IEEE (2021)","DOI":"10.23919\/DATE51398.2021.9473957"},{"key":"4_CR20","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"42","DOI":"10.1007\/978-3-030-88494-9_3","volume-title":"Runtime Verification","author":"A Lukina","year":"2021","unstructured":"Lukina, A., Schilling, C., Henzinger, T.A.: Into the unknown: active monitoring of\u00a0neural networks. In: Feng, L., Fisman, D. (eds.) RV 2021. LNCS, vol. 12974, pp. 42\u201361. Springer, Cham (2021). https:\/\/doi.org\/10.1007\/978-3-030-88494-9_3"},{"key":"4_CR21","doi-asserted-by":"publisher","first-page":"397","DOI":"10.1007\/978-3-031-19992-9_26","volume-title":"International Symposium on Automated Technology for Verification and Analysis","author":"C-H Cheng","year":"2022","unstructured":"Cheng, C.-H., Wu, C., Seferis, E., Bensalem, S.: Prioritizing corners in OoD detectors via symbolic string manipulation. In: Bouajjani, A., Holik, L., Wu, Z. (eds.) ATVA 2022. LNCS, pp. 397\u2013413. Springer, Heidelberg (2022). https:\/\/doi.org\/10.1007\/978-3-031-19992-9_26"},{"key":"4_CR22","doi-asserted-by":"crossref","unstructured":"Fremont, D.J., Dreossi, T., Ghosh, S., Yue, X., Sangiovanni-Vincentelli, A.L., Seshia, S.A.: Scenic: a language for scenario specification and scene generation. In: Proceedings of the 40th ACM SIGPLAN Conference on Programming Language Design and Implementation, pp. 63\u201378 (2019)","DOI":"10.1145\/3314221.3314633"},{"issue":"19","key":"4_CR23","first-page":"12741","volume":"55","author":"S Zhong","year":"2021","unstructured":"Zhong, S., et al.: Machine learning: new ideas and tools in environmental science and engineering. Environ. Sci. Technol. 55(19), 12741\u201312754 (2021)","journal-title":"Environ. Sci. Technol."},{"key":"4_CR24","doi-asserted-by":"crossref","unstructured":"Brunton, S.L., Kutz, J.N.: Data-driven Science and Engineering: Machine Learning, Dynamical Systems, and Control. Cambridge University Press, Cambridge (2019)","DOI":"10.1017\/9781108380690"},{"key":"4_CR25","unstructured":"Zelaya, C.V.G.: Towards explaining the effects of data preprocessing on machine learning. In: IEEE 35th International Conference on Data Engineering (ICDE), vol. 2019, pp. 2086\u20132090. IEEE (2019)"},{"issue":"4","key":"4_CR26","doi-asserted-by":"publisher","first-page":"1328","DOI":"10.1109\/TKDE.2019.2946162","volume":"33","author":"Y Roh","year":"2019","unstructured":"Roh, Y., Heo, G., Whang, S.E.: A survey on data collection for machine learning: a big data-AI integration perspective. IEEE Trans. Knowl. Data Eng. 33(4), 1328\u20131347 (2019)","journal-title":"IEEE Trans. Knowl. Data Eng."},{"key":"4_CR27","doi-asserted-by":"crossref","unstructured":"Bensalem, S., et al.: Formal specification for learning-enabled autonomous systems. In: FoMLAS2022 (2022)","DOI":"10.1007\/978-3-031-21222-2_8"},{"issue":"2","key":"4_CR28","doi-asserted-by":"publisher","first-page":"14","DOI":"10.1109\/52.199724","volume":"10","author":"JD Musa","year":"1993","unstructured":"Musa, J.D.: Operational profiles in software-reliability engineering. IEEE Softw. 10(2), 14\u201332 (1993)","journal-title":"IEEE Softw."},{"key":"4_CR29","volume-title":"Introduction to Statistical Pattern Recognition","author":"K Fukunaga","year":"2013","unstructured":"Fukunaga, K.: Introduction to Statistical Pattern Recognition. Elsevier, Amsterdam (2013)"},{"key":"4_CR30","doi-asserted-by":"crossref","unstructured":"Nakkiran, P., Kaplun, G., Bansal, Y., Yang, T., Barak, B., Sutskever, I.: Deep double descent: where bigger models and more data hurt. In: International Conference on Learning Representations (2020)","DOI":"10.1088\/1742-5468\/ac3a74"},{"key":"4_CR31","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"296","DOI":"10.1007\/978-3-030-32304-2_15","volume-title":"Static Analysis","author":"J Li","year":"2019","unstructured":"Li, J., Liu, J., Yang, P., Chen, L., Huang, X., Zhang, L.: Analyzing deep neural networks with symbolic propagation: towards higher precision and faster verification. In: Chang, B.-Y.E. (ed.) SAS 2019. LNCS, vol. 11822, pp. 296\u2013319. Springer, Cham (2019). https:\/\/doi.org\/10.1007\/978-3-030-32304-2_15"},{"key":"4_CR32","doi-asserted-by":"crossref","unstructured":"Li, R., et al.: Prodeep: a platform for robustness verification of deep neural networks. In: Proceedings of the 28th ACM Joint Meeting on European Software Engineering Conference and Symposium on the Foundations of Software Engineering, ESEC\/FSE 2020, pp. 1630\u20131634. ACM, New York (2020)","DOI":"10.1145\/3368089.3417918"},{"issue":"3","key":"4_CR33","doi-asserted-by":"publisher","first-page":"407","DOI":"10.1007\/s00165-021-00548-1","volume":"33","author":"P Yang","year":"2021","unstructured":"Yang, P., et al.: Enhancing robustness verification for deep neural networks via symbolic propagation. Form. Asp. Comput. 33(3), 407\u2013435 (2021)","journal-title":"Form. Asp. Comput."},{"key":"4_CR34","doi-asserted-by":"crossref","unstructured":"Ruan, W., Huang, X., Kwiatkowska, M.: Reachability analysis of deep neural networks with provable guarantees. In: Proceedings of the 27th International Joint Conference on Artificial Intelligence, IJCAI-18, pp. 2651\u20132659 (2018)","DOI":"10.24963\/ijcai.2018\/368"},{"key":"4_CR35","doi-asserted-by":"crossref","unstructured":"Ruan, W., Wu, M., Sun, Y., Huang, X., Kroening, D., Kwiatkowska, M.: Global robustness evaluation of deep neural networks with provable guarantees for the hamming distance. In: Proceedings of the 28th International Joint Conference on Artificial Intelligence, IJCAI-19, pp. 5944\u20135952 (2019)","DOI":"10.24963\/ijcai.2019\/824"},{"key":"4_CR36","doi-asserted-by":"crossref","unstructured":"Xu, P., Ruan, W., Huang, X.: Quantifying safety risks of deep neural networks. In: Complex & Intelligent Systems (2022)","DOI":"10.1007\/s40747-022-00790-x"},{"issue":"32","key":"4_CR37","doi-asserted-by":"publisher","first-page":"15849","DOI":"10.1073\/pnas.1903070116","volume":"116","author":"M Belkin","year":"2019","unstructured":"Belkin, M., Hsu, D., Ma, S., Mandal, S.: Reconciling modern machine-learning practice and the classical bias-variance trade-off. Proc. Natl. Acad. Sci. 116(32), 15849\u201315854 (2019)","journal-title":"Proc. Natl. Acad. Sci."},{"key":"4_CR38","doi-asserted-by":"crossref","unstructured":"Huang, X., et al.: A survey of safety and trustworthiness of large language models through the lens of verification and validation. arXiv:2305.11391 (2023)","DOI":"10.1007\/s10462-024-10824-0"},{"issue":"5","key":"4_CR39","doi-asserted-by":"publisher","first-page":"1178","DOI":"10.1109\/TSE.2011.80","volume":"38","author":"B Littlewood","year":"2012","unstructured":"Littlewood, B., Rushby, J.: Reasoning about the reliability of diverse two-channel systems in which one channel is \u201cpossibly perfect\u2019\u2019. IEEE Trans. Softw. Eng. 38(5), 1178\u20131194 (2012)","journal-title":"IEEE Trans. Softw. Eng."},{"key":"4_CR40","doi-asserted-by":"crossref","unstructured":"Rushby, J.: Software verification and system assurance. In: 7th International Conference on Software Engineering and Formal Methods, pp. 3\u201310. IEEE, Hanoi (2009)","DOI":"10.1109\/SEFM.2009.39"},{"key":"4_CR41","doi-asserted-by":"publisher","first-page":"230","DOI":"10.1016\/j.ress.2016.09.002","volume":"158","author":"X Zhao","year":"2017","unstructured":"Zhao, X., Littlewood, B., Povyakalo, A., Strigini, L., Wright, D.: Modeling the probability of failure on demand (pfd) of a 1-out-of-2 system in which one channel is \u201cquasi-perfect\u2019\u2019. Reliabil. Eng. Syst. Safety 158, 230\u2013245 (2017)","journal-title":"Reliabil. Eng. Syst. Safety"},{"key":"4_CR42","doi-asserted-by":"crossref","unstructured":"Huang, W., Zhao, X., Jin, G., Huang, X.: Safari: versatile and efficient evaluations for robustness of interpretability. In: International Conference on Computer Vision (ICCV 2023) (2023)","DOI":"10.1109\/ICCV51070.2023.00190"},{"key":"4_CR43","doi-asserted-by":"crossref","unstructured":"Dutle, A., et al.: Pressburger, from requirements to autonomous flight: an overview of the monitoring ICAROUS project. In: Proceedings of 2nd Workshop on Formal Methods for Autonomous Systems, vol. 329 of EPTCS, pp. 23\u201330 (2020)","DOI":"10.4204\/EPTCS.329.3"},{"key":"4_CR44","doi-asserted-by":"publisher","unstructured":"Balakrishnan, A., et al.: Specifying and evaluating quality metrics for vision-based perception systems. In: Design, Automation & Test in Europe Conference & Exhibition (DATE), pp. 1433\u20131438 (2019). https:\/\/doi.org\/10.23919\/DATE.2019.8715114","DOI":"10.23919\/DATE.2019.8715114"},{"key":"4_CR45","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"297","DOI":"10.1007\/978-3-030-88494-9_18","volume-title":"Runtime Verification","author":"A Balakrishnan","year":"2021","unstructured":"Balakrishnan, A., Deshmukh, J., Hoxha, B., Yamaguchi, T., Fainekos, G.: PerceMon: online monitoring for\u00a0perception systems. In: Feng, L., Fisman, D. (eds.) RV 2021. LNCS, vol. 12974, pp. 297\u2013308. Springer, Cham (2021). https:\/\/doi.org\/10.1007\/978-3-030-88494-9_18"},{"issue":"3","key":"4_CR46","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1145\/3570918","volume":"22","author":"Y Dong","year":"2023","unstructured":"Dong, Y., et al.: Reliability assessment and safety arguments for machine learning components in system assurance. ACM Trans. Embedded Comput. Syst. 22(3), 1\u201348 (2023)","journal-title":"ACM Trans. Embedded Comput. Syst."},{"key":"4_CR47","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1007\/978-3-031-17244-1_1","volume-title":"Formal Methods and Software Engineering","author":"X Huang","year":"2022","unstructured":"Huang, X., Ruan, W., Tang, Q., Zhao, X.: Bridging formal methods and machine learning with global optimisation. In: Riesco, A., Zhang, M. (eds.) ICFEM 2022. LNCS, vol. 13478, pp. 1\u201319. Springer, Cham (2022). https:\/\/doi.org\/10.1007\/978-3-031-17244-1_1"},{"key":"4_CR48","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"244","DOI":"10.1007\/978-3-030-54549-9_16","volume-title":"Computer Safety, Reliability, and Security","author":"X Zhao","year":"2020","unstructured":"Zhao, X., et al.: A safety framework for critical systems utilising deep neural networks. In: Casimiro, A., Ortmeier, F., Bitsch, F., Ferreira, P. (eds.) SAFECOMP 2020. LNCS, vol. 12234, pp. 244\u2013259. Springer, Cham (2020). https:\/\/doi.org\/10.1007\/978-3-030-54549-9_16"},{"key":"4_CR49","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"97","DOI":"10.1007\/978-3-319-63387-9_5","volume-title":"Computer Aided Verification","author":"G Katz","year":"2017","unstructured":"Katz, G., Barrett, C., Dill, D.L., Julian, K., Kochenderfer, M.J.: Reluplex: an efficient SMT solver for verifying deep neural networks. In: Majumdar, R., Kun\u010dak, V. (eds.) CAV 2017. LNCS, vol. 10426, pp. 97\u2013117. Springer, Cham (2017). https:\/\/doi.org\/10.1007\/978-3-319-63387-9_5"},{"key":"4_CR50","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"269","DOI":"10.1007\/978-3-319-68167-2_19","volume-title":"Automated Technology for Verification and Analysis","author":"R Ehlers","year":"2017","unstructured":"Ehlers, R.: Formal verification of piece-wise linear feed-forward neural networks. In: D\u2019Souza, D., Narayan Kumar, K. (eds.) ATVA 2017. LNCS, vol. 10482, pp. 269\u2013286. Springer, Cham (2017). https:\/\/doi.org\/10.1007\/978-3-319-68167-2_19"},{"key":"4_CR51","doi-asserted-by":"crossref","unstructured":"Narodytska, N.: Formal analysis of deep binarized neural networks. In: IJCAI, pp. 5692\u20135696 (2018)","DOI":"10.24963\/ijcai.2018\/811"},{"key":"4_CR52","doi-asserted-by":"crossref","unstructured":"Narodytska, N., Kasiviswanathan, S., Ryzhyk, L., Sagiv, M., Walsh, T.: Verifying properties of binarized deep neural networks. In: Proceedings of the AAAI Conference on Artificial Intelligence, vol. 32 (2018)","DOI":"10.1609\/aaai.v32i1.12206"},{"key":"4_CR53","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"279","DOI":"10.1007\/978-3-030-03592-1_16","volume-title":"Verified Software. Theories, Tools, and Experiments","author":"C-H Cheng","year":"2018","unstructured":"Cheng, C.-H., N\u00fchrenberg, G., Huang, C.-H., Ruess, H.: Verification of binarized neural networks via inter-neuron factoring. In: Piskac, R., R\u00fcmmer, P. (eds.) VSTTE 2018. LNCS, vol. 11294, pp. 279\u2013290. Springer, Cham (2018). https:\/\/doi.org\/10.1007\/978-3-030-03592-1_16"},{"key":"4_CR54","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"251","DOI":"10.1007\/978-3-319-68167-2_18","volume-title":"Automated Technology for Verification and Analysis","author":"C-H Cheng","year":"2017","unstructured":"Cheng, C.-H., N\u00fchrenberg, G., Ruess, H.: Maximum resilience of artificial neural networks. In: D\u2019Souza, D., Narayan Kumar, K. (eds.) ATVA 2017. LNCS, vol. 10482, pp. 251\u2013268. Springer, Cham (2017). https:\/\/doi.org\/10.1007\/978-3-319-68167-2_18"},{"key":"4_CR55","unstructured":"Lomuscio, A., Maganti, L.: An approach to reachability analysis for feed-forward relu neural networks. arXiv preprint arXiv:1706.07351 (2017)"},{"key":"4_CR56","doi-asserted-by":"crossref","unstructured":"Gehr, T., Mirman, M., Drachsler-Cohen, D., Tsankov, P., Chaudhuri, S., Vechev, M.: Ai2: safety and robustness certification of neural networks with abstract interpretation. In: IEEE Symposium on Security and Privacy (SP), vol. 2018, pp. 3\u201318. IEEE (2018)","DOI":"10.1109\/SP.2018.00058"},{"key":"4_CR57","unstructured":"Mirman, M., Gehr, T., Vechev, M.: Differentiable abstract interpretation for provably robust neural networks. In: International Conference on Machine Learning, pp. 3575\u20133583 (2018)"},{"key":"4_CR58","unstructured":"Wong, E., Kolter, Z.: Provable defenses against adversarial examples via the convex outer adversarial polytope. In: International Conference on Machine Learning, pp. 5283\u20135292 (2018)"},{"key":"4_CR59","unstructured":"Dvijotham, K., Stanforth, R., Gowal, S., Mann, T.A., Kohli, P.: A dual approach to scalable verification of deep networks. In: UAI, vol. 1, p. 3 (2018)"},{"key":"4_CR60","unstructured":"Wang, S., Pei, K., Whitehouse, J., Yang, J., Jana, S.: Formal security analysis of neural networks using symbolic intervals. In: 27th $$\\{$$USENIX$$\\}$$ Security Symposium ($$\\{$$USENIX$$\\}$$ Security 2018), pp. 1599\u20131614 (2018)"},{"key":"4_CR61","unstructured":"Peck, J., Roels, J., Goossens, B., Saeys, Y.: Lower bounds on the robustness to adversarial perturbations. Adv. Neural Inf. Process. Syst. 30 (2017)"},{"key":"4_CR62","doi-asserted-by":"publisher","first-page":"283","DOI":"10.1007\/s10107-003-0433-3","volume":"99","author":"A Neumaier","year":"2004","unstructured":"Neumaier, A., Shcherbina, O.: Safe bounds in linear and mixed-integer linear programming. Math. Program. 99, 283\u2013296 (2004)","journal-title":"Math. Program."},{"key":"4_CR63","doi-asserted-by":"crossref","unstructured":"Ruan, W., Huang, X., Kwiatkowska, M.: Reachability analysis of deep neural networks with provable guarantees. arXiv preprint arXiv:1805.02242 (2018)","DOI":"10.24963\/ijcai.2018\/368"},{"key":"4_CR64","unstructured":"Weng, T.-W., et al.: Evaluating the robustness of neural networks: an extreme value theory approach. In: ICLR 2018 (2018)"},{"key":"4_CR65","unstructured":"Webb, S., Rainforth, T., Teh, Y.W., Kumar, M.P.: A statistical approach to assessing neural network robustness. In: International Conference on Learning Representations (2018)"},{"key":"4_CR66","unstructured":"Wang, B., Webb, S., Rainforth, T.: Statistically robust neural network classification. In: Uncertainty in Artificial Intelligence, pp. 1735\u20131745. PMLR (2021)"},{"key":"4_CR67","unstructured":"Zhao, X., et al.: Assessing the reliability of deep learning classifiers through robustness evaluation and operational profiles. In: Workshop on AI Safety at IJCAI-21 (2021)"},{"key":"4_CR68","doi-asserted-by":"crossref","unstructured":"Pei, K., Cao, Y., Yang, J., Jana, S.: Deepxplore: automated whitebox testing of deep learning systems. In: proceedings of the 26th Symposium on Operating Systems Principles, pp. 1\u201318 (2017)","DOI":"10.1145\/3132747.3132785"},{"key":"4_CR69","doi-asserted-by":"crossref","unstructured":"Ma, L., et al.: DeepGauge: comprehensive and multi-granularity testing criteria for gauging the robustness of deep learning systems. In: 33rd IEEE\/ACM International Conference on Automated Software Engineering (ASE) (2018)","DOI":"10.1145\/3238147.3238202"},{"key":"4_CR70","doi-asserted-by":"crossref","unstructured":"Sun, Y., Wu, M., Ruan, W., Huang, X., Kwiatkowska, M., Kroening, D.: Deepconcolic: testing and debugging deep neural networks. In: ICSE 2019 (2019)","DOI":"10.1109\/ICSE-Companion.2019.00051"},{"key":"4_CR71","doi-asserted-by":"crossref","unstructured":"Du, X., Xie, X., Li, Y., Ma, L., Liu, Y., Zhao, J.: Deepstellar: model-based quantitative analysis of stateful deep learning systems. In: Proceedings of the 27th ACM Joint Meeting on European Software Engineering Conference and Symposium on the Foundations of Software Engineering, pp. 477\u2013487 (2019)","DOI":"10.1145\/3338906.3338954"},{"issue":"3","key":"4_CR72","doi-asserted-by":"publisher","first-page":"1191","DOI":"10.1109\/TR.2021.3080664","volume":"71","author":"W Huang","year":"2021","unstructured":"Huang, W., et al.: Coverage-guided testing for recurrent neural networks. IEEE Trans. Reliab. 71(3), 1191\u20131206 (2021)","journal-title":"IEEE Trans. Reliab."},{"key":"4_CR73","doi-asserted-by":"crossref","unstructured":"Berend, D.: Distribution awareness for AI system testing. In: 2021 IEEE\/ACM 43rd International Conference on Software Engineering: Companion Proceedings (ICSE-Companion), pp. 96\u201398. IEEE (2021)","DOI":"10.1109\/ICSE-Companion52605.2021.00045"},{"key":"4_CR74","doi-asserted-by":"crossref","unstructured":"Dola, S., Dwyer, M.B., Soffa, M.L.: Distribution-aware testing of neural networks using generative models. In: 2021 IEEE\/ACM 43rd International Conference on Software Engineering (ICSE), pp. 226\u2013237. IEEE (2021)","DOI":"10.1109\/ICSE43902.2021.00032"},{"key":"4_CR75","doi-asserted-by":"crossref","unstructured":"Byun, T., Vijayakumar, A., Rayadurgam, S., Cofer, D.: Manifold-based test generation for image classifiers. In: IEEE International Conference on Artificial Intelligence Testing (AITest), vol. 2020, pp. 15\u201322. IEEE (2020)","DOI":"10.1109\/AITEST49225.2020.00010"},{"key":"4_CR76","doi-asserted-by":"crossref","unstructured":"Toledo, F., Shriver, D., Elbaum, S., Dwyer, M.B.: Distribution models for falsification and verification of dnns. In: 2021 36th IEEE\/ACM International Conference on Automated Software Engineering (ASE), pp. 317\u2013329. IEEE (2021)","DOI":"10.1109\/ASE51524.2021.9678590"},{"key":"4_CR77","doi-asserted-by":"crossref","unstructured":"Huang, W., Zhao, X., Banks, A., Cox, V., Huang, X.: Hierarchical distribution-aware testing of deep learning. arXiv preprint arXiv:2205.08589 (2022)","DOI":"10.1145\/3625290"}],"container-title":["Lecture Notes in Computer Science","Bridging the Gap Between AI and Reality"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-031-46002-9_4","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,11,5]],"date-time":"2024-11-05T22:10:49Z","timestamp":1730844649000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-3-031-46002-9_4"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023,12,14]]},"ISBN":["9783031460012","9783031460029"],"references-count":77,"URL":"https:\/\/doi.org\/10.1007\/978-3-031-46002-9_4","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"value":"0302-9743","type":"print"},{"value":"1611-3349","type":"electronic"}],"subject":[],"published":{"date-parts":[[2023,12,14]]},"assertion":[{"value":"14 December 2023","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"AISoLA","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"International Conference on Bridging the Gap between AI and Reality","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Crete","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Greece","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2023","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"23 October 2023","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"28 October 2023","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"1","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"aisola2023","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"https:\/\/2023-aisola.isola-conference.org\/","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}}]}}