{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,6,21]],"date-time":"2025-06-21T22:40:08Z","timestamp":1750545608142,"version":"3.41.0"},"publisher-location":"Cham","reference-count":24,"publisher":"Springer International Publishing","isbn-type":[{"type":"print","value":"9783030791490"},{"type":"electronic","value":"9783030791506"}],"license":[{"start":{"date-parts":[[2021,1,1]],"date-time":"2021-01-01T00:00:00Z","timestamp":1609459200000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"},{"start":{"date-parts":[[2021,1,1]],"date-time":"2021-01-01T00:00:00Z","timestamp":1609459200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2021]]},"DOI":"10.1007\/978-3-030-79150-6_21","type":"book-chapter","created":{"date-parts":[[2021,6,21]],"date-time":"2021-06-21T23:35:17Z","timestamp":1624318517000},"page":"256-267","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":7,"title":["Robustness Testing of AI Systems: A Case Study for Traffic Sign Recognition"],"prefix":"10.1007","author":[{"given":"Christian","family":"Berghoff","sequence":"first","affiliation":[]},{"given":"Pavol","family":"Bielik","sequence":"additional","affiliation":[]},{"given":"Matthias","family":"Neu","sequence":"additional","affiliation":[]},{"given":"Petar","family":"Tsankov","sequence":"additional","affiliation":[]},{"given":"Arndt","family":"von Twickel","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2021,6,22]]},"reference":[{"key":"21_CR1","unstructured":"Balunovic, M., Baader, M., Singh, G., Gehr, T., Vechev, M.: Certifying geometric robustness of neural networks. In: Wallach, H., Larochelle, H., Beygelzimer, A., d\u2019Alch\u00e9 Buc, F., Fox, E., Garnett, R. (eds.) Advances in Neural Information Processing Systems, vol. 32. Curran Associates, Inc. (2019)"},{"key":"21_CR2","doi-asserted-by":"publisher","first-page":"23","DOI":"10.3389\/fdata.2020.00023","volume":"3","author":"C Berghoff","year":"2020","unstructured":"Berghoff, C., Neu, M., von Twickel, A.: Vulnerabilities of connectionist AI applications: evaluation and defense. Front. Big Data 3, 23 (2020). https:\/\/doi.org\/10.3389\/fdata.2020.00023","journal-title":"Front. Big Data"},{"key":"21_CR3","unstructured":"Bielik, P., Tsankov, P., Krause, A., Vechev, M.: Reliability assessment of traffic sign classifiers. Technica report, Bundesamt f\u00fcr Sicherheit in der Informationstechnik (2020). https:\/\/www.bsi.bund.de\/ki"},{"key":"21_CR4","doi-asserted-by":"publisher","first-page":"317","DOI":"10.1016\/j.patcog.2018.07.023","volume":"84","author":"B Biggio","year":"2018","unstructured":"Biggio, B., Roli, F.: Wild patterns: ten years after the rise of adversarial machine learning. Pattern Recognit. 84, 317\u2013331 (2018). https:\/\/doi.org\/10.1016\/j.patcog.2018.07.023","journal-title":"Pattern Recognit."},{"key":"21_CR5","unstructured":"Carlini, N., et al.: On evaluating adversarial robustness. CoRR abs\/1902.06705 (2019)"},{"key":"21_CR6","doi-asserted-by":"publisher","unstructured":"Dalvi, N.N., Domingos, P.M., Sanghai, S.K., Verma, D.: Adversarial classification. In: Kim, W., Kohavi, R., Gehrke, J., DuMouchel, W. (eds.) Proceedings of the Tenth ACM SIGKDD International Conference on Knowledge Discovery and Data Mining, pp. 99\u2013108. ACM (2004). https:\/\/doi.org\/10.1145\/1014052.1014066","DOI":"10.1145\/1014052.1014066"},{"key":"21_CR7","unstructured":"D\u2019Amour, A., et al.: Under specification presents challenges for credibility in modern machine learning. CoRR abs\/2011.03395 (2020)"},{"key":"21_CR8","doi-asserted-by":"publisher","unstructured":"Deng, J., Dong, W., Socher, R., Li, L., Li, K., Li, F.: ImageNet: a large-scale hierarchical image database. In: 2009 IEEE Computer Society Conference on Computer Vision and Pattern Recognition, pp. 248\u2013255. IEEE Computer Society (2009). https:\/\/doi.org\/10.1109\/CVPR.2009.5206848","DOI":"10.1109\/CVPR.2009.5206848"},{"key":"21_CR9","unstructured":"Engstrom, L., Tran, B., Tsipras, D., Schmidt, L., Madry, A.: Exploring the landscape of spatial robustness. In: Proceedings of the 36th International Conference on Machine Learning, vol. 97, pp. 1802\u20131811. PMLR (2019)"},{"key":"21_CR10","doi-asserted-by":"publisher","unstructured":"Fawzi, A., Moosavi-Dezfooli, S.M., Frossard, P., Soatto, S.: Empirical study of the topology and geometry of deep networks. In: 2018 IEEE\/CVF Conference on Computer Vision and Pattern Recognition, pp. 3762\u20133770 (2018). https:\/\/doi.org\/10.1109\/CVPR.2018.00396","DOI":"10.1109\/CVPR.2018.00396"},{"key":"21_CR11","doi-asserted-by":"publisher","unstructured":"Gehr, T., Mirman, M., Drachsler-Cohen, D., Tsankov, P., Chaudhuri, S., Vechev, M.T.: AI2: safety and robustness certification of neural networks with abstract interpretation. In: 2018 IEEE Symposium on Security and Privacy, pp. 3\u201318. IEEE Computer Society (2018). https:\/\/doi.org\/10.1109\/SP.2018.00058","DOI":"10.1109\/SP.2018.00058"},{"key":"21_CR12","doi-asserted-by":"publisher","first-page":"665","DOI":"10.1038\/s42256-020-00257-z","volume":"2","author":"R Geirhos","year":"2020","unstructured":"Geirhos, R., et al.: Shortcut learning in deep neural networks. Nature Mach. Intell. 2, 665\u2013673 (2020)","journal-title":"Nature Mach. Intell."},{"key":"21_CR13","doi-asserted-by":"publisher","unstructured":"Houben, S., Stallkamp, J., Salmen, J., Schlipsing, M., Igel, C.: Detection of traffic signs in real-world images: the German traffic sign detection benchmark. In: The 2013 International Joint Conference on Neural Networks, pp. 1\u20138. IEEE (2013). https:\/\/doi.org\/10.1109\/IJCNN.2013.6706807","DOI":"10.1109\/IJCNN.2013.6706807"},{"key":"21_CR14","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"3","DOI":"10.1007\/978-3-319-63387-9_1","volume-title":"Computer Aided Verification","author":"X Huang","year":"2017","unstructured":"Huang, X., Kwiatkowska, M., Wang, S., Wu, M.: Safety verification of deep neural networks. In: Majumdar, R., Kun\u010dak, V. (eds.) CAV 2017. LNCS, vol. 10426, pp. 3\u201329. Springer, Cham (2017). https:\/\/doi.org\/10.1007\/978-3-319-63387-9_1"},{"key":"21_CR15","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"97","DOI":"10.1007\/978-3-319-63387-9_5","volume-title":"Computer Aided Verification","author":"G Katz","year":"2017","unstructured":"Katz, G., Barrett, C., Dill, D.L., Julian, K., Kochenderfer, M.J.: Reluplex: an efficient SMT solver for verifying deep neural networks. In: Majumdar, R., Kun\u010dak, V. (eds.) CAV 2017. LNCS, vol. 10426, pp. 97\u2013117. Springer, Cham (2017). https:\/\/doi.org\/10.1007\/978-3-319-63387-9_5"},{"key":"21_CR16","doi-asserted-by":"publisher","DOI":"10.1049\/ipr2.12159","author":"Y Kim","year":"2021","unstructured":"Kim, Y., Hwang, H., Shin, J.: Robust object detection under harsh autonomous-driving environments. IET Image Process. (2021). https:\/\/doi.org\/10.1049\/ipr2.12159","journal-title":"IET Image Process."},{"key":"21_CR17","unstructured":"Michaelis, C., et al.: Benchmarking robustness in object detection: autonomous driving when winter is coming. CoRR abs\/1907.07484 (2019)"},{"issue":"13","key":"21_CR18","doi-asserted-by":"publisher","first-page":"3699","DOI":"10.3390\/s20133699","volume":"20","author":"T Ponn","year":"2020","unstructured":"Ponn, T., Kr\u00f6ger, T., Diermeyer, F.: Identification and explanation of challenging conditions for camera-based object detection of automated vehicles. Sensors 20(13), 3699 (2020). https:\/\/doi.org\/10.3390\/s20133699","journal-title":"Sensors"},{"key":"21_CR19","doi-asserted-by":"publisher","unstructured":"Singh, G., Gehr, T., P\u00fcschel, M., Vechev, M.T.: An abstract domain for certifying neural networks. In: Proceedings of ACM Program Language, 3(POPL), pp. 1\u201330 (2019). https:\/\/doi.org\/10.1145\/3290354","DOI":"10.1145\/3290354"},{"key":"21_CR20","doi-asserted-by":"publisher","unstructured":"Szegedy, C., Vanhoucke, V., Ioffe, S., Shlens, J., Wojna, Z.: Rethinking the inception architecture for computer vision. In: 2016 IEEE Conference on Computer Vision and Pattern Recognition, pp. 2818\u20132826. IEEE Computer Society (2016). https:\/\/doi.org\/10.1109\/CVPR.2016.308","DOI":"10.1109\/CVPR.2016.308"},{"key":"21_CR21","unstructured":"Szegedy, C., et al.: Intriguing properties of neural networks. In: Bengio, Y., LeCun, Y. (eds.) 2nd International Conference on Learning Representations (2014). http:\/\/arxiv.org\/abs\/1312.6199"},{"key":"21_CR22","doi-asserted-by":"publisher","unstructured":"Temel, D., Chen, M., AlRegib, G.: Traffic sign detection under challenging conditions: a deeper look into performance variations and spectral characteristics. IEEE Transactions on Intelligent Transportation Systems, pp. 1\u201311 (2019). https:\/\/doi.org\/10.1109\/TITS.2019.2931429","DOI":"10.1109\/TITS.2019.2931429"},{"key":"21_CR23","doi-asserted-by":"crossref","unstructured":"Temel, D., Kwon, G., Prabhushankar, M., AlRegib, G.: CURE-TSR: challenging unreal and real environments for traffic sign recognition. In: Neural Information Processing Systems (NeurIPS) Workshop on Machine Learning for Intelligent Transportation Systems (2017)","DOI":"10.1109\/ICMLA.2018.00028"},{"key":"21_CR24","unstructured":"Tramer, F., Carlini, N., Brendel, W., Madry, A.: On adaptive attacks to adversarial example defenses. In: Advances in Neural Information Processing Systems, vol. 33, pp. 1633\u20131645. Curran Associates, Inc. (2020)"}],"container-title":["IFIP Advances in Information and Communication Technology","Artificial Intelligence Applications and Innovations"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-030-79150-6_21","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,21]],"date-time":"2025-06-21T22:02:23Z","timestamp":1750543343000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-3-030-79150-6_21"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2021]]},"ISBN":["9783030791490","9783030791506"],"references-count":24,"URL":"https:\/\/doi.org\/10.1007\/978-3-030-79150-6_21","relation":{},"ISSN":["1868-4238","1868-422X"],"issn-type":[{"type":"print","value":"1868-4238"},{"type":"electronic","value":"1868-422X"}],"subject":[],"published":{"date-parts":[[2021]]},"assertion":[{"value":"22 June 2021","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"AIAI","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"IFIP International Conference on Artificial Intelligence Applications and Innovations","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Hersonissos, Crete","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Greece","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2021","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"25 June 2021","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"27 June 2021","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"17","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"aiai2021","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"http:\/\/www.aiai2021.eu\/","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Single-blind","order":1,"name":"type","label":"Type","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"easyacademia.org","order":2,"name":"conference_management_system","label":"Conference Management System","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"113","order":3,"name":"number_of_submissions_sent_for_review","label":"Number of Submissions Sent for Review","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"50","order":4,"name":"number_of_full_papers_accepted","label":"Number of Full Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"11","order":5,"name":"number_of_short_papers_accepted","label":"Number of Short Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"44% - The value is computed by the equation \"Number of Full Papers Accepted \/ Number of Submissions Sent for Review * 100\" and then rounded to a whole number.","order":6,"name":"acceptance_rate_of_full_papers","label":"Acceptance Rate of Full Papers","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"2.7","order":7,"name":"average_number_of_reviews_per_paper","label":"Average Number of Reviews per Paper","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"2.8","order":8,"name":"average_number_of_papers_per_reviewer","label":"Average Number of Papers per Reviewer","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"No","order":9,"name":"external_reviewers_involved","label":"External Reviewers Involved","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}}]}}