{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,9,10]],"date-time":"2026-09-10T23:14:13Z","timestamp":1789082053154,"version":"build-2803163510"},"publisher-location":"New York, NY, USA","reference-count":56,"publisher":"ACM","license":[{"start":{"date-parts":[[2022,5,21]],"date-time":"2022-05-21T00:00:00Z","timestamp":1653091200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"name":"NSERC"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2022,5,21]]},"DOI":"10.1145\/3510003.3510109","type":"proceedings-article","created":{"date-parts":[[2022,7,5]],"date-time":"2022-07-05T22:42:59Z","timestamp":1657060979000},"page":"1145-1156","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":21,"title":["If a human can see it, so should your system"],"prefix":"10.1145","author":[{"given":"Boyue Caroline","family":"Hu","sequence":"first","affiliation":[{"name":"University of Toronto, Toronto, Ontario, Canada"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Lina","family":"Marsso","sequence":"additional","affiliation":[{"name":"University of Toronto, Toronto, Ontario, Canada"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Krzysztof","family":"Czarnecki","sequence":"additional","affiliation":[{"name":"University of Waterloo, Waterloo, Ontario, Canada"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Rick","family":"Salay","sequence":"additional","affiliation":[{"name":"University of Waterloo, Waterloo, Ontario, Canada"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Huakun","family":"Shen","sequence":"additional","affiliation":[{"name":"University of Toronto, Toronto, Ontario, Canada"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Marsha","family":"Chechik","sequence":"additional","affiliation":[{"name":"University of Toronto, Toronto, Ontario, Canada"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2022,7,5]]},"reference":[{"key":"e_1_3_2_1_1_1","first-page":"1","volume":"12","author":"Standard IEEE","year":"1990","unstructured":"IEEE Standard Glossary of Software Engineering Terminology. IEEE Std 610.12-1990, pages 1--84, 1990.","journal-title":"IEEE Std 610."},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.1214\/aos\/1176350371"},{"key":"e_1_3_2_1_3_1","volume-title":"NeurIPS'16","author":"Bastani Osbert","year":"2016","unstructured":"Osbert Bastani, Yani A Ioannou, Leonidas Lampropoulos, Dimitrios Vytiniotis, Aditya V. Nori, and Antonio Criminisi. Measuring Neural Net Robustness with Constraints. In NeurIPS'16, 2016."},{"key":"e_1_3_2_1_4_1","volume-title":"August","author":"Boudette N","year":"2021","unstructured":"N Boudette. 'It Happened So Fast': Inside a Fatal Tesla Autopilot Accident. https:\/\/www.nytimes.com\/2021\/08\/17\/business\/tesla-autopilot-accident.html, August 2021."},{"key":"e_1_3_2_1_5_1","volume-title":"August","author":"Boudette N","year":"2021","unstructured":"N Boudette and N. Chokshi. U.S. Will Investigate Tesla's Autopilot System Over Crashes With Emergency Vehicles. https:\/\/www.nytimes.com\/2021\/08\/16\/business\/tesla-autopilot-nhtsa.html, August 2021."},{"key":"e_1_3_2_1_6_1","volume-title":"Albumentations: Fast and Flexible Image Augmentations. Information, 11(2)","author":"Buslaev Alexander","year":"2020","unstructured":"Alexander Buslaev, Vladimir I. Iglovikov, Eugene Khvedchenya, Alex Parinov, Mikhail Druzhinin, and Alexandr A. Kalinin. Albumentations: Fast and Flexible Image Augmentations. Information, 11(2), 2020."},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"publisher","DOI":"10.1109\/TIP.2007.901820"},{"key":"e_1_3_2_1_8_1","series-title":"CEUR Workshop Proceedings","first-page":"23","volume-title":"SafeAI@AAAI","author":"Picardi Chiara","year":"2020","unstructured":"Chiara Picardi and Colin Paterson and Richard Hawkins and Radu Calinescu and Ibrahim Habli. Assurance Argument Patterns and Processes for Machine Learning in Safety-Related Systems. In SafeAI@AAAI, volume 2560 of CEUR Workshop Proceedings, pages 23--30. CEUR-WS.org, 2020."},{"key":"e_1_3_2_1_9_1","volume-title":"Certified Adversarial Robustness via Randomized Smoothing. ArXiv, abs\/1902.02918","author":"Cohen Jeremy M.","year":"2019","unstructured":"Jeremy M. Cohen, Elan Rosenfeld, and J. Zico Kolter. Certified Adversarial Robustness via Randomized Smoothing. ArXiv, abs\/1902.02918, 2019."},{"key":"e_1_3_2_1_10_1","volume-title":"RobustBench: a standardized adversarial robustness benchmark. arXiv preprint arXiv:2010.09670","author":"Croce Francesco","year":"2020","unstructured":"Francesco Croce, Maksym Andriushchenko, Vikash Sehwag, Edoardo Debenedetti, Nicolas Flammarion, Mung Chiang, Prateek Mittal, and Matthias Hein. RobustBench: a standardized adversarial robustness benchmark. arXiv preprint arXiv:2010.09670, 2020."},{"key":"e_1_3_2_1_11_1","first-page":"321","volume-title":"Jun Zhu. Benchmarking Adversarial Robustness on Image Classification. In Proc. of CVPR'20","author":"Dong Yinpeng","year":"2020","unstructured":"Yinpeng Dong, Qi-An Fu, Xiao Yang, Tianyu Pang, Hang Su, Zihao Xiao, and Jun Zhu. Benchmarking Adversarial Robustness on Image Classification. In Proc. of CVPR'20, pages 321--331, 2020."},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.1145\/3213846.3213858"},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1201\/9780429246593"},{"key":"e_1_3_2_1_14_1","first-page":"26562","volume-title":"Performance vs. Competence in Human-Machine Comparisons","author":"Firestone Chaz","year":"2020","unstructured":"Chaz Firestone. Performance vs. Competence in Human-Machine Comparisons. volume 117, pages 26562--26571. National Academy of Sciences, 2020."},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-54549-9_13"},{"key":"e_1_3_2_1_16_1","first-page":"7549","volume-title":"FA Wichmann. Generalisation in Humans and Deep Neural Networks. In Proc. of NeurIPS'18","author":"Geirhos R","year":"2019","unstructured":"R Geirhos, CR Medina Temme, J Rauber, HH Sch\u00fctt, M Bethge, and FA Wichmann. Generalisation in Humans and Deep Neural Networks. In Proc. of NeurIPS'18, pages 7549--7561. Curran, 2019."},{"key":"e_1_3_2_1_17_1","first-page":"770","volume-title":"Jian Sun. Deep Residual Learning for Image Recognition. 2016 IEEE Conference on Computer Vision and Pattern Recognition (CVPR)","author":"He Kaiming","year":"2016","unstructured":"Kaiming He, X. Zhang, Shaoqing Ren, and Jian Sun. Deep Residual Learning for Image Recognition. 2016 IEEE Conference on Computer Vision and Pattern Recognition (CVPR), pages 770--778, 2016."},{"key":"e_1_3_2_1_18_1","volume-title":"Samyak Parajuli, Michael Guo, D. Song, J. Steinhardt, and J. Gilmer. The many faces of robustness: A critical analysis of out-of-distribution generalization. ArXiv, abs\/2006.16241","author":"Hendrycks Dan","year":"2020","unstructured":"Dan Hendrycks, Steven Basart, Norman Mu, Saurav Kadavath, Frank Wang, Evan Dorundo, Rahul Desai, Tyler Lixuan Zhu, Samyak Parajuli, Michael Guo, D. Song, J. Steinhardt, and J. Gilmer. The many faces of robustness: A critical analysis of out-of-distribution generalization. ArXiv, abs\/2006.16241, 2020."},{"key":"e_1_3_2_1_19_1","volume-title":"Proc. of ICLR'19","author":"Hendrycks Dan","year":"2019","unstructured":"Dan Hendrycks and Thomas Dietterich. Benchmarking Neural Network Robustness to Common Corruptions and Perturbations. Proc. of ICLR'19, 2019."},{"key":"e_1_3_2_1_20_1","volume-title":"AugMix: A Simple Data Processing Method to Improve Robustness and Uncertainty. ArXiv, abs\/1912.02781","author":"Hendrycks Dan","year":"2020","unstructured":"Dan Hendrycks, Norman Mu, E. D. Cubuk, Barret Zoph, J. Gilmer, and Balaji Lakshminarayanan. AugMix: A Simple Data Processing Method to Improve Robustness and Uncertainty. ArXiv, abs\/1912.02781, 2020."},{"key":"e_1_3_2_1_21_1","volume-title":"CIFAR10 to Compare Visual Recognition Performance between Deep Neural Networks and Humans. ArXiv, abs\/1811.07270","author":"Ho-Phuoc T.","year":"2018","unstructured":"T. Ho-Phuoc. CIFAR10 to Compare Visual Recognition Performance between Deep Neural Networks and Humans. ArXiv, abs\/1811.07270, 2018."},{"key":"e_1_3_2_1_22_1","first-page":"48","volume-title":"Towards Requirements Specification for Machine-learned Perception Based on Human Performance. In AIRE'20","author":"Hu B. C.","year":"2020","unstructured":"B. C. Hu, R. Salay, K. Czarnecki, M. Rahimi, G. Selim, and M. Chechik. Towards Requirements Specification for Machine-learned Perception Based on Human Performance. In AIRE'20, pages 48--51, 2020."},{"key":"e_1_3_2_1_23_1","first-page":"3","volume-title":"Min Wu. Safety Verification of Deep Neural Networks. In CAV'17","author":"Huang Xiaowei","year":"2017","unstructured":"Xiaowei Huang, Marta Kwiatkowska, Sen Wang, and Min Wu. Safety Verification of Deep Neural Networks. In CAV'17, pages 3--29, 2017."},{"key":"e_1_3_2_1_24_1","first-page":"2","volume-title":"How Do Engineers Perceive Difficulties in Engineering of Machine-learning Systems?: Questionnaire Survey","author":"Ishikawa Fuyuki","year":"2019","unstructured":"Fuyuki Ishikawa and Nobukazu Yoshioka. How Do Engineers Perceive Difficulties in Engineering of Machine-learning Systems?: Questionnaire Survey. In Marcus Ciolkowski, Dusica Marijan, Matthias Galster, Weiyi Shang, Andreas Jedlitschka, Rakesh Shukla, and Kanchana Padmanabhan, editors, CESSER-IP@ICSE 2019, pages 2--9. IEEE \/ ACM, 2019."},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.1109\/AIRE51212.2020.00012"},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.1038\/srep32672"},{"key":"e_1_3_2_1_27_1","volume-title":"On the Effectiveness of Adversarial Training Against Common Corruptions. ArXiv, abs\/2103.02325","author":"Kireev Klim","year":"2021","unstructured":"Klim Kireev, Maksym Andriushchenko, and Nicolas Flammarion. On the Effectiveness of Adversarial Training Against Common Corruptions. ArXiv, abs\/2103.02325, 2021."},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"publisher","DOI":"10.1093\/biomet\/81.4.673"},{"issue":"4","key":"e_1_3_2_1_29_1","first-page":"392","article-title":"Medical Image Data and Datasets in the Era of Machine Learning","volume":"30","author":"Kohli Marc","year":"2017","unstructured":"Marc Kohli, Ronald Summers, and Jr Geis. Medical Image Data and Datasets in the Era of Machine Learning. JDI, 30(4):392--399, 2017.","journal-title":"JDI"},{"key":"e_1_3_2_1_30_1","volume-title":"Learning Multiple Layers of Features from Tiny Images","author":"Krizhevsky Alex","year":"2009","unstructured":"Alex Krizhevsky, Geoffrey Hinton, et al. Learning Multiple Layers of Features from Tiny Images. 2009."},{"key":"e_1_3_2_1_31_1","first-page":"165","volume-title":"Hakjoo Oh. Effective White-Box Testing of Deep Neural Networks with Adaptive Neuron-Selection Strategy. ISSTA 2020","author":"Lee Seokhyun","year":"2020","unstructured":"Seokhyun Lee, Sooyoung Cha, Dain Lee, and Hakjoo Oh. Effective White-Box Testing of Deep Neural Networks with Adaptive Neuron-Selection Strategy. ISSTA 2020, page 165--176, New York, NY, USA, 2020. Association for Computing Machinery."},{"key":"e_1_3_2_1_32_1","doi-asserted-by":"publisher","DOI":"10.1109\/MET.2019.00016"},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICCVW.2017.94"},{"key":"e_1_3_2_1_34_1","volume-title":"Benchmarking Robustness in Object Detection: Autonomous Driving when Winter is Coming. arXiv preprint arXiv:1907.07484","author":"Michaelis Claudio","year":"2019","unstructured":"Claudio Michaelis, Benjamin Mitzkus, Robert Geirhos, Evgenia Rusak, Oliver Bringmann, Alexander S. Ecker, Matthias Bethge, and Wieland Brendel. Benchmarking Robustness in Object Detection: Autonomous Driving when Winter is Coming. arXiv preprint arXiv:1907.07484, 2019."},{"key":"e_1_3_2_1_35_1","first-page":"180","volume-title":"Vittorio Ferrari. Training Object Class Detectors with Click Supervision. In 2017 IEEE Conference on Computer Vision and Pattern Recognition, CVPR 2017","author":"Papadopoulos Dim P.","year":"2017","unstructured":"Dim P. Papadopoulos, Jasper R. R. Uijlings, Frank Keller, and Vittorio Ferrari. Training Object Class Detectors with Click Supervision. In 2017 IEEE Conference on Computer Vision and Pattern Recognition, CVPR 2017, Honolulu, HI, USA, July 21--26,2017, pages 180--189. IEEE Computer Society, 2017."},{"key":"e_1_3_2_1_36_1","first-page":"241","volume-title":"Marsha Chechik. Toward Requirements Specification for Machine-Learned Components. In AIRE'19","author":"Rahimi Mona","year":"2019","unstructured":"Mona Rahimi, Jin L. C. Guo, Sahar Kokaly, and Marsha Chechik. Toward Requirements Specification for Machine-Learned Components. In AIRE'19, pages 241--244, 2019."},{"key":"e_1_3_2_1_37_1","volume-title":"Increasing the robustness of dnns against image corruptions by playing the game of noise. ArXiv, abs\/2001.06057","author":"Rusak E.","year":"2020","unstructured":"E. Rusak, Lukas Schott, Roland S. Zimmermann, Julian Bitterwolf, O. Bringmann, M. Bethge, and Wieland Brendel. Increasing the robustness of dnns against image corruptions by playing the game of noise. ArXiv, abs\/2001.06057, 2020."},{"key":"e_1_3_2_1_38_1","doi-asserted-by":"publisher","DOI":"10.1007\/s11263-015-0816-y"},{"key":"e_1_3_2_1_39_1","volume-title":"Using Machine Learning Safely in Automotive Software: An Assessment and Adaption of Software Process Requirements in ISO 26262. ArXiv, abs\/1808.01614","author":"Salay Rick","year":"2018","unstructured":"Rick Salay and Krzysztof Czarnecki. Using Machine Learning Safely in Automotive Software: An Assessment and Adaption of Software Process Requirements in ISO 26262. ArXiv, abs\/1808.01614, 2018."},{"key":"e_1_3_2_1_40_1","volume-title":"Adversarial Examples on Object Recognition: A Comprehensive Survey. ACM Computing Surveys (CSUR), 53(3):1--38","author":"Serban Alex","year":"2020","unstructured":"Alex Serban, Erik Poll, and Joost Visser. Adversarial Examples on Object Recognition: A Comprehensive Survey. ACM Computing Surveys (CSUR), 53(3):1--38, 2020."},{"key":"e_1_3_2_1_41_1","volume-title":"Seshia and Dorsa Sadigh. Towards Verified Artificial Intelligence. ArXiv, abs\/1606.08514","author":"Sanjit","year":"2016","unstructured":"Sanjit A. Seshia and Dorsa Sadigh. Towards Verified Artificial Intelligence. ArXiv, abs\/1606.08514, 2016."},{"key":"e_1_3_2_1_42_1","doi-asserted-by":"publisher","DOI":"10.1109\/TIP.2005.859378"},{"key":"e_1_3_2_1_43_1","doi-asserted-by":"publisher","DOI":"10.1109\/TIP.2006.881959"},{"key":"e_1_3_2_1_44_1","doi-asserted-by":"publisher","DOI":"10.1007\/s11263-020-01423-x"},{"issue":"323","key":"e_1_3_2_1_45_1","first-page":"332","article-title":"Man vs. computer: Benchmarking machine learning algorithms for traffic sign recognition","volume":"32","author":"Stallkamp J.","year":"2012","unstructured":"J. Stallkamp, M. Schlipsing, J. Salmen, and C. Igel. Man vs. computer: Benchmarking machine learning algorithms for traffic sign recognition. Neural Networks, 32:323 - 332, 2012. Selected Papers from IJCNN 2011.","journal-title":"Neural Networks"},{"key":"e_1_3_2_1_46_1","doi-asserted-by":"publisher","DOI":"10.1109\/REW.2019.00050"},{"key":"e_1_3_2_1_47_1","first-page":"300","volume-title":"Peng Cheng. RobOT: Robustness-Oriented Testing for Deep Learning Systems. 2021 IEEE\/ACM 43rd International Conference on Software Engineering (ICSE)","author":"Wang Jingyi","year":"2021","unstructured":"Jingyi Wang, Jialuo Chen, Youcheng Sun, Xingjun Ma, Dongxia Wang, Jun Sun, and Peng Cheng. RobOT: Robustness-Oriented Testing for Deep Learning Systems. 2021 IEEE\/ACM 43rd International Conference on Software Engineering (ICSE), pages 300--311, 2021."},{"key":"e_1_3_2_1_48_1","doi-asserted-by":"publisher","DOI":"10.1109\/MSP.2008.930649"},{"key":"e_1_3_2_1_49_1","doi-asserted-by":"publisher","DOI":"10.1109\/TIP.2003.819861"},{"key":"e_1_3_2_1_50_1","first-page":"408","volume-title":"Marta Kwiatkowska. Feature-Guided Black-Box Safety Testing of Deep Neural Networks. In Proc. of TACAS'18","author":"Wicker Matthew","year":"2018","unstructured":"Matthew Wicker, Xiaowei Huang, and Marta Kwiatkowska. Feature-Guided Black-Box Safety Testing of Deep Neural Networks. In Proc. of TACAS'18, pages 408--426. Springer, 2018."},{"key":"e_1_3_2_1_51_1","first-page":"125","volume-title":"Irwin King. Deep Validation: Toward Detecting Real-World Corner Cases for Deep Neural Networks. In 2019 49th Annual IEEE\/IFIP International Conference on Dependable Systems and Networks (DSN)","author":"Wu Weibin","year":"2019","unstructured":"Weibin Wu, Hui Xu, Sanqiang Zhong, Michael R. Lyu, and Irwin King. Deep Validation: Toward Detecting Real-World Corner Cases for Deep Neural Networks. In 2019 49th Annual IEEE\/IFIP International Conference on Dependable Systems and Networks (DSN), pages 125--137, 2019."},{"key":"e_1_3_2_1_52_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.jss.2010.11.920"},{"key":"e_1_3_2_1_53_1","first-page":"1","volume-title":"Archives of Computational Methods in Engineering","author":"Xu S.","year":"2020","unstructured":"S. Xu, J. Wang, W. Shou, T. Ngo, A. Sadick, and X. Wang. Computer Vision Techniques in Construction: a Critical Review. Archives of Computational Methods in Engineering, pages 1--15, 2020."},{"key":"e_1_3_2_1_54_1","volume-title":"Wide residual networks. ArXiv, abs\/1605.07146","author":"Zagoruyko Sergey","year":"2016","unstructured":"Sergey Zagoruyko and N. Komodakis. Wide residual networks. ArXiv, abs\/1605.07146, 2016."},{"key":"e_1_3_2_1_55_1","first-page":"2066","volume-title":"Wolfgang Herzner. CV-HAZOP: Introducing Test Data Validation for Computer Vision. In ICCV'15","author":"Zendel Oliver","year":"2015","unstructured":"Oliver Zendel, Markus Murschitz, Martin Humenberger, and Wolfgang Herzner. CV-HAZOP: Introducing Test Data Validation for Computer Vision. In ICCV'15, pages 2066--2074, 2015."},{"key":"e_1_3_2_1_56_1","first-page":"132","volume-title":"DeepRoad: GAN-based Metamorphic Testing and Input Validation Framework for Autonomous Driving Systems. In 2018 33rd IEEE\/ACM International Conference on Automated Software Engineering (ASE)","author":"Zhang Mengshi","year":"2018","unstructured":"Mengshi Zhang, Yuqun Zhang, Lingming Zhang, Cong Liu, and Sarfraz Khurshid. DeepRoad: GAN-based Metamorphic Testing and Input Validation Framework for Autonomous Driving Systems. In 2018 33rd IEEE\/ACM International Conference on Automated Software Engineering (ASE), pages 132--142. IEEE, 2018."}],"event":{"name":"ICSE '22: 44th International Conference on Software Engineering","location":"Pittsburgh Pennsylvania","acronym":"ICSE '22","sponsor":["SIGSOFT ACM Special Interest Group on Software Engineering","IEEE CS"]},"container-title":["Proceedings of the 44th International Conference on Software Engineering"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3510003.3510109","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3510003.3510109","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T18:10:23Z","timestamp":1750183823000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3510003.3510109"}},"subtitle":["reliability requirements for machine vision components"],"short-title":[],"issued":{"date-parts":[[2022,5,21]]},"references-count":56,"alternative-id":["10.1145\/3510003.3510109","10.1145\/3510003"],"URL":"https:\/\/doi.org\/10.1145\/3510003.3510109","relation":{},"subject":[],"published":{"date-parts":[[2022,5,21]]},"assertion":[{"value":"2022-07-05","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}