{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,3,28]],"date-time":"2026-03-28T20:31:38Z","timestamp":1774729898535,"version":"3.50.1"},"publisher-location":"New York, NY, USA","reference-count":45,"publisher":"ACM","license":[{"start":{"date-parts":[[2023,11,30]],"date-time":"2023-11-30T00:00:00Z","timestamp":1701302400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"funder":[{"DOI":"10.13039\/100000001","name":"NSF (National Science Foundation)","doi-asserted-by":"publisher","award":["2019239,2332991"],"award-info":[{"award-number":["2019239,2332991"]}],"id":[{"id":"10.13039\/100000001","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2023,11,30]]},"DOI":"10.1145\/3611643.3613079","type":"proceedings-article","created":{"date-parts":[[2023,11,30]],"date-time":"2023-11-30T23:14:38Z","timestamp":1701386078000},"page":"2122-2126","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":5,"title":["Deeper Notions of Correctness in Image-Based DNNs: Lifting Properties from Pixel to Entities"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-5632-7518","authenticated-orcid":false,"given":"Felipe","family":"Toledo","sequence":"first","affiliation":[{"name":"University of Virginia, Charlottesville, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-0208-6517","authenticated-orcid":false,"given":"David","family":"Shriver","sequence":"additional","affiliation":[{"name":"University of Virginia, Charlottesville, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-9592-1352","authenticated-orcid":false,"given":"Sebastian","family":"Elbaum","sequence":"additional","affiliation":[{"name":"University of Virginia, Charlottesville, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-1937-1544","authenticated-orcid":false,"given":"Matthew B.","family":"Dwyer","sequence":"additional","affiliation":[{"name":"University of Virginia, Charlottesville, USA"}]}],"member":"320","published-online":{"date-parts":[[2023,11,30]]},"reference":[{"key":"e_1_3_2_2_1_1","unstructured":"Yahoo News Aaron Cole. July 1 2016. Fatal crash in Florida is first reported Tesla Autopilot death. https:\/\/news.yahoo.com\/fatal-crash-florida-first-reported-tesla-autopilot-death-180000689.html"},{"key":"e_1_3_2_2_2_1","unstructured":"Akirasosa. 2020. Real-Time Semantic Segmentation in Mobile device. https:\/\/github.com\/akirasosa\/mobile-semantic-segmentation"},{"key":"e_1_3_2_2_3_1","doi-asserted-by":"publisher","DOI":"10.1109\/TITS.2020.2993926"},{"key":"e_1_3_2_2_4_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICRA.2018.8460487"},{"key":"e_1_3_2_2_5_1","unstructured":"Scheme Color. [n. d.]. Skin color palette. https:\/\/www.schemecolor.com\/real-skin-tones-color-palette.php"},{"key":"e_1_3_2_2_6_1","unstructured":"Yahoo News Daniel Howley. June 29 2015. Google Photos Mislabels 2 Black Americans as Gorillas. https:\/\/finance.yahoo.com\/news\/google-photos-mislabels-two-black-americans-as-122793782784.html"},{"key":"e_1_3_2_2_7_1","doi-asserted-by":"publisher","DOI":"10.1145\/3460319.3464801"},{"key":"e_1_3_2_2_8_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-68167-2_19"},{"key":"e_1_3_2_2_9_1","volume-title":"Advances in Neural Information Processing Systems 27","author":"Goodfellow Ian","unstructured":"Ian Goodfellow, Jean Pouget-Abadie, Mehdi Mirza, Bing Xu, David Warde-Farley, Sherjil Ozair, Aaron Courville, and Yoshua Bengio. 2014. Generative Adversarial Nets. In Advances in Neural Information Processing Systems 27, Z. Ghahramani, M. Welling, C. Cortes, N. D. Lawrence, and K. Q. Weinberger (Eds.). Curran Associates, Inc., 2672\u20132680."},{"key":"e_1_3_2_2_10_1","volume-title":"3rd International Conference on Learning Representations, ICLR","author":"Goodfellow Ian J.","year":"2015","unstructured":"Ian J. Goodfellow, Jonathon Shlens, and Christian Szegedy. 2015. Explaining and Harnessing Adversarial Examples. In 3rd International Conference on Learning Representations, ICLR 2015, San Diego, CA, USA, May 7-9, 2015, Conference Track Proceedings, Yoshua Bengio and Yann LeCun (Eds.)."},{"key":"e_1_3_2_2_11_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2019.00550"},{"key":"e_1_3_2_2_12_1","volume-title":"Deep Residual Learning for Image Recognition. In The IEEE Conference on Computer Vision and Pattern Recognition (CVPR).","author":"He Kaiming","year":"2016","unstructured":"Kaiming He, Xiangyu Zhang, Shaoqing Ren, and Jian Sun. 2016. Deep Residual Learning for Image Recognition. In The IEEE Conference on Computer Vision and Pattern Recognition (CVPR)."},{"key":"e_1_3_2_2_13_1","volume-title":"Jeff Larson and ProPublica Lauren Kirchner","author":"Julia Angwin Surya Mattu","year":"2023","unstructured":"Surya Mattu Julia Angwin, Jeff Larson and ProPublica Lauren Kirchner. April 04, 2023. Machine Bias. https:\/\/en.wikipedia.org\/wiki\/COMPAS_(software)"},{"key":"e_1_3_2_2_14_1","doi-asserted-by":"publisher","DOI":"10.1109\/WACV48630.2021.00159"},{"key":"e_1_3_2_2_15_1","volume-title":"CAV 2017, Heidelberg, Germany, July 24-28, 2017, Proceedings, Part I. 97\u2013117","author":"Katz Guy","unstructured":"Guy Katz, Clark W. Barrett, David L. Dill, Kyle Julian, and Mykel J. Kochenderfer. 2017. Reluplex: An Efficient SMT Solver for Verifying Deep Neural Networks. In Computer Aided Verification - 29th International Conference, CAV 2017, Heidelberg, Germany, July 24-28, 2017, Proceedings, Part I. 97\u2013117."},{"key":"e_1_3_2_2_16_1","volume-title":"Auto-Encoding Variational Bayes. In 2nd International Conference on Learning Representations, ICLR 2014, Banff, AB, Canada, April 14-16, 2014, Conference Track Proceedings.","author":"Diederik","unstructured":"Diederik P. Kingma and Max Welling. 2014. Auto-Encoding Variational Bayes. In 2nd International Conference on Learning Representations, ICLR 2014, Banff, AB, Canada, April 14-16, 2014, Conference Track Proceedings."},{"key":"e_1_3_2_2_17_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR42600.2020.01426"},{"key":"e_1_3_2_2_18_1","volume-title":"5th International Conference on Learning Representations, ICLR 2017, Toulon, France, April 24-26, 2017, Workshop Track Proceedings. OpenReview.net.","author":"Kurakin Alexey","year":"2017","unstructured":"Alexey Kurakin, Ian J. Goodfellow, and Samy Bengio. 2017. Adversarial examples in the physical world. In 5th International Conference on Learning Representations, ICLR 2017, Toulon, France, April 24-26, 2017, Workshop Track Proceedings. OpenReview.net."},{"key":"e_1_3_2_2_19_1","doi-asserted-by":"publisher","DOI":"10.1109\/lra.2018.2795643"},{"key":"e_1_3_2_2_20_1","volume-title":"6th International Conference on Learning Representations, ICLR 2018, Vancouver, BC, Canada, April 30 - May 3, 2018, Conference Track Proceedings. OpenReview.net.","author":"Madry Aleksander","year":"2018","unstructured":"Aleksander Madry, Aleksandar Makelov, Ludwig Schmidt, Dimitris Tsipras, and Adrian Vladu. 2018. Towards Deep Learning Models Resistant to Adversarial Attacks. In 6th International Conference on Learning Representations, ICLR 2018, Vancouver, BC, Canada, April 30 - May 3, 2018, Conference Track Proceedings. OpenReview.net."},{"key":"e_1_3_2_2_21_1","unstructured":"ABC News Mark Osborne. March 31 2018. Tesla car was on autopilot prior to fatal crash in California company says. https:\/\/abcnews.go.com\/US\/tesla-car-autopilot-prior-fatal-crash-california-company\/story?id=54142891"},{"key":"e_1_3_2_2_22_1","unstructured":"Algorithm Watch Nicolas Kayser-Bril. April 7 2020. Google apologizes after its Vision AI produced racist results. https:\/\/algorithmwatch.org\/en\/google-vision-racism\/"},{"key":"e_1_3_2_2_23_1","volume-title":"Proceedings of the 36th International Conference on Machine Learning, Kamalika Chaudhuri and Ruslan Salakhutdinov (Eds.) (Proceedings of Machine Learning Research","volume":"4911","author":"Odena Augustus","year":"2019","unstructured":"Augustus Odena, Catherine Olsson, David Andersen, and Ian Goodfellow. 2019. TensorFuzz: Debugging Neural Networks with Coverage-Guided Fuzzing. In Proceedings of the 36th International Conference on Machine Learning, Kamalika Chaudhuri and Ruslan Salakhutdinov (Eds.) (Proceedings of Machine Learning Research, Vol. 97). PMLR, Long Beach, California, USA. 4901\u20134911."},{"key":"e_1_3_2_2_24_1","unstructured":"Department of Homeland Security. December 4 2017. Passenger Screening Algorithm Challenge. https:\/\/www.kaggle.com\/competitions\/passenger-screening-algorithm-challenge\/overview\/timeline"},{"key":"e_1_3_2_2_25_1","unstructured":"ONNX. 2017. Open Neural Network Exchange. https:\/\/github.com\/onnx\/onnx"},{"key":"e_1_3_2_2_26_1","doi-asserted-by":"publisher","DOI":"10.1145\/3132747.3132785"},{"key":"e_1_3_2_2_27_1","doi-asserted-by":"publisher","DOI":"10.1145\/3368089.3409730"},{"key":"e_1_3_2_2_28_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-46475-6_7"},{"key":"e_1_3_2_2_29_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-81685-8_6"},{"key":"e_1_3_2_2_30_1","volume-title":"Proceedings of the International Conference on Software Engineering.","author":"Shriver David","unstructured":"David Shriver, Sebastian G. Elbaum, and Matthew B. Dwyer. 2021. Reducing DNN Properties to Enable Falsification with Adversarial Attacks. In Proceedings of the International Conference on Software Engineering."},{"key":"e_1_3_2_2_31_1","volume-title":"Danilo Vasconcellos Vargas, and Kouichi Sakurai","author":"Su Jiawei","year":"2017","unstructured":"Jiawei Su, Danilo Vasconcellos Vargas, and Kouichi Sakurai. 2017. One pixel attack for fooling deep neural networks. CoRR, abs\/1710.08864 (2017)."},{"key":"e_1_3_2_2_32_1","volume-title":"2nd International Conference on Learning Representations, ICLR 2014, Banff, AB, Canada, April 14-16, 2014, Conference Track Proceedings, Yoshua Bengio and Yann LeCun (Eds.).","author":"Szegedy Christian","year":"2014","unstructured":"Christian Szegedy, Wojciech Zaremba, Ilya Sutskever, Joan Bruna, Dumitru Erhan, Ian J. Goodfellow, and Rob Fergus. 2014. Intriguing properties of neural networks. In 2nd International Conference on Learning Representations, ICLR 2014, Banff, AB, Canada, April 14-16, 2014, Conference Track Proceedings, Yoshua Bengio and Yann LeCun (Eds.)."},{"key":"e_1_3_2_2_33_1","doi-asserted-by":"publisher","DOI":"10.1145\/3180155.3180220"},{"key":"e_1_3_2_2_34_1","volume-title":"Evaluating Robustness of Neural Networks with Mixed Integer Programming. In International Conference on Learning Representations.","author":"Tjeng Vincent","year":"2019","unstructured":"Vincent Tjeng, Kai Y. Xiao, and Russ Tedrake. 2019. Evaluating Robustness of Neural Networks with Mixed Integer Programming. In International Conference on Learning Representations."},{"key":"e_1_3_2_2_35_1","doi-asserted-by":"publisher","DOI":"10.1109\/ASE51524.2021.9678590"},{"key":"e_1_3_2_2_36_1","doi-asserted-by":"crossref","unstructured":"Gulshan V Peng L Coram M Stumpe MC Wu D Narayanaswamy A Venugopalan S Widner K Madams T Cuadros J Kim R Raman R Nelson PC Mega JL and Webster DR. December 16 2016. Development and Validation of a Deep Learning Algorithm for Detection of Diabetic Retinopathy in Retinal Fundus Photographs.","DOI":"10.1001\/jama.2016.17216"},{"key":"e_1_3_2_2_37_1","unstructured":"Shiqi Wang Kexin Pei Justin Whitehouse Junfeng Yang and Suman Jana. 2018. Efficient Formal Safety Analysis of Neural Networks. In NeurIPS. 6369\u20136379."},{"key":"e_1_3_2_2_38_1","unstructured":"Wikipedia. May 23 2016. COMPAS (software). https:\/\/www.propublica.org\/article\/machine-bias-risk-assessments-in-criminal-sentencing"},{"key":"e_1_3_2_2_39_1","doi-asserted-by":"publisher","DOI":"10.1145\/3510003.3510212"},{"key":"e_1_3_2_2_40_1","unstructured":"Yuxin Wu Alexander Kirillov Francisco Massa Wan-Yen Lo and Ross Girshick. 2019. Detectron2. https:\/\/github.com\/facebookresearch\/detectron2"},{"key":"e_1_3_2_2_41_1","first-page":"4939","article-title":"Efficient Neural Network Robustness Certification with General Activation Functions","volume":"31","author":"Zhang Huan","year":"2018","unstructured":"Huan Zhang, Tsui-Wei Weng, Pin-Yu Chen, Cho-Jui Hsieh, and Luca Daniel. 2018. Efficient Neural Network Robustness Certification with General Activation Functions. Advances in Neural Information Processing Systems, 31 (2018), 4939\u20134948.","journal-title":"Advances in Neural Information Processing Systems"},{"key":"e_1_3_2_2_42_1","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2019.2962027"},{"key":"e_1_3_2_2_43_1","doi-asserted-by":"publisher","DOI":"10.1145\/3377811.3380422"},{"key":"e_1_3_2_2_44_1","doi-asserted-by":"publisher","DOI":"10.1109\/TPAMI.2022.3225573"},{"key":"e_1_3_2_2_45_1","doi-asserted-by":"publisher","DOI":"10.1145\/3460319.3464811"}],"event":{"name":"ESEC\/FSE '23: 31st ACM Joint European Software Engineering Conference and Symposium on the Foundations of Software Engineering","location":"San Francisco CA USA","acronym":"ESEC\/FSE '23","sponsor":["SIGSOFT ACM Special Interest Group on Software Engineering"]},"container-title":["Proceedings of the 31st ACM Joint European Software Engineering Conference and Symposium on the Foundations of Software Engineering"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3611643.3613079","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3611643.3613079","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T16:37:10Z","timestamp":1750178230000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3611643.3613079"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023,11,30]]},"references-count":45,"alternative-id":["10.1145\/3611643.3613079","10.1145\/3611643"],"URL":"https:\/\/doi.org\/10.1145\/3611643.3613079","relation":{},"subject":[],"published":{"date-parts":[[2023,11,30]]},"assertion":[{"value":"2023-11-30","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}