{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,9]],"date-time":"2026-04-09T02:25:14Z","timestamp":1775701514811,"version":"3.50.1"},"publisher-location":"New York, NY, USA","reference-count":59,"publisher":"ACM","license":[{"start":{"date-parts":[[2020,6,27]],"date-time":"2020-06-27T00:00:00Z","timestamp":1593216000000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"name":"Qdai-jump Research Program","award":["No.01277"],"award-info":[{"award-number":["No.01277"]}]},{"DOI":"10.13039\/501100012659","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["No.61772038, 61532019"],"award-info":[{"award-number":["No.61772038, 61532019"]}],"id":[{"id":"10.13039\/501100012659","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100007162","name":"Guangdong Science and Technology Department","doi-asserted-by":"publisher","award":["No.2018B010107004"],"award-info":[{"award-number":["No.2018B010107004"]}],"id":[{"id":"10.13039\/501100007162","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100001381","name":"National Research Foundation Singapore","doi-asserted-by":"publisher","award":["No.NRF2018NCR-NCR005-0001, No.NRF2018NCR-NSOE003-0001"],"award-info":[{"award-number":["No.NRF2018NCR-NCR005-0001, No.NRF2018NCR-NSOE003-0001"]}],"id":[{"id":"10.13039\/501100001381","id-type":"DOI","asserted-by":"publisher"}]},{"name":"JSPS KAKENHI","award":["No.19K24348, 19H04086, 18H04097"],"award-info":[{"award-number":["No.19K24348, 19H04086, 18H04097"]}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2020,6,27]]},"DOI":"10.1145\/3377811.3380368","type":"proceedings-article","created":{"date-parts":[[2020,10,1]],"date-time":"2020-10-01T18:25:34Z","timestamp":1601576734000},"page":"739-751","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":57,"title":["Towards characterizing adversarial defects of deep learning software from the lens of uncertainty"],"prefix":"10.1145","author":[{"given":"Xiyue","family":"Zhang","sequence":"first","affiliation":[{"name":"Peking University, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Xiaofei","family":"Xie","sequence":"additional","affiliation":[{"name":"Nanyang Technological University, Singapore"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Lei","family":"Ma","sequence":"additional","affiliation":[{"name":"Kyushu University, Japan"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Xiaoning","family":"Du","sequence":"additional","affiliation":[{"name":"Nanyang Technological University, Singapore"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Qiang","family":"Hu","sequence":"additional","affiliation":[{"name":"Kyushu University, Japan"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Yang","family":"Liu","sequence":"additional","affiliation":[{"name":"Nanyang Technological University, Singapore"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Jianjun","family":"Zhao","sequence":"additional","affiliation":[{"name":"Kyushu University, Japan"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Meng","family":"Sun","sequence":"additional","affiliation":[{"name":"Peking University, China"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2020,10]]},"reference":[{"key":"e_1_3_2_1_1_1","volume-title":"12th USENIX Symposium on Operating Systems Design and Implementation (OSDI 16)","author":"Abadi Martin","year":"2016","unstructured":"Martin Abadi, Paul Barham, Jianmin Chen, Zhifeng Chen, Andy Davis, Jeffrey Dean, Matthieu Devin, Sanjay Ghemawat, Geoffrey Irving, Michael Isard, Manjunath Kudlur, Josh Levenberg, Rajat Monga, Sherry Moore, Derek G. Murray, Benoit Steiner, Paul Tucker, Vijay Vasudevan, Pete Warden, Martin Wicke, Yuan Yu, and Xiaoqiang Zheng. 2016. TensorFlow: A system for large-scale machine learning. In 12th USENIX Symposium on Operating Systems Design and Implementation (OSDI 16). 265--283."},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-40994-3_25"},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1145\/3128572.3140444"},{"key":"e_1_3_2_1_4_1","volume-title":"Towards Evaluating the Robustness of Neural Networks. In IEEE Symposium on Security and Privacy. 39--57","author":"Carlini Nicholas","year":"2017","unstructured":"Nicholas Carlini and David Wagner. 2017. Towards Evaluating the Robustness of Neural Networks. In IEEE Symposium on Security and Privacy. 39--57."},{"key":"e_1_3_2_1_5_1","unstructured":"Fran\u00e7ois Chollet et al. 2015. Keras. https:\/\/github.com\/fchollet\/keras. (2015)."},{"key":"e_1_3_2_1_6_1","volume-title":"Deep learning for aspect-based sentiment analysis: a comparative review. Expert Systems With Applications","author":"Dohaiha Hai Ha","year":"2018","unstructured":"Hai Ha Dohaiha, PWC Prasad, Angelika Maag, and Abeer Alsadoon. 2018. Deep learning for aspect-based sentiment analysis: a comparative review. Expert Systems With Applications (2018)."},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"publisher","DOI":"10.1145\/3338906.3338954"},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2018.00175"},{"key":"e_1_3_2_1_9_1","volume-title":"An introduction to ROC analysis. Pattern recognition letters 27, 8","author":"Fawcett Tom","year":"2006","unstructured":"Tom Fawcett. 2006. An introduction to ROC analysis. Pattern recognition letters 27, 8 (2006), 861--874."},{"key":"e_1_3_2_1_10_1","volume-title":"Detecting adversarial samples from artifacts. arXiv preprint arXiv:1703.00410","author":"Feinman Reuben","year":"2017","unstructured":"Reuben Feinman, Ryan R Curtin, Saurabh Shintre, and Andrew B Gardner. 2017. Detecting adversarial samples from artifacts. arXiv preprint arXiv:1703.00410 (2017)."},{"key":"e_1_3_2_1_12_1","volume-title":"international conference on machine learning. 1050--1059","author":"Gal Yarin","year":"2016","unstructured":"Yarin Gal and Zoubin Ghahramani. 2016. Dropout as a bayesian approximation: Representing model uncertainty in deep learning. In international conference on machine learning. 1050--1059."},{"key":"e_1_3_2_1_13_1","volume-title":"Adversarial and clean data are not twins. arXiv preprint arXiv:1704.04960","author":"Gong Zhitao","year":"2017","unstructured":"Zhitao Gong, Wenlu Wang, and Wei-Shinn Ku. 2017. Adversarial and clean data are not twins. arXiv preprint arXiv:1704.04960 (2017)."},{"key":"e_1_3_2_1_14_1","volume-title":"Explaining and Harnessing Adversarial Examples. In International Conference on Learning Representations. http:\/\/arxiv.org\/abs\/1412","author":"Goodfellow Ian","year":"2015","unstructured":"Ian Goodfellow, Jonathon Shlens, and Christian Szegedy. 2015. Explaining and Harnessing Adversarial Examples. In International Conference on Learning Representations. http:\/\/arxiv.org\/abs\/1412.6572"},{"key":"e_1_3_2_1_15_1","volume-title":"Countering adversarial images using input transformations. arXiv preprint arXiv:1711.00117","author":"Guo Chuan","year":"2017","unstructured":"Chuan Guo, Mayank Rana, Moustapha Cisse, and Laurens van der Maaten. 2017. Countering adversarial images using input transformations. arXiv preprint arXiv:1711.00117 (2017)."},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.1109\/ASE.2019.00080"},{"key":"e_1_3_2_1_17_1","volume-title":"Optimization, and Statistics","author":"Hazan Tamir","unstructured":"Tamir Hazan, George Papandreou, and Daniel Tarlow. 2016. Perturbations, Optimization, and Statistics. MIT Press."},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2016.90"},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","DOI":"10.1109\/MSP.2012.2205597"},{"key":"e_1_3_2_1_20_1","volume-title":"Mobilenets: Efficient convolutional neural networks for mobile vision applications. arXiv preprint arXiv:1704.04861","author":"Howard Andrew G","year":"2017","unstructured":"Andrew G Howard, Menglong Zhu, Bo Chen, Dmitry Kalenichenko, Weijun Wang, Tobias Weyand, Marco Andreetto, and Hartwig Adam. 2017. Mobilenets: Efficient convolutional neural networks for mobile vision applications. arXiv preprint arXiv:1704.04861 (2017)."},{"key":"e_1_3_2_1_21_1","volume-title":"CAV 2017, Heidelberg, Germany, July 24--28, 2017, Proceedings, Part I. 97--117","author":"Katz Guy","unstructured":"Guy Katz, Clark W. Barrett, David L. Dill, Kyle Julian, and Mykel J. Kochenderfer. 2017. Reluplex: An Efficient SMT Solver for Verifying Deep Neural Networks. In Computer Aided Verification - 29th International Conference, CAV 2017, Heidelberg, Germany, July 24--28, 2017, Proceedings, Part I. 97--117."},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE.2019.00108"},{"key":"e_1_3_2_1_23_1","volume-title":"d.]. CIFAR-10","author":"Krizhevsky Alex","unstructured":"Alex Krizhevsky, Vinod Nair, and Geoffrey Hinton. [n. d.]. CIFAR-10 (Canadian Institute for Advanced Research). ([n. d.]). http:\/\/www.cs.toronto.edu\/~kriz\/cifar.html"},{"key":"e_1_3_2_1_24_1","volume-title":"Adversarial examples in the physical world. arXiv preprint arXiv:1607.02533","author":"Kurakin Alexey","year":"2016","unstructured":"Alexey Kurakin, Ian Goodfellow, and Samy Bengio. 2016. Adversarial examples in the physical world. arXiv preprint arXiv:1607.02533 (2016)."},{"key":"e_1_3_2_1_25_1","volume-title":"Adversarial Machine Learning at Scale. arXiv preprint arXiv:1611.01236","author":"Kurakin Alexey","year":"2016","unstructured":"Alexey Kurakin, Ian Goodfellow, and Samy Bengio. 2016. Adversarial Machine Learning at Scale. arXiv preprint arXiv:1611.01236 (2016)."},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.1109\/5.726791"},{"key":"e_1_3_2_1_27_1","volume-title":"Network in network. arXiv preprint arXiv:1312.4400","author":"Lin Min","year":"2013","unstructured":"Min Lin, Qiang Chen, and Shuicheng Yan. 2013. Network in network. arXiv preprint arXiv:1312.4400 (2013)."},{"key":"e_1_3_2_1_28_1","volume-title":"DeepCT: Tomographic Combinatorial Testing for Deep Learning Systems. In 2019 IEEE 26th International Conference on Software Analysis, Evolution and Reengineering (SANER). 614--618","author":"Ma Lei","year":"2019","unstructured":"Lei Ma, Felix Juefei-Xu, Minhui Xue, Bo Li, Li Li, Yang Liu, and Jianjun Zhao. 2019. DeepCT: Tomographic Combinatorial Testing for Deep Learning Systems. In 2019 IEEE 26th International Conference on Software Analysis, Evolution and Reengineering (SANER). 614--618."},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.1145\/3238147.3238202"},{"key":"e_1_3_2_1_30_1","volume-title":"DeepMutation: Mutation Testing of Deep Learning Systems. In 29th IEEE International Symposium on Software Reliability Engineering (ISSRE)","author":"Ma Lei","year":"2018","unstructured":"Lei Ma, Fuyuan Zhang, Jiyuan Sun, Minhui Xue, Bo Li, Felix Juefei-Xu, Chao Xie, Li Li, Yang Liu, Jianjun Zhao, and Yadong Wang. [n. d.]. DeepMutation: Mutation Testing of Deep Learning Systems. In 29th IEEE International Symposium on Software Reliability Engineering (ISSRE), Memphis, USA, Oct. 15--18, 2018. 100--111."},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"publisher","DOI":"10.1145\/3236024.3236082"},{"key":"e_1_3_2_1_32_1","volume-title":"Test Selection for Deep Learning Systems. arXiv preprint arXiv:1904.13195","author":"Ma Wei","year":"2019","unstructured":"Wei Ma, Mike Papadakis, Anestis Tsakmalis, Maxime Cordy, and Yves Le Traon. 2019. Test Selection for Deep Learning Systems. arXiv preprint arXiv:1904.13195 (2019)."},{"key":"e_1_3_2_1_33_1","volume-title":"A practical bayesian framework for backpropagation networks. Neural computation","author":"MacKay David JC","year":"1992","unstructured":"JC MacKay David. 1992. A practical bayesian framework for backpropagation networks. Neural computation (1992)."},{"key":"e_1_3_2_1_34_1","volume-title":"6th International Conference on Learning Representations, ICLR 2018, Vancouver, BC, Canada, April 30 - May 3, 2018, Conference Track Proceedings.","author":"Madry Aleksander","year":"2018","unstructured":"Aleksander Madry, Aleksandar Makelov, Ludwig Schmidt, Dimitris Tsipras, and Adrian Vladu. 2018. Towards Deep Learning Models Resistant to Adversarial Attacks. In 6th International Conference on Learning Representations, ICLR 2018, Vancouver, BC, Canada, April 30 - May 3, 2018, Conference Track Proceedings."},{"key":"e_1_3_2_1_35_1","unstructured":"Brad L Miller David E Goldberg et al. 1995. Genetic algorithms tournament selection and the effects of noise. Complex systems 9 3 (1995) 193--212."},{"key":"e_1_3_2_1_36_1","doi-asserted-by":"crossref","unstructured":"Volodymyr Mnih Koray Kavukcuoglu David Silver Andrei A. Rusu Joel Veness and et al. 2015. Human-level control through deep reinforcement learning. Nature 518 7540 (2015) 529--533.","DOI":"10.1038\/nature14236"},{"key":"e_1_3_2_1_37_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2016.282"},{"key":"e_1_3_2_1_38_1","volume-title":"TensorFuzz: Debugging Neural Networks with Coverage-Guided Fuzzing. In International Conference on Machine Learning. 4901--4911","author":"Odena Augustus","year":"2019","unstructured":"Augustus Odena, Catherine Olsson, David Andersen, and Ian Goodfellow. 2019. TensorFuzz: Debugging Neural Networks with Coverage-Guided Fuzzing. In International Conference on Machine Learning. 4901--4911."},{"key":"e_1_3_2_1_39_1","doi-asserted-by":"publisher","DOI":"10.1109\/EuroSP.2016.36"},{"key":"e_1_3_2_1_40_1","volume-title":"2016 IEEE Symposium on Security and Privacy (SP). 582--597","author":"Papernot N.","unstructured":"N. Papernot, P. McDaniel, X. Wu, S. Jha, and A. Swami. 2016. Distillation as a Defense to Adversarial Perturbations Against Deep Neural Networks. In 2016 IEEE Symposium on Security and Privacy (SP). 582--597."},{"key":"e_1_3_2_1_41_1","doi-asserted-by":"publisher","DOI":"10.1145\/3132747.3132785"},{"key":"e_1_3_2_1_42_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE.2019.00107"},{"key":"e_1_3_2_1_43_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2018.00894"},{"key":"e_1_3_2_1_44_1","volume-title":"Foolbox: A Python toolbox to benchmark the robustness of machine learning models. arXiv preprint arXiv:1707.04131","author":"Rauber Jonas","year":"2017","unstructured":"Jonas Rauber, Wieland Brendel, and Matthias Bethge. 2017. Foolbox: A Python toolbox to benchmark the robustness of machine learning models. arXiv preprint arXiv:1707.04131 (2017)."},{"key":"e_1_3_2_1_45_1","doi-asserted-by":"publisher","DOI":"10.1007\/s11263-015-0816-y"},{"key":"e_1_3_2_1_46_1","volume-title":"Understanding measures of uncertainty for adversarial example detection. arXiv preprint arXiv:1803.08533","author":"Smith Lewis","year":"2018","unstructured":"Lewis Smith and Yarin Gal. 2018. Understanding measures of uncertainty for adversarial example detection. arXiv preprint arXiv:1803.08533 (2018)."},{"key":"e_1_3_2_1_47_1","volume-title":"Testing Deep Neural Networks. arXiv preprint arXiv:1803.04792","author":"Sun Youcheng","year":"2018","unstructured":"Youcheng Sun, Xiaowei Huang, and Daniel Kroening. 2018. Testing Deep Neural Networks. arXiv preprint arXiv:1803.04792 (2018)."},{"key":"e_1_3_2_1_48_1","unstructured":"Youcheng Sun Min Wu Wenjie Ruan Xiaowei Huang Marta Kwiatkowska and Daniel Kroening. 2018. Concolic Testing for Deep Neural Networks. (2018). https:\/\/doi.org\/arXiv:1805.00089v2 arXiv:1805.00089"},{"key":"e_1_3_2_1_49_1","volume-title":"Intriguing properties of neural networks. arXiv preprint arXiv:1312.6199","author":"Szegedy Christian","year":"2013","unstructured":"Christian Szegedy, Wojciech Zaremba, Ilya Sutskever, Joan Bruna, Dumitru Erhan, Ian Goodfellow, and Rob Fergus. 2013. Intriguing properties of neural networks. arXiv preprint arXiv:1312.6199 (2013)."},{"key":"e_1_3_2_1_50_1","first-page":"7728","article-title":"Attacks Meet Interpretability","volume":"31","author":"Tao Guanhong","year":"2018","unstructured":"Guanhong Tao, Shiqing Ma, Yingqi Liu, and Xiangyu Zhang. 2018. Attacks Meet Interpretability: Attribute-steered Detection of Adversarial Samples. In Advances in Neural Information Processing Systems 31. 7728--7739.","journal-title":"Attribute-steered Detection of Adversarial Samples. In Advances in Neural Information Processing Systems"},{"key":"e_1_3_2_1_51_1","unstructured":"The BBC. 2016. AI image recognition fooled by single pixel change. (2016). https:\/\/www.bbc.com\/news\/technology-41845878"},{"key":"e_1_3_2_1_52_1","doi-asserted-by":"publisher","DOI":"10.1145\/3180155.3180220"},{"key":"e_1_3_2_1_53_1","volume-title":"Adversarial Sample Detection for Deep Neural Network through Model Mutation Testing. arXiv preprint arXiv:1812.05793","author":"Wang Jingyi","year":"2018","unstructured":"Jingyi Wang, Guoliang Dong, Jun Sun, Xinyu Wang, and Peixin Zhang. 2018. Adversarial Sample Detection for Deep Neural Network through Model Mutation Testing. arXiv preprint arXiv:1812.05793 (2018)."},{"key":"e_1_3_2_1_54_1","volume-title":"https:\/\/sites.google.com\/view\/modeluncertainty","author":"Project Website Model Uncertainty","year":"2019","unstructured":"Model Uncertainty Project Website. 2019. (2019). https:\/\/sites.google.com\/view\/modeluncertainty"},{"key":"e_1_3_2_1_55_1","doi-asserted-by":"publisher","DOI":"10.1145\/3293882.3330579"},{"key":"e_1_3_2_1_56_1","doi-asserted-by":"publisher","DOI":"10.24963\/ijcai.2019\/800"},{"key":"e_1_3_2_1_57_1","volume-title":"Feature squeezing: Detecting adversarial examples in deep neural networks. arXiv preprint arXiv:1704.01155","author":"Xu Weilin","year":"2017","unstructured":"Weilin Xu, David Evans, and Yanjun Qi. 2017. Feature squeezing: Detecting adversarial examples in deep neural networks. arXiv preprint arXiv:1704.01155 (2017)."},{"key":"e_1_3_2_1_58_1","doi-asserted-by":"publisher","DOI":"10.1145\/3238147.3238187"},{"key":"e_1_3_2_1_59_1","volume-title":"Article 5 (Feb.","author":"Zhang Shuai","year":"2019","unstructured":"Shuai Zhang, Lina Yao, Aixin Sun, and Yi Tay. 2019. Deep Learning Based Recommender System: A Survey and New Perspectives. ACM Comput. Surv. 52, 1, Article 5 (Feb. 2019), 38 pages."},{"key":"e_1_3_2_1_60_1","doi-asserted-by":"publisher","DOI":"10.1145\/3213846.3213866"}],"event":{"name":"ICSE '20: 42nd International Conference on Software Engineering","location":"Seoul South Korea","acronym":"ICSE '20","sponsor":["SIGSOFT ACM Special Interest Group on Software Engineering","KIISE Korean Institute of Information Scientists and Engineers","IEEE CS"]},"container-title":["Proceedings of the ACM\/IEEE 42nd International Conference on Software Engineering"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3377811.3380368","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3377811.3380368","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T22:41:39Z","timestamp":1750200099000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3377811.3380368"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2020,6,27]]},"references-count":59,"alternative-id":["10.1145\/3377811.3380368","10.1145\/3377811"],"URL":"https:\/\/doi.org\/10.1145\/3377811.3380368","relation":{},"subject":[],"published":{"date-parts":[[2020,6,27]]},"assertion":[{"value":"2020-10-01","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}