{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,3,27]],"date-time":"2026-03-27T16:30:00Z","timestamp":1774629000357,"version":"3.50.1"},"publisher-location":"New York, NY, USA","reference-count":59,"publisher":"ACM","license":[{"start":{"date-parts":[[2020,12,21]],"date-time":"2020-12-21T00:00:00Z","timestamp":1608508800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"name":"Singapore National Research Foundation","award":["NSOE003-0001"],"award-info":[{"award-number":["NSOE003-0001"]}]},{"name":"NVIDIA AI Tech Center (NVAITC)"},{"name":"National Science Foundation of China","award":["61872262, 61572349"],"award-info":[{"award-number":["61872262, 61572349"]}]},{"name":"NRF Investigatorship","award":["NRFI06-2020-0022"],"award-info":[{"award-number":["NRFI06-2020-0022"]}]},{"name":"Singapore Ministry of Education Academic Research Fund Tier 1","award":["2018-T1-002-069"],"award-info":[{"award-number":["2018-T1-002-069"]}]},{"name":"National Research Foundation","award":["NRF2018NCR-NCR005-0001"],"award-info":[{"award-number":["NRF2018NCR-NCR005-0001"]}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2020,12,21]]},"DOI":"10.1145\/3324884.3416571","type":"proceedings-article","created":{"date-parts":[[2021,1,27]],"date-time":"2021-01-27T23:38:56Z","timestamp":1611790736000},"page":"486-498","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":71,"title":["Audee"],"prefix":"10.1145","author":[{"given":"Qianyu","family":"Guo","sequence":"first","affiliation":[{"name":"Tianjin University, China"}]},{"given":"Xiaofei","family":"Xie","sequence":"additional","affiliation":[{"name":"Nanyang Technological University, Singapore"}]},{"given":"Yi","family":"Li","sequence":"additional","affiliation":[{"name":"Nanyang Technological University, Singapore"}]},{"given":"Xiaoyu","family":"Zhang","sequence":"additional","affiliation":[{"name":"Xi'an Jiaotong University, China"}]},{"given":"Yang","family":"Liu","sequence":"additional","affiliation":[{"name":"Nanyang Technological University, Singapore"}]},{"given":"Xiaohong","family":"Li","sequence":"additional","affiliation":[{"name":"Tianjin University, China"}]},{"given":"Chao","family":"Shen","sequence":"additional","affiliation":[{"name":"Xi'an Jiaotong University, China"}]}],"member":"320","published-online":{"date-parts":[[2021,1,27]]},"reference":[{"key":"e_1_3_2_1_1_1","unstructured":"2018. Uber is giving up on self-driving cars in California after deadly crash. https:\/\/www.vice.com\/en_us\/article\/9kga85\/uber-is-giving-up-on-self-driving-cars-in-california-after-deadly-crash"},{"key":"e_1_3_2_1_2_1","unstructured":"2019. IMDb Dataset. https:\/\/www.imdb.com\/interfaces\/"},{"key":"e_1_3_2_1_3_1","unstructured":"2020. AUDEE. https:\/\/sites.google.com\/view\/audee"},{"key":"e_1_3_2_1_4_1","unstructured":"2020. Keras: The Python Deep Learning library. https:\/\/keras.io"},{"key":"e_1_3_2_1_5_1","unstructured":"2020. List of self-driving car fatalities. https:\/\/en.wikipedia.org\/wiki\/List_of_self-driving_car_fatalities#cite_note-15"},{"key":"e_1_3_2_1_6_1","volume-title":"12th USENIX Symposium on Operating Systems Design and Implementation (OSDI 16)","author":"Abadi Martin","year":"2016","unstructured":"Martin Abadi, Paul Barham, Jianmin Chen, Zhifeng Chen, Andy Davis, Jeffrey Dean, Matthieu Devin, Sanjay Ghemawat, Geoffrey Irving, Michael Isard, Manjunath Kudlur, Josh Levenberg, Rajat Monga, Sherry Moore, Derek G. Murray, Benoit Steiner, Paul Tucker, Vijay Vasudevan, Pete Warden, Martin Wicke, Yuan Yu, and Xiaoqiang Zheng. 2016. TensorFlow: A system for large-scale machine learning. In 12th USENIX Symposium on Operating Systems Design and Implementation (OSDI 16). 265--283."},{"key":"e_1_3_2_1_7_1","volume-title":"Towards evaluating the robustness of neural networks. In 2017 ieee symposium on security and privacy (sp)","author":"Carlini Nicholas","unstructured":"Nicholas Carlini and David Wagner. 2017. Towards evaluating the robustness of neural networks. In 2017 ieee symposium on security and privacy (sp). IEEE, 39--57."},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2015.312"},{"key":"e_1_3_2_1_9_1","unstructured":"CNTK. 2020. CNTK has supporting issues with GRU(unroll=true). https:\/\/github.com\/microsoft\/CNTK\/issues\/3800"},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.1145\/3338906.3338954"},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.1145\/3236024.3236057"},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.1145\/3213846.3213858"},{"key":"e_1_3_2_1_13_1","unstructured":"Facebook. 2020. ONNX. https:\/\/github.com\/onnx\/onnx"},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.1145\/2025113.2025179"},{"key":"e_1_3_2_1_15_1","volume-title":"Explaining and harnessing adversarial examples. arXiv preprint arXiv:1412.6572","author":"Goodfellow Ian J","year":"2014","unstructured":"Ian J Goodfellow, Jonathon Shlens, and Christian Szegedy. 2014. Explaining and harnessing adversarial examples. arXiv preprint arXiv:1412.6572 (2014)."},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICASSP.2013.6638947"},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.1109\/ASE.2019.00080"},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2016.90"},{"key":"e_1_3_2_1_19_1","volume-title":"Taxonomy of Real Faults in Deep Learning Systems. arXiv","author":"Humbatova Nargiz","year":"2019","unstructured":"Nargiz Humbatova, Gunel Jahangirova, Gabriele Bavota, Vincenzo Riccio, Andrea Stocco, and Paolo Tonella. 2019. Taxonomy of Real Faults in Deep Learning Systems. arXiv (2019), arXiv-1910."},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-63387-9_5"},{"key":"e_1_3_2_1_21_1","unstructured":"Keras. 2020. Keras has supporting issues with GRU (unroll=true) on the CNTK backend. https:\/\/github.com\/keras-team\/keras\/issues\/13852"},{"key":"e_1_3_2_1_22_1","unstructured":"Nair Krizhevsky Hinton Vinod Christopher Geoffrey Mike Papadakis and Anthony Ventresque. 2014. CIFAR-10 dataset. http:\/\/www.cs.toronto.edu\/kriz\/cifar.html."},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.1109\/5.726791"},{"key":"e_1_3_2_1_24_1","unstructured":"Yann LeCun and Corrina Cortes. 1998. The MNIST database of handwritten digits."},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICDCS.2018.00125"},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.1109\/ISBI.2014.6868045"},{"key":"e_1_3_2_1_27_1","unstructured":"Microsoft. 2020. MMdnn. https:\/\/github.com\/Microsoft\/MMdnn"},{"key":"e_1_3_2_1_28_1","unstructured":"M. Zalewski. [n.d.]. american fuzzy lop. http:\/\/lcamtuf.coredump.cx\/afl\/."},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.1109\/ASE.2019.00078"},{"key":"e_1_3_2_1_30_1","volume-title":"Tensorfuzz: Debugging neural networks with coverage-guided fuzzing. arXiv preprint arXiv:1807.10875","author":"Odena Augustus","year":"2018","unstructured":"Augustus Odena and Ian Goodfellow. 2018. Tensorfuzz: Debugging neural networks with coverage-guided fuzzing. arXiv preprint arXiv:1807.10875 (2018)."},{"key":"e_1_3_2_1_31_1","volume-title":"Improving Adversarial Robustness via Promoting Ensemble Diversity. CoRR abs\/1901.08846","author":"Pang Tianyu","year":"2019","unstructured":"Tianyu Pang, Kun Xu, Chao Du, Ning Chen, and Jun Zhu. 2019. Improving Adversarial Robustness via Promoting Ensemble Diversity. CoRR abs\/1901.08846 (2019). http:\/\/arxiv.org\/abs\/1901.08846"},{"key":"e_1_3_2_1_32_1","volume-title":"Automatic differentiation in PyTorch. openreview","author":"Paszke Adam","year":"2017","unstructured":"Adam Paszke, Sam Gross, Soumith Chintala, Gregory Chanan, Edward Yang, Zachary DeVito, Zeming Lin, Alban Desmaison, Luca Antiga, and Adam Lerer. 2017. Automatic differentiation in PyTorch. openreview (2017)."},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"publisher","DOI":"10.1145\/3132747.3132785"},{"key":"e_1_3_2_1_34_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE.2019.00107"},{"key":"e_1_3_2_1_35_1","unstructured":"Pytorch. 2020. AvgPool: Ensure all cells are valid in ceil mode. https:\/\/github.com\/pytorch\/pytorch\/pull\/41368"},{"key":"e_1_3_2_1_36_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2018.00474"},{"key":"e_1_3_2_1_37_1","doi-asserted-by":"publisher","DOI":"10.1145\/2939672.2945397"},{"key":"e_1_3_2_1_38_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICST.2019.00022"},{"key":"e_1_3_2_1_39_1","volume-title":"Very deep convolutional networks for large-scale image recognition. arXiv preprint arXiv:1409.1556","author":"Simonyan Karen","year":"2014","unstructured":"Karen Simonyan and Andrew Zisserman. 2014. Very deep convolutional networks for large-scale image recognition. arXiv preprint arXiv:1409.1556 (2014)."},{"key":"e_1_3_2_1_40_1","volume-title":"Workshops at the Thirty-Second AAAI Conference on Artificial Intelligence.","author":"Srisakaokul Siwakorn","year":"2018","unstructured":"Siwakorn Srisakaokul, Zhengkai Wu, Angello Astorga, Oreoluwa Alebiosu, and Tao Xie. 2018. Multiple-implementation testing of supervised learning software. In Workshops at the Thirty-Second AAAI Conference on Artificial Intelligence."},{"key":"e_1_3_2_1_41_1","unstructured":"Yi Sun Yuheng Chen Xiaogang Wang and Xiaoou Tang. 2014. Deep learning face representation by joint identification-verification. In Advances in neural information processing systems. 1988--1996."},{"key":"e_1_3_2_1_42_1","volume-title":"Theano: A Python framework for fast computation of mathematical expressions. arXiv preprint arXiv:1605.02688","author":"Development Team The Theano","year":"2016","unstructured":"The Theano Development Team, Rami Al-Rfou, Guillaume Alain, Amjad Almahairi, Christof Angermueller, Dzmitry Bahdanau, Nicolas Ballas, Fr\u00e9d\u00e9ric Bastien, Justin Bayer, Anatoly Belikov, et al. 2016. Theano: A Python framework for fast computation of mathematical expressions. arXiv preprint arXiv:1605.02688 (2016)."},{"key":"e_1_3_2_1_43_1","unstructured":"TensorFlow. 2019. SoftmaxOp leads to overflow. https:\/\/github.com\/uTensor\/uTensor\/issues\/175"},{"key":"e_1_3_2_1_44_1","unstructured":"TensorFlow. 2020. Checking if Kernel_size=0 in conv2d and reports error accordingly. https:\/\/github.com\/tensorflow\/tensorflow\/pull\/37395"},{"key":"e_1_3_2_1_45_1","unstructured":"TensorFlow. 2020. The fix of corner cases for the value None processing. https:\/\/github.com\/tensorflow\/tensorflow\/commit\/3db8df8ffafe5bcd83a12b92bc4c8287cd80237f"},{"key":"e_1_3_2_1_46_1","unstructured":"TensorFlow. 2020. The fix of missing check for the unreasonable parameter input_dim=0 in the layer Embedding. https:\/\/github.com\/tensorflow\/tensorflow\/commit\/f61175812426009a4c96e51befb2951612990903"},{"key":"e_1_3_2_1_47_1","unstructured":"TensorFlow. 2020. The output of Batch Normalization may contain Nan under certain parameters. https:\/\/github.com\/tensorflow\/tensorflow\/issues\/38644"},{"key":"e_1_3_2_1_48_1","unstructured":"TensorFlow. 2020. Tensorflow can build and even run a model with Conv2D kerne_size=0. https:\/\/github.com\/tensorflow\/tensorflow\/issues\/37334"},{"key":"e_1_3_2_1_49_1","unstructured":"Theano. 2020. Theano lacks a check for unreasonable parameters like dilation_rate=0 in Conv2D or DepthwiseConv2D. https:\/\/github.com\/Theano\/Theano\/issues\/6745"},{"key":"e_1_3_2_1_50_1","doi-asserted-by":"publisher","DOI":"10.1145\/3180155.3180220"},{"key":"e_1_3_2_1_51_1","unstructured":"Florian Tram\u00e8r Alexey Kurakin Nicolas Papernot Ian Goodfellow Dan Boneh and Patrick McDaniel. 2017. Ensemble Adversarial Training: Attacks and Defenses. arXiv:stat.ML\/1705.07204"},{"key":"e_1_3_2_1_52_1","doi-asserted-by":"publisher","DOI":"10.1145\/2955129.2955178"},{"key":"e_1_3_2_1_53_1","doi-asserted-by":"publisher","DOI":"10.1145\/3377811.3380358"},{"key":"e_1_3_2_1_54_1","unstructured":"Yonghui Wu Mike Schuster Zhifeng Chen Quoc V Le Mohammad Norouzi Wolfgang Macherey Maxim Krikun Yuan Cao Qin Gao Klaus Macherey et al. 2016. Google's neural machine translation system: Bridging the gap between human and machine translation. arXiv preprint arXiv:1609.08144 (2016)."},{"key":"e_1_3_2_1_55_1","doi-asserted-by":"publisher","DOI":"10.1145\/3293882.3330579"},{"key":"e_1_3_2_1_56_1","doi-asserted-by":"publisher","DOI":"10.24963\/ijcai.2019\/800"},{"key":"e_1_3_2_1_57_1","doi-asserted-by":"crossref","unstructured":"Xiyue Zhang Xiaofei Xie Lei Ma Xiaoning Du Qiang Hu Yang Liu Jianjun Zhao and Meng Sun. 2020. Towards Characterizing Adversarial Defects of Deep Learning Software from the Lens of Uncertainty.","DOI":"10.1145\/3377811.3380368"},{"key":"e_1_3_2_1_58_1","doi-asserted-by":"publisher","DOI":"10.1145\/3213846.3213866"},{"key":"e_1_3_2_1_59_1","first-page":"1","article-title":"IEEE standard for floating-point arithmetic","volume":"754","author":"Zuras Dan","year":"2008","unstructured":"Dan Zuras, Mike Cowlishaw, Alex Aiken, Matthew Applegate, David Bailey, Steve Bass, Dileep Bhandarkar, Mahesh Bhat, David Bindel, Sylvie Boldo, et al. 2008. IEEE standard for floating-point arithmetic. IEEE Std 754, 2008 (2008), 1--70.","journal-title":"IEEE Std"}],"event":{"name":"ASE '20: 35th IEEE\/ACM International Conference on Automated Software Engineering","location":"Virtual Event Australia","acronym":"ASE '20","sponsor":["SIGAI ACM Special Interest Group on Artificial Intelligence","SIGSOFT ACM Special Interest Group on Software Engineering","IEEE CS"]},"container-title":["Proceedings of the 35th IEEE\/ACM International Conference on Automated Software Engineering"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3324884.3416571","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3324884.3416571","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T22:01:38Z","timestamp":1750197698000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3324884.3416571"}},"subtitle":["automated testing for deep learning frameworks"],"short-title":[],"issued":{"date-parts":[[2020,12,21]]},"references-count":59,"alternative-id":["10.1145\/3324884.3416571","10.1145\/3324884"],"URL":"https:\/\/doi.org\/10.1145\/3324884.3416571","relation":{},"subject":[],"published":{"date-parts":[[2020,12,21]]},"assertion":[{"value":"2021-01-27","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}