{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,3,28]],"date-time":"2026-03-28T10:38:27Z","timestamp":1774694307661,"version":"3.50.1"},"reference-count":71,"publisher":"IEEE","license":[{"start":{"date-parts":[[2021,10,1]],"date-time":"2021-10-01T00:00:00Z","timestamp":1633046400000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2021,10,1]],"date-time":"2021-10-01T00:00:00Z","timestamp":1633046400000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2021,10]]},"DOI":"10.1109\/issre52982.2021.00041","type":"proceedings-article","created":{"date-parts":[[2022,2,12]],"date-time":"2022-02-12T01:01:25Z","timestamp":1644627685000},"page":"309-320","source":"Crossref","is-referenced-by-count":10,"title":["Black-Box Testing of Deep Neural Networks"],"prefix":"10.1109","author":[{"given":"Taejoon","family":"Byun","sequence":"first","affiliation":[{"name":"University of Minnesota,Minneapolis,MN,United States"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Sanjai","family":"Rayadurgam","sequence":"additional","affiliation":[{"name":"University of Minnesota,Minneapolis,MN,United States"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Mats P.E.","family":"Heimdahl","sequence":"additional","affiliation":[{"name":"University of Minnesota,Minneapolis,MN,United States"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"263","reference":[{"key":"ref71","doi-asserted-by":"publisher","DOI":"10.1109\/DASC52595.2021.9594360"},{"key":"ref70","doi-asserted-by":"publisher","DOI":"10.1109\/SBST52555.2021.00020"},{"key":"ref39","doi-asserted-by":"publisher","DOI":"10.1145\/2950290.2950324"},{"key":"ref38","doi-asserted-by":"publisher","DOI":"10.1145\/2661136.2661157"},{"key":"ref33","first-page":"97","article-title":"Manifold for Machine Learning Assurance","author":"byun","year":"2020","journal-title":"2020 IEEE\/ACM 42nd International Conference on Software Engineering New Ideas and Emerging Results (ICSE-NIER) ICSE-NIER"},{"key":"ref32","first-page":"178","article-title":"Specification test coverage adequacy criteria = specification test generation inadequacy criteria","author":"heimdahl","year":"0","journal-title":"Eighth IEEE International Symposium on High Assurance Systems Engineering 2004 Proceedings"},{"key":"ref31","author":"tan","year":"2020","journal-title":"Efficientnet Rethinking model scaling for convolutional neural networks"},{"key":"ref30","author":"gerasimou","year":"2020","journal-title":"Importance-driven deep learning system testing"},{"key":"ref37","doi-asserted-by":"crossref","first-page":"34","DOI":"10.1126\/science.153.3731.34","article-title":"Dynamic programming","volume":"153","author":"bellman","year":"1966","journal-title":"Science"},{"key":"ref36","article-title":"Tutorial on variational autoencoders","author":"doersch","year":"2016","journal-title":"ArXiv"},{"key":"ref35","doi-asserted-by":"publisher","DOI":"10.1109\/TPAMI.2013.50"},{"key":"ref34","doi-asserted-by":"publisher","DOI":"10.1109\/ICSTW.2013.77"},{"key":"ref60","first-page":"1","author":"recht","year":"2019","journal-title":"Do imagenet classifiers generalize to imagenet?"},{"key":"ref62","doi-asserted-by":"publisher","DOI":"10.1145\/3368089.3409754"},{"key":"ref61","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE-NIER.2019.00031"},{"key":"ref63","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-55583-2_21"},{"key":"ref28","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE-NIER.2019.00030"},{"key":"ref64","article-title":"Intriguing properties of neural networks","author":"szegedy","year":"2013","journal-title":"ArXiv Preprint"},{"key":"ref27","author":"du","year":"2018","journal-title":"Deepcruiser Automated guided testing for stateful deep learning systems"},{"key":"ref65","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2015.7298640"},{"key":"ref66","doi-asserted-by":"publisher","DOI":"10.1109\/TNNLS.2018.2886017"},{"key":"ref29","first-page":"1","author":"ma","year":"2018","journal-title":"Combinatorial testing for deep learning systems"},{"key":"ref67","doi-asserted-by":"publisher","DOI":"10.1145\/3238147.3238187"},{"key":"ref68","article-title":"Manifold-based test generation for image classifiers","author":"byun","year":"0","journal-title":"2020 IEEE International"},{"key":"ref69","first-page":"1178","article-title":"Adversarial vulnerability for any classifier","author":"fawzi","year":"2018","journal-title":"Advances in neural information processing systems"},{"key":"ref2","doi-asserted-by":"publisher","DOI":"10.1145\/1146238.1146242"},{"key":"ref1","doi-asserted-by":"publisher","DOI":"10.1109\/ASE.2006.31"},{"key":"ref20","doi-asserted-by":"publisher","DOI":"10.1145\/3132747.3132785"},{"key":"ref22","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE.2019.00108"},{"key":"ref21","doi-asserted-by":"publisher","DOI":"10.1145\/3238147.3238202"},{"key":"ref24","doi-asserted-by":"publisher","DOI":"10.1109\/SANER.2019.8668044"},{"key":"ref23","doi-asserted-by":"publisher","DOI":"10.1109\/AITest.2019.00013"},{"key":"ref26","article-title":"Testing deep neural networks","volume":"abs 1803 4792","author":"sun","year":"2018","journal-title":"CoRR"},{"key":"ref25","doi-asserted-by":"publisher","DOI":"10.1145\/3238147.3238172"},{"key":"ref50","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2015.2421011"},{"key":"ref51","doi-asserted-by":"publisher","DOI":"10.1109\/ISSRE.2017.33"},{"key":"ref59","author":"ioffe","year":"2015","journal-title":"Batch Normalization Accelerating Deep Network Training by Reducing Internal Covariate Shift"},{"key":"ref58","first-page":"2172","article-title":"Infogan: Interpretable representation learning by information maximizing generative adversarial nets","volume":"29","author":"chen","year":"2016","journal-title":"Advances in neural information processing systems"},{"key":"ref57","article-title":"Diagnosing and enhancing VAE models","volume":"abs 1903 5789","year":"2019","journal-title":"CoRR"},{"key":"ref56","author":"dai","year":"2019","journal-title":"TwoStageVAE"},{"key":"ref55","doi-asserted-by":"crossref","first-page":"818","DOI":"10.1007\/978-3-319-10590-1_53","article-title":"Visualizing and understanding convolutional networks","author":"zeiler","year":"2014","journal-title":"Computer Vision - ECCV 2014"},{"key":"ref54","author":"mu","year":"2019","journal-title":"Mnist-c A robustness benchmark for computer vision"},{"key":"ref53","article-title":"EMNIST: an extension of MNIST to handwritten letters","volume":"abs 1702 5373","author":"cohen","year":"2017","journal-title":"CoRR"},{"key":"ref52","doi-asserted-by":"publisher","DOI":"10.1145\/2568225.2568271"},{"key":"ref10","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE-SEIP.2019.00042"},{"key":"ref11","doi-asserted-by":"publisher","DOI":"10.1145\/1764810.1764814"},{"key":"ref40","first-page":"537","article-title":"Are mutation scores correlated with real fault detection? a large scale empirical study on the relationship between mutants and real faults","author":"papadakis","year":"0","journal-title":"2018 IEEE\/ACM 40th International Conference on Software Engineering (ICSE)"},{"key":"ref12","author":"kurakin","year":"2016","journal-title":"Adversarial machine learning at scale"},{"key":"ref13","doi-asserted-by":"publisher","DOI":"10.1016\/j.infsof.2012.06.002"},{"key":"ref14","doi-asserted-by":"publisher","DOI":"10.4204\/EPTCS.257.3"},{"key":"ref15","first-page":"1","article-title":"Benchmarking neural network robust-ness to common corruptions and perturbations","author":"hendrycks","year":"0","journal-title":"7th International Conference on Learning Representations ICLR 2019"},{"key":"ref16","author":"kawaguchi","year":"2020","journal-title":"Generalization in deep learning"},{"key":"ref17","first-page":"1","author":"alzantot","year":"2018","journal-title":"Generating natural adversarial examples"},{"key":"ref18","author":"hendrycks","year":"2019","journal-title":"Natural adversarial examples"},{"key":"ref19","author":"beizer","year":"2003","journal-title":"Software Testing Techniques"},{"key":"ref4","author":"krizhevsky","year":"2009","journal-title":"Learning multiple layers of features from tiny images"},{"key":"ref3","author":"lecun","year":"1998","journal-title":"The MNIST Database of Handwritten Digits"},{"key":"ref6","doi-asserted-by":"publisher","DOI":"10.1109\/AITest.2019.000-6"},{"key":"ref5","article-title":"Become a self-driving car engineer","year":"2015","journal-title":"Udacity"},{"key":"ref8","article-title":"Machine learning testing: Survey, landscapes and horizons","volume":"abs 1906 10742","author":"zhang","year":"2019","journal-title":"CoRR"},{"key":"ref7","first-page":"1","author":"recht","year":"2018","journal-title":"Do CIFAR-10 Classifiers Generalize to CIFAR-10?"},{"key":"ref49","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-28872-2_28"},{"key":"ref9","doi-asserted-by":"publisher","DOI":"10.1145\/3377811.3380395"},{"key":"ref46","doi-asserted-by":"publisher","DOI":"10.1109\/5.726791"},{"key":"ref45","author":"li","year":"2017","journal-title":"cifar-10-cnn Play deep learning with cifar datasets"},{"key":"ref48","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2016.90"},{"key":"ref47","author":"lin","year":"2013","journal-title":"Network in Network"},{"key":"ref42","first-page":"1050","article-title":"Dropout as a bayesian approximation: Representing model uncertainty in deep learning","author":"gal","year":"0","journal-title":"Proceedings of the 33rd International Conference on International Conference on Machine Learning - Volume 48 ser ICML'16"},{"key":"ref41","doi-asserted-by":"publisher","DOI":"10.1145\/3395363.3397357"},{"key":"ref44","first-page":"265","article-title":"Tensorflow: A system for large-scale machine learning","author":"abadi","year":"0","journal-title":"12th USENIX Symposium on Operating Systems Design and Implementation ( OSDI 16)"},{"key":"ref43","article-title":"What uncertainties do we need in bayesian deep learning for computer vision?","volume":"abs 1703 4977","author":"kendall","year":"2017","journal-title":"CoRR"}],"event":{"name":"2021 IEEE 32nd International Symposium on Software Reliability Engineering (ISSRE)","location":"Wuhan, China","start":{"date-parts":[[2021,10,25]]},"end":{"date-parts":[[2021,10,28]]}},"container-title":["2021 IEEE 32nd International Symposium on Software Reliability Engineering (ISSRE)"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx7\/9700160\/9700163\/09700360.pdf?arnumber=9700360","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2022,6,13]],"date-time":"2022-06-13T21:07:22Z","timestamp":1655154442000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/9700360\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2021,10]]},"references-count":71,"URL":"https:\/\/doi.org\/10.1109\/issre52982.2021.00041","relation":{},"subject":[],"published":{"date-parts":[[2021,10]]}}}