{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,1,13]],"date-time":"2026-01-13T15:24:16Z","timestamp":1768317856913,"version":"3.49.0"},"publisher-location":"New York, NY, USA","reference-count":68,"publisher":"ACM","license":[{"start":{"date-parts":[[2022,5,21]],"date-time":"2022-05-21T00:00:00Z","timestamp":1653091200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"name":"National Natural Science Foundation of China","award":["62002158, 61832009, 61932012"],"award-info":[{"award-number":["62002158, 61832009, 61932012"]}]},{"name":"Science, Technology and Innovation commission of Shenzhen Municipality","award":["CJGJZD20200617103001003"],"award-info":[{"award-number":["CJGJZD20200617103001003"]}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2022,5,21]]},"DOI":"10.1145\/3510003.3510232","type":"proceedings-article","created":{"date-parts":[[2022,7,5]],"date-time":"2022-07-05T22:42:59Z","timestamp":1657060979000},"page":"73-85","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":49,"title":["Adaptive test selection for deep neural networks"],"prefix":"10.1145","author":[{"given":"Xinyu","family":"Gao","sequence":"first","affiliation":[{"name":"Nanjing University, Nanjing, China"}]},{"given":"Yang","family":"Feng","sequence":"additional","affiliation":[{"name":"Nanjing University, Nanjing, China"}]},{"given":"Yining","family":"Yin","sequence":"additional","affiliation":[{"name":"Nanjing University, Nanjing, China"}]},{"given":"Zixi","family":"Liu","sequence":"additional","affiliation":[{"name":"Nanjing University, Nanjing, China"}]},{"given":"Zhenyu","family":"Chen","sequence":"additional","affiliation":[{"name":"Nanjing University, Nanjing, China"}]},{"given":"Baowen","family":"Xu","sequence":"additional","affiliation":[{"name":"Nanjing University, Nanjing, China"}]}],"member":"320","published-online":{"date-parts":[[2022,7,5]]},"reference":[{"key":"e_1_3_2_1_1_1","unstructured":"[n.d.]. Amazon promises fix for creepy Alexa laugh - BBC News. https:\/\/www.bbc.com\/news\/technology-43325230. (Accessed on 08\/23\/2021)."},{"key":"e_1_3_2_1_2_1","unstructured":"[n.d.]. A Google self-driving car caused a crash for the first time - The Verge https:\/\/www.theverge.com\/2016\/2\/29\/11134344\/google-self-driving-car-crash-report. (Accessed on 09\/04\/2021)."},{"key":"e_1_3_2_1_3_1","unstructured":"[n.d.]. Tesla's Latest Autopilot Death Looks Just Like a Prior Crash | WIRED. https:\/\/www.wired.com\/story\/teslas-latest-autopilot-death-looks-like-prior-crash\/. (Accessed on 09\/04\/2021)."},{"key":"e_1_3_2_1_4_1","volume-title":"Data diversity: An approach to software fault tolerance. Ieee transactions on computers 37, 4","author":"Ammann Paul Eric","year":"1988","unstructured":"Paul Eric Ammann and John C Knight. 1988. Data diversity: An approach to software fault tolerance. Ieee transactions on computers 37, 4 (1988), 418--425."},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.jss.2013.02.061"},{"key":"e_1_3_2_1_6_1","volume-title":"Neural machine translation by jointly learning to align and translate. arXiv preprint arXiv:1409.0473","author":"Bahdanau Dzmitry","year":"2014","unstructured":"Dzmitry Bahdanau, Kyunghyun Cho, and Yoshua Bengio. 2014. Neural machine translation by jointly learning to align and translate. arXiv preprint arXiv:1409.0473 (2014)."},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"publisher","DOI":"10.1109\/TC.2016.2547380"},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.1109\/FTCS.1993.627312"},{"key":"e_1_3_2_1_9_1","volume-title":"Davide Del Testa","author":"Bojarski Mariusz","year":"2016","unstructured":"Mariusz Bojarski, Davide Del Testa, Daniel Dworakowski, Bernhard Firner, Beat Flepp, Prasoon Goyal, Lawrence D Jackel, Mathew Monfort, Urs Muller, Jiakai Zhang, et al. 2016. End to end learning for self-driving cars. arXiv preprint arXiv:1604.07316 (2016)."},{"key":"e_1_3_2_1_10_1","volume-title":"Linear algebra with applications","author":"Bretscher Otto","unstructured":"Otto Bretscher. 1997. Linear algebra with applications. Prentice Hall Eaglewood Cliffs, NJ."},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.ins.2011.01.025"},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.1016\/0950-5849(96)01103-2"},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1142\/S0218194006002926"},{"key":"e_1_3_2_1_14_1","volume-title":"Deep neural network test coverage: How far are we? arXiv preprint arXiv:2010.04946","author":"Chen Junjie","year":"2020","unstructured":"Junjie Chen, Ming Yan, Zan Wang, Yuning Kang, and Zhuo Wu. 2020. Deep neural network test coverage: How far are we? arXiv preprint arXiv:2010.04946 (2020)."},{"key":"e_1_3_2_1_15_1","unstructured":"Tsong Yueh Chen Hing Leung and I. K. Mak. 2004. Adaptive Random Testing. (2004)."},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.1109\/QSIC.2004.1357947"},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.1109\/TR.2007.903293"},{"key":"e_1_3_2_1_18_1","volume-title":"Quality Software, 2004. QSIC 2004. Proceedings. Fourth International Conference on.","author":"Chen T. Y.","unstructured":"T. Y. Chen, R. G. Merkel, P. K. Wong, and G. Eddy. 2004. Adaptive random testing through dynamic partitioning. In Quality Software, 2004. QSIC 2004. Proceedings. Fourth International Conference on."},{"key":"e_1_3_2_1_19_1","first-page":"746","article-title":"Reducing labeling effort for structured prediction tasks","volume":"5","author":"Culotta Aron","year":"2005","unstructured":"Aron Culotta and Andrew McCallum. 2005. Reducing labeling effort for structured prediction tasks. In AAAI, Vol. 5. 746--751.","journal-title":"AAAI"},{"key":"e_1_3_2_1_20_1","volume-title":"A Wilcoxon-type test for trend. Statistics in medicine 4, 1","author":"Cuzick Jack","year":"1985","unstructured":"Jack Cuzick. 1985. A Wilcoxon-type test for trend. Statistics in medicine 4, 1 (1985), 87--90."},{"key":"e_1_3_2_1_21_1","volume-title":"On stochastic orderings of the Wilcoxon rank sum test statistic---with applications to reproducibility probability estimation testing. Statistics & probability letters 81, 8","author":"De Capitani L","year":"2011","unstructured":"L De Capitani and D De Martini. 2011. On stochastic orderings of the Wilcoxon rank sum test statistic---with applications to reproducibility probability estimation testing. Statistics & probability letters 81, 8 (2011), 937--946."},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICST.2013.27"},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.1145\/3395363.3397357"},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"publisher","DOI":"10.1016\/0951-8320(91)90052-9"},{"key":"e_1_3_2_1_25_1","volume-title":"Foundations of set theory","author":"Fraenkel Abraham Adolf","unstructured":"Abraham Adolf Fraenkel, Yehoshua Bar-Hillel, and Azriel Levy. 1973. Foundations of set theory. Elsevier."},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.1145\/3236024.3264835"},{"key":"e_1_3_2_1_27_1","doi-asserted-by":"publisher","DOI":"10.1145\/3368089.3409754"},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2016.90"},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.1109\/ASE.2019.00126"},{"key":"e_1_3_2_1_30_1","volume-title":"An Empirical Evaluation of Mutation Operators for Deep Learning Systems. In 2020 IEEE 13th International Conference on Software Testing, Validation and Verification (ICST).","author":"Jahangirova Gunel","year":"2020","unstructured":"Gunel Jahangirova and Paolo Tonella. 2020. An Empirical Evaluation of Mutation Operators for Deep Learning Systems. In 2020 IEEE 13th International Conference on Software Testing, Validation and Verification (ICST)."},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"publisher","DOI":"10.1109\/ASE.2009.77"},{"key":"e_1_3_2_1_32_1","volume-title":"Guiding Deep Learning System Testing Using Surprise Adequacy. In 2019 IEEE\/ACM 41st International Conference on Software Engineering (ICSE).","author":"Kim Jinhan","year":"2019","unstructured":"Jinhan Kim, Robert Feldt, and Shin Yoo. 2019. Guiding Deep Learning System Testing Using Surprise Adequacy. In 2019 IEEE\/ACM 41st International Conference on Software Engineering (ICSE)."},{"key":"e_1_3_2_1_33_1","unstructured":"A. Krizhevsky and G. Hinton. 2009. Learning multiple layers of features from tiny images. Handbook of Systemic Autoimmune Diseases 1 4 (2009)."},{"key":"e_1_3_2_1_34_1","unstructured":"Alex Krizhevsky Vinod Nair and Geoffrey Hinton. 2009. CIFAR10. https:\/\/www.cs.toronto.edu\/~kriz\/cifar.html."},{"key":"e_1_3_2_1_35_1","unstructured":"Alexey Kurakin Ian Goodfellow Samy Bengio et al. 2016. Adversarial examples in the physical world."},{"key":"e_1_3_2_1_36_1","article-title":"Exploring strategies for training deep neural networks","volume":"10","author":"Larochelle Hugo","year":"2009","unstructured":"Hugo Larochelle, Yoshua Bengio, J\u00e9r\u00f4me Louradour, and Pascal Lamblin. 2009. Exploring strategies for training deep neural networks. Journal of machine learning research 10, 1 (2009).","journal-title":"Journal of machine learning research"},{"key":"e_1_3_2_1_37_1","doi-asserted-by":"publisher","DOI":"10.1109\/5.726791"},{"key":"e_1_3_2_1_38_1","volume-title":"SIGIR' 94","author":"Lewis David D","unstructured":"David D Lewis and William A Gale. 1994. A sequential algorithm for training text classifiers. In SIGIR' 94. Springer, 3--12."},{"key":"e_1_3_2_1_39_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE-NIER.2019.00031"},{"key":"e_1_3_2_1_40_1","doi-asserted-by":"publisher","DOI":"10.1145\/3338906.3338930"},{"key":"e_1_3_2_1_41_1","doi-asserted-by":"publisher","DOI":"10.1145\/3238147.3238202"},{"key":"e_1_3_2_1_42_1","doi-asserted-by":"publisher","DOI":"10.1109\/ISSRE.2018.00021"},{"key":"e_1_3_2_1_43_1","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1145\/3417330","article-title":"Test selection for deep learning systems","volume":"30","author":"Ma Wei","year":"2021","unstructured":"Wei Ma, Mike Papadakis, Anestis Tsakmalis, Maxime Cordy, and Yves Le Traon. 2021. Test selection for deep learning systems. ACM Transactions on Software Engineering and Methodology (TOSEM) 30, 2 (2021), 1--22.","journal-title":"ACM Transactions on Software Engineering and Methodology (TOSEM)"},{"key":"e_1_3_2_1_44_1","doi-asserted-by":"publisher","DOI":"10.1109\/ISSRE.1995.497647"},{"key":"e_1_3_2_1_45_1","doi-asserted-by":"publisher","DOI":"10.1145\/1101908.1101963"},{"key":"e_1_3_2_1_46_1","volume-title":"Data augmentation for improving deep learning in image classification problem. In 2018 international interdisciplinary PhD workshop (IIPhDW)","author":"Miko\u0142ajczyk Agnieszka","unstructured":"Agnieszka Miko\u0142ajczyk and Micha\u0142 Grochowski. 2018. Data augmentation for improving deep learning in image classification problem. In 2018 international interdisciplinary PhD workshop (IIPhDW). IEEE, 117--122."},{"key":"e_1_3_2_1_47_1","unstructured":"Yuval Netzer Tao Wang Adam Coates Alessandro Bissacco Bo Wu and Andrew Y Ng. 2011. Reading digits in natural images with unsupervised feature learning. (2011)."},{"key":"e_1_3_2_1_48_1","doi-asserted-by":"publisher","DOI":"10.1109\/EuroSP.2016.36"},{"key":"e_1_3_2_1_49_1","doi-asserted-by":"publisher","DOI":"10.1145\/3132747.3132785"},{"key":"e_1_3_2_1_50_1","unstructured":"Burr Settles. 2009. Active learning literature survey. (2009)."},{"key":"e_1_3_2_1_51_1","volume-title":"Active Learning and Experimental Design workshop In conjunction with AISTATS 2010 (Proceedings of Machine Learning Research)","author":"Settles Burr","unstructured":"Burr Settles. 2011. From Theories to Queries: Active Learning in Practice. In Active Learning and Experimental Design workshop In conjunction with AISTATS 2010 (Proceedings of Machine Learning Research), Isabelle Guyon, Gavin Cawley, Gideon Dror, Vincent Lemaire, and Alexander Statnikov (Eds.), Vol. 16. PMLR, Sardinia, Italy, 1--18. https:\/\/proceedings.mlr.press\/v16\/settles11a.html"},{"key":"e_1_3_2_1_52_1","doi-asserted-by":"publisher","DOI":"10.3115\/1613715.1613855"},{"key":"e_1_3_2_1_53_1","doi-asserted-by":"publisher","DOI":"10.1186\/s40537-019-0197-0"},{"key":"e_1_3_2_1_54_1","volume-title":"Very deep convolutional networks for large-scale image recognition. arXiv preprint arXiv:1409.1556","author":"Simonyan Karen","year":"2014","unstructured":"Karen Simonyan and Andrew Zisserman. 2014. Very deep convolutional networks for large-scale image recognition. arXiv preprint arXiv:1409.1556 (2014)."},{"key":"e_1_3_2_1_55_1","unstructured":"Satinder P Singh Michael J Kearns Diane J Litman and Marilyn A Walker. 2000. Reinforcement learning for spoken dialogue systems. In Advances in Neural Information Processing Systems. 956--962."},{"key":"e_1_3_2_1_56_1","volume-title":"Introduction to linear algebra","author":"Strang Gilbert","unstructured":"Gilbert Strang, Gilbert Strang, Gilbert Strang, and Gilbert Strang. 1993. Introduction to linear algebra. Vol. 3. Wellesley-Cambridge Press Wellesley, MA."},{"key":"e_1_3_2_1_57_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE-Companion.2019.00051"},{"key":"e_1_3_2_1_58_1","unstructured":"Youcheng Sun Xiaowei Huang Daniel Kroening James Sharp Matthew Hill and Rob Ashmore. 2019. Testing Deep Neural Networks. arXiv:cs.LG\/1803.04792"},{"key":"e_1_3_2_1_59_1","doi-asserted-by":"crossref","unstructured":"Yuchi Tian Kexin Pei Suman Jana and Baishakhi Ray. 2017. DeepTest: Automated Testing of Deep-Neural-Network-driven Autonomous Cars. (2017).","DOI":"10.1145\/3180155.3180220"},{"key":"e_1_3_2_1_60_1","volume-title":"Proceedings of the Korea Society of Information Technology Applications Conference. The Korea Society of Information Technology Applications, 138--138","author":"Towey Dave","year":"2007","unstructured":"Dave Towey. 2007. Adaptive random testing; ubiquitous testing to support ubiquitous computing. In Proceedings of the Korea Society of Information Technology Applications Conference. The Korea Society of Information Technology Applications, 138--138."},{"key":"e_1_3_2_1_61_1","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.1980.234486"},{"key":"e_1_3_2_1_62_1","unstructured":"Han Xiao Kashif Rasul and Roland Vollgraf. 2017. Fashion-MNIST: a Novel Image Dataset for Benchmarking Machine Learning Algorithms. arXiv:cs.LG\/cs.LG\/1708.07747"},{"key":"e_1_3_2_1_63_1","doi-asserted-by":"publisher","DOI":"10.1145\/3293882.3330579"},{"key":"e_1_3_2_1_64_1","volume-title":"Corinna Cortes.","author":"Christopher","year":"1998","unstructured":"Christopher J.C. Burges Yann LeCun, Corinna Cortes. 1998. MNIST. http:\/\/yann.lecun.com\/exdb\/mnist\/."},{"key":"e_1_3_2_1_65_1","doi-asserted-by":"publisher","DOI":"10.1002\/stv.430"},{"key":"e_1_3_2_1_66_1","unstructured":"Xueying Zhan Huan Liu Qing Li and Antoni B Chan. [n.d.]. A Comparative Survey: Benchmarking for Pool-based Active Learning ([n. d.])."},{"key":"e_1_3_2_1_67_1","doi-asserted-by":"publisher","DOI":"10.1145\/3238147.3238187"},{"key":"e_1_3_2_1_68_1","doi-asserted-by":"publisher","DOI":"10.1002\/sam.11161"}],"event":{"name":"ICSE '22: 44th International Conference on Software Engineering","location":"Pittsburgh Pennsylvania","acronym":"ICSE '22","sponsor":["SIGSOFT ACM Special Interest Group on Software Engineering","IEEE CS"]},"container-title":["Proceedings of the 44th International Conference on Software Engineering"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3510003.3510232","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3510003.3510232","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T20:12:24Z","timestamp":1750191144000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3510003.3510232"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2022,5,21]]},"references-count":68,"alternative-id":["10.1145\/3510003.3510232","10.1145\/3510003"],"URL":"https:\/\/doi.org\/10.1145\/3510003.3510232","relation":{},"subject":[],"published":{"date-parts":[[2022,5,21]]},"assertion":[{"value":"2022-07-05","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}