{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,6,19]],"date-time":"2025-06-19T01:10:11Z","timestamp":1750295411609,"version":"3.41.0"},"publisher-location":"New York, NY, USA","reference-count":43,"publisher":"ACM","license":[{"start":{"date-parts":[[2023,10,27]],"date-time":"2023-10-27T00:00:00Z","timestamp":1698364800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2023,10,27]]},"DOI":"10.1145\/3650215.3650237","type":"proceedings-article","created":{"date-parts":[[2024,4,16]],"date-time":"2024-04-16T22:11:20Z","timestamp":1713305480000},"page":"113-119","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":1,"title":["Research on Deep Neural Network Testing Techniques"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-4004-1259","authenticated-orcid":false,"given":"Qianchao","family":"Hu","sequence":"first","affiliation":[{"name":"Institute of Systems Engineering, Academy of Military Sciences, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-2271-4197","authenticated-orcid":false,"given":"Feng","family":"Wang","sequence":"additional","affiliation":[{"name":"Institute of Systems Engineering, Academy of Military Sciences, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-6914-1424","authenticated-orcid":false,"given":"Binglin","family":"Liu","sequence":"additional","affiliation":[{"name":"Institute of Systems Engineering, Academy of Military Sciences, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-0946-8382","authenticated-orcid":false,"given":"Haitian","family":"Liu","sequence":"additional","affiliation":[{"name":"Institute of Systems Engineering, Academy of Military Sciences, China"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2024,4,16]]},"reference":[{"key":"e_1_3_2_1_1_1","volume-title":"Delving deep into rectifiers: Surpassing human-level performance on imagenet classification[C]\/\/Proceedings of the IEEE international conference on computer vision","author":"He K","year":"2015","unstructured":"He K, Zhang X, Ren S, Delving deep into rectifiers: Surpassing human-level performance on imagenet classification[C]\/\/Proceedings of the IEEE international conference on computer vision. 2015, 1026-1034."},{"key":"e_1_3_2_1_2_1","volume-title":"Intriguing properties of neural networks [J]. arXiv preprint arXiv:1312.6199","author":"Szegedy C","year":"2013","unstructured":"Szegedy C, Zaremba W, Sutskever I, Intriguing properties of neural networks [J]. arXiv preprint arXiv:1312.6199, 2013."},{"issue":"9","key":"e_1_3_2_1_3_1","first-page":"369","article-title":"semi autonomous driving, an uptick for future autonomy [J]","volume":"3","author":"Ingle S","year":"2016","unstructured":"Ingle S, Phute M. Tesla autopilot: semi autonomous driving, an uptick for future autonomy [J]. International Research Journal of Engineering and Technology, 2016, 3(9): 369-372.","journal-title":"International Research Journal of Engineering and Technology"},{"key":"e_1_3_2_1_4_1","volume-title":"Tencent Keen Security Lab","author":"Lab T K S","year":"2019","unstructured":"Lab T K S. Experimental security research of Tesla autopilot[J]. Tencent Keen Security Lab, 2019."},{"key":"e_1_3_2_1_5_1","volume-title":"The art of software testing[M]. Feng Wang","author":"Glenford J.","year":"2006","unstructured":"Glenford J. Myers, Tom Badgett, Corey Sandler. The art of software testing[M]. Feng Wang, Jie Chen, translation. Beijing, 2006."},{"key":"e_1_3_2_1_6_1","volume-title":"Research progress of software testing[J]","author":"Jinhui Shan","year":"2005","unstructured":"Jinhui Shan, Ying Jang, Ping Sun. Research progress of software testing[J]. Journal of Peking University (Natural Science), 2005, (1): 134-145."},{"key":"e_1_3_2_1_7_1","volume-title":"Changguo Guo, translation Beijing","author":"Aditya P.","year":"2011","unstructured":"Aditya P. Mathur. Software testing basics tutorial[M]. Feng Wang, Changguo Guo, translation Beijing: China Machine Press, 2011."},{"key":"e_1_3_2_1_8_1","volume-title":"Junfeng Yang","author":"Kexin Pei","year":"2017","unstructured":"Kexin Pei, Yinzhi Cao, Junfeng Yang, Deepxplore: Automated whitebox testing of deep learning systems[C]\/\/proceedings of the 26th Symposium on Operating Systems Principles. 2017, 1-18."},{"key":"e_1_3_2_1_9_1","volume-title":"Testing deep neural networks [J]. arXiv preprint arXiv:1803.04792","author":"Sun Y","year":"2018","unstructured":"Sun Y, Huang X, Kroening D, Testing deep neural networks [J]. arXiv preprint arXiv:1803.04792, 2018."},{"key":"e_1_3_2_1_10_1","volume-title":"Zhang F","author":"Ma L","year":"2018","unstructured":"Ma L, Juefei-Xu F, Zhang F, Deepgauge: Multi-granularity testing criteria for deep learning systems[C]\/\/Proceedings of the 33rd ACM\/IEEE international conference on automated software engineering. 2018, 120-131."},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.1109\/MSP.2012.2211477"},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2009.5206848"},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1109\/5.726791"},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2016.90"},{"key":"e_1_3_2_1_15_1","volume-title":"Computer Science","author":"Simonyan K","year":"2014","unstructured":"Simonyan K, Zisserman A. Very Deep Convolutional Networks for Large-Scale Image Recognition [J]. Computer Science, 2014."},{"key":"e_1_3_2_1_16_1","volume-title":"Explaining and harnessing adversarial examples [J]. arXiv preprint arXiv: 1412.6572","author":"Goodfellow I J","year":"2014","unstructured":"Goodfellow I J, Shlens J, Szegedy C. Explaining and harnessing adversarial examples [J]. arXiv preprint arXiv: 1412.6572, 2014."},{"key":"e_1_3_2_1_17_1","volume-title":"Towards Evaluating the Robustness of Neural Networks[C]\/\/2017 IEEE Symposium on Security and Privacy (SP)","author":"Carlini N","year":"2017","unstructured":"Carlini N, Wagner D. Towards Evaluating the Robustness of Neural Networks[C]\/\/2017 IEEE Symposium on Security and Privacy (SP). 2017."},{"key":"e_1_3_2_1_18_1","volume-title":"The Limitations of Deep Learning in Adversarial Settings [J]","author":"Papernot N","year":"2015","unstructured":"Papernot N, Mcdaniel P, Jha S, The Limitations of Deep Learning in Adversarial Settings [J]. IEEE, 2015."},{"key":"e_1_3_2_1_19_1","volume-title":"Adversarial examples in the physical world[M]\/\/Artificial intelligence safety and security","author":"Kurakin A","year":"2018","unstructured":"Kurakin A, Goodfellow I J, Bengio S. Adversarial examples in the physical world[M]\/\/Artificial intelligence safety and security. Chapman and Hall\/CRC, 2018, 99-112."},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"crossref","unstructured":"Ma L Juefei-Xu F Xue M DeepCT: Tomographic Combinatorial Testing for Deep Learning Systems[C]\/\/2019 IEEE 26th International Conference on Software Analysis Evolution and Reengineering (SANER). 2019.","DOI":"10.1109\/SANER.2019.8668044"},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE.2019.00108"},{"key":"e_1_3_2_1_22_1","volume-title":"Towards Improved Testing For Deep Learning[C]\/\/2019 IEEE\/ACM 41st International Conference on Software Engineering: New Ideas and Emerging Results (ICSE-NIER)","author":"Sekhon J","year":"2019","unstructured":"Sekhon J, Fleming C. Towards Improved Testing For Deep Learning[C]\/\/2019 IEEE\/ACM 41st International Conference on Software Engineering: New Ideas and Emerging Results (ICSE-NIER). 2019."},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.1145\/3395363.3397346"},{"key":"e_1_3_2_1_24_1","volume-title":"DeepPath: Path-Driven Testing Criteria for Deep Neural Networks[C]\/\/2019 IEEE International Conference On Artificial Intelligence Testing (AITest)","author":"Wang D","year":"2019","unstructured":"Wang D, Wang Z, Fang C, DeepPath: Path-Driven Testing Criteria for Deep Neural Networks[C]\/\/2019 IEEE International Conference On Artificial Intelligence Testing (AITest). 2019."},{"key":"e_1_3_2_1_25_1","volume-title":"DeepCruiser: Automated Guided Testing for Stateful Deep Learning Systems[J]","author":"Du X","year":"2018","unstructured":"Du X, Xie X, Li Y, DeepCruiser: Automated Guided Testing for Stateful Deep Learning Systems[J]. 2018."},{"key":"e_1_3_2_1_26_1","volume-title":"Importance-driven deep learning system testing[C]\/\/Proceedings of the ACM\/IEEE 42nd International Conference on Software Engineering","author":"Gerasimou S","year":"2020","unstructured":"Gerasimou S, Eniser H F, Sen A, Importance-driven deep learning system testing[C]\/\/Proceedings of the ACM\/IEEE 42nd International Conference on Software Engineering. 2020, 702-713."},{"key":"e_1_3_2_1_27_1","volume-title":"DeepMutation: Mutation Testing of Deep Learning Systems[J]","author":"Ma L","year":"2018","unstructured":"Ma L, Zhang F, Sun J, DeepMutation: Mutation Testing of Deep Learning Systems[J]. IEEE, 2018."},{"key":"e_1_3_2_1_28_1","volume-title":"MuNN: Mutation Analysis of Neural Networks[C]\/\/2018 IEEE International Conference on Software Quality, Reliability and Security Companion (QRS-C)","author":"Shen W","year":"2018","unstructured":"Shen W, Wan J, Chen Z. MuNN: Mutation Analysis of Neural Networks[C]\/\/2018 IEEE International Conference on Software Quality, Reliability and Security Companion (QRS-C). 2018, 108-115."},{"key":"e_1_3_2_1_29_1","volume-title":"Mutation Testing for Artificial Neural Networks: An Empirical Evaluation[C]\/\/2020 IEEE 20th International Conference on Software Quality, Reliability and Security (QRS)","author":"Klampfl L","year":"2020","unstructured":"Klampfl L, Chetouane N, Wotawa F. Mutation Testing for Artificial Neural Networks: An Empirical Evaluation[C]\/\/2020 IEEE 20th International Conference on Software Quality, Reliability and Security (QRS). 2020, 356-365."},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.infsof.2022.107129"},{"key":"e_1_3_2_1_31_1","volume-title":"Zhao Y","author":"Guo J","year":"2018","unstructured":"Guo J, Jiang Y, Zhao Y, Dlfuzz: Differential fuzzing testing of deep learning systems[C]\/\/Proceedings of the 2018 26th ACM Joint Meeting on European Software Engineering Conference and Symposium on the Foundations of Software Engineering. 2018, 739-743."},{"key":"e_1_3_2_1_32_1","first-page":"4901","volume-title":"PMLR","author":"Odena A","year":"2019","unstructured":"Odena A, Olsson C, Andersen D, Tensorfuzz: Debugging neural networks with coverage-guided fuzzing[C]\/\/International Conference on Machine Learning. PMLR, 2019, 4901-4911."},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.infsof.2023.107266"},{"key":"e_1_3_2_1_34_1","first-page":"53","volume-title":"IEEE","author":"Chen T Y","year":"2015","unstructured":"Chen T Y. Metamorphic testing: A simple method for alleviating the test oracle problem[C]\/\/2015 IEEE\/ACM 10th International Workshop on Automation of Software Test. IEEE, 2015, 53-54."},{"issue":"17","key":"e_1_3_2_1_35_1","first-page":"69","article-title":"Framework for machine Learning image classification programs [J]","volume":"56","author":"Liu Jialuo","year":"2020","unstructured":"Liu Jialuo, Yao Yi, Huang Song, Metamorphosis testing Framework for machine Learning image classification programs [J]. Computer Engineering and Applications, 2020, 56(17): 69-77.","journal-title":"Computer Engineering and Applications"},{"key":"e_1_3_2_1_36_1","volume-title":"Metamorphic testing: a new approach for generating next test cases[J]. arXiv preprint arXiv:2002.12543","author":"Chen T Y","year":"2020","unstructured":"Chen T Y, Cheung S C, Yiu S M. Metamorphic testing: a new approach for generating next test cases[J]. arXiv preprint arXiv:2002.12543, 2020."},{"key":"e_1_3_2_1_37_1","volume-title":"DeepRoad: GAN-Based Metamorphic Testing and Input Validation Framework for Autonomous Driving Systems[C]\/\/IEEE\/ACM International Conference on Automated Software Engineering","author":"Zhang M","year":"2018","unstructured":"Zhang M, Zhang Y, Zhang L, DeepRoad: GAN-Based Metamorphic Testing and Input Validation Framework for Autonomous Driving Systems[C]\/\/IEEE\/ACM International Conference on Automated Software Engineering. 2018."},{"key":"e_1_3_2_1_38_1","volume-title":"Jana S","author":"Tian Y","year":"2018","unstructured":"Tian Y, Pei K, Jana S, Deeptest: Automated testing of deep-neural-network-driven autonomous cars[C]\/\/Proceedings of the 40th international conference on software engineering. 2018: 303-314."},{"key":"e_1_3_2_1_39_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.infsof.2021.106701"},{"key":"e_1_3_2_1_40_1","doi-asserted-by":"publisher","DOI":"10.1145\/3508035"},{"volume-title":"Verification and Validation Workshops (ICSTW). IEEE","author":"Chandrasekaran J","key":"e_1_3_2_1_41_1","unstructured":"Chandrasekaran J, Lei Y, Kacker R, A combinatorial approach to testing deep neural network-based autonomous driving systems[C]\/\/2021 IEEE International Conference on Software Testing, Verification and Validation Workshops (ICSTW). IEEE, 2021: 57-66."},{"volume-title":"IEEE International Conference on Artificial Intelligence Testing (AITest). IEEE","author":"Chandrasekaran J","key":"e_1_3_2_1_42_1","unstructured":"Chandrasekaran J, Patel A R, Lei Y, Evaluation of T-Way Testing of DNNs in Autonomous Driving Systems[C]\/\/2021 IEEE International Conference on Artificial Intelligence Testing (AITest). IEEE, 2021: 17-18."},{"key":"e_1_3_2_1_43_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-21251-2_7"}],"event":{"name":"ICMLCA 2023: 2023 4th International Conference on Machine Learning and Computer Application","acronym":"ICMLCA 2023","location":"Hangzhou China"},"container-title":["2023 4th International Conference on Machine Learning and Computer Application"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3650215.3650237","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3650215.3650237","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,19]],"date-time":"2025-06-19T00:58:10Z","timestamp":1750294690000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3650215.3650237"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023,10,27]]},"references-count":43,"alternative-id":["10.1145\/3650215.3650237","10.1145\/3650215"],"URL":"https:\/\/doi.org\/10.1145\/3650215.3650237","relation":{},"subject":[],"published":{"date-parts":[[2023,10,27]]},"assertion":[{"value":"2024-04-16","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}