{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,3,27]],"date-time":"2025-03-27T18:28:47Z","timestamp":1743100127527,"version":"3.40.3"},"publisher-location":"Singapore","reference-count":50,"publisher":"Springer Nature Singapore","isbn-type":[{"type":"print","value":"9789819615476"},{"type":"electronic","value":"9789819615483"}],"license":[{"start":{"date-parts":[[2025,1,1]],"date-time":"2025-01-01T00:00:00Z","timestamp":1735689600000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2025,1,1]],"date-time":"2025-01-01T00:00:00Z","timestamp":1735689600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2025]]},"DOI":"10.1007\/978-981-96-1548-3_12","type":"book-chapter","created":{"date-parts":[[2025,2,16]],"date-time":"2025-02-16T09:15:54Z","timestamp":1739697354000},"page":"181-191","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":0,"title":["A Comprehensive Review on\u00a0Deep Learning System Testing"],"prefix":"10.1007","author":[{"given":"Ying","family":"Li","sequence":"first","affiliation":[]},{"given":"Chun","family":"Shan","sequence":"additional","affiliation":[]},{"given":"Zhen","family":"Liu","sequence":"additional","affiliation":[]},{"given":"Shuyan","family":"Liao","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2025,2,17]]},"reference":[{"key":"12_CR1","volume":"65","author":"S Bhattacharya","year":"2021","unstructured":"Bhattacharya, S., Maddikunta, P.K.R., Pham, Q.-V., et al.: Deep learning and medical image processing for coronavirus (covid-19) pandemic: a survey. Comput. Biol. Med. 65, 102589 (2021)","journal-title":"Comput. Biol. Med."},{"key":"12_CR2","doi-asserted-by":"crossref","unstructured":"Maqueda, A.I., Loquercio, A., Gallego, G., et\u00a0al.: Event-based vision meets deep learning on steering prediction for self-driving cars. In: 2018 IEEE\/CVF Conference on Computer Vision and Pattern Recognition (CVPR). IEEE Computer Society (2018)","DOI":"10.1109\/CVPR.2018.00568"},{"key":"12_CR3","doi-asserted-by":"crossref","unstructured":"Pei, K., Cao, Y., Yang, J., Jana, S.: Deepxplore: automated whitebox testing of deep learning systems. In: Proceedings of the 26th Symposium on Operating Systems Principles, pp. 1\u201318. ACM (2017)","DOI":"10.1145\/3132747.3132785"},{"issue":"1","key":"12_CR4","doi-asserted-by":"publisher","first-page":"53","DOI":"10.1109\/MSP.2017.2765202","volume":"35","author":"A Creswell","year":"2018","unstructured":"Creswell, A., White, T., Dumoulin, V., et al.: Generative adversarial networks: an overview. IEEE Signal Process. Mag. 35(1), 53\u201365 (2018)","journal-title":"IEEE Signal Process. Mag."},{"issue":"3","key":"12_CR5","first-page":"46","volume":"7","author":"TAN Qingyin","year":"2021","unstructured":"Qingyin, T.A.N., Yingming, Z.E.N.G., Ye, H.A.N., et al.: Survey on backdoor attacks targeted on neural network. Chin. J. Netw. Inf. Secur. 7(3), 46\u201358 (2021)","journal-title":"Chin. J. Netw. Inf. Secur."},{"key":"12_CR6","doi-asserted-by":"publisher","first-page":"131166","DOI":"10.1109\/ACCESS.2021.3114202","volume":"9","author":"F Rustamov","year":"2021","unstructured":"Rustamov, F., Kim, J., Jihyeon, Yu., Yun, J.: Exploratory review of hybrid fuzzing for automated vulnerability detection. IEEE Access 9, 131166\u2013131190 (2021)","journal-title":"IEEE Access"},{"key":"12_CR7","doi-asserted-by":"crossref","unstructured":"Papadakis, M., Kintis, M., Zhang, J., Jia, Y., Traon, Y.L., Harman, M.: Mutation testing advances: an analysis and survey. In: Advances in Computers, vol. 112, pp. 275\u2013378. Elsevier (2019)","DOI":"10.1016\/bs.adcom.2018.03.015"},{"key":"12_CR8","unstructured":"Goodfellow, I.J., Shlens, J., Szegedy, C.: Explaining and harnessing adversarial examples. arXiv preprint arxiv:1412.6572 (2014)"},{"key":"12_CR9","doi-asserted-by":"crossref","unstructured":"Cheng, C.H., Huang, C.H., Ruess, H., et\u00a0al.: Towards dependability metrics for neural networks. In: 2018 16th ACM\/IEEE International Conference on Formal Methods and Models for System Design (MEMOCODE). IEEE (2018)","DOI":"10.1109\/MEMCOD.2018.8556962"},{"key":"12_CR10","unstructured":"Wang, J., Sun, J., Zhang, P., et\u00a0al.: Detecting adversarial samples for deep neural networks through mutation testing. arXiv preprint arxiv:1805.00001 (2018)"},{"key":"12_CR11","doi-asserted-by":"crossref","unstructured":"Xu, W., Evans, D., Qi, Y.: Feature squeezing: detecting adversarial examples in deep neural networks. arXiv preprint arxiv:1704.01155 (2017)","DOI":"10.14722\/ndss.2018.23198"},{"key":"12_CR12","doi-asserted-by":"publisher","first-page":"182","DOI":"10.1016\/j.ins.2019.05.084","volume":"501","author":"F Guo","year":"2019","unstructured":"Guo, F., et al.: Detecting adversarial examples via prediction difference for deep neural networks. Inf. Sci. 501, 182\u2013192 (2019)","journal-title":"Inf. Sci."},{"key":"12_CR13","doi-asserted-by":"crossref","unstructured":"Wang, J., Dong, G., Sun, J., Wang, X., Zhang, P.: Adversarial sample detection for deep neural network through model mutation testing. In: 2019 IEEE\/ACM 41st International Conference on Software Engineering (ICSE), pp. 1245\u20131256. IEEE (2019)","DOI":"10.1109\/ICSE.2019.00126"},{"key":"12_CR14","doi-asserted-by":"publisher","first-page":"10339","DOI":"10.1007\/s11042-020-09167-z","volume":"80","author":"H Kwon","year":"2021","unstructured":"Kwon, H., Kim, Y., Yoon, H., Choi, D.: Classification score approach for detecting adversarial example in deep neural network. Multimedia Tools Appl. 80, 10339\u201310360 (2021)","journal-title":"Multimedia Tools Appl."},{"key":"12_CR15","doi-asserted-by":"crossref","unstructured":"Shen, Q., Ma, H., Chen, J., et\u00a0al.: A comprehensive study of deep learning compiler bugs. In: Proceedings of the 29th ACM Joint Meeting on European Software Engineering Conference and Symposium on the Foundations of Software Engineering (2021)","DOI":"10.1145\/3468264.3468591"},{"key":"12_CR16","unstructured":"Harzevili, N.S., Shin, J., Wang, J., et\u00a0al.: Characterizing and understanding software security vulnerabilities in machine learning libraries. arXiv preprint arxiv:2202.00001 (2022)"},{"key":"12_CR17","unstructured":"Lagouvardos, S., Dolby, J., Grech, N., et\u00a0al.: Static analysis of shape in tensorflow programs. In: 34th European Conference on Object-Oriented Programming (ECOOP 2020). Schloss Dagstuhl-Leibniz-Zentrum f\u00fcr Informatik (2020)"},{"key":"12_CR18","doi-asserted-by":"crossref","unstructured":"Liu, C., Lu, J., Li, G., et\u00a0al.: Detecting tensorflow program bugs in real-world industrial environment. In: 2021 36th IEEE\/ACM International Conference on Automated Software Engineering (ASE). IEEE (2021)","DOI":"10.1109\/ASE51524.2021.9678891"},{"key":"12_CR19","doi-asserted-by":"crossref","unstructured":"Pham, H.V., Lutellier, T., Qi, W., et\u00a0al.: Cradle: cross-backend validation to detect and localize bugs in deep learning libraries. In: 2019 IEEE\/ACM 41st International Conference on Software Engineering (ICSE). IEEE (2019)","DOI":"10.1109\/ICSE.2019.00107"},{"key":"12_CR20","doi-asserted-by":"crossref","unstructured":"Guo, Q., Xie, X., Li, Y., et\u00a0al.: Audee: automated testing for deep learning frameworks. In: 2020 35th IEEE\/ACM International Conference on Automated Software Engineering (ASE). IEEE (2020)","DOI":"10.1145\/3324884.3416571"},{"key":"12_CR21","doi-asserted-by":"crossref","unstructured":"Wang, Z., Yan, M., Chen, J., Liu, S., Zhang, D.: Deep learning library testing via effective model generation. In: Proceedings of the 28th ACM Joint Meeting on European Software Engineering Conference and Symposium on the Foundations of Software Engineering, pp. 788\u2013799 (2020)","DOI":"10.1145\/3368089.3409761"},{"key":"12_CR22","doi-asserted-by":"crossref","unstructured":"Gu, J., Luo, X., Zhou, Y., Wang, X.: Muffin: testing deep learning libraries via neural architecture fuzzing. In: Proceedings of the 44th International Conference on Software Engineering, pp. 1418\u20131430 (2022)","DOI":"10.1145\/3510003.3510092"},{"issue":"5","key":"12_CR23","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1145\/3583566","volume":"32","author":"M Li","year":"2023","unstructured":"Li, M., Cao, J., Tian, Y., Li, T.O., Wen, M., Cheung, S.C.: Comet: Coverage-guided model generation for deep learning library testing. ACM Trans. Softw. Eng. Methodol. 32(5), 1\u201334 (2023)","journal-title":"ACM Trans. Softw. Eng. Methodol."},{"key":"12_CR24","doi-asserted-by":"crossref","unstructured":"Liu, J., Peng, J., Wang, Y., Zhang, L.: Neuri: diversifying dnn generation via inductive rule inference. In: Proceedings of the 31st ACM Joint European Software Engineering Conference and Symposium on the Foundations of Software Engineering, pp. 657\u2013669 (2023)","DOI":"10.1145\/3611643.3616337"},{"key":"12_CR25","doi-asserted-by":"publisher","DOI":"10.1016\/j.jss.2023.111681","volume":"201","author":"Y Zou","year":"2023","unstructured":"Zou, Y., Sun, H., Fang, C., Liu, J., Zhang, Z.: Deep learning framework testing via hierarchical and heuristic model generation. J. Syst. Softw. 201, 111681 (2023)","journal-title":"J. Syst. Softw."},{"key":"12_CR26","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"604","DOI":"10.1007\/978-3-030-59410-7_40","volume-title":"Database Systems for Advanced Applications","author":"L Jia","year":"2020","unstructured":"Jia, L., Zhong, H., Wang, X., Huang, L., Lu, X.: An empirical study on bugs inside TensorFlow. In: Nah, Y., Cui, B., Lee, S.-W., Yu, J.X., Moon, Y.-S., Whang, S.E. (eds.) DASFAA 2020. LNCS, vol. 12112, pp. 604\u2013620. Springer, Cham (2020). https:\/\/doi.org\/10.1007\/978-3-030-59410-7_40"},{"key":"12_CR27","doi-asserted-by":"crossref","unstructured":"Zhang, X., et al.: Predoo: precision testing of deep learning operators. In: Proceedings of the 30th ACM SIGSOFT International Symposium on Software Testing and Analysis, pp. 400\u2013412 (2021)","DOI":"10.1145\/3460319.3464843"},{"key":"12_CR28","doi-asserted-by":"crossref","unstructured":"Wei, A., Deng, Y., Yang, C.Y., et\u00a0al.: Free lunch for testing: fuzzing deep-learning libraries from open source. arXiv preprint arxiv:2201.00001 (2022)","DOI":"10.1145\/3510003.3510041"},{"key":"12_CR29","doi-asserted-by":"crossref","unstructured":"Xie, D., Li, Y., Kim, M., et\u00a0al.: Docter: documentation-guided fuzzing for testing deep learning api functions. In: Proceedings of the 31st ACM SIGSOFT International Symposium on Software Testing and Analysis (2022)","DOI":"10.1145\/3533767.3534220"},{"key":"12_CR30","doi-asserted-by":"crossref","unstructured":"Deng, Y., Yang, C., Wei, A., Zhang, L.: Fuzzing deep-learning libraries via automated relational api inference. In: Proceedings of the 30th ACM Joint European Software Engineering Conference and Symposium on the Foundations of Software Engineering, pp. 44\u201356 (2022)","DOI":"10.1145\/3540250.3549085"},{"key":"12_CR31","doi-asserted-by":"crossref","unstructured":"Deng, Y., Xia, C.S., Peng, H., Yang, C., Zhang, L.: Large language models are zero-shot fuzzers: fuzzing deep-learning libraries via large language models. In: Proceedings of the 32nd ACM SIGSOFT International Symposium on Software Testing and Analysis, pp. 423\u2013435 (2023)","DOI":"10.1145\/3597926.3598067"},{"key":"12_CR32","doi-asserted-by":"crossref","unstructured":"Deng, Y., Xia, C.S., Yang, C., Zhang, S.D., Yang, S., Zhang, L.: Large language models are edge-case fuzzers: testing deep learning libraries via fuzzgpt. arXiv preprint arxiv:2304.02014 (2023)","DOI":"10.1145\/3597926.3598067"},{"key":"12_CR33","unstructured":"Deng, Z., Meng, G., Chen, K., Liu, T., Xiang, L., Chen, C.: Differential testing of cross deep learning framework $$\\{$$APIs$$\\}$$: revealing inconsistencies and vulnerabilities. In: 32nd USENIX Security Symposium (USENIX Security 23), pp. 7393\u20137410 (2023)"},{"key":"12_CR34","doi-asserted-by":"crossref","unstructured":"Wang, B., et al.: Neural cleanse: identifying and mitigating backdoor attacks in neural networks. In: 2019 IEEE Symposium on Security and Privacy (SP), pp. 707\u2013723. IEEE (2019)","DOI":"10.1109\/SP.2019.00031"},{"key":"12_CR35","doi-asserted-by":"crossref","unstructured":"Hossain, K.M., Oates, T.: Advancing security in AI systems: a novel approach to detecting backdoors in deep neural networks. arXiv e-prints, pages arXiv\u20132403 (2024)","DOI":"10.1109\/ICC51166.2024.10622312"},{"key":"12_CR36","volume-title":"Robust Statistics","author":"PJ Huber","year":"2011","unstructured":"Huber, P.J., Ronchetti, E.M.: Robust Statistics. John Wiley & Sons, Hoboken (2011)"},{"key":"12_CR37","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"313","DOI":"10.1007\/978-3-030-71500-7_16","volume-title":"Fundamental Approaches to Software Engineering","author":"Z Zhong","year":"2021","unstructured":"Zhong, Z., Tian, Y., Ray, B.: Understanding local robustness of deep neural networks under natural variations. In: FASE 2021. LNCS, vol. 12649, pp. 313\u2013337. Springer, Cham (2021). https:\/\/doi.org\/10.1007\/978-3-030-71500-7_16"},{"key":"12_CR38","doi-asserted-by":"crossref","unstructured":"Nanda, V., et\u00a0al.: Fairness through robustness: investigating robustness disparity in deep learning. In: Proceedings of the 2021 ACM Conference on Fairness, Accountability, and Transparency (2021)","DOI":"10.1145\/3442188.3445910"},{"key":"12_CR39","unstructured":"Guo, J., Bao, W., Wang, J., et\u00a0al.: A comprehensive evaluation framework for deep model robustness. arXiv preprint arxiv:2104.00710 (2021)"},{"key":"12_CR40","unstructured":"Ganin, Y., Lempitsky, V.: Unsupervised domain adaptation by backpropagation. In: International Conference on Machine Learning. PMLR (2015)"},{"issue":"3","key":"12_CR41","first-page":"578","volume":"50","author":"DENG Wei","year":"2020","unstructured":"Wei, D.E.N.G., Yuhan, X.I.N.G., Yifan, L.I., et al.: Survey on fair machine learning. CAAI Trans. Intell. Syst. 50(3), 578\u2013586 (2020)","journal-title":"CAAI Trans. Intell. Syst."},{"key":"12_CR42","unstructured":"Qian, S., et\u00a0al.: Are my deep learning systems fair? an empirical study of fixed-seed training. Adv. Neural Inf. Process. Syst. (2021)"},{"issue":"9","key":"12_CR43","doi-asserted-by":"publisher","first-page":"3593","DOI":"10.1109\/TSE.2021.3101478","volume":"48","author":"P Zhang","year":"2021","unstructured":"Zhang, P., et al.: Automatic fairness testing of neural classifiers through adversarial sampling. IEEE Trans. Softw. Eng. 48(9), 3593\u20133612 (2021)","journal-title":"IEEE Trans. Softw. Eng."},{"key":"12_CR44","doi-asserted-by":"crossref","unstructured":"Zhang, L., Zhang, Y., Zhang, M.: Efficient white-box fairness testing through gradient search. In: Proceedings of the 30th ACM SIGSOFT International Symposium on Software Testing and Analysis (2021)","DOI":"10.1145\/3460319.3464820"},{"key":"12_CR45","doi-asserted-by":"crossref","unstructured":"Zheng, H., et\u00a0al.: Neuronfair: interpretable white-box fairness testing through biased neuron identification. In: Proceedings of the 44th International Conference on Software Engineering (2022)","DOI":"10.1145\/3510003.3510123"},{"key":"12_CR46","doi-asserted-by":"crossref","unstructured":"Goyal, P., et al.: Fairness indicators for systematic assessments of visual feature extractors. In: Proceedings of the 2022 ACM Conference on Fairness, Accountability, and Transparency, pp. 70\u201388 (2022)","DOI":"10.1145\/3531146.3533074"},{"issue":"5","key":"12_CR47","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1145\/3236009","volume":"51","author":"R Guidotti","year":"2018","unstructured":"Guidotti, R., Monreale, A., Ruggieri, S., Turini, F., Giannotti, F., Pedreschi, D.: A survey of methods for explaining black box models. ACM Comput. Surv. (CSUR) 51(5), 1\u201342 (2018)","journal-title":"ACM Comput. Surv. (CSUR)"},{"key":"12_CR48","doi-asserted-by":"crossref","unstructured":"Selvaraju, R.R., Cogswell, M., Das, A., et\u00a0al.: Grad-cam: visual explanations from deep networks via gradient-based localization. In: Proceedings of the IEEE International Conference on Computer Vision (2017)","DOI":"10.1109\/ICCV.2017.74"},{"key":"12_CR49","unstructured":"Ramaswamy, H.G.: Ablation-cam: visual explanations for deep convolutional network via gradient-free localization. In: Proceedings of the IEEE\/CVF Winter Conference on Applications of Computer Vision (2020)"},{"key":"12_CR50","unstructured":"You, J., Leskovec, J., He, K., et\u00a0al.: Graph structure of neural networks. In: International Conference on Machine Learning. PMLR (2020)"}],"container-title":["Lecture Notes in Computer Science","Algorithms and Architectures for Parallel Processing"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-981-96-1548-3_12","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,2,16]],"date-time":"2025-02-16T09:16:39Z","timestamp":1739697399000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-981-96-1548-3_12"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025]]},"ISBN":["9789819615476","9789819615483"],"references-count":50,"URL":"https:\/\/doi.org\/10.1007\/978-981-96-1548-3_12","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"type":"print","value":"0302-9743"},{"type":"electronic","value":"1611-3349"}],"subject":[],"published":{"date-parts":[[2025]]},"assertion":[{"value":"17 February 2025","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"ICA3PP","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"International Conference on Algorithms and Architectures for Parallel Processing","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Macau","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"China","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2024","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"30 October 2024","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"1 November 2024","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"24","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"ica3pp2024","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"https:\/\/ica3pp2024.scimeeting.cn\/en\/web\/index\/","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}}]}}