{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,2,21]],"date-time":"2025-02-21T07:19:39Z","timestamp":1740122379246,"version":"3.37.3"},"reference-count":66,"publisher":"Springer Science and Business Media LLC","issue":"5","license":[{"start":{"date-parts":[[2024,7,25]],"date-time":"2024-07-25T00:00:00Z","timestamp":1721865600000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2024,7,25]],"date-time":"2024-07-25T00:00:00Z","timestamp":1721865600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"funder":[{"DOI":"10.13039\/501100001809","name":"Joint Funds of the National Natural Science Foundation of China","doi-asserted-by":"crossref","award":["U224120044"],"award-info":[{"award-number":["U224120044"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"crossref"}]},{"DOI":"10.13039\/501100012226","name":"Fundamental Research Funds for the Central Universities","doi-asserted-by":"publisher","award":["NT2022027"],"award-info":[{"award-number":["NT2022027"]}],"id":[{"id":"10.13039\/501100012226","id-type":"DOI","asserted-by":"publisher"}]},{"name":"The Open Foundation of State Key Laboratory for Novel Software Technology of China","award":["KFKT2021B32"],"award-info":[{"award-number":["KFKT2021B32"]}]}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Empir Software Eng"],"published-print":{"date-parts":[[2024,9]]},"DOI":"10.1007\/s10664-024-10524-x","type":"journal-article","created":{"date-parts":[[2024,7,25]],"date-time":"2024-07-25T08:01:54Z","timestamp":1721894514000},"update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":0,"title":["Neuron importance-aware coverage analysis for deep neural network testing"],"prefix":"10.1007","volume":"29","author":[{"ORCID":"https:\/\/orcid.org\/0000-0001-5286-874X","authenticated-orcid":false,"given":"Hongjing","family":"Guo","sequence":"first","affiliation":[]},{"given":"Chuanqi","family":"Tao","sequence":"additional","affiliation":[]},{"given":"Zhiqiu","family":"Huang","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2024,7,25]]},"reference":[{"key":"10524_CR1","unstructured":"Uda (2016) The udacity open source self-driving car project. https:\/\/github.com\/ udacity\/self-driving-car"},{"issue":"7","key":"10524_CR2","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1371\/journal.pone.0130140","volume":"10","author":"S Bach","year":"2015","unstructured":"Bach S, Binder A, Montavon G, Klauschen F, M\u00fcller KR, Samek W (2015) On pixel-wise explanations for non-linear classifier decisions by layer-wise relevance propagation. PLOS ONE 10(7):1\u201346. https:\/\/doi.org\/10.1371\/journal.pone.0130140","journal-title":"PLOS ONE"},{"key":"10524_CR3","doi-asserted-by":"publisher","unstructured":"Carlini N, Wagner DA (2017) Towards evaluating the robustness of neural networks. In: 2017 IEEE Symposium on security and privacy, SP 2017, San Jose, CA, USA, May 22\u201326, 2017, IEEE Computer Society, pp 39\u201357. https:\/\/doi.org\/10.1109\/SP.2017.49","DOI":"10.1109\/SP.2017.49"},{"key":"10524_CR4","doi-asserted-by":"publisher","unstructured":"Chen J, Wu Z, Wang Z, You H, Zhang L, Yan M (2020a) Practical accuracy estimation for efficient deep neural network testing. ACM Trans Softw Eng Methodol 29(4):30:1\u201330:35. https:\/\/doi.org\/10.1145\/3394112","DOI":"10.1145\/3394112"},{"key":"10524_CR5","unstructured":"Chen J, Yan M, Wang Z, Kang Y, Wu Z (2020b) Deep neural network test coverage: How far are we? CoRR arXiv:2010.04946"},{"key":"10524_CR6","doi-asserted-by":"publisher","unstructured":"Chen Z, Huang X (2017) End-to-end learning for lane keeping of self-driving cars. In: IEEE Intelligent vehicles symposium, IV 2017, Los Angeles, CA, USA, June 11-14, 2017, IEEE, pp 1856\u20131860. https:\/\/doi.org\/10.1109\/IVS.2017.7995975","DOI":"10.1109\/IVS.2017.7995975"},{"key":"10524_CR7","doi-asserted-by":"publisher","unstructured":"Du X, Xie X, Li Y, Ma L, Liu Y, Zhao J (2019) Deepstellar: model-based quantitative analysis of stateful deep learning systems. In: Dumas M, Pfahl D, Apel S, Russo A (eds) Proceedings of the ACM joint meeting on european software engineering conference and symposium on the foundations of software engineering, ESEC\/SIGSOFT FSE 2019, Tallinn, Estonia, August 26\u201330, 2019, ACM, pp 477\u2013487. https:\/\/doi.org\/10.1145\/3338906.3338954","DOI":"10.1145\/3338906.3338954"},{"issue":"7639","key":"10524_CR8","doi-asserted-by":"publisher","first-page":"115","DOI":"10.1038\/nature21056","volume":"542","author":"A Esteva","year":"2017","unstructured":"Esteva A, Kuprel B, Novoa RA, Ko J, Swetter SM, Blau HM, Thrun S (2017) Dermatologist-level classification of skin cancer with deep neural networks. Nat 542(7639):115\u2013118. https:\/\/doi.org\/10.1038\/nature21056","journal-title":"Nat"},{"key":"10524_CR9","doi-asserted-by":"publisher","unstructured":"Feng Y, Shi Q, Gao X, Wan J, Fang C, Chen Z (2020) Deepgini: prioritizing massive tests to enhance the robustness of deep neural networks. In: Khurshid S, Pasareanu CS (eds) ISSTA \u201920: 29th ACM SIGSOFT International symposium on software testing and analysis, virtual event, USA, July 18\u201322, 2020, ACM, pp 177\u2013188. https:\/\/doi.org\/10.1145\/3395363.3397357","DOI":"10.1145\/3395363.3397357"},{"key":"10524_CR10","unstructured":"Gal Y, Ghahramani Z (2016) Dropout as a bayesian approximation: Representing model uncertainty in deep learning. In: Balcan M, Weinberger KQ (eds) Proceedings of the 33nd international conference on machine learning, ICML 2016, New York City, NY, USA, June 19-24, 2016, JMLR.org, JMLR Workshop and Conference Proceedings, vol\u00a048, pp 1050\u20131059. http:\/\/proceedings.mlr.press\/v48\/gal16.html"},{"key":"10524_CR11","doi-asserted-by":"publisher","unstructured":"Gerasimou S, Eniser HF, Sen A, Cakan A (2020) Importance-driven deep learning system testing. In: Rothermel G, Bae D (eds) ICSE \u201920: 42nd International conference on software engineering, Seoul, South Korea, 27 June - 19 July, 2020, ACM, pp 702\u2013713. https:\/\/doi.org\/10.1145\/3377811.3380391","DOI":"10.1145\/3377811.3380391"},{"key":"10524_CR12","unstructured":"Goodfellow IJ, Shlens J, Szegedy C (2015) Explaining and harnessing adversarial examples. In: Bengio Y, LeCun Y (eds) 3rd International conference on learning representations, ICLR 2015, San Diego, CA, USA, May 7-9, 2015, Conference Track Proceedings. arXiv:1412.6572"},{"key":"10524_CR13","unstructured":"Goodfellow IJ, Papernot N, McDaniel PD (2016) cleverhans v0.1: an adversarial machine learning library. CoRR arXiv:1610.00768"},{"key":"10524_CR14","doi-asserted-by":"publisher","unstructured":"Guo J, Jiang Y, Zhao Y, Chen Q, Sun J (2018) Dlfuzz: differential fuzzing testing of deep learning systems. In: Leavens GT, Garcia A, Pasareanu CS (eds) Proceedings of the 2018 ACM joint meeting on european software engineering conference and symposium on the foundations of software engineering, ESEC\/SIGSOFT FSE 2018, Lake Buena Vista, FL, USA, November 04-09, 2018, ACM, pp 739\u2013743. https:\/\/doi.org\/10.1145\/3236024.3264835","DOI":"10.1145\/3236024.3264835"},{"issue":"6","key":"10524_CR15","doi-asserted-by":"publisher","first-page":"141","DOI":"10.1007\/s10664-022-10172-z","volume":"27","author":"FU Haq","year":"2022","unstructured":"Haq FU, Shin D, Nejati S, Briand LC (2022) Correction to: Can offline testing of deep neural networks replace their online testing? Empir Softw Eng 27(6):141. https:\/\/doi.org\/10.1007\/s10664-022-10172-z","journal-title":"Empir Softw Eng"},{"key":"10524_CR16","doi-asserted-by":"publisher","unstructured":"Harel-Canada F, Wang L, Gulzar MA, Gu Q, Kim M (2020) Is neuron coverage a meaningful measure for testing deep neural networks? In: Devanbu P, Cohen MB, Zimmermann T (eds) ESEC\/FSE \u201920: 28th ACM joint european software engineering conference and symposium on the foundations of software engineering, virtual event, USA, November 8\u201313, 2020, ACM, pp 851\u2013862. https:\/\/doi.org\/10.1145\/3368089.3409754","DOI":"10.1145\/3368089.3409754"},{"key":"10524_CR17","doi-asserted-by":"publisher","unstructured":"Houle ME (2017) Local intrinsic dimensionality II: multivariate analysis and distributional support. In: Beecks C, Borutta F, Kr\u00f6ger P, Seidl T (eds) Similarity Search and Applications - 10th International Conference, SISAP 2017, Munich, Germany, October 4-6, 2017, Proceedings, Springer, Lecture Notes in Computer Science, vol 10609, pp 80\u201395. https:\/\/doi.org\/10.1007\/978-3-319-68474-1_6","DOI":"10.1007\/978-3-319-68474-1_6"},{"key":"10524_CR18","doi-asserted-by":"publisher","unstructured":"Hu Q, Ma L, Xie X, Yu B, Liu Y, Zhao J (2019) Deepmutation++: A mutation testing framework for deep learning systems. In: 34th IEEE\/ACM International conference on automated software engineering, ASE 2019, San Diego, CA, USA, November 11\u201315, 2019, IEEE, pp 1158\u20131161. https:\/\/doi.org\/10.1109\/ASE.2019.00126","DOI":"10.1109\/ASE.2019.00126"},{"key":"10524_CR19","doi-asserted-by":"crossref","unstructured":"Huang W, Sun Y, Zhao XE, Sharp J, Ruan W, Meng J, Huang X (2021) Coverage-guided testing for recurrent neural networks. IEEE Transactions on Reliability","DOI":"10.1109\/TR.2021.3080664"},{"key":"10524_CR20","doi-asserted-by":"publisher","DOI":"10.1016\/j.cosrev.2020.100270","volume":"37","author":"X Huang","year":"2020","unstructured":"Huang X, Kroening D, Ruan W, Sharp J, Sun Y, Thamo E, Wu M, Yi X (2020) A survey of safety and trustworthiness of deep neural networks: Verification, testing, adversarial attack and defence, and interpretability. Comput Sci Rev 37:100270. https:\/\/doi.org\/10.1016\/j.cosrev.2020.100270","journal-title":"Comput Sci Rev"},{"key":"10524_CR21","doi-asserted-by":"publisher","unstructured":"Humbatova N, Jahangirova G, Tonella P (2021) Deepcrime: mutation testing of deep learning systems based on real faults. In: Cadar C, Zhang X (eds) ISSTA \u201921: 30th ACM SIGSOFT International symposium on software testing and analysis, virtual event, Denmark, July 11\u201317, 2021, ACM, pp 67\u201378. https:\/\/doi.org\/10.1145\/3460319.3464825","DOI":"10.1145\/3460319.3464825"},{"issue":"3","key":"10524_CR22","doi-asserted-by":"publisher","first-page":"598","DOI":"10.2514\/1.G003724","volume":"42","author":"KD Julian","year":"2019","unstructured":"Julian KD, Kochenderfer MJ, Owen MP (2019) Deep neural network compression for aircraft collision avoidance systems. J Guid Control Dyn 42(3):598\u2013608. https:\/\/doi.org\/10.2514\/1.G003724","journal-title":"J Guid Control Dyn"},{"key":"10524_CR23","doi-asserted-by":"publisher","unstructured":"Karger DR, Ruhl M (2002) Finding nearest neighbors in growth-restricted metrics. In: Reif JH (ed) Proceedings on 34th Annual ACM Symposium on Theory of Computing, May 19\u201321, 2002, Montr\u00e9al, Qu\u00e9bec, Canada, ACM, pp 741\u2013750. https:\/\/doi.org\/10.1145\/509907.510013","DOI":"10.1145\/509907.510013"},{"key":"10524_CR24","doi-asserted-by":"publisher","unstructured":"Kim J, Feldt R, Yoo S (2019) Guiding deep learning system testing using surprise adequacy. In: Atlee JM, Bultan T, Whittle J (eds) Proceedings of the 41st international conference on software engineering, ICSE 2019, Montreal, QC, Canada, May 25\u201331, 2019, IEEE \/ ACM, pp 1039\u20131049. https:\/\/doi.org\/10.1109\/ICSE.2019.00108","DOI":"10.1109\/ICSE.2019.00108"},{"key":"10524_CR25","doi-asserted-by":"publisher","unstructured":"Kim J, Ju J, Feldt R, Yoo S (2020) Reducing DNN labelling cost using surprise adequacy: an industrial case study for autonomous driving. In: Devanbu P, Cohen MB, Zimmermann T (eds) ESEC\/FSE \u201920: 28th ACM Joint european software engineering conference and symposium on the foundations of software engineering, virtual event, USA, November 8\u201313, 2020, ACM, pp 1466\u20131476. https:\/\/doi.org\/10.1145\/3368089.3417065","DOI":"10.1145\/3368089.3417065"},{"key":"10524_CR26","doi-asserted-by":"publisher","unstructured":"Kim J, Feldt R, Yoo S (2023) Evaluating surprise adequacy for deep learning system testing. ACM Trans Softw Eng Methodol 32(2):42:1\u201342:29. https:\/\/doi.org\/10.1145\/3546947","DOI":"10.1145\/3546947"},{"key":"10524_CR27","doi-asserted-by":"publisher","unstructured":"Kim S, Yoo S (2020) Evaluating surprise adequacy for question answering. In: ICSE \u201920: 42nd International conference on software engineering, Workshops, Seoul, Republic of Korea, 27 June - 19 July, 2020, ACM, pp 197\u2013202. https:\/\/doi.org\/10.1145\/3387940.3391465","DOI":"10.1145\/3387940.3391465"},{"key":"10524_CR28","doi-asserted-by":"publisher","unstructured":"Kim S, Yoo S (2021) Multimodal surprise adequacy analysis of inputs for natural language processing DNN models. In: 2nd IEEE\/ACM International conference on automation of software test, AST@ICSE 2021, Madrid, Spain, May 20-21, 2021, IEEE, pp 80\u201389. https:\/\/doi.org\/10.1109\/AST52587.2021.00017","DOI":"10.1109\/AST52587.2021.00017"},{"key":"10524_CR29","unstructured":"Krizhevsky N, Vinod H, Geoffrey C, Papadakis M, Ventresque A (2014a) The cifar-10 dataset. http:\/\/www.cs.toronto.edu\/~kriz\/cifar.html"},{"key":"10524_CR30","unstructured":"Krizhevsky N, Vinod H, Geoffrey C, Papadakis M, Ventresque A (2014b) The cifar-100 dataset. http:\/\/www.cs.toronto.edu\/~kriz\/cifar.html"},{"key":"10524_CR31","unstructured":"Kurakin A, Goodfellow IJ, Bengio S (2017) Adversarial examples in the physical world. In: 5th International conference on learning representations, ICLR 2017, Toulon, France, April 24-26, 2017, Workshop Track Proceedings, OpenReview.net. https:\/\/openreview.net\/forum?id=HJGU3Rodl"},{"key":"10524_CR32","unstructured":"Lakshminarayanan B, Pritzel A, Blundell C (2017) Simple and scalable predictive uncertainty estimation using deep ensembles. In: Guyon I, von Luxburg U, Bengio S, Wallach HM, Fergus R, Vishwanathan SVN, Garnett R (eds) Advances in neural information processing systems 30: Annual Conference on Neural Information Processing Systems 2017, December 4-9, 2017, Long Beach, CA, USA, pp 6402\u20136413. https:\/\/proceedings.neurips.cc\/paper\/2017\/hash\/9ef2ed4b7fd2c810847ffa5fa85bce38-Abstract.html"},{"key":"10524_CR33","unstructured":"LeCun Y, Cortes C (1998) The mnist database of handwritten digits. http:\/\/yann.lecun.com\/exdb\/mnist\/"},{"issue":"11","key":"10524_CR34","doi-asserted-by":"publisher","first-page":"2278","DOI":"10.1109\/5.726791","volume":"86","author":"Y LeCun","year":"1998","unstructured":"LeCun Y, Bottou L, Bengio Y, Haffner P (1998) Gradient-based learning applied to document recognition. Proc IEEE 86(11):2278\u20132324. https:\/\/doi.org\/10.1109\/5.726791","journal-title":"Proc IEEE"},{"key":"10524_CR35","doi-asserted-by":"publisher","unstructured":"Lee S, Cha S, Lee D, Oh H (2020) Effective white-box testing of deep neural networks with adaptive neuron-selection strategy. In: Khurshid S, Pasareanu CS (eds) ISSTA \u201920: 29th ACM SIGSOFT International symposium on software testing and analysis, virtual event, USA, July 18-22, 2020, ACM, pp 165\u2013176. https:\/\/doi.org\/10.1145\/3395363.3397346","DOI":"10.1145\/3395363.3397346"},{"key":"10524_CR36","doi-asserted-by":"publisher","unstructured":"Li Z, Ma X, Xu C, Cao C (2019a) Structural coverage criteria for neural networks could be misleading. In: Sarma A, Murta L (eds) Proceedings of the 41st International Conference on Software Engineering: New Ideas and Emerging Results, ICSE (NIER) 2019, Montreal, QC, Canada, May 29-31, 2019, IEEE \/ ACM, pp 89\u201392. https:\/\/doi.org\/10.1109\/ICSE-NIER.2019.00031","DOI":"10.1109\/ICSE-NIER.2019.00031"},{"key":"10524_CR37","doi-asserted-by":"publisher","unstructured":"Li Z, Ma X, Xu C, Cao C, Xu J, L\u00fc J (2019b) Boosting operational DNN testing efficiency through conditioning. In: Dumas M, Pfahl D, Apel S, Russo A (eds) Proceedings of the ACM joint meeting on european software engineering conference and symposium on the foundations of software engineering, ESEC\/SIGSOFT FSE 2019, Tallinn, Estonia, August 26\u201330, 2019, ACM, pp 499\u2013509. https:\/\/doi.org\/10.1145\/3338906.3338930","DOI":"10.1145\/3338906.3338930"},{"key":"10524_CR38","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-14267-3","author":"T Liu","year":"2011","unstructured":"Liu T (2011) Learning to Rank for Information Retrieval. Springer. https:\/\/doi.org\/10.1007\/978-3-642-14267-3","journal-title":"Springer"},{"key":"10524_CR39","doi-asserted-by":"publisher","unstructured":"Ma L, Juefei-Xu F, Zhang F, Sun J, Xue M, Li B, Chen C, Su T, Li L, Liu Y, Zhao J, Wang Y (2018a) Deepgauge: multi-granularity testing criteria for deep learning systems. In: Huchard M, K\u00e4stner C, Fraser G (eds) Proceedings of the 33rd ACM\/IEEE international conference on automated software engineering, ASE 2018, Montpellier, France, September 3-7, 2018, ACM, pp 120\u2013131. https:\/\/doi.org\/10.1145\/3238147.3238202","DOI":"10.1145\/3238147.3238202"},{"key":"10524_CR40","doi-asserted-by":"publisher","unstructured":"Ma L, Zhang F, Sun J, Xue M, Li B, Juefei-Xu F, Xie C, Li L, Liu Y, Zhao J, Wang Y (2018b) Deepmutation: Mutation testing of deep learning systems. In: Ghosh S, Natella R, Cukic B, Poston RS, Laranjeiro N (eds) 29th IEEE International symposium on software reliability engineering, ISSRE 2018, Memphis, TN, USA, October 15-18, 2018, IEEE Computer Society, pp 100\u2013111. https:\/\/doi.org\/10.1109\/ISSRE.2018.00021","DOI":"10.1109\/ISSRE.2018.00021"},{"key":"10524_CR41","doi-asserted-by":"publisher","unstructured":"Ma L, Juefei-Xu F, Xue M, Li B, Li L, Liu Y, Zhao J (2019) Deepct: Tomographic combinatorial testing for deep learning systems. In: Wang X, Lo D, Shihab E (eds) 26th IEEE International conference on software analysis, evolution and reengineering, SANER 2019, Hangzhou, China, February 24-27, 2019, IEEE, pp 614\u2013618. https:\/\/doi.org\/10.1109\/SANER.2019.8668044","DOI":"10.1109\/SANER.2019.8668044"},{"key":"10524_CR42","doi-asserted-by":"publisher","unstructured":"Ma W, Papadakis M, Tsakmalis A, Cordy M, Traon YL (2021) Test selection for deep learning systems. ACM Trans Softw Eng Methodol 30(2):13:1\u201313:22. https:\/\/doi.org\/10.1145\/3417330","DOI":"10.1145\/3417330"},{"key":"10524_CR43","unstructured":"Ma X, Li B, Wang Y, Erfani SM, Wijewickrema SNR, Schoenebeck G, Song D, Houle ME, Bailey J (2018c) Characterizing adversarial subspaces using local intrinsic dimensionality. In: 6th International conference on learning representations, ICLR 2018, Vancouver, BC, Canada, April 30 - May 3, 2018, Conference Track Proceedings, OpenReview.net. https:\/\/openreview.net\/forum?id=B1gJ1L2aW"},{"key":"10524_CR44","unstructured":"Netzer Y, Wang T, Coates A, Bissacco A, Wu B, Ng A (2011) Reading digits in natural images with unsupervised feature learning"},{"key":"10524_CR45","unstructured":"Odena A, Olsson C, Andersen DG, Goodfellow IJ (2019) Tensorfuzz: Debugging neural networks with coverage-guided fuzzing. In: Chaudhuri K, Salakhutdinov R (eds) Proceedings of the 36th international conference on machine learning, ICML 2019, 9-15 June 2019, Long Beach, California, USA, PMLR, Proceedings of Machine Learning Research, vol\u00a097, pp 4901\u20134911. http:\/\/proceedings.mlr.press\/v97\/odena19a.html"},{"key":"10524_CR46","doi-asserted-by":"publisher","unstructured":"Ouyang T, Isobe Y, Marco VS, Ogata J, Seo Y, Oiwa Y (2021) AI robustness analysis with consideration of corner cases. In: 2021 IEEE International Conference on Artificial Intelligence Testing, AITest 2021, Oxford, United Kingdom, August 23-26, 2021, IEEE, pp 29\u201336. https:\/\/doi.org\/10.1109\/AITEST52744.2021.00016","DOI":"10.1109\/AITEST52744.2021.00016"},{"key":"10524_CR47","doi-asserted-by":"publisher","unstructured":"Papernot N, McDaniel PD, Jha S, Fredrikson M, Celik ZB, Swami A (2016) The limitations of deep learning in adversarial settings. In: IEEE European symposium on security and privacy, EuroS &P 2016, Saarbr\u00fccken, Germany, March 21-24, 2016, IEEE, pp 372\u2013387. https:\/\/doi.org\/10.1109\/EuroSP.2016.36","DOI":"10.1109\/EuroSP.2016.36"},{"key":"10524_CR48","doi-asserted-by":"publisher","unstructured":"Pei K, Cao Y, Yang J, Jana S (2017) Deepxplore: Automated whitebox testing of deep learning systems. In: Proceedings of the 26th Symposium on Operating Systems Principles, Shanghai, China, October 28-31, 2017, ACM, pp 1\u201318. https:\/\/doi.org\/10.1145\/3132747.3132785","DOI":"10.1145\/3132747.3132785"},{"issue":"6","key":"10524_CR49","doi-asserted-by":"publisher","first-page":"5193","DOI":"10.1007\/s10664-020-09881-0","volume":"25","author":"V Riccio","year":"2020","unstructured":"Riccio V, Jahangirova G, Stocco A, Humbatova N, Weiss M, Tonella P (2020) Testing machine learning based systems: a systematic mapping. Empir Softw Eng 25(6):5193\u20135254. https:\/\/doi.org\/10.1007\/s10664-020-09881-0","journal-title":"Empir Softw Eng"},{"key":"10524_CR50","unstructured":"Romano J, Kromrey JD, Coraggio J, Skowronek J, Devine L (2006) Exploring methods for evaluating group differences on the nsse and other surveys: Are the t-test and cohen\u2019sd indices the most appropriate choices. In: Annual meeting of the southern association for institutional research, Citeseer, pp 1\u201351"},{"key":"10524_CR51","doi-asserted-by":"publisher","unstructured":"Sekhon J, Fleming C (2019) Towards improved testing for deep learning. In: Sarma A, Murta L (eds) Proceedings of the 41st international conference on software engineering: new ideas and emerging results, ICSE (NIER) 2019, Montreal, QC, Canada, May 29-31, 2019, IEEE \/ ACM, pp 85\u201388. https:\/\/doi.org\/10.1109\/ICSE-NIER.2019.00030","DOI":"10.1109\/ICSE-NIER.2019.00030"},{"key":"10524_CR52","doi-asserted-by":"publisher","unstructured":"Shen W, Li Y, Chen L, Han Y, Zhou Y, Xu B (2020) Multiple-boundary clustering and prioritization to promote neural network retraining. In: 35th IEEE\/ACM International conference on automated software engineering, ASE 2020, Melbourne, Australia, September 21-25, 2020, IEEE, pp 410\u2013422. https:\/\/doi.org\/10.1145\/3324884.3416621","DOI":"10.1145\/3324884.3416621"},{"key":"10524_CR53","unstructured":"Simonyan K, Zisserman A (2015) Very deep convolutional networks for large-scale image recognition. arXiv:1409.1556"},{"key":"10524_CR54","doi-asserted-by":"publisher","unstructured":"Sun Y, Wu M, Ruan W, Huang X, Kwiatkowska M, Kroening D (2018) Concolic testing for deep neural networks. In: Huchard M, K\u00e4stner C, Fraser G (eds) Proceedings of the 33rd ACM\/IEEE international conference on automated software engineering, ASE 2018, Montpellier, France, September 3-7, 2018, ACM, pp 109\u2013119. https:\/\/doi.org\/10.1145\/3238147.3238172","DOI":"10.1145\/3238147.3238172"},{"key":"10524_CR55","doi-asserted-by":"publisher","unstructured":"Sun Y, Huang X, Kroening D, Sharp J, Hill M, Ashmore R (2019) Structural test coverage criteria for deep neural networks. In: Atlee JM, Bultan T, Whittle J (eds) Proceedings of the 41st international conference on software engineering: companion proceedings, ICSE 2019, Montreal, QC, Canada, May 25-31, 2019, IEEE \/ ACM, pp 320\u2013321. https:\/\/doi.org\/10.1109\/ICSE-Companion.2019.00134","DOI":"10.1109\/ICSE-Companion.2019.00134"},{"key":"10524_CR56","doi-asserted-by":"publisher","unstructured":"Tian Y, Pei K, Jana S, Ray B (2018) Deeptest: automated testing of deep-neural-network-driven autonomous cars. In: Chaudron M, Crnkovic I, Chechik M, Harman M (eds) Proceedings of the 40th international conference on software engineering, ICSE 2018, Gothenburg, Sweden, May 27 - June 03, 2018, ACM, pp 303\u2013314. https:\/\/doi.org\/10.1145\/3180155.3180220","DOI":"10.1145\/3180155.3180220"},{"key":"10524_CR57","doi-asserted-by":"publisher","unstructured":"Wang Z, You H, Chen J, Zhang Y, Dong X, Zhang W (2021) Prioritizing test inputs for deep neural networks via mutation analysis. In: 43rd IEEE\/ACM International conference on software engineering, ICSE 2021, Madrid, Spain, 22-30 May 2021, IEEE, pp 397\u2013409. https:\/\/doi.org\/10.1109\/ICSE43902.2021.00046","DOI":"10.1109\/ICSE43902.2021.00046"},{"key":"10524_CR58","doi-asserted-by":"publisher","unstructured":"Weiss M, Tonella P (2021) Fail-safe execution of deep learning based systems through uncertainty monitoring. In: 14th IEEE Conference on software testing, verification and validation, ICST 2021, Porto de Galinhas, Brazil, April 12-16, 2021, IEEE, pp 24\u201335. https:\/\/doi.org\/10.1109\/ICST49551.2021.00015","DOI":"10.1109\/ICST49551.2021.00015"},{"key":"10524_CR59","doi-asserted-by":"publisher","unstructured":"Weiss M, Chakraborty R, Tonella P (2021) A review and refinement of surprise adequacy. In: 3rd IEEE\/ACM International workshop on deep learning for testing and testing for deep learning, DeepTest@ICSE 2021, Madrid, Spain, June 1, 2021, IEEE, pp 17\u201324. https:\/\/doi.org\/10.1109\/DeepTest52559.2021.00009","DOI":"10.1109\/DeepTest52559.2021.00009"},{"key":"10524_CR60","doi-asserted-by":"crossref","unstructured":"Wilcoxon F (1992) Individual comparisons by ranking methods. In: Breakthroughs in statistics, Springer, pp 196\u2013202","DOI":"10.1007\/978-1-4612-4380-9_16"},{"key":"10524_CR61","unstructured":"Xiao H, Rasul K, Vollgraf R (2019) Fashion-mnist is a dataset of zalando\u2019s article images. https:\/\/github.com\/zalandoresearch\/fashion-mnist"},{"key":"10524_CR62","doi-asserted-by":"publisher","unstructured":"Xie X, Ma L, Juefei-Xu F, Xue M, Chen H, Liu Y, Zhao J, Li B, Yin J, See S (2019) Deephunter: a coverage-guided fuzz testing framework for deep neural networks. In: Zhang D, M\u00f8ller A (eds) Proceedings of the 28th ACM SIGSOFT International symposium on software testing and analysis, ISSTA 2019, Beijing, China, July 15-19, 2019, ACM, pp 146\u2013157. https:\/\/doi.org\/10.1145\/3293882.3330579","DOI":"10.1145\/3293882.3330579"},{"key":"10524_CR63","doi-asserted-by":"publisher","unstructured":"Xie X, Li T, Wang J, Ma L, Guo Q, Juefei-Xu F, Liu Y (2022) NPC: neuron path coverage via characterizing decision logic of deep neural networks. ACM Trans Softw Eng Methodol 31(3):47:1\u201347:27. https:\/\/doi.org\/10.1145\/3490489","DOI":"10.1145\/3490489"},{"key":"10524_CR64","doi-asserted-by":"publisher","unstructured":"Yan S, Tao G, Liu X, Zhai J, Ma S, Xu L, Zhang X (2020) Correlations between deep neural network model coverage criteria and model quality. In: Devanbu P, Cohen MB, Zimmermann T (eds) ESEC\/FSE \u201920: 28th ACM Joint european software engineering conference and symposium on the foundations of software engineering, virtual event, USA, November 8-13, 2020, ACM, pp 775\u2013787. https:\/\/doi.org\/10.1145\/3368089.3409671","DOI":"10.1145\/3368089.3409671"},{"key":"10524_CR65","doi-asserted-by":"publisher","unstructured":"Yuan Y, Pang Q, Wang S (2023) Revisiting neuron coverage for DNN testing: A layer-wise and distribution-aware criterion. In: 45th IEEE\/ACM International conference on software engineering, ICSE 2023, Melbourne, Australia, May 14-20, 2023, IEEE, pp 1200\u20131212. https:\/\/doi.org\/10.1109\/ICSE48619.2023.00107","DOI":"10.1109\/ICSE48619.2023.00107"},{"issue":"2","key":"10524_CR66","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1109\/TSE.2019.2962027","volume":"48","author":"JM Zhang","year":"2022","unstructured":"Zhang JM, Harman M, Ma L, Liu Y (2022) Machine learning testing: Survey, landscapes and horizons. IEEE Trans Software Eng 48(2):1\u201336. https:\/\/doi.org\/10.1109\/TSE.2019.2962027","journal-title":"IEEE Trans Software Eng"}],"container-title":["Empirical Software Engineering"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10664-024-10524-x.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s10664-024-10524-x\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10664-024-10524-x.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,9,6]],"date-time":"2024-09-06T03:52:36Z","timestamp":1725594756000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s10664-024-10524-x"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,7,25]]},"references-count":66,"journal-issue":{"issue":"5","published-print":{"date-parts":[[2024,9]]}},"alternative-id":["10524"],"URL":"https:\/\/doi.org\/10.1007\/s10664-024-10524-x","relation":{},"ISSN":["1382-3256","1573-7616"],"issn-type":[{"type":"print","value":"1382-3256"},{"type":"electronic","value":"1573-7616"}],"subject":[],"published":{"date-parts":[[2024,7,25]]},"assertion":[{"value":"2 July 2024","order":1,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"25 July 2024","order":2,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Declarations"}},{"value":"No conflict of interest exists in the submission of this manuscript, and this manuscript is approved by all authors for publication.","order":2,"name":"Ethics","group":{"name":"EthicsHeading","label":"Conflicts of interest"}}],"article-number":"118"}}