{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,1,20]],"date-time":"2026-01-20T01:39:51Z","timestamp":1768873191442,"version":"3.49.0"},"reference-count":52,"publisher":"Springer Science and Business Media LLC","issue":"3","license":[{"start":{"date-parts":[[2025,2,11]],"date-time":"2025-02-11T00:00:00Z","timestamp":1739232000000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2025,2,11]],"date-time":"2025-02-11T00:00:00Z","timestamp":1739232000000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"funder":[{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["62002162"],"award-info":[{"award-number":["62002162"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]},{"name":"Natural Science Foundation of Jiangsu Province, China","award":["BK20200442"],"award-info":[{"award-number":["BK20200442"]}]}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Empir Software Eng"],"published-print":{"date-parts":[[2025,5]]},"DOI":"10.1007\/s10664-025-10613-5","type":"journal-article","created":{"date-parts":[[2025,2,11]],"date-time":"2025-02-11T05:32:44Z","timestamp":1739251964000},"update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":1,"title":["A fine-grained evaluation of mutation operators to boost mutation testing for deep learning systems"],"prefix":"10.1007","volume":"30","author":[{"given":"Zhiyi","family":"Zhang","sequence":"first","affiliation":[]},{"given":"Yichun","family":"Wang","sequence":"additional","affiliation":[]},{"given":"Yongming","family":"Yao","sequence":"additional","affiliation":[]},{"given":"Ziyuan","family":"Wang","sequence":"additional","affiliation":[]},{"given":"Zhiqiu","family":"Huang","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2025,2,11]]},"reference":[{"key":"10613_CR1","doi-asserted-by":"crossref","unstructured":"AbouTrab MS, Counsell S, Hierons RM (2012) Specification mutation analysis for validating timed testing approaches based on timed automata. In: 2012 IEEE 36th Annual computer software and applications conference, IEEE, pp 660\u2013669","DOI":"10.1109\/COMPSAC.2012.93"},{"key":"10613_CR2","doi-asserted-by":"crossref","unstructured":"Aghababaeyan Z, Abdellatif M, Briand L, Ramesh S, Bagherzadeh M (2023) Black-box testing of deep neural networks through test case diversity. IEEE Trans Softw Eng","DOI":"10.1109\/TSE.2023.3243522"},{"key":"10613_CR3","unstructured":"Budd TA (1980) Mutation analysis of program test data. Yale University"},{"key":"10613_CR4","doi-asserted-by":"crossref","unstructured":"Carlini N, Wagner D (2017) Towards evaluating the robustness of neural networks. In: 2017 ieee symposium on security and privacy (sp), IEEE, pp 39\u201357","DOI":"10.1109\/SP.2017.49"},{"key":"10613_CR5","unstructured":"Chen J, Yan M, Wang Z, Kang Y, Wu Z (2020) Deep neural network test coverage: How far are we? arXiv:2010.04946"},{"key":"10613_CR6","doi-asserted-by":"crossref","unstructured":"Coles H, Laurent T, Henard C, Papadakis M, Ventresque A (2016) Pit: a practical mutation testing tool for java. In: Proceedings of the 25th international symposium on software testing and analysis, pp 449\u2013452","DOI":"10.1145\/2931037.2948707"},{"key":"10613_CR7","doi-asserted-by":"crossref","unstructured":"Dan H, Hierons RM (2012) Smt-c: A semantic mutation testing tools for c. In: 2012 IEEE Fifth international conference on software testing, verification and validation, IEEE, pp 654\u2013663","DOI":"10.1109\/ICST.2012.155"},{"key":"10613_CR8","doi-asserted-by":"crossref","unstructured":"Delamaro ME, Maldonado JC, Mathur AP (1996) Integration testing using interface mutation. In: Proceedings of ISSRE\u201996: 7th international symposium on software reliability engineering, IEEE, pp 112\u2013121","DOI":"10.1109\/ISSRE.1996.558719"},{"issue":"4\u20135","key":"10613_CR9","doi-asserted-by":"publisher","DOI":"10.1002\/stvr.1630","volume":"27","author":"P Delgado-P\u00e9rez","year":"2017","unstructured":"Delgado-P\u00e9rez P, Segura S, Medina-Bulo I (2017) Assessment of c++ object-oriented mutation operators: A selective mutation approach. Softw Testing, Verification Reliability 27(4\u20135):e1630","journal-title":"Softw Testing, Verification Reliability"},{"key":"10613_CR10","doi-asserted-by":"publisher","first-page":"823","DOI":"10.1007\/s11219-018-9425-7","volume":"27","author":"P Delgado-P\u00e9rez","year":"2019","unstructured":"Delgado-P\u00e9rez P, Rose LM, Medina-Bulo I (2019) Coverage-based quality metric of mutation operators for test suite improvement. Softw Quality J 27:823\u2013859","journal-title":"Softw Quality J"},{"key":"10613_CR11","doi-asserted-by":"crossref","unstructured":"Derezi\u0144ska A (2016) Evaluation of deletion mutation operators in mutation testing of c# programs. In: Dependability engineering and complex systems: proceedings of the eleventh international conference on dependability and complex systems DepCoS-RELCOMEX. June 27\u2013July 1, 2016, Brun\u00f3w, Poland, Springer, pp 97\u2013108","DOI":"10.1007\/978-3-319-39639-2_9"},{"key":"10613_CR12","doi-asserted-by":"crossref","unstructured":"Derezinska A, Halas K (2014) Experimental evaluation of mutation testing approaches to python programs. In: 2014 IEEE Seventh international conference on software testing, verification and validation workshops, IEEE, pp 156\u2013164","DOI":"10.1109\/ICSTW.2014.24"},{"issue":"5\u20137","key":"10613_CR13","doi-asserted-by":"publisher","first-page":"536","DOI":"10.1002\/stvr.1528","volume":"25","author":"A Estero-Botaro","year":"2015","unstructured":"Estero-Botaro A, Palomo-Lozano F, Medina-Bulo I, Dom\u00ednguez-Jim\u00e9nez JJ, Garc\u00eda-Dom\u00ednguez A (2015) Quality metrics for mutation testing with applications to ws-bpel compositions. Softw Testing, Verification Reliability 25(5\u20137):536\u2013571","journal-title":"Softw Testing, Verification Reliability"},{"issue":"3","key":"10613_CR14","first-page":"601","volume":"23","author":"LC Feng","year":"2022","unstructured":"Feng LC, Wang XY, Zhang SY, Gao RZ, Zhao ZH (2022) Mutation operator reduction for cost-effective deep learning software testing via decision boundary change measurement. J Int Technol 23(3):601\u2013610","journal-title":"J Int Technol"},{"key":"10613_CR15","doi-asserted-by":"crossref","unstructured":"Gao X, Feng Y, Yin Y, Liu Z, Chen Z, Xu B (2022) Adaptive test selection for deep neural networks. In: Proceedings of the 44th international conference on software engineering, pp 73\u201385","DOI":"10.1145\/3510003.3510232"},{"key":"10613_CR16","doi-asserted-by":"crossref","unstructured":"Gerasimou S, Eniser HF, Sen A, Cakan A (2020) Importance-driven deep learning system testing. In: 2020 ieee\/acm 42nd international conference on software engineering (icse). IEEE, 702\u015b713","DOI":"10.1145\/3377811.3380391"},{"key":"10613_CR17","doi-asserted-by":"crossref","unstructured":"Gligoric M, Zhang L, Pereira C, Pokam G (2013) Selective mutation testing for concurrent code. In: Proceedings of the 2013 international symposium on software testing and analysis, pp 224\u2013234","DOI":"10.1145\/2483760.2483773"},{"key":"10613_CR18","doi-asserted-by":"crossref","unstructured":"Hu Q, Ma L, Xie X, Yu B, Liu Y, Zhao J (2019) Deepmutation++: A mutation testing framework for deep learning systems. In: 2019 34th IEEE\/ACM international conference on automated software engineering (ASE), IEEE, pp 1158\u20131161","DOI":"10.1109\/ASE.2019.00126"},{"key":"10613_CR19","unstructured":"Hussain S (2008) Mutation clustering. Ms. Th., Kings College London, Strand, London, pp\u00a09"},{"key":"10613_CR20","doi-asserted-by":"crossref","unstructured":"Jahangirova G, Tonella P (2020) An empirical evaluation of mutation operators for deep learning systems. In: 2020 IEEE 13th International conference on software testing, validation and verification (ICST), IEEE, pp 74\u201384","DOI":"10.1109\/ICST46399.2020.00018"},{"issue":"5","key":"10613_CR21","doi-asserted-by":"publisher","first-page":"649","DOI":"10.1109\/TSE.2010.62","volume":"37","author":"Y Jia","year":"2010","unstructured":"Jia Y, Harman M (2010) An analysis and survey of the development of mutation testing. IEEE Trans Softw Eng 37(5):649\u2013678","journal-title":"IEEE Trans Softw Eng"},{"key":"10613_CR22","doi-asserted-by":"crossref","unstructured":"Kaufman SJ, Featherman R, Alvin J, Kurtz B, Ammann P, Just R (2022) Prioritizing mutants to guide mutation testing. pp 1743\u20131754","DOI":"10.1145\/3510003.3510187"},{"key":"10613_CR23","doi-asserted-by":"crossref","unstructured":"Kim J, Feldt R, Yoo S (2019) Guiding deep learning system testing using surprise adequacy. In: 2019 IEEE\/ACM 41st international conference on software engineering (ICSE), IEEE, pp 1039\u20131049","DOI":"10.1109\/ICSE.2019.00108"},{"key":"10613_CR24","unstructured":"Krizhevsky A, Nair V, Hinton G (2023) The cifar-10 dataset. http:\/\/www.cs.toronto.edu\/~kriz\/cifar.html. Accessed January 15"},{"key":"10613_CR25","doi-asserted-by":"crossref","unstructured":"Kurtz B, Ammann P, Delamaro ME, Offutt J, Deng L (2014) Mutant subsumption graphs. In: 2014 IEEE Seventh international conference on software testing, verification and validation workshops, IEEE, pp 176\u2013185","DOI":"10.1109\/ICSTW.2014.20"},{"key":"10613_CR26","unstructured":"LeCun Y (1998) The mnist database of handwritten digits. http:\/\/yann.lecun.com\/exdb\/mnist\/"},{"issue":"11","key":"10613_CR27","doi-asserted-by":"publisher","first-page":"2278","DOI":"10.1109\/5.726791","volume":"86","author":"Y LeCun","year":"1998","unstructured":"LeCun Y, Bottou L, Bengio Y, Haffner P (1998) Gradient-based learning applied to document recognition. Proceed IEEE 86(11):2278\u20132324","journal-title":"Proceed IEEE"},{"key":"10613_CR28","doi-asserted-by":"publisher","DOI":"10.1016\/j.infsof.2022.106954","volume":"150","author":"Y Li","year":"2022","unstructured":"Li Y, Shen W, Wu T, Chen L, Wu D, Zhou Y, Xu B (2022) How higher order mutant testing performs for deep learning models: A fine-grained evaluation of test effectiveness and efficiency improved from second-order mutant-classification tuples. Inf Softw Technol 150:106954","journal-title":"Inf Softw Technol"},{"key":"10613_CR29","doi-asserted-by":"publisher","first-page":"112","DOI":"10.1016\/j.infsof.2016.04.007","volume":"81","author":"B Lindstr\u00f6m","year":"2017","unstructured":"Lindstr\u00f6m B, Offutt J, Sundmark D, Andler SF, Pettersson P (2017) Using mutation to design tests for aspect-oriented models. Inf Softw Technol 81:112\u2013130","journal-title":"Inf Softw Technol"},{"key":"10613_CR30","unstructured":"Lipton R (1971) Fault diagnosis of computer programs. student report. Carnegie Mellon University 2:2"},{"key":"10613_CR31","doi-asserted-by":"crossref","unstructured":"Ma L, Juefei-Xu F, Zhang F, Sun J, Xue M, Li B, Chen C, Su T, Li L, Liu Y, Zhao J (2018a) Deepgauge: Multi-granularity testing criteria for deep learning systems. In: Proceedings of the 33rd ACM\/IEEE international conference on automated software engineering, pp 120\u2013131","DOI":"10.1145\/3238147.3238202"},{"key":"10613_CR32","doi-asserted-by":"crossref","unstructured":"Ma L, Zhang F, Sun J, Xue M, Li B, Juefei-Xu F, Xie C, Li L, Liu Y, Zhao J, Wang Y (2018b) Deepmutation: Mutation testing of deep learning systems. In: 2018 IEEE 29th international symposium on software reliability engineering (ISSRE), IEEE, pp 100\u2013111","DOI":"10.1109\/ISSRE.2018.00021"},{"key":"10613_CR33","doi-asserted-by":"crossref","unstructured":"Ma L, Juefei-Xu F, Xue M, Li B, Li L, Liu Y, Zhao J (2019) Deepct: Tomographic combinatorial testing for deep learning systems. In: 2019 IEEE 26th International conference on software analysis, evolution and reengineering (SANER), IEEE, pp 614\u2013618","DOI":"10.1109\/SANER.2019.8668044"},{"issue":"2","key":"10613_CR34","doi-asserted-by":"publisher","first-page":"169","DOI":"10.1016\/j.infsof.2009.08.007","volume":"52","author":"L Madeyski","year":"2010","unstructured":"Madeyski L (2010) The impact of test-first programming on branch coverage and mutation score indicator of unit tests: An experiment. Inf Softw Technol 52(2):169\u2013184","journal-title":"Inf Softw Technol"},{"key":"10613_CR35","doi-asserted-by":"crossref","unstructured":"Mathur AP (1991) Performance, effectiveness, and reliability issues in software testing. In: 1991 The fifteenth annual international computer software & applications conference, IEEE Computer Society, pages 604\u2013605","DOI":"10.1109\/CMPSAC.1991.170248"},{"issue":"1","key":"10613_CR36","doi-asserted-by":"publisher","first-page":"9","DOI":"10.1002\/stvr.4370040104","volume":"4","author":"AP Mathur","year":"1994","unstructured":"Mathur AP, Wong WE (1994) An empirical comparison of data flow and mutation-based test adequacy criteria. Software Testing, Verification Reliability 4(1):9\u201331","journal-title":"Software Testing, Verification Reliability"},{"key":"10613_CR37","unstructured":"Netzer Y, Wang T, Coates A, Bissacco A, Wu B, Ng AY (2011) Reading digits in natural images with unsupervised feature learning"},{"key":"10613_CR38","doi-asserted-by":"crossref","unstructured":"Offutt AJ, Rothermel G, Zapf C (1993) An experimental evaluation of selective mutation. In: Proceedings of 1993 15th international conference on software engineering, IEEE, pp 100\u2013107","DOI":"10.1109\/ICSE.1993.346062"},{"key":"10613_CR39","doi-asserted-by":"crossref","unstructured":"Palomo-Lozano F, Estero-Botaro A, Medina-Bulo I, N\u00fa\u00f1ez M (2018) Test suite minimization for mutation testing of ws-bpel compositions. In: Proceedings of the genetic and evolutionary computation conference, pp 1427\u20131434","DOI":"10.1145\/3205455.3205533"},{"key":"10613_CR40","doi-asserted-by":"crossref","unstructured":"Pei K, Cao Y, Yang J, Jana S (2017) Deepxplore: Automated whitebox testing of deep learning systems. In: Proceedings of the 26th Symposium on Operating Systems Principles, pp 1\u201318","DOI":"10.1145\/3132747.3132785"},{"issue":"2","key":"10613_CR41","doi-asserted-by":"publisher","first-page":"111","DOI":"10.1002\/stvr.392","volume":"19","author":"M Polo","year":"2009","unstructured":"Polo M, Piattini M, Garc\u00eda-Rodr\u00edguez I (2009) Decreasing the cost of mutation testing with second-order mutants. Software Testing, Verification Reliability 19(2):111\u2013131","journal-title":"Software Testing, Verification Reliability"},{"key":"10613_CR42","doi-asserted-by":"crossref","unstructured":"Shen W, Wan J, Chen Z (2018) Munn: Mutation analysis of neural networks. In: 2018 IEEE International conference on software quality, reliability and security companion (QRS-C), IEEE, pp 108\u2013115","DOI":"10.1109\/QRS-C.2018.00032"},{"key":"10613_CR43","doi-asserted-by":"publisher","DOI":"10.1016\/j.infsof.2020.106413","volume":"130","author":"W Shen","year":"2021","unstructured":"Shen W, Li Y, Han Y, Chen L, Wu D, Zhou Y, Xu B (2021) Boundary sampling to boost mutation testing for deep learning models. Inf Softw Technol 130:106413","journal-title":"Inf Softw Technol"},{"key":"10613_CR44","doi-asserted-by":"crossref","unstructured":"Silveira B, Durelli V, Santos S, Durelli R, Delamaro M, Souza S (2023) Test data selection based on applying mutation testing to decision tree models. In: Proceedings of the 8th Brazilian symposium on systematic and automated software testing, pp 38\u201346","DOI":"10.1145\/3624032.3624038"},{"key":"10613_CR45","unstructured":"Smits JPG (2022) Callisto-selecting effective mutation operators for mutation testing. Master\u2019s thesis, University of Twente"},{"key":"10613_CR46","unstructured":"Sohn J, Kang S, Yoo S (2019) Search based repair of deep neural networks. arXiv:1912.12463"},{"key":"10613_CR47","doi-asserted-by":"crossref","unstructured":"Sun Y, Huang X, Kroening D, Sharp J, Hill M, Ashmore R (2018) Testing deep neural networks","DOI":"10.1145\/3238147.3238172"},{"key":"10613_CR48","doi-asserted-by":"crossref","unstructured":"Wang J, Dong G, Sun J, Wang X, Zhang P (2019) Adversarial sample detection for deep neural network through model mutation testing. In: 2019 IEEE\/ACM 41st International conference on software engineering (ICSE), IEEE, pp 1245\u20131256","DOI":"10.1109\/ICSE.2019.00126"},{"key":"10613_CR49","doi-asserted-by":"crossref","unstructured":"Wang Y, Zhang Z, Yao Y, Huang Z (2023) A fine-grained evaluation of mutation operators for deep learning systems: A selective mutation approach. In: Proceedings of the 14th Asia-Pacific symposium on internetware, pp 123\u2013133","DOI":"10.1145\/3609437.3609453"},{"key":"10613_CR50","doi-asserted-by":"crossref","unstructured":"Wang Z, You H, Chen J, Zhang Y, Dong X, Zhang W (2021) Prioritizing test inputs for deep neural networks via mutation analysis. In: 2021 IEEE\/ACM 43rd International conference on software engineering (ICSE), IEEE, pp 397\u2013409","DOI":"10.1109\/ICSE43902.2021.00046"},{"key":"10613_CR51","doi-asserted-by":"crossref","unstructured":"Wong WE (2001) Mutation testing for the new century, volume\u00a024. Springer Science & Business Media","DOI":"10.1007\/978-1-4757-5939-6"},{"key":"10613_CR52","doi-asserted-by":"crossref","unstructured":"Xiao C, Li B, Zhu JY, He W, Liu M, Song D (2018) Generating adversarial examples with adversarial networks. arXiv:1801.02610","DOI":"10.24963\/ijcai.2018\/543"}],"container-title":["Empirical Software Engineering"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10664-025-10613-5.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s10664-025-10613-5\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10664-025-10613-5.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,11,20]],"date-time":"2025-11-20T13:27:16Z","timestamp":1763645236000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s10664-025-10613-5"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,2,11]]},"references-count":52,"journal-issue":{"issue":"3","published-print":{"date-parts":[[2025,5]]}},"alternative-id":["10613"],"URL":"https:\/\/doi.org\/10.1007\/s10664-025-10613-5","relation":{},"ISSN":["1382-3256","1573-7616"],"issn-type":[{"value":"1382-3256","type":"print"},{"value":"1573-7616","type":"electronic"}],"subject":[],"published":{"date-parts":[[2025,2,11]]},"assertion":[{"value":"10 January 2025","order":1,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"11 February 2025","order":2,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Declarations"}},{"value":"The authors declare that they have no conflict of interest.","order":2,"name":"Ethics","group":{"name":"EthicsHeading","label":"Conflict of interest"}}],"article-number":"63"}}