{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,6,6]],"date-time":"2026-06-06T20:59:24Z","timestamp":1780779564754,"version":"3.54.1"},"reference-count":55,"publisher":"Elsevier BV","license":[{"start":{"date-parts":[[2026,10,1]],"date-time":"2026-10-01T00:00:00Z","timestamp":1790812800000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.elsevier.com\/tdm\/userlicense\/1.0\/"},{"start":{"date-parts":[[2026,10,1]],"date-time":"2026-10-01T00:00:00Z","timestamp":1790812800000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.elsevier.com\/legal\/tdmrep-license"},{"start":{"date-parts":[[2026,10,1]],"date-time":"2026-10-01T00:00:00Z","timestamp":1790812800000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-017"},{"start":{"date-parts":[[2026,10,1]],"date-time":"2026-10-01T00:00:00Z","timestamp":1790812800000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"},{"start":{"date-parts":[[2026,10,1]],"date-time":"2026-10-01T00:00:00Z","timestamp":1790812800000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-012"},{"start":{"date-parts":[[2026,10,1]],"date-time":"2026-10-01T00:00:00Z","timestamp":1790812800000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2026,10,1]],"date-time":"2026-10-01T00:00:00Z","timestamp":1790812800000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-004"}],"funder":[{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["elsevier.com","sciencedirect.com"],"crossmark-restriction":true},"short-container-title":["Journal of Systems and Software"],"published-print":{"date-parts":[[2026,10]]},"DOI":"10.1016\/j.jss.2026.112936","type":"journal-article","created":{"date-parts":[[2026,5,10]],"date-time":"2026-05-10T22:16:50Z","timestamp":1778451410000},"page":"112936","update-policy":"https:\/\/doi.org\/10.1016\/elsevier_cm_policy","source":"Crossref","is-referenced-by-count":0,"special_numbering":"C","title":["A multi-dimensional test case evaluation framework based on clustering and differential testing"],"prefix":"10.1016","volume":"240","author":[{"given":"Daguang","family":"Jiang","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Jiajun","family":"Zhou","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Xin","family":"Wang","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Xiaojie","family":"Fan","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-5884-2089","authenticated-orcid":false,"given":"Hengyuan","family":"Liu","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-1754-3039","authenticated-orcid":false,"given":"Yong","family":"Liu","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"78","reference":[{"key":"10.1016\/j.jss.2026.112936_b1","doi-asserted-by":"crossref","DOI":"10.1016\/j.infsof.2020.106426","article-title":"Statement frequency coverage: a code coverage criterion for assessing test suite effectiveness","volume":"129","author":"Aghamohammadi","year":"2021","journal-title":"Inf. Softw. Technol."},{"issue":"6","key":"10.1016\/j.jss.2026.112936_b2","doi-asserted-by":"crossref","first-page":"742","DOI":"10.1109\/TSE.2009.52","article-title":"A systematic review of the application and empirical investigation of search-based test case generation","volume":"36","author":"Ali","year":"2009","journal-title":"IEEE Trans. Softw. Eng."},{"issue":"8","key":"10.1016\/j.jss.2026.112936_b3","doi-asserted-by":"crossref","first-page":"1978","DOI":"10.1016\/j.jss.2013.02.061","article-title":"An orchestrated survey of methodologies for automated software test case generation","volume":"86","author":"Anand","year":"2013","journal-title":"J. Syst. Softw."},{"key":"10.1016\/j.jss.2026.112936_b4","series-title":"Code coverage based test case selection and prioritization","author":"Beena","year":"2013"},{"key":"10.1016\/j.jss.2026.112936_b5","series-title":"Proc. 33rd ACM SIGSOFT Int. Symp. Software Testing and Analysis","first-page":"1262","article-title":"Wasmaker: Differential testing of webassembly runtimes via semantic-aware binary generation","author":"Cao","year":"2024"},{"key":"10.1016\/j.jss.2026.112936_b6","series-title":"2025 25th International Conference on Software Quality, Reliability and Security","first-page":"588","article-title":"DiFuzzNMT: A differential fuzzing framework for neural machine translation","author":"Chen","year":"2025"},{"key":"10.1016\/j.jss.2026.112936_b7","doi-asserted-by":"crossref","DOI":"10.1016\/j.infsof.2024.107468","article-title":"Effective test generation using pre-trained large language models and mutation testing","volume":"171","author":"Dakhel","year":"2024","journal-title":"Inf. Softw. Technol."},{"issue":"4","key":"10.1016\/j.jss.2026.112936_b8","doi-asserted-by":"crossref","first-page":"34","DOI":"10.1109\/C-M.1978.218136","article-title":"Hints on test data selection: Help for the practicing programmer","volume":"11","author":"DeMillo","year":"2006","journal-title":"Computer"},{"key":"10.1016\/j.jss.2026.112936_b9","series-title":"Proc. 30th ACM Joint Eur. Software Eng. Conf. Symp. Foundations Software Eng.","first-page":"44","article-title":"Fuzzing deep-learning libraries via automated relational api inference","author":"Deng","year":"2022"},{"key":"10.1016\/j.jss.2026.112936_b10","series-title":"Kdd","first-page":"226","article-title":"A density-based algorithm for discovering clusters in large spatial databases with noise","volume":"vol. 96","author":"Ester","year":"1996"},{"key":"10.1016\/j.jss.2026.112936_b11","series-title":"Proc. 6th Joint Meeting Eur. Software Eng. Conf. ACM SIGSOFT Symp. Foundations Software Eng.: Companion Papers","first-page":"549","article-title":"Differential testing: a new approach to change detection","author":"Evans","year":"2007"},{"key":"10.1016\/j.jss.2026.112936_b12","series-title":"Proc. 6th IEEE Int. Conf. Software Testing, Verification and Validation","first-page":"362","article-title":"Evosuite: On the challenges of test case generation in the real world","author":"Fraser","year":"2013"},{"key":"10.1016\/j.jss.2026.112936_b13","series-title":"Proc. 2015 IEEE 26th Int. Symp. Software Reliability Engineering","first-page":"216","article-title":"How hard does mutation analysis have to be, anyway?","author":"Gopinath","year":"2015"},{"key":"10.1016\/j.jss.2026.112936_b14","series-title":"Proc. 29th Int. Conf. Software Engineering","first-page":"621","article-title":"Randomized differential testing as a prelude to formal verification","author":"Groce","year":"2007"},{"key":"10.1016\/j.jss.2026.112936_b15","series-title":"Proc. 35th IEEE\/ACM Int. Conf. Automated Software Engineering","first-page":"486","article-title":"Audee: Automated testing for deep learning frameworks","author":"Guo","year":"2020"},{"issue":"5","key":"10.1016\/j.jss.2026.112936_b16","doi-asserted-by":"crossref","first-page":"490","DOI":"10.1109\/TSE.2015.2496939","article-title":"To be optimal or not in test-case prioritization","volume":"42","author":"Hao","year":"2015","journal-title":"IEEE Trans. Softw. Eng."},{"key":"10.1016\/j.jss.2026.112936_b17","series-title":"Proc. 2015 IEEE Int. Conf. Software Quality, Reliability and Security","first-page":"151","article-title":"How effective are code coverage criteria?","author":"Hemmati","year":"2015"},{"key":"10.1016\/j.jss.2026.112936_b18","series-title":"Int. Workshop Data Privacy Management","first-page":"289","article-title":"Practical mutation testing for smart contracts","author":"Honig","year":"2019"},{"key":"10.1016\/j.jss.2026.112936_b19","doi-asserted-by":"crossref","DOI":"10.1016\/j.jss.2020.110712","article-title":"Regression test case prioritization by code combinations coverage","volume":"169","author":"Huang","year":"2020","journal-title":"J. Syst. Softw."},{"key":"10.1016\/j.jss.2026.112936_b20","series-title":"Proc. 2019 27th ACM Joint Meeting Eur. Software Eng. Conf. Symp. Foundations Software Eng.","first-page":"955","article-title":"Code coverage at google","author":"Ivankovi\u0107","year":"2019"},{"issue":"1","key":"10.1016\/j.jss.2026.112936_b21","first-page":"1","article-title":"An effective PSO-driven method for test data generation in branch coverage software testing","volume":"21","author":"Jaiswalx","year":"2025","journal-title":"Int. J. Perform. Eng."},{"issue":"10","key":"10.1016\/j.jss.2026.112936_b22","doi-asserted-by":"crossref","first-page":"1379","DOI":"10.1016\/j.infsof.2009.04.016","article-title":"Higher order mutation testing","volume":"51","author":"Jia","year":"2009","journal-title":"Inf. Softw. Technol."},{"key":"10.1016\/j.jss.2026.112936_b23","series-title":"Proc. 2016 Int. Conf. Software Analysis, Testing and Evolution","first-page":"72","article-title":"Identify coincidental correct test cases based on fuzzy classification","author":"Li","year":"2016"},{"key":"10.1016\/j.jss.2026.112936_b24","doi-asserted-by":"crossref","DOI":"10.1016\/j.jss.2024.112189","article-title":"GBSR: Graph-based suspiciousness refinement for improving fault localization","volume":"218","author":"Li","year":"2024","journal-title":"J. Syst. Softw."},{"key":"10.1016\/j.jss.2026.112936_b25","doi-asserted-by":"crossref","DOI":"10.1016\/j.jss.2023.111822","article-title":"VsusFL: Variable-suspiciousness-based fault localization for novice programs","volume":"205","author":"Li","year":"2023","journal-title":"J. Syst. Softw."},{"key":"10.1016\/j.jss.2026.112936_b26","series-title":"Proc. 2020 IEEE 44th Annual Computers, Software, and Applications Conf.","first-page":"178","article-title":"Using fine-grained test cases for improving novice program fault localization","author":"Li","year":"2020"},{"key":"10.1016\/j.jss.2026.112936_b27","series-title":"Proc. 2021 8th Int. Conf. Dependable Systems and their Applications","first-page":"54","article-title":"Fault localization-guided test data generation approach for novice programs","author":"Li","year":"2021"},{"key":"10.1016\/j.jss.2026.112936_b28","series-title":"Proc. 2023 38th IEEE\/ACM Int. Conf. Automated Software Engineering","first-page":"14","article-title":"Nuances are the key: Unlocking chatgpt to find failure-inducing tests with differential prompting","author":"Li","year":"2023"},{"key":"10.1016\/j.jss.2026.112936_b29","series-title":"Proc. 32nd ACM SIGSOFT Int. Symp. Software Testing and Analysis","first-page":"334","article-title":"Who judges the judge: An empirical study on online judge tests","author":"Liu","year":"2023"},{"key":"10.1016\/j.jss.2026.112936_b30","doi-asserted-by":"crossref","DOI":"10.1016\/j.jss.2024.112281","article-title":"Integrating neural mutation into mutation-based fault localization: A hybrid approach","volume":"221","author":"Liu","year":"2025","journal-title":"J. Syst. Softw."},{"issue":"4","key":"10.1016\/j.jss.2026.112936_b31","doi-asserted-by":"crossref","DOI":"10.1002\/stvr.1872","article-title":"Delta4Ms: Improving mutation-based fault localization by eliminating mutant bias","volume":"34","author":"Liu","year":"2024","journal-title":"Softw. Test. Verif. Reliab."},{"issue":"4","key":"10.1016\/j.jss.2026.112936_b32","first-page":"1061","article-title":"CRMF: A fault localization approach based on class reduction and method call frequency","volume":"53","author":"Liu","year":"2023","journal-title":"Softw.: Pr. Exp."},{"key":"10.1016\/j.jss.2026.112936_b33","series-title":"Proc. 17th Int. Conf. Computer Science and Education","article-title":"BUCTOL:VS code based integrated online programming and plagiarism monitoring platform","author":"Liu","year":"2022"},{"key":"10.1016\/j.jss.2026.112936_b34","doi-asserted-by":"crossref","first-page":"572","DOI":"10.1016\/j.ins.2017.09.006","article-title":"An optimal mutation execution strategy for cost reduction of mutation-based fault localization","volume":"422","author":"Liu","year":"2018","journal-title":"Inform. Sci."},{"key":"10.1016\/j.jss.2026.112936_b35","doi-asserted-by":"crossref","first-page":"21558","DOI":"10.52202\/075280-0943","article-title":"Is your code generated by ChatGPT really correct? Rigorous evaluation of large language models for code generation","volume":"36","author":"Liu","year":"2023","journal-title":"Adv. Neural Inf. Process. Syst."},{"issue":"10","key":"10.1016\/j.jss.2026.112936_b36","article-title":"Multi-objective optimization-based and fault localization-oriented test case generation for novice programs","volume":"36","author":"Liu","year":"2024","journal-title":"J. Softw.: Evol. Process."},{"issue":"4","key":"10.1016\/j.jss.2026.112936_b37","doi-asserted-by":"crossref","first-page":"420","DOI":"10.1109\/TR.2002.804489","article-title":"Software reliability growth with test coverage","volume":"51","author":"Malaiya","year":"2002","journal-title":"IEEE Trans. Reliab."},{"key":"10.1016\/j.jss.2026.112936_b38","series-title":"Simp\u00f3sio Brasileiro de Engenharia de Software","first-page":"320","article-title":"Evaluation of the cost of alternate mutation strategies","author":"Mathur","year":"1993"},{"issue":"1","key":"10.1016\/j.jss.2026.112936_b39","doi-asserted-by":"crossref","first-page":"9","DOI":"10.1002\/stvr.4370040104","article-title":"An empirical comparison of data flow and mutation-based test adequacy criteria","volume":"4","author":"Mathur","year":"1994","journal-title":"Softw. Test. Verif. Reliab."},{"issue":"1","key":"10.1016\/j.jss.2026.112936_b40","first-page":"100","article-title":"Differential testing for software","volume":"10","author":"McKeeman","year":"1998","journal-title":"Digit. Tech. J."},{"issue":"1","key":"10.1016\/j.jss.2026.112936_b41","doi-asserted-by":"crossref","first-page":"5","DOI":"10.1145\/125489.125473","article-title":"Investigations of the software testing coupling effect","volume":"1","author":"Offutt","year":"1992","journal-title":"ACM Trans. Softw. Eng. Methodol. (TOSEM)"},{"key":"10.1016\/j.jss.2026.112936_b42","series-title":"Proc. 2019 IEEE\/ACM 41st Int. Conf. Software Engineering","first-page":"1027","article-title":"CRADLE: cross-backend validation to detect and localize bugs in deep learning libraries","author":"Pham","year":"2019"},{"issue":"10","key":"10.1016\/j.jss.2026.112936_b43","first-page":"131","article-title":"Competitive learning in informatics: The UVa online judge experience","volume":"2","author":"Revilla","year":"2008","journal-title":"Olymp. Inform."},{"key":"10.1016\/j.jss.2026.112936_b44","series-title":"Proc. 30th IEEE\/ACM Int. Conf. Automated Software Engineering","first-page":"201","article-title":"Do automatically generated unit tests find real faults? an empirical study of effectiveness and challenges (t)","author":"Shamshiri","year":"2015"},{"key":"10.1016\/j.jss.2026.112936_b45","series-title":"Proc. 2019 IEEE 26th Int. Conf. Software Analysis, Evolution and Reengineering","first-page":"584","article-title":"Program state coverage: A test coverage metric based on executed program states","author":"Someoliayi","year":"2019"},{"key":"10.1016\/j.jss.2026.112936_b46","series-title":"VALTEST: Automated validation of language model generated test cases","author":"Taherkhani","year":"2024"},{"key":"10.1016\/j.jss.2026.112936_b47","series-title":"Proc. ACM\/IEEE 42nd Int. Conf. Software Engineering: Companion Proc.","first-page":"29","article-title":"Deepmutation: A neural mutation tool","author":"Tufano","year":"2020"},{"key":"10.1016\/j.jss.2026.112936_b48","series-title":"Automated unit test case generation: A systematic literature review","author":"Wang","year":"2025"},{"key":"10.1016\/j.jss.2026.112936_b49","series-title":"Towards more effective fault detection in LLM-based unit test generation","author":"Wang","year":"2025"},{"key":"10.1016\/j.jss.2026.112936_b50","series-title":"Proc. 2021 36th IEEE\/ACM Int. Conf. Automated Software Engineering","first-page":"692","article-title":"QDiff: Differential testing of quantum software stacks","author":"Wang","year":"2021"},{"issue":"1","key":"10.1016\/j.jss.2026.112936_b51","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1145\/3143560","article-title":"A survey on online judge systems and their applications","volume":"51","author":"Wasik","year":"2018","journal-title":"ACM Comput. Surv."},{"key":"10.1016\/j.jss.2026.112936_b52","series-title":"Proc. 2016 Int. Symp. Educational Technology","first-page":"57","article-title":"Online judge system and its applications in c language teaching","author":"Wu","year":"2016"},{"key":"10.1016\/j.jss.2026.112936_b53","series-title":"A survey of deep learning library testing methods","author":"Zhang","year":"2024"},{"key":"10.1016\/j.jss.2026.112936_b54","series-title":"2025 25th International Conference on Software Quality, Reliability and Security","first-page":"485","article-title":"PCBFen: Bug detection in PCB design tool chain through functionally equivalent netlist mutation","author":"Zhao","year":"2025"},{"issue":"4","key":"10.1016\/j.jss.2026.112936_b55","doi-asserted-by":"crossref","first-page":"366","DOI":"10.1145\/267580.267590","article-title":"Software unit test coverage and adequacy","volume":"29","author":"Zhu","year":"1997","journal-title":"Acm Comput. Surv. (Csur)"}],"container-title":["Journal of Systems and Software"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/api.elsevier.com\/content\/article\/PII:S016412122600169X?httpAccept=text\/xml","content-type":"text\/xml","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/api.elsevier.com\/content\/article\/PII:S016412122600169X?httpAccept=text\/plain","content-type":"text\/plain","content-version":"vor","intended-application":"text-mining"}],"deposited":{"date-parts":[[2026,6,6]],"date-time":"2026-06-06T20:23:36Z","timestamp":1780777416000},"score":1,"resource":{"primary":{"URL":"https:\/\/linkinghub.elsevier.com\/retrieve\/pii\/S016412122600169X"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026,10]]},"references-count":55,"alternative-id":["S016412122600169X"],"URL":"https:\/\/doi.org\/10.1016\/j.jss.2026.112936","relation":{},"ISSN":["0164-1212"],"issn-type":[{"value":"0164-1212","type":"print"}],"subject":[],"published":{"date-parts":[[2026,10]]},"assertion":[{"value":"Elsevier","name":"publisher","label":"This article is maintained by"},{"value":"A multi-dimensional test case evaluation framework based on clustering and differential testing","name":"articletitle","label":"Article Title"},{"value":"Journal of Systems and Software","name":"journaltitle","label":"Journal Title"},{"value":"https:\/\/doi.org\/10.1016\/j.jss.2026.112936","name":"articlelink","label":"CrossRef DOI link to publisher maintained version"},{"value":"article","name":"content_type","label":"Content Type"},{"value":"\u00a9 2026 Elsevier Inc. All rights are reserved, including those for text and data mining, AI training, and similar technologies.","name":"copyright","label":"Copyright"}],"article-number":"112936"}}