{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,1]],"date-time":"2026-04-01T04:34:05Z","timestamp":1775018045907,"version":"3.50.1"},"reference-count":56,"publisher":"Springer Science and Business Media LLC","issue":"3","license":[{"start":{"date-parts":[[2014,2,9]],"date-time":"2014-02-09T00:00:00Z","timestamp":1391904000000},"content-version":"tdm","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["Empir Software Eng"],"published-print":{"date-parts":[[2015,6]]},"DOI":"10.1007\/s10664-013-9299-z","type":"journal-article","created":{"date-parts":[[2014,2,8]],"date-time":"2014-02-08T07:45:10Z","timestamp":1391845510000},"page":"783-812","source":"Crossref","is-referenced-by-count":88,"title":["Achieving scalable mutation-based generation of whole test suites"],"prefix":"10.1007","volume":"20","author":[{"given":"Gordon","family":"Fraser","sequence":"first","affiliation":[]},{"given":"Andrea","family":"Arcuri","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2014,2,9]]},"reference":[{"key":"9299_CR1","unstructured":"Acree AT (1980) On mutation. Phd thesis, Georgia Institute of Technology, Atlanta, Georgia"},{"key":"9299_CR2","doi-asserted-by":"crossref","unstructured":"Adamopoulos K, Harman M, Hierons RM (2004) How to overcome the equivalent mutant problem and achieve tailored selective mutation using co-evolution. In: Genetic and evolutionary computation conference (GECCO). Seattle, Washington, pp 1338\u20131349","DOI":"10.1007\/978-3-540-24855-2_155"},{"key":"9299_CR3","doi-asserted-by":"crossref","unstructured":"Andrews JH, Briand LC, Labiche Y (2005) Is mutation an appropriate tool for testing experiments? In: Proceedings of the 27th international conference on software engineering, ICSE 05. ACM, pp 402\u2013411","DOI":"10.1145\/1062455.1062530"},{"issue":"2","key":"9299_CR4","doi-asserted-by":"crossref","first-page":"119","DOI":"10.1002\/stvr.457","volume":"23","author":"A Arcuri","year":"2013","unstructured":"Arcuri A (2013) It really does matter how you normalize the branch distance in search-based software testing. Softw Test Verification Reliab (STVR) 23(2):119\u2013147","journal-title":"Softw Test Verification Reliab (STVR)"},{"key":"9299_CR5","doi-asserted-by":"crossref","unstructured":"Arcuri A, Briand L (2012) A hitchhiker\u2019s guide to statistical tests for assessing randomized algorithms in software engineering. In: Software testing verification and reliability (STVR). doi: 10.1002\/stvr.1486","DOI":"10.1002\/stvr.1486"},{"key":"9299_CR6","doi-asserted-by":"crossref","unstructured":"Arcuri A, Fraser G (2013) Parameter tuning or default values? An empirical investigation in search-based software engineering. In: Empirical software engineering (EMSE). pp 1\u201330. doi: 10.1007\/s10664-013-9249-9","DOI":"10.1007\/s10664-013-9249-9"},{"key":"9299_CR7","doi-asserted-by":"crossref","unstructured":"Ayari K, Bouktif S, Antoniol G (2007) Automatic mutation test input data generation via ant colony. In: Genetic and evolutionary computation conference (GECCO). ACM, New York, pp 1074\u20131081","DOI":"10.1145\/1276958.1277172"},{"key":"9299_CR8","doi-asserted-by":"crossref","unstructured":"Baker R, Habli I (2012) An empirical evaluation of mutation testing for improving the test quality of safety-critical software. In: IEEE transactions on software engineering (TSE)","DOI":"10.1109\/TSE.2012.56"},{"key":"9299_CR9","doi-asserted-by":"crossref","unstructured":"Baldwin D, Sayward FG (1979) Heuristics for determining equivalence of program mutations. Technical Report 276, Yale University, New Haven, Connecticut","DOI":"10.21236\/ADA071795"},{"issue":"2","key":"9299_CR10","doi-asserted-by":"crossref","first-page":"76","DOI":"10.1109\/MS.2005.30","volume":"22","author":"B Baudry","year":"2005","unstructured":"Baudry B, Fleurey F, Jzquel JM, Le Traon Y (2005) Automatic test cases optimization: a bacteriologic algorithm. IEEE Softw 22(2):76\u201382","journal-title":"IEEE Softw"},{"key":"9299_CR11","doi-asserted-by":"crossref","unstructured":"Bauersfeld S, Vos T, Lakhotia K, Poulding S, Condori N (2013) Unit testing tool competition. In: International workshop on search-based software testing (SBST). pp 414\u2013420","DOI":"10.1109\/ICSTW.2013.55"},{"key":"9299_CR12","unstructured":"Bottaci L (2001) A genetic algorithm fitness function for mutation testing. In: International workshop on software engineering using metaheuristic inovative algorithms, a workshop at 23rd International conference on software engineering, SEMINAL 2001. pp 3\u20137"},{"key":"9299_CR13","unstructured":"Budd TA (1980) Mutation analysis of program test data. Phd thesis, Yale University, New Haven, Connecticut"},{"issue":"9","key":"9299_CR14","doi-asserted-by":"crossref","first-page":"900","DOI":"10.1109\/32.92910","volume":"17","author":"RA DeMillo","year":"1991","unstructured":"DeMillo RA, Offutt AJ (1991) Constraint-based automatic test data generation. IEEE Trans Softw Eng 17(9):900\u2013910","journal-title":"IEEE Trans Softw Eng"},{"issue":"4","key":"9299_CR15","doi-asserted-by":"crossref","first-page":"34","DOI":"10.1109\/C-M.1978.218136","volume":"11","author":"RA DeMillo","year":"1978","unstructured":"DeMillo RA, Lipton RJ, Sayward FG (1978) Hints on test data selection: help for the practicing programmer. Computer 11(4):34\u201341","journal-title":"Computer"},{"key":"9299_CR16","doi-asserted-by":"crossref","unstructured":"Deng L, Offutt J, Li N (2013) Empirical evaluation of the statement deletion mutation operator. In: IEEE International conference on software testing, verification and validation (ICST)","DOI":"10.1109\/ICST.2013.20"},{"key":"9299_CR17","doi-asserted-by":"crossref","unstructured":"Fleyshgakker VN, Weiss SN (1994) Efficient mutation analysis: a new approach. In: Proceedings of the international symposium on software testing and analysis, ISSTA \u201994. Seattle, Washington, pp 185\u2013195","DOI":"10.1145\/186258.187179"},{"issue":"3","key":"9299_CR18","doi-asserted-by":"crossref","first-page":"235","DOI":"10.1016\/S0164-1212(96)00154-9","volume":"38","author":"PG Frankl","year":"1997","unstructured":"Frankl PG, Weiss SN, Hu C (1997) All-uses vs mutation testing: an experimental comparison of effectiveness. J Syst Softw 38(3):235\u2013253","journal-title":"J Syst Softw"},{"key":"9299_CR19","doi-asserted-by":"crossref","unstructured":"Fraser G, Arcuri A (2011a) EvoSuite: Automatic test suite generation for object-oriented software. In: ACM symposium on the foundations of software engineering (FSE). pp 416\u2013419","DOI":"10.1145\/2025113.2025179"},{"key":"9299_CR20","doi-asserted-by":"crossref","unstructured":"Fraser G, Arcuri A (2011b) It is not the length that matters, it is how you control it. In: IEEE international conference on software testing, verification and validation (ICST). pp 150\u2013159","DOI":"10.1109\/ICST.2011.54"},{"key":"9299_CR21","doi-asserted-by":"crossref","unstructured":"Fraser G, Arcuri A (2012a) The seed is strong: Seeding strategies in search-based software testing. In: IEEE international conference on software testing, verification and validation (ICST). pp 121\u2013130","DOI":"10.1109\/ICST.2012.92"},{"key":"9299_CR22","doi-asserted-by":"crossref","unstructured":"Fraser G, Arcuri A (2012b) Sound empirical evidence in software testing. In: ACM\/IEEE international conference on software engineering (ICSE). pp 178\u2013188","DOI":"10.1109\/ICSE.2012.6227195"},{"key":"9299_CR23","doi-asserted-by":"crossref","unstructured":"Fraser G, Arcuri A (2013a) Evosuite at the SBST 2013 tool competition. In: International workshop on search-based software testing (SBST). pp 406\u2013409","DOI":"10.1109\/ICSTW.2013.53"},{"key":"9299_CR24","doi-asserted-by":"crossref","unstructured":"Fraser G, Arcuri A (2013b) EvoSuite: On the challenges of test case generation in the real world (tool paper). In: IEEE international conference on software testing, verification and validation (ICST)","DOI":"10.1109\/ICST.2013.51"},{"key":"9299_CR25","doi-asserted-by":"crossref","unstructured":"Fraser G, Arcuri A (2013c) Whole test suite generation, vol 39","DOI":"10.1109\/TSE.2012.14"},{"issue":"2","key":"9299_CR26","doi-asserted-by":"crossref","first-page":"278","DOI":"10.1109\/TSE.2011.93","volume":"28","author":"G Fraser","year":"2012","unstructured":"Fraser G, Zeller A (2012) Mutation-driven generation of unit tests and oracles. IEEE Trans Softw Eng (TSE) 28(2):278\u2013292","journal-title":"IEEE Trans Softw Eng (TSE)"},{"key":"9299_CR27","doi-asserted-by":"crossref","unstructured":"Fraser G, Arcuri A, McMinn P (2013) Test suite generation with memetic algorithms. In: Genetic and evolutionary computation conference (GECCO)","DOI":"10.1145\/2463372.2463548"},{"key":"9299_CR28","doi-asserted-by":"crossref","unstructured":"Godefroid P, Klarlund N, Sen K (2005) DART: directed automated random testing. In: Proceedings of the 2005 ACM SIGPLAN conference on programming language design and implementation, PLDI05. ACM, pp 213223","DOI":"10.1145\/1065010.1065036"},{"issue":"4","key":"9299_CR29","doi-asserted-by":"crossref","first-page":"279","DOI":"10.1109\/TSE.1977.231145","volume":"3","author":"RG Hamlet","year":"1977","unstructured":"Hamlet RG (1977) Testing programs with the aid of a compiler. IEEE Trans Softw Eng 3(4):279\u2013290","journal-title":"IEEE Trans Softw Eng"},{"key":"9299_CR30","doi-asserted-by":"crossref","unstructured":"Harman M, Jia Y, Langdon WB (2011) Strong higher order mutation-based test data generation. In: Proceedings of the 19th ACM SIGSOFT symposium and the 13th European conference on Foundations of software engineering, ESEC\/FSE 11. ACM, pp 212\u2013222","DOI":"10.1145\/2025113.2025144"},{"issue":"4","key":"9299_CR31","doi-asserted-by":"crossref","first-page":"233","DOI":"10.1002\/(SICI)1099-1689(199912)9:4<233::AID-STVR191>3.0.CO;2-3","volume":"9","author":"RM Hierons","year":"1999","unstructured":"Hierons RM, Harman M, Danicic S (1999) Using program slicing to assist in the detection of equivalent mutants. Softw Test Verification Reliab 9(4):233\u2013262","journal-title":"Softw Test Verification Reliab"},{"issue":"4","key":"9299_CR32","doi-asserted-by":"crossref","first-page":"371","DOI":"10.1109\/TSE.1982.235571","volume":"8","author":"WE Howden","year":"1982","unstructured":"Howden WE (1982) Weak mutation testing and completeness of test sets. IEEE Trans Softw Eng 8(4):371\u2013379","journal-title":"IEEE Trans Softw Eng"},{"issue":"10","key":"9299_CR33","doi-asserted-by":"crossref","first-page":"1379","DOI":"10.1016\/j.infsof.2009.04.016","volume":"51","author":"Y Jia","year":"2009","unstructured":"Jia Y, Harman M (2009) Higher order mutation testing. J Informat Softw Technol 51(10):1379\u20131393","journal-title":"J Informat Softw Technol"},{"issue":"5","key":"9299_CR34","doi-asserted-by":"crossref","first-page":"649","DOI":"10.1109\/TSE.2010.62","volume":"37","author":"Y Jia","year":"2011","unstructured":"Jia Y, Harman M (2011) An analysis and survey of the development of mutation testing. IEEE Trans Softw Eng (TSE) 37(5):649\u2013678","journal-title":"IEEE Trans Softw Eng (TSE)"},{"key":"9299_CR35","doi-asserted-by":"crossref","unstructured":"Just R, Kapfhammer GM, Schweiggert F (2012) Using non-redundant mutation operators and test suite prioritization to achieve efficient and scalable mutation analysis. In: Proceedings of the 2012 IEEE 23rd international symposium on software reliability engineering, ISSRE 12. IEEE Computer Society, pp11\u201320","DOI":"10.1109\/ISSRE.2012.31"},{"key":"9299_CR36","unstructured":"Just R, Ernst MD, Fraser G (2013) Using state infection conditions to detect equivalent mutants and speed up mutation analysis. arXiv preprint arXiv: 13032784"},{"key":"9299_CR37","doi-asserted-by":"crossref","unstructured":"Korel B (1990) Automated software test data generation. In: IEEE Transactions on software engineering, pp 870\u2013879","DOI":"10.1109\/32.57624"},{"key":"9299_CR38","unstructured":"Mateo PR, Usaola MP, Aleman JLF (2012) Validating 2nd-order mutation at system level. In: IEEE Transactions on software engineering (TSE)"},{"issue":"2","key":"9299_CR39","doi-asserted-by":"crossref","first-page":"105","DOI":"10.1002\/stvr.294","volume":"14","author":"P McMinn","year":"2004","unstructured":"McMinn P (2004) Search-based software test data generation: a survey. Softw Test Verification Reliab 14(2):105\u2013156","journal-title":"Softw Test Verification Reliab"},{"issue":"1","key":"9299_CR40","doi-asserted-by":"crossref","first-page":"5","DOI":"10.1145\/125489.125473","volume":"1","author":"AJ Offutt","year":"1992","unstructured":"Offutt AJ (1992) Investigations of the software testing coupling effect. ACM Trans Softw Eng Methodol 1(1):5\u201320","journal-title":"ACM Trans Softw Eng Methodol"},{"issue":"3","key":"9299_CR41","doi-asserted-by":"crossref","first-page":"131","DOI":"10.1002\/stvr.4370040303","volume":"4","author":"AJ Offutt","year":"1994","unstructured":"Offutt AJ, Craft WM (1994) Using compiler optimization techniques to detect equivalent mutants. Softw Test Verification Reliab 4(3):131\u2013154","journal-title":"Softw Test Verification Reliab"},{"key":"9299_CR42","unstructured":"Offutt AJ, Lee SD (1991) How strong is weak mutation? In: Proceedings of the symposium on testing, analysis, and verification. TAV4. ACM, pp 200\u2013213"},{"issue":"5","key":"9299_CR43","doi-asserted-by":"crossref","first-page":"337","DOI":"10.1109\/32.286422","volume":"20","author":"AJ Offutt","year":"1994","unstructured":"Offutt AJ, Lee SD (1994) An empirical evaluation of weak mutation. IEEE Trans Softw Eng 20(5):337\u2013344","journal-title":"IEEE Trans Softw Eng"},{"issue":"3","key":"9299_CR44","doi-asserted-by":"crossref","first-page":"165","DOI":"10.1002\/(SICI)1099-1689(199709)7:3<165::AID-STVR143>3.0.CO;2-U","volume":"7","author":"AJ Offutt","year":"1997","unstructured":"Offutt AJ, Pan J (1997) Automatically detecting equivalent mutants and infeasible paths. Softw Test Verification Reliab 7(3):165\u2013192","journal-title":"Softw Test Verification Reliab"},{"key":"9299_CR45","unstructured":"Offutt AJ, Untch RH (2001) Mutation testing for the new century. Chap mutation 2000: uniting the orthogonal. Kluwer Academic Publishers, Norwell, MA, pp 34\u201344"},{"key":"9299_CR46","doi-asserted-by":"crossref","unstructured":"Offutt AJ, Rothermel G, Zapf C (1993) An experimental evaluation of selective mutation. In: Proceedings of the 15th international conference on software engineering, ICSE \u201993. Baltimore, Maryland, pp 100\u2013107","DOI":"10.1109\/ICSE.1993.346062"},{"issue":"5","key":"9299_CR47","first-page":"1","volume":"29","author":"AJ Offutt","year":"2004","unstructured":"Offutt AJ, Ma YS, Kwon YR (2004) An experimental mutation system for Java. ACM SIGSOFT Softw Eng Notes 29(5):1\u20134","journal-title":"ACM SIGSOFT Softw Eng Notes"},{"key":"9299_CR48","doi-asserted-by":"crossref","unstructured":"Pacheco C, Ernst MD (2007) Randoop: feedback-directed random testing for Java. In: Companion to the 22nd ACM SIGPLAN conference on object-oriented programming systems and application, OOPSLA07. ACM, pp 815816","DOI":"10.1145\/1297846.1297902"},{"key":"9299_CR49","doi-asserted-by":"crossref","unstructured":"Papadakis M, Malevris N (2010) Automatic mutation test case generation via dynamic symbolic execution. In: IEEE 21st International symposium on software reliability engineering (ISSRE). pp 121\u2013130","DOI":"10.1109\/ISSRE.2010.38"},{"key":"9299_CR50","doi-asserted-by":"crossref","unstructured":"Patrick M, Alexander R, Oriol M, Clark JA (2013) Using mutation analysis to evolve subdomains for random testing. In: International workshop on mutation analysis","DOI":"10.1109\/ICSTW.2013.14"},{"key":"9299_CR51","doi-asserted-by":"crossref","unstructured":"Schuler D, Zeller A (2010) (Un-)Covering equivalent mutants. In: Proceedings of the 3rd international conference on software testing, verification, and validation, ICST \u201910. IEEE Computer Society, pp 45\u201354","DOI":"10.1109\/ICST.2010.30"},{"key":"9299_CR52","doi-asserted-by":"crossref","unstructured":"Staats M, Whalen MW, Heimdahl MP (2011) Programs, tests, and oracles: the foundations of testing revisited. In: ACM\/IEEE international conference on software engineering (ICSE). pp 391\u2013400","DOI":"10.1145\/1985793.1985847"},{"key":"9299_CR53","doi-asserted-by":"crossref","unstructured":"Untch RH (1992) Mutation-based software testing using program schemata. In: Proceedings of the 30th annual southeast regional conference (ACM-SE\u201992). Raleigh, North Carolina, pp 285\u2013291","DOI":"10.1145\/503720.503749"},{"key":"9299_CR54","unstructured":"Walsh PJ (1985) A measure of test case completeness (software, engineering). PhD thesis, State University of New York at Binghamton, Binghamton, NY"},{"key":"9299_CR55","doi-asserted-by":"crossref","unstructured":"Wong WE, Mathur AP, Maldonado JC (1995) Mutation versus all-uses: an empirical evaluation of cost, strength and effectiveness. In: Software quality and productivity: theory, practice and training. Chapman & Hall, London, pp 258\u2013265","DOI":"10.1007\/978-0-387-34848-3_40"},{"key":"9299_CR56","doi-asserted-by":"crossref","unstructured":"Zhang L, Xie T, Zhang L, Tillmann N, de Halleux J, Mei H (2010) Test generation via dynamic symbolic execution for mutation testing. In: Proceedings of the 2010 IEEE international conference on software maintenance, ICSM 10. IEEE Computer Society, pp 110","DOI":"10.1109\/ICSM.2010.5609672"}],"container-title":["Empirical Software Engineering"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s10664-013-9299-z.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/article\/10.1007\/s10664-013-9299-z\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s10664-013-9299-z","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2019,8,7]],"date-time":"2019-08-07T13:15:34Z","timestamp":1565183734000},"score":1,"resource":{"primary":{"URL":"http:\/\/link.springer.com\/10.1007\/s10664-013-9299-z"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2014,2,9]]},"references-count":56,"journal-issue":{"issue":"3","published-print":{"date-parts":[[2015,6]]}},"alternative-id":["9299"],"URL":"https:\/\/doi.org\/10.1007\/s10664-013-9299-z","relation":{},"ISSN":["1382-3256","1573-7616"],"issn-type":[{"value":"1382-3256","type":"print"},{"value":"1573-7616","type":"electronic"}],"subject":[],"published":{"date-parts":[[2014,2,9]]}}}