{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,3,17]],"date-time":"2026-03-17T18:55:35Z","timestamp":1773773735321,"version":"3.50.1"},"reference-count":98,"publisher":"Springer Science and Business Media LLC","issue":"4","license":[{"start":{"date-parts":[[2020,5,16]],"date-time":"2020-05-16T00:00:00Z","timestamp":1589587200000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"},{"start":{"date-parts":[[2020,5,16]],"date-time":"2020-05-16T00:00:00Z","timestamp":1589587200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Int J Softw Tools Technol Transfer"],"published-print":{"date-parts":[[2020,8]]},"DOI":"10.1007\/s10009-020-00567-y","type":"journal-article","created":{"date-parts":[[2020,5,16]],"date-time":"2020-05-16T04:02:53Z","timestamp":1589601773000},"page":"365-388","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":17,"title":["Comparing mutation coverage against branch coverage in an industrial setting"],"prefix":"10.1007","volume":"22","author":[{"ORCID":"https:\/\/orcid.org\/0000-0001-8525-8198","authenticated-orcid":false,"given":"Ali","family":"Parsai","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-4463-2945","authenticated-orcid":false,"given":"Serge","family":"Demeyer","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2020,5,16]]},"reference":[{"key":"567_CR1","doi-asserted-by":"publisher","unstructured":"Aaltonen, K., Ihantola, P., Sepp\u00e4l\u00e4, O.: Mutation analysis vs. code coverage in automated assessment of students\u2019 testing skills. In: Proceedings of the ACM International Conference Companion on Object Oriented Programming Systems Languages and Applications Companion\u2014SPLASH \u201910, OOPSLA \u201910, pp. 153\u2013160. ACM Press, New York (2010). https:\/\/doi.org\/10.1145\/1869542.1869567","DOI":"10.1145\/1869542.1869567"},{"key":"567_CR2","doi-asserted-by":"publisher","unstructured":"Abdi, H.: Kendall rank correlation coefficient. In: The Concise Encyclopedia of Statistics, pp. 278\u2013281. Springer New York (2007). https:\/\/doi.org\/10.1007\/978-0-387-32833-1_211","DOI":"10.1007\/978-0-387-32833-1_211"},{"issue":"1","key":"567_CR3","doi-asserted-by":"publisher","first-page":"94","DOI":"10.1109\/TSE.2008.73","volume":"35","author":"R Abraham","year":"2009","unstructured":"Abraham, R., Erwig, M.: Mutation operators for spreadsheets. IEEE Trans. Softw. Eng. 35(1), 94\u2013108 (2009). https:\/\/doi.org\/10.1109\/TSE.2008.73","journal-title":"IEEE Trans. Softw. Eng."},{"key":"567_CR4","unstructured":"Acree, A.T., Jr.: On mutation. Ph.D. thesis, Georgia Institute of Technology, Atlanta, GA (1980)"},{"key":"567_CR5","doi-asserted-by":"publisher","DOI":"10.1002\/9780470594001","volume-title":"Analysis of Ordinal Categorical Data","author":"A Agresti","year":"2010","unstructured":"Agresti, A.: Analysis of Ordinal Categorical Data, vol. 656. Wiley, New York (2010). https:\/\/doi.org\/10.1002\/9780470594001"},{"key":"567_CR6","doi-asserted-by":"publisher","unstructured":"Ahmed, Z., Zahoor, M., Younas, I.: Mutation operators for object-oriented systems: a survey. In: 2010 The 2nd International Conference on Computer and Automation Engineering (ICCAE), vol.\u00a02, pp. 614\u2013618. IEEE (2010). https:\/\/doi.org\/10.1109\/ICCAE.2010.5451692","DOI":"10.1109\/ICCAE.2010.5451692"},{"key":"567_CR7","doi-asserted-by":"publisher","unstructured":"Ammann, P., Delamaro, M.E., Offutt, J.: Establishing theoretical minimal sets of mutants. In: 2014 IEEE Seventh International Conference on Software Testing, Verification and Validation, pp. 21\u201330. IEEE (2014). https:\/\/doi.org\/10.1109\/ICST.2014.13","DOI":"10.1109\/ICST.2014.13"},{"key":"567_CR8","doi-asserted-by":"publisher","DOI":"10.1017\/9781316771273","volume-title":"Introduction to Software Testing","author":"P Ammann","year":"2016","unstructured":"Ammann, P., Offutt, J.: Introduction to Software Testing, 2nd edn. Cambridge University Press, Cambridge (2016). https:\/\/doi.org\/10.1017\/9781316771273","edition":"2"},{"key":"567_CR9","doi-asserted-by":"publisher","unstructured":"Andrews, J.H., Briand, L.C., Labiche, Y.: Is mutation an appropriate tool for testing experiments? In: Proceedings of the 27th International Conference on Software Engineering\u2014ICSE \u201905, pp. 402\u2013411. ACM Press, New York (2005). https:\/\/doi.org\/10.1145\/1062455.1062530","DOI":"10.1145\/1062455.1062530"},{"issue":"8","key":"567_CR10","doi-asserted-by":"publisher","first-page":"608","DOI":"10.1109\/tse.2006.83","volume":"32","author":"JH Andrews","year":"2006","unstructured":"Andrews, J.H., Briand, L.C., Labiche, Y., Namin, A.S.: Using mutation analysis for assessing and comparing testing coverage criteria. IEEE Trans. Softw. Eng. 32(8), 608\u2013624 (2006). https:\/\/doi.org\/10.1109\/tse.2006.83","journal-title":"IEEE Trans. Softw. Eng."},{"key":"567_CR11","doi-asserted-by":"publisher","unstructured":"Assylbekov, B., Gaspar, E., Uddin, N., Egan, P.: Investigating the correlation between mutation score and coverage score. In: 2013 UKSim 15th International Conference on Computer Modelling and Simulation, pp. 347\u2013352. IEEE (2013). https:\/\/doi.org\/10.1109\/UKSim.2013.28","DOI":"10.1109\/UKSim.2013.28"},{"key":"567_CR12","doi-asserted-by":"publisher","unstructured":"Baudry, B., Fleurey, F., Traon, Y.L.: Improving test suites for efficient fault localization. In: Proceeding of the 28th International Conference on Software Engineering\u2014ICSE \u201906, pp. 82\u201391. ACM Press, New York (2006). https:\/\/doi.org\/10.1145\/1134285.1134299","DOI":"10.1145\/1134285.1134299"},{"key":"567_CR13","doi-asserted-by":"publisher","unstructured":"Bauersfeld, S., Vos, T.E.J., Lakhotia, K.: Unit testing tool competitions\u2014lessons learned. In: Vos, E.T., Lakhotia, K., Bauersfeld, S. (eds.) Future Internet Testing: First International Workshop, FITTEST 2013, Istanbul, 12 Nov 2013, Revised Selected Papers, pp. 75\u201394. Springer, Cham (2014). https:\/\/doi.org\/10.1007\/978-3-319-07785-7_5","DOI":"10.1007\/978-3-319-07785-7_5"},{"key":"567_CR14","volume-title":"Test Driven Development: By Example","author":"K Beck","year":"2002","unstructured":"Beck, K.: Test Driven Development: By Example. Addison-Wesley Longman Publishing Co. Inc, Boston (2002)"},{"key":"567_CR15","doi-asserted-by":"publisher","unstructured":"Bjerke-Gulstuen, K., Larsen, E.W., St\u00e5lhane, T., Dings\u00f8yr, T.: High level test driven development\u2014shift left. In: Lassenius, C., Dings\u00f8yr, T., Paasivaara, M. (eds.) Agile Processes in Software Engineering and Extreme Programming, pp. 239\u2013247. Springer, Cham (2015). https:\/\/doi.org\/10.1007\/978-3-319-18612-2_23","DOI":"10.1007\/978-3-319-18612-2_23"},{"issue":"4","key":"567_CR16","doi-asserted-by":"publisher","first-page":"1203","DOI":"10.1007\/s11219-016-9328-4","volume":"25","author":"V Blondeau","year":"2017","unstructured":"Blondeau, V., Etien, A., Anquetil, N., Cresson, S., Croisy, P., Ducasse, S.: Test case selection in industry: an analysis of issues related to static approaches. Softw. Qual. J. 25(4), 1203\u20131237 (2017). https:\/\/doi.org\/10.1007\/s11219-016-9328-4","journal-title":"Softw. Qual. J."},{"key":"567_CR17","unstructured":"Booch, G.: Object Oriented Design: With Applications. The Benjamin\/Cummings Series in Ada and Software Engineering. Benjamin\/Cummings Pub. (1991). http:\/\/books.google.be\/books?id=w5VQAAAAMAAJ"},{"key":"567_CR18","doi-asserted-by":"publisher","unstructured":"Bradbury, J.S., Cordy, J.R., Dingel, J.: ExMAn: a generic and customizable framework for experimental mutation analysis. In: Second Workshop on Mutation Analysis (Mutation 2006\u2014ISSRE Workshops 2006), MUTATION \u201906, p.\u00a04. IEEE, Washington, DC (2006). https:\/\/doi.org\/10.1109\/mutation.2006.5","DOI":"10.1109\/mutation.2006.5"},{"issue":"4","key":"567_CR19","doi-asserted-by":"publisher","first-page":"10","DOI":"10.1109\/mc.1987.1663532","volume":"20","author":"FP Brooks","year":"1987","unstructured":"Brooks, F.P.: No silver bullet essence and accidents of software engineering. Computer 20(4), 10\u201319 (1987). https:\/\/doi.org\/10.1109\/mc.1987.1663532","journal-title":"Computer"},{"issue":"3","key":"567_CR20","doi-asserted-by":"publisher","first-page":"5","DOI":"10.1145\/2070336.2070341","volume":"31","author":"B Brosgol","year":"2011","unstructured":"Brosgol, B.: Do-178C: the next avionics safety standard. ACM SIGAda Ada Lett. 31(3), 5\u20136 (2011). https:\/\/doi.org\/10.1145\/2070336.2070341","journal-title":"ACM SIGAda Ada Lett."},{"key":"567_CR21","unstructured":"Budd, T.A.: Mutation analysis of program test data. Ph.D. thesis, Yale University, New Haven (1980). AAI8025191"},{"key":"567_CR22","doi-asserted-by":"publisher","unstructured":"Chen, B., Song, J., Xu, P., Hu, X., Jiang, Z.M.J.: An automated approach to estimating code coverage measures via execution logs. In: Proceedings of the 33rd ACM\/IEEE International Conference on Automated Software Engineering, ASE 2018, pp. 305\u2013316. ACM, New York (2018). https:\/\/doi.org\/10.1145\/3238147.3238214","DOI":"10.1145\/3238147.3238214"},{"key":"567_CR23","doi-asserted-by":"publisher","unstructured":"Chen, H.Y., Hu, S.: Two new kinds of class level mutants for object-oriented programs. In: 2006 IEEE International Conference on Systems, Man and Cybernetics, vol.\u00a03, pp. 2173\u20132178. IEEE (2006). https:\/\/doi.org\/10.1109\/icsmc.2006.385183","DOI":"10.1109\/icsmc.2006.385183"},{"issue":"9","key":"567_CR24","doi-asserted-by":"publisher","first-page":"900","DOI":"10.1109\/32.92910","volume":"17","author":"R DeMilli","year":"1991","unstructured":"DeMilli, R., Offutt, A.: Constraint-based automatic test data generation. IEEE Trans. Softw. Eng. 17(9), 900\u2013910 (1991). https:\/\/doi.org\/10.1109\/32.92910","journal-title":"IEEE Trans. Softw. Eng."},{"issue":"4","key":"567_CR25","doi-asserted-by":"publisher","first-page":"34","DOI":"10.1109\/C-M.1978.218136","volume":"11","author":"RA DeMillo","year":"1978","unstructured":"DeMillo, R.A., Lipton, R.J., Sayward, F.G.: Hints on test data selection: help for the practicing programmer. Computer 11(4), 34\u201341 (1978). https:\/\/doi.org\/10.1109\/C-M.1978.218136","journal-title":"Computer"},{"key":"567_CR26","doi-asserted-by":"publisher","unstructured":"Derezi\u0144ska, A., Rudnik, M.: Quality evaluation of object-oriented and standard mutation operators applied to c# programs. In: Furia, C.A., Nanz, S. (eds.) Objects, Models, Components, Patterns, pp. 42\u201357. Springer, Berlin (2012). https:\/\/doi.org\/10.1007\/978-3-642-30561-0_5","DOI":"10.1007\/978-3-642-30561-0_5"},{"issue":"9","key":"567_CR27","doi-asserted-by":"publisher","first-page":"733","DOI":"10.1109\/tse.2006.92","volume":"32","author":"H Do","year":"2006","unstructured":"Do, H., Rothermel, G.: On the use of mutation faults in empirical assessments of test case prioritization techniques. IEEE Trans. Softw. Eng. 32(9), 733\u2013752 (2006). https:\/\/doi.org\/10.1109\/tse.2006.92","journal-title":"IEEE Trans. Softw. Eng."},{"issue":"8","key":"567_CR28","doi-asserted-by":"publisher","first-page":"861","DOI":"10.1016\/j.patrec.2005.10.010","volume":"27","author":"T Fawcett","year":"2006","unstructured":"Fawcett, T.: An introduction to ROC analysis. Pattern Recognit. Lett. 27(8), 861\u2013874 (2006). https:\/\/doi.org\/10.1016\/j.patrec.2005.10.010","journal-title":"Pattern Recognit. Lett."},{"key":"567_CR29","unstructured":"Fowler, M.: Continuous integration. Tech. rep., http:\/\/www.martinfowler.com\/ (2006). http:\/\/www.martinfowler.com\/articles\/continuousIntegration.html. Accessed 9 May 2020"},{"issue":"8","key":"567_CR30","doi-asserted-by":"publisher","first-page":"774","DOI":"10.1109\/32.238581","volume":"19","author":"PG Frankl","year":"1993","unstructured":"Frankl, P.G., Weiss, S.N.: An experimental comparison of the effectiveness of branch testing and data flow testing. IEEE Trans. Softw. Eng. 19(8), 774\u2013787 (1993). https:\/\/doi.org\/10.1109\/32.238581","journal-title":"IEEE Trans. Softw. Eng."},{"issue":"3","key":"567_CR31","doi-asserted-by":"publisher","first-page":"235","DOI":"10.1016\/s0164-1212(96)00154-9","volume":"38","author":"PG Frankl","year":"1997","unstructured":"Frankl, P.G., Weiss, S.N., Hu, C.: All-uses vs mutation testing: an experimental comparison of effectiveness. J. Syst. Softw. 38(3), 235\u2013253 (1997). https:\/\/doi.org\/10.1016\/s0164-1212(96)00154-9","journal-title":"J. Syst. Softw."},{"issue":"2","key":"567_CR32","doi-asserted-by":"publisher","first-page":"278","DOI":"10.1109\/TSE.2011.93","volume":"38","author":"G Fraser","year":"2012","unstructured":"Fraser, G., Zeller, A.: Mutation-driven generation of unit tests and oracles. IEEE Trans. Softw. Eng. 38(2), 278\u2013292 (2012). https:\/\/doi.org\/10.1109\/TSE.2011.93","journal-title":"IEEE Trans. Softw. Eng."},{"key":"567_CR33","doi-asserted-by":"publisher","unstructured":"Garousi, V., Yildirim, E.: Introducing automated GUI testing and observing its benefits: an industrial case study in the context of law-practice management software. In: 2018 IEEE International Conference on Software Testing, Verification and Validation Workshops\u2014NEXTA Workshop, pp. 138\u2013145. IEEE (2018). https:\/\/doi.org\/10.1109\/icstw.2018.00042","DOI":"10.1109\/icstw.2018.00042"},{"issue":"8","key":"567_CR34","doi-asserted-by":"publisher","first-page":"803","DOI":"10.1109\/tse.2015.2421011","volume":"41","author":"G Gay","year":"2015","unstructured":"Gay, G., Staats, M., Whalen, M., Heimdahl, M.P.E.: The risks of coverage-directed test case generation. IEEE Trans. Softw. Eng. 41(8), 803\u2013819 (2015). https:\/\/doi.org\/10.1109\/tse.2015.2421011","journal-title":"IEEE Trans. Softw. Eng."},{"key":"567_CR35","doi-asserted-by":"publisher","unstructured":"Gligoric, M., Groce, A., Zhang, C., Sharma, R., Alipour, M.A., Marinov, D.: Comparing non-adequate test suites using coverage criteria. In: Proceedings of the 2013 International Symposium on Software Testing and Analysis\u2014ISSTA 2013, ISSTA 2013, pp. 302\u2013313. ACM Press, New York (2013). https:\/\/doi.org\/10.1145\/2483760.2483769","DOI":"10.1145\/2483760.2483769"},{"key":"567_CR36","doi-asserted-by":"publisher","unstructured":"Gopinath, R., Jensen, C., Groce, A.: Code coverage for suite evaluation by developers. In: Proceedings of the 36th International Conference on Software Engineering\u2014ICSE 2014, ICSE 2014, pp. 72\u201382. ACM Press, New York (2014). https:\/\/doi.org\/10.1145\/2568225.2568278","DOI":"10.1145\/2568225.2568278"},{"key":"567_CR37","doi-asserted-by":"publisher","unstructured":"Gr\u00fcn, B.J.M., Schuler, D., Zeller, A.: The impact of equivalent mutants. In: 2009 International Conference on Software Testing, Verification, and Validation Workshops\u2014mutation workshop, pp. 192\u2013199. IEEE (2009). https:\/\/doi.org\/10.1109\/ICSTW.2009.37","DOI":"10.1109\/ICSTW.2009.37"},{"key":"567_CR38","doi-asserted-by":"publisher","unstructured":"Harman, M., Jia, Y., Langdon, W.B.: Strong higher order mutation-based test data generation. In: Proceedings of the 19th ACM SIGSOFT Symposium and the 13th European Conference on Foundations of Software Engineering\u2014SIGSOFT\/FSE \u201911, ESEC\/FSE \u201911, pp. 212\u2013222. ACM Press, New York (2011). https:\/\/doi.org\/10.1145\/2025113.2025144","DOI":"10.1145\/2025113.2025144"},{"key":"567_CR39","doi-asserted-by":"publisher","unstructured":"Haschemi, S., Wei\u00dfleder, S.: A generic approach to run mutation analysis. In: Bottaci, L., Fraser, G. (eds.) Testing\u2014Practice and Research Techniques, Lecture Notes in Computer Science, vol. 6303, pp. 155\u2013164. Springer, Berlin (2010). https:\/\/doi.org\/10.1007\/978-3-642-15585-7_15","DOI":"10.1007\/978-3-642-15585-7_15"},{"key":"567_CR40","doi-asserted-by":"publisher","unstructured":"Hayhurst, K., Veerhusen, D.: A practical approach to modified condition\/decision coverage. In: 20th DASC. 20th Digital Avionics Systems Conference (Cat. No.01CH37219), vol.\u00a01, pp. 1B2\/1\u20131B2\/10 vol.1. IEEE (2001). https:\/\/doi.org\/10.1109\/dasc.2001.963305","DOI":"10.1109\/dasc.2001.963305"},{"key":"567_CR41","doi-asserted-by":"publisher","unstructured":"Hemmati, H.: How effective are code coverage criteria? In: 2015 IEEE International Conference on Software Quality, Reliability and Security, pp. 151\u2013156. IEEE (2015). https:\/\/doi.org\/10.1109\/QRS.2015.30","DOI":"10.1109\/QRS.2015.30"},{"key":"567_CR42","volume-title":"Eating the IT Elephant: Moving from Greenfield Development to Brownfield","author":"R Hopkins","year":"2008","unstructured":"Hopkins, R., Jenkins, K.: Eating the IT Elephant: Moving from Greenfield Development to Brownfield. IBM Press, Indianapolis (2008)"},{"key":"567_CR43","doi-asserted-by":"publisher","unstructured":"Hutchins, M., Foster, H., Goradia, T., Ostrand, T.: Experiments on the effectiveness of dataflow- and control-flow-based test adequacy criteria. In: Proceedings of 16th International Conference on Software Engineering, ICSE \u201994, pp. 191\u2013200. IEEE Computer Society Press, Los Alamitos, CA (1994). https:\/\/doi.org\/10.1109\/ICSE.1994.296778","DOI":"10.1109\/ICSE.1994.296778"},{"key":"567_CR44","doi-asserted-by":"publisher","unstructured":"Inozemtseva, L., Holmes, R.: Coverage is not strongly correlated with test suite effectiveness. In: Proceedings of the 36th International Conference on Software Engineering\u2014ICSE 2014, ICSE 2014, pp. 435\u2013445. ACM Press, New York (2014). https:\/\/doi.org\/10.1145\/2568225.2568271","DOI":"10.1145\/2568225.2568271"},{"key":"567_CR45","unstructured":"ISO: Road vehicles\u2014Functional safety (2011)"},{"key":"567_CR46","doi-asserted-by":"publisher","unstructured":"Janjic, W., Atkinson, C.: Utilizing software reuse experience for automated test recommendation. In: 2013 8th International Workshop on Automation of Software Test (AST), AST \u201913, pp. 100\u2013106. IEEE, Piscataway (2013). https:\/\/doi.org\/10.1109\/iwast.2013.6595799. http:\/\/dl.acm.org\/citation.cfm?id=2662413.2662436","DOI":"10.1109\/iwast.2013.6595799"},{"issue":"5","key":"567_CR47","doi-asserted-by":"publisher","first-page":"649","DOI":"10.1109\/TSE.2010.62","volume":"37","author":"Y Jia","year":"2011","unstructured":"Jia, Y., Harman, M.: An analysis and survey of the development of mutation testing. IEEE Trans. Softw. Eng. 37(5), 649\u2013678 (2011). https:\/\/doi.org\/10.1109\/TSE.2010.62","journal-title":"IEEE Trans. Softw. Eng."},{"key":"567_CR48","doi-asserted-by":"publisher","unstructured":"Just, R.: The Major mutation framework: efficient and scalable mutation analysis for java. In: Proceedings of the 2014 International Symposium on Software Testing and Analysis\u2014ISSTA 2014, ISSTA 2014, pp. 433\u2013436. ACM Press, New York (2014). https:\/\/doi.org\/10.1145\/2610384.2628053","DOI":"10.1145\/2610384.2628053"},{"key":"567_CR49","volume-title":"Practical Unit Testing with TestNG and Mockito","author":"T Kaczanowski","year":"2012","unstructured":"Kaczanowski, T.: Practical Unit Testing with TestNG and Mockito. Tomasz Kaczanowski, Cracow (2012)"},{"issue":"3","key":"567_CR50","doi-asserted-by":"publisher","first-page":"261","DOI":"10.1007\/s00607-014-0418-5","volume":"97","author":"S Kandl","year":"2014","unstructured":"Kandl, S., Chandrashekar, S.: Reasonability of MC\/DC for safety-relevant software implemented in programming languages with short-circuit evaluation. Computing 97(3), 261\u2013279 (2014). https:\/\/doi.org\/10.1007\/s00607-014-0418-5","journal-title":"Computing"},{"issue":"7","key":"567_CR51","doi-asserted-by":"publisher","first-page":"685","DOI":"10.1002\/spe.4380210704","volume":"21","author":"KN King","year":"1991","unstructured":"King, K.N., Offutt, A.J.: A fortran language system for mutation-based software testing. Softw. Pract. Exp. 21(7), 685\u2013718 (1991). https:\/\/doi.org\/10.1002\/spe.4380210704","journal-title":"Softw. Pract. Exp."},{"key":"567_CR52","doi-asserted-by":"publisher","unstructured":"Kracht, J.S., Petrovic, J.Z., Walcott-Justice, K.R.: Empirically evaluating the quality of automatically generated and manually written test suites. In: 2014 14th International Conference on Quality Software, pp. 256\u2013265. IEEE (2014). https:\/\/doi.org\/10.1109\/qsic.2014.33","DOI":"10.1109\/qsic.2014.33"},{"key":"567_CR53","doi-asserted-by":"publisher","unstructured":"Kurtz, B., Ammann, P., Delamaro, M.E., Offutt, J., Deng, L.: Mutant subsumption graphs. In: Proceedings of the 2014 IEEE International Conference on Software Testing, Verification, and Validation Workshops\u2014Mutation Workshop, ICST \u201914, pp. 176\u2013185. IEEE Computer Society, Washington, DC, USA (2014). https:\/\/doi.org\/10.1109\/ICSTW.2014.20","DOI":"10.1109\/ICSTW.2014.20"},{"key":"567_CR54","doi-asserted-by":"publisher","unstructured":"Lee, H.J., Ma, Y.S., Kwon, Y.R.: Empirical evaluation of orthogonality of class mutation operators. In: 11th Asia-Pacific Software Engineering Conference, APSEC \u201904, pp. 512\u2013518. IEEE, Washington, DC (2004). https:\/\/doi.org\/10.1109\/apsec.2004.49","DOI":"10.1109\/apsec.2004.49"},{"key":"567_CR55","doi-asserted-by":"publisher","unstructured":"Li, N., Meng, X., Offutt, J., Deng, L.: Is bytecode instrumentation as good as source code instrumentation: An empirical study with industrial tools (experience report). In: 2013 IEEE 24th International Symposium on Software Reliability Engineering (ISSRE), pp. 380\u2013389. IEEE (2013). https:\/\/doi.org\/10.1109\/ISSRE.2013.6698891","DOI":"10.1109\/ISSRE.2013.6698891"},{"issue":"4","key":"567_CR56","doi-asserted-by":"publisher","first-page":"372","DOI":"10.1109\/tse.2016.2597136","volume":"43","author":"N Li","year":"2017","unstructured":"Li, N., Offutt, J.: Test oracle strategies for model-based testing. IEEE Trans. Softw. Eng. 43(4), 372\u2013395 (2017). https:\/\/doi.org\/10.1109\/tse.2016.2597136","journal-title":"IEEE Trans. Softw. Eng."},{"key":"567_CR57","doi-asserted-by":"publisher","unstructured":"Li, N., Praphamontripong, U., Offutt, J.: An experimental comparison of four unit test criteria: Mutation, edge-pair, all-uses and prime path coverage. In: 2009 International Conference on Software Testing, Verification, and Validation Workshops\u2014Mutation Workshop, pp. 220\u2013229. IEEE (2009). https:\/\/doi.org\/10.1109\/ICSTW.2009.30","DOI":"10.1109\/ICSTW.2009.30"},{"key":"567_CR58","doi-asserted-by":"publisher","unstructured":"Li, N., West, M., Escalona, A., Durelli, V.H.S.: Mutation testing in practice using ruby. In: 2015 IEEE Eighth International Conference on Software Testing, Verification and Validation Workshops\u2014Mutation Workshop, pp. 1\u20136. IEEE (2015). https:\/\/doi.org\/10.1109\/ICSTW.2015.7107453","DOI":"10.1109\/ICSTW.2015.7107453"},{"key":"567_CR59","doi-asserted-by":"publisher","unstructured":"Ma, Y.S., Kwon, Y.R., Offutt, J.: Inter-class mutation operators for java. In: 13th International Symposium on Software Reliability Engineering, 2002. Proceedings, pp. 352\u2013363. IEEE Comput. Soc (2002). https:\/\/doi.org\/10.1109\/issre.2002.1173287","DOI":"10.1109\/issre.2002.1173287"},{"key":"567_CR60","doi-asserted-by":"publisher","unstructured":"Ma, Y.S., Offutt, J., Kwon, Y.R.: MuJava. In: Proceeding of the 28th International Conference on Software Engineering\u2014ICSE \u201906, ICSE \u201906, pp. 827\u2013830. ACM Press, New York (2006). https:\/\/doi.org\/10.1145\/1134285.1134425","DOI":"10.1145\/1134285.1134425"},{"issue":"1","key":"567_CR61","doi-asserted-by":"publisher","first-page":"23","DOI":"10.1109\/TSE.2013.44","volume":"40","author":"L Madeyski","year":"2014","unstructured":"Madeyski, L., Orzeszyna, W., Torkar, R., Jozala, M.: Overcoming the equivalent mutant problem: a systematic literature review and a comparative experiment of second order mutation. IEEE Trans. Softw. Eng. 40(1), 23\u201342 (2014). https:\/\/doi.org\/10.1109\/TSE.2013.44","journal-title":"IEEE Trans. Softw. Eng."},{"key":"567_CR62","unstructured":"Marick, B.: Experience with the cost of different coverage goals for testing. In: Proceedings of the Ninth Pacific Northwest Software Quality Conference, pp. 147\u2013164 (1991)"},{"key":"567_CR63","unstructured":"Marick, B., Foundations, T.: How to misuse code coverage. In: 16th International Conference and Exposition on Testing Computer Software (1999)"},{"issue":"1","key":"567_CR64","doi-asserted-by":"publisher","first-page":"9","DOI":"10.1002\/stvr.4370040104","volume":"4","author":"AP Mathur","year":"1994","unstructured":"Mathur, A.P., Wong, W.E.: An empirical comparison of data flow and mutation-based test adequacy criteria. Softw. Test. Verif. Reliab. 4(1), 9\u201331 (1994). https:\/\/doi.org\/10.1002\/stvr.4370040104","journal-title":"Softw. Test. Verif. Reliab."},{"issue":"4","key":"567_CR65","doi-asserted-by":"publisher","first-page":"7","DOI":"10.5381\/jot.2007.6.4.c1","volume":"6","author":"JD McGregor","year":"2007","unstructured":"McGregor, J.D.: Test early, test often. J. Object Technol. 6(4), 7\u201314 (2007). https:\/\/doi.org\/10.5381\/jot.2007.6.4.c1","journal-title":"J. Object Technol."},{"key":"567_CR66","doi-asserted-by":"publisher","unstructured":"Memon, A., Gao, Z., Nguyen, B., Dhanda, S., Nickell, E., Siemborski, R., Micco, J.: Taming google-scale continuous testing. In: Proceedings of the 39th International Conference on Software Engineering: Software Engineering in Practice Track, ICSE-SEIP \u201917, pp. 233\u2013242. IEEE Press, Piscataway (2017). https:\/\/doi.org\/10.1109\/ICSE-SEIP.2017.16","DOI":"10.1109\/ICSE-SEIP.2017.16"},{"issue":"2","key":"567_CR67","doi-asserted-by":"publisher","first-page":"99","DOI":"10.1145\/227607.227610","volume":"5","author":"AJ Offutt","year":"1996","unstructured":"Offutt, A.J., Lee, A., Rothermel, G., Untch, R.H., Zapf, C.: An experimental determination of sufficient mutant operators. ACM Trans. Softw. Eng. Methodol. 5(2), 99\u2013118 (1996). https:\/\/doi.org\/10.1145\/227607.227610","journal-title":"ACM Trans. Softw. Eng. Methodol."},{"issue":"3","key":"567_CR68","doi-asserted-by":"publisher","first-page":"165","DOI":"10.1002\/(sici)1099-1689(199709)7:3<165::aid-stvr143>3.0.co;2-u","volume":"7","author":"AJ Offutt","year":"1997","unstructured":"Offutt, A.J., Pan, J.: Automatically detecting equivalent mutants and infeasible paths. Softw. Test. Verif. Reliab. 7(3), 165\u2013192 (1997). https:\/\/doi.org\/10.1002\/(sici)1099-1689(199709)7:3<165::aid-stvr143>3.0.co;2-u","journal-title":"Softw. Test. Verif. Reliab."},{"issue":"2","key":"567_CR69","doi-asserted-by":"publisher","first-page":"165","DOI":"10.1002\/(SICI)1097-024X(199602)26:2<165::AID-SPE5>3.0.CO;2-K","volume":"26","author":"AJ Offutt","year":"1996","unstructured":"Offutt, A.J., Pan, J., Tewary, K., Zhang, T.: An experimental evaluation of data flow and mutation testing. Softw. Pract. Exp. 26(2), 165\u2013176 (1996). https:\/\/doi.org\/10.1002\/(SICI)1097-024X(199602)26:2<165::AID-SPE5>3.0.CO;2-K","journal-title":"Softw. Pract. Exp."},{"key":"567_CR70","series-title":"The Springer International Series on Advances in Database Systems","doi-asserted-by":"publisher","first-page":"34","DOI":"10.1007\/978-1-4757-5939-6_7","volume-title":"Mutation Testing for the New Century","author":"AJ Offutt","year":"2001","unstructured":"Offutt, A.J., Untch, R.H.: Mutation 2000: uniting the orthogonal. In: Wong, W. (ed.) Mutation Testing for the New Century. The Springer International Series on Advances in Database Systems, vol. 24, pp. 34\u201344. Springer, New York (2001). https:\/\/doi.org\/10.1007\/978-1-4757-5939-6_7"},{"key":"567_CR71","unstructured":"Offutt, A.J., Voas, J.M.: Subsumption of condition coverage techniques by mutation testing. Tech. rep., George Mason University (1996)"},{"key":"567_CR72","doi-asserted-by":"publisher","unstructured":"Papadakis, M., Henard, C., Harman, M., Jia, Y., Traon, Y.L.: Threats to the validity of mutation-based test assessment. In: Proceedings of the 25th International Symposium on Software Testing and Analysis\u2014ISSTA 2016, ISSTA 2016, pp. 354\u2013365. ACM Press, New York (2016). https:\/\/doi.org\/10.1145\/2931037.2931040","DOI":"10.1145\/2931037.2931040"},{"key":"567_CR73","doi-asserted-by":"publisher","unstructured":"Papadakis, M., Jia, Y., Harman, M., Traon, Y.L.: Trivial compiler equivalence: a large scale empirical study of a simple, fast and effective equivalent mutant detection technique. In: 2015 IEEE\/ACM 37th IEEE International Conference on Software Engineering, vol.\u00a01, pp. 936\u2013946. IEEE (2015). https:\/\/doi.org\/10.1109\/icse.2015.103","DOI":"10.1109\/icse.2015.103"},{"key":"567_CR74","doi-asserted-by":"publisher","DOI":"10.1016\/bs.adcom","author":"M Papadakis","year":"2018","unstructured":"Papadakis, M., Kintis, M., Zhang, J., Jia, Y., Traon, Y.L., Harman, M.: Mutation testing advances: an analysis and survey. Adv. Comput. (2018). https:\/\/doi.org\/10.1016\/bs.adcom","journal-title":"Adv. Comput."},{"key":"567_CR75","doi-asserted-by":"publisher","unstructured":"Parsai, A., Demeyer, S.: Do null-type mutation operators help prevent null-type faults? In: Catania, B., Kr\u00e1lovi\u010d, R., Nawrocki, J., Pighizzini, G. (eds.) SOFSEM 2019: Theory and Practice of Computer Science, pp. 419\u2013434. Springer, Cham (2019). https:\/\/doi.org\/10.1007\/978-3-030-10801-4_33. http:\/\/link.springer.com\/chapter\/10.1007%2F978-3-030-10801-4_33","DOI":"10.1007\/978-3-030-10801-4_33"},{"key":"567_CR76","doi-asserted-by":"publisher","unstructured":"Parsai, A., Demeyer, S., Busser, S.D.: C++11\/14 mutation operators based on common fault patterns. In: Medina-Bulo, I., Merayo, M.G., Hierons, R. (eds.) Testing Software and Systems, pp. 102\u2013118. Springer, Cham (2018). https:\/\/doi.org\/10.1007\/978-3-319-99927-2_9. http:\/\/link.springer.com\/chapter\/10.1007%2F978-3-319-99927-2_9","DOI":"10.1007\/978-3-319-99927-2_9"},{"key":"567_CR77","doi-asserted-by":"publisher","unstructured":"Parsai, A., Murgia, A., Demeyer, S.: Evaluating random mutant selection at class-level in projects with non-adequate test suites. In: Proceedings of the 20th International Conference on Evaluation and Assessment in Software Engineering\u2014EASE \u201916, EASE \u201916, pp. 11:1\u201311:10. ACM Press, New York (2016). https:\/\/doi.org\/10.1145\/2915970.2915992","DOI":"10.1145\/2915970.2915992"},{"key":"567_CR78","doi-asserted-by":"publisher","unstructured":"Parsai, A., Murgia, A., Demeyer, S.: LittleDarwin: A feature-rich and extensible mutation testing framework for large and complex java systems. In: Dastani, M., Sirjani, M. (eds.) Fundamentals of Software Engineering: 7th International Conference, FSEN 2017, Tehran, 26\u201328 April 2017, Revised Selected Papers, pp. 148\u2013163. Springer (2017). https:\/\/doi.org\/10.1007\/978-3-319-68972-2_10","DOI":"10.1007\/978-3-319-68972-2_10"},{"key":"567_CR79","doi-asserted-by":"publisher","first-page":"166","DOI":"10.1007\/978-3-319-14358-3_14","volume-title":"Lecture Notes in Business Information Processing","author":"A Parsai","year":"2014","unstructured":"Parsai, A., Soetens, Q.D., Murgia, A., Demeyer, S.: Considering polymorphism in change-based test suite reduction. In: Dings\u00f8yr, T., Moe, N.B., Tonelli, R., Counsell, S., Gencel, C., Petersen, K. (eds.) Lecture Notes in Business Information Processing, pp. 166\u2013181. Springer, Cham (2014). https:\/\/doi.org\/10.1007\/978-3-319-14358-3_14"},{"issue":"4","key":"567_CR80","doi-asserted-by":"publisher","first-page":"26","DOI":"10.1109\/52.854065","volume":"17","author":"L Rising","year":"2000","unstructured":"Rising, L., Janoff, N.S.: The scrum software development process for small teams. IEEE Softw. 17(4), 26\u201332 (2000). https:\/\/doi.org\/10.1109\/52.854065","journal-title":"IEEE Softw."},{"issue":"2","key":"567_CR81","doi-asserted-by":"publisher","first-page":"131","DOI":"10.1007\/s10664-008-9102-8","volume":"14","author":"P Runeson","year":"2008","unstructured":"Runeson, P., H\u00f6st, M.: Guidelines for conducting and reporting case study research in software engineering. Empir. Softw. Eng. 14(2), 131\u2013164 (2008). https:\/\/doi.org\/10.1007\/s10664-008-9102-8","journal-title":"Empir. Softw. Eng."},{"key":"567_CR82","unstructured":"Sahinoglu, M., Spafford, E.H.: A bayes sequential statistical procedure for approving software products. In: Proceedings of the IFIP Conference on Approving Software Products (ASP\u201990), pp. 43\u201356 (1990)"},{"key":"567_CR83","doi-asserted-by":"publisher","first-page":"172","DOI":"10.1007\/978-3-319-14130-5_13","volume-title":"Lecture Notes in Computer Science","author":"I Saleh","year":"2014","unstructured":"Saleh, I., Nagi, K.: HadoopMutator: a cloud-based mutation testing framework. In: Schaefer, I., Stamelos, I. (eds.) Lecture Notes in Computer Science, pp. 172\u2013187. Springer, Cham (2014). https:\/\/doi.org\/10.1007\/978-3-319-14130-5_13"},{"key":"567_CR84","doi-asserted-by":"publisher","unstructured":"Shahriar, H., Zulkernine, M.: Mutation-based testing of format string bugs. In: 2008 11th IEEE High Assurance Systems Engineering Symposium, HASE \u201908, pp. 229\u2013238. IEEE, Washington, DC (2008). https:\/\/doi.org\/10.1109\/hase.2008.8","DOI":"10.1109\/hase.2008.8"},{"key":"567_CR85","doi-asserted-by":"publisher","unstructured":"Silva, R.A., do\u00a0Rocio Senger\u00a0de Souza, S., de\u00a0Souza, P.S.L.: Mutation operators for concurrent programs in MPI. In: 2012 13th Latin American Test Workshop (LATW), LATW \u201912, pp. 1\u20136. IEEE, Washington, DC (2012). https:\/\/doi.org\/10.1109\/latw.2012.6261240","DOI":"10.1109\/latw.2012.6261240"},{"issue":"3","key":"567_CR86","doi-asserted-by":"publisher","first-page":"341","DOI":"10.1007\/s10664-008-9083-7","volume":"14","author":"BH Smith","year":"2008","unstructured":"Smith, B.H., Williams, L.: On guiding the augmentation of an automated test suite via mutation analysis. Empir. Softw. Eng. 14(3), 341\u2013369 (2008). https:\/\/doi.org\/10.1007\/s10664-008-9083-7","journal-title":"Empir. Softw. Eng."},{"issue":"11","key":"567_CR87","doi-asserted-by":"publisher","first-page":"1819","DOI":"10.1016\/j.jss.2009.06.031","volume":"82","author":"BH Smith","year":"2009","unstructured":"Smith, B.H., Williams, L.: Should software testers use mutation analysis to augment a test set? J. Syst. Softw. 82(11), 1819\u20131832 (2009). https:\/\/doi.org\/10.1016\/j.jss.2009.06.031","journal-title":"J. Syst. Softw."},{"key":"567_CR88","doi-asserted-by":"publisher","unstructured":"Tengeri, D., Beszedes, A., Havas, D., Gyimothy, T.: Toolset and program repository for code coverage-based test suite analysis and manipulation. In: 2014 IEEE 14th International Working Conference on Source Code Analysis and Manipulation, pp. 47\u201352. IEEE (2014). https:\/\/doi.org\/10.1109\/SCAM.2014.38","DOI":"10.1109\/SCAM.2014.38"},{"key":"567_CR89","doi-asserted-by":"publisher","unstructured":"Tengeri, D., Horvath, F., Beszedes, A., Gergely, T., Gyimothy, T.: Negative effects of bytecode instrumentation on java source code coverage. In: 2016 IEEE 23rd International Conference on Software Analysis, Evolution, and Reengineering (SANER), vol.\u00a01, pp. 225\u2013235. IEEE (2016). https:\/\/doi.org\/10.1109\/saner.2016.61","DOI":"10.1109\/saner.2016.61"},{"key":"567_CR90","doi-asserted-by":"publisher","DOI":"10.1002\/0471722324","volume-title":"Software Quality Engineering: Testing, Quality Assurance and Quantifiable Improvement","author":"J Tian","year":"2009","unstructured":"Tian, J.: Software Quality Engineering: Testing, Quality Assurance and Quantifiable Improvement. Wiley, New York (2009). https:\/\/doi.org\/10.1002\/0471722324"},{"key":"567_CR91","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"194","DOI":"10.1007\/978-3-642-25231-0_5","volume-title":"Empirical Software Engineering and Verification","author":"Y Wei","year":"2012","unstructured":"Wei, Y., Meyer, B., Oriol, M.: Is branch coverage a good measure of testing effectiveness? In: Meyer, B., Nordio, M. (eds.) Empirical Software Engineering and Verification. Lecture Notes in Computer Science, vol. 7007, pp. 194\u2013212. Springer, Berlin (2012). https:\/\/doi.org\/10.1007\/978-3-642-25231-0_5"},{"issue":"3","key":"567_CR92","doi-asserted-by":"publisher","first-page":"185","DOI":"10.1016\/0164-1212(94)00098-0","volume":"31","author":"W Wong","year":"1995","unstructured":"Wong, W., Mathur, A.P.: Reducing the cost of mutation testing: an empirical study. J. Syst. Softw. 31(3), 185\u2013196 (1995). https:\/\/doi.org\/10.1016\/0164-1212(94)00098-0","journal-title":"J. Syst. Softw."},{"key":"567_CR93","unstructured":"Wong, W.E.: On mutation and data flow. Ph.D. thesis, Purdue University, West Lafayette (1993). UMI Order No. GAX94-20921"},{"issue":"5","key":"567_CR94","doi-asserted-by":"publisher","first-page":"589","DOI":"10.1093\/comjnl\/bxm021","volume":"52","author":"Q Yang","year":"2007","unstructured":"Yang, Q., Li, J.J., Weiss, D.M.: A survey of coverage-based testing tools. Comput. J. 52(5), 589\u2013597 (2007). https:\/\/doi.org\/10.1093\/comjnl\/bxm021","journal-title":"Comput. J."},{"key":"567_CR95","doi-asserted-by":"publisher","unstructured":"Zeng, F., Mao, L., Chen, Z., Cao, Q.: Mutation-based testing of integer overflow vulnerabilities. In: 2009 5th International Conference on Wireless Communications, Networking and Mobile Computing, WiCOM\u201909, pp. 4416\u20134419. IEEE, Piscataway (2009). https:\/\/doi.org\/10.1109\/wicom.2009.5302048","DOI":"10.1109\/wicom.2009.5302048"},{"key":"567_CR96","doi-asserted-by":"publisher","unstructured":"Zhang, L., Gligoric, M., Marinov, D., Khurshid, S.: Operator-based and random mutant selection: better together. In: 2013 28th IEEE\/ACM International Conference on Automated Software Engineering (ASE), pp. 92\u2013102. IEEE (2013). https:\/\/doi.org\/10.1109\/ASE.2013.6693070","DOI":"10.1109\/ASE.2013.6693070"},{"key":"567_CR97","doi-asserted-by":"publisher","unstructured":"Zhang, L., Hou, S.S., Hu, J.J., Xie, T., Mei, H.: Is operator-based mutant selection superior to random mutant selection? In: Proceedings of the 32nd ACM\/IEEE International Conference on Software Engineering\u2014ICSE \u201910, ICSE \u201910, pp. 435\u2013444. ACM Press, New York (2010). https:\/\/doi.org\/10.1145\/1806799.1806863","DOI":"10.1145\/1806799.1806863"},{"issue":"4","key":"567_CR98","doi-asserted-by":"publisher","first-page":"366","DOI":"10.1145\/267580.267590","volume":"29","author":"H Zhu","year":"1997","unstructured":"Zhu, H., Hall, P.A.V., May, J.H.R.: Software unit test coverage and adequacy. ACM Comput. Surv. 29(4), 366\u2013427 (1997). https:\/\/doi.org\/10.1145\/267580.267590","journal-title":"ACM Comput. Surv."}],"container-title":["International Journal on Software Tools for Technology Transfer"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10009-020-00567-y.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s10009-020-00567-y\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10009-020-00567-y.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2021,5,15]],"date-time":"2021-05-15T23:30:09Z","timestamp":1621121409000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s10009-020-00567-y"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2020,5,16]]},"references-count":98,"journal-issue":{"issue":"4","published-print":{"date-parts":[[2020,8]]}},"alternative-id":["567"],"URL":"https:\/\/doi.org\/10.1007\/s10009-020-00567-y","relation":{},"ISSN":["1433-2779","1433-2787"],"issn-type":[{"value":"1433-2779","type":"print"},{"value":"1433-2787","type":"electronic"}],"subject":[],"published":{"date-parts":[[2020,5,16]]},"assertion":[{"value":"16 May 2020","order":1,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}}]}}