{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,25]],"date-time":"2026-10-25T16:31:38Z","timestamp":1792945898102,"version":"4.5.0"},"reference-count":70,"publisher":"Springer Science and Business Media LLC","issue":"2","license":[{"start":{"date-parts":[[2013,7,11]],"date-time":"2013-07-11T00:00:00Z","timestamp":1373500800000},"content-version":"tdm","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["Empir Software Eng"],"published-print":{"date-parts":[[2014,4]]},"DOI":"10.1007\/s10664-013-9266-8","type":"journal-article","created":{"date-parts":[[2013,7,9]],"date-time":"2013-07-09T23:57:52Z","timestamp":1373414272000},"page":"303-342","source":"Crossref","is-referenced-by-count":50,"title":["Are test cases needed? Replicated comparison between exploratory and test-case-based software testing"],"prefix":"10.1007","volume":"19","author":[{"given":"Juha","family":"Itkonen","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Mika V.","family":"M\u00e4ntyl\u00e4","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2013,7,11]]},"reference":[{"key":"9266_CR1","volume-title":"Guide to the software engineering body of knowledge 2004 version","author":"A Abran","year":"2004","unstructured":"Abran A, Moore JW, Bourque P et al (2004) Guide to the software engineering body of knowledge 2004 version. IEEE Computer Society, Los Alamitos"},{"key":"9266_CR2","doi-asserted-by":"crossref","unstructured":"Andersson C, Runeson P (2002) Verification and validation in industry\u2014a qualitative survey on the state of practice. Proceedings of International Symposium on Empirical Software Engineering. pp 37\u201347","DOI":"10.1109\/ISESE.2002.1166923"},{"key":"9266_CR3","unstructured":"Bach J (1999) General functionality and stability test procedure for certified for microsoft windows logo. http:\/\/www.satisfice.com\/tools\/procedure.pdf . Accessed 8 May 2013"},{"key":"9266_CR4","unstructured":"Bach J (2000) Session-based test management. In: software testing and quality engineering. http:\/\/www.satisfice.com\/articles\/sbtm.pdf . Accessed 8 May 2013"},{"key":"9266_CR5","unstructured":"Bach J (2003) Exploratory testing explained. http:\/\/www.satisfice.com\/articles\/et-article.pdf . Accessed 8 May 2013"},{"key":"9266_CR6","first-page":"253","volume-title":"The testing practitioner","author":"J Bach","year":"2004","unstructured":"Bach J (2004) Exploratory testing. In: van Veenendaal E (ed) The testing practitioner. Second. UTN Publishers, Den Bosch, pp 253\u2013265"},{"key":"9266_CR7","unstructured":"Baeza-Yates R, Ribeiro-Neto B (1999) Modern information retrieval. Addison-Wesley Longman Publishing Co., Inc., New York, USA"},{"key":"9266_CR8","doi-asserted-by":"crossref","first-page":"1278","DOI":"10.1109\/TSE.1987.232881","volume":"13","author":"VR Basili","year":"1987","unstructured":"Basili VR, Selby RW (1987) Comparing the effectiveness of software testing strategies. IEEE Trans Softw Eng 13:1278\u20131296","journal-title":"IEEE Trans Softw Eng"},{"key":"9266_CR9","volume-title":"Software testing techniques","author":"B Beizer","year":"1990","unstructured":"Beizer B (1990) Software testing techniques. Van Nostrand Reinhold, New York"},{"key":"9266_CR10","unstructured":"Berner S, Weber R, Keller RK (2005) Observations and lessons learned from automated testing. Proceedings of International Conference on Software Engineering. pp 571\u2013579"},{"key":"9266_CR11","unstructured":"Bolton M (2005) Testing without a map. Better Software 7(1)"},{"key":"9266_CR12","unstructured":"Carver JC (2010) Towards reporting guidelines for experimental replications: a proposal. 1st International Workshop on Replication in Empirical Software Engineering"},{"key":"9266_CR13","doi-asserted-by":"crossref","first-page":"437","DOI":"10.1109\/32.605761","volume":"23","author":"DM Cohen","year":"1997","unstructured":"Cohen DM, Dalal SR, Fredman ML, Patton GC (1997) The AETG system: an approach to testing based on combinatorial design. IEEE Trans Softw Eng 23:437\u2013444. doi: 10.1109\/32.605761","journal-title":"IEEE Trans Softw Eng"},{"key":"9266_CR14","volume-title":"A practitioner\u2019s guide to software test design","author":"L Copeland","year":"2004","unstructured":"Copeland L (2004) A practitioner\u2019s guide to software test design. Artech House Publishers, Boston"},{"key":"9266_CR15","volume-title":"Systematic software testing","author":"RD Craig","year":"2002","unstructured":"Craig RD, Jaskiel SP (2002) Systematic software testing. Artech House Publishers, Boston"},{"key":"9266_CR16","volume-title":"Agile testing: A practical guide for testers and agile teams","author":"L Crispin","year":"2009","unstructured":"Crispin L, Gregory J (2009) Agile testing: A practical guide for testers and agile teams. Addison-Wesley, Boston"},{"key":"9266_CR17","doi-asserted-by":"crossref","unstructured":"Do Nascimento LHO, Machado PDL (2007) An experimental evaluation of approaches to feature testing in the mobile phone applications domain. Proceedings of the Workshop on Domain Specific Approaches to Software Test Automation. pp 27\u201333","DOI":"10.1145\/1294921.1294926"},{"key":"9266_CR18","doi-asserted-by":"crossref","unstructured":"Ellis PD (2010) The essential guide to effect sizes: statistical power, meta-analysis, and the interpretation of research results, 1st edn. Cambridge University Press, New York, USA","DOI":"10.1017\/CBO9780511761676"},{"key":"9266_CR19","doi-asserted-by":"crossref","unstructured":"Engstr\u00f6m E, Runeson P (2010) A qualitative survey of regression testing practices. Proceedings of International Conference on Product-Focused Software Process Improvement. pp 3\u201316","DOI":"10.1007\/978-3-642-13792-1_3"},{"key":"9266_CR20","doi-asserted-by":"crossref","first-page":"581","DOI":"10.1016\/j.infsof.2012.04.009","volume":"55","author":"E Engstr\u00f6m","year":"2013","unstructured":"Engstr\u00f6m E, Runeson P (2013) Test overlay in an emerging software product line\u2014An industrial case study. Inf Softw Technol 55:581\u2013594. doi: 10.1016\/j.infsof.2012.04.009","journal-title":"Inf Softw Technol"},{"key":"9266_CR21","unstructured":"Field A (2005) Discovering statistics using SPSS, 2nd edn. Sage Publications Ltd, London, UK"},{"key":"9266_CR22","first-page":"112","volume":"19","author":"RL Glass","year":"2002","unstructured":"Glass RL (2002) Project retrospectives, and why they never happen. IEEE Softw 19:112. doi: 10.1109\/MS.2002.1032872","journal-title":"IEEE Softw"},{"key":"9266_CR23","doi-asserted-by":"crossref","unstructured":"Grechanik M, Jones JA, Orso A, van der Hoek A (2010) Bridging gaps between developers and testers in globally-distributed software development. Proceedings of the FSE\/SDP workshop on Future of software engineering research. ACM, New York, NY, USA, pp 149\u2013154","DOI":"10.1145\/1882362.1882394"},{"key":"9266_CR24","doi-asserted-by":"crossref","first-page":"637","DOI":"10.1142\/S0218194002001128","volume":"12","author":"F Houdek","year":"2002","unstructured":"Houdek F, Schwinn T, Ernst D (2002) Defect detection for executable specifications\u2014an experiment. Int J Softw Eng Knowl Eng 12:637\u2013655","journal-title":"Int J Softw Eng Knowl Eng"},{"key":"9266_CR25","doi-asserted-by":"crossref","first-page":"88","DOI":"10.1109\/MS.2006.127","volume":"23","author":"L Huang","year":"2006","unstructured":"Huang L, Boehm B (2006) How much software quality investment is enough: a value-based approach. IEEE Softw 23:88\u201395. doi: 10.1109\/MS.2006.127","journal-title":"IEEE Softw"},{"key":"9266_CR26","unstructured":"Itkonen J (2008) Do test cases really matter? An experiment comparing test case based and exploratory testing. Licentiate Thesis, Helsinki University of Technology"},{"key":"9266_CR27","unstructured":"Itkonen J (2013) ET vs. TCT Experiment replication dataset. In: Figshare.com. http:\/\/dx.doi.org\/10.6084\/m9.figshare.689809 . Accessed 29 Apr 2013"},{"key":"9266_CR28","doi-asserted-by":"crossref","unstructured":"Itkonen J, Rautiainen K (2005) Exploratory testing: a multiple case study. Proceedings of International Symposium on Empirical Software Engineering. pp 84\u201393","DOI":"10.1109\/ISESE.2005.1541817"},{"key":"9266_CR29","doi-asserted-by":"crossref","unstructured":"Itkonen J, M\u00e4ntyl\u00e4 MV, Lassenius C (2007) Defect detection efficiency: test case based vs. exploratory testing. Proceedings of International Symposium on Empirical Software Engineering and Measurement. pp 61\u201370","DOI":"10.1109\/ESEM.2007.56"},{"key":"9266_CR30","doi-asserted-by":"crossref","unstructured":"Itkonen J, M\u00e4ntyl\u00e4 MV, Lassenius C (2009) How do testers do it? An exploratory study on manual testing practices. Empirical Software Engineering and Measurement, 2009. ESEM 2009. 3rd International Symposium on. pp 494\u2013497","DOI":"10.1109\/ESEM.2009.5314240"},{"key":"9266_CR31","doi-asserted-by":"crossref","first-page":"707","DOI":"10.1109\/TSE.2012.55","volume":"39","author":"J Itkonen","year":"2013","unstructured":"Itkonen J, M\u00e4ntyl\u00e4 MV, Lassenius C (2013) The role of the tester\u2019s knowledge in exploratory software testing. IEEE Trans Softw Eng 39:707\u2013724. doi: 10.1109\/TSE.2012.55","journal-title":"IEEE Trans Softw Eng"},{"key":"9266_CR32","doi-asserted-by":"crossref","DOI":"10.1007\/978-1-4757-3304-4","volume-title":"Basics of software engineering experimentation","author":"N Juristo","year":"2001","unstructured":"Juristo N, Moreno AM (2001) Basics of software engineering experimentation. Kluwer Academic Publishers, Boston"},{"key":"9266_CR33","doi-asserted-by":"crossref","first-page":"7","DOI":"10.1023\/B:EMSE.0000013513.48963.1b","volume":"9","author":"N Juristo","year":"2004","unstructured":"Juristo N, Moreno AM, Vegas S (2004) Reviewing 25 years of testing technique experiments. Empir Softw Eng 9:7\u201344","journal-title":"Empir Softw Eng"},{"key":"9266_CR34","doi-asserted-by":"crossref","unstructured":"Juristo N, Vegas S, Solari M et al (2012) Comparing the effectiveness of equivalence partitioning, branch testing and code reading by stepwise abstraction applied by subjects. Proceedings of Fifth International Conference on Software Testing, Verification and Validation. pp 330\u2013339","DOI":"10.1109\/ICST.2012.113"},{"key":"9266_CR35","doi-asserted-by":"crossref","unstructured":"Kamsties E, Lott CM (1995) An empirical evaluation of three defect-detection techniques. In: Sch\u00e4fer W, Botella P (eds) Proceedings of ESEC\u201995. Springer Berlin Heidelberg, pp 362\u2013383","DOI":"10.1007\/3-540-60406-5_25"},{"key":"9266_CR36","volume-title":"Testing computer software","author":"C Kaner","year":"1999","unstructured":"Kaner C, Falk J, Nguyen HQ (1999) Testing computer software. Wiley, New York"},{"key":"9266_CR37","volume-title":"Lessons learned in software testing","author":"C Kaner","year":"2002","unstructured":"Kaner C, Bach J, Pettichord B (2002) Lessons learned in software testing. Wiley, New York"},{"key":"9266_CR38","doi-asserted-by":"crossref","first-page":"219","DOI":"10.1007\/s10664-008-9061-0","volume":"13","author":"B Kitchenham","year":"2008","unstructured":"Kitchenham B (2008) The role of replications in empirical software engineering\u2014a word of warning. Empir Softw Eng 13:219\u2013221. doi: 10.1007\/s10664-008-9061-0","journal-title":"Empir Softw Eng"},{"key":"9266_CR39","unstructured":"Lyndsay J, van Eeden N (2003) Adventures in session-based testing. http:\/\/www.workroom-productions.com\/papers\/AiSBTv1.2.pdf . Accessed 20 Jun 2012"},{"key":"9266_CR40","doi-asserted-by":"crossref","first-page":"986","DOI":"10.1016\/j.infsof.2012.12.004","volume":"55","author":"MV M\u00e4ntyl\u00e4","year":"2013","unstructured":"M\u00e4ntyl\u00e4 MV, Itkonen J (2013) More testers\u2014The effect of crowd size and time restriction in software testing. Inf Softw Technol 55:986\u20131003. doi: 10.1016\/j.infsof.2012.12.004","journal-title":"Inf Softw Technol"},{"key":"9266_CR41","doi-asserted-by":"crossref","unstructured":"M\u00e4ntyl\u00e4 MV, Vanhanen J (2011) Software deployment activities and challenges\u2014a case study of four software product companies. Proceedings of the 15th European Conference on Software Maintenance and Reengineering. pp 131\u2013140","DOI":"10.1109\/CSMR.2011.19"},{"key":"9266_CR42","doi-asserted-by":"crossref","unstructured":"Martin D, Rooksby J, Rouncefield M, Sommerville I (2007) \u201cGood\u201d organisational reasons for \u201cbad\u201d software testing: an ethnographic study of testing in a small software company. Proceedings of International Conference on Software Engineering. pp 602\u2013611","DOI":"10.1109\/ICSE.2007.1"},{"key":"9266_CR43","unstructured":"McConnell S (2004) Code complete. Microsoft Press, Redmond, WA, USA"},{"key":"9266_CR44","doi-asserted-by":"crossref","first-page":"267","DOI":"10.2307\/2491150","volume":"28","author":"LS McDaniel","year":"1990","unstructured":"McDaniel LS (1990) The effects of time pressure and audit program structure on audit performance. J Account Res 28:267\u2013285. doi: 10.2307\/2491150","journal-title":"J Account Res"},{"key":"9266_CR45","doi-asserted-by":"crossref","first-page":"161","DOI":"10.1109\/TSE.2010.32","volume":"37","author":"S Mouchawrab","year":"2011","unstructured":"Mouchawrab S, Briand LC, Labiche Y, Di Penta M (2011) Assessing, comparing, and combining state machine-based testing and structural testing: a series of experiments. IEEE Trans Softw Eng 37:161\u2013187. doi: 10.1109\/TSE.2010.32","journal-title":"IEEE Trans Softw Eng"},{"key":"9266_CR46","doi-asserted-by":"crossref","first-page":"760","DOI":"10.1145\/359588.359602","volume":"21","author":"GJ Myers","year":"1978","unstructured":"Myers GJ (1978) A controlled experiment in program testing and code walkthroughs\/inspections. Commun ACM 21:760\u2013768. doi: 10.1145\/359588.359602","journal-title":"Commun ACM"},{"key":"9266_CR47","volume-title":"The art of software testing","author":"GJ Myers","year":"1979","unstructured":"Myers GJ (1979) The art of software testing. Wiley, New York"},{"key":"9266_CR48","unstructured":"Ng SP, Murnane T, Reed K, et al (2004) A preliminary survey on software testing practices in Australia. Proceedings of the Australian Software Engineering Conference. pp 116\u2013125"},{"key":"9266_CR49","unstructured":"Page A, Johnston K, Rollison B (2008) How we test software at microsoft. Microsoft Press, Redmond, WA, USA"},{"key":"9266_CR50","unstructured":"Pichler J, Ramler R (2008) How to test the intangible properties of graphical user interfaces? Proceedings of 1st International Conference on Software Testing, Verification, and Validation. pp 494\u2013497"},{"key":"9266_CR51","doi-asserted-by":"crossref","unstructured":"Rafi DM, Moses KRK, Petersen K, Mantyla MV (2012) Benefits and limitations of automated software testing: Systematic literature review and practitioner survey. 2012 7th International Workshop on Automation of Software Test (AST). pp 36\u201342","DOI":"10.1109\/IWAST.2012.6228988"},{"key":"9266_CR52","doi-asserted-by":"crossref","unstructured":"Ramasubbu N, Balan RK (2009) The impact of process choice in high maturity environments: An empirical analysis. Proceedings of 31st International Conference on Software Engineering. pp 529\u2013539","DOI":"10.1109\/ICSE.2009.5070551"},{"key":"9266_CR53","doi-asserted-by":"crossref","first-page":"82","DOI":"10.1109\/MS.2006.89","volume":"23","author":"P Runeson","year":"2006","unstructured":"Runeson P, Andersson C, Thelin T et al (2006) What do we know about defect detection methods? IEEE Softw 23:82\u201390. doi: 10.1109\/MS.2006.89","journal-title":"IEEE Softw"},{"key":"9266_CR54","doi-asserted-by":"crossref","unstructured":"Shah SMA, Morisio M, Torchiano M (2012) The impact of process maturity on defect density. Proceedings of International symposium on empirical software engineering and measurement. ACM, New York, NY, USA, pp 315\u2013318","DOI":"10.1145\/2372251.2372308"},{"key":"9266_CR55","doi-asserted-by":"crossref","unstructured":"Shoaib L, Nadeem A, Akbar A (2009) An empirical evaluation of the influence of human personality on exploratory software testing. Proceedings of IEEE International Multitopic Conference. pp 1\u20136","DOI":"10.1109\/INMIC.2009.5383088"},{"key":"9266_CR56","doi-asserted-by":"crossref","first-page":"211","DOI":"10.1007\/s10664-008-9060-1","volume":"13","author":"FJ Shull","year":"2008","unstructured":"Shull FJ, Carver JC, Vegas S, Juristo N (2008) The role of replications in Empirical Software Engineering. Empir Softw Eng 13:211\u2013218. doi: 10.1007\/s10664-008-9060-1","journal-title":"Empir Softw Eng"},{"key":"9266_CR57","unstructured":"Spolsky J (2001) Big Macs vs. the naked chef. In: Joel on Software. http:\/\/www.joelonsoftware.com\/articles\/fog0000000024.html . Accessed 28 Jun 2012"},{"key":"9266_CR58","doi-asserted-by":"crossref","first-page":"309","DOI":"10.1023\/A:1009844119158","volume":"5","author":"WF Tichy","year":"2000","unstructured":"Tichy WF (2000) Hints for reviewing empirical work in software engineering. Empir Softw Eng 5:309\u2013312. doi: 10.1023\/A:1009844119158","journal-title":"Empir Softw Eng"},{"key":"9266_CR59","unstructured":"Tinkham A, Kaner C (2003a) Exploring exploratory testing. Proceedings of the Software Testing Analysis & Review Conference. p 9"},{"key":"9266_CR60","unstructured":"Tinkham A, Kaner C (2003b) Learning styles and exploratory testing. Proceedings of the Pacific Northwest Software Quality Conference"},{"key":"9266_CR61","doi-asserted-by":"crossref","first-page":"759","DOI":"10.5465\/amr.1999.2553252","volume":"24","author":"EWK Tsang","year":"1999","unstructured":"Tsang EWK, Kwan K-M (1999) Replication and theory development in organizational science: a critical realist perspective. Acad Manag Rev 24:759\u2013780","journal-title":"Acad Manag Rev"},{"key":"9266_CR62","doi-asserted-by":"crossref","unstructured":"Tuomikoski J, Tervonen I (2009) Absorbing software testing into the scrum method. Proceedings of 10th International Conference on Product-Focused Software Process Improvement 32 LNBIP","DOI":"10.1007\/978-3-642-02152-7_16"},{"key":"9266_CR63","doi-asserted-by":"crossref","first-page":"23","DOI":"10.1007\/978-3-642-56333-1_2","volume-title":"Software quality and software testing in internet times","author":"J V\u00e5ga","year":"2002","unstructured":"V\u00e5ga J, Amland S (2002) Managing high-speed web testing. In: Meyerhoff D, Laibarra B, van der Pouw Kraan R, Wallet A (eds) Software quality and software testing in internet times. Springer, Berlin, pp 23\u201330"},{"key":"9266_CR64","doi-asserted-by":"crossref","unstructured":"Vegas S, Juristo N, Moreno A et al (2006) Analysis of the influence of communication between researchers on experiment replication. Proceedings of the 2006 ACM\/IEEE international symposium on Empirical Software Engineering. ACM, New York, NY, USA, pp 28\u201337","DOI":"10.1145\/1159733.1159741"},{"key":"9266_CR65","volume-title":"How to break software a practical guide to testing","author":"JA Whittaker","year":"2003","unstructured":"Whittaker JA (2003) How to break software a practical guide to testing. Addison Wesley, Boston"},{"key":"9266_CR66","unstructured":"Whittaker JA (2009) Exploratory software testing: tips, tricks, tours, and techniques to guide test design. Addison-Wesley Professional, Boston, MA, USA"},{"key":"9266_CR67","doi-asserted-by":"crossref","DOI":"10.1007\/978-1-4615-4625-2","volume-title":"Experimentation in software engineering: An introduction","author":"C Wohlin","year":"2000","unstructured":"Wohlin C, Runeson P, H\u00f6st M et al (2000) Experimentation in software engineering: An introduction. Kluwer Academic Publishers, Boston"},{"key":"9266_CR68","first-page":"90","volume":"25","author":"B Wood","year":"2003","unstructured":"Wood B, James D (2003) Applying session-based testing to medical software. Med Device Diagn Ind 25:90","journal-title":"Med Device Diagn Ind"},{"key":"9266_CR69","doi-asserted-by":"crossref","first-page":"262","DOI":"10.1145\/267896.267915","volume":"22","author":"M Wood","year":"1997","unstructured":"Wood M, Roper M, Brooks A, Miller J (1997) Comparing and combining software defect detection techniques: a replicated empirical study. ACM SIGSOFT Softw Eng Notes 22:262\u2013277. doi: 10.1145\/267895.267915","journal-title":"ACM SIGSOFT Softw Eng Notes"},{"key":"9266_CR70","unstructured":"Yatani K (2010) Statistics for HCI Research: Mann-Whitney\u2019s U test. In: Statistics for HCI research. http:\/\/yatani.jp\/HCIstats\/MannWhitney . Accessed 28 Jun 2012"}],"container-title":["Empirical Software Engineering"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s10664-013-9266-8.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/article\/10.1007\/s10664-013-9266-8\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s10664-013-9266-8","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2019,7,17]],"date-time":"2019-07-17T17:39:43Z","timestamp":1563385183000},"score":1,"resource":{"primary":{"URL":"http:\/\/link.springer.com\/10.1007\/s10664-013-9266-8"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2013,7,11]]},"references-count":70,"journal-issue":{"issue":"2","published-print":{"date-parts":[[2014,4]]}},"alternative-id":["9266"],"URL":"https:\/\/doi.org\/10.1007\/s10664-013-9266-8","relation":{},"ISSN":["1382-3256","1573-7616"],"issn-type":[{"value":"1382-3256","type":"print"},{"value":"1573-7616","type":"electronic"}],"subject":[],"published":{"date-parts":[[2013,7,11]]}}}