{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,10,16]],"date-time":"2025-10-16T03:50:44Z","timestamp":1760586644027},"reference-count":47,"publisher":"Springer Science and Business Media LLC","issue":"6","license":[{"start":{"date-parts":[[2013,8,18]],"date-time":"2013-08-18T00:00:00Z","timestamp":1376784000000},"content-version":"tdm","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["Empir Software Eng"],"published-print":{"date-parts":[[2014,12]]},"DOI":"10.1007\/s10664-013-9262-z","type":"journal-article","created":{"date-parts":[[2013,8,17]],"date-time":"2013-08-17T05:11:47Z","timestamp":1376716307000},"page":"1781-1808","source":"Crossref","is-referenced-by-count":14,"title":["Variation factors in the design and analysis of replicated controlled experiments"],"prefix":"10.1007","volume":"19","author":[{"given":"Per","family":"Runeson","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Andreas","family":"Stefik","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Anneliese","family":"Andrews","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2013,8,18]]},"reference":[{"issue":"2","key":"9262_CR1","doi-asserted-by":"crossref","first-page":"193","DOI":"10.1214\/aoms\/1177729437","volume":"23","author":"T Anderson","year":"1952","unstructured":"Anderson T, Darling D (1952) Asymptotic theory of certain \u201cgoodness of fit\u201d criteria based on stochastic processes. Ann Math Stat 23(2):193\u2013212","journal-title":"Ann Math Stat"},{"issue":"12","key":"9262_CR2","doi-asserted-by":"crossref","first-page":"1278","DOI":"10.1109\/TSE.1987.232881","volume":"13","author":"VR Basili","year":"1987","unstructured":"Basili VR, Selby RW (1987) Comparing the effectiveness of software testing strategies. IEEE Trans Softw Eng 13(12):1278\u20131296","journal-title":"IEEE Trans Softw Eng"},{"issue":"4","key":"9262_CR3","doi-asserted-by":"crossref","first-page":"456","DOI":"10.1109\/32.799939","volume":"25","author":"VR Basili","year":"1999","unstructured":"Basili VR, Shull F, Lanubile F (1999) Building knowledge through families of experiments. IEEE Trans Softw Eng 25(4):456\u2013473","journal-title":"IEEE Trans Softw Eng"},{"issue":"3","key":"9262_CR4","doi-asserted-by":"crossref","first-page":"177","DOI":"10.1049\/ip-sen:20030483","volume":"150","author":"T Berling","year":"2003","unstructured":"Berling T, Runeson P (2003) Evaluation of a perspective based review method applied in an industrial setting. IEE Proc SW 150(3):177\u2013184","journal-title":"IEE Proc SW"},{"issue":"1","key":"9262_CR5","doi-asserted-by":"crossref","first-page":"143","DOI":"10.1215\/00182702-23-1-143","volume":"23","author":"N Cartwright","year":"1991","unstructured":"Cartwright N (1991) Replicability, reproducibility, and robustness: comments on Harry Collins. Hist Polit Econ 23(1):143\u2013155","journal-title":"Hist Polit Econ"},{"issue":"5","key":"9262_CR6","doi-asserted-by":"crossref","first-page":"433","DOI":"10.1016\/j.infsof.2011.12.003","volume":"54","author":"P Clarke","year":"2012","unstructured":"Clarke P, O\u2019Connor RV (2012) The situational factors that affect the software development process: towards a comprehensive reference framework. Inf Softw Technol 54(5):433\u2013447","journal-title":"Inf Softw Technol"},{"key":"9262_CR8","author":"FQB Silva da","year":"2012","unstructured":"da Silva FQB, Suassuna M, Fran\u00e7a ACC, Grubb AM, Gouveia TB, Monteiro CVF, dos Santos IE (2012) Replication of empirical studies in software engineering research: a systematic mapping study. Empir Softw Eng. doi: 10.1007\/s10664-012-9227-7","journal-title":"Empir Softw Eng."},{"key":"9262_CR10","doi-asserted-by":"crossref","unstructured":"Dyb\u00e5 T, Sj\u00f8berg DIK, Cruzes DS (2012) What works for whom, where, when, and why?: on the role of context in empirical software engineering. In: Proceedings of the 11th international symposium on empirical software engineering and measurement, pp\u00a019\u201328","DOI":"10.1145\/2372251.2372256"},{"key":"9262_CR12","doi-asserted-by":"crossref","unstructured":"Gomez OS, Juristo N, Vegas S (2010) Replications types in experimental disciplines. In: Proceedings of the fourth international symposium on empirical software engineering and measurement","DOI":"10.1145\/1852786.1852790"},{"issue":"2","key":"9262_CR13","doi-asserted-by":"crossref","first-page":"242","DOI":"10.1109\/TSE.2008.13","volume":"34","author":"J Hannay","year":"2008","unstructured":"Hannay J, J\u00f8rgensen M (2008) The role of deliberate artificial design elements in software engineering experiments. IEEE Trans Softw Eng 34(2):242\u2013259","journal-title":"IEEE Trans Softw Eng"},{"issue":"3","key":"9262_CR14","doi-asserted-by":"crossref","first-page":"10","DOI":"10.1145\/1036508.1036511","volume":"3","author":"W Hetzel","year":"1972","unstructured":"Hetzel W (1972) An experimental analysis of program verification problem solving capabilities as they relate to programmer efficiency. Comput Pers 3(3):10\u201315","journal-title":"Comput Pers"},{"issue":"3","key":"9262_CR15","doi-asserted-by":"crossref","first-page":"112","DOI":"10.1080\/00031305.1975.10477393","volume":"29","author":"D Hoaglin","year":"1975","unstructured":"Hoaglin D, Andrews D (1975) The reporting of computation-based results in statistics. Am Stat 29(3):112\u2013126","journal-title":"Am Stat"},{"key":"9262_CR16","volume-title":"A discipline for software engineering","author":"WS Humphrey","year":"1995","unstructured":"Humphrey WS (1995) A discipline for software engineering. Addison-Wesley, Reading, MA"},{"key":"9262_CR17","doi-asserted-by":"crossref","unstructured":"Jedlitschka A, Pfahl D (2005) Reporting guidelines for controlled experiments in software engineering. In: Proceedings of the 4th international symposium on empirical software engineering, pp\u00a095\u2013104","DOI":"10.1109\/ISESE.2005.1541818"},{"issue":"5","key":"9262_CR18","doi-asserted-by":"crossref","first-page":"695","DOI":"10.1109\/TSE.2010.78","volume":"37","author":"M J\u00f8rgensen","year":"2011","unstructured":"J\u00f8rgensen M, Grimstad S (2011) The impact of irrelevant and misleading information on software development effort estimates: a randomized controlled field experiment. IEEE Trans Softw Eng 37(5):695\u2013707","journal-title":"IEEE Trans Softw Eng"},{"issue":"3","key":"9262_CR19","doi-asserted-by":"crossref","first-page":"677","DOI":"10.1109\/TSE.2011.40","volume":"38","author":"M J\u00f8rgensen","year":"2012","unstructured":"J\u00f8rgensen M, Grimstad S (2012) Software development estimation biases: the role of interdependence. IEEE Trans Softw Eng 38(3):677\u2013693","journal-title":"IEEE Trans Softw Eng"},{"issue":"3","key":"9262_CR20","doi-asserted-by":"crossref","first-page":"368","DOI":"10.1109\/TSE.2009.2","volume":"35","author":"M J\u00f8rgensen","year":"2009","unstructured":"J\u00f8rgensen M, Gruschke T (2009) The impact of lessons-learned sessions on effort estimation and uncertainty assessments. IEEE Trans Softw Eng 35(3):368\u2013383","journal-title":"IEEE Trans Softw Eng"},{"key":"9262_CR21","doi-asserted-by":"crossref","first-page":"33","DOI":"10.1109\/TSE.2007.256943","volume":"33","author":"M J\u00f8rgensen","year":"2007","unstructured":"J\u00f8rgensen M, Shepperd M (2007) A systematic review of software development cost estimation studies. IEEE Trans Softw Eng 33:33\u201353","journal-title":"IEEE Trans Softw Eng"},{"key":"9262_CR22","doi-asserted-by":"crossref","unstructured":"Juristo N, Gomez OS (2012) Replication of software engineering experiments. In: Meyer B, Nordio M (eds) Empirical software engineering and verification. LNCS, vol\u00a07007. Springer, pp\u00a060\u201388","DOI":"10.1007\/978-3-642-25231-0_2"},{"issue":"3","key":"9262_CR23","doi-asserted-by":"crossref","first-page":"295","DOI":"10.1007\/s10664-010-9141-9","volume":"16","author":"N Juristo","year":"2011","unstructured":"Juristo N, Vegas S (2011) The role of non-exact replications in software engineering experiments. Empir Softw Eng 16(3):295\u2013324","journal-title":"Empir Softw Eng"},{"issue":"1\u20132","key":"9262_CR24","doi-asserted-by":"crossref","first-page":"7","DOI":"10.1023\/B:EMSE.0000013513.48963.1b","volume":"9","author":"N Juristo","year":"2004","unstructured":"Juristo N, Moreno AM, Vegas S (2004) Reviewing 25 years of testing technique experiments. Empir Softw Eng 9(1\u20132):7\u201344","journal-title":"Empir Softw Eng"},{"key":"9262_CR25","doi-asserted-by":"crossref","first-page":"72","DOI":"10.1109\/MS.2006.166","volume":"23","author":"N Juristo","year":"2006","unstructured":"Juristo N, Moreno AM, Vegas S, Solari M (2006) In search of what we experimentally know about unit testing. IEEE Softw 23:72\u201380","journal-title":"IEEE Softw"},{"key":"9262_CR26","doi-asserted-by":"crossref","unstructured":"Juristo N, Vegas S, Solari M, Abrahao S, Ramos I (2012) Comparing the effectiveness of equivalence partitioning, branch testing and code reading be stepwise abstraction applied by subjects. In: Proceedings fifth IEEE international conference on software testing, verification and validation, Montreal, Canada, pp\u00a0330\u2013339","DOI":"10.1109\/ICST.2012.113"},{"key":"9262_CR28","doi-asserted-by":"crossref","unstructured":"Kitchenham BA, Fry J, Linkman SG (2003) The case against cross-over designs in software engineering. In: 11th international workshop on software technology and engineering practice (STEP 2003), Amsterdam, The Netherlands, pp\u00a065\u201367","DOI":"10.1109\/STEP.2003.32"},{"key":"9262_CR29","doi-asserted-by":"crossref","first-page":"219","DOI":"10.1007\/s10664-008-9061-0","volume":"13","author":"BA Kitchenham","year":"2008","unstructured":"Kitchenham, BA (2008) The role of replications in empirical software engineering\u2014a word of warning. Empir Softw Eng 13:219\u2013221","journal-title":"Empir Softw Eng"},{"issue":"1","key":"9262_CR30","doi-asserted-by":"crossref","first-page":"97","DOI":"10.1007\/s10664-007-9053-5","volume":"13","author":"BA Kitchenham","year":"2007","unstructured":"Kitchenham BA, Al-Khilidar H, Babar MA, Berry M, Cox K, Keung J, Kurniawati F, Staples M, Zhang H, Zhu L (2007) Evaluating guidelines for reporting empirical software engineering studies. Empir Softw Eng 13(1):97\u2013121","journal-title":"Empir Softw Eng"},{"issue":"1","key":"9262_CR31","doi-asserted-by":"crossref","first-page":"7","DOI":"10.1016\/j.infsof.2008.09.009","volume":"51","author":"B Kitchenham","year":"2009","unstructured":"Kitchenham B, PearlBrereton O, Budgen D, Turner M, Bailey J, Linkman S (2009) Systematic literature reviews in software engineering\u2014a systematic literature review. Inf Softw Technol 51(1):7\u201315","journal-title":"Inf Softw Technol"},{"key":"9262_CR32","doi-asserted-by":"crossref","unstructured":"Laitenberger O (1998) Studying the effects of code inspection and structural testing on software quality. In: Proceedings 9th international symposium on software reliability engineering, pp\u00a0237\u2013246","DOI":"10.1109\/ISSRE.1998.730887"},{"issue":"3","key":"9262_CR33","doi-asserted-by":"crossref","first-page":"217","DOI":"10.1080\/00031305.1993.10475983","volume":"47","author":"RM Lindsay","year":"1993","unstructured":"Lindsay RM, Ehrenberg ASC (1993) The design of replicated studies. Am Stat 47(3):217\u2013227","journal-title":"Am Stat"},{"key":"9262_CR34","unstructured":"M\u00e4ntyl\u00e4 MV, Lasseinus C, Vanhanen J (2010) Rethinking replication in software engineering: can we see the forest for the trees? In: Knutson C, Krein J (eds) 1st international workshop on replication in empirical software engineering research, Cape Town, South Africa"},{"issue":"1","key":"9262_CR35","doi-asserted-by":"crossref","first-page":"29","DOI":"10.1016\/S0164-1212(00)00024-8","volume":"54","author":"J Miller","year":"2000","unstructured":"Miller J (2000) Applying meta-analytical procedures to software engineering experiments. J Syst Softw 54(1):29\u201339","journal-title":"J Syst Softw"},{"issue":"4","key":"9262_CR36","doi-asserted-by":"crossref","first-page":"233","DOI":"10.1016\/j.infsof.2004.08.005","volume":"47","author":"J Miller","year":"2005","unstructured":"Miller J (2005) Replicating software engineering experiments: a poisoned chalice or the holy grail. Inf Softw Technol 47(4):233\u2013244","journal-title":"Inf Softw Technol"},{"key":"9262_CR37","volume-title":"Design and analysis of experiments","author":"DC Montgomery","year":"2001","unstructured":"Montgomery DC (2001) Design and analysis of experiments, 5th edn. Wiley, New York","edition":"5"},{"issue":"14","key":"9262_CR38","doi-asserted-by":"crossref","first-page":"811","DOI":"10.1016\/S0950-5849(98)00101-3","volume":"40","author":"L Pickard","year":"1998","unstructured":"Pickard L, Kitchenham BA, Jones P (1998) Combining empirical results in software engineering. Inf Softw Technol 40(14):811\u2013821","journal-title":"Inf Softw Technol"},{"key":"9262_CR40","doi-asserted-by":"crossref","unstructured":"Runeson P, Andrews A (2003) Detection or isolation of defects? An experimental comparison of unit testing and code inspection. In: 14th international symposium on software reliability engineering, pp\u00a03\u201313","DOI":"10.1109\/ISSRE.2003.1251026"},{"issue":"3","key":"9262_CR41","doi-asserted-by":"crossref","first-page":"82","DOI":"10.1109\/MS.2006.89","volume":"23","author":"P Runeson","year":"2006","unstructured":"Runeson P, Anderson C, Thelin T, Andrews A, Berling T (2006) What do we know about defect detection methods? IEEE Softw 23(3):82\u201390","journal-title":"IEEE Softw"},{"key":"9262_CR42","doi-asserted-by":"crossref","unstructured":"Runeson P, Stefik A, Andrews A, Gr\u00f6nblom S, Porres I, Siebert S (2011) A comparative analysis of three replicated experiments comparing inspection and unit testing. In: Proceedings 2nd international workshop on replication in empirical software engineering research, Banff, Canada, pp\u00a035\u201342","DOI":"10.1109\/RESER.2011.13"},{"key":"9262_CR43","doi-asserted-by":"crossref","DOI":"10.1002\/9781118181034","volume-title":"Case study research in software engineering\u2014guidelines and examples","author":"P Runeson","year":"2012","unstructured":"Runeson P, H\u00f6st M, Rainer A, Regnell B (2012) Case study research in software engineering\u2014guidelines and examples. Wiley, New York"},{"issue":"2","key":"9262_CR44","doi-asserted-by":"crossref","first-page":"90","DOI":"10.1037\/a0015108","volume":"13","author":"S Schmidt","year":"2009","unstructured":"Schmidt S (2009) Shall we really do it again? The powerful concept of replication is neglected in the social sciences. Rev Gen Psychol 13(2):90\u2013100","journal-title":"Rev Gen Psychol"},{"key":"9262_CR45","doi-asserted-by":"crossref","unstructured":"Shull F, Basili VR, Carver J, Maldonado JC, Travassos GH, Mendonca M, Fabbri S (2002) Replicating software engineering experiments: addressing the tacit knowledge problem. In: Proceedings of the 1st international symposium empirical software engineering, pp\u00a07\u201316","DOI":"10.1109\/ISESE.2002.1166920"},{"issue":"2","key":"9262_CR46","doi-asserted-by":"crossref","first-page":"211","DOI":"10.1007\/s10664-008-9060-1","volume":"13","author":"FJ Shull","year":"2008","unstructured":"Shull FJ, Carver J, Vegas S, Juristo N (2008) The role of replications in empirical software engineering. Empir Softw Eng 13(2):211\u2013218","journal-title":"Empir Softw Eng"},{"key":"9262_CR47","volume-title":"Nonparametric statistics for the behavioural sciences","author":"S Siegel","year":"1956","unstructured":"Siegel S, Castellan N (1956) Nonparametric statistics for the behavioural sciences. McGraw-Hill, New York"},{"key":"9262_CR48","doi-asserted-by":"crossref","unstructured":"Sj\u00f8berg DIK (2007) Knowledge acquisition in software engineering requires sharing of data and artifacts. In: Basili V, Rombach H, Schneider K, Kitchenham B, Pfahl D, Selby R (eds) Empirical software engineering issues: critical assessment and future directions. LNCS, vol\u00a04336. Springer, pp\u00a077\u201382","DOI":"10.1007\/978-3-540-71301-2_23"},{"issue":"3","key":"9262_CR49","doi-asserted-by":"crossref","first-page":"155","DOI":"10.1002\/stvr.248","volume":"12","author":"S So","year":"2002","unstructured":"So S, Cha S, Shimeall T, Kwon Y (2002) An empirical evaluation of six methods to detect faults in software. SW Test Ver Rel 12(3):155\u2013171","journal-title":"SW Test Ver Rel"},{"issue":"1","key":"9262_CR50","doi-asserted-by":"crossref","first-page":"142","DOI":"10.1037\/0021-9010.79.1.142","volume":"79","author":"BE Teasley","year":"1994","unstructured":"Teasley BE, Leventhal LM, Mynatt CR, Rohlman DS (1994) Why software testing is sometimes ineffective: two applied studies of positive test strategy. J Appl Psychol 79(1):142\u2013155","journal-title":"J Appl Psychol"},{"key":"9262_CR51","doi-asserted-by":"crossref","unstructured":"Wohlin C, Runeson P, H\u00f6st M, Ohlsson MC, Regnell B, Wesslen A (2012) Experimentation in software engineering. Springer","DOI":"10.1007\/978-3-642-29044-2"},{"key":"9262_CR52","volume-title":"Case study research design and methods","author":"RK Yin","year":"2009","unstructured":"Yin RK (2009) Case study research design and methods, 4th edn. Sage Publications, Beverly Hills, CA","edition":"4"}],"container-title":["Empirical Software Engineering"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s10664-013-9262-z.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/article\/10.1007\/s10664-013-9262-z\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s10664-013-9262-z","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,5,16]],"date-time":"2024-05-16T19:52:17Z","timestamp":1715889137000},"score":1,"resource":{"primary":{"URL":"http:\/\/link.springer.com\/10.1007\/s10664-013-9262-z"}},"subtitle":["Three (dis)similar studies on inspections versus unit testing"],"short-title":[],"issued":{"date-parts":[[2013,8,18]]},"references-count":47,"journal-issue":{"issue":"6","published-print":{"date-parts":[[2014,12]]}},"alternative-id":["9262"],"URL":"https:\/\/doi.org\/10.1007\/s10664-013-9262-z","relation":{},"ISSN":["1382-3256","1573-7616"],"issn-type":[{"value":"1382-3256","type":"print"},{"value":"1573-7616","type":"electronic"}],"subject":[],"published":{"date-parts":[[2013,8,18]]}}}