{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,3,3]],"date-time":"2026-03-03T00:49:53Z","timestamp":1772498993067,"version":"3.50.1"},"reference-count":79,"publisher":"Springer Science and Business Media LLC","issue":"2","license":[{"start":{"date-parts":[[2021,2,2]],"date-time":"2021-02-02T00:00:00Z","timestamp":1612224000000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"},{"start":{"date-parts":[[2021,2,2]],"date-time":"2021-02-02T00:00:00Z","timestamp":1612224000000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"}],"funder":[{"DOI":"10.13039\/100014440","name":"Ministerio de Ciencia, Innovaci\u00f3n y Universidades","doi-asserted-by":"publisher","award":["PGC2018- 097265-B-I00"],"award-info":[{"award-number":["PGC2018- 097265-B-I00"]}],"id":[{"id":"10.13039\/100014440","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Empir Software Eng"],"published-print":{"date-parts":[[2021,3]]},"DOI":"10.1007\/s10664-020-09907-7","type":"journal-article","created":{"date-parts":[[2021,2,2]],"date-time":"2021-02-02T18:32:37Z","timestamp":1612290757000},"update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":14,"title":["Comparing the results of replications in software engineering"],"prefix":"10.1007","volume":"26","author":[{"given":"Adrian","family":"Santos","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-8535-9386","authenticated-orcid":false,"given":"Sira","family":"Vegas","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Markku","family":"Oivo","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Natalia","family":"Juristo","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2021,2,2]]},"reference":[{"key":"9907_CR1","doi-asserted-by":"crossref","unstructured":"Arcuri A, Briand L (2011) A practical guide for using statistical tests to assess randomized algorithms in software engineering. In: 2011 33rd International conference on software engineering (ICSE). IEEE, pp 1\u201310","DOI":"10.1145\/1985793.1985795"},{"key":"9907_CR2","doi-asserted-by":"crossref","unstructured":"Badampudi D, Wohlin C, Gorschek T (2019) Contextualizing research evidence through knowledge translation in software engineering. In: Proceedings of the evaluation and assessment on software engineering, EASE 2019, Copenhagen, Denmark, April 15\u201317, 2019. ACM, pp 306\u2013311","DOI":"10.1109\/ESEM.2019.8870165"},{"issue":"7604","key":"9907_CR3","doi-asserted-by":"publisher","first-page":"452","DOI":"10.1038\/533452a","volume":"533","author":"M Baker","year":"2016","unstructured":"Baker M (2016) Is there a reproducibility crisis? a nature survey lifts the lid on how researchers view the\u2019crisis rocking science and what they think will help. Nature 533(7604):452\u2013455","journal-title":"Nature"},{"issue":"4","key":"9907_CR4","doi-asserted-by":"publisher","first-page":"456","DOI":"10.1109\/32.799939","volume":"25","author":"VR Basili","year":"1999","unstructured":"Basili VR, Shull F, Lanubile F (1999) Building knowledge through families of experiments. IEEE Trans Softw Eng 25(4):456\u2013473","journal-title":"IEEE Trans Softw Eng"},{"key":"9907_CR5","volume-title":"Test-driven development: by example","author":"K Beck","year":"2003","unstructured":"Beck K (2003) Test-driven development: by example. Addison-Wesley Professional, Boston"},{"key":"9907_CR6","doi-asserted-by":"crossref","unstructured":"Bezerra RM, da Silva FQ, Santana AM, Magalhaes CV, Santos RE (2015) Replication of empirical studies in software engineering: an update of a systematic mapping study. In: Proceedings of the 2015 9th international symposium on empirical software engineering and measurement (ESEM). IEEE, pp 1\u20134","DOI":"10.1109\/ESEM.2015.7321213"},{"key":"9907_CR7","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-25655-9","volume-title":"Umbrella reviews: evidence synthesis with overviews of reviews and meta-epidemiologic studies","author":"G Biondi-Zoccai","year":"2016","unstructured":"Biondi-Zoccai G (2016) Umbrella reviews: evidence synthesis with overviews of reviews and meta-epidemiologic studies. Springer, Berlin"},{"key":"9907_CR8","volume-title":"Introduction to meta-analysis","author":"M Borenstein","year":"2011","unstructured":"Borenstein M, Hedges LV, Higgins JP, Rothstein HR (2011) Introduction to meta-analysis. Wiley, Chichester"},{"issue":"5","key":"9907_CR9","doi-asserted-by":"publisher","first-page":"72","DOI":"10.1109\/MS.2017.3571562","volume":"34","author":"L Briand","year":"2017","unstructured":"Briand L, Bianculli D, Nejati S, Pastore F, Sabetzadeh M (2017) The case for context-driven software engineering research: generalizability is overrated. IEEE Softw 34(5):72\u201375","journal-title":"IEEE Softw"},{"key":"9907_CR10","unstructured":"Brooks A, Roper M, Wood M, Daly J, Miller J (2003) Replication of software engineering experiments. Empirical Foundations of Computer Science Technical Report, EfoCS-51-2003. Department of Computer and Information Sciences University of Strathclyde"},{"key":"9907_CR11","doi-asserted-by":"publisher","DOI":"10.1002\/9781118778210","volume-title":"Applied mixed models in medicine","author":"H Brown","year":"2014","unstructured":"Brown H, Prescott R (2014) Applied mixed models in medicine. Wiley, Chichester"},{"issue":"5","key":"9907_CR12","doi-asserted-by":"publisher","first-page":"365","DOI":"10.1038\/nrn3475","volume":"14","author":"KS Button","year":"2013","unstructured":"Button KS, Ioannidis JP, Mokrysz C, Nosek BA, Flint J, Robinson ES, Munaf\u00f2 MR (2013) Power failure: why small sample size undermines the reliability of neuroscience. Nat Rev Neurosci 14(5):365","journal-title":"Nat Rev Neurosci"},{"issue":"9","key":"9907_CR13","doi-asserted-by":"publisher","first-page":"637","DOI":"10.1038\/s41562-018-0399-z","volume":"2","author":"CF Camerer","year":"2018","unstructured":"Camerer CF, Dreber A, Holzmeister F, Ho TH, Huber J, Johannesson M, Kirchler M, Nave G, Nosek BA, Pfeiffer T et al (2018) Evaluating the replicability of social science experiments in nature and science between 2010 and 2015. Nat Hum Behav 2(9):637\u2013644","journal-title":"Nat Hum Behav"},{"key":"9907_CR14","doi-asserted-by":"publisher","DOI":"10.1201\/b14872","volume-title":"Applied meta-analysis with R","author":"DGD Chen","year":"2013","unstructured":"Chen DGD, Peace KE (2013) Applied meta-analysis with R. CRC Press, Boca Raton"},{"key":"9907_CR15","first-page":"20","volume-title":"Statistical power analysis for the behavioral sciences","author":"J Cohen","year":"1988","unstructured":"Cohen J (1988) Statistical power analysis for the behavioral sciences. Lawrence Earlbaum Associates, Hillsdale, pp 20\u201326"},{"key":"9907_CR16","doi-asserted-by":"publisher","DOI":"10.4324\/9780203807002","volume-title":"Understanding the new statistics: effect sizes, confidence intervals, and meta-analysis","author":"G Cumming","year":"2013","unstructured":"Cumming G (2013) Understanding the new statistics: effect sizes, confidence intervals, and meta-analysis. Routledge, New York"},{"issue":"1","key":"9907_CR17","doi-asserted-by":"publisher","first-page":"7","DOI":"10.1177\/0956797613504966","volume":"25","author":"G Cumming","year":"2014","unstructured":"Cumming G (2014) The new statistics: why and how. Psychol Sci 25(1):7\u201329","journal-title":"Psychol Sci"},{"issue":"3","key":"9907_CR18","first-page":"501","volume":"19","author":"FQ Da Silva","year":"2014","unstructured":"Da Silva FQ, Suassuna M, Fran\u00e7a A C C, Grubb AM, Gouveia TB, Monteiro CV, dos Santos IE (2014) Replication of empirical studies in software engineering research: a systematic mapping study. Empir Softw Eng 19 (3):501\u2013557","journal-title":"Empir Softw Eng"},{"issue":"3","key":"9907_CR19","doi-asserted-by":"publisher","first-page":"1302","DOI":"10.1007\/s10664-015-9386-4","volume":"21","author":"BBN de Fran\u00e7a","year":"2016","unstructured":"de Fran\u00e7a B B N, Travassos GH (2016) Experimentation with dynamic simulation models in software engineering: planning and reporting guidelines. Empir Softw Eng 21(3):1302\u20131345","journal-title":"Empir Softw Eng"},{"key":"9907_CR20","doi-asserted-by":"publisher","first-page":"76","DOI":"10.1016\/j.infsof.2015.02.001","volume":"64","author":"CV de Magalh\u00e3es","year":"2015","unstructured":"de Magalh\u00e3es C V, da Silva FQ, Santos RE, Suassuna M (2015) Investigations about replication of empirical studies in software engineering: a systematic mapping study. Inf Softw Technol 64:76\u2013101","journal-title":"Inf Softw Technol"},{"issue":"5","key":"9907_CR21","doi-asserted-by":"publisher","first-page":"2457","DOI":"10.1007\/s10664-016-9471-3","volume":"22","author":"O Dieste","year":"2017","unstructured":"Dieste O, Aranda AM, Uyaguari F, Turhan B, Tosun A, Fucci D, Oivo M, Juristo N (2017) Empirical evaluation of the effects of experience on code quality and programmer productivity: an exploratory study. Empir Softw Eng 22(5):2457\u20132542","journal-title":"Empir Softw Eng"},{"key":"9907_CR22","doi-asserted-by":"publisher","first-page":"438","DOI":"10.1109\/TSE.1984.5010257","volume":"SE-10(4)","author":"J Duran","year":"1984","unstructured":"Duran J, Ntafos S (1984) An evaluation of random testing. IEEE Trans Softw Eng SE-10(4):438\u2013444","journal-title":"IEEE Trans Softw Eng"},{"issue":"8","key":"9907_CR23","doi-asserted-by":"publisher","first-page":"745","DOI":"10.1016\/j.infsof.2005.08.009","volume":"48","author":"T Dyb\u00e5","year":"2006","unstructured":"Dyb\u00e5 T, Kampenes VB, Sj\u00f8berg DI (2006) A systematic review of statistical power in software engineering experiments. Inf Softw Technol 48(8):745\u2013755","journal-title":"Inf Softw Technol"},{"key":"9907_CR24","volume-title":"Systematic reviews in health care: meta-analysis in context","author":"M Egger","year":"2008","unstructured":"Egger M, Davey-Smith G, Altman D (2008) Systematic reviews in health care: meta-analysis in context. Wiley, New York"},{"key":"9907_CR25","doi-asserted-by":"publisher","DOI":"10.1017\/CBO9780511761676","volume-title":"The essential guide to effect sizes: statistical power, meta-analysis, and the interpretation of research results","author":"PD Ellis","year":"2010","unstructured":"Ellis PD (2010) The essential guide to effect sizes: statistical power, meta-analysis, and the interpretation of research results. Cambridge University Press, Cambridge"},{"issue":"5","key":"9907_CR26","doi-asserted-by":"publisher","first-page":"383","DOI":"10.3109\/00952990.2011.600386","volume":"37","author":"DJ Feaster","year":"2011","unstructured":"Feaster DJ, Mikulich-Gilbertson S, Brincks AM (2011) Modeling site effects in the design and analysis of multi-site trials. Am J Drug Alcohol Abuse 37(5):383\u2013391","journal-title":"Am J Drug Alcohol Abuse"},{"key":"9907_CR27","volume-title":"Discovering statistics using IBM SPSS statistics","author":"A Field","year":"2013","unstructured":"Field A (2013) Discovering statistics using IBM SPSS statistics. Sage, Thousand Oaks"},{"issue":"9","key":"9907_CR28","doi-asserted-by":"publisher","first-page":"949","DOI":"10.1016\/j.jclinepi.2010.11.016","volume":"64","author":"D Fisher","year":"2011","unstructured":"Fisher D, Copas A, Tierney J, Parmar M (2011) A critical review of methods for the assessment of patient-level interactions in individual participant data meta-analysis of randomized trials, and guidance for practitioners. J Clin Epidemiol 64 (9):949\u2013967","journal-title":"J Clin Epidemiol"},{"issue":"1","key":"9907_CR29","doi-asserted-by":"publisher","first-page":"111","DOI":"10.1186\/1471-2288-12-111","volume":"12","author":"JJ Gagnier","year":"2012","unstructured":"Gagnier JJ, Moher D, Boon H, Beyene J, Bombardier C (2012) Investigating clinical heterogeneity in systematic reviews: a methodologic review of guidance in the literature. BMC Med Res Methodol 12(1):111","journal-title":"BMC Med Res Methodol"},{"key":"9907_CR30","volume-title":"Theory of probability","author":"BV Gnedenko","year":"2020","unstructured":"Gnedenko BV (2020) Theory of probability, 6th edn. CRC Press, Boca Raton","edition":"6th edn."},{"key":"9907_CR31","doi-asserted-by":"crossref","unstructured":"G\u00f3mez O S, Juristo N, Vegas S (2010) Replications types in experimental disciplines. In: Proceedings of the 2010 ACM-IEEE international symposium on empirical software engineering and measurement. ACM, p 3","DOI":"10.1145\/1852786.1852790"},{"issue":"8","key":"9907_CR32","doi-asserted-by":"publisher","first-page":"1033","DOI":"10.1016\/j.infsof.2014.04.004","volume":"56","author":"OS Gomez","year":"2014","unstructured":"Gomez OS, Juristo N, Vegas S (2014) Understanding replication of experiments in software engineering: a classification. Inf Softw Technol 56(8):1033\u20131048","journal-title":"Inf Softw Technol"},{"issue":"1","key":"9907_CR33","doi-asserted-by":"publisher","first-page":"43","DOI":"10.1186\/1471-2288-10-43","volume":"10","author":"RH Groenwold","year":"2010","unstructured":"Groenwold RH, Rovers MM, Lubsen J, van der Heijden GJ (2010) Subgroup effects despite homogeneous heterogeneity test results. BMC Med Res Methodol 10(1):43","journal-title":"BMC Med Res Methodol"},{"key":"9907_CR34","volume-title":"Cochrane handbook for systematic reviews of interventions, vol 4","author":"JP Higgins","year":"2011","unstructured":"Higgins JP, Green S (2011) Cochrane handbook for systematic reviews of interventions, vol 4. Chichester, Wiley"},{"issue":"7414","key":"9907_CR35","doi-asserted-by":"publisher","first-page":"557","DOI":"10.1136\/bmj.327.7414.557","volume":"327","author":"JP Higgins","year":"2003","unstructured":"Higgins JP, Thompson SG, Deeks JJ, Altman DG (2003) Measuring inconsistency in meta-analyses. BMJ 327(7414):557\u2013560","journal-title":"BMJ"},{"key":"9907_CR36","doi-asserted-by":"crossref","unstructured":"H\u00f6st M, Wohlin C, Thelin T (2005) Experimental context classification: incentives and experience of subjects. In: 27th International conference on software engineering (ICSE 2005), 15\u201321 May 2005, St. Louis, Missouri, USA. ACM, pp 470\u2013478","DOI":"10.1145\/1062455.1062539"},{"issue":"8","key":"9907_CR37","doi-asserted-by":"publisher","first-page":"e124","DOI":"10.1371\/journal.pmed.0020124","volume":"2","author":"JP Ioannidis","year":"2005","unstructured":"Ioannidis JP (2005) Why most published research findings are false. PLoS Med 2(8):e124","journal-title":"PLoS Med"},{"issue":"7658","key":"9907_CR38","doi-asserted-by":"publisher","first-page":"1413","DOI":"10.1136\/bmj.a117","volume":"336","author":"J Ioannidis","year":"2008","unstructured":"Ioannidis J, Patsopoulos N, Rothstein H (2008) Research methodology: reasons or excuses for avoiding meta-analysis in forest plots. BMJ: Br Med J 336 (7658):1413\u20131415","journal-title":"BMJ: Br Med J"},{"key":"9907_CR39","volume-title":"Guide to advanced empirical software engineering, chap. Reporting Controlled Experiments in Software Engineering","author":"A Jedlitschka","year":"2008","unstructured":"Jedlitschka A, Ciolkowski M, Pfahl D (2008) Guide to advanced empirical software engineering, chap. Reporting Controlled Experiments in Software Engineering. Springer, Berlin"},{"key":"9907_CR40","doi-asserted-by":"publisher","first-page":"133","DOI":"10.1016\/j.jss.2015.03.065","volume":"116","author":"M J\u00f8rgensen","year":"2016","unstructured":"J\u00f8rgensen M, Dyb\u00e5 T, Liest\u00f8l K, Sj\u00f8berg DI (2016) Incorrect results in software engineering experiments: how to improve research practices. J Syst Softw 116:133\u2013145","journal-title":"J Syst Softw"},{"key":"9907_CR41","doi-asserted-by":"crossref","unstructured":"Juristo N (2016) Once is not enough: why we need replication. In: Menzies T, Williams L, Zimmermann T (eds) Perspectives on data science for software engineering. Morgan Kaufmann","DOI":"10.1016\/B978-0-12-804206-9.00054-4"},{"key":"9907_CR42","unstructured":"Juristo N, Moreno AM (2011) Basics of software engineering experimentation. Springer Science & Business Media"},{"key":"9907_CR43","doi-asserted-by":"crossref","unstructured":"Juristo N, Vegas S (2009) Using differences among replications of software engineering experiments to gain knowledge. In: Proceedings of the 2009 3rd international symposium on empirical software engineering and measurement (ESEM). IEEE, pp 356\u2013366","DOI":"10.1109\/ESEM.2009.5314236"},{"issue":"3","key":"9907_CR44","doi-asserted-by":"publisher","first-page":"295","DOI":"10.1007\/s10664-010-9141-9","volume":"16","author":"N Juristo","year":"2011","unstructured":"Juristo N, Vegas S (2011) The role of non-exact replications in software engineering experiments. Empir Softw Eng 16(3):295\u2013324","journal-title":"Empir Softw Eng"},{"key":"9907_CR45","doi-asserted-by":"crossref","unstructured":"Juristo N, Vegas S, Solari M, Abrahao S, Ramos I (2012) Comparing the effectiveness of equivalence partitioning, branch testing and code reading by stepwise abstraction applied by subjects. In: 2012 IEEE fifth international conference on software testing, verification and validation. IEEE, pp 330\u2013339","DOI":"10.1109\/ICST.2012.113"},{"issue":"11","key":"9907_CR46","doi-asserted-by":"publisher","first-page":"1073","DOI":"10.1016\/j.infsof.2007.02.015","volume":"49","author":"VB Kampenes","year":"2007","unstructured":"Kampenes VB, Dyb\u00e5 T, Hannay JE, Sj\u00f8berg DI (2007) A systematic review of effect size in software engineering experiments. Inf Softw Technol 49(11):1073\u20131086","journal-title":"Inf Softw Technol"},{"issue":"2","key":"9907_CR47","doi-asserted-by":"publisher","first-page":"219","DOI":"10.1007\/s10664-008-9061-0","volume":"13","author":"B Kitchenham","year":"2008","unstructured":"Kitchenham B (2008) The role of replications in empirical software engineering\u2014a word of warning. Empir Softw Eng 13(2):219\u2013221","journal-title":"Empir Softw Eng"},{"issue":"2","key":"9907_CR48","doi-asserted-by":"publisher","first-page":"579","DOI":"10.1007\/s10664-016-9437-5","volume":"22","author":"B Kitchenham","year":"2017","unstructured":"Kitchenham B, Madeyski L, Budgen D, Keung J, Brereton P, Charters S, Gibbs S, Pohthong A (2017) Robust statistical methods for empirical software engineering. Empir Softw Eng 22(2):579\u2013630","journal-title":"Empir Softw Eng"},{"key":"9907_CR49","doi-asserted-by":"crossref","unstructured":"Langan D, Higgins JP, Jackson D, Bowden J, Veroniki AA, Kontopantelis E, Viechtbauer W, Simmonds M (2018) A comparison of heterogeneity variance estimators in simulated random-effects meta-analyses. Research synthesis methods","DOI":"10.1002\/jrsm.1316"},{"issue":"9096","key":"9907_CR50","doi-asserted-by":"publisher","first-page":"123","DOI":"10.1016\/S0140-6736(97)08468-7","volume":"351","author":"J Lau","year":"1998","unstructured":"Lau J, Ioannidis JP, Schmid CH (1998) Summing up evidence: one answer is not always enough. Lancet 351(9096):123\u2013127","journal-title":"Lancet"},{"key":"9907_CR51","volume-title":"Meta-analysis in medical research: the handbook for the understanding and practice of meta-analysis","author":"G Leandro","year":"2008","unstructured":"Leandro G (2008) Meta-analysis in medical research: the handbook for the understanding and practice of meta-analysis. Wiley, Hoboken"},{"issue":"6","key":"9907_CR52","doi-asserted-by":"publisher","first-page":"537","DOI":"10.1177\/1745691612460688","volume":"7","author":"MC Makel","year":"2012","unstructured":"Makel MC, Plucker JA, Hegarty B (2012) Replications in psychology research: how often do they really occur? Perspec Psychol Sci 7(6):537\u2013542","journal-title":"Perspec Psychol Sci"},{"issue":"6","key":"9907_CR53","doi-asserted-by":"publisher","first-page":"487","DOI":"10.1037\/a0039400","volume":"70","author":"SE Maxwell","year":"2015","unstructured":"Maxwell SE, Lau MY, Howard GS (2015) Is psychology suffering from a replication crisis? what does \u201cfailure to replicate\u201d really mean? Am Psychol 70(6):487","journal-title":"Am Psychol"},{"issue":"2","key":"9907_CR54","doi-asserted-by":"publisher","first-page":"295","DOI":"10.1007\/s10648-014-9287-x","volume":"28","author":"DM McNeish","year":"2016","unstructured":"McNeish DM, Stapleton LM (2016) The effect of small sample size on two-level model estimates: a review and illustration. Educ Psychol Rev 28(2):295\u2013314","journal-title":"Educ Psychol Rev"},{"issue":"1\u20132","key":"9907_CR55","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1007\/s10664-011-9193-5","volume":"17","author":"T Menzies","year":"2012","unstructured":"Menzies T, Shepperd M (2012) Special issue on repeatable results in software engineering prediction. Empirical Softw Eng 17 (1\u20132):1\u201317. https:\/\/doi.org\/10.1007\/s10664-011-9193-5","journal-title":"Empirical Softw Eng"},{"issue":"4","key":"9907_CR56","doi-asserted-by":"publisher","first-page":"233","DOI":"10.1016\/j.infsof.2004.08.005","volume":"47","author":"J Miller","year":"2005","unstructured":"Miller J (2005) Replicating software engineering experiments: a poisoned chalice or the holy grail. Inf Softw Technol 47(4):233\u2013244","journal-title":"Inf Softw Technol"},{"issue":"1","key":"9907_CR57","doi-asserted-by":"publisher","first-page":"105","DOI":"10.1037\/1082-989X.7.1.105","volume":"7","author":"SB Morris","year":"2002","unstructured":"Morris SB, DeShon RP (2002) Combining effect size estimates in meta-analysis with repeated measures and independent-groups designs. Psychol Methods 7(1):105","journal-title":"Psychol Methods"},{"issue":"11","key":"9907_CR58","doi-asserted-by":"publisher","first-page":"2074","DOI":"10.1002\/sim.8086","volume":"38","author":"TP Morris","year":"2019","unstructured":"Morris TP, White IR, Michael JC (2019) Using simulation studies to evaluate statistical methods. Stat Med 38(11):2074\u20132102","journal-title":"Stat Med"},{"key":"9907_CR59","doi-asserted-by":"crossref","unstructured":"Murphy GC (2019) Beyond integrated development environments: adding context to software development. In: Proceedings of the 41st international conference on software engineering: new ideas and emerging results (ICSE-NIER), IEEE, pp 73\u201376","DOI":"10.1109\/ICSE-NIER.2019.00027"},{"key":"9907_CR60","volume-title":"The art of software testing","author":"GJ Myers","year":"2011","unstructured":"Myers GJ, Sandler C, Badgett T (2011) The art of software testing. Wiley, New York"},{"key":"9907_CR61","doi-asserted-by":"crossref","unstructured":"Ntafos S (1998) On random and partition testing. In: Proceedings of ACM SIGSOFT international symposium on software testing and analysis. ACM Press, pp 42\u201348","DOI":"10.1145\/271775.271785"},{"issue":"6","key":"9907_CR62","doi-asserted-by":"publisher","first-page":"528","DOI":"10.1177\/1745691612465253","volume":"7","author":"H Pashler","year":"2012","unstructured":"Pashler H, Wagenmakers EJ (2012) Editors\u2019 introduction to the special section on replicability in psychological science: a crisis of confidence? Perspect Psychol Sci 7(6):528\u2013530","journal-title":"Perspect Psychol Sci"},{"issue":"4","key":"9907_CR63","doi-asserted-by":"publisher","first-page":"539","DOI":"10.1177\/1745691616646366","volume":"11","author":"P Patil","year":"2016","unstructured":"Patil P, Peng RD, Leek JT (2016) What should researchers expect when they replicate studies? A statistical view of replicability in psychological science. Perspect Psychol Sci 11(4):539\u2013544","journal-title":"Perspect Psychol Sci"},{"key":"9907_CR64","doi-asserted-by":"crossref","unstructured":"Petersen K, Wohlin C (2009) Context in industrial software engineering research. In: Proceedings of the third international symposium on empirical software engineering and measurement, ESEM 2009, October 15\u201316, 2009, Lake Buena Vista, Florida, USA. IEEE Computer Society, pp 401\u2013404","DOI":"10.1109\/ESEM.2009.5316010"},{"key":"9907_CR65","doi-asserted-by":"crossref","unstructured":"Petitti DB (2000) Meta-analysis, decision analysis, and cost-effectiveness analysis: methods for quantitative synthesis in medicine. 31 OUP USA","DOI":"10.1093\/acprof:oso\/9780195133646.001.0001"},{"issue":"2","key":"9907_CR66","doi-asserted-by":"publisher","first-page":"131","DOI":"10.1007\/s10664-008-9102-8","volume":"14","author":"P Runeson","year":"2009","unstructured":"Runeson P, H\u00f6st M (2009) Guidelines for conducting and reporting case study research in software engineering. Empir Softw Eng 14(2):131","journal-title":"Empir Softw Eng"},{"issue":"4","key":"9907_CR67","doi-asserted-by":"publisher","first-page":"387","DOI":"10.1177\/009286150403800409","volume":"38","author":"F Ruvuna","year":"2004","unstructured":"Ruvuna F (2004) Unequal center sizes, sample size, and power in multicenter clinical trials. Drug Inf J 38(4):387\u2013394","journal-title":"Drug Inf J"},{"key":"9907_CR68","doi-asserted-by":"publisher","unstructured":"Santos A, G\u00f3mez OS, Juristo N (2018) Analyzing families of experiments in se: a systematic mapping study. IEEE Trans Softw Eng, pp 1\u20131. https:\/\/doi.org\/10.1109\/TSE.2018.2864633","DOI":"10.1109\/TSE.2018.2864633"},{"key":"9907_CR69","doi-asserted-by":"publisher","unstructured":"Santos A, Vegas S, Oivo M, Juristo N (2019) A procedure and guidelines for analyzing groups of software engineering replications. IEEE Trans Softw Eng, pp 1\u20131. https:\/\/doi.org\/10.1109\/TSE.2019.2935720","DOI":"10.1109\/TSE.2019.2935720"},{"key":"9907_CR70","unstructured":"Shadish WR, Cook TD, Campbell DT (2002) Experimental and quasi-experimental designs for generalized causal inference. Wadsworth Cengage Learning"},{"key":"9907_CR71","doi-asserted-by":"publisher","first-page":"120","DOI":"10.1016\/j.infsof.2018.01.006","volume":"99","author":"M Shepperd","year":"2018","unstructured":"Shepperd M, Ajienka N, Counsell S (2018) The role and value of replication in empirical software engineering results. Inf Softw Technol 99:120\u2013132","journal-title":"Inf Softw Technol"},{"issue":"1\u20132","key":"9907_CR72","doi-asserted-by":"publisher","first-page":"111","DOI":"10.1023\/B:EMSE.0000013516.80487.33","volume":"9","author":"F Shull","year":"2004","unstructured":"Shull F, Mendonc\u00e7a M G, Basili V, Carver J, Maldonado JC, Fabbri S, Travassos GH, Ferreira MC (2004) Knowledge-sharing issues in experimental software engineering. Empir Softw Eng 9(1\u20132):111\u2013137","journal-title":"Empir Softw Eng"},{"issue":"3","key":"9907_CR73","doi-asserted-by":"publisher","first-page":"209","DOI":"10.1191\/1740774505cn087oa","volume":"2","author":"MC Simmonds","year":"2005","unstructured":"Simmonds MC, Higginsa JP, Stewartb LA, Tierneyb JF, Clarke MJ, Thompson SG (2005) Meta-analysis of individual patient data from randomized trials: a review of methods used in practice. Clin Trials 2(3):209\u2013217","journal-title":"Clin Trials"},{"key":"9907_CR74","doi-asserted-by":"crossref","unstructured":"Snijders TA (2011) Multilevel analysis. In: International encyclopedia of statistical science. Springer, pp 879\u2013882","DOI":"10.1007\/978-3-642-04898-2_387"},{"issue":"2","key":"9907_CR75","doi-asserted-by":"publisher","first-page":"157","DOI":"10.1111\/j.1467-6494.1994.tb00289.x","volume":"62","author":"B Thompson","year":"1994","unstructured":"Thompson B (1994) The pivotal role of replication in psychological research: empirically evaluating the replicability of sample results. J Pers 62 (2):157\u2013176","journal-title":"J Pers"},{"issue":"6","key":"9907_CR76","doi-asserted-by":"publisher","first-page":"2763","DOI":"10.1007\/s10664-016-9490-0","volume":"22","author":"A Tosun","year":"2017","unstructured":"Tosun A, Dieste O, Fucci D, Vegas S, Turhan B, Erdogmus H, Santos A, Oivo M, Toro K, Jarvinen J et al (2017) An industry experiment on the effects of test-driven development on external quality and productivity. Empir Softw Eng 22(6):2763\u20132805","journal-title":"Empir Softw Eng"},{"key":"9907_CR77","unstructured":"Viechtbauer W (2010) Metafor: meta-analysis package for r. R package version 2010, 1\u20130"},{"key":"9907_CR78","doi-asserted-by":"publisher","DOI":"10.1002\/0470854200","volume-title":"Meta-analysis of controlled clinical trials, vol 7","author":"A Whitehead","year":"2002","unstructured":"Whitehead A (2002) Meta-analysis of controlled clinical trials, vol 7. Wiley, New York"},{"key":"9907_CR79","doi-asserted-by":"crossref","unstructured":"Wohlin C, Runeson P, H\u00f6st M, Ohlsson MC, Regnell B, Wessl\u00e9n A (2012) Experimentation in software engineering. Springer Science & Business Media","DOI":"10.1007\/978-3-642-29044-2"}],"container-title":["Empirical Software Engineering"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10664-020-09907-7.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s10664-020-09907-7\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10664-020-09907-7.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2022,10,26]],"date-time":"2022-10-26T05:04:58Z","timestamp":1666760698000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s10664-020-09907-7"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2021,2,2]]},"references-count":79,"journal-issue":{"issue":"2","published-print":{"date-parts":[[2021,3]]}},"alternative-id":["9907"],"URL":"https:\/\/doi.org\/10.1007\/s10664-020-09907-7","relation":{},"ISSN":["1382-3256","1573-7616"],"issn-type":[{"value":"1382-3256","type":"print"},{"value":"1573-7616","type":"electronic"}],"subject":[],"published":{"date-parts":[[2021,2,2]]},"assertion":[{"value":"13 November 2020","order":1,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"2 February 2021","order":2,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}}],"article-number":"13"}}