{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,24]],"date-time":"2026-10-24T19:20:27Z","timestamp":1792869627425,"version":"4.5.0"},"reference-count":43,"publisher":"Springer Science and Business Media LLC","license":[{"start":{"date-parts":[[2012,9,1]],"date-time":"2012-09-01T00:00:00Z","timestamp":1346457600000},"content-version":"tdm","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["Empir Software Eng"],"DOI":"10.1007\/s10664-012-9227-7","type":"journal-article","created":{"date-parts":[[2012,8,31]],"date-time":"2012-08-31T07:56:36Z","timestamp":1346399796000},"source":"Crossref","is-referenced-by-count":40,"title":["Replication of empirical studies in software engineering research: a systematic mapping study"],"prefix":"10.1007","author":[{"given":"Fabio Q. B.","family":"da Silva","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Marcos","family":"Suassuna","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"A. C\u00e9sar C.","family":"Fran\u00e7a","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Alicia M.","family":"Grubb","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Tatiana B.","family":"Gouveia","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Cleviton V. F.","family":"Monteiro","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Igor Ebrahim","family":"dos Santos","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2012,9,1]]},"reference":[{"key":"9227_CR1","unstructured":"Abran A, Moore J, Bourque P, Dupuis T (Eds.) (2004) Guide to software engineering body of knowledge, IEEE Computer Society. 204"},{"key":"9227_CR2","unstructured":"Almqvist JPF (2006) Replication of controlled experiments in empirical software engineering \u2014a survey. Master\u2019s Thesis, Department of Computer Science, Faculty of Science, Lund University, Sweden. 129"},{"key":"9227_CR3","doi-asserted-by":"crossref","first-page":"19","DOI":"10.1080\/1364557032000119616","volume":"8","author":"H Arksey","year":"2005","unstructured":"Arksey H, O\u2019Malley L (2005) Scoping studies: towards a methodological framework. Int J Soc Res Meth 8:19\u201332","journal-title":"Int J Soc Res Meth"},{"key":"9227_CR4","doi-asserted-by":"crossref","first-page":"456","DOI":"10.1109\/32.799939","volume":"25","author":"V Basili","year":"1999","unstructured":"Basili V et al (1999) Building knowledge through families of experiments. IEEE Trans Software Eng 25:456\u2013473. doi: 10.1109\/32.799939","journal-title":"IEEE Trans Software Eng"},{"key":"9227_CR5","unstructured":"Brooks A et al. (1995) Replication of Experimental Results in Software Engineering. Technical Report, EFoCS-17-95 [RR\/95\/193], Dept. of Computer Science, Univ. of Strathclyde. 38"},{"key":"9227_CR6","unstructured":"Brooks A et al. (2007) Replication\u2019s role in software engineering. In F Shull, J Singer, and DIK Sj\u00f8berg (eds) Guide to Advanced Empirical Software Engineering. Springer, pp 365\u2013379"},{"key":"9227_CR7","unstructured":"Carver JC. (2010) Towards Reporting Guidelines for Experimental Replications: A Proposal. In RESER\u20192010: Proceedings of the 1st International Workshop on Replication in Empirical Software Engineering Research, Cape Town, South Africa. 4"},{"key":"9227_CR8","doi-asserted-by":"crossref","unstructured":"Carver JC et al. (2003) Issues in using students in empirical studies in soft \u2014ware engineering education. In Proceedings of the 9th International Software Metrics Symposium (METRICS2003), pp239\u2013249","DOI":"10.1109\/METRIC.2003.1232471"},{"key":"9227_CR9","doi-asserted-by":"crossref","unstructured":"Ciolkowski M et al. (2004) Using academic courses for empirical validation of software development processes. In Proceedings of the 30th Euromicro Conference, pp 354\u2013361","DOI":"10.1109\/EURMIC.2004.1333390"},{"issue":"1","key":"9227_CR10","doi-asserted-by":"crossref","first-page":"37","DOI":"10.1177\/001316446002000104","volume":"20","author":"J Cohen","year":"1960","unstructured":"Cohen J (1960) A coefficient of agreement for nominal scales. Educ Psychol Meas 20(1):37\u201346. doi: 10.1177\/001316446002000104","journal-title":"Educ Psychol Meas"},{"issue":"9","key":"9227_CR11","doi-asserted-by":"crossref","first-page":"899","DOI":"10.1016\/j.infsof.2011.04.004","volume":"53","author":"FQB Silva da","year":"2011","unstructured":"da Silva FQB et al (2011a) Six years of systematic literature reviews in software engineering: an updated tertiary study. Inform Software Tech 53(9):899\u2013913. doi: 10.1016\/j.infsof.2011.04.004","journal-title":"Inform Software Tech"},{"key":"9227_CR12","doi-asserted-by":"crossref","unstructured":"da Silva FQB et al. (2011b) Replication of empirical studies in software engineering: Preliminary findings from a systematic mapping study. Proceedings of the 2nd International Workshop on Replication in Empirical Software Engineering Research RESER\u20192011, pp 61\u201370","DOI":"10.1109\/RESER.2011.14"},{"key":"9227_CR13","doi-asserted-by":"crossref","unstructured":"Daly J, Brooks A, Miller J, Roper M, Wood M (1994) Verification of Results in Software Maintenance Through External Replication. IEEE International Conference on Software Maintenance, pp. 50\u201357","DOI":"10.1109\/ICSM.1994.336790"},{"issue":"7","key":"9227_CR14","doi-asserted-by":"crossref","first-page":"707","DOI":"10.1016\/j.infsof.2010.03.003","volume":"52","author":"MK Davidsen","year":"2010","unstructured":"Davidsen MK, Krogstie J (2010) A longitudinal study of development and maintenance. Inform Software Tech 52(7):707\u2013719","journal-title":"Inform Software Tech"},{"key":"9227_CR15","doi-asserted-by":"crossref","first-page":"833","DOI":"10.1016\/j.infsof.2008.01.006","volume":"50","author":"T Dyb\u00e5","year":"2008","unstructured":"Dyb\u00e5 T, Dings\u00f8yr T (2008) Empirical studies of agile software development: a systematic review. Inform Software Tech 50:833\u2013859","journal-title":"Inform Software Tech"},{"key":"9227_CR16","unstructured":"Easterbrook SM et al. (2007) Selecting Empirical Methods for Software Engineering Research.. In: F Shull, J Singer and D Sj\u00f8berg (eds.) Guide to Advanced Empirical Software Engineering. Springer, pp 285\u2013311"},{"key":"9227_CR17","unstructured":"Fran\u00e7a A C\u00e9sar C et al. (2010) The Effect of Reasoning Strategies on Success in Early Learning of Programming: Lessons Learned from an External Experiment Replication. In EASE\u20192010: 14th International Conference on Evaluation and Assessment in Software Engineering, Keele University, UK. 10"},{"key":"9227_CR18","doi-asserted-by":"crossref","unstructured":"G\u00f3mez G, Omar S, Juristo N, Vegas N (2010a) Replication, Reproduction and Re-analysis: Three ways for verifying experimental findings. In RESER\u20192010: Proceedings of the 1st International Workshop on Replication in Empirical Software Engineering Research. Cape Town, South Africa. pp 42\u201344","DOI":"10.1145\/1838687.1838698"},{"key":"9227_CR19","doi-asserted-by":"crossref","unstructured":"G\u00f3mez G, Omar S, Juristo N, Vegas N (2010b) Replications Types in Experimental Disciplines. In ESEM\u20192010: Proceedings of the ACM\/IEEE 4th International Symposium on Empirical Software Engineering and Measurement, September 16\u201317, Bolzano-Bozen, Italy. pp. 1\u201310","DOI":"10.1145\/1852786.1852790"},{"key":"9227_CR20","volume-title":"A dictionary of the social sciences","year":"1964","unstructured":"Gould J, Kolb WL (eds) (1964) A dictionary of the social sciences. Tavistock Publications, London, 761"},{"key":"9227_CR21","doi-asserted-by":"crossref","first-page":"687","DOI":"10.1016\/S0950-5849(00)00111-7","volume":"42","author":"KK Holgeid","year":"2000","unstructured":"Holgeid KK, Krogstie J, Sj\u00f8berg DIK (2000) A study of development and maintenance in Norway: assessing the efficiency of information systems support using functional maintenance. Inform Software Tech 42:687\u2013700","journal-title":"Inform Software Tech"},{"key":"9227_CR22","doi-asserted-by":"crossref","unstructured":"Juristo N, Vegas S (2009) Using differences among replications of software engineering experiments to gain knowledge. In ESEM\u201909: Proceedings of the ACM\/IEEE 3rd International Symposium on Empirical Software Engineering and Measurement. IEEE Computer Society, Washington, DC, USA, pp 356\u2013366","DOI":"10.1109\/ESEM.2009.5314236"},{"key":"9227_CR23","doi-asserted-by":"crossref","first-page":"219","DOI":"10.1007\/s10664-008-9061-0","volume":"13","author":"B Kitchenham","year":"2008","unstructured":"Kitchenham B (2008) The role of replications in empirical software engineering\u2014a word of warning. Empir Software Eng 13:219\u2013221","journal-title":"Empir Software Eng"},{"key":"9227_CR24","unstructured":"Kitchenham B, Charters S (2007) Guidelines for performing systematic literature reviews in software engineering, Technical Report EBSE-2007-01, School of Computer Science and Mathematics, Keele University"},{"key":"9227_CR25","doi-asserted-by":"crossref","unstructured":"Kitchenham BA, Pfleeger SL (2007) Personal Opinion Surveys. In: F Shull, J Singer, D. Sj\u00f8berg (eds), pp. 63\u201392, Guide to Advanced Empirical Software Engineering, Springer","DOI":"10.1007\/978-1-84800-044-5_3"},{"key":"9227_CR26","doi-asserted-by":"crossref","unstructured":"Kitchenham B, Dyb\u00e5 T, J\u00f8rgensen M (2004) Evidence-based Software Engineering. In ICSE\u20192004: Proceedings of the 26th International Conference on Software Engineering, Washington DC, USA. pp 273\u2013281","DOI":"10.1109\/ICSE.2004.1317449"},{"key":"9227_CR27","doi-asserted-by":"crossref","first-page":"792","DOI":"10.1016\/j.infsof.2010.03.006","volume":"52","author":"B Kitchenham","year":"2010","unstructured":"Kitchenham B et al (2010) Literature reviews in software engineering\u2014a tertiary study. Inform Software Tech 52:792\u2013805","journal-title":"Inform Software Tech"},{"key":"9227_CR28","unstructured":"Krein Jonathan L, Knutson Charles D (2010) A Case for Replication: Synthesizing Research Methodologies in Software Engineering. In RESER\u20192010: Proceedings of the 1st International Workshop on Replication in Empirical Software Engineering Research, Cape Town, South Africa. 10"},{"key":"9227_CR29","doi-asserted-by":"crossref","unstructured":"Krogstie J, S\u00f8lvberg A (1994) Software Maintenance in Norway: a survey investigation. In ICSM\u20191994: Proceedings of the International Conference on Software Maintenance. pp 304\u2013313","DOI":"10.1109\/ICSM.1994.336764"},{"key":"9227_CR30","doi-asserted-by":"crossref","first-page":"993","DOI":"10.1016\/j.infsof.2005.12.008","volume":"48","author":"J Krogstie","year":"2006","unstructured":"Krogstie J, Jahr A, Sj\u00f8berg DIK (2006) A longitudinal study of development and maintenance in Norway: report from the 2003 investigation. Inform Softw Technol 48:993\u20131005","journal-title":"Inform Softw Technol"},{"issue":"2","key":"9227_CR31","doi-asserted-by":"crossref","first-page":"219","DOI":"10.1111\/j.1533-8525.1972.tb00805.x","volume":"13","author":"MA Sorte La","year":"1972","unstructured":"La Sorte MA (1972) Replication as a verification technique in survey research: a paradigm. Socio Q 13(2):219\u2013227","journal-title":"Socio Q"},{"issue":"3","key":"9227_CR32","doi-asserted-by":"crossref","first-page":"217","DOI":"10.1080\/00031305.1993.10475983","volume":"47","author":"RM Lindsay","year":"1993","unstructured":"Lindsay RM, Ehrenberg A (1993) The design of replicated studies. Am Stat 47(3):217\u2013228","journal-title":"Am Stat"},{"key":"9227_CR33","doi-asserted-by":"crossref","unstructured":"Lung J et al. (2008) On the difficulty of replicating human subjects studies in software engineering. In ICSE\u20192008: Proceedings of the 13th international conference on Software engineering, New York, USA: ACM Press. pp 191\u2013201","DOI":"10.1145\/1368088.1368115"},{"key":"9227_CR34","doi-asserted-by":"crossref","unstructured":"Petticrew M, Roberts H (2006) Systematic Reviews in the Social Sciences. Blackwell Publishing. 336","DOI":"10.1002\/9780470754887"},{"key":"9227_CR35","doi-asserted-by":"crossref","unstructured":"Popper K (1959) The Logic of Scientific Discovery. Hutchinson & Co. 513","DOI":"10.1063\/1.3060577"},{"key":"9227_CR36","doi-asserted-by":"crossref","first-page":"90","DOI":"10.1037\/a0015108","volume":"13","author":"S Schmidt","year":"2009","unstructured":"Schmidt S (2009) Shall we really do it again? The powerful concept of replication is neglected in the social sciences. Rev Gen Psychol 13:90\u2013100. doi: 10.1037\/a0015108","journal-title":"Rev Gen Psychol"},{"key":"9227_CR37","doi-asserted-by":"crossref","unstructured":"Shull F, Basili V, Carver J, Maldonado JC, Travassos GH, Mendon\u00e7a M, Fabbri S (2002) Replicating software engineering experiments: Addressing the tacit knowledge problem. In ISESE\u20192002: Proc. Int. Symp. on Empirical Softw. Eng., Washington, DC, USA, IEEE Computer Society. 10","DOI":"10.1109\/ISESE.2002.1166920"},{"key":"9227_CR38","doi-asserted-by":"crossref","first-page":"211","DOI":"10.1007\/s10664-008-9060-1","volume":"13","author":"F Shull","year":"2008","unstructured":"Shull F, Carver J, Vegas S, Juristo N (2008) The Role of Replications in Empirical Software Engineering. Empir Software Eng 13:211\u2013218","journal-title":"Empir Software Eng"},{"issue":"9","key":"9227_CR39","doi-asserted-by":"crossref","first-page":"62","DOI":"10.1145\/1810891.1810911","volume":"53","author":"D Sj\u00f8berg","year":"2010","unstructured":"Sj\u00f8berg D (2010) Confronting the myth of rapid obsolescence in computing research. Commun ACM 53(9):62\u201367","journal-title":"Commun ACM"},{"key":"9227_CR40","doi-asserted-by":"crossref","first-page":"733","DOI":"10.1109\/TSE.2005.97","volume":"31","author":"D Sj\u00f8berg","year":"2005","unstructured":"Sj\u00f8berg D et al (2005) A survey of controlled experiments in software engineering. IEEE Trans Software Eng 31:733\u2013753","journal-title":"IEEE Trans Software Eng"},{"key":"9227_CR41","doi-asserted-by":"crossref","unstructured":"Vegas S et al. (2006) Analysis of the Influence of Communication between Researchers on Experiment Replication. In ISESE\u20192006: Proceedings of the 5th International Symposium on Empirical Software Engineering. September 20\u201321, Rio de Janeiro, Brazil. pp 28\u201337","DOI":"10.1145\/1159733.1159741"},{"key":"9227_CR42","volume-title":"Case study research: Design and methods","author":"RK Yin","year":"2009","unstructured":"Yin RK (2009) Case study research: Design and methods, 4th edn. Sage Publications, London, 240","edition":"4"},{"issue":"6","key":"9227_CR43","doi-asserted-by":"crossref","first-page":"625","DOI":"10.1016\/j.infsof.2010.12.010","volume":"53","author":"H Zhang","year":"2010","unstructured":"Zhang H, Babarb MA, Tell P (2010) Identifying relevant studies in software engineering. Inform Software Tech 53(6):625\u2013637, http:\/\/dx.doi.org\/10.1016\/j.infsof.2010.12.010","journal-title":"Inform Software Tech"}],"container-title":["Empirical Software Engineering"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s10664-012-9227-7.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/article\/10.1007\/s10664-012-9227-7\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s10664-012-9227-7","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,4,7]],"date-time":"2025-04-07T16:14:37Z","timestamp":1744042477000},"score":1,"resource":{"primary":{"URL":"http:\/\/link.springer.com\/10.1007\/s10664-012-9227-7"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2012,9,1]]},"references-count":43,"alternative-id":["9227"],"URL":"https:\/\/doi.org\/10.1007\/s10664-012-9227-7","relation":{},"ISSN":["1382-3256","1573-7616"],"issn-type":[{"value":"1382-3256","type":"print"},{"value":"1573-7616","type":"electronic"}],"subject":[],"published":{"date-parts":[[2012,9,1]]}}}