{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,2,18]],"date-time":"2026-02-18T22:58:52Z","timestamp":1771455532287,"version":"3.50.1"},"reference-count":58,"publisher":"Springer Science and Business Media LLC","issue":"3","license":[{"start":{"date-parts":[[2021,3,23]],"date-time":"2021-03-23T00:00:00Z","timestamp":1616457600000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"},{"start":{"date-parts":[[2021,3,23]],"date-time":"2021-03-23T00:00:00Z","timestamp":1616457600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"}],"funder":[{"DOI":"10.13039\/100014440","name":"Ministerio de Ciencia, Innovaci\u00f3n y Universidades","doi-asserted-by":"publisher","award":["PGC2018-097265-B-I00"],"award-info":[{"award-number":["PGC2018-097265-B-I00"]}],"id":[{"id":"10.13039\/100014440","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Empir Software Eng"],"published-print":{"date-parts":[[2021,5]]},"DOI":"10.1007\/s10664-020-09895-8","type":"journal-article","created":{"date-parts":[[2021,3,23]],"date-time":"2021-03-23T13:02:43Z","timestamp":1616504563000},"update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":28,"title":["A family of experiments on test-driven development"],"prefix":"10.1007","volume":"26","author":[{"given":"Adrian","family":"Santos","sequence":"first","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0001-8535-9386","authenticated-orcid":false,"given":"Sira","family":"Vegas","sequence":"additional","affiliation":[]},{"given":"Oscar","family":"Dieste","sequence":"additional","affiliation":[]},{"given":"Fernando","family":"Uyaguari","sequence":"additional","affiliation":[]},{"given":"Ay\u015fe","family":"Tosun","sequence":"additional","affiliation":[]},{"given":"Davide","family":"Fucci","sequence":"additional","affiliation":[]},{"given":"Burak","family":"Turhan","sequence":"additional","affiliation":[]},{"given":"Giuseppe","family":"Scanniello","sequence":"additional","affiliation":[]},{"given":"Simone","family":"Romano","sequence":"additional","affiliation":[]},{"given":"Itir","family":"Karac","sequence":"additional","affiliation":[]},{"given":"Marco","family":"Kuhrmann","sequence":"additional","affiliation":[]},{"given":"Vladimir","family":"Mandi\u0107","sequence":"additional","affiliation":[]},{"given":"Robert","family":"Rama\u010d","sequence":"additional","affiliation":[]},{"given":"Dietmar","family":"Pfahl","sequence":"additional","affiliation":[]},{"given":"Christian","family":"Engblom","sequence":"additional","affiliation":[]},{"given":"Jarno","family":"Kyykka","sequence":"additional","affiliation":[]},{"given":"Kerli","family":"Rungi","sequence":"additional","affiliation":[]},{"given":"Carolina","family":"Palomeque","sequence":"additional","affiliation":[]},{"given":"Jaroslav","family":"Spisak","sequence":"additional","affiliation":[]},{"given":"Markku","family":"Oivo","sequence":"additional","affiliation":[]},{"given":"Natalia","family":"Juristo","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2021,3,23]]},"reference":[{"key":"9895_CR1","unstructured":"Astels D (2003) Test driven development: A practical guide. Prentice Hall Professional Technical Reference"},{"key":"9895_CR2","doi-asserted-by":"crossref","unstructured":"Baltes S, Diehl S (2018) Towards a theory of software development expertise. arXiv:1807.06087","DOI":"10.1145\/3236024.3236061"},{"key":"9895_CR3","unstructured":"Basili V R (1992) Software modeling and measurement: the goal\/question\/metric paradigm"},{"issue":"4","key":"9895_CR4","doi-asserted-by":"publisher","first-page":"456","DOI":"10.1109\/32.799939","volume":"25","author":"VR Basili","year":"1999","unstructured":"Basili V R, Shull F, Lanubile F (1999) Building knowledge through families of experiments. IEEE Trans Softw Eng 25(4):456\u2013473","journal-title":"IEEE Trans Softw Eng"},{"key":"9895_CR5","unstructured":"Beck K (2003) Test-driven development: by example. Addison-Wesley Professional"},{"issue":"12","key":"9895_CR6","doi-asserted-by":"publisher","first-page":"1163","DOI":"10.1109\/TSE.2014.2348997","volume":"40","author":"GR Bergersen","year":"2014","unstructured":"Bergersen GR, Sjoberg DIK, Dyba T (2014) Construction and validation of an instrument for measuring programming skill. IEEE Trans Softw Eng 40 (12):1163\u20131184","journal-title":"IEEE Trans Softw Eng"},{"key":"9895_CR7","doi-asserted-by":"crossref","unstructured":"Bertolino A (2007) Software testing research: Achievements, challenges, dreams. In: 2007 Future of Software Engineering. IEEE Computer Society, pp 85\u2013103","DOI":"10.1109\/FOSE.2007.25"},{"key":"9895_CR8","doi-asserted-by":"publisher","first-page":"45","DOI":"10.1016\/j.infsof.2016.02.004","volume":"74","author":"W Bissi","year":"2016","unstructured":"Bissi W, Neto A G S S, Emer M C F P (2016) The effects of test driven development on internal quality, external quality and productivity: A systematic review. Inf Softw Technol 74:45\u201354","journal-title":"Inf Softw Technol"},{"key":"9895_CR9","unstructured":"Borenstein M, Hedges L V, Higgins JPT, Rothstein H R (2011) Introduction to meta-analysis. Wiley"},{"key":"9895_CR10","doi-asserted-by":"crossref","unstructured":"Brown H, Prescott R (2014) Applied mixed models in medicine. Wiley","DOI":"10.1002\/9781118778210"},{"key":"9895_CR11","doi-asserted-by":"crossref","unstructured":"Causevic A, Sundmark D, Punnekkat S (2011) Factors limiting industrial adoption of test driven development: A systematic review. In: 2011 IEEE Fourth International Conference on Software Testing, Verification and Validation (ICST). IEEE, pp 337\u2013346","DOI":"10.1109\/ICST.2011.19"},{"issue":"2","key":"9895_CR12","doi-asserted-by":"publisher","first-page":"165","DOI":"10.1037\/a0015565","volume":"14","author":"H Cooper","year":"2009","unstructured":"Cooper H, Patall E A (2009) The relative benefits of meta-analysis conducted with individual participant data versus aggregated data. Psychol Methods 14(2):165","journal-title":"Psychol Methods"},{"key":"9895_CR13","doi-asserted-by":"crossref","unstructured":"Cumming G (2013) Understanding the new statistics: Effect sizes, confidence intervals, and meta-analysis. Routledge","DOI":"10.4324\/9780203807002"},{"key":"9895_CR14","unstructured":"de Winter JCF (2013) Using the student\u2019s t-test with extremely small sample sizes. Pract Assess Res Eval 18(10). [Online; accessed 28-August-2018]"},{"issue":"5","key":"9895_CR15","doi-asserted-by":"publisher","first-page":"2457","DOI":"10.1007\/s10664-016-9471-3","volume":"22","author":"O Dieste","year":"2017","unstructured":"Dieste O, Aranda A M, Uyaguari F, Turhan B, Tosun A, Fucci D, Oivo M, Juristo N (2017) Empirical evaluation of the effects of experience on code quality and programmer productivity: an exploratory study. Empir Softw Eng 22(5):2457\u20132542","journal-title":"Empir Softw Eng"},{"issue":"1","key":"9895_CR16","doi-asserted-by":"publisher","first-page":"452","DOI":"10.1007\/s10664-017-9523-3","volume":"23","author":"D Falessi","year":"2018","unstructured":"Falessi D, Juristo N, Wohlin C, Turhan B, M\u00fcnch J, Jedlitschka A, Oivo M (2018) Empirical software engineering experts on the use of students and professionals in experiments. Empir Softw Eng 23(1):452\u2013489","journal-title":"Empir Softw Eng"},{"key":"9895_CR17","doi-asserted-by":"crossref","unstructured":"Feigenspan J, K\u00e4stner C, Liebig J, Apel S, Hanenberg S (2012) Measuring programming experience. In: 2012 IEEE 20th International Conference on Program Comprehension (ICPC). IEEE, pp 73\u201382","DOI":"10.1109\/ICPC.2012.6240511"},{"key":"9895_CR18","unstructured":"Field A (2013) Discovering statistics using ibm spss statistics. Sage"},{"issue":"9","key":"9895_CR19","doi-asserted-by":"publisher","first-page":"949","DOI":"10.1016\/j.jclinepi.2010.11.016","volume":"64","author":"DJ Fisher","year":"2011","unstructured":"Fisher DJ, Copas AJ, Tierney JF, Parmar MKB (2011) A critical review of methods for the assessment of patient-level interactions in individual participant data meta-analysis of randomized trials, and guidance for practitioners. J Clin Epidemiol 64(9):949\u2013967","journal-title":"J Clin Epidemiol"},{"issue":"7","key":"9895_CR20","doi-asserted-by":"publisher","first-page":"597","DOI":"10.1109\/TSE.2016.2616877","volume":"43","author":"D Fucci","year":"2017","unstructured":"Fucci D, Erdogmus H, Turhan B, Oivo M, Juristo N (2017) A dissection of the test-driven development process: does it really matter to test-first or to test-last?. IEEE Trans Softw Eng 43(7):597\u2013614","journal-title":"IEEE Trans Softw Eng"},{"issue":"8","key":"9895_CR21","doi-asserted-by":"publisher","first-page":"1033","DOI":"10.1016\/j.infsof.2014.04.004","volume":"56","author":"OS G\u00f3mez","year":"2014","unstructured":"G\u00f3mez O S, Juristo N, Vegas S (2014) Understanding replication of experiments in software engineering: A classification. Inf Softw Technol 56 (8):1033\u20131048","journal-title":"Inf Softw Technol"},{"issue":"7695","key":"9895_CR22","doi-asserted-by":"publisher","first-page":"175","DOI":"10.1038\/nature25753","volume":"555","author":"J Gurevitch","year":"2018","unstructured":"Gurevitch J, Koricheva J, Nakagawa S, Stewart G (2018) Meta-analysis and the science of research synthesis. Nature 555(7695):175","journal-title":"Nature"},{"key":"9895_CR23","doi-asserted-by":"crossref","unstructured":"Higgins JPT, Green S, et al. (2008) Cochrane handbook for systematic reviews of interventions, vol 5. Wiley Online Library","DOI":"10.1002\/9780470712184"},{"key":"9895_CR24","unstructured":"ISO\/IEC 25010:2011 (2011) https:\/\/www.iso.org\/obp\/ui\/#iso:std:iso-iec:25010:ed-1:v1:en"},{"issue":"5","key":"9895_CR25","doi-asserted-by":"publisher","first-page":"649","DOI":"10.1109\/TSE.2010.62","volume":"37","author":"Y Jia","year":"2011","unstructured":"Jia Y, Harman M (2011) An analysis and survey of the development of mutation testing. IEEE Trans Softw Eng 37(5):649\u2013678","journal-title":"IEEE Trans Softw Eng"},{"key":"9895_CR26","doi-asserted-by":"crossref","unstructured":"Jung J, Hoefig K, Domis D, Jedlitschka A, Hiller M (2013) Experimental comparison of two safety analysis methods and its replication. In: 2013 ACM\/IEEE International Symposium on Empirical Software Engineering and Measurement. IEEE, pp 223\u2013232","DOI":"10.1109\/ESEM.2013.59"},{"key":"9895_CR27","doi-asserted-by":"crossref","unstructured":"Juristo N, Moreno A M (2001) Basics of software engineering experimentation. Springer Science & Business Media","DOI":"10.1007\/978-1-4757-3304-4"},{"key":"9895_CR28","doi-asserted-by":"crossref","unstructured":"Juristo N, Vegas S (2009) Using differences among replications of software engineering experiments to gain knowledge. In: Proceedings of the 2009 3rd International Symposium on Empirical Software Engineering and Measurement. IEEE Computer Society, pp 356\u2013366","DOI":"10.1109\/ESEM.2009.5314236"},{"issue":"4","key":"9895_CR29","doi-asserted-by":"publisher","first-page":"81","DOI":"10.1109\/MS.2018.2801554","volume":"35","author":"I Karac","year":"2018","unstructured":"Karac I, Turhan B (2018) What do we (really) know about test-driven development?. IEEE Softw 35(4):81\u201385","journal-title":"IEEE Softw"},{"key":"9895_CR30","doi-asserted-by":"crossref","unstructured":"Karac E I, Turhan B, Juristo N (2019) A controlled experiment with novice developers on the impact of task description granularity on software quality in test-driven development. IEEE Transactions on Software Engineering","DOI":"10.1109\/TSE.2019.2920377"},{"issue":"2","key":"9895_CR31","doi-asserted-by":"publisher","first-page":"219","DOI":"10.1007\/s10664-008-9061-0","volume":"13","author":"B Kitchenham","year":"2008","unstructured":"Kitchenham B (2008) The role of replications in empirical software engineering, a word of warning. Empir Softw Eng 13(2):219\u2013221","journal-title":"Empir Softw Eng"},{"key":"9895_CR32","unstructured":"Kollanus S (2010) Test-driven development-still a promising approach? In: Quality of Information and Communications Technology (QUATIC), 2010 Seventh International Conference on the. , pp 403\u2013408"},{"issue":"6","key":"9895_CR33","doi-asserted-by":"publisher","first-page":"1121","DOI":"10.1037\/0022-3514.77.6.1121","volume":"77","author":"J Kruger","year":"1999","unstructured":"Kruger J, Dunning D (1999) Unskilled and unaware of it: how difficulties in recognizing one\u2019s own incompetence lead to inflated self-assessments. J Person Soc Psychol 77(6):1121","journal-title":"J Person Soc Psychol"},{"issue":"9096","key":"9895_CR34","doi-asserted-by":"publisher","first-page":"123","DOI":"10.1016\/S0140-6736(97)08468-7","volume":"351","author":"J Lau","year":"1998","unstructured":"Lau J, Ioannidis John PA, Schmid C H (1998) Summing up evidence: one answer is not always enough. Lancet 351(9096):123\u2013127","journal-title":"Lancet"},{"issue":"1","key":"9895_CR35","doi-asserted-by":"publisher","first-page":"151","DOI":"10.1146\/annurev.publhealth.23.100901.140546","volume":"23","author":"T Lumley","year":"2002","unstructured":"Lumley T, Diehr P, Emerson S, Chen L (2002) The importance of the normality assumption in large public health data sets. Ann Rev Public Health 23(1):151\u2013169","journal-title":"Ann Rev Public Health"},{"key":"9895_CR36","doi-asserted-by":"crossref","unstructured":"M\u00e4kinen S, M\u00fcnch J (2014) Effects of test-driven development: A comparative analysis of empirical studies. In: International Conference on Software Quality. Springer, pp 155\u2013169","DOI":"10.1007\/978-3-319-03602-1_10"},{"key":"9895_CR37","unstructured":"Martin CR (2001) Advanced principles, patterns and process of software development. Prentice Hall"},{"issue":"4","key":"9895_CR38","doi-asserted-by":"publisher","first-page":"375","DOI":"10.1016\/j.infsof.2014.01.002","volume":"56","author":"H Munir","year":"2014","unstructured":"Munir H, Moayyed M, Petersen K (2014) Considering rigor and relevance when evaluating test driven development: A systematic review. Inf Softw Technol 56(4):375\u2013394","journal-title":"Inf Softw Technol"},{"key":"9895_CR39","doi-asserted-by":"crossref","unstructured":"Myers G J, Sandler C, Badgett T (2011) The art of software testing. Wiley","DOI":"10.1002\/9781119202486"},{"issue":"5","key":"9895_CR40","doi-asserted-by":"publisher","first-page":"625","DOI":"10.1007\/s10459-010-9222-y","volume":"15","author":"G Norman","year":"2010","unstructured":"Norman G (2010) Likert scales, levels of measurement and the laws of statistics. Adv Health Sci Educ 15(5):625\u2013632","journal-title":"Adv Health Sci Educ"},{"issue":"4","key":"9895_CR41","doi-asserted-by":"publisher","first-page":"e1670","DOI":"10.1002\/stvr.1670","volume":"28","author":"J Offutt","year":"2018","unstructured":"Offutt J (2018) Why don\u2019t we publish more TDD research papers?. Softw Test Verif Reliab 28(4):e1670","journal-title":"Softw Test Verif Reliab"},{"key":"9895_CR42","doi-asserted-by":"crossref","unstructured":"Quinn G P, Keough M J (2002) Experimental design and data analysis for biologists. Cambridge University Press","DOI":"10.1017\/CBO9780511806384"},{"issue":"6","key":"9895_CR43","doi-asserted-by":"publisher","first-page":"835","DOI":"10.1109\/TSE.2012.28","volume":"39","author":"Y Rafique","year":"2013","unstructured":"Rafique Y, Mi\u0161i\u0107 V B (2013) The effects of test-driven development on external quality and productivity: A meta-analysis. IEEE Trans Softw Eng 39(6):835\u2013856","journal-title":"IEEE Trans Softw Eng"},{"key":"9895_CR44","doi-asserted-by":"publisher","first-page":"c221","DOI":"10.1136\/bmj.c221","volume":"340","author":"RD Riley","year":"2010","unstructured":"Riley R D, Lambert P C, Abo-Zaid G (2010) Meta-analysis of individual participant data: rationale, conduct, and reporting. Bmj 340:c221","journal-title":"Bmj"},{"key":"9895_CR45","doi-asserted-by":"crossref","unstructured":"Rosenthal R (1991) Meta-analytic procedures for social research, vol 6. Sage","DOI":"10.4135\/9781412984997"},{"key":"9895_CR46","doi-asserted-by":"publisher","unstructured":"Santos A, Gomez O S, Juristo N (2018a) Analyzing families of experiments in SE: a systematic mapping study. IEEE Trans Softw Eng:1. https:\/\/doi.org\/10.1109\/TSE.2018.2864633","DOI":"10.1109\/TSE.2018.2864633"},{"key":"9895_CR47","doi-asserted-by":"crossref","unstructured":"Santos A, Jarvinen J, Partanen J, Oivo M, Juristo N (2018b) Does the performance of tdd hold across software companies and premises? a group of industrial experiments on tdd. In: International Conference on Product-Focused Software Process Improvement. Springer, pp 227\u2013242","DOI":"10.1007\/978-3-030-03673-7_17"},{"key":"9895_CR48","unstructured":"Santos A, Vegas S, Oivo M, Juristo N (2018c) Guidelines for analyzing families of experiments in SE. Submitted to IEEE Transactions on Software Engineering"},{"key":"9895_CR49","doi-asserted-by":"crossref","unstructured":"Schmider E, Ziegler M, Danay E, Beyer L, B\u00fchner M (2010) Is it really robust? Methodology","DOI":"10.1027\/1614-2241\/a000016"},{"issue":"6","key":"9895_CR50","doi-asserted-by":"publisher","first-page":"16","DOI":"10.1109\/MS.2010.152","volume":"27","author":"F Shull","year":"2010","unstructured":"Shull F, Melnik G, Turhan B, Layman L, Diep M, Erdogmus H (2010) What do we know about test-driven development?. IEEE Softw 27(6):16\u201319","journal-title":"IEEE Softw"},{"key":"9895_CR51","unstructured":"Sj\u00f8berg D IK, Bergersen G R (2018) The price of using students comments on empirical software engineering experts on the use of students and professionals in experiments. CoRR, arXiv:1810.10791"},{"issue":"7","key":"9895_CR52","doi-asserted-by":"publisher","first-page":"e39471","DOI":"10.1371\/journal.pone.0039471","volume":"7","author":"K Thorlund","year":"2012","unstructured":"Thorlund K, Imberger G, Johnston B C, Walsh M, Awad T, Thabane L, Gluud C, Devereaux PJ, Wetterslev J (2012) Evolution of heterogeneity (i2) estimates and their 95% confidence intervals in large meta-analyses. PloS One 7(7):e39471","journal-title":"PloS One"},{"issue":"6","key":"9895_CR53","doi-asserted-by":"publisher","first-page":"2763","DOI":"10.1007\/s10664-016-9490-0","volume":"22","author":"A Tosun","year":"2017","unstructured":"Tosun A, Dieste O, Fucci D, Vegas S, Turhan B, Erdogmus H, Santos A, Oivo M, Toro K, Jarvinen J et al (2017) An industry experiment on the effects of test-driven development on external quality and productivity. Empir Softw Eng 22(6):2763\u20132805","journal-title":"Empir Softw Eng"},{"key":"9895_CR54","unstructured":"Tosun A, Dieste O, Vegas S, Pfahl D, Rungi K, Juristo N (In press) Investigating the impact of development task on external quality in test-driven development: An industry experiment. IEEE Transactions on Software Engineering"},{"key":"9895_CR55","doi-asserted-by":"crossref","unstructured":"Vegas S, Dieste O, Juristo N (2015) Difficulties in running experiments in the software industry: experiences from the trenches. In: Proceedings of the Third International Workshop on Conducting Empirical Studies in Industry at ICSE. IEEE Press, pp 3\u20139","DOI":"10.1109\/CESI.2015.8"},{"issue":"1","key":"9895_CR56","doi-asserted-by":"publisher","first-page":"35","DOI":"10.1186\/1471-2288-5-35","volume":"5","author":"AJ Vickers","year":"2005","unstructured":"Vickers A J (2005) Parametric versus non-parametric statistics in the analysis of randomized trials with non-normally distributed data. BMC Med Res Methodol 5(1):35","journal-title":"BMC Med Res Methodol"},{"key":"9895_CR57","unstructured":"Williams L, Kessler R (2002) Pair programming illuminated. Addison-Wesley Longman Publishing Co., Inc."},{"key":"9895_CR58","doi-asserted-by":"crossref","unstructured":"Wohlin C, Runeson P, H\u00f6st M, Ohlsson M C, Regnell B, Wessl\u00e9n A (2012) Experimentation in software engineering. Springer Science & Business Media","DOI":"10.1007\/978-3-642-29044-2"}],"container-title":["Empirical Software Engineering"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10664-020-09895-8.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s10664-020-09895-8\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10664-020-09895-8.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2021,4,25]],"date-time":"2021-04-25T07:46:49Z","timestamp":1619336809000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s10664-020-09895-8"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2021,3,23]]},"references-count":58,"journal-issue":{"issue":"3","published-print":{"date-parts":[[2021,5]]}},"alternative-id":["9895"],"URL":"https:\/\/doi.org\/10.1007\/s10664-020-09895-8","relation":{},"ISSN":["1382-3256","1573-7616"],"issn-type":[{"value":"1382-3256","type":"print"},{"value":"1573-7616","type":"electronic"}],"subject":[],"published":{"date-parts":[[2021,3,23]]},"assertion":[{"value":"27 November 2020","order":1,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"23 March 2021","order":2,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}}],"article-number":"42"}}