{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,6,10]],"date-time":"2026-06-10T12:01:15Z","timestamp":1781092875417,"version":"3.54.1"},"reference-count":46,"publisher":"Springer Science and Business Media LLC","issue":"1","license":[{"start":{"date-parts":[[2019,7,30]],"date-time":"2019-07-30T00:00:00Z","timestamp":1564444800000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0"},{"start":{"date-parts":[[2019,7,30]],"date-time":"2019-07-30T00:00:00Z","timestamp":1564444800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0"}],"funder":[{"DOI":"10.13039\/501100004569","name":"Ministerstwo Nauki i Szkolnictwa Wy\u017bszego","doi-asserted-by":"publisher","award":["Wroclaw University of Science and Technology Grant 0401\/0201\/18"],"award-info":[{"award-number":["Wroclaw University of Science and Technology Grant 0401\/0201\/18"]}],"id":[{"id":"10.13039\/501100004569","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Empir Software Eng"],"published-print":{"date-parts":[[2020,1]]},"DOI":"10.1007\/s10664-019-09747-0","type":"journal-article","created":{"date-parts":[[2019,7,30]],"date-time":"2019-07-30T17:02:31Z","timestamp":1564506151000},"page":"353-401","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":27,"title":["Meta-analysis for families of experiments in software engineering: a systematic review and reproducibility and validity assessment"],"prefix":"10.1007","volume":"25","author":[{"given":"Barbara","family":"Kitchenham","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-3907-3357","authenticated-orcid":false,"given":"Lech","family":"Madeyski","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Pearl","family":"Brereton","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2019,7,30]]},"reference":[{"issue":"16","key":"9747_CR1","doi-asserted-by":"publisher","first-page":"3356","DOI":"10.1016\/j.ins.2011.04.005","volume":"181","author":"S Abrah\u00e3o","year":"2011","unstructured":"Abrah\u00e3o S, Insfr\u00e1n E, Cars\u00ed JA, Genero M (2011) Evaluating requirements modeling methods based on user perceptions: a family of experiments. Inf Sci 181 (16):3356\u20133378","journal-title":"Inf Sci"},{"issue":"3","key":"9747_CR2","doi-asserted-by":"publisher","first-page":"327","DOI":"10.1109\/TSE.2012.27","volume":"39","author":"S Abrahao","year":"2013","unstructured":"Abrahao S, Gravino C, Insfran Pelozo E, Scanniello G, Tortora G (2013) Assessing the effectiveness of sequence diagrams in the comprehension of functional requirements: results from a family of five experiments. IEEE Trans Softw Eng 39 (3):327\u2013342","journal-title":"IEEE Trans Softw Eng"},{"issue":"4","key":"9747_CR3","doi-asserted-by":"publisher","first-page":"456","DOI":"10.1109\/32.799939","volume":"25","author":"VR Basili","year":"1999","unstructured":"Basili VR, Shull F, Lanubile F (1999) Building knowledge through families of experiments. IEEE Trans Softw Eng 25(4):456\u2013473","journal-title":"IEEE Trans Softw Eng"},{"key":"9747_CR4","unstructured":"BioStat (2006) Comprehensive meta-analysis (cma) v2.0. \nhttps:\/\/www.meta-analysis.com\/pages\/v2download.php"},{"key":"9747_CR5","doi-asserted-by":"publisher","DOI":"10.1002\/9780470743386","volume-title":"Introduction to meta-analysis","author":"M Borenstein","year":"2009","unstructured":"Borenstein M, Hedges LV, Higgins JPT, Rothstein HT (2009) Introduction to meta-analysis. Wiley, UK"},{"key":"9747_CR6","volume-title":"Design and analysis of bioavailability and bioequivalence studies","author":"S Chow","year":"1992","unstructured":"Chow S, Liu J (1992) Design and analysis of bioavailability and bioequivalence studies. Taylor-Francis, New York"},{"key":"9747_CR7","doi-asserted-by":"publisher","unstructured":"Ciolkowski M (2009) What do we know about perspective-based reading? An approach for quantitative aggregation in software engineering. In: Proceedings of the 2009 3rd international symposium on empirical software engineering and measurement. ESEM \u201909. \nhttps:\/\/doi.org\/10.1109\/ESEM.2009.5316026\n\n. IEEE Computer Society, Washington, DC, pp 133\u2013144","DOI":"10.1109\/ESEM.2009.5316026"},{"issue":"6","key":"9747_CR8","doi-asserted-by":"publisher","first-page":"685","DOI":"10.1007\/s10664-009-9106-z","volume":"14","author":"JA Cruz-Lemus","year":"2009","unstructured":"Cruz-Lemus JA, Genero M, Manso ME, Morasca S, Piattini M (2009) Assessing the understandability of UML statechart diagrams with composite states\u2014a family of empirical studies. Empir Softw Eng 14(6):685\u2013719","journal-title":"Empir Softw Eng"},{"issue":"12","key":"9747_CR9","doi-asserted-by":"publisher","first-page":"1391","DOI":"10.1016\/j.infsof.2011.07.002","volume":"53","author":"JA Cruz-Lemus","year":"2011","unstructured":"Cruz-Lemus JA, Genero M, Caivano D, Abrah\u00e3o S, Insfr\u00e1n E, Cars\u00ed JA (2011) Assessing the influence of stereotypes on the comprehension of UML sequence diagrams: a family of experiments. Inf Softw Technol 53(12):1391\u20131403","journal-title":"Inf Softw Technol"},{"key":"9747_CR10","volume-title":"Understanding the new statistics effect sizes, confidence intervals and meta-analysis","author":"G Cumming","year":"2012","unstructured":"Cumming G (2012) Understanding the new statistics effect sizes, confidence intervals and meta-analysis. Routledge, UK"},{"key":"9747_CR11","unstructured":"Dahl DB, Scott D, Roosen C, Magnusson A, Swinton J (2018) xtable: export tables to LaTeX or HTML. \nhttps:\/\/CRAN.R-project.org\/package=xtable\n\n, r package version 1.8-3"},{"issue":"1","key":"9747_CR12","doi-asserted-by":"publisher","first-page":"161","DOI":"10.1016\/j.jss.2012.07.043","volume":"86","author":"A Fernandez","year":"2013","unstructured":"Fernandez A, Abrah\u00e3o S, Insfran E (2013) Empirical validation of a usability inspection method for model-driven Web development. J Syst Softw 86(1):161\u2013186","journal-title":"J Syst Softw"},{"key":"9747_CR13","doi-asserted-by":"publisher","first-page":"644","DOI":"10.1016\/j.infsof.2014.05.014","volume":"57","author":"AM Fern\u00e1ndez-S\u00e1ez","year":"2015","unstructured":"Fern\u00e1ndez-S\u00e1ez AM, Genero M, Chaudronand MRV, Caivano D, Ramos I (2015) Are forward design or reverse-engineered UML diagrams more helpful for code maintenance?: a family of experiments. Inf Softw Technol 57:644\u2013663","journal-title":"Inf Softw Technol"},{"issue":"1","key":"9747_CR14","doi-asserted-by":"publisher","first-page":"212","DOI":"10.1007\/s10664-014-9354-4","volume":"21","author":"AM Fern\u00e1ndez-S\u00e1ez","year":"2016","unstructured":"Fern\u00e1ndez-S\u00e1ez AM, Genero M, Caivano D, Chaudron MRV (2016) Does the level of detail of UML diagrams affect the maintainability of source code?: a family of experiments. Empir Softw Eng 21(1):212\u2013259","journal-title":"Empir Softw Eng"},{"key":"9747_CR15","first-page":"1","volume":"1","author":"R Fisher","year":"1921","unstructured":"Fisher R (1921) On the probable error of a coefficient of correlation deduced from a small sample. Metron 1:1\u201332","journal-title":"Metron"},{"key":"9747_CR16","doi-asserted-by":"publisher","first-page":"405","DOI":"10.1016\/j.infsof.2014.05.018","volume":"57","author":"J Gonzalez-Huerta","year":"2015","unstructured":"Gonzalez-Huerta J, Insfr\u00e1n E, Abrah\u00e3o S M, Scanniello G (2015) Validating a model-driven software architecture evaluation and improvement method: a family of experiments. Inf Softw Technol 57:405\u2013429","journal-title":"Inf Softw Technol"},{"issue":"10","key":"9747_CR17","doi-asserted-by":"publisher","first-page":"1823","DOI":"10.1016\/j.infsof.2013.05.003","volume":"55","author":"I Hadar","year":"2013","unstructured":"Hadar I, Reinhartz-Berger I, Kuflik T, Perini A, Ricca F, Susi A (2013) Comparing the comprehensibility of requirements models expressed in Use Case and Tropos: results from a family of experiments. Inf Softw Technol 55(10):1823\u20131843","journal-title":"Inf Softw Technol"},{"key":"9747_CR18","volume-title":"Statistical methods for meta-analysis","author":"LV Hedges","year":"1985","unstructured":"Hedges LV, Olkin I (1985) Statistical methods for meta-analysis. Academic Press, Orlando"},{"issue":"11","key":"9747_CR19","doi-asserted-by":"publisher","first-page":"1539","DOI":"10.1002\/sim.1186","volume":"21","author":"JPT Higgins","year":"2002","unstructured":"Higgins JPT, Thompson SG (2002) Quantifying heterogeneity in a meta-analysis. Stat Med 21(11):1539\u20131558","journal-title":"Stat Med"},{"key":"9747_CR20","volume-title":"Methods of meta-analysis: correcting error and bias in research findings","author":"J Hunter","year":"1990","unstructured":"Hunter J, Schmidt F (1990) Methods of meta-analysis: correcting error and bias in research findings. Sage, Newbury Park"},{"issue":"3\u20134","key":"9747_CR21","doi-asserted-by":"publisher","first-page":"362","DOI":"10.1093\/biomet\/31.3-4.362","volume":"31","author":"NL Johnson","year":"1940","unstructured":"Johnson NL, Welch BL (1940) Applications of the non-central t-distribution. Biometrika 31(3\u20134):362\u2013389","journal-title":"Biometrika"},{"issue":"1","key":"9747_CR22","doi-asserted-by":"publisher","first-page":"21","DOI":"10.5277\/e-Inf150102","volume":"9","author":"M Jureczko","year":"2015","unstructured":"Jureczko M, Madeyski L (2015) Cross\u2013project defect prediction with respect to code ownership model: an empirical study. e-Informatica Softw Eng J 9(1):21\u201335. \nhttps:\/\/doi.org\/10.5277\/e-Inf150102","journal-title":"e-Informatica Softw Eng J"},{"key":"9747_CR23","doi-asserted-by":"crossref","DOI":"10.1201\/b19467","volume-title":"Evidence-based software engineering and systematic reviews","author":"B Kitchenham","year":"2015","unstructured":"Kitchenham B, Budgen D, Brereton P (2015) Evidence-based software engineering and systematic reviews. CRC Press, Boca Raton"},{"issue":"2","key":"9747_CR24","doi-asserted-by":"publisher","first-page":"579","DOI":"10.1007\/s10664-016-9437-5","volume":"22","author":"B Kitchenham","year":"2017","unstructured":"Kitchenham B, Madeyski L, Budgen D, Keung J, Brereton P, Charters S, Gibbs S, Pohthong A (2017) Robust statistical methods for empirical software engineering. Empir Softw Eng 22(2):579\u2013630","journal-title":"Empir Softw Eng"},{"issue":"2","key":"9747_CR25","doi-asserted-by":"publisher","first-page":"320","DOI":"10.1002\/sim.7379","volume":"37","author":"B Kitchenham","year":"2018","unstructured":"Kitchenham B, Madeyski L, Curtin F (2018) Corrections to effect size variances for continuous outcomes of cross-over clinical trials. Stat Med 37 (2):320\u2013323. \nhttps:\/\/doi.org\/10.1002\/sim.7379\n\n. \nhttp:\/\/madeyski.e-informatyka.pl\/download\/KitchenhamMadeyskiCurtinSIM.pdf","journal-title":"Stat Med"},{"key":"9747_CR26","doi-asserted-by":"publisher","unstructured":"Kitchenham B, Madeyski L, Brereton P (2019a) Problems with statistical practice in human-centric software engineering experiments. In: Proceedings of the 2019 international conference on evaluation and assessment in software engineering (EASE). \nhttps:\/\/doi.org\/10.1145\/3319008.3319009\n\n, pp 134\u2013143","DOI":"10.1145\/3319008.3319009"},{"key":"9747_CR27","doi-asserted-by":"crossref","unstructured":"Kitchenham B, Madeyski L, Brereton P (2019b) Supplementary materials for the paper \u201cMeta-analysis for families of experiments in software engineering: a systematic review and reproducibility and validity assessment\u201d. \nhttp:\/\/madeyski.e-informatyka.pl\/download\/KitchenhamMadeyskiBrereton19Supplement.pdf","DOI":"10.1007\/s10664-019-09747-0"},{"issue":"5","key":"9747_CR28","doi-asserted-by":"publisher","first-page":"387","DOI":"10.1109\/32.922713","volume":"27","author":"O Laitenberger","year":"2001","unstructured":"Laitenberger O, Emam KE, Harbich TG (2001) An internally replicated quasi-experimental comparison of checklist and perspective-based reading of code documents. IEEE Trans Softw Eng 27(5):387\u2013418","journal-title":"IEEE Trans Softw Eng"},{"issue":"Article 863","key":"9747_CR29","first-page":"1","volume":"4","author":"D Lakens","year":"2013","unstructured":"Lakens D (2013) Calculating and reporting effect sizes to facilitate cumulative science: a practical primer for t-tests and anovas. Front Psychol 4(Article 863):1\u201312","journal-title":"Front Psychol"},{"key":"9747_CR30","volume-title":"Practical mete-analysis","author":"MW Lipsey","year":"2001","unstructured":"Lipsey MW, Wilson DB (2001) Practical mete-analysis. Sage Publications Inc., UK"},{"issue":"3","key":"9747_CR31","doi-asserted-by":"publisher","first-page":"393","DOI":"10.1007\/s11219-014-9241-7","volume":"23","author":"L Madeyski","year":"2015","unstructured":"Madeyski L, Jureczko M (2015) Which process metrics can significantly improve defect prediction models? An empirical study. Softw Qual J 23(3):393\u2013422. \nhttps:\/\/doi.org\/10.1007\/s11219-014-9241-7","journal-title":"Softw Qual J"},{"issue":"2","key":"9747_CR32","doi-asserted-by":"publisher","first-page":"1509","DOI":"10.3233\/JIFS-169146","volume":"32","author":"L Madeyski","year":"2017","unstructured":"Madeyski L, Kitchenham B (2017) Would wider adoption of reproducible research be beneficial for empirical software engineering research J Intell Fuzzy Syst 32(2):1509\u20131521. \nhttps:\/\/doi.org\/10.3233\/JIFS-169146\n\n. \nhttp:\/\/madeyski.e-informatyka.pl\/download\/MadeyskiKitchenham17JIFS.pdf","journal-title":"J Intell Fuzzy Syst"},{"key":"9747_CR33","doi-asserted-by":"publisher","unstructured":"Madeyski L, Kitchenham B (2018a) Effect sizes and their variance for ab\/ba crossover design studies. In: Proceedings of the ACM\/IEEE 40th international conference on software engineering (May 27\u2013June 3, 2018). ACM, Gothenburg, p 420, DOI \nhttps:\/\/doi.org\/10.1145\/3180155.3182556\n\n, (to appear in Print)","DOI":"10.1145\/3180155.3182556"},{"issue":"4","key":"9747_CR36","doi-asserted-by":"publisher","first-page":"1982","DOI":"10.1007\/s10664-017-9574-5","volume":"23","author":"L Madeyski","year":"2018","unstructured":"Madeyski L, Kitchenham BA (2018b) Effect sizes and their variance for AB\/BA crossover design studies. Empir Softw Eng 23(4):1982\u20132017. \nhttps:\/\/doi.org\/10.1007\/s10664-017-9574-5","journal-title":"Empir Softw Eng"},{"key":"9747_CR34","unstructured":"Madeyski L, Kitchenham B (2019) Reproducer: reproduce statistical analyses and meta-analyses. \nhttp:\/\/madeyski.e-informatyka.pl\/reproducible-research\/\n\n, R package version 0.3.0 (\nhttp:\/\/CRAN.R-project.org\/package=reproducer\n\n)"},{"key":"9747_CR35","doi-asserted-by":"publisher","first-page":"82","DOI":"10.1016\/j.jss.2015.12.056","volume":"114","author":"JM Morales","year":"2016","unstructured":"Morales JM, Navarro E, S\u00e1nchez-Palma P, Alonso D (2016) A family of experiments to evaluate the understandability of TRiStar and i* for modeling teleo-reactive systems. J Syst Softw 114:82\u2013100","journal-title":"J Syst Softw"},{"key":"9747_CR37","doi-asserted-by":"publisher","first-page":"17","DOI":"10.1348\/000711000159150","volume":"53","author":"SB Morris","year":"2000","unstructured":"Morris S B (2000) Distribution of the standardized mean change effect size for meta-analysis on repeated measures. Br J Math Stat Psychol 53:17\u201329","journal-title":"Br J Math Stat Psychol"},{"issue":"1","key":"9747_CR38","doi-asserted-by":"publisher","first-page":"105","DOI":"10.1037\/\/1082-989X.7.1.105","volume":"7","author":"SB Morris","year":"2002","unstructured":"Morris S B, DeShon R P (2002) Combining effect size estimates in meta-analysis with repeated measures and independent-groups designs. Psychol Methods 7(1):105\u2013125. \nhttps:\/\/doi.org\/10.1037\/\/1082-989X.7.1.105","journal-title":"Psychol Methods"},{"issue":"2","key":"9747_CR39","doi-asserted-by":"publisher","first-page":"127","DOI":"10.1016\/S0950-5849(03)00115-0","volume":"46","author":"D Pfahl","year":"2004","unstructured":"Pfahl D, Laitenberger O, Ruhe G, Dorsch J, Krivobokova T (2004) Evaluating the learning effectiveness of using simulations in software project management education: results from a twice replicated experiment. Inf Softw Technol 46(2):127\u2013147","journal-title":"Inf Softw Technol"},{"key":"9747_CR40","doi-asserted-by":"publisher","DOI":"10.4135\/9781412984997","volume-title":"Meta-analytic procedures for social research","author":"R Rosenthal","year":"1991","unstructured":"Rosenthal R (1991) Meta-analytic procedures for social research. Sage, UK"},{"key":"9747_CR41","unstructured":"Santos A, G\u00f3mez OS, Juristo N (2018) Analyzing families of experiments in SE: a systematic mapping study. CoRR arXiv:\n1805.09009"},{"issue":"2","key":"9747_CR42","doi-asserted-by":"publisher","first-page":"13:1","DOI":"10.1145\/2491912","volume":"23","author":"G Scanniello","year":"2014","unstructured":"Scanniello G, Gravino C, Genero M, Cruz-Lemus J A, Tortora G (2014) On the impact of UML analysis models on source-code comprehensibility and modifiability. ACM Trans Softw Eng Methodol 23(2):13:1\u201313:26. \nhttps:\/\/doi.org\/10.1145\/2491912","journal-title":"ACM Trans Softw Eng Methodol"},{"key":"9747_CR43","doi-asserted-by":"publisher","DOI":"10.1002\/0470854596","volume-title":"Cross-over trials in clinical research, 2nd edn","author":"S Senn","year":"2002","unstructured":"Senn S (2002) Cross-over trials in clinical research, 2nd edn. Wiley, UK"},{"issue":"11","key":"9747_CR44","doi-asserted-by":"publisher","first-page":"1215","DOI":"10.1016\/j.infsof.2012.06.001","volume":"54","author":"MA Teruel","year":"2012","unstructured":"Teruel MA, Navarro E, L\u00f3pez-Jaquero V, Montero F, Jaen J, Gonz\u00e1lez P (2012) Analyzing the understandability of requirements engineering languages for CSCW systems: a family of experiments. Inf Softw Technol 54(11):1215\u20131228","journal-title":"Inf Softw Technol"},{"issue":"2","key":"9747_CR45","doi-asserted-by":"publisher","first-page":"120","DOI":"10.1109\/TSE.2015.2467378","volume":"42","author":"S Vegas","year":"2016","unstructured":"Vegas S, Apa C, Juristo N (2016) Crossover designs in software engineering experiments: benefits and perils. IEEE Trans Softw Eng 42(2):120\u2013135. \nhttps:\/\/doi.org\/10.1109\/TSE.2015.2467378","journal-title":"IEEE Trans Softw Eng"},{"issue":"3","key":"9747_CR46","doi-asserted-by":"publisher","first-page":"1","DOI":"10.18637\/jss.v036.i03","volume":"36","author":"W Viechtbauer","year":"2010","unstructured":"Viechtbauer W (2010) Conducting meta-analysis in R with the metafor package. J Stat Softw 36(3):1\u201348","journal-title":"J Stat Softw"}],"container-title":["Empirical Software Engineering"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s10664-019-09747-0.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/article\/10.1007\/s10664-019-09747-0\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s10664-019-09747-0.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2020,7,28]],"date-time":"2020-07-28T23:33:29Z","timestamp":1595979209000},"score":1,"resource":{"primary":{"URL":"http:\/\/link.springer.com\/10.1007\/s10664-019-09747-0"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2019,7,30]]},"references-count":46,"journal-issue":{"issue":"1","published-print":{"date-parts":[[2020,1]]}},"alternative-id":["9747"],"URL":"https:\/\/doi.org\/10.1007\/s10664-019-09747-0","relation":{},"ISSN":["1382-3256","1573-7616"],"issn-type":[{"value":"1382-3256","type":"print"},{"value":"1573-7616","type":"electronic"}],"subject":[],"published":{"date-parts":[[2019,7,30]]},"assertion":[{"value":"30 July 2019","order":1,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}}]}}