{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,8,26]],"date-time":"2026-08-26T19:13:04Z","timestamp":1787771584833,"version":"build-2784847793"},"reference-count":88,"publisher":"Springer Science and Business Media LLC","issue":"6","license":[{"start":{"date-parts":[[2023,11,1]],"date-time":"2023-11-01T00:00:00Z","timestamp":1698796800000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2023,11,1]],"date-time":"2023-11-01T00:00:00Z","timestamp":1698796800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Empir Software Eng"],"published-print":{"date-parts":[[2023,11]]},"DOI":"10.1007\/s10664-023-10370-3","type":"journal-article","created":{"date-parts":[[2023,11,13]],"date-time":"2023-11-13T08:02:01Z","timestamp":1699862521000},"update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":10,"title":["Operationalizing validity of empirical software engineering studies"],"prefix":"10.1007","volume":"28","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-7461-2320","authenticated-orcid":false,"given":"Johannes","family":"H\u00e4rtel","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Ralf","family":"L\u00e4mmel","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2023,11,13]]},"reference":[{"key":"10370_CR1","doi-asserted-by":"publisher","first-page":"199","DOI":"10.1007\/978-1-4612-1694-0_15","volume-title":"Selected papers of hirotugu akaike","author":"H Akaike","year":"1998","unstructured":"Akaike H (1998) Information theory and an extension of the maximum likelihood principle. In: Parzen E, Tanabe K, Kitagawa G (eds) Selected papers of hirotugu akaike. Springer, pp 199\u2013213"},{"key":"10370_CR2","doi-asserted-by":"crossref","unstructured":"Alali A, Kagdi HH, Maletic JI (2008) What\u2019s a typical commit? A characterization of open source software repositories. In: ICPC, pp 182\u2013191. IEEE Computer society","DOI":"10.1109\/ICPC.2008.24"},{"issue":"1","key":"10370_CR3","doi-asserted-by":"publisher","first-page":"241","DOI":"10.1007\/s10664-012-9221-0","volume":"19","author":"\u00d6 Albayrak","year":"2014","unstructured":"Albayrak \u00d6, Carver JC (2014) Investigation of individual factors impacting the effectiveness of requirements inspections: a replicated experiment. Empir Softw Eng 19(1):241\u2013266","journal-title":"Empir Softw Eng"},{"issue":"3","key":"10370_CR4","doi-asserted-by":"publisher","first-page":"285","DOI":"10.1007\/s10664-005-1289-3","volume":"10","author":"B Anda","year":"2005","unstructured":"Anda B, Sj\u00f8berg DIK (2005) Investigating the role of use cases in the construction of class diagrams. Empir Softw Eng 10(3):285\u2013309","journal-title":"Empir Softw Eng"},{"issue":"2","key":"10370_CR5","doi-asserted-by":"publisher","first-page":"378","DOI":"10.1007\/s10664-013-9267-7","volume":"19","author":"C Apa","year":"2014","unstructured":"Apa C, Dieste O, Espinosa GEG, Fonseca CER (2014) Effectiveness for detecting faults within and outside the scope of testing techniques: an independent replication. Empir Softw Eng 19(2):378\u2013417","journal-title":"Empir Softw Eng"},{"issue":"4","key":"10370_CR6","doi-asserted-by":"publisher","first-page":"390","DOI":"10.1016\/j.jml.2007.12.005","volume":"59","author":"RH Baayen","year":"2008","unstructured":"Baayen RH, Davidson DJ, Bates DM (2008) Mixed-effects modeling with crossed random effects for subjects and items. J Memory Lang 59(4):390\u2013412","journal-title":"J Memory Lang"},{"key":"10370_CR7","doi-asserted-by":"crossref","unstructured":"Bangash AA, Sahar H, Hindle A, Ali K (2020) On the time-based conclusion stability of cross-project defect prediction models. Empirical Software Engineering pp 1\u201338","DOI":"10.1007\/s10664-020-09878-9"},{"key":"10370_CR8","doi-asserted-by":"crossref","unstructured":"Bar\u00f3n MM, Wyrich M, Graziotin D, Wagner S (2023) Evidence profiles for validity threats in program comprehension experiments. In: ICSE, pp 1907\u20131919. IEEE","DOI":"10.1109\/ICSE48619.2023.00162"},{"issue":"3","key":"10370_CR9","doi-asserted-by":"publisher","first-page":"255","DOI":"10.1016\/j.jml.2012.11.001","volume":"368","author":"DJ Barr","year":"2013","unstructured":"Barr DJ, Levy R, Scheepers C, Tily HJ (2013) Random effects structure for confirmatory hypothesis testing: Keep it maximal. J Memory Lang 368(3):255\u2013278","journal-title":"J Memory Lang"},{"issue":"7866","key":"10370_CR10","doi-asserted-by":"publisher","first-page":"1476","DOI":"10.1038\/s41586-021-03655-4","volume":"595","author":"B Beheim","year":"2021","unstructured":"Beheim B, Atkinson QD, Bulbulia J, Gervais W, Gray RD, Henrich J, Lang M, Monroe MW, Muthukrishna M, Norenzayan A, Purzycki BG, Shariff A, Slingerland E, Spicer R, Willard AK (2021) Treatment of missing data determined conclusions regarding moralizing gods. Nature 595(7866):1476\u20134687","journal-title":"Nature"},{"key":"10370_CR11","doi-asserted-by":"publisher","first-page":"100070","DOI":"10.1016\/j.osnem.2020.100070","volume":"16","author":"NH Bidoki","year":"2020","unstructured":"Bidoki NH, Schiappa M, Sukthankar G, Garibay I (2020) Modeling social coding dynamics with sampled historical data. Online Soc Netw Med 16:100070","journal-title":"Online Soc Netw Med"},{"key":"10370_CR12","doi-asserted-by":"crossref","unstructured":"Bird C, Bachmann A, Aune E, Duffy J, Bernstein A, Filkov V, Devanbu PT (2009) Fair and balanced?: bias in bug-fix datasets. In: ESEC\/SIGSOFT FSE, pp 121\u2013130. ACM","DOI":"10.1145\/1595696.1595716"},{"key":"10370_CR13","doi-asserted-by":"crossref","unstructured":"Blythe J, Bollenbacher J, Huang D, Hui P, Krohn R, Pacheco D, Muric G, Sapienza A, Tregubov A, Ahn Y, Flammini A, Lerman K, Menczer F, Weninger T, Ferrara E (2019) Massive multi-agent data-driven simulations of the GitHub ecosystem. In: PAAMS, Lecture notes in computer science, vol 11523, pp 3\u201315. Springer","DOI":"10.1007\/978-3-030-24209-1_1"},{"issue":"8","key":"10370_CR14","first-page":"1315","volume":"53","author":"WF Boh","year":"2007","unstructured":"Boh WF, Slaughter S, Espinosa JA (2007) Learning from experience in software development: A multilevel analysis. Manag Sci 53(8):1315\u20131331","journal-title":"Manag Sci"},{"key":"10370_CR15","doi-asserted-by":"crossref","unstructured":"Borges H, Hora AC, Valente MT (2016) Predicting the popularity of GitHub repositories. In: PROMISE, pp 9:1\u20139:10. ACM","DOI":"10.1145\/2972958.2972966"},{"issue":"4","key":"10370_CR16","doi-asserted-by":"publisher","first-page":"1931","DOI":"10.1007\/s10664-017-9576-3","volume":"23","author":"NC Borle","year":"2018","unstructured":"Borle NC, Feghhi M, Stroulia E, Greiner R, Hindle A (2018) Analyzing the effects of test driven development in GitHub. Empir Softw Eng 23(4):1931\u20131958","journal-title":"Empir Softw Eng"},{"issue":"24","key":"10370_CR17","doi-asserted-by":"publisher","first-page":"4279","DOI":"10.1002\/sim.2673","volume":"25","author":"A Burton","year":"2006","unstructured":"Burton A, Altman DG, Royston P, Holder RL (2006) The design of simulation studies in medical statistics. Stat Med 25(24):4279\u20134292","journal-title":"Stat Med"},{"issue":"4","key":"10370_CR18","doi-asserted-by":"publisher","first-page":"426","DOI":"10.1002\/stvr.1570","volume":"25","author":"G Canfora","year":"2015","unstructured":"Canfora G, Lucia AD, Penta MD, Oliveto R, Panichella A, Panichella S (2015) Defect prediction as a multiobjective optimization problem. Softw Test Verification Reliab 25(4):426\u2013459","journal-title":"Softw Test Verification Reliab"},{"key":"10370_CR19","doi-asserted-by":"crossref","unstructured":"Casalnuovo C, Devanbu PT, Oliveira A, Filkov V, Ray B (2015) Assert use in GitHub projects. In: ICSE (1), pp 755\u2013766. IEEE Computer Society","DOI":"10.1109\/ICSE.2015.88"},{"key":"10370_CR20","doi-asserted-by":"crossref","unstructured":"Clyburne-Sherin A, Fei X, Green SA (2019) Computational reproducibility via containers in psychology. Meta-psychology 3","DOI":"10.15626\/MP.2018.892"},{"key":"10370_CR21","doi-asserted-by":"publisher","DOI":"10.4324\/9780203774441","volume-title":"Applied multiple regression\/correlation analysis for the behavioral sciences","author":"J Cohen","year":"2013","unstructured":"Cohen J, Cohen P, West SG, Aiken LS (2013) Applied multiple regression\/correlation analysis for the behavioral sciences. Routledge"},{"key":"10370_CR22","doi-asserted-by":"crossref","unstructured":"Cosentino V, Izquierdo JLC, Cabot J (2016) Findings from GitHub: methods, datasets and limitations. In: Proceedings MSR, pp 137\u2013141","DOI":"10.1145\/2901739.2901776"},{"key":"10370_CR23","doi-asserted-by":"crossref","unstructured":"Dias M, Bacchelli A, Gousios G, Cassou D, Ducasse S (2015) Untangling fine-grained code changes. In: SANER, pp 341\u2013350. IEEE Computer society","DOI":"10.1109\/SANER.2015.7081844"},{"key":"10370_CR24","doi-asserted-by":"crossref","unstructured":"Falc\u00e3o F, Barbosa C, Fonseca B, Garcia A, Ribeiro M, Gheyi R (2020) On relating technical, social factors, and the introduction of bugs. In: SANER, pp 378\u2013388. IEEE","DOI":"10.1109\/SANER48275.2020.9054824"},{"key":"10370_CR25","doi-asserted-by":"crossref","unstructured":"Fang H, Lamba H, Herbsleb JD, Vasilescu B (2022) This is damn slick! estimating the impact of tweets on open source project popularity and new contributors. In: ICSE, pp 2116\u20132129. ACM","DOI":"10.1145\/3510003.3510121"},{"key":"10370_CR26","doi-asserted-by":"crossref","unstructured":"Gabel M, Su, Z (2010) A study of the uniqueness of source code. In: SIGSOFT FSE, pp 147\u2013156. ACM","DOI":"10.1145\/1882291.1882315"},{"issue":"1","key":"10370_CR27","doi-asserted-by":"publisher","first-page":"5","DOI":"10.1111\/stan.12188","volume":"74","author":"A Gasparini","year":"2020","unstructured":"Gasparini A, Abrams KR, Barrett JK, Major RW, Sweeting MJ, Brunskill NJ, Crowther MJ (2020) Mixed-effects models for health care longitudinal data with an informative visiting process: A Monte Carlo simulation study. Statistica Neerlandica 74(1):5\u201323","journal-title":"Statistica Neerlandica"},{"key":"10370_CR28","doi-asserted-by":"publisher","DOI":"10.1017\/CBO9780511790942","volume-title":"Data analysis using regression and multilevel\/hierarchical models","author":"A Gelman","year":"2006","unstructured":"Gelman A, Hill J (2006) Data analysis using regression and multilevel\/hierarchical models. Cambridge University Press"},{"key":"10370_CR29","doi-asserted-by":"publisher","DOI":"10.1017\/9781139161879","volume-title":"Regression and other stories","author":"A Gelman","year":"2020","unstructured":"Gelman A, Hill J, Vehtari A (2020) Regression and other stories. Cambridge University Press"},{"issue":"4","key":"10370_CR30","doi-asserted-by":"publisher","first-page":"2102","DOI":"10.1007\/s10664-019-09695-9","volume":"24","author":"TA Ghaleb","year":"2019","unstructured":"Ghaleb TA, da Costa DA, Zou Y (2019) An empirical study of the long duration of continuous integration builds. Empir Softw Eng 24(4):2102\u20132139","journal-title":"Empir Softw Eng"},{"key":"10370_CR31","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-19425-7","volume-title":"Regression modeling strategies: with applications to linear models, logistic and ordinal regression, and survival analysis,","author":"FE Harrell","year":"2015","unstructured":"Harrell FE (2015) Regression modeling strategies: with applications to linear models, logistic and ordinal regression, and survival analysis, vol 2. Springer"},{"key":"10370_CR32","doi-asserted-by":"crossref","unstructured":"H\u00e4rtel J, L\u00e4mmel R (2020) Incremental map-reduce on repository history. In: SANER, pp 320\u2013331. IEEE","DOI":"10.1109\/SANER48275.2020.9054811"},{"key":"10370_CR33","doi-asserted-by":"crossref","unstructured":"H\u00e4rtel J, L\u00e4mmel R (2022) Operationalizing threats to MSR studies by simulation-based testing. In: MSR, pp 86\u201397. IEEE","DOI":"10.1145\/3524842.3527960"},{"key":"10370_CR34","doi-asserted-by":"crossref","unstructured":"He Z, Peters F, Menzies T, Yang Y (2013) Learning from open-source projects: An empirical study on defect prediction. In: ESEM, pp 45\u201354. IEEE Computer society","DOI":"10.1109\/ESEM.2013.20"},{"key":"10370_CR35","doi-asserted-by":"crossref","unstructured":"Herzig K, Zeller A (2013) The impact of tangled code changes. In: MSR, pp 121\u2013130. IEEE Computer society","DOI":"10.1109\/MSR.2013.6624018"},{"key":"10370_CR36","doi-asserted-by":"crossref","unstructured":"Honsel, V (2015) Statistical learning and software mining for agent based simulation of software evolution. In: ICSE (2), pp 863\u2013866. IEEE Computer society","DOI":"10.1109\/ICSE.2015.279"},{"key":"10370_CR37","doi-asserted-by":"crossref","unstructured":"Honsel V, Honsel D, Grabowski J (2014) Software process simulation based on mining software repositories. In: ICDM Workshops, pp 828\u2013831. IEEE Computer society","DOI":"10.1109\/ICDMW.2014.35"},{"key":"10370_CR38","doi-asserted-by":"crossref","unstructured":"Honsel V, Honsel D, Herbold S, Grabowski J, Waack S (2015) Mining software dependency networks for agent-based simulation of software evolution. In: ASE Workshops, pp 102\u2013108. IEEE Computer society","DOI":"10.1109\/ASEW.2015.9"},{"key":"10370_CR39","doi-asserted-by":"publisher","DOI":"10.1017\/CBO9781139025751","volume-title":"Causal inference in statistics, social, and biomedical sciences","author":"GW Imbens","year":"2015","unstructured":"Imbens GW, Rubin DB (2015) Causal inference in statistics, social, and biomedical sciences. Cambridge University Press"},{"key":"10370_CR40","unstructured":"Iyer RN, Yun SA, Nagappan M, Hoey J (2019) Effects of personality traits on pull request acceptance. IEEE Transactions on Software Engineering"},{"key":"10370_CR41","doi-asserted-by":"crossref","unstructured":"Jamie DM (2002) Using computer simulation methods to teach statistics: A review of the literature. Journal of Statistics Education 10(1)","DOI":"10.1080\/10691898.2002.11910548"},{"issue":"5","key":"10370_CR42","doi-asserted-by":"publisher","first-page":"1261","DOI":"10.1007\/s10664-013-9275-7","volume":"19","author":"A Jbara","year":"2014","unstructured":"Jbara A, Matan A, Feitelson DG (2014) High-MCC functions in the Linux kernel. Empir Softw Eng 19(5):1261\u20131298","journal-title":"Empir Softw Eng"},{"issue":"2","key":"10370_CR43","doi-asserted-by":"publisher","first-page":"320","DOI":"10.1109\/TSE.2019.2891758","volume":"47","author":"J Jiarpakdee","year":"2021","unstructured":"Jiarpakdee J, Tantithamthavorn C, Hassan AE (2021) The impact of correlated metrics on the interpretation of defect models. IEEE Trans Softw Eng 47(2):320\u2013331","journal-title":"IEEE Trans Softw Eng"},{"key":"10370_CR44","doi-asserted-by":"crossref","unstructured":"Johnson J, Lubo S, Yedla N, Aponte J, Sharif B (2019) An empirical study assessing source code readability in comprehension. In: ICSME, pp 513\u2013523. IEEE","DOI":"10.1109\/ICSME.2019.00085"},{"issue":"6","key":"10370_CR45","doi-asserted-by":"publisher","first-page":"4427","DOI":"10.1007\/s10664-020-09835-6","volume":"25","author":"R Jolak","year":"2020","unstructured":"Jolak R, Savary-Leblanc M, Dalibor M, Wortmann A, Hebig R, Vincur J, Pol\u00e1sek I, Pallec XL, G\u00e9rard S, Chaudron MRV (2020) Software engineering whispers: The effect of textual vs. graphical software design descriptions on software design communication. Empir Softw Eng 25(6):4427\u20134471","journal-title":"Empir Softw Eng"},{"issue":"6","key":"10370_CR46","doi-asserted-by":"publisher","first-page":"757","DOI":"10.1109\/TSE.2012.70","volume":"39","author":"Y Kamei","year":"2013","unstructured":"Kamei Y, Shihab E, Adams B, Hassan AE, Mockus A, Sinha A, Ubayashi N (2013) A large-scale empirical study of just-in-time quality assurance. IEEE Trans Softw Eng 39(6):757\u2013773","journal-title":"IEEE Trans Softw Eng"},{"key":"10370_CR47","doi-asserted-by":"crossref","unstructured":"Kochhar PS, Lo D (2017) Revisiting assert use in GitHub projects. In: EASE, pp 298\u2013307. ACM","DOI":"10.1145\/3084226.3084259"},{"issue":"5","key":"10370_CR48","doi-asserted-by":"publisher","first-page":"587","DOI":"10.1007\/s10664-010-9142-8","volume":"16","author":"A Martens","year":"2011","unstructured":"Martens A, Koziolek H, Prechelt L, Reussner RH (2011) From monolithic to component-based performance evaluation of software architectures - A series of experiments analysing accuracy and effort. Empir Softw Eng 16(5):587\u2013622","journal-title":"Empir Softw Eng"},{"key":"10370_CR49","doi-asserted-by":"crossref","unstructured":"McChesney IR, Bond RR (2020) Observations on the linear order of program code reading patterns in programmers with dyslexia. In: EASE, pp 81\u201389. ACM","DOI":"10.1145\/3383219.3383228"},{"key":"10370_CR50","doi-asserted-by":"crossref","unstructured":"McElreath, R (2020) Statistical rethinking: A Bayesian course with examples in R and Stan. CRC press","DOI":"10.1201\/9780429029608"},{"issue":"5807","key":"10370_CR51","doi-asserted-by":"publisher","first-page":"1856","DOI":"10.1126\/science.314.5807.1856","volume":"314","author":"G Miller","year":"2006","unstructured":"Miller G (2006) A Scientist\u2019s nightmare: Software problem leads to five retractions. Science 314(5807):1856\u20131857","journal-title":"Science"},{"key":"10370_CR52","doi-asserted-by":"crossref","unstructured":"Mockus, A (2010) Organizational volatility and its effects on software defects. In: SIGSOFT FSE, pp 117\u2013126. ACM","DOI":"10.1145\/1882291.1882311"},{"issue":"2","key":"10370_CR53","doi-asserted-by":"publisher","first-page":"169","DOI":"10.1002\/bltj.2229","volume":"5","author":"A Mockus","year":"2000","unstructured":"Mockus A, Weiss DM (2000) Predicting risk of software changes. Bell Labs Tech J 5(2):169\u2013180","journal-title":"Bell Labs Tech J"},{"issue":"11","key":"10370_CR54","doi-asserted-by":"publisher","first-page":"2074","DOI":"10.1002\/sim.8086","volume":"38","author":"TP Morris","year":"2019","unstructured":"Morris TP, White IR, Crowther MJ (2019) Using simulation studies to evaluate statistical methods. Stat Med 38(11):2074\u20132102","journal-title":"Stat Med"},{"key":"10370_CR55","doi-asserted-by":"crossref","unstructured":"Nagappan, N, Zeller, A, Zimmermann, T, Herzig, K, Murphy, B (2010) Change bursts as defect predictors. In: ISSRE, pp 309\u2013318. IEEE Computer society","DOI":"10.1109\/ISSRE.2010.25"},{"issue":"9","key":"10370_CR56","doi-asserted-by":"publisher","first-page":"874","DOI":"10.1109\/TSE.2017.2720603","volume":"44","author":"J Nam","year":"2018","unstructured":"Nam J, Fu W, Kim S, Menzies T, Tan L (2018) Heterogeneous defect prediction. IEEE Trans Softw Eng 44(9):874\u2013896","journal-title":"IEEE Trans Softw Eng"},{"key":"10370_CR57","doi-asserted-by":"crossref","unstructured":"Nam J, Pan SJ, Kim S (2013) Transfer defect learning. In: ICSE, pp 382\u2013391. IEEE Computer society","DOI":"10.1109\/ICSE.2013.6606584"},{"issue":"12","key":"10370_CR58","doi-asserted-by":"publisher","first-page":"1373","DOI":"10.1016\/S0895-4356(96)00236-3","volume":"49","author":"P Peduzzi","year":"1996","unstructured":"Peduzzi P, Concato J, Kemper E, Holford TR, Feinstein AR (1996) A simulation study of the number of events per variable in logistic regression analysis. J Clin Epidemiol 49(12):1373\u20131379","journal-title":"J Clin Epidemiol"},{"key":"10370_CR59","doi-asserted-by":"crossref","unstructured":"Penta MD, Cerulo L, Gu\u00e9h\u00e9neuc Y, Antoniol G (2008) An empirical study of the relationships between design pattern roles and class change proneness. In: ICSM, pp 217\u2013226. IEEE Computer society","DOI":"10.1109\/ICSM.2008.4658070"},{"key":"10370_CR60","doi-asserted-by":"crossref","unstructured":"Posnett D, Filkov V, Devanbu, PT (2011) Ecological inference in empirical software engineering. In: ASE, pp 362\u2013371. IEEE Computer society","DOI":"10.1109\/ASE.2011.6100074"},{"key":"10370_CR61","doi-asserted-by":"crossref","unstructured":"Rahman F, Devanbu PT (2011) Ownership, experience and defects: a fine-grained study of authorship. In: ICSE, pp 491\u2013500. ACM","DOI":"10.1145\/1985793.1985860"},{"key":"10370_CR62","doi-asserted-by":"crossref","unstructured":"Rahman F, Posnett D, Devanbu PT (2012) Recalling the \"imprecision\" of cross-project defect prediction. In: SIGSOFT FSE, p 61. ACM","DOI":"10.1145\/2393596.2393669"},{"key":"10370_CR63","doi-asserted-by":"crossref","unstructured":"Rahman MM, Roy CK, Collins JA (2016) CoRReCT: code reviewer recommendation in GitHub based on cross-project and technology experience. In: ICSE (Companion Volume), pp 222\u2013231. ACM","DOI":"10.1145\/2889160.2889244"},{"key":"10370_CR64","doi-asserted-by":"crossref","unstructured":"Reyes RP, Dieste O, Fonseca ER, Juristo N (2018) Statistical errors in software engineering experiments: a preliminary literature review. In: ICSE, pp 1195\u20131206. ACM","DOI":"10.29007\/964b"},{"issue":"8","key":"10370_CR65","doi-asserted-by":"publisher","first-page":"913","DOI":"10.1111\/ecog.02881","volume":"40","author":"DR Roberts","year":"2017","unstructured":"Roberts DR, Bahn V, Ciuti S, Boyce MS, Elith J, Guillera-Arroita G, Hauenstein S, Lahoz-Monfort JJ, Schr\u00f6der B, Thuiller W, Warton DI, Wintle BA, Hartig F, Dormann CF (2017) Cross-validation strategies for data with temporal, spatial, hierarchical, or phylogenetic structure. Ecography 40(8):913\u2013929","journal-title":"Ecography"},{"issue":"2","key":"10370_CR66","doi-asserted-by":"publisher","first-page":"1259","DOI":"10.1007\/s10664-019-09790-x","volume":"25","author":"M Sayagh","year":"2020","unstructured":"Sayagh M, Kerzazi N, Petrillo F, Bennani K, Adams B (2020) What should your run-time configuration framework do to help developers? Empir Softw Eng 25(2):1259\u20131293","journal-title":"Empir Softw Eng"},{"issue":"2","key":"10370_CR67","doi-asserted-by":"publisher","first-page":"642","DOI":"10.1007\/s10664-015-9406-4","volume":"21","author":"I Scholtes","year":"2016","unstructured":"Scholtes I, Mavrodiev P, Schweitzer F (2016) From Aristotle to Ringelmann: a large-scale analysis of team productivity and coordination in Open Source Software projects. Empir Softw Eng 21(2):642\u2013683","journal-title":"Empir Softw Eng"},{"key":"10370_CR68","doi-asserted-by":"crossref","unstructured":"Seifer P, H\u00e4rtel J, Leinberger M, L\u00e4mmel R, Staab S (2019) Empirical study on the usage of graph query languages in open source Java projects. In: SLE, pp 152\u2013166. ACM","DOI":"10.1145\/3357766.3359541"},{"key":"10370_CR69","unstructured":"Seo T, Lee H (2009) Agent-based simulation model for the evolution process of open source software. In: SEKE, pp 170\u2013177. Knowledge systems institute graduate school"},{"key":"10370_CR70","unstructured":"Shadish WR, Cook TD, Campbell DT (2002) Experimental and quasi-experimental designs for generalized causal inference. Houghton mifflin company"},{"issue":"422","key":"10370_CR71","doi-asserted-by":"publisher","first-page":"486","DOI":"10.1080\/01621459.1993.10476299","volume":"88","author":"J Shao","year":"1993","unstructured":"Shao J (1993) Linear model selection by cross-validation. J Am Stat Assoc 88(422):486\u2013494","journal-title":"J Am Stat Assoc"},{"issue":"9","key":"10370_CR72","doi-asserted-by":"publisher","first-page":"733","DOI":"10.1109\/TSE.2005.97","volume":"31","author":"DIK Sj\u00f8berg","year":"2005","unstructured":"Sj\u00f8berg DIK, Hannay JE, Hansen O, Kampenes VB, Karahasanovic A, Liborg N, Rekdal AC (2005) A survey of controlled experiments in software engineering. IEEE Trans Softw Eng 31(9):733\u2013753","journal-title":"IEEE Trans Softw Eng"},{"key":"10370_CR73","doi-asserted-by":"crossref","unstructured":"Sliwerski J, Zimmermann T, Zeller A (2005) When do changes induce fixes? In: MSR. ACM","DOI":"10.1145\/1083142.1083147"},{"issue":"11","key":"10370_CR74","doi-asserted-by":"publisher","first-page":"2584","DOI":"10.1073\/pnas.1708290115","volume":"115","author":"V Stodden","year":"2018","unstructured":"Stodden V, Seiler J, Ma Z (2018) An empirical analysis of journal policy effectiveness for computational reproducibility. Proc Natl Acad Sci USA 115(11):2584\u20132589","journal-title":"Proc Natl Acad Sci USA"},{"key":"10370_CR75","doi-asserted-by":"crossref","unstructured":"Tan M, Tan L, Dara S, Mayeux C (2015) Online defect prediction for imbalanced data. In: ICSE (2), pp 99\u2013108. IEEE Computer society","DOI":"10.1109\/ICSE.2015.139"},{"key":"10370_CR76","doi-asserted-by":"crossref","unstructured":"Tantithamthavorn C, Hassan AE (2018) An experience report on defect modelling in practice: pitfalls and challenges. In: ICSE (SEIP), pp 286\u2013295. ACM","DOI":"10.1145\/3183519.3183547"},{"issue":"1","key":"10370_CR77","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1109\/TSE.2016.2584050","volume":"43","author":"C Tantithamthavorn","year":"2017","unstructured":"Tantithamthavorn C, McIntosh S, Hassan AE, Matsumoto K (2017) An empirical comparison of model validation techniques for defect prediction models. IEEE Trans Softw Eng 43(1):1\u201318","journal-title":"IEEE Trans Softw Eng"},{"key":"10370_CR78","doi-asserted-by":"crossref","unstructured":"Thongtanunam P, McIntosh S, Hassan AE, Iida H (2016) Revisiting code ownership and its relationship with software quality in the scope of modern code review. In: ICSE, pp 1039\u20131050. ACM","DOI":"10.1145\/2884781.2884852"},{"issue":"1","key":"10370_CR79","doi-asserted-by":"publisher","first-page":"9","DOI":"10.1016\/0164-1212(94)00111-Y","volume":"28","author":"WF Tichy","year":"1995","unstructured":"Tichy WF, Lukowicz P, Prechelt L, Heinz EA (1995) Experimental evaluation in computer science: A quantitative study. J Syst Softw 28(1):9\u201318","journal-title":"J Syst Softw"},{"key":"10370_CR80","doi-asserted-by":"crossref","unstructured":"Tsay J, Dabbish L, Herbsleb JD (2014) Influence of social and technical factors for evaluating contribution in GitHub. In: ICSE, pp 356\u2013366. ACM","DOI":"10.1145\/2568225.2568315"},{"key":"10370_CR81","doi-asserted-by":"crossref","unstructured":"Tufano M, Bavota G, Poshyvanyk D, Penta MD, Oliveto R, Lucia AD (2017) An empirical study on developer-related factors characterizing fix-inducing commits. J Softw Evol Process 29(1)","DOI":"10.1002\/smr.1797"},{"key":"10370_CR82","doi-asserted-by":"crossref","unstructured":"Vasilescu B, Posnett D, Ray B, van den Brand MGJ, Serebrenik A, Devanbu PT, Filkov V (2015) Gender and tenure diversity in GitHub teams. In: CHI, pp 3789\u20133798. ACM","DOI":"10.1145\/2702123.2702549"},{"issue":"12","key":"10370_CR83","doi-asserted-by":"publisher","first-page":"904","DOI":"10.1109\/TSE.2004.99","volume":"30","author":"M Vok\u00e1c","year":"2004","unstructured":"Vok\u00e1c M (2004) Defect frequency and design patterns: An empirical study of industrial code. IEEE Trans Softw Eng 30(12):904\u2013917","journal-title":"IEEE Trans Softw Eng"},{"key":"10370_CR84","doi-asserted-by":"crossref","unstructured":"Wood M (2005) The role of simulation approaches in statistics. Journal of Statistics Education 13(3)","DOI":"10.1080\/10691898.2005.11910562"},{"key":"10370_CR85","doi-asserted-by":"crossref","unstructured":"Yan M, Xia X, Fan Y, Lo D, Hassan AE, Zhang X (2020) Effort-aware just-in-time defect identification in practice: a case study at Alibaba. In: ESEC\/SIGSOFT FSE, pp 1308\u20131319. ACM","DOI":"10.1145\/3368089.3417048"},{"issue":"5","key":"10370_CR86","doi-asserted-by":"publisher","first-page":"476","DOI":"10.1109\/TSE.2016.2599161","volume":"43","author":"F Zhang","year":"2017","unstructured":"Zhang F, Hassan AE, McIntosh S, Zou Y (2017) The use of summation to aggregate software metrics hinders the performance of defect prediction models. IEEE Trans Softw Eng 43(5):476\u2013491","journal-title":"IEEE Trans Softw Eng"},{"key":"10370_CR87","doi-asserted-by":"crossref","unstructured":"Zimmermann T, Nagappan N (2008) Predicting defects using network analysis on dependency graphs. In: ICSE, pp 531\u2013540. ACM","DOI":"10.1145\/1368088.1368161"},{"key":"10370_CR88","doi-asserted-by":"crossref","unstructured":"Zimmermann T, Premraj R, Zeller A (2007) Predicting defects for eclipse. In: PROMISE 2007, p 76. IEEE","DOI":"10.1109\/PROMISE.2007.10"}],"container-title":["Empirical Software Engineering"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10664-023-10370-3.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s10664-023-10370-3\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10664-023-10370-3.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,11,1]],"date-time":"2024-11-01T21:57:17Z","timestamp":1730498237000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s10664-023-10370-3"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023,11]]},"references-count":88,"journal-issue":{"issue":"6","published-print":{"date-parts":[[2023,11]]}},"alternative-id":["10370"],"URL":"https:\/\/doi.org\/10.1007\/s10664-023-10370-3","relation":{},"ISSN":["1382-3256","1573-7616"],"issn-type":[{"value":"1382-3256","type":"print"},{"value":"1573-7616","type":"electronic"}],"subject":[],"published":{"date-parts":[[2023,11]]},"assertion":[{"value":"13 July 2023","order":1,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"13 November 2023","order":2,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Declarations"}},{"value":"The authors have no conflict of interest.","order":2,"name":"Ethics","group":{"name":"EthicsHeading","label":"Conflicts of interest"}}],"article-number":"153"}}