{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,1,13]],"date-time":"2026-01-13T21:13:45Z","timestamp":1768338825931,"version":"3.49.0"},"reference-count":26,"publisher":"Elsevier BV","license":[{"start":{"date-parts":[[2023,6,1]],"date-time":"2023-06-01T00:00:00Z","timestamp":1685577600000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.elsevier.com\/tdm\/userlicense\/1.0\/"},{"start":{"date-parts":[[2023,6,1]],"date-time":"2023-06-01T00:00:00Z","timestamp":1685577600000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.elsevier.com\/legal\/tdmrep-license"},{"start":{"date-parts":[[2023,2,15]],"date-time":"2023-02-15T00:00:00Z","timestamp":1676419200000},"content-version":"vor","delay-in-days":0,"URL":"http:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"content-domain":{"domain":["elsevier.com","sciencedirect.com"],"crossmark-restriction":true},"short-container-title":["Information and Software Technology"],"published-print":{"date-parts":[[2023,6]]},"DOI":"10.1016\/j.infsof.2023.107174","type":"journal-article","created":{"date-parts":[[2023,2,22]],"date-time":"2023-02-22T23:01:52Z","timestamp":1677106912000},"page":"107174","update-policy":"https:\/\/doi.org\/10.1016\/elsevier_cm_policy","source":"Crossref","is-referenced-by-count":6,"special_numbering":"C","title":["Double-counting in software engineering tertiary studies \u2014 An overlooked threat to validity"],"prefix":"10.1016","volume":"158","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-0639-4234","authenticated-orcid":false,"given":"J\u00fcrgen","family":"B\u00f6rstler","sequence":"first","affiliation":[]},{"given":"Nauman","family":"bin Ali","sequence":"additional","affiliation":[]},{"given":"Kai","family":"Petersen","sequence":"additional","affiliation":[]}],"member":"78","reference":[{"issue":"1","key":"10.1016\/j.infsof.2023.107174_b1","doi-asserted-by":"crossref","first-page":"7","DOI":"10.1016\/j.infsof.2008.09.009","article-title":"Systematic literature reviews in software engineering\u00a0\u2014\u00a0A systematic literature review","volume":"51","author":"Kitchenham","year":"2009","journal-title":"Inf. Softw. Technol."},{"key":"10.1016\/j.infsof.2023.107174_b2","doi-asserted-by":"crossref","first-page":"396","DOI":"10.1016\/j.jss.2018.12.018","article-title":"Landscaping systematic mapping studies in software engineering: A tertiary study","volume":"149","author":"Khan","year":"2019","journal-title":"J. Syst. Softw."},{"key":"10.1016\/j.infsof.2023.107174_b3","series-title":"Guidelines for Performing Systematic Literature Reviews in Software Engineering","author":"Kitchenham","year":"2007"},{"key":"10.1016\/j.infsof.2023.107174_b4","doi-asserted-by":"crossref","first-page":"162","DOI":"10.1016\/j.jss.2016.06.043","article-title":"The impacts of agile and lean practices on project constraints: A tertiary study","volume":"119","author":"Nurdiani","year":"2016","journal-title":"J. Syst. Softw."},{"key":"10.1016\/j.infsof.2023.107174_b5","doi-asserted-by":"crossref","DOI":"10.1016\/j.infsof.2021.106620","article-title":"Assessing test artifact quality \u2013 A tertiary study","volume":"139","author":"Tran","year":"2021","journal-title":"Inf. Softw. Technol."},{"issue":"4","key":"10.1016\/j.infsof.2023.107174_b6","doi-asserted-by":"crossref","first-page":"368","DOI":"10.1016\/j.jclinepi.2013.11.007","article-title":"Systematic review finds overlapping reviews were not mentioned in every other overview","volume":"67","author":"Pieper","year":"2014","journal-title":"J. Clin. Epidemiol."},{"issue":"1","key":"10.1016\/j.infsof.2023.107174_b7","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1186\/s12874-021-01269-y","article-title":"Managing overlap of primary study results across systematic reviews: Practical considerations for authors of overviews of reviews","volume":"21","author":"Lunny","year":"2021","journal-title":"BMC Med. Res. Methodol."},{"issue":"1","key":"10.1016\/j.infsof.2023.107174_b8","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1186\/1471-2288-9-10","article-title":"Overstating the evidence \u2013 double counting in meta-analysis and related problems","volume":"9","author":"Senn","year":"2009","journal-title":"BMC Med. Res. Methodol."},{"issue":"1","key":"10.1016\/j.infsof.2023.107174_b9","doi-asserted-by":"crossref","first-page":"54","DOI":"10.1016\/j.infsof.2013.06.005","article-title":"Risks and risk mitigation in global software development: A tertiary study","volume":"56","author":"Verner","year":"2014","journal-title":"Inf. Softw. Technol."},{"issue":"5","key":"10.1016\/j.infsof.2023.107174_b10","doi-asserted-by":"crossref","first-page":"601","DOI":"10.1157\/13074852","article-title":"Duplicate or redundant publication: Can we afford it?","volume":"58","author":"Alfonso","year":"2005","journal-title":"Rev. Esp. Cardiol."},{"key":"10.1016\/j.infsof.2023.107174_b11","series-title":"The PROMISE Repository of Software Engineering Databases","author":"Sayyad\u00a0Shirabad","year":"2005"},{"issue":"4","key":"10.1016\/j.infsof.2023.107174_b12","doi-asserted-by":"crossref","first-page":"405","DOI":"10.1007\/s10664-005-3861-2","article-title":"Supporting controlled experimentation with testing techniques: An infrastructure and its potential impact","volume":"10","author":"Do","year":"2005","journal-title":"Empir. Softw. Eng."},{"issue":"4","key":"10.1016\/j.infsof.2023.107174_b13","doi-asserted-by":"crossref","first-page":"2020","DOI":"10.1007\/s10664-018-9670-1","article-title":"On the search for industry-relevant regression testing research","volume":"24","author":"bin Ali","year":"2019","journal-title":"Empir. Softw. Eng."},{"key":"10.1016\/j.infsof.2023.107174_b14","doi-asserted-by":"crossref","first-page":"26773","DOI":"10.1109\/ACCESS.2019.2952191","article-title":"Replication of studies in empirical software engineering: A systematic mapping study, from 2013 to 2018","volume":"8","author":"Cruz","year":"2019","journal-title":"IEEE Access"},{"issue":"4","key":"10.1016\/j.infsof.2023.107174_b15","doi-asserted-by":"crossref","first-page":"375","DOI":"10.1016\/j.infsof.2014.01.002","article-title":"Considering rigor and relevance when evaluating test driven development: A systematic review","volume":"56","author":"Munir","year":"2014","journal-title":"Inf. Softw. Technol."},{"key":"10.1016\/j.infsof.2023.107174_b16","doi-asserted-by":"crossref","first-page":"201","DOI":"10.1016\/j.infsof.2018.10.006","article-title":"Identifying, categorizing and mitigating threats to validity in software engineering secondary studies","volume":"106","author":"Ampatzoglou","year":"2019","journal-title":"Inf. Softw. Technol."},{"key":"10.1016\/j.infsof.2023.107174_b17","doi-asserted-by":"crossref","unstructured":"D. Costal, C. Farr\u00e9, X. Franch, C. Quer, Inclusion and Exclusion Criteria in Software Engineering Tertiary Studies: A Systematic Mapping and Emerging Framework, in: F. Lanubile, M. Kalinowski, M.T. Baldassarre (Eds.), Proceedings of the 15th ACM\/IEEE International Symposium on Empirical Software Engineering and Measurement, 2021, pp. 30:1\u201330:6.","DOI":"10.1145\/3475716.3484190"},{"key":"10.1016\/j.infsof.2023.107174_b18","series-title":"Evidence-Based Software Engineering and Systematic Reviews","author":"Kitchenham","year":"2016"},{"key":"10.1016\/j.infsof.2023.107174_b19","doi-asserted-by":"crossref","first-page":"117","DOI":"10.1016\/j.infsof.2018.05.010","article-title":"A tertiary study on technical debt: Types, management strategies, research trends, and base information for practitioners","volume":"102","author":"Rios","year":"2018","journal-title":"Inf. Softw. Technol."},{"key":"10.1016\/j.infsof.2023.107174_b20","series-title":"Cochrane Handbook for Systematic Reviews of Interventions, Version 6.2","article-title":"Chapter V: Overviews of reviews","author":"Pollock","year":"2021"},{"issue":"1","key":"10.1016\/j.infsof.2023.107174_b21","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1186\/s13643-020-01509-0","article-title":"Guidance for overviews of reviews continues to accumulate, but important challenges remain: a scoping review","volume":"9","author":"Gates","year":"2020","journal-title":"Syst. Rev."},{"key":"10.1016\/j.infsof.2023.107174_b22","unstructured":"D. Costal, C. Farr\u00e9, X. Franch, C. Quer, How tertiary studies perform quality assessment of secondary studies in software engineering, in: T. Conte, M. Solari, S.S. Reinehr, R. Prikladnicki, N. Condori-Fern\u00e1ndez, V.E.S. Souza, N.M.C. Valentim, S. Mart\u00ednez-Fern\u00e1ndez, M. Genero, M. Kalinowski, M. Jenkins, A. Mart\u00ednez, C. Quesada-L\u00f3pez (Eds.), Proceedings of the XXIV Iberoamerican Conference on Software Engineering, 2021, pp. 56\u201369."},{"key":"10.1016\/j.infsof.2023.107174_b23","series-title":"How tertiary studies perform quality assessment of secondary studies in software engineering \u2013 Replication package","author":"Costal","year":"2021"},{"key":"10.1016\/j.infsof.2023.107174_b24","doi-asserted-by":"crossref","first-page":"34","DOI":"10.1016\/j.jclinepi.2020.12.004","article-title":"Methods for depicting overlap in overviews of systematic reviews: An introduction to static tabular and graphical displays","volume":"132","author":"Bougioukas","year":"2021","journal-title":"J. Clin. Epidemiol."},{"issue":"6","key":"10.1016\/j.infsof.2023.107174_b25","doi-asserted-by":"crossref","first-page":"835","DOI":"10.1109\/TSE.2012.28","article-title":"The effects of test-driven development on external quality and productivity: A meta-analysis","volume":"39","author":"Rafique","year":"2012","journal-title":"Trans. Softw. Eng."},{"key":"10.1016\/j.infsof.2023.107174_b26","series-title":"A quality assessment instrument for systematic literature reviews in software engineering, CoRR abs\/2109.10134","author":"Usman","year":"2021"}],"container-title":["Information and Software Technology"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/api.elsevier.com\/content\/article\/PII:S0950584923000289?httpAccept=text\/xml","content-type":"text\/xml","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/api.elsevier.com\/content\/article\/PII:S0950584923000289?httpAccept=text\/plain","content-type":"text\/plain","content-version":"vor","intended-application":"text-mining"}],"deposited":{"date-parts":[[2025,10,16]],"date-time":"2025-10-16T03:39:26Z","timestamp":1760585966000},"score":1,"resource":{"primary":{"URL":"https:\/\/linkinghub.elsevier.com\/retrieve\/pii\/S0950584923000289"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023,6]]},"references-count":26,"alternative-id":["S0950584923000289"],"URL":"https:\/\/doi.org\/10.1016\/j.infsof.2023.107174","relation":{},"ISSN":["0950-5849"],"issn-type":[{"value":"0950-5849","type":"print"}],"subject":[],"published":{"date-parts":[[2023,6]]},"assertion":[{"value":"Elsevier","name":"publisher","label":"This article is maintained by"},{"value":"Double-counting in software engineering tertiary studies \u2014 An overlooked threat to validity","name":"articletitle","label":"Article Title"},{"value":"Information and Software Technology","name":"journaltitle","label":"Journal Title"},{"value":"https:\/\/doi.org\/10.1016\/j.infsof.2023.107174","name":"articlelink","label":"CrossRef DOI link to publisher maintained version"},{"value":"article","name":"content_type","label":"Content Type"},{"value":"\u00a9 2023 The Author(s). Published by Elsevier B.V.","name":"copyright","label":"Copyright"}],"article-number":"107174"}}