{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,11,1]],"date-time":"2025-11-01T09:36:22Z","timestamp":1761989782301,"version":"3.44.0"},"reference-count":51,"publisher":"Springer Science and Business Media LLC","issue":"5","license":[{"start":{"date-parts":[[2025,6,13]],"date-time":"2025-06-13T00:00:00Z","timestamp":1749772800000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2025,6,13]],"date-time":"2025-06-13T00:00:00Z","timestamp":1749772800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Empir Software Eng"],"published-print":{"date-parts":[[2025,9]]},"DOI":"10.1007\/s10664-025-10682-6","type":"journal-article","created":{"date-parts":[[2025,6,13]],"date-time":"2025-06-13T09:28:27Z","timestamp":1749806907000},"update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":1,"title":["On the need to monitor continuous integration practices"],"prefix":"10.1007","volume":"30","author":[{"ORCID":"https:\/\/orcid.org\/0009-0007-8217-0642","authenticated-orcid":false,"given":"Jadson","family":"Santos","sequence":"first","affiliation":[]},{"given":"Daniel Alencar","family":"da Costa","sequence":"additional","affiliation":[]},{"given":"Shane","family":"McIntosh","sequence":"additional","affiliation":[]},{"given":"Uir\u00e1","family":"Kulesza","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2025,6,13]]},"reference":[{"key":"10682_CR1","doi-asserted-by":"publisher","unstructured":"Allamanis M, Peng H, Sutton C (2016) A Convolutional Attention Network for Extreme Summarization of Source Code. In: Proceedings of the 33rd international conference on machine learning, proceedings of machine learning research. PMLR, vol\u00a048, pp 2091\u20132100. https:\/\/doi.org\/10.48550\/arXiv.1602.03001","DOI":"10.48550\/arXiv.1602.03001"},{"key":"10682_CR2","doi-asserted-by":"publisher","unstructured":"Bach T, Andrzejak A, Pannemans R, Lo D (2017) The Impact of Coverage on Bug Density in a Large Industrial Software Project. In: 2017 ACM\/IEEE international symposium on empirical software engineering and measurement (ESEM), pp 307\u2013313. https:\/\/doi.org\/10.1109\/ESEM.2017.44","DOI":"10.1109\/ESEM.2017.44"},{"key":"10682_CR3","doi-asserted-by":"publisher","unstructured":"Bernardo JaH, da\u00a0Costa DA, Kulesza U (2018) Studying the impact of adopting continuous integration on the delivery time of pull requests. In: Proceedings of the 15th international conference on mining software repositories, pp 131\u2013141. https:\/\/doi.org\/10.1145\/3196398.3196421","DOI":"10.1145\/3196398.3196421"},{"key":"10682_CR4","doi-asserted-by":"crossref","unstructured":"Besbes A (2013) Github actions - everything you need to know to get started. https:\/\/towardsdatascience.com\/github-actions-everything-you-need-to-know-to-get-started-537f1dffa0ed","DOI":"10.1002\/9781119196891.ch1"},{"key":"10682_CR5","doi-asserted-by":"publisher","unstructured":"Betz RM, Walker RC (2013) Implementing continuous integration software in an established computational chemistry software package. In: 2013 5th International workshop on software engineering for computational science and engineering (SE-CSE). IEEE, IEEE Press, pp. 68\u201374. https:\/\/doi.org\/10.1109\/SECSE.2013.6615101","DOI":"10.1109\/SECSE.2013.6615101"},{"issue":"2","key":"10682_CR6","doi-asserted-by":"publisher","first-page":"27","DOI":"10.3316\/QRJ0902027","volume":"9","author":"GA Bowen","year":"2009","unstructured":"Bowen GA (2009) Document analysis as a qualitative research method. Qual Res J 9(2):27\u201340. https:\/\/doi.org\/10.3316\/QRJ0902027","journal-title":"Qual Res J"},{"key":"10682_CR7","doi-asserted-by":"publisher","unstructured":"Cassee N, Vasilescu B, Serebrenik A (2020) The silent helper: the impact of continuous integration on code reviews. In: 2020 IEEE 27th international conference on software analysis, evolution and reengineering (SANER), pp 423\u2013434. https:\/\/doi.org\/10.1109\/SANER48275.2020.9054818","DOI":"10.1109\/SANER48275.2020.9054818"},{"key":"10682_CR8","doi-asserted-by":"publisher","unstructured":"Chen B, Chen L, Zhang C, Peng X (2020) BUILDFAST: History-aware build outcome prediction for fast feedback and reduced cost in continuous integration. In: Proceedings of the 35th IEEE\/ACM international conference on automated software engineering, pp 42\u201353. https:\/\/doi.org\/10.1145\/3324884.3416616","DOI":"10.1145\/3324884.3416616"},{"issue":"4","key":"10682_CR9","doi-asserted-by":"publisher","first-page":"213","DOI":"10.1037\/h0026256","volume":"70","author":"J Cohen","year":"1968","unstructured":"Cohen J (1968) Weighted kappa: nominal scale agreement provision for scaled disagreement or partial credit. Psychol Bull 70(4):213. https:\/\/doi.org\/10.1037\/h0026256","journal-title":"Psychol Bull"},{"key":"10682_CR10","doi-asserted-by":"publisher","unstructured":"de Oliveira MC, Freitas D, Bonif\u00e1cio R, Pinto G, Lo D (2019) Finding needles in a haystack: Leveraging co-change dependencies to recommend refactorings 110420. https:\/\/doi.org\/10.1016\/j.jss.2019.110420","DOI":"10.1016\/j.jss.2019.110420"},{"key":"10682_CR11","doi-asserted-by":"publisher","unstructured":"de\u00a0Oliveira MC (2017) DRACO: discovering refactorings that improve architecture using fine-grained co-change dependencies. In: Proceedings of the 2017 11th joint meeting on foundations of software engineering, ESEC\/FSE 2017, pp 1018\u20131021. https:\/\/doi.org\/10.1145\/3106237.3119872","DOI":"10.1145\/3106237.3119872"},{"key":"10682_CR12","doi-asserted-by":"publisher","unstructured":"Decan A, Mens T, Mazrae PR, Golzadeh M (2022) On the Use of GitHub Actions in Software Development Repositories. In: 2022 IEEE international conference on software maintenance and evolution (ICSME), pp 235\u2013245. https:\/\/doi.org\/10.1109\/ICSME55016.2022.00029","DOI":"10.1109\/ICSME55016.2022.00029"},{"key":"10682_CR13","doi-asserted-by":"publisher","unstructured":"dos Santos\u00a0J\u00fanior PS, Perini\u00a0Barcellos M, Borges\u00a0Ruy F (2021) Tell me: am i going to heaven? A diagnosis instrument of continuous software engineering practices adoption. In: Evaluation and assessment in software engineering. Association for Computing Machinery, pp 30\u201339. https:\/\/doi.org\/10.1145\/3463274.3463324","DOI":"10.1145\/3463274.3463324"},{"key":"10682_CR14","unstructured":"Duvall P, Matyas SM, Glover A (2007) Continuous integration: improving software quality and reducing risk (The Addison-Wesley Signature Series). Addison-Wesley Professional"},{"issue":"3","key":"10682_CR15","doi-asserted-by":"publisher","first-page":"94","DOI":"10.1109\/MS.2016.68","volume":"33","author":"C Ebert","year":"2016","unstructured":"Ebert C, Gallardo G, Hernantes J, Serrano N (2016) DevOps 33(3):94\u2013100. https:\/\/doi.org\/10.1109\/MS.2016.68","journal-title":"DevOps"},{"key":"10682_CR16","doi-asserted-by":"crossref","unstructured":"Elazhary O, Werner C, Li ZS, Lowlind D, Ernst NA, Storey MA (2021) Uncovering the benefits and challenges of continuous integration practices. IEEE Trans Softw Eng 48(7):2570\u20132583. https:\/\/doi.ieeecomputersociety.org\/10.1109\/TSE.2021.3064953","DOI":"10.1109\/TSE.2021.3064953"},{"key":"10682_CR17","doi-asserted-by":"publisher","unstructured":"Felidr\u00e9 W, Furtado L, Costa DAd, Cartaxo B, Pinto G (2019) Continuous integration theater. In: 2019 ACM\/IEEE international symposium on empirical software engineering and measurement (ESEM), pp 1\u201310. https:\/\/doi.org\/10.1109\/ESEM.2019.8870152","DOI":"10.1109\/ESEM.2019.8870152"},{"key":"10682_CR18","doi-asserted-by":"publisher","unstructured":"Fitzgerald B, Stol KJ (2014) Continuous software engineering and beyond: trends and challenges. In: Proceedings of the 1st international workshop on rapid continuous software engineering. Association for Computing Machinery, pp 1\u20139. https:\/\/doi.org\/10.1145\/2593812.2593813","DOI":"10.1145\/2593812.2593813"},{"key":"10682_CR19","unstructured":"Fowler M, Foemmel M (2006) Continuous integration. https:\/\/martinfowler.com\/articles\/continuousIntegration.html"},{"key":"10682_CR20","doi-asserted-by":"publisher","first-page":"101","DOI":"10.1016\/j.infsof.2018.09.006","volume":"106","author":"V Garousi","year":"2019","unstructured":"Garousi V, Felderer M, M\u00e4ntyl\u00e4 MV (2019) Guidelines for including grey literature and conducting multivocal literature reviews in software engineering. Inf Softw Technol 106:101\u2013121. https:\/\/doi.org\/10.1016\/j.infsof.2018.09.006","journal-title":"Inf Softw Technol"},{"key":"10682_CR21","doi-asserted-by":"publisher","unstructured":"Garousi V, Felderer M, M\u00e4ntyl\u00e4 MV, Rainer A (2020) Benefitting from the grey literature in software engineering research. In: Contemporary empirical methods in software engineering. Springer, pp 385\u2013413. https:\/\/doi.org\/10.1007\/978-3-030-32489-6_14","DOI":"10.1007\/978-3-030-32489-6_14"},{"key":"10682_CR22","doi-asserted-by":"publisher","first-page":"2102","DOI":"10.1007\/s10664-019-09695-9","volume":"24","author":"TA Ghaleb","year":"2019","unstructured":"Ghaleb TA, Da Costa DA, Zou Y (2019) An empirical study of the long duration of continuous integration builds. Empir Softw Eng 24:2102\u20132139. https:\/\/doi.org\/10.1007\/s10664-019-09695-9","journal-title":"Empir Softw Eng"},{"key":"10682_CR23","doi-asserted-by":"publisher","unstructured":"Golzadeh M, Decan A, Mens T (2022) On the rise and fall of CI services in GitHub. In: 2022 IEEE international conference on software analysis, evolution and reengineering (SANER), pp 662\u2013672. https:\/\/doi.org\/10.1109\/SANER53432.2022.00084","DOI":"10.1109\/SANER53432.2022.00084"},{"key":"10682_CR24","doi-asserted-by":"publisher","unstructured":"Hilton M, Nelson N, Tunnell T, Marinov D, Dig D (2017) Trade-offs in continuous integration: assurance, security, and flexibility. Association for computing machinery, New York, NY, USA. https:\/\/doi.org\/10.1145\/3106237.3106270","DOI":"10.1145\/3106237.3106270"},{"key":"10682_CR25","doi-asserted-by":"crossref","unstructured":"Hilton M, Tunnell T, Huang K, Marinov D, Dig D (2016) Usage, costs, and benefits of continuous integration in open-source projects. In: Proceedings of the 31st IEEE\/ACM international conference on automated software engineering, pp 426\u2013437. https:\/\/ieeexplore.ieee.org\/document\/7582778","DOI":"10.1145\/2970276.2970358"},{"key":"10682_CR26","doi-asserted-by":"publisher","first-page":"106609","DOI":"10.1016\/j.infsof.2021.106609","volume":"138","author":"F Kamei","year":"2021","unstructured":"Kamei F, Wiese I, Lima C, Polato I, Nepomuceno V, Ferreira W, Ribeiro M, Pena C, Cartaxo B, Pinto G, Soares S (2021) Grey literature in software engineering: a critical review. Inf Softw Technol 138:106609. https:\/\/doi.org\/10.1016\/j.infsof.2021.106609","journal-title":"Inf Softw Technol"},{"issue":"7","key":"10682_CR27","doi-asserted-by":"publisher","first-page":"363","DOI":"10.3390\/info11070363","volume":"11","author":"I Karamitsos","year":"2020","unstructured":"Karamitsos I, Albarhami S, Apostolopoulos C (2020) Applying DevOps practices of continuous automation for machine learning. Information 11(7):363. https:\/\/doi.org\/10.3390\/info11070363","journal-title":"Information"},{"key":"10682_CR28","doi-asserted-by":"publisher","unstructured":"Kavaler D, Trockman A, Vasilescu B, Filkov V (2019) Tool choice matters: JavaScript quality assurance tools and usage outcomes in GitHub projects. In: 2019 IEEE\/ACM 41st international conference on software engineering (ICSE). IEEE, pp 476\u2013487. https:\/\/doi.org\/10.1109\/ICSE.2019.00060","DOI":"10.1109\/ICSE.2019.00060"},{"key":"10682_CR29","unstructured":"Kibuacha F (2021) How to determine sample size for a research study. https:\/\/www.geopoll.com\/blog\/sample-size-research\/"},{"key":"10682_CR30","doi-asserted-by":"publisher","unstructured":"Laukkanen E, Paasivaara M, Arvonen T (2015) Stakeholder perceptions of the adoption of continuous integration \u2013 a case study. In: 2015 Agile conference, pp 11\u201320. https:\/\/doi.org\/10.1109\/Agile.2015.15","DOI":"10.1109\/Agile.2015.15"},{"key":"10682_CR31","unstructured":"LeanIX (2023) Dora metrics: how to measure software delivery performance. https:\/\/www.leanix.net\/en\/wiki\/vsm\/dora-metrics"},{"key":"10682_CR32","unstructured":"Linaker J, Sulaman SM, H\u00f6st M, de\u00a0Mello RM (2015) Guidelines for conducting surveys in software engineering v. 1.1. https:\/\/lucris.lub.lu.se\/ws\/portalfiles\/portal\/6062997\/5463412.pdf"},{"key":"10682_CR33","doi-asserted-by":"publisher","unstructured":"Lwakatare LE, Kuvaja P, Oivo M (2016) Relationship of devops to agile, lean and continuous deployment: a multivocal literature review study. In: Product-focused software process improvement: 17th international conference, PROFES 2016, Trondheim, Norway, November 22-24, 2016, Proceedings 17. Springer, pp 399\u2013415. https:\/\/doi.org\/10.1007\/978-3-319-49094-6_27","DOI":"10.1007\/978-3-319-49094-6_27"},{"key":"10682_CR34","doi-asserted-by":"publisher","unstructured":"Maguire M, Delahunt B (2017) Doing a thematic analysis: a practical, step-by-step guide for learning and teaching scholars. All Ireland J Higher Educ 9(3). https:\/\/doi.org\/10.62707\/aishej.v9i3.335","DOI":"10.62707\/aishej.v9i3.335"},{"key":"10682_CR35","doi-asserted-by":"publisher","first-page":"2146","DOI":"10.1007\/s10664-015-9381-9","volume":"21","author":"S McIntosh","year":"2016","unstructured":"McIntosh S, Kamei Y, Adams B, Hassan AE (2016) An empirical study of the impact of modern code review practices on software quality. Empir Softw Eng 21:2146\u20132189. https:\/\/doi.org\/10.1007\/s10664-015-9381-9","journal-title":"Empir Softw Eng"},{"key":"10682_CR36","doi-asserted-by":"publisher","unstructured":"Morgan H (2022) Conducting a Qualitative Document Analysis. Qual Rep 27(1). https:\/\/doi.org\/10.46743\/2160-3715\/2022.5044","DOI":"10.46743\/2160-3715\/2022.5044"},{"key":"10682_CR37","doi-asserted-by":"publisher","unstructured":"Poncin W, Serebrenik A, van\u00a0den Brand M (2011) Process mining software repositories. In: 2011 15th European conference on software maintenance and reengineering, pp 5\u201314. https:\/\/doi.org\/10.1109\/CSMR.2011.5","DOI":"10.1109\/CSMR.2011.5"},{"key":"10682_CR38","unstructured":"Rehn A, Palmborg T, Bostr\u00f6m P (2013) The continuous delivery maturity model. https:\/\/www.infoq.com\/articles\/Continuous-Delivery-Maturity-Model\/"},{"key":"10682_CR39","doi-asserted-by":"publisher","unstructured":"Sallin M, Kropp M, Anslow C, Quilty JW, Meier A (2021) Measuring software delivery performance using the four key metrics of devops. In: International conference on agile software development. Springer International Publishing Cham, pp 103\u2013119. https:\/\/doi.org\/10.1007\/978-3-030-78098-2_7","DOI":"10.1007\/978-3-030-78098-2_7"},{"key":"10682_CR40","doi-asserted-by":"publisher","unstructured":"Santos J, Alencar\u00a0da Costa D, Kulesza U (2022) Investigating the impact of continuous integration practices on the productivity and quality of open-source projects. In: Proceedings of the 16th ACM \/ IEEE international symposium on empirical software engineering and measurement, ESEM \u201922, pp 137\u2013147. https:\/\/doi.org\/10.1145\/3544902.3546244","DOI":"10.1145\/3544902.3546244"},{"key":"10682_CR41","doi-asserted-by":"publisher","unstructured":"Smith E, Loftin R, Murphy-Hill E, Bird C, Zimmermann T (2013) Improving developer participation rates in surveys. In: 2013 6th International workshop on cooperative and human aspects of software engineering (CHASE), pp 89\u201392. https:\/\/doi.org\/10.1109\/CHASE.2013.6614738","DOI":"10.1109\/CHASE.2013.6614738"},{"key":"10682_CR42","doi-asserted-by":"publisher","unstructured":"Soares E, Sizilio G, Santos J, da\u00a0Costa DA, Kulesza U (2022) The effects of continuous integration on software development: a systematic literature review 27(3). https:\/\/doi.org\/10.1007\/s10664-021-10114-1","DOI":"10.1007\/s10664-021-10114-1"},{"key":"10682_CR43","doi-asserted-by":"publisher","first-page":"48","DOI":"10.1016\/j.jss.2013.08.032","volume":"87","author":"D St\u00e5hl","year":"2014","unstructured":"St\u00e5hl D, Bosch J (2014) Modeling continuous integration practice differences in industry software development. J Syst Softw 87:48\u201359. https:\/\/doi.org\/10.1016\/j.jss.2013.08.032","journal-title":"J Syst Softw"},{"key":"10682_CR44","doi-asserted-by":"publisher","unstructured":"St\u00e5hl D, Bosch J (2013) Experienced benefits of continuous integration in industry software product development: a case study. In: The 12th iasted international conference on software engineering, (innsbruck, austria, 2013), pp 736\u2013743. https:\/\/doi.org\/10.2316\/P.2013.796-012","DOI":"10.2316\/P.2013.796-012"},{"key":"10682_CR45","doi-asserted-by":"publisher","unstructured":"Thompson C (2017) Large-scale analysis of modern code review practices and software security in open source software. In: Proceedings of the 13th international conference on predictive models and data analytics in software engineering (2017), pp 83\u201392. https:\/\/doi.org\/10.1145\/3127005.3127014","DOI":"10.1145\/3127005.3127014"},{"key":"10682_CR46","unstructured":"Vadapalli S (2018) DevOps: continuous delivery, integration, and deployment with DevOps dive into the core DevOps strategies. Packt Publishing. https:\/\/dl.acm.org\/doi\/10.5555\/3235131"},{"key":"10682_CR47","doi-asserted-by":"publisher","unstructured":"Vasilescu B, Yu Y, Wang H, Devanbu P, Filkov V (2015) Quality and productivity outcomes relating to continuous integration in GitHub. In: Proceedings of the 2015 10th joint meeting on foundations of software engineering, pp 805\u2013816. https:\/\/doi.org\/10.1145\/2786805.2786850","DOI":"10.1145\/2786805.2786850"},{"key":"10682_CR48","doi-asserted-by":"publisher","unstructured":"Vassallo C, Palomba F, Bacchelli A, Gall HC (2018) Continuous code quality: are we (really) doing that? In: Proceedings of the 33rd ACM\/IEEE international conference on automated software engineering, ASE 2018, pp 790\u2013795. https:\/\/doi.org\/10.1145\/3238147.3240729","DOI":"10.1145\/3238147.3240729"},{"key":"10682_CR49","doi-asserted-by":"publisher","unstructured":"Widder DG, Hilton M, K\u00e4stner C, Vasilescu B (2019) A conceptual replication of continuous integration pain points in the context of travis CI. Association for Computing Machinery, New York, NY, USA. https:\/\/doi.org\/10.1145\/3338906.3338922","DOI":"10.1145\/3338906.3338922"},{"key":"10682_CR50","doi-asserted-by":"publisher","unstructured":"Yarlagadda RT (2018) Understanding DevOps & bridging the gap from continuous integration to continuous delivery. Understanding DevOps & bridging the gap from continuous integration to continuous delivery\u2019, International Journal of Emerging Technologies and Innovative Research (www. jetir. org), ISSN pp 2349\u20135162. https:\/\/doi.org\/10.1109\/INTECH.2015.7173368","DOI":"10.1109\/INTECH.2015.7173368"},{"key":"10682_CR51","doi-asserted-by":"publisher","unstructured":"Zhao Y, Serebrenik A, Zhou Y, Filkov V, Vasilescu B (2017) The impact of continuous integration on other software development practices: A large-scale empirical study. In: 2017 32nd IEEE\/ACM international conference on automated software engineering (ASE), pp 60\u201371. https:\/\/doi.org\/10.1109\/ASE.2017.8115619","DOI":"10.1109\/ASE.2017.8115619"}],"container-title":["Empirical Software Engineering"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10664-025-10682-6.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s10664-025-10682-6\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10664-025-10682-6.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,9,13]],"date-time":"2025-09-13T08:55:38Z","timestamp":1757753738000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s10664-025-10682-6"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,6,13]]},"references-count":51,"journal-issue":{"issue":"5","published-print":{"date-parts":[[2025,9]]}},"alternative-id":["10682"],"URL":"https:\/\/doi.org\/10.1007\/s10664-025-10682-6","relation":{},"ISSN":["1382-3256","1573-7616"],"issn-type":[{"type":"print","value":"1382-3256"},{"type":"electronic","value":"1573-7616"}],"subject":[],"published":{"date-parts":[[2025,6,13]]},"assertion":[{"value":"3 June 2025","order":1,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"13 June 2025","order":2,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Declarations"}},{"value":"Informed consent was obtained from all participants prior to completing the survey.","order":2,"name":"Ethics","group":{"name":"EthicsHeading","label":"Informed Consent"}},{"value":"The authors declare that they have no conflicts of interest or competing interests.","order":3,"name":"Ethics","group":{"name":"EthicsHeading","label":"Competing Interests"}}],"article-number":"125"}}