{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,14]],"date-time":"2026-10-14T21:38:06Z","timestamp":1792013886124,"version":"4.4.0"},"reference-count":63,"publisher":"Elsevier BV","license":[{"start":{"date-parts":[[2026,6,1]],"date-time":"2026-06-01T00:00:00Z","timestamp":1780272000000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.elsevier.com\/tdm\/userlicense\/1.0\/"},{"start":{"date-parts":[[2026,6,1]],"date-time":"2026-06-01T00:00:00Z","timestamp":1780272000000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.elsevier.com\/legal\/tdmrep-license"},{"start":{"date-parts":[[2026,6,1]],"date-time":"2026-06-01T00:00:00Z","timestamp":1780272000000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-017"},{"start":{"date-parts":[[2026,6,1]],"date-time":"2026-06-01T00:00:00Z","timestamp":1780272000000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"},{"start":{"date-parts":[[2026,6,1]],"date-time":"2026-06-01T00:00:00Z","timestamp":1780272000000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-012"},{"start":{"date-parts":[[2026,6,1]],"date-time":"2026-06-01T00:00:00Z","timestamp":1780272000000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2026,6,1]],"date-time":"2026-06-01T00:00:00Z","timestamp":1780272000000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-004"}],"funder":[{"DOI":"10.13039\/501100002661","name":"FRS-FNRS","doi-asserted-by":"publisher","award":["F.4515.23"],"award-info":[{"award-number":["F.4515.23"]}],"id":[{"id":"10.13039\/501100002661","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100002661","name":"FRS-FNRS","doi-asserted-by":"publisher","award":["T.0149.22"],"award-info":[{"award-number":["T.0149.22"]}],"id":[{"id":"10.13039\/501100002661","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["elsevier.com","sciencedirect.com"],"crossmark-restriction":true},"short-container-title":["Journal of Systems and Software"],"published-print":{"date-parts":[[2026,6]]},"DOI":"10.1016\/j.jss.2026.112824","type":"journal-article","created":{"date-parts":[[2026,2,18]],"date-time":"2026-02-18T16:14:52Z","timestamp":1771431292000},"page":"112824","update-policy":"https:\/\/doi.org\/10.1016\/elsevier_cm_policy","source":"Crossref","is-referenced-by-count":1,"special_numbering":"C","title":["An empirical study of the evolution of GitHub actions workflows"],"prefix":"10.1016","volume":"236","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-4859-1546","authenticated-orcid":false,"given":"Pooya","family":"Rostami Mazrae","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-5824-5823","authenticated-orcid":false,"given":"Alexandre","family":"Decan","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-3636-5020","authenticated-orcid":false,"given":"Tom","family":"Mens","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-8619-726X","authenticated-orcid":false,"given":"Mairieli","family":"Wessel","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"78","reference":[{"key":"10.1016\/j.jss.2026.112824_bib0001","series-title":"International Symposium on Empirical Software Engineering and Measurement","first-page":"1","article-title":"On the abandonment and survival of open source projects: an empirical investigation","author":"Avelino","year":"2019"},{"key":"10.1016\/j.jss.2026.112824_bib0002","series-title":"Extreme Programming Explained: Embrace Change","author":"Beck","year":"2000"},{"key":"10.1016\/j.jss.2026.112824_bib0003","series-title":"International Conference on Mining Software Repositories (MSR)","first-page":"356","article-title":"Oops, my tests broke the build: an explorative analysis of travis ci with github","author":"Beller","year":"2017"},{"key":"10.1016\/j.jss.2026.112824_bib0004","series-title":"International Workshop on Software Engineering for Computational Science and Engineering","first-page":"68","article-title":"Implementing continuous integration software in an established computational chemistry software package","author":"Betz","year":"2013"},{"issue":"6","key":"10.1016\/j.jss.2026.112824_bib0005","doi-asserted-by":"crossref","DOI":"10.1145\/3582083","article-title":"Taking flight with copilot: early insights and opportunities of ai-powered pair-programming tools","volume":"20","author":"Bird","year":"2023","journal-title":"ACM Queue"},{"key":"10.1016\/j.jss.2026.112824_bib0006","series-title":"International Working Conference on Mining Software Repositories (MSR)","first-page":"1","article-title":"The promises and perils of mining git","author":"Bird","year":"2009"},{"key":"10.1016\/j.jss.2026.112824_bib0007","series-title":"International Conference on Software Engineering (ICSE)","article-title":"Resource usage and optimization opportunities in workflows of github actions","author":"Bouzenia","year":"2024"},{"issue":"4","key":"10.1016\/j.jss.2026.112824_bib0008","doi-asserted-by":"crossref","first-page":"271","DOI":"10.1016\/j.aucc.2012.07.002","article-title":"Sample size: how many is enough?","volume":"25","author":"Burmeister","year":"2012","journal-title":"Australian Critical Care"},{"key":"10.1016\/j.jss.2026.112824_bib0009","series-title":"International Conference on Mining Software Repositories (MSR)","author":"Cardoen","year":"2024"},{"key":"10.1016\/j.jss.2026.112824_bib0010","series-title":"International Conference on Software Analysis, Evolution and Reengineering (SANER)","article-title":"An empirical analysis of code clones in github actions workflows","author":"Cardoen","year":"2026"},{"issue":"2","key":"10.1016\/j.jss.2026.112824_bib0011","first-page":"50","article-title":"Continuous delivery: huge benefits, but challenges too","volume":"32","author":"Chen","year":"2015","journal-title":"IEEE Software - Special Issue on Release Engineering"},{"key":"10.1016\/j.jss.2026.112824_bib0012","doi-asserted-by":"crossref","first-page":"72","DOI":"10.1016\/j.jss.2017.02.013","article-title":"Continuous delivery: overcoming adoption challenges","volume":"128","author":"Chen","year":"2017","journal-title":"J. Syst. Soft."},{"key":"10.1016\/j.jss.2026.112824_bib0013","series-title":"International Conference on Software Quality, Reliability and Security Companion (QRS-C)","first-page":"1","article-title":"Let\u2019s supercharge the workflows: an empirical study of github actions","author":"Chen","year":"2021"},{"issue":"2","key":"10.1016\/j.jss.2026.112824_bib0014","doi-asserted-by":"crossref","first-page":"101","DOI":"10.1007\/s11334-017-0303-4","article-title":"An empirical comparison of developer retention in the rubygems and npm software ecosystems","volume":"13","author":"Constantinou","year":"2017","journal-title":"Innov. Syst. Softw. Eng."},{"key":"10.1016\/j.jss.2026.112824_bib0015","series-title":"International Conference on Mining Software Repositories (MSR)","first-page":"181","article-title":"On the impact of security vulnerabilities in the npm package dependency network","author":"Decan","year":"2018"},{"key":"10.1016\/j.jss.2026.112824_bib0016","doi-asserted-by":"crossref","DOI":"10.1016\/j.jss.2023.111827","article-title":"On the outdatedness of workflows in the github actions ecosystem","volume":"206","author":"Decan","year":"2023","journal-title":"J. Syst. Softw."},{"key":"10.1016\/j.jss.2026.112824_bib0017","doi-asserted-by":"crossref","first-page":"381","DOI":"10.1007\/s10664-017-9589-y","article-title":"An empirical comparison of dependency network evolution in seven software packaging ecosystems","volume":"24","author":"Decan","year":"2019","journal-title":"Empir. Softw. Eng."},{"key":"10.1016\/j.jss.2026.112824_bib0018","series-title":"International Conference on Software Maintenance and Evolution (ICSME)","article-title":"On the use of github actions in software development repositories","author":"Decan","year":"2022"},{"key":"10.1016\/j.jss.2026.112824_bib0019","series-title":"International Conference on Mining Software Repositories (MSR)","first-page":"692","article-title":"Quantifying security issues in reusable javascript actions in github workflows","author":"Delicheh","year":"2024"},{"key":"10.1016\/j.jss.2026.112824_bib0020","series-title":"International Conference on Software Maintenance and Evolution (ICSME)","first-page":"291","article-title":"An analysis of 35 million jobs of travis CI","author":"Durieux","year":"2019"},{"key":"10.1016\/j.jss.2026.112824_bib0021","series-title":"Continuous Integration: Improving Software Quality And Reducing Risk","author":"Duvall","year":"2007"},{"issue":"7","key":"10.1016\/j.jss.2026.112824_bib0022","doi-asserted-by":"crossref","first-page":"2570","DOI":"10.1109\/TSE.2021.3064953","article-title":"Uncovering the benefits and challenges of continuous integration practices","volume":"48","author":"Elazhary","year":"2022","journal-title":"IEEE Trans. Softw. Eng."},{"issue":"1","key":"10.1016\/j.jss.2026.112824_bib0023","doi-asserted-by":"crossref","first-page":"33","DOI":"10.1109\/TSE.2018.2838131","article-title":"Use and misuse of continuous integration features: an empirical study of projects that (mis) use travis ci","volume":"46","author":"Gallaba","year":"2018","journal-title":"Trans. Softw. Eng."},{"issue":"1","key":"10.1016\/j.jss.2026.112824_bib0024","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1016\/j.iheduc.2005.11.001","article-title":"Revisiting methodological issues in transcript analysis: negotiated coding and reliability","volume":"9","author":"Garrison","year":"2006","journal-title":"Inte. Higher Educ."},{"key":"10.1016\/j.jss.2026.112824_bib0025","series-title":"Transactions on Software Engineering and Methodology","article-title":"CI\/CD configuration practices in open-source android apps: an empirical study","author":"Ghaleb","year":"2024"},{"key":"10.1016\/j.jss.2026.112824_bib0026","series-title":"International Conference on Software Testing, Verification and Validation Workshops (ICSTW)","first-page":"1","article-title":"Automated testing in the continuous delivery pipeline: a case study of an online company","author":"Gmeiner","year":"2015"},{"key":"10.1016\/j.jss.2026.112824_bib0027","series-title":"International Conference on Software Analysis, Evolution and Reengineering (SANER)","article-title":"On the rise and fall of ci services in github","author":"Golzadeh","year":"2022"},{"issue":"4","key":"10.1016\/j.jss.2026.112824_bib0028","doi-asserted-by":"crossref","first-page":"773","DOI":"10.1093\/biomet\/75.4.773","article-title":"Some implications of an alternative definition of the multiple comparison problem","volume":"75","author":"Halperin","year":"1988","journal-title":"Biometrika"},{"key":"10.1016\/j.jss.2026.112824_bib0029","series-title":"International Conference on Automated Software Engineering (ASE)","first-page":"426","article-title":"Dig, usage, costs, and benefits of continuous integration in open-source projects","author":"Hilton","year":"2016"},{"key":"10.1016\/j.jss.2026.112824_bib0030","series-title":"International Working Conference on Source Code Analysis and Manipulation (SCAM)","first-page":"61","article-title":"Cigar: contrastive learning for github action recommendation","author":"Huang","year":"2023"},{"key":"10.1016\/j.jss.2026.112824_bib0031","series-title":"PipelineAsCode: A CI\/CD workflow management system through configuration files at ByteDance, in: International Conference on Software Analysis, Evolution and Reengineering (SANER)","author":"Jin","year":"2024"},{"issue":"5","key":"10.1016\/j.jss.2026.112824_bib0032","doi-asserted-by":"crossref","first-page":"2035","DOI":"10.1007\/s10664-015-9393-5","article-title":"An in-depth study of the promises and perils of mining github","volume":"21","author":"Kalliamvakou","year":"2016","journal-title":"Empir. Softw. Eng."},{"key":"10.1016\/j.jss.2026.112824_bib0033","series-title":"Insights into Developers\u2019 Abandonment in FLOSS Projects, in: Intelligent Sustainable Systems: Selected Papers of WorldS4 2021","volume":"Vol. 1","author":"Kaur","year":"2022"},{"key":"10.1016\/j.jss.2026.112824_bib0034","series-title":"International Working Conference on Source Code Analysis and Manipulation (SCAM)","article-title":"Catching smells in the act: a github actions workflow investigation","author":"Khatami","year":"2024"},{"key":"10.1016\/j.jss.2026.112824_bib0035","series-title":"International Conference on Mining Software Repositories (MSR)","first-page":"420","article-title":"How do software developers use github actions to automate their workflows?","author":"Kinsman","year":"2021"},{"key":"10.1016\/j.jss.2026.112824_bib0036","series-title":"Practical Experience With Test-Driven Development During Commissioning of the Multi-Star AO System ARGOS","volume":"Vol. 9152","author":"Kulas","year":"2014"},{"key":"10.1016\/j.jss.2026.112824_bib0037","series-title":"European Workshop on Software Process Technology (EWPST)","first-page":"108","article-title":"Laws of software evolution revisited","author":"Lehman","year":"1996"},{"key":"10.1016\/j.jss.2026.112824_bib0038","series-title":"International Conference on Power System Technology","first-page":"2441","article-title":"Implementation of continuous integration and automated testing in software development of smart grid scheduling support system","author":"Lu","year":"2014"},{"key":"10.1016\/j.jss.2026.112824_bib0039","series-title":"Proceedings of the 21st International Conference on Mining Software Repositories (MSR \u201924)","first-page":"682","article-title":"gawd: A differencing tool for GitHub Actions workflows","author":"Rostami Mazrae","year":"2024"},{"key":"10.1016\/j.jss.2026.112824_bib0040","article-title":"A preliminary study of github actions workflow changes","volume":"3483","author":"Rostami Mazrae","year":"2023","journal-title":"CEUR Workshop Proc."},{"issue":"2","key":"10.1016\/j.jss.2026.112824_bib0041","doi-asserted-by":"crossref","first-page":"52","DOI":"10.1007\/s10664-022-10285-5","article-title":"On the usage, co-usage and migration of CI\/CD tools: a qualitative analysis","volume":"28","author":"Rostami Mazrae","year":"2023","journal-title":"Empir. Softw. Eng."},{"key":"10.1016\/j.jss.2026.112824_bib0042","series-title":"International Conference on Mining Software Repositories (MSR)","first-page":"234","article-title":"Investigating the resolution of vulnerable dependencies with dependabot security updates","author":"Mohayeji","year":"2023"},{"key":"10.1016\/j.jss.2026.112824_bib0043","series-title":"International Conference on Mining Software Repositories (MSR) - Data and Tool Showcase Track","article-title":"Ghalogs: large-scale dataset of github actions runs","author":"Moriconi","year":"2025"},{"key":"10.1016\/j.jss.2026.112824_bib0044","series-title":"International Symposium on Software Reliability Engineering","first-page":"309","article-title":"Change bursts as defect predictors","author":"Nagappan","year":"2010"},{"key":"10.1016\/j.jss.2026.112824_bib0045","series-title":"International Conference on Mining Software Repositories (MSR)","article-title":"Promises, perils, and (timely) heuristics for mining coding agent activity","author":"Robbes","year":"2026"},{"key":"10.1016\/j.jss.2026.112824_bib0046","series-title":"Annual Meeting of the Southern Association for Institutional Research","article-title":"Exploring methods for evaluating group differences on the nsse and other surveys: are the t-test and cohen\u2019s d indices the most appropriate choices?","author":"Romano","year":"2006"},{"key":"10.1016\/j.jss.2026.112824_bib0047","series-title":"International Conference on Evaluation and Assessment in Software Engineering (EASE)","article-title":"Developers\u2019 Perception of github actions: a survey analysis","author":"Saroar","year":"2023"},{"key":"10.1016\/j.jss.2026.112824_bib0048","series-title":"International Conference on Software Engineering (ICSE)","first-page":"21","article-title":"Continuous deployment at facebook and OANDA","author":"Savor","year":"2016"},{"key":"10.1016\/j.jss.2026.112824_bib0049","doi-asserted-by":"crossref","first-page":"3909","DOI":"10.1109\/ACCESS.2017.2685629","article-title":"Continuous integration, delivery and deployment: a systematic review on approaches, tools, challenges and practices","volume":"5","author":"Shahin","year":"2017","journal-title":"IEEE Access"},{"issue":"3","key":"10.1016\/j.jss.2026.112824_bib0050","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1007\/s10664-021-10114-1","article-title":"The effects of continuous integration on software development: a systematic literature review","volume":"27","author":"Soares","year":"2022","journal-title":"Empir. Softw. Eng."},{"key":"10.1016\/j.jss.2026.112824_bib0051","series-title":"Card sorting: Designing usable categories","author":"Spencer","year":"2009"},{"key":"10.1016\/j.jss.2026.112824_bib0052","series-title":"Quality in qualitative evaluation: a framework for assessing research evidence","author":"Spencer","year":"2003"},{"key":"10.1016\/j.jss.2026.112824_bib0053","series-title":"International Conference on Software Analysis, Evolution and Reengineering (SANER)","first-page":"123","article-title":"Evolution of github action workflows","author":"Valenzuela-Toledo","year":"2022"},{"key":"10.1016\/j.jss.2026.112824_bib0054","series-title":"International Conference on Source Code Analysis and Manipulation (SCAM)","article-title":"The hidden costs of automation: an empirical study on github actions workflow maintenance","author":"Valenzuela-Toledo","year":"2024"},{"key":"10.1016\/j.jss.2026.112824_bib0055","series-title":"Joint Meeting on Foundations of Software Engineering (FSE)","first-page":"805","article-title":"Quality and productivity outcomes relating to continuous integration in github","author":"Vasilescu","year":"2015"},{"key":"10.1016\/j.jss.2026.112824_bib0056","series-title":"International Conference on Software Engineering (ICSE)","first-page":"105","article-title":"Automated reporting of anti-patterns and decay in continuous integration","author":"Vassallo","year":"2019"},{"key":"10.1016\/j.jss.2026.112824_bib0057","series-title":"Mixed Methods Research","author":"Watkins","year":"2015"},{"key":"10.1016\/j.jss.2026.112824_bib0058","series-title":"Software Ecosystems: Tooling and Analytics","article-title":"The github development workflow automation ecosystems","author":"Wessel","year":"2023"},{"key":"10.1016\/j.jss.2026.112824_bib0059","series-title":"Experimentation in Software Engineering","volume":"Vol. 236","author":"Wohlin","year":"2012"},{"key":"10.1016\/j.jss.2026.112824_bib0060","series-title":"International Conference on Software Maintenance and Evolution (ICSME)","first-page":"471","article-title":"CI\/CD pipelines evolution and restructuring: a qualitative and quantitative study","author":"Zampetti","year":"2021"},{"key":"10.1016\/j.jss.2026.112824_bib0061","series-title":"International Conference on Mining Software Repositories (MSR)","first-page":"334","article-title":"How open source projects use static code analysis tools in continuous integration pipelines","author":"Zampetti","year":"2017"},{"key":"10.1016\/j.jss.2026.112824_bib0062","doi-asserted-by":"crossref","first-page":"1095","DOI":"10.1007\/s10664-019-09785-8","article-title":"An empirical characterization of bad practices in continuous integration","volume":"25","author":"Zampetti","year":"2020","journal-title":"Empir. Softw. Eng."},{"key":"10.1016\/j.jss.2026.112824_bib0063","series-title":"International Conference on Availability, Reliability and Security (ARES), ARES \u201924","article-title":"On the effectiveness of large language models for github workflows","author":"Zhang","year":"2024"}],"container-title":["Journal of Systems and Software"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/api.elsevier.com\/content\/article\/PII:S0164121226000580?httpAccept=text\/xml","content-type":"text\/xml","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/api.elsevier.com\/content\/article\/PII:S0164121226000580?httpAccept=text\/plain","content-type":"text\/plain","content-version":"vor","intended-application":"text-mining"}],"deposited":{"date-parts":[[2026,10,8]],"date-time":"2026-10-08T20:44:31Z","timestamp":1791492271000},"score":1,"resource":{"primary":{"URL":"https:\/\/linkinghub.elsevier.com\/retrieve\/pii\/S0164121226000580"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026,6]]},"references-count":63,"alternative-id":["S0164121226000580"],"URL":"https:\/\/doi.org\/10.1016\/j.jss.2026.112824","relation":{},"ISSN":["0164-1212"],"issn-type":[{"value":"0164-1212","type":"print"}],"subject":[],"published":{"date-parts":[[2026,6]]},"assertion":[{"value":"Elsevier","name":"publisher","label":"This article is maintained by"},{"value":"An empirical study of the evolution of GitHub actions workflows","name":"articletitle","label":"Article Title"},{"value":"Journal of Systems and Software","name":"journaltitle","label":"Journal Title"},{"value":"https:\/\/doi.org\/10.1016\/j.jss.2026.112824","name":"articlelink","label":"CrossRef DOI link to publisher maintained version"},{"value":"article","name":"content_type","label":"Content Type"},{"value":"\u00a9 2026 Elsevier Inc. All rights are reserved, including those for text and data mining, AI training, and similar technologies.","name":"copyright","label":"Copyright"}],"article-number":"112824"}}