{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,1,23]],"date-time":"2026-01-23T18:55:34Z","timestamp":1769194534002,"version":"3.49.0"},"publisher-location":"New York, NY, USA","reference-count":56,"publisher":"ACM","license":[{"start":{"date-parts":[[2022,9,19]],"date-time":"2022-09-19T00:00:00Z","timestamp":1663545600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"DOI":"10.13039\/100000001","name":"NSF (National Science Foundation)","doi-asserted-by":"publisher","award":["2152819"],"award-info":[{"award-number":["2152819"]}],"id":[{"id":"10.13039\/100000001","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2022,9,19]]},"DOI":"10.1145\/3544902.3546237","type":"proceedings-article","created":{"date-parts":[[2022,9,7]],"date-time":"2022-09-07T04:07:45Z","timestamp":1662523665000},"page":"69-79","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":9,"title":["Characterizing the Usage of CI Tools in ML Projects"],"prefix":"10.1145","author":[{"given":"Dhia Elhaq","family":"Rzig","sequence":"first","affiliation":[{"name":"University of Michigan-Dearborn, United States"}]},{"given":"Foyzul","family":"Hassan","sequence":"additional","affiliation":[{"name":"University of Michigan-Dearborn, United States"}]},{"given":"Chetan","family":"Bansal","sequence":"additional","affiliation":[{"name":"Microsoft Research, United States"}]},{"given":"Nachiappan","family":"Nagappan","sequence":"additional","affiliation":[{"name":"Microsoft Research, United States"}]}],"member":"320","published-online":{"date-parts":[[2022,9,19]]},"reference":[{"key":"e_1_3_2_1_1_1","unstructured":"2000. Continuous Integration. https:\/\/martinfowler.com\/articles\/continuousIntegration.html Accessed: 2021-08-25.  2000. Continuous Integration. https:\/\/martinfowler.com\/articles\/continuousIntegration.html Accessed: 2021-08-25."},{"key":"e_1_3_2_1_2_1","unstructured":"2017. After failure output gets truncated unless sleep is used - Issue #6018 - travis-ci\/travis-ci. https:\/\/github.com\/travis-ci\/travis-ci\/issues\/6018 Accessed: 2021-08-27.  2017. After failure output gets truncated unless sleep is used - Issue #6018 - travis-ci\/travis-ci. https:\/\/github.com\/travis-ci\/travis-ci\/issues\/6018 Accessed: 2021-08-27."},{"key":"e_1_3_2_1_3_1","unstructured":"2018. DigitalOcean Blog. https:\/\/www.digitalocean.com\/blog\/currents-march-2018\/  2018. DigitalOcean Blog. https:\/\/www.digitalocean.com\/blog\/currents-march-2018\/"},{"key":"e_1_3_2_1_4_1","unstructured":"[\n  4\n  ]  2021. https:\/\/neptune.ai\/blog\/machine-learning-model-management  [4] 2021. https:\/\/neptune.ai\/blog\/machine-learning-model-management"},{"key":"e_1_3_2_1_5_1","unstructured":"2021. Amazon SageMaker \u2013 Machine Learning \u2013 Amazon Web Services. https:\/\/aws.amazon.com\/sagemaker\/ accessed 08-28-2021.  2021. Amazon SageMaker \u2013 Machine Learning \u2013 Amazon Web Services. https:\/\/aws.amazon.com\/sagemaker\/ accessed 08-28-2021."},{"key":"e_1_3_2_1_6_1","unstructured":"2021. API Developer Documentation - Travis CI. https:\/\/developer.travis-ci.com\/ Accessed: 2021-08-20.  2021. API Developer Documentation - Travis CI. https:\/\/developer.travis-ci.com\/ Accessed: 2021-08-20."},{"key":"e_1_3_2_1_7_1","unstructured":"2021. Buildbot - The Continuous Integration Framework. https:\/\/www.buildbot.net\/ accessed 08-31-2021.  2021. Buildbot - The Continuous Integration Framework. https:\/\/www.buildbot.net\/ accessed 08-31-2021."},{"key":"e_1_3_2_1_8_1","unstructured":"2021. CircleCI-Continuous Integration and Delivery. https:\/\/circleci.com\/ accessed 08-31-2021.  2021. CircleCI-Continuous Integration and Delivery. https:\/\/circleci.com\/ accessed 08-31-2021."},{"key":"e_1_3_2_1_9_1","unstructured":"2021. Cloud Build Serverless CI\/CD Platform. https:\/\/cloud.google.com\/build accessed 08-31-2021.  2021. Cloud Build Serverless CI\/CD Platform. https:\/\/cloud.google.com\/build accessed 08-31-2021."},{"key":"e_1_3_2_1_10_1","unstructured":"2021. Continuous Integration and Deployment service for Windows Linux and macOS. https:\/\/www.appveyor.com\/ accessed 08-31-2021.  2021. Continuous Integration and Deployment service for Windows Linux and macOS. https:\/\/www.appveyor.com\/ accessed 08-31-2021."},{"key":"e_1_3_2_1_11_1","unstructured":"2021. Continuous Integration with Visual Studio Team Services. https:\/\/microsoft.github.io\/PartsUnlimitedMRP\/cicd\/200.3x-CICD-CI.html accessed 08-31-2021.  2021. Continuous Integration with Visual Studio Team Services. https:\/\/microsoft.github.io\/PartsUnlimitedMRP\/cicd\/200.3x-CICD-CI.html accessed 08-31-2021."},{"key":"e_1_3_2_1_12_1","unstructured":"2021. DocConverter: A Java library to parse yaml file. https:\/\/github.com\/assimbly\/docconverter Accessed: 2021-08-20.  2021. DocConverter: A Java library to parse yaml file. https:\/\/github.com\/assimbly\/docconverter Accessed: 2021-08-20."},{"key":"e_1_3_2_1_13_1","unstructured":"2021. Docker: open source containerization platform. https:\/\/www.docker.com\/ accessed 08-31-2021.  2021. Docker: open source containerization platform. https:\/\/www.docker.com\/ accessed 08-31-2021."},{"key":"e_1_3_2_1_14_1","unstructured":"2021. ease.ml\\ci. http:\/\/ease.ml\/ci accessed 08-31-2021.  2021. ease.ml\\ci. http:\/\/ease.ml\/ci accessed 08-31-2021."},{"key":"e_1_3_2_1_15_1","unstructured":"2021. GitHub Actions Documentation. https:\/\/docs.github.com\/en\/actions accessed 08-31-2021.  2021. GitHub Actions Documentation. https:\/\/docs.github.com\/en\/actions accessed 08-31-2021."},{"key":"e_1_3_2_1_16_1","unstructured":"2021. GitHub REST API. https:\/\/docs.github.com\/en\/rest Accessed: 2021-08-27.  2021. GitHub REST API. https:\/\/docs.github.com\/en\/rest Accessed: 2021-08-27."},{"key":"e_1_3_2_1_17_1","unstructured":"2021. GitLab CI\/CD. https:\/\/docs.gitlab.com\/ee\/ci\/ accessed 08-31-2021.  2021. GitLab CI\/CD. https:\/\/docs.gitlab.com\/ee\/ci\/ accessed 08-31-2021."},{"key":"e_1_3_2_1_18_1","unstructured":"2021. How to use unittest-based tests with pytest \u2014 pytest documentation. https:\/\/doc.pytest.org\/en\/latest\/how-to\/unittest.html Accessed: 2021-08-27.  2021. How to use unittest-based tests with pytest \u2014 pytest documentation. https:\/\/doc.pytest.org\/en\/latest\/how-to\/unittest.html Accessed: 2021-08-27."},{"key":"e_1_3_2_1_19_1","unstructured":"2021. Introduction to Kubeflow. https:\/\/www.kubeflow.org\/docs\/about\/kubeflow\/ accessed 08-28-2021.  2021. Introduction to Kubeflow. https:\/\/www.kubeflow.org\/docs\/about\/kubeflow\/ accessed 08-28-2021."},{"key":"e_1_3_2_1_20_1","unstructured":"2021. Jenkins \u2013 an open source automation server which enables developers around the world to reliably build test and deploy their software. https:\/\/www.jenkins.io\/https:\/www.jenkins.io\/ accessed 08-31-2021.  2021. Jenkins \u2013 an open source automation server which enables developers around the world to reliably build test and deploy their software. https:\/\/www.jenkins.io\/https:\/www.jenkins.io\/ accessed 08-31-2021."},{"key":"e_1_3_2_1_21_1","unstructured":"2021. PyTravisCI documentation. https:\/\/pytravisci.readthedocs.io\/en\/latest\/usage\/ Accessed: 2021-08-23.  2021. PyTravisCI documentation. https:\/\/pytravisci.readthedocs.io\/en\/latest\/usage\/ Accessed: 2021-08-23."},{"key":"e_1_3_2_1_22_1","unstructured":"2021. Travis CI | Test and Deploy With Confidence. https:\/\/www.travis-ci.com\/ accessed 08-31-2021.  2021. Travis CI | Test and Deploy With Confidence. https:\/\/www.travis-ci.com\/ accessed 08-31-2021."},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"crossref","unstructured":"2021. Travis CI Documentation. https:\/\/docs.travis-ci.com\/user\/deployment\/ accessed 08-31-2021.  2021. Travis CI Documentation. https:\/\/docs.travis-ci.com\/user\/deployment\/ accessed 08-31-2021.","DOI":"10.1093\/oso\/9780198844129.003.0003"},{"key":"e_1_3_2_1_24_1","unstructured":"2021. Travis CI Documentation - Using YAML as a build configuration language. https:\/\/docs.travis-ci.com\/user\/build-config-yaml\/ accessed 08-31-2021.  2021. Travis CI Documentation - Using YAML as a build configuration language. https:\/\/docs.travis-ci.com\/user\/build-config-yaml\/ accessed 08-31-2021."},{"key":"e_1_3_2_1_25_1","unstructured":"2021. What is AWS CodeBuild? - AWS CodeBuild. https:\/\/docs.aws.amazon.com\/codebuild\/latest\/userguide\/welcome.html accessed 08-31-2021.  2021. What is AWS CodeBuild? - AWS CodeBuild. https:\/\/docs.aws.amazon.com\/codebuild\/latest\/userguide\/welcome.html accessed 08-31-2021."},{"key":"e_1_3_2_1_26_1","unstructured":"2022. Replication package for this work. https:\/\/figshare.com\/s\/13f6afeb3fbc668cfbe4  2022. Replication package for this work. https:\/\/figshare.com\/s\/13f6afeb3fbc668cfbe4"},{"key":"e_1_3_2_1_27_1","volume-title":"My Tests Broke the Build: An Explorative Analysis of Travis CI with GitHub. In 2017 IEEE\/ACM 14th International Conference on Mining Software Repositories (MSR). IEEE, 356\u2013367","author":"Beller Moritz","year":"2017","unstructured":"Moritz Beller , Georgios Gousios , and Andy Zaidman . 2017 . Oops , My Tests Broke the Build: An Explorative Analysis of Travis CI with GitHub. In 2017 IEEE\/ACM 14th International Conference on Mining Software Repositories (MSR). IEEE, 356\u2013367 . https:\/\/doi.org\/10.1109\/MSR.2017.62 10.1109\/MSR.2017.62 Moritz Beller, Georgios Gousios, and Andy Zaidman. 2017. Oops, My Tests Broke the Build: An Explorative Analysis of Travis CI with GitHub. In 2017 IEEE\/ACM 14th International Conference on Mining Software Repositories (MSR). IEEE, 356\u2013367. https:\/\/doi.org\/10.1109\/MSR.2017.62"},{"key":"e_1_3_2_1_28_1","unstructured":"G. Booch. 1991. Object Oriented Design with Applications. Benjamin\/Cummings Publishing Company. https:\/\/books.google.com\/books?id=w5VQAAAAMAAJ  G. Booch. 1991. Object Oriented Design with Applications. Benjamin\/Cummings Publishing Company. https:\/\/books.google.com\/books?id=w5VQAAAAMAAJ"},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.1188\/14.ONF.545-547"},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSME.2019.00044"},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"publisher","DOI":"10.1145\/3379597.3387460"},{"key":"e_1_3_2_1_32_1","doi-asserted-by":"publisher","DOI":"10.1111\/j.1468-0394.2005.00299.x"},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"publisher","DOI":"10.1177\/001316447303300309"},{"key":"e_1_3_2_1_34_1","doi-asserted-by":"publisher","DOI":"10.1145\/356674.356676"},{"key":"#cr-split#-e_1_3_2_1_35_1.1","doi-asserted-by":"crossref","unstructured":"Keheliya Gallaba Christian Macho Martin Pinzger and Shane McIntosh. 2018. Noise and heterogeneity in historical build data: an empirical study of Travis CI. ACM 87-97. https:\/\/doi.org\/10.1145\/3238147.3238171 10.1145\/3238147.3238171","DOI":"10.1145\/3238147.3238171"},{"key":"#cr-split#-e_1_3_2_1_35_1.2","doi-asserted-by":"crossref","unstructured":"Keheliya Gallaba Christian Macho Martin Pinzger and Shane McIntosh. 2018. Noise and heterogeneity in historical build data: an empirical study of Travis CI. ACM 87-97. https:\/\/doi.org\/10.1145\/3238147.3238171","DOI":"10.1145\/3238147.3238171"},{"key":"e_1_3_2_1_36_1","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2018.2838131"},{"key":"e_1_3_2_1_37_1","doi-asserted-by":"publisher","DOI":"10.1145\/3379597.3387473"},{"key":"e_1_3_2_1_38_1","volume-title":"Work Practices and Challenges in Pull-Based Development: The Integrator\u2019s Perspective. In 2015 IEEE\/ACM 37th IEEE International Conference on Software Engineering. IEEE, 358\u2013368","author":"Gousios Georgios","year":"2015","unstructured":"Georgios Gousios , Andy Zaidman , Margaret-Anne Storey , and Arie\u00a0van Deursen . 2015 . Work Practices and Challenges in Pull-Based Development: The Integrator\u2019s Perspective. In 2015 IEEE\/ACM 37th IEEE International Conference on Software Engineering. IEEE, 358\u2013368 . https:\/\/doi.org\/10.1109\/ICSE.2015.55 10.1109\/ICSE.2015.55 Georgios Gousios, Andy Zaidman, Margaret-Anne Storey, and Arie\u00a0van Deursen. 2015. Work Practices and Challenges in Pull-Based Development: The Integrator\u2019s Perspective. In 2015 IEEE\/ACM 37th IEEE International Conference on Software Engineering. IEEE, 358\u2013368. https:\/\/doi.org\/10.1109\/ICSE.2015.55"},{"key":"e_1_3_2_1_39_1","doi-asserted-by":"publisher","DOI":"10.1145\/2970276.2970358"},{"key":"e_1_3_2_1_40_1","doi-asserted-by":"publisher","DOI":"10.1145\/3394486.3403290"},{"key":"e_1_3_2_1_41_1","first-page":"2009","article-title":"Open coding","volume":"23","author":"Khandkar Shahedul\u00a0Huq","year":"2009","unstructured":"Shahedul\u00a0Huq Khandkar . 2009 . Open coding . University of Calgary 23 (2009), 2009 . Shahedul\u00a0Huq Khandkar. 2009. Open coding. University of Calgary 23(2009), 2009.","journal-title":"University of Calgary"},{"key":"e_1_3_2_1_42_1","volume-title":"Article 127 (Nov.","author":"Leite Leonardo","year":"2019","unstructured":"Leonardo Leite , Carla Rocha , Fabio Kon , Dejan Milojicic , and Paulo Meirelles . 2019. A Survey of DevOps Concepts and Challenges. ACM Comput. Surv. 52, 6 , Article 127 (Nov. 2019 ), 35\u00a0pages. https:\/\/doi.org\/10.1145\/3359981 10.1145\/3359981 Leonardo Leite, Carla Rocha, Fabio Kon, Dejan Milojicic, and Paulo Meirelles. 2019. A Survey of DevOps Concepts and Challenges. ACM Comput. Surv. 52, 6, Article 127 (Nov. 2019), 35\u00a0pages. https:\/\/doi.org\/10.1145\/3359981"},{"key":"e_1_3_2_1_43_1","doi-asserted-by":"publisher","DOI":"10.23919\/SoftCOM50211.2020.9238323"},{"key":"e_1_3_2_1_44_1","doi-asserted-by":"publisher","DOI":"10.1145\/1083142.1083143"},{"key":"e_1_3_2_1_45_1","article-title":"Unit testing: Test early, test often","author":"Olan Michael","year":"2003","unstructured":"Michael Olan . 2003 . Unit testing: Test early, test often . Journal of Computing Sciences in Colleges - JCSC 19 (Jan 2003). Michael Olan. 2003. Unit testing: Test early, test often. Journal of Computing Sciences in Colleges - JCSC 19 (Jan 2003).","journal-title":"Journal of Computing Sciences in Colleges - JCSC 19"},{"key":"e_1_3_2_1_46_1","first-page":"12","article-title":"Work practices and challenges in continuous integration: A survey with Travis CI users: Work practices and challenges: A survey with Travis CI users. Software","volume":"48","author":"Pinto Gustavo","year":"2018","unstructured":"Gustavo Pinto , Fernando Castor , Rodrigo Bonifacio , and Marcel Rebou\u00e7as . 2018 . Work practices and challenges in continuous integration: A survey with Travis CI users: Work practices and challenges: A survey with Travis CI users. Software : Practice and Experience 48 , 12 (Dec 2018), 2223\u20132236. https:\/\/doi.org\/10.1002\/spe.2637 10.1002\/spe.2637 Gustavo Pinto, Fernando Castor, Rodrigo Bonifacio, and Marcel Rebou\u00e7as. 2018. Work practices and challenges in continuous integration: A survey with Travis CI users: Work practices and challenges: A survey with Travis CI users. Software: Practice and Experience 48, 12 (Dec 2018), 2223\u20132236. https:\/\/doi.org\/10.1002\/spe.2637","journal-title":"Practice and Experience"},{"key":"e_1_3_2_1_47_1","unstructured":"Quantilus. 2020. Why is Machine Learning Important and How will it Impact Business?https:\/\/quantilus.com\/why-is-machine-learning-important-and-how-will-it-impact-business\/ Accessed: 2021-08-24.  Quantilus. 2020. Why is Machine Learning Important and How will it Impact Business?https:\/\/quantilus.com\/why-is-machine-learning-important-and-how-will-it-impact-business\/ Accessed: 2021-08-24."},{"key":"e_1_3_2_1_48_1","volume-title":"An Empirical Analysis of Build Failures in the Continuous Integration Workflows of Java-Based Open-Source Software. In 2017 IEEE\/ACM 14th International Conference on Mining Software Repositories (MSR)","author":"Rausch Thomas","year":"2017","unstructured":"Thomas Rausch , Waldemar Hummer , Philipp Leitner , and Stefan Schulte . 2017. An Empirical Analysis of Build Failures in the Continuous Integration Workflows of Java-Based Open-Source Software. In 2017 IEEE\/ACM 14th International Conference on Mining Software Repositories (MSR) . IEEE , 345\u2013355. https:\/\/doi.org\/10.1109\/MSR. 2017 .54 10.1109\/MSR.2017.54 Thomas Rausch, Waldemar Hummer, Philipp Leitner, and Stefan Schulte. 2017. An Empirical Analysis of Build Failures in the Continuous Integration Workflows of Java-Based Open-Source Software. In 2017 IEEE\/ACM 14th International Conference on Mining Software Repositories (MSR). IEEE, 345\u2013355. https:\/\/doi.org\/10.1109\/MSR.2017.54"},{"key":"e_1_3_2_1_49_1","doi-asserted-by":"publisher","DOI":"10.1145\/2568225.2568255"},{"key":"e_1_3_2_1_50_1","doi-asserted-by":"publisher","DOI":"10.1145\/3001878.3001882"},{"key":"e_1_3_2_1_51_1","first-page":"4","article-title":"There and back again: Can you compile that snapshot?: There and Back Again: Can you Compile that Snapshot?Journal of Software","volume":"29","author":"Tufano Michele","year":"2017","unstructured":"Michele Tufano , Fabio Palomba , Gabriele Bavota , Massimiliano Di\u00a0Penta , Rocco Oliveto , Andrea De\u00a0Lucia , and Denys Poshyvanyk . 2017 . There and back again: Can you compile that snapshot?: There and Back Again: Can you Compile that Snapshot?Journal of Software : Evolution and Process 29 , 4 (Apr 2017), e1838. https:\/\/doi.org\/10.1002\/smr.1838 10.1002\/smr.1838 Michele Tufano, Fabio Palomba, Gabriele Bavota, Massimiliano Di\u00a0Penta, Rocco Oliveto, Andrea De\u00a0Lucia, and Denys Poshyvanyk. 2017. There and back again: Can you compile that snapshot?: There and Back Again: Can you Compile that Snapshot?Journal of Software: Evolution and Process 29, 4 (Apr 2017), e1838. https:\/\/doi.org\/10.1002\/smr.1838","journal-title":"Evolution and Process"},{"key":"e_1_3_2_1_52_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSME.2014.62"},{"key":"e_1_3_2_1_53_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSME.2017.67"},{"key":"#cr-split#-e_1_3_2_1_54_1.1","doi-asserted-by":"crossref","unstructured":"David\u00a0Gray Widder Michael Hilton Christian K\u00e4stner and Bogdan Vasilescu. 2019. A conceptual replication of continuous integration pain points in the context of Travis CI. ACM 647-658. https:\/\/doi.org\/10.1145\/3338906.3338922 10.1145\/3338906.3338922","DOI":"10.1145\/3338906.3338922"},{"key":"#cr-split#-e_1_3_2_1_54_1.2","doi-asserted-by":"crossref","unstructured":"David\u00a0Gray Widder Michael Hilton Christian K\u00e4stner and Bogdan Vasilescu. 2019. A conceptual replication of continuous integration pain points in the context of Travis CI. ACM 647-658. https:\/\/doi.org\/10.1145\/3338906.3338922","DOI":"10.1145\/3338906.3338922"}],"event":{"name":"ESEM '22: ACM \/ IEEE International Symposium on Empirical Software Engineering and Measurement","location":"Helsinki Finland","acronym":"ESEM '22","sponsor":["SIGSOFT ACM Special Interest Group on Software Engineering"]},"container-title":["Proceedings of the 16th ACM \/ IEEE International Symposium on Empirical Software Engineering and Measurement"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3544902.3546237","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3544902.3546237","content-type":"application\/pdf","content-version":"vor","intended-application":"syndication"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3544902.3546237","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T19:00:07Z","timestamp":1750186807000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3544902.3546237"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2022,9,19]]},"references-count":56,"alternative-id":["10.1145\/3544902.3546237","10.1145\/3544902"],"URL":"https:\/\/doi.org\/10.1145\/3544902.3546237","relation":{},"subject":[],"published":{"date-parts":[[2022,9,19]]},"assertion":[{"value":"2022-09-19","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}