{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,5,12]],"date-time":"2026-05-12T13:02:14Z","timestamp":1778590934243,"version":"3.51.4"},"publisher-location":"New York, NY, USA","reference-count":58,"publisher":"ACM","license":[{"start":{"date-parts":[[2024,7,30]],"date-time":"2024-07-30T00:00:00Z","timestamp":1722297600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"funder":[{"DOI":"10.13039\/501100006374","name":"Defense Advanced Research Projects Agency","doi-asserted-by":"publisher","award":["N6600120C4031, N660012224037"],"award-info":[{"award-number":["N6600120C4031, N660012224037"]}],"id":[{"id":"10.13039\/501100006374","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100006374","name":"NSF (National Science Foundation)","doi-asserted-by":"publisher","award":["CNS-2247686"],"award-info":[{"award-number":["CNS-2247686"]}],"id":[{"id":"10.13039\/501100006374","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2024,7,30]]},"DOI":"10.1145\/3664476.3664497","type":"proceedings-article","created":{"date-parts":[[2024,7,25]],"date-time":"2024-07-25T12:35:50Z","timestamp":1721910950000},"page":"1-14","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":11,"title":["On the Effectiveness of Large Language Models for GitHub Workflows"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0009-0000-9106-1148","authenticated-orcid":false,"given":"Xinyu","family":"Zhang","sequence":"first","affiliation":[{"name":"Purdue University, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0009-0002-0205-7214","authenticated-orcid":false,"given":"Siddharth","family":"Muralee","sequence":"additional","affiliation":[{"name":"Purdue University, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0009-0002-7199-1879","authenticated-orcid":false,"given":"Sourag","family":"Cherupattamoolayil","sequence":"additional","affiliation":[{"name":"Purdue University, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-5124-6818","authenticated-orcid":false,"given":"Aravind","family":"Machiry","sequence":"additional","affiliation":[{"name":"Purdue University, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2024,7,30]]},"reference":[{"key":"e_1_3_2_1_1_1","unstructured":"2023. actionlint. https:\/\/github.com\/rhysd\/actionlint."},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.1109\/TIFS.2024.3374558"},{"key":"e_1_3_2_1_3_1","unstructured":"Simon Arvidsson and Johan Axell. 2023. Prompt engineering guidelines for LLMs in Requirements Engineering. (2023)."},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.14722\/aiscc.2024.23015"},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1145\/3560835.3564554"},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"publisher","DOI":"10.5555\/3495724.3495883"},{"key":"e_1_3_2_1_7_1","unstructured":"Mark Chen Jerry Tworek Heewoo Jun Qiming Yuan Henrique\u00a0Ponde de Oliveira\u00a0Pinto Jared Kaplan Harri Edwards Yuri Burda Nicholas Joseph Greg Brockman Alex Ray Raul Puri Gretchen Krueger Michael Petrov Heidy Khlaaf Girish Sastry Pamela Mishkin Brooke Chan Scott Gray Nick Ryder Mikhail Pavlov Alethea Power Lukasz Kaiser Mohammad Bavarian Clemens Winter Philippe Tillet Felipe\u00a0Petroski Such Dave Cummings Matthias Plappert Fotios Chantzis Elizabeth Barnes Ariel Herbert-Voss William\u00a0Hebgen Guss Alex Nichol Alex Paino Nikolas Tezak Jie Tang Igor Babuschkin Suchir Balaji Shantanu Jain William Saunders Christopher Hesse Andrew\u00a0N. Carr Jan Leike Josh Achiam Vedant Misra Evan Morikawa Alec Radford Matthew Knight Miles Brundage Mira Murati Katie Mayer Peter Welinder Bob McGrew Dario Amodei Sam McCandlish Ilya Sutskever and Wojciech Zaremba. 2021. Evaluating Large Language Models Trained on Code. arxiv:2107.03374\u00a0[cs.LG]"},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSME55016.2022.00029"},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/N19-1423"},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.36227\/techrxiv.22683919.v2"},{"key":"e_1_3_2_1_11_1","volume-title":"Analyzing Prompt Influence on Automated Method Generation: An Empirical Study with Copilot. arXiv preprint arXiv:2402.08430","author":"Fagadau Ionut\u00a0Daniel","year":"2024","unstructured":"Ionut\u00a0Daniel Fagadau, Leonardo Mariani, Daniela Micucci, and Oliviero Riganelli. 2024. Analyzing Prompt Influence on Automated Method Generation: An Empirical Study with Copilot. arXiv preprint arXiv:2402.08430 (2024)."},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.1145\/3524842.3528452"},{"key":"e_1_3_2_1_13_1","unstructured":"Yujia Fu Peng Liang Amjed Tahir Zengyang Li Mojtaba Shahin and Jiaxin Yu. 2023. Security Weaknesses of Copilot Generated Code in GitHub. arxiv:2310.02059\u00a0[cs.SE]"},{"key":"e_1_3_2_1_14_1","unstructured":"Zeyu Gao Hao Wang Yuchen Zhou Wenyu Zhu and Chao Zhang. 2023. How Far Have We Gone in Vulnerability Detection Using Large Language Models. arxiv:2311.12420\u00a0[cs.AI]"},{"key":"e_1_3_2_1_15_1","unstructured":"GitHub Security Code Injection Finder [n. d.]. GitHub Security Code Injection Finder. https:\/\/github.com\/github\/codeql\/blob\/main\/javascript\/ql\/src\/Security\/CWE-094\/ExpressionInjection.ql."},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.1109\/SP46215.2023.10179471"},{"key":"e_1_3_2_1_17_1","unstructured":"Jez Humble and David Farley. 2010. Continuous delivery: reliable software releases through build test and deployment automation. Pearson Education."},{"key":"e_1_3_2_1_18_1","unstructured":"Adnan Khan. 2023. One Supply Chain Attack to Rule Them All \u2013 Poisoning GitHub\u2019s Runner Images. https:\/\/adnanthekhan.com\/2023\/12\/20\/one-supply-chain-attack-to-rule-them-all\/."},{"key":"e_1_3_2_1_19_1","unstructured":"Avishree Khare Saikat Dutta Ziyang Li Alaia Solko-Breslin Rajeev Alur and Mayur Naik. 2023. Understanding the Effectiveness of Large Language Models in Detecting Security Vulnerabilities. arxiv:2311.16169\u00a0[cs.CR]"},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.1109\/MSR52588.2021.00054"},{"key":"e_1_3_2_1_21_1","volume-title":"Characterizing the Security of Github CI Workflows. In 31st USENIX Security Symposium (USENIX Security 22)","author":"Koishybayev Igibek","year":"2022","unstructured":"Igibek Koishybayev, Aleksandr Nahapetyan, Raima Zachariah, Siddharth Muralee, Bradley Reaves, Alexandros Kapravelos, and Aravind Machiry. 2022. Characterizing the Security of Github CI Workflows. In 31st USENIX Security Symposium (USENIX Security 22). USENIX Association, Boston, MA, 2747\u20132763. https:\/\/www.usenix.org\/conference\/usenixsecurity22\/presentation\/koishybayev"},{"key":"e_1_3_2_1_22_1","unstructured":"Raymond Li Loubna\u00a0Ben allal Yangtian Zi Niklas Muennighoff Denis Kocetkov Chenghao Mou Marc Marone Christopher Akiki Jia LI Jenny Chim Qian Liu Evgenii Zheltonozhskii Terry\u00a0Yue Zhuo Thomas Wang Olivier Dehaene Joel Lamy-Poirier Joao Monteiro Nicolas Gontier Ming-Ho Yee Logesh\u00a0Kumar Umapathi Jian Zhu Ben Lipkin Muhtasham Oblokulov Zhiruo Wang Rudra Murthy Jason\u00a0T Stillerman Siva\u00a0Sankalp Patel Dmitry Abulkhanov Marco Zocca Manan Dey Zhihan Zhang Urvashi Bhattacharyya Wenhao Yu Sasha Luccioni Paulo Villegas Fedor Zhdanov Tony Lee Nadav Timor Jennifer Ding Claire\u00a0S Schlesinger Hailey Schoelkopf Jan Ebert Tri Dao Mayank Mishra Alex Gu Carolyn\u00a0Jane Anderson Brendan Dolan-Gavitt Danish Contractor Siva Reddy Daniel Fried Dzmitry Bahdanau Yacine Jernite Carlos\u00a0Mu\u00f1oz Ferrandis Sean Hughes Thomas Wolf Arjun Guha Leandro\u00a0Von Werra and Harm de Vries. 2023. StarCoder: may the source be with you!Transactions on Machine Learning Research (2023). https:\/\/openreview.net\/forum?id=KoFOg41haE Reproducibility Certification."},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.1145\/3560815"},{"key":"e_1_3_2_1_24_1","volume-title":"Proceedings of the 40th International Conference on Machine Learning(Proceedings of Machine Learning Research, Vol.\u00a0202)","author":"Longpre Shayne","year":"2023","unstructured":"Shayne Longpre, Le Hou, Tu Vu, Albert Webson, Hyung\u00a0Won Chung, Yi Tay, Denny Zhou, Quoc\u00a0V Le, Barret Zoph, Jason Wei, and Adam Roberts. 2023. The Flan Collection: Designing Data and Methods for Effective Instruction Tuning. In Proceedings of the 40th International Conference on Machine Learning(Proceedings of Machine Learning Research, Vol.\u00a0202), Andreas Krause, Emma Brunskill, Kyunghyun Cho, Barbara Engelhardt, Sivan Sabato, and Jonathan Scarlett (Eds.). PMLR, 22631\u201322648. https:\/\/proceedings.mlr.press\/v202\/longpre23a.html"},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.1145\/3395363.3397369"},{"key":"e_1_3_2_1_26_1","unstructured":"Meta. 2023. Llama 2: Open Foundation and Fine-Tuned Chat Models. arxiv:2307.09288\u00a0[cs.CL]"},{"key":"e_1_3_2_1_27_1","volume-title":"ARGUS: A Framework for Staged Static Taint Analysis of GitHub Workflows and Actions. In 32nd USENIX Security Symposium (USENIX Security 23)","author":"Muralee Siddharth","year":"2023","unstructured":"Siddharth Muralee, Igibek Koishybayev, Aleksandr Nahapetyan, Greg Tystahl, Brad Reaves, Antonio Bianchi, William Enck, Alexandros Kapravelos, and Aravind Machiry. 2023. ARGUS: A Framework for Staged Static Taint Analysis of GitHub Workflows and Actions. In 32nd USENIX Security Symposium (USENIX Security 23). USENIX Association, Anaheim, CA, 6983\u20137000. https:\/\/www.usenix.org\/conference\/usenixsecurity23\/presentation\/muralee"},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"publisher","DOI":"10.1145\/3597503.3639187"},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.1145\/349299.349314"},{"key":"e_1_3_2_1_30_1","volume-title":"CodeGen: An Open Large Language Model for Code with Multi-Turn Program Synthesis. In The Eleventh International Conference on Learning Representations. https:\/\/openreview.net\/forum?id=iaYcJKpY2B_","author":"Nijkamp Erik","year":"2023","unstructured":"Erik Nijkamp, Bo Pang, Hiroaki Hayashi, Lifu Tu, Huan Wang, Yingbo Zhou, Silvio Savarese, and Caiming Xiong. 2023. CodeGen: An Open Large Language Model for Code with Multi-Turn Program Synthesis. In The Eleventh International Conference on Learning Representations. https:\/\/openreview.net\/forum?id=iaYcJKpY2B_"},{"key":"e_1_3_2_1_31_1","unstructured":"OpenAI. 2022. Introducing ChatGPT. https:\/\/openai.com\/blog\/chatgpt"},{"key":"e_1_3_2_1_32_1","unstructured":"OpenAI. 2024. GPT-4 Technical Report. arxiv:2303.08774\u00a0[cs.CL]"},{"key":"e_1_3_2_1_33_1","unstructured":"OWASP. 2022. OWASP Top 10 CI\/CD Security Risks. https:\/\/owasp.org\/www-project-top-10-ci-cd-security-risks\/."},{"key":"e_1_3_2_1_34_1","doi-asserted-by":"publisher","DOI":"10.3115\/1073083.1073135"},{"key":"e_1_3_2_1_35_1","doi-asserted-by":"publisher","DOI":"10.1109\/SP46214.2022.9833571"},{"key":"e_1_3_2_1_36_1","unstructured":"Baolin Peng Chunyuan Li Pengcheng He Michel Galley and Jianfeng Gao. 2023. Instruction Tuning with GPT-4. arxiv:2304.03277\u00a0[cs.CL]"},{"key":"e_1_3_2_1_37_1","doi-asserted-by":"publisher","DOI":"10.1109\/DAC56929.2023.10247987"},{"key":"e_1_3_2_1_38_1","unstructured":"Alec Radford Jeff Wu Rewon Child David Luan Dario Amodei and Ilya Sutskever. 2019. Language Models are Unsupervised Multitask Learners. https:\/\/api.semanticscholar.org\/CorpusID:160025533"},{"key":"e_1_3_2_1_39_1","doi-asserted-by":"publisher","DOI":"10.1145\/3411763.3451760"},{"key":"e_1_3_2_1_40_1","volume-title":"Code Llama: Open Foundation Models for Code. arxiv:2308.12950\u00a0[cs.CL]","author":"Rozi\u00e8re Baptiste","year":"2024","unstructured":"Baptiste Rozi\u00e8re, Jonas Gehring, Fabian Gloeckle, Sten Sootla, Itai Gat, Xiaoqing\u00a0Ellen Tan, Yossi Adi, Jingyu Liu, Romain Sauvestre, Tal Remez, J\u00e9r\u00e9my Rapin, Artyom Kozhevnikov, Ivan Evtimov, Joanna Bitton, Manish Bhatt, Cristian\u00a0Canton Ferrer, Aaron Grattafiori, Wenhan Xiong, Alexandre D\u00e9fossez, Jade Copet, Faisal Azhar, Hugo Touvron, Louis Martin, Nicolas Usunier, Thomas Scialom, and Gabriel Synnaeve. 2024. Code Llama: Open Foundation Models for Code. arxiv:2308.12950\u00a0[cs.CL]"},{"key":"e_1_3_2_1_41_1","volume-title":"In-Context Impersonation Reveals Large Language Models\u2019 Strengths and Biases. In Thirty-seventh Conference on Neural Information Processing Systems. https:\/\/openreview.net\/forum?id=CbsJ53LdKc","author":"Salewski Leonard","year":"2023","unstructured":"Leonard Salewski, Stephan Alaniz, Isabel Rio-Torto, Eric Schulz, and Zeynep Akata. 2023. In-Context Impersonation Reveals Large Language Models\u2019 Strengths and Biases. In Thirty-seventh Conference on Neural Information Processing Systems. https:\/\/openreview.net\/forum?id=CbsJ53LdKc"},{"key":"e_1_3_2_1_42_1","volume-title":"The 5th Workshop on Energy Efficient Machine Learning and Cognitive Computing @ NeurIPS 2019","author":"Sanh Victor","year":"2019","unstructured":"Victor Sanh, Lysandre Debut, Julien Chaumond, and Thomas Wolf. 2019. DistilBERT, a distilled version of BERT: smaller, faster, cheaper and lighter. In The 5th Workshop on Energy Efficient Machine Learning and Cognitive Computing @ NeurIPS 2019. arxiv:1910.01108http:\/\/arxiv.org\/abs\/1910.01108"},{"key":"e_1_3_2_1_43_1","volume-title":"The truth of the F-measure. Teach tutor mater 1, 5","author":"Yutaka Sasaki","year":"2007","unstructured":"Yutaka Sasaki 2007. The truth of the F-measure. Teach tutor mater 1, 5 (2007), 1\u20135."},{"key":"e_1_3_2_1_44_1","doi-asserted-by":"publisher","DOI":"10.1109\/APR59189.2023.00012"},{"key":"e_1_3_2_1_45_1","unstructured":"John Stawinski. 2023. Playing with Fire \u2013 How We Executed a Critical Supply Chain Attack on PyTorch. https:\/\/johnstawinski.com\/2024\/01\/11\/playing-with-fire-how-we-executed-a-critical-supply-chain-attack-on-pytorch\/."},{"key":"e_1_3_2_1_46_1","doi-asserted-by":"publisher","DOI":"10.1145\/3564625.3567985"},{"key":"e_1_3_2_1_47_1","volume-title":"Creating a Coding Assistant with StarCoder. Hugging Face Blog","author":"Tunstall Lewis","year":"2023","unstructured":"Lewis Tunstall, Nathan Lambert, Nazneen Rajani, Edward Beeching, Teven Le\u00a0Scao, Leandro von Werra, Sheon Han, Philipp Schmid, and Alexander Rush. 2023. Creating a Coding Assistant with StarCoder. Hugging Face Blog (2023). https:\/\/huggingface.co\/blog\/starchat."},{"key":"e_1_3_2_1_48_1","volume-title":"LLMs with Industrial Lens: Deciphering the Challenges and Prospects\u2013A Survey. arXiv preprint arXiv:2402.14558","author":"Urlana Ashok","year":"2024","unstructured":"Ashok Urlana, Charaka\u00a0Vinayak Kumar, Ajeet\u00a0Kumar Singh, Bala\u00a0Mallikarjunarao Garlapati, Srinivasa\u00a0Rao Chalamala, and Rahul Mishra. 2024. LLMs with Industrial Lens: Deciphering the Challenges and Prospects\u2013A Survey. arXiv preprint arXiv:2402.14558 (2024)."},{"key":"e_1_3_2_1_49_1","doi-asserted-by":"publisher","DOI":"10.1109\/SANER53432.2022.00026"},{"key":"e_1_3_2_1_50_1","doi-asserted-by":"publisher","DOI":"10.15446\/dyna.v90n230.111700"},{"key":"e_1_3_2_1_51_1","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2024.3368208"},{"key":"e_1_3_2_1_52_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2021.emnlp-main.685"},{"key":"e_1_3_2_1_53_1","volume-title":"International Conference on Learning Representations. https:\/\/openreview.net\/forum?id=gEZrGCozdqR","author":"Wei Jason","year":"2022","unstructured":"Jason Wei, Maarten Bosma, Vincent Zhao, Kelvin Guu, Adams\u00a0Wei Yu, Brian Lester, Nan Du, Andrew\u00a0M. Dai, and Quoc\u00a0V Le. 2022. Finetuned Language Models are Zero-Shot Learners. In International Conference on Learning Representations. https:\/\/openreview.net\/forum?id=gEZrGCozdqR"},{"key":"e_1_3_2_1_54_1","doi-asserted-by":"publisher","DOI":"10.1145\/3597926.3598135"},{"key":"e_1_3_2_1_55_1","doi-asserted-by":"publisher","DOI":"10.1145\/3643540"},{"key":"e_1_3_2_1_56_1","unstructured":"Yifei Xu Yuning Chen Xumiao Zhang Xianshang Lin Pan Hu Yunfei Ma Songwu Lu Wan Du Zhuoqing Mao Ennan Zhai and Dennis Cai. 2023. CloudEval-YAML: A Practical Benchmark for Cloud Configuration Generation. arxiv:2401.06786\u00a0[cs.DC]"},{"key":"e_1_3_2_1_57_1","doi-asserted-by":"publisher","DOI":"10.1109\/TDSC.2023.3308897"},{"key":"e_1_3_2_1_58_1","doi-asserted-by":"crossref","unstructured":"Xinyu Zhang Siddharth Muralee Sourag Cherupattamoolayil and Aravind Machiry. 2024. On the Effectiveness of Large Language Models for GitHub Workflows.","DOI":"10.1145\/3664476.3664497"}],"event":{"name":"ARES 2024: The 19th International Conference on Availability, Reliability and Security","location":"Vienna Austria","acronym":"ARES 2024"},"container-title":["Proceedings of the 19th International Conference on Availability, Reliability and Security"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3664476.3664497","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3664476.3664497","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,8,22]],"date-time":"2025-08-22T16:52:00Z","timestamp":1755881520000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3664476.3664497"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,7,30]]},"references-count":58,"alternative-id":["10.1145\/3664476.3664497","10.1145\/3664476"],"URL":"https:\/\/doi.org\/10.1145\/3664476.3664497","relation":{},"subject":[],"published":{"date-parts":[[2024,7,30]]},"assertion":[{"value":"2024-07-30","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}