{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,5,4]],"date-time":"2026-05-04T11:05:39Z","timestamp":1777892739351,"version":"3.51.4"},"publisher-location":"New York, NY, USA","reference-count":42,"publisher":"ACM","license":[{"start":{"date-parts":[[2024,4,20]],"date-time":"2024-04-20T00:00:00Z","timestamp":1713571200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2024,4,20]]},"DOI":"10.1145\/3643795.3648380","type":"proceedings-article","created":{"date-parts":[[2024,9,10]],"date-time":"2024-09-10T13:46:19Z","timestamp":1725975979000},"page":"88-93","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":5,"title":["Investigating the Proficiency of Large Language Models in Formative Feedback Generation for Student Programmers"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-7583-1795","authenticated-orcid":false,"given":"Smitha","family":"S Kumar","sequence":"first","affiliation":[{"name":"Heriot-Watt University -UAE, Dubai, United Arab Emirates"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-2745-9896","authenticated-orcid":false,"given":"Michael","family":"Adam Lones","sequence":"additional","affiliation":[{"name":"Heriot-Watt University- UK, Edinburgh, United Kingdom"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-6233-6341","authenticated-orcid":false,"given":"Manuel","family":"Maarek","sequence":"additional","affiliation":[{"name":"Heriot-Watt University -UK, Edinburgh, United Kingdom"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-3804-0513","authenticated-orcid":false,"given":"Hind","family":"Zantout","sequence":"additional","affiliation":[{"name":"Heriot-Watt University -UAE, Dubai, United Arab Emirates"}]}],"member":"320","published-online":{"date-parts":[[2024,9,10]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"publisher","DOI":"10.1145\/3587102.3588852"},{"key":"e_1_3_2_1_2_1","volume-title":"Grounded Copilot: How Programmers Interact with Code-Generating Models. arXiv:2206.15000 [cs.HC]","author":"Barke Shraddha","year":"2022","unstructured":"Shraddha Barke, Michael B. James, and Nadia Polikarpova. 2022. Grounded Copilot: How Programmers Interact with Code-Generating Models. arXiv:2206.15000 [cs.HC]"},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1145\/3545945.3569759"},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.1145\/3344429.3372508"},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1145\/2538862.2538924"},{"key":"e_1_3_2_1_6_1","unstructured":"Yihan Cao Siyu Li Yixin Liu Zhiling Yan Yutong Dai Philip S. Yu and Lichao Sun. 2023. A Comprehensive Survey of AI-Generated Content (AIGC): A History of Generative AI from GAN to ChatGPT. arXiv:2303.04226 [cs.AI]"},{"key":"e_1_3_2_1_7_1","volume-title":"Molly Q Feldman, Arjun Guha, Michael Greenberg, and Abhinav Jangda.","author":"Cassano Federico","year":"2022","unstructured":"Federico Cassano, John Gouwar, Daniel Nguyen, Sydney Nguyen, Luna Phipps-Costin, Donald Pinckney, Ming-Ho Yee, Yangtian Zi, Carolyn Jane Anderson, Molly Q Feldman, Arjun Guha, Michael Greenberg, and Abhinav Jangda. 2022. MultiPL-E: A Scalable and Extensible Approach to Benchmarking Neural Code Generation. arXiv:2208.08227 [cs.LG]"},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"crossref","unstructured":"Yupeng Chang Xu Wang Jindong Wang Yuan Wu Linyi Yang Kaijie Zhu Hao Chen Xiaoyuan Yi Cunxiang Wang Yidong Wang Wei Ye Yue Zhang Yi Chang Philip S. Yu Qiang Yang and Xing Xie. 2023. A Survey on Evaluation of Large Language Models. arXiv:2307.03109 [cs.CL]","DOI":"10.1145\/3641289"},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1145\/2325296.2325318"},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.1145\/1953163.1953299"},{"key":"e_1_3_2_1_11_1","unstructured":"Thomas Durieux and Martin Monperrus. 2016. IntroClassJava: A benchmark of 297 small and buggy Java programs. https:\/\/hal.archives-ouvertes.fr\/hal-01272126"},{"key":"e_1_3_2_1_12_1","unstructured":"Mark Chen et al. 2021. Evaluating Large Language Models Trained on Code. arXiv:2107.03374 [cs.LG]"},{"key":"e_1_3_2_1_13_1","unstructured":"Raymond Li et al. 2023. StarCoder: may the source be with you! arXiv:2305.06161 [cs.CL]"},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.1145\/3160489.3160493"},{"key":"e_1_3_2_1_15_1","volume-title":"Zhang","author":"Fan Angela","year":"2023","unstructured":"Angela Fan, Beliz Gokkaya, Mark Harman, Mitya Lyubarskiy, Shubho Sengupta, Shin Yoo, and Jie M. Zhang. 2023. Large Language Models for Software Engineering: Survey and Open Problems. arXiv:2310.03533 [cs.SE]"},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.1145\/3511861.3511863"},{"key":"e_1_3_2_1_17_1","unstructured":"Nat Friedman. 2021. Introducing GitHub Copilot: your AI pair programmer. https:\/\/github.blog\/2021-06-29-introducing-github-copilot-ai-pair-programmer"},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1145\/1352135.1352226"},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","DOI":"10.1145\/3192366.3192387"},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.1145\/3544548.3580919"},{"key":"e_1_3_2_1_21_1","volume-title":"Barbara J. Ericson, David Weintrop, and Tovi Grossman.","author":"Kazemitabaar Majeed","year":"2023","unstructured":"Majeed Kazemitabaar, Xinying Hou, Austin Henley, Barbara J. Ericson, David Weintrop, and Tovi Grossman. 2023. How Novices Use LLM-Based Code Generators to Solve CS1 Coding Tasks in a Self-Paced Learning Environment. arXiv:2309.14049 [cs.HC]"},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.1145\/3231711"},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.1145\/3135932.3135941"},{"key":"e_1_3_2_1_24_1","volume-title":"Handbook of research on educational communications and technology","author":"Narciss Susanne","unstructured":"Susanne Narciss. 2008. Feedback strategies for interactive learning tasks. In Handbook of research on educational communications and technology. Routledge, 125--143."},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.1080\/08874417.2023.2261010"},{"key":"e_1_3_2_1_26_1","volume-title":"https:\/\/openai.com\/blog\/chatgpt Accessed on","author":"Introducing AI.","year":"2023","unstructured":"OpenAI. 2022. Introducing ChatGPT. https:\/\/openai.com\/blog\/chatgpt Accessed on August 21, 2023."},{"key":"e_1_3_2_1_27_1","volume-title":"https:\/\/openai.com\/blog\/openai-codex Accessed on","author":"Codex AI.","year":"2023","unstructured":"OpenAI. 2022. OpenAI Codex. https:\/\/openai.com\/blog\/openai-codex Accessed on August 21, 2023."},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.dib.2023.108887"},{"key":"e_1_3_2_1_29_1","unstructured":"Tung Phung Jos\u00e9 Cambronero Sumit Gulwani Tobias Kohn Rupak Majumdar Adish Singla and Gustavo Soares. 2023. Generating High-Precision Feedback for Programming Syntax Errors using Large Language Models. (2023). http:\/\/arxiv.org\/abs\/2302.04662"},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"crossref","unstructured":"James Prather Paul Denny Juho Leinonen Brett A. Becker Ibrahim Albluwi Michelle Craig Hieke Keuning Natalie Kiesler Tobias Kohn Andrew Luxton-Reilly Stephen MacNeil Andrew Peterson Raymond Pettit Brent N. Reeves and Jaromir Savelka. 2023. The Robots are Here: Navigating the Generative AI Revolution in Computing Education. arXiv:2310.00658 [cs.CY]","DOI":"10.1145\/3623762.3633499"},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"publisher","DOI":"10.1007\/s40593-019-00177-z"},{"key":"e_1_3_2_1_32_1","unstructured":"Baptiste Rozi\u00e8re Jonas Gehring Fabian Gloeckle Sten Sootla Itai Gat Xiaoqing Ellen Tan Yossi Adi Jingyu Liu Tal Remez J\u00e9r\u00e9my Rapin Artyom Kozhevnikov Ivan Evtimov Joanna Bitton Manish Bhatt Cristian Canton Ferrer Aaron Grattafiori Wenhan Xiong Alexandre D\u00e9fossez Jade Copet Faisal Azhar Hugo Touvron Louis Martin Nicolas Usunier Thomas Scialom and Gabriel Synnaeve. 2023. Code Llama: Open Foundation Models for Code. arXiv:2308.12950 [cs.CL]"},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"publisher","DOI":"10.1145\/3501385.3543957"},{"key":"e_1_3_2_1_34_1","doi-asserted-by":"publisher","DOI":"10.1177\/088840640402700407"},{"key":"e_1_3_2_1_35_1","doi-asserted-by":"publisher","DOI":"10.1145\/3573051.3596191"},{"key":"e_1_3_2_1_36_1","doi-asserted-by":"publisher","DOI":"10.1109\/APR59189.2023.00012"},{"key":"e_1_3_2_1_37_1","unstructured":"Ashish Vaswani Noam Shazeer Niki Parmar Jakob Uszkoreit Llion Jones Aidan N. Gomez Lukasz Kaiser and Illia Polosukhin. 2023. Attention Is All You Need. arXiv:1706.03762 [cs.CL]"},{"key":"e_1_3_2_1_38_1","doi-asserted-by":"publisher","DOI":"10.1145\/3545945.3569830"},{"key":"e_1_3_2_1_39_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE48619.2023.00129"},{"key":"e_1_3_2_1_40_1","unstructured":"Chunqiu Steven Xia and Lingming Zhang. 2023. Conversational automated program repair. (2023). http:\/\/arxiv.org\/abs\/2301.13246"},{"key":"e_1_3_2_1_41_1","doi-asserted-by":"publisher","DOI":"10.1145\/3106237.3106262"},{"key":"e_1_3_2_1_42_1","unstructured":"Jialu Zhang Jos\u00e9 Cambronero Sumit Gulwani Vu Le Ruzica Piskac Gustavo Soares and Gust Verbruggen. 2022. Repairing bugs in python assignments using large language models. (2022). https:\/\/doi.org\/abs\/2209.14876"}],"event":{"name":"LLM4Code '24: 1st International Workshop on Large Language Models for Code","location":"Lisbon Portugal","acronym":"LLM4Code '24","sponsor":["SIGSOFT ACM Special Interest Group on Software Engineering","IEEE CS","Faculty of Engineering of University of Porto"]},"container-title":["Proceedings of the 1st International Workshop on Large Language Models for Code"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3643795.3648380","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3643795.3648380","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,19]],"date-time":"2025-06-19T00:57:45Z","timestamp":1750294665000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3643795.3648380"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,4,20]]},"references-count":42,"alternative-id":["10.1145\/3643795.3648380","10.1145\/3643795"],"URL":"https:\/\/doi.org\/10.1145\/3643795.3648380","relation":{},"subject":[],"published":{"date-parts":[[2024,4,20]]},"assertion":[{"value":"2024-09-10","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}