{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,6,11]],"date-time":"2026-06-11T08:02:34Z","timestamp":1781164954955,"version":"3.54.1"},"publisher-location":"New York, NY, USA","reference-count":35,"publisher":"ACM","license":[{"start":{"date-parts":[[2024,3,18]],"date-time":"2024-03-18T00:00:00Z","timestamp":1710720000000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"funder":[{"DOI":"10.13039\/501100006374","name":"European Research Council","doi-asserted-by":"publisher","award":["ERC, TOPS, 101039090"],"award-info":[{"award-number":["ERC, TOPS, 101039090"]}],"id":[{"id":"10.13039\/501100006374","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2024,3,18]]},"DOI":"10.1145\/3636555.3636846","type":"proceedings-article","created":{"date-parts":[[2024,3,5]],"date-time":"2024-03-05T18:11:20Z","timestamp":1709662280000},"page":"12-23","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":49,"title":["Automating Human Tutor-Style Programming Feedback: Leveraging GPT-4 Tutor Model for Hint Generation and GPT-3.5 Student Model for Hint Validation"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0009-0005-2683-434X","authenticated-orcid":false,"given":"Tung","family":"Phung","sequence":"first","affiliation":[{"name":"MPI-SWS, Germany"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0000-7903-6054","authenticated-orcid":false,"given":"Victor-Alexandru","family":"P\u0103durean","sequence":"additional","affiliation":[{"name":"MPI-SWS, Germany"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0006-4092-4779","authenticated-orcid":false,"given":"Anjali","family":"Singh","sequence":"additional","affiliation":[{"name":"University of Michigan, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-0875-0204","authenticated-orcid":false,"given":"Christopher","family":"Brooks","sequence":"additional","affiliation":[{"name":"University of Michigan, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-0713-6141","authenticated-orcid":false,"given":"Jos\u00e9","family":"Cambronero","sequence":"additional","affiliation":[{"name":"Microsoft, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-9226-9634","authenticated-orcid":false,"given":"Sumit","family":"Gulwani","sequence":"additional","affiliation":[{"name":"Microsoft, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-9922-0668","authenticated-orcid":false,"given":"Adish","family":"Singla","sequence":"additional","affiliation":[{"name":"MPI-SWS, Germany"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-8061-9000","authenticated-orcid":false,"given":"Gustavo","family":"Soares","sequence":"additional","affiliation":[{"name":"Microsoft, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2024,3,18]]},"reference":[{"key":"e_1_3_2_1_1_1","unstructured":"Khan Academy. 2023. Khanmigo. https:\/\/www.khanacademy.org\/khan-labs."},{"key":"e_1_3_2_1_2_1","volume-title":"Multimodal Evaluation of ChatGPT on Reasoning, Hallucination, and Interactivity. CoRR abs\/2302.04023","author":"Multitask Yejin Bang","year":"2023","unstructured":"Yejin Bang 2023. A Multitask, Multilingual, Multimodal Evaluation of ChatGPT on Reasoning, Hallucination, and Interactivity. CoRR abs\/2302.04023 (2023)."},{"key":"e_1_3_2_1_3_1","unstructured":"Georg Brandl Matth\u00e4us Chajdas and Jean Abou-Samra. 2006. Pygments. https:\/\/pygments.org\/."},{"key":"e_1_3_2_1_4_1","volume-title":"Sparks of Artificial General Intelligence: Early Experiments with GPT-4. CoRR abs\/2303.12712","author":"S\u00e9bastien Bubeck","year":"2023","unstructured":"S\u00e9bastien Bubeck 2023. Sparks of Artificial General Intelligence: Early Experiments with GPT-4. CoRR abs\/2303.12712 (2023)."},{"key":"e_1_3_2_1_5_1","volume-title":"Teaching Large Language Models to Self-Debug. CoRR abs\/2304.05128","author":"Chen Xinyun","year":"2023","unstructured":"Xinyun Chen, Maxwell Lin, Nathanael Sch\u00e4rli, and Denny Zhou. 2023. Teaching Large Language Models to Self-Debug. CoRR abs\/2304.05128 (2023)."},{"key":"e_1_3_2_1_6_1","volume-title":"The \u03c7 2 Test of Goodness of Fit. The Annals of Mathematical Statistics","author":"Cochran G","year":"1952","unstructured":"William\u00a0G Cochran. 1952. The \u03c7 2 Test of Goodness of Fit. The Annals of Mathematical Statistics (1952)."},{"key":"e_1_3_2_1_7_1","volume-title":"A coefficient of agreement for nominal scales. Educational and psychological measurement 20, 1","author":"Cohen Jacob","year":"1960","unstructured":"Jacob Cohen. 1960. A coefficient of agreement for nominal scales. Educational and psychological measurement 20, 1 (1960), 37\u201346."},{"key":"e_1_3_2_1_8_1","unstructured":"geeksforgeeks.org. 2009. GeeksforGeeks: A Computer Science Portal for Geeks. https:\/\/www.geeksforgeeks.org\/."},{"key":"e_1_3_2_1_9_1","unstructured":"GitHub. 2022. GitHub Copilot: Your AI Pair Programmer. https:\/\/github.com\/features\/copilot."},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"crossref","unstructured":"Sumit Gulwani Ivan Radicek and Florian Zuleger. 2018. Automated Clustering and Program Repair for Introductory Programming Assignments. In PLDI.","DOI":"10.1145\/3192366.3192387"},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"crossref","unstructured":"Andrew Head Elena\u00a0L. Glassman Gustavo Soares Ryo Suzuki Lucas Figueredo Loris D\u2019Antoni and Bj\u00f6rn Hartmann. 2017. Writing Reusable Code Feedback at Scale with Mixed-Initiative Program Synthesis. In Learning @ Scale.","DOI":"10.1145\/3051457.3051467"},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"crossref","unstructured":"Natalie Kiesler Dominic Lohr and Hieke Keuning. 2023. Exploring the Potential of Large Language Models to Generate Formative Programming Feedback. In FIE.","DOI":"10.1109\/FIE58773.2023.10343457"},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"crossref","unstructured":"Juho Leinonen Arto Hellas Sami Sarsa Brent\u00a0N. Reeves Paul Denny James Prather and Brett\u00a0A. Becker. 2023. Using Large Language Models to Enhance Programming Error Messages. In SIGCSE.","DOI":"10.1145\/3545945.3569770"},{"key":"e_1_3_2_1_14_1","unstructured":"Tiffany\u00a0Wenting Li Silas Hsu Max Fowler Zhilin Zhang Craig\u00a0B. Zilles and Karrie Karahalios. 2023. Am I Wrong or Is the Autograder Wrong? Effects of AI Grading Mistakes on Learning. In ICER."},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"crossref","unstructured":"Stephen MacNeil Andrew Tran Arto Hellas Joanne Kim Sami Sarsa Paul Denny Seth Bernstein and Juho Leinonen. 2023. Experiences from Using Code Explanations Generated by Large Language Models in a Web Software Development E-Book. In SIGCSE.","DOI":"10.1145\/3545945.3569785"},{"key":"e_1_3_2_1_16_1","volume-title":"Self-Refine: Iterative Refinement with Self-Feedback. CoRR abs\/2303.17651","author":"Aman Madaan","year":"2023","unstructured":"Aman Madaan 2023. Self-Refine: Iterative Refinement with Self-Feedback. CoRR abs\/2303.17651 (2023)."},{"key":"e_1_3_2_1_17_1","first-page":"1","article-title":"pandas: A Foundational Python Library for Data Analysis and Statistics","volume":"14","author":"McKinney Wes","year":"2011","unstructured":"Wes McKinney 2011. pandas: A Foundational Python Library for Data Analysis and Statistics. Python for High Performance and Scientific Computing 14, 9 (2011), 1\u20139.","journal-title":"Python for High Performance and Scientific Computing"},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"crossref","unstructured":"Samim Mirhosseini Austin\u00a0Z. Henley and Chris Parnin. 2023. What is Your Biggest Pain Point? An Investigation of CS Instructor Obstacles Workarounds and Desires. In SIGCSE.","DOI":"10.1145\/3545945.3569816"},{"key":"e_1_3_2_1_19_1","volume-title":"Reading Between the Lines: Modeling User Behavior and Costs in AI-Assisted Programming. CoRR abs\/2210.14306","author":"Mozannar Hussein","year":"2022","unstructured":"Hussein Mozannar, Gagan Bansal, Adam Fourney, and Eric Horvitz. 2022. Reading Between the Lines: Modeling User Behavior and Costs in AI-Assisted Programming. CoRR abs\/2210.14306 (2022)."},{"key":"e_1_3_2_1_20_1","volume-title":"Demystifying GPT Self-Repair for Code Generation. CoRR abs\/2306.09896","author":"Olausson X.","year":"2023","unstructured":"Theo\u00a0X. Olausson, Jeevana\u00a0Priya Inala, Chenglong Wang, Jianfeng Gao, and Armando Solar-Lezama. 2023. Demystifying GPT Self-Repair for Code Generation. CoRR abs\/2306.09896 (2023)."},{"key":"e_1_3_2_1_21_1","unstructured":"OpenAI. 2022. Codex-Edit. https:\/\/beta.openai.com\/playground?mode=edit&model=code-davinci-edit-001."},{"key":"e_1_3_2_1_22_1","unstructured":"OpenAI. 2023. ChatGPT. https:\/\/openai.com\/blog\/chatgpt."},{"key":"e_1_3_2_1_23_1","unstructured":"OpenAI. 2023. GPT-4 Technical Report. CoRR abs\/2303.08774 (2023)."},{"key":"e_1_3_2_1_24_1","volume-title":"Neural Task Synthesis for Visual Programming. CoRR abs\/2305.18342","author":"P\u0103durean Victor-Alexandru","year":"2023","unstructured":"Victor-Alexandru P\u0103durean, Georgios Tzannetos, and Adish Singla. 2023. Neural Task Synthesis for Visual Programming. CoRR abs\/2305.18342 (2023)."},{"key":"e_1_3_2_1_25_1","volume-title":"Large Language Models (GPT) for Automating Feedback on Programming Assignments. CoRR abs\/2307.00150","author":"Pankiewicz Maciej","year":"2023","unstructured":"Maciej Pankiewicz and Ryan\u00a0Shaun Baker. 2023. Large Language Models (GPT) for Automating Feedback on Programming Assignments. CoRR abs\/2307.00150 (2023)."},{"key":"e_1_3_2_1_26_1","unstructured":"Tung Phung Jos\u00e9 Cambronero Sumit Gulwani Tobias Kohn Rupak Majumdar Adish Singla and Gustavo Soares. 2023. Generating High-Precision Feedback for Programming Syntax Errors using Large Language Models. In EDM."},{"key":"e_1_3_2_1_27_1","doi-asserted-by":"crossref","unstructured":"Tung Phung Victor-Alexandru P\u0103durean Jos\u00e9 Cambronero Sumit Gulwani Tobias Kohn Rupak Majumdar Adish Singla and Gustavo Soares. 2023. Generative AI for Programming Education: Benchmarking ChatGPT GPT-4 and Human Tutors. In ICER V.2.","DOI":"10.1145\/3568812.3603476"},{"key":"e_1_3_2_1_28_1","unstructured":"Quizlet. 2023. Q-Chat. https:\/\/quizlet.com\/qchat-personal-ai-tutor."},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"crossref","unstructured":"Sami Sarsa Paul Denny Arto Hellas and Juho Leinonen. 2022. Automatic Generation of Programming Exercises and Code Explanations Using Large Language Models. In ICER.","DOI":"10.1145\/3501385.3543957"},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"crossref","unstructured":"Jarom\u00edr Savelka Arav Agarwal Christopher Bogart Yifan Song and Majd Sakr. 2023. Can Generative Pre-trained Transformers (GPT) Pass Assessments in Higher Education Programming Courses?. In ITiCSE.","DOI":"10.1145\/3587102.3588792"},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"crossref","unstructured":"Rishabh Singh Sumit Gulwani and Armando Solar-Lezama. 2013. Automated Feedback Generation for Introductory Programming Assignments. In PLDI.","DOI":"10.1145\/2491956.2462195"},{"key":"e_1_3_2_1_32_1","doi-asserted-by":"crossref","unstructured":"Adish Singla. 2023. Evaluating ChatGPT and GPT-4 for Visual Programming. In ICER V.2.","DOI":"10.1145\/3568812.3603474"},{"key":"e_1_3_2_1_33_1","volume-title":"Llama 2: Open Foundation and Fine-Tuned Chat Models. CoRR abs\/2307.09288","author":"Hugo Touvron","year":"2023","unstructured":"Hugo Touvron 2023. Llama 2: Open Foundation and Fine-Tuned Chat Models. CoRR abs\/2307.09288 (2023)."},{"key":"e_1_3_2_1_34_1","unstructured":"Jason Wei 2022. Chain-of-Thought Prompting Elicits Reasoning in Large Language Models. In NeurIPS."},{"key":"e_1_3_2_1_35_1","volume-title":"Repairing Bugs in Python Assignments Using Large Language Models. CoRR abs\/2209.14876","author":"Zhang Jialu","year":"2022","unstructured":"Jialu Zhang, Jos\u00e9 Cambronero, Sumit Gulwani, Vu Le, Ruzica Piskac, Gustavo Soares, and Gust Verbruggen. 2022. Repairing Bugs in Python Assignments Using Large Language Models. CoRR abs\/2209.14876 (2022)."}],"event":{"name":"LAK '24: The 14th Learning Analytics and Knowledge Conference","location":"Kyoto Japan","acronym":"LAK '24"},"container-title":["Proceedings of the 14th Learning Analytics and Knowledge Conference"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3636555.3636846","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3636555.3636846","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,9,17]],"date-time":"2025-09-17T01:20:23Z","timestamp":1758072023000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3636555.3636846"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,3,18]]},"references-count":35,"alternative-id":["10.1145\/3636555.3636846","10.1145\/3636555"],"URL":"https:\/\/doi.org\/10.1145\/3636555.3636846","relation":{},"subject":[],"published":{"date-parts":[[2024,3,18]]},"assertion":[{"value":"2024-03-18","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}