{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,25]],"date-time":"2026-04-25T21:31:56Z","timestamp":1777152716372,"version":"3.51.4"},"publisher-location":"New York, NY, USA","reference-count":39,"publisher":"ACM","license":[{"start":{"date-parts":[[2023,11,13]],"date-time":"2023-11-13T00:00:00Z","timestamp":1699833600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2023,11,13]]},"DOI":"10.1145\/3631802.3631830","type":"proceedings-article","created":{"date-parts":[[2024,2,6]],"date-time":"2024-02-06T12:45:51Z","timestamp":1707223551000},"page":"1-11","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":156,"title":["CodeHelp: Using Large Language Models with Guardrails for Scalable Support in Programming Classes"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0009-0004-1512-7829","authenticated-orcid":false,"given":"Mark","family":"Liffiton","sequence":"first","affiliation":[{"name":"Computer Science, Illinois Wesleyan University, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-8762-5622","authenticated-orcid":false,"given":"Brad E","family":"Sheese","sequence":"additional","affiliation":[{"name":"Computer Science, Illinois Wesleyan University, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-3674-5456","authenticated-orcid":false,"given":"Jaromir","family":"Savelka","sequence":"additional","affiliation":[{"name":"Carnegie Mellon University, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-5150-9806","authenticated-orcid":false,"given":"Paul","family":"Denny","sequence":"additional","affiliation":[{"name":"School of Computer Science, The University of Auckland, New Zealand"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2024,2,6]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"publisher","DOI":"10.1145\/3545945.3569759"},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.1191\/1478088706qp063oa"},{"key":"e_1_3_2_1_3_1","unstructured":"Peter Brusilovsky Barbara\u00a0J Ericson Cay\u00a0S Horstmann and Christian Servin. 2023. The Future of Computing Education Materials. (2023)."},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.1109\/SIIE56031.2022.9982349"},{"key":"e_1_3_2_1_5_1","unstructured":"Bei Chen Fengji Zhang Anh Nguyen Daoguang Zan Zeqi Lin Jian-Guang Lou and Weizhu Chen. 2022. CodeT: Code Generation with Generated Tests. arxiv:2207.10397\u00a0[cs.CL]"},{"key":"e_1_3_2_1_6_1","volume-title":"Jared Kaplan, Harri Edwards, Yuri Burda","author":"Chen Mark","year":"2021","unstructured":"Mark Chen, Jerry Tworek, Heewoo Jun, Qiming Yuan, Henrique Ponde de\u00a0Oliveira Pinto, Jared Kaplan, Harri Edwards, Yuri Burda, Nicholas Joseph, Greg Brockman, 2021. Evaluating large language models trained on code. arxiv:2107.03374\u00a0[cs.LG]"},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"publisher","DOI":"10.1177\/00317217231168266"},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.1145\/3160489.3160492"},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1145\/3545945.3569823"},{"key":"e_1_3_2_1_10_1","volume-title":"Promptly: Using Prompt Problems to Teach Learners How to Effectively Utilize AI Code Generators. arxiv:2307.16364\u00a0[cs.HC]","author":"Denny Paul","year":"2023","unstructured":"Paul Denny, Juho Leinonen, James Prather, Andrew Luxton-Reilly, Thezyrie Amarouche, Brett\u00a0A. Becker, and Brent\u00a0N. Reeves. 2023. Promptly: Using Prompt Problems to Teach Learners How to Effectively Utilize AI Code Generators. arxiv:2307.16364\u00a0[cs.HC]"},{"key":"e_1_3_2_1_11_1","unstructured":"Paul Denny James Prather Brett\u00a0A. Becker James Finnie-Ansley Arto Hellas Juho Leinonen Andrew Luxton-Reilly Brent\u00a0N. Reeves Eddie\u00a0Antonio Santos and Sami Sarsa. 2023. Computing Education in the Era of Generative AI. arxiv:2306.02608\u00a0[cs.CY]"},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.1145\/3511861.3511863"},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1145\/3478431.3499334"},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"crossref","unstructured":"Arto Hellas Juho Leinonen Sami Sarsa Charles Koutcheme Lilja Kujanp\u00e4\u00e4 and Juha Sorva. 2023. Exploring the Responses of Large Language Models to Beginner Programmers\u2019 Help Requests. arxiv:2306.05715\u00a0[cs.CY]","DOI":"10.1145\/3568813.3600139"},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.1109\/icstw58534.2023.00078"},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.lindif.2023.102274"},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.1145\/3544548.3580919"},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1145\/3231711"},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","DOI":"10.1109\/MIPRO.2015.7160406"},{"key":"e_1_3_2_1_20_1","volume-title":"Artificial Intelligence in Education, Ning Wang, Genaro Rebolledo-Mendez, Noboru Matsuda, Olga\u00a0C","author":"Koutcheme Charles","unstructured":"Charles Koutcheme, Sami Sarsa, Juho Leinonen, Arto Hellas, and Paul Denny. 2023. Automated Program Repair Using Generative Models for\u00a0Code Infilling. In Artificial Intelligence in Education, Ning Wang, Genaro Rebolledo-Mendez, Noboru Matsuda, Olga\u00a0C. Santos, and Vania Dimitrova (Eds.). Springer Nature Switzerland, Cham, 798\u2013803."},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"crossref","unstructured":"Juho Leinonen Paul Denny Stephen MacNeil Sami Sarsa Seth Bernstein Joanne Kim Andrew Tran and Arto Hellas. 2023. Comparing Code Explanations Created by Students and Large Language Models. arxiv:2304.03938\u00a0[cs.CY]","DOI":"10.1145\/3587102.3588785"},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.procs.2021.12.259"},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.1145\/3469885"},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"publisher","DOI":"10.1145\/3524842.3528470"},{"key":"e_1_3_2_1_25_1","volume-title":"Python-Bot: A Chatbot for Teaching Python Programming. Engineering Letters 29 (02","author":"Okonkwo Chinedu\u00a0Wilfred","year":"2021","unstructured":"Chinedu\u00a0Wilfred Okonkwo and Abejide Ade-Ibijola. 2021. Python-Bot: A Chatbot for Teaching Python Programming. Engineering Letters 29 (02 2021), 25\u201334."},{"key":"e_1_3_2_1_26_1","article-title":"Revision-Bot: A Chatbot for Studying Past Questions in Introductory Programming","volume":"49","author":"Okonkwo Chinedu\u00a0Wilfred","year":"2022","unstructured":"Chinedu\u00a0Wilfred Okonkwo and Abejide Ade-Ibijola. 2022. Revision-Bot: A Chatbot for Studying Past Questions in Introductory Programming. IAENG International Journal of Computer Science 49, 3 (2022).","journal-title":"IAENG International Journal of Computer Science"},{"key":"e_1_3_2_1_27_1","volume-title":"Pardos and Shreya Bhandari","author":"A.","year":"2023","unstructured":"Zachary\u00a0A. Pardos and Shreya Bhandari. 2023. Learning gain differences between ChatGPT and human tutor generated algebra hints. arxiv:2302.06871\u00a0[cs.CY]"},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"publisher","DOI":"10.1371\/journal.pcbi.1011511"},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"crossref","unstructured":"James Prather Paul Denny Juho Leinonen Brett\u00a0A. Becker Ibrahim Albluwi Michelle Craig Hieke Keuning Natalie Kiesler Tobias Kohn Andrew Luxton-Reilly Stephen MacNeil Andrew Peterson Raymond Pettit Brent\u00a0N. Reeves and Jaromir Savelka. 2023. The Robots are Here: Navigating the Generative AI Revolution in Computing Education. arxiv:2310.00658\u00a0[cs.CY]","DOI":"10.1145\/3623762.3633499"},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"crossref","unstructured":"James Prather Brent\u00a0N. Reeves Paul Denny Brett\u00a0A. Becker Juho Leinonen Andrew Luxton-Reilly Garrett Powell James Finnie-Ansley and Eddie\u00a0Antonio Santos. 2023. \"It\u2019s Weird That it Knows What I Want\": Usability and Interactions with Copilot for Novice Programmers. arxiv:2304.02491\u00a0[cs.HC]","DOI":"10.1145\/3617367"},{"key":"e_1_3_2_1_31_1","volume-title":"Duckbot: A chatbot to assist students in programming tutorials. Master\u2019s thesis","author":"Rutgers Margot","year":"2021","unstructured":"Margot Rutgers. 2021. Duckbot: A chatbot to assist students in programming tutorials. Master\u2019s thesis. University of Twente."},{"key":"e_1_3_2_1_32_1","doi-asserted-by":"publisher","DOI":"10.1145\/3501385.3543957"},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"publisher","DOI":"10.1145\/3568813.3600142"},{"key":"e_1_3_2_1_34_1","doi-asserted-by":"crossref","unstructured":"Jaromir Savelka Arav Agarwal Christopher Bogart and Majd Sakr. 2023. Large Language Models (GPT) Struggle to Answer Multiple-Choice Questions about Code. arxiv:2303.08033\u00a0[cs.CL]","DOI":"10.5220\/0011996900003470"},{"key":"e_1_3_2_1_35_1","unstructured":"Haoye Tian Weiqi Lu Tsz\u00a0On Li Xunzhu Tang Shing-Chi Cheung Jacques Klein and Tegawend\u00e9\u00a0F. Bissyand\u00e9. 2023. Is ChatGPT the Ultimate Programming Assistant \u2013 How far is it?arxiv:2304.11938\u00a0[cs.SE]"},{"key":"e_1_3_2_1_36_1","volume-title":"Proceedings of the EDSIG Conference ISSN, Vol.\u00a02473","author":"Walden James","year":"2022","unstructured":"James Walden, Nicholas Caporusso, and Ludiana Atnafu. 2022. A Chatbot for Teaching Secure Programming. In Proceedings of the EDSIG Conference ISSN, Vol.\u00a02473. 4901."},{"key":"e_1_3_2_1_37_1","volume-title":"Chi, Quoc Le, and Denny Zhou","author":"Wei Jason","year":"2023","unstructured":"Jason Wei, Xuezhi Wang, Dale Schuurmans, Maarten Bosma, Brian Ichter, Fei Xia, Ed Chi, Quoc Le, and Denny Zhou. 2023. Chain-of-Thought Prompting Elicits Reasoning in Large Language Models. arxiv:2201.11903\u00a0[cs.CL]"},{"key":"e_1_3_2_1_38_1","doi-asserted-by":"publisher","DOI":"10.1145\/3531146.3533088"},{"key":"e_1_3_2_1_39_1","unstructured":"Terry\u00a0Yue Zhuo Yujin Huang Chunyang Chen and Zhenchang Xing. 2023. Red teaming ChatGPT via Jailbreaking: Bias Robustness Reliability and Toxicity. arxiv:2301.12867\u00a0[cs.CL]"}],"event":{"name":"Koli Calling '23: 23rd Koli Calling International Conference on Computing Education Research","location":"Koli Finland","acronym":"Koli Calling '23"},"container-title":["Proceedings of the 23rd Koli Calling International Conference on Computing Education Research"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3631802.3631830","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3631802.3631830","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,8,22]],"date-time":"2025-08-22T16:20:52Z","timestamp":1755879652000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3631802.3631830"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023,11,13]]},"references-count":39,"alternative-id":["10.1145\/3631802.3631830","10.1145\/3631802"],"URL":"https:\/\/doi.org\/10.1145\/3631802.3631830","relation":{},"subject":[],"published":{"date-parts":[[2023,11,13]]},"assertion":[{"value":"2024-02-06","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}