{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,5,18]],"date-time":"2026-05-18T15:38:02Z","timestamp":1779118682296,"version":"3.51.4"},"publisher-location":"New York, NY, USA","reference-count":38,"publisher":"ACM","license":[{"start":{"date-parts":[[2023,3,2]],"date-time":"2023-03-02T00:00:00Z","timestamp":1677715200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2023,3,2]]},"DOI":"10.1145\/3545945.3569770","type":"proceedings-article","created":{"date-parts":[[2023,3,3]],"date-time":"2023-03-03T23:40:27Z","timestamp":1677886827000},"page":"563-569","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":191,"title":["Using Large Language Models to Enhance Programming Error Messages"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0001-6829-9449","authenticated-orcid":false,"given":"Juho","family":"Leinonen","sequence":"first","affiliation":[{"name":"Aalto University, Espoo, Finland"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-6502-209X","authenticated-orcid":false,"given":"Arto","family":"Hellas","sequence":"additional","affiliation":[{"name":"Aalto University, Espoo, Finland"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-7277-9282","authenticated-orcid":false,"given":"Sami","family":"Sarsa","sequence":"additional","affiliation":[{"name":"Aalto University, Espoo, Finland"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-5781-1136","authenticated-orcid":false,"given":"Brent","family":"Reeves","sequence":"additional","affiliation":[{"name":"Abilene Christian University, Abilene, TX, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-5150-9806","authenticated-orcid":false,"given":"Paul","family":"Denny","sequence":"additional","affiliation":[{"name":"The University of Auckland, Auckland, New Zealand"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-2807-6042","authenticated-orcid":false,"given":"James","family":"Prather","sequence":"additional","affiliation":[{"name":"Abilene Christian University, Abilene, TX, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-1446-647X","authenticated-orcid":false,"given":"Brett A.","family":"Becker","sequence":"additional","affiliation":[{"name":"University College Dublin, Dublin, Ireland"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2023,3,3]]},"reference":[{"key":"e_1_3_2_2_1_1","volume-title":"Noah Rose Ledesma, and Premkumar Devanbu","author":"Ahmed Toufique","year":"2021","unstructured":"Toufique Ahmed, Noah Rose Ledesma, and Premkumar Devanbu. 2021. SYNFIX: Automatically Fixing Syntax Errors using Compiler Diagnostics. arXiv preprint arXiv:2104.14671 (2021)."},{"key":"e_1_3_2_2_2_1","doi-asserted-by":"publisher","DOI":"10.1145\/3183377.3183383"},{"key":"e_1_3_2_2_3_1","volume-title":"Error Messages as Rational Reconstructions. Ph.,D. Dissertation","author":"Barik Titus","unstructured":"Titus Barik. 2018. Error Messages as Rational Reconstructions. Ph.,D. Dissertation. North Carolina State University."},{"key":"e_1_3_2_2_4_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE.2017.59"},{"key":"e_1_3_2_2_5_1","doi-asserted-by":"publisher","DOI":"10.1145\/2839509.2844584"},{"key":"e_1_3_2_2_6_1","doi-asserted-by":"publisher","DOI":"10.1145\/3469115"},{"key":"e_1_3_2_2_7_1","doi-asserted-by":"publisher","DOI":"10.1145\/3344429.3372508"},{"key":"e_1_3_2_2_8_1","volume-title":"Towards Assessing the Readability of Programming Error Messages. In Australasian Computing Education Conference. ACM, 181--188","author":"Becker Brett A.","year":"2021","unstructured":"Brett A. Becker, Paul Denny, James Prather, Raymond Pettit, Robert Nix, and Catherine Mooney. 2021. Towards Assessing the Readability of Programming Error Messages. In Australasian Computing Education Conference. ACM, 181--188."},{"key":"e_1_3_2_2_9_1","doi-asserted-by":"publisher","DOI":"10.1080\/08993408.2016.1225464"},{"key":"e_1_3_2_2_10_1","doi-asserted-by":"publisher","DOI":"10.1145\/3159450.3159461"},{"key":"e_1_3_2_2_11_1","volume-title":"Categorizing Compiler Error Messages with Principal Component Analysis. In 12th China-Europe International Symposium on Software Engineering Education (CEISEE","author":"Becker Brett A","year":"2016","unstructured":"Brett A Becker and Catherine Mooney. 2016. Categorizing Compiler Error Messages with Principal Component Analysis. In 12th China-Europe International Symposium on Software Engineering Education (CEISEE 2016)."},{"key":"e_1_3_2_2_12_1","first-page":"1877","article-title":"Language Models are Few-shot Learners","volume":"33","author":"Brown Tom","year":"2020","unstructured":"Tom Brown, Benjamin Mann, Nick Ryder, Melanie Subbiah, Jared D Kaplan, Prafulla Dhariwal, Arvind Neelakantan, Pranav Shyam, Girish Sastry, Amanda Askell, et al. 2020. Language Models are Few-shot Learners. Advances in Neural Information Processing Systems, Vol. 33 (2020), 1877--1901.","journal-title":"Advances in Neural Information Processing Systems"},{"key":"e_1_3_2_2_13_1","volume-title":"Jared Kaplan, Harri Edwards, Yuri Burda, Nicholas Joseph, Greg Brockman, et al.","author":"Chen Mark","year":"2021","unstructured":"Mark Chen, Jerry Tworek, Heewoo Jun, Qiming Yuan, Henrique Ponde de Oliveira Pinto, Jared Kaplan, Harri Edwards, Yuri Burda, Nicholas Joseph, Greg Brockman, et al. 2021. Evaluating Large Language Models Trained on Code. arXiv preprint arXiv:2107.03374 (2021)."},{"key":"e_1_3_2_2_14_1","doi-asserted-by":"publisher","DOI":"10.1145\/2591708.2591748"},{"key":"e_1_3_2_2_15_1","doi-asserted-by":"publisher","DOI":"10.1145\/1999747.1999807"},{"key":"e_1_3_2_2_16_1","doi-asserted-by":"publisher","DOI":"10.1145\/3341525.3387384"},{"key":"e_1_3_2_2_17_1","volume-title":"Proceedings of the 2021 CHI Conference on Human Factors in Computing Systems. ACM.","author":"Denny Paul","unstructured":"Paul Denny, James Prather, Brett A. Becker, Catherine Mooney, John Homer, Zachary C Albrecht, and Garrett B. Powell. 2021. On Designing Programming Error Messages for Novices: Readability and Its Constituent Factors. In Proceedings of the 2021 CHI Conference on Human Factors in Computing Systems. ACM."},{"key":"e_1_3_2_2_18_1","volume-title":"Robosourcing Educational Resources--Leveraging Large Language Models for Learnersourcing. arXiv preprint arXiv:2211.04715","author":"Denny Paul","year":"2022","unstructured":"Paul Denny, Sami Sarsa, Arto Hellas, and Juho Leinonen. 2022. Robosourcing Educational Resources--Leveraging Large Language Models for Learnersourcing. arXiv preprint arXiv:2211.04715 (2022)."},{"key":"e_1_3_2_2_19_1","volume-title":"The Robots Are Coming: Exploring the Implications of OpenAI Codex on Introductory Programming. In Australasian Computing Education Conference. 10--19","author":"Finnie-Ansley James","year":"2022","unstructured":"James Finnie-Ansley, Paul Denny, Brett A Becker, Andrew Luxton-Reilly, and James Prather. 2022. The Robots Are Coming: Exploring the Implications of OpenAI Codex on Introductory Programming. In Australasian Computing Education Conference. 10--19."},{"key":"e_1_3_2_2_20_1","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v33i01.3301930"},{"key":"e_1_3_2_2_21_1","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v31i1.10742"},{"key":"e_1_3_2_2_22_1","doi-asserted-by":"crossref","unstructured":"Slava Kalyuga. 2009. The Expertise Reversal Effect. In Managing cognitive load in adaptive multimedia learning. IGI Global 58--80.","DOI":"10.4018\/978-1-60566-048-6.ch003"},{"key":"e_1_3_2_2_23_1","volume-title":"Proceedings of the 51st ACM Technical Symposium on Computer Science Education. ACM, 759--765","author":"Karvelas Ioannis","unstructured":"Ioannis Karvelas, Annie Li, and Brett A. Becker. 2020. The Effects of Compilation Mechanisms and Error Message Presentation on Novice Programmer Behavior. In Proceedings of the 51st ACM Technical Symposium on Computer Science Education. ACM, 759--765."},{"key":"e_1_3_2_2_24_1","doi-asserted-by":"publisher","DOI":"10.1145\/3287324.3287381"},{"key":"e_1_3_2_2_25_1","volume-title":"The Measurement of Observer Agreement for Categorical Data. biometrics","author":"Richard Landis J","year":"1977","unstructured":"J Richard Landis and Gary G Koch. 1977. The Measurement of Observer Agreement for Categorical Data. biometrics (1977), 159--174."},{"key":"e_1_3_2_2_26_1","doi-asserted-by":"publisher","DOI":"10.1145\/3490443"},{"key":"e_1_3_2_2_27_1","doi-asserted-by":"publisher","DOI":"10.1145\/3287324.3287503"},{"key":"e_1_3_2_2_28_1","volume-title":"A Systematic Survey of Prompting Methods in Natural Language Processing. arXiv preprint arXiv:2107.13586","author":"Liu Pengfei","year":"2021","unstructured":"Pengfei Liu, Weizhe Yuan, Jinlan Fu, Zhengbao Jiang, Hiroaki Hayashi, and Graham Neubig. 2021. Pre-train, Prompt, and Predict: A Systematic Survey of Prompting Methods in Natural Language Processing. arXiv preprint arXiv:2107.13586 (2021)."},{"key":"e_1_3_2_2_29_1","doi-asserted-by":"publisher","DOI":"10.1145\/3501709.3544280"},{"key":"e_1_3_2_2_30_1","doi-asserted-by":"publisher","DOI":"10.1109\/SP46214.2022.9833571"},{"key":"e_1_3_2_2_31_1","doi-asserted-by":"publisher","DOI":"10.1145\/3017680.3017768"},{"key":"e_1_3_2_2_32_1","doi-asserted-by":"publisher","DOI":"10.1145\/3230977.3230981"},{"key":"e_1_3_2_2_33_1","doi-asserted-by":"publisher","DOI":"10.1145\/3105726.3106169"},{"key":"e_1_3_2_2_34_1","doi-asserted-by":"publisher","DOI":"10.1145\/3291279.3339423"},{"key":"e_1_3_2_2_35_1","doi-asserted-by":"publisher","DOI":"10.1145\/365660.365671"},{"key":"e_1_3_2_2_36_1","doi-asserted-by":"publisher","DOI":"10.1145\/3501385.3543957"},{"key":"e_1_3_2_2_37_1","doi-asserted-by":"publisher","DOI":"10.1145\/2534973"},{"key":"e_1_3_2_2_38_1","doi-asserted-by":"publisher","DOI":"10.1145\/3491101.3519665"}],"event":{"name":"SIGCSE 2023: The 54th ACM Technical Symposium on Computer Science Education","location":"Toronto ON Canada","acronym":"SIGCSE 2023","sponsor":["SIGCSE ACM Special Interest Group on Computer Science Education"]},"container-title":["Proceedings of the 54th ACM Technical Symposium on Computer Science Education V. 1"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3545945.3569770","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3545945.3569770","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T18:10:00Z","timestamp":1750183800000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3545945.3569770"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023,3,2]]},"references-count":38,"alternative-id":["10.1145\/3545945.3569770","10.1145\/3545945"],"URL":"https:\/\/doi.org\/10.1145\/3545945.3569770","relation":{},"subject":[],"published":{"date-parts":[[2023,3,2]]},"assertion":[{"value":"2023-03-03","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}