{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,6,23]],"date-time":"2026-06-23T03:04:50Z","timestamp":1782183890501,"version":"3.54.5"},"publisher-location":"New York, NY, USA","reference-count":48,"publisher":"ACM","license":[{"start":{"date-parts":[[2024,9,5]],"date-time":"2024-09-05T00:00:00Z","timestamp":1725494400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by-sa\/4.0\/"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2024,9,5]]},"DOI":"10.1145\/3689535.3689554","type":"proceedings-article","created":{"date-parts":[[2024,11,6]],"date-time":"2024-11-06T15:39:49Z","timestamp":1730907589000},"page":"1-7","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":12,"title":["Not the Silver Bullet: LLM-enhanced Programming Error Messages are Ineffective in Practice"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0001-5337-715X","authenticated-orcid":false,"given":"Eddie Antonio","family":"Santos","sequence":"first","affiliation":[{"name":"School of Computer Science, University College Dublin, Ireland"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-1446-647X","authenticated-orcid":false,"given":"Brett A.","family":"Becker","sequence":"additional","affiliation":[{"name":"School of Computer Science, University College Dublin, Ireland"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2024,11,6]]},"reference":[{"key":"e_1_3_2_1_1_1","volume-title":"Error Messages as Rational Reconstructions. Ph.\u00a0D. Dissertation","author":"Barik Titus","unstructured":"Titus Barik. 2018. Error Messages as Rational Reconstructions. Ph.\u00a0D. Dissertation. North Carolina State University."},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.1093\/comjnl\/19.1.93-a"},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1145\/2839509.2844584"},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.1145\/3545945.3569759"},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1145\/3344429.3372508"},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"publisher","DOI":"10.1145\/3159450.3159461"},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"publisher","DOI":"10.1145\/3159450.3159453"},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.1145\/2163.358083"},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1093\/comjnl\/15.4.302"},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.1145\/3626252.3630909"},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.1145\/2591708.2591748"},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.1145\/3411764.3445696"},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1145\/3511861.3511863"},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.1145\/3576123.3576134"},{"key":"e_1_3_2_1_15_1","volume-title":"Examining the Effects of Enhanced Compilers on Student Productivity. Master\u2019s thesis","author":"Harker Devon","unstructured":"Devon Harker. 2017. Examining the Effects of Enhanced Compilers on Student Productivity. Master\u2019s thesis. University of Northern British Columbia."},{"key":"e_1_3_2_1_16_1","unstructured":"James\u00a0J. Horning. 1974. What the Compiler Should Tell the User. In Compiler Construction: An Advanced Course F.\u00a0L. Bauer F.\u00a0L. De\u00a0Remer M.\u00a0Griffiths U.\u00a0Hill J.\u00a0J. Horning C.\u00a0H.\u00a0A. Koster W.\u00a0M. McKeeman P.\u00a0C. Poole W.\u00a0M. Waite F.\u00a0L. Bauer and J.\u00a0Eickel (Eds.). Springer Berlin Heidelberg 525\u2013548."},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.1145\/3626253.3635404"},{"key":"e_1_3_2_1_18_1","unstructured":"Lei Huang Weijiang Yu Weitao Ma Weihong Zhong Zhangyin Feng Haotian Wang Qianglong Chen Weihua Peng Xiaocheng Feng Bing Qin and Ting Liu. 2023. A Survey on Hallucination in Large Language Models: Principles Taxonomy Challenges and Open Questions. arxiv:2311.05232\u00a0[cs.CL]"},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","DOI":"10.1145\/3287324.3287381"},{"key":"e_1_3_2_1_20_1","volume-title":"Proceedings of the 51st ACM Technical Symposium on Computer Science Education. ACM, NY, NY, USA, 1054\u20131060","author":"Kohn Tobias","year":"2020","unstructured":"Tobias Kohn and Bill Manaris. 2020. Tell Me What\u2019s Wrong: A Python IDE with Error Messages. In Proceedings of the 51st ACM Technical Symposium on Computer Science Education. ACM, NY, NY, USA, 1054\u20131060."},{"key":"e_1_3_2_1_21_1","volume-title":"\u201cBan It Till We Understand It","author":"Lau Sam","unstructured":"Sam Lau and Philip Guo. 2023. From \u201cBan It Till We Understand It\u201d to \u201cResistance is Futile\u201d: How University Programming Instructors Plan to Adapt as More Students Use AI Code Generation and Explanation Tools such as ChatGPT and GitHub Copilot. In Proceedings of the 2023 ACM Conference on International Computing Education Research - Volume 1 (Chicago, IL, USA) (ICER \u201923). Association for Computing Machinery, New York, NY, USA, 106\u2013121."},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.1145\/3545945.3569770"},{"key":"e_1_3_2_1_23_1","volume-title":"Retrieved","author":"Meredith Caleb","year":"2019","unstructured":"Caleb Meredith. 2019. Writing Good Compiler Error Messages. Code \u2192 Software. Retrieved August 24, 2022 from https:\/\/calebmer.com\/2019\/07\/01\/writing-good-compiler-error-messages.html"},{"key":"e_1_3_2_1_24_1","volume-title":"Retrieved","year":"2024","unstructured":"Microsoft. 2024. Monaco Editor. Retrieved April 10, 2024 from https:\/\/microsoft.github.io\/monaco-editor\/"},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.1145\/363018.363060"},{"key":"e_1_3_2_1_26_1","unstructured":"Hussein Mozannar Gagan Bansal Adam Fourney and Eric Horvitz. 2022. Reading Between the Lines: Modeling User Behavior and Costs in AI-Assisted Programming. arxiv:2210.14306\u00a0[cs]"},{"key":"e_1_3_2_1_27_1","unstructured":"Hussein Mozannar Valerie Chen Mohammed Alsobay Subhro Das Sebastian Zhao Dennis Wei Manish Nagireddy Prasanna Sattigeri Ameet Talwalkar and David Sontag. 2024. The RealHumanEval: Evaluating Large Language Models\u2019 Abilities to Support Programmers. arxiv:2404.02806\u00a0[cs.SE]"},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"crossref","unstructured":"Sydney Nguyen Hannah\u00a0McLean Babe Yangtian Zi Arjun Guha Carolyn\u00a0Jane Anderson and Molly\u00a0Q Feldman. 2024. How Beginning Programmers and Code LLMs (Mis)read Each Other. arxiv:2401.15232\u00a0[cs.HC]","DOI":"10.1145\/3613904.3642706"},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.1145\/1352322.1352192"},{"key":"e_1_3_2_1_30_1","volume-title":"Retrieved","author":"AI.","year":"2023","unstructured":"OpenAI. 2023. GPT-4 System Card. Retrieved May 14, 2023 from https:\/\/cdn.openai.com\/papers\/gpt-4-system-card.pdf"},{"key":"e_1_3_2_1_31_1","volume-title":"ChatGPT Is Here to Help, Not to Replace Anybody","author":"Pereira\u00a0Cipriano Bruno","unstructured":"Bruno Pereira\u00a0Cipriano and Pedro Alves. 2024. \u201cChatGPT Is Here to Help, Not to Replace Anybody\u201d \u2014 An Evaluation of Students\u2019 Opinions On Integrating ChatGPT In CS Courses. arxiv:2404.17443\u00a0[cs.ET]"},{"key":"e_1_3_2_1_32_1","doi-asserted-by":"publisher","DOI":"10.1145\/3017680.3017768"},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"publisher","DOI":"10.1145\/3576882.3617921"},{"key":"e_1_3_2_1_34_1","doi-asserted-by":"publisher","DOI":"10.1145\/3105726.3106169"},{"key":"e_1_3_2_1_35_1","doi-asserted-by":"publisher","DOI":"10.1145\/3617367"},{"key":"e_1_3_2_1_36_1","volume-title":"The Full Story of Large Language Models and RLHF. Retrieved","author":"Ramponi Marco","year":"2024","unstructured":"Marco Ramponi. 2023. The Full Story of Large Language Models and RLHF. Retrieved June 5, 2024 from https:\/\/www.assemblyai.com\/blog\/the-full-story-of-large-language-models-and-rlhf\/"},{"key":"e_1_3_2_1_37_1","first-page":"11","article-title":"PUFFT - The Purdue University Fast FORTRAN","volume":"8","author":"Rosen Saul","year":"1965","unstructured":"Saul Rosen, Robert\u00a0A. Spurgeon, and Joel\u00a0K. Donnelly. 1965. PUFFT - The Purdue University Fast FORTRAN Translator. Commun. ACM 8, 11 (nov 1965), 661\u2013666.","journal-title":"Translator. Commun. ACM"},{"key":"e_1_3_2_1_38_1","volume-title":"Errors and Lints \u2014 Rust Compiler Development Guide. Retrieved","author":"Rustc","year":"2024","unstructured":"Rustc developers. 2024. Errors and Lints \u2014 Rust Compiler Development Guide. Retrieved March 4, 2024 from https:\/\/rustc-dev-guide.rust-lang.org\/diagnostics"},{"key":"e_1_3_2_1_39_1","doi-asserted-by":"publisher","DOI":"10.1145\/3576882.3617909"},{"key":"e_1_3_2_1_40_1","doi-asserted-by":"publisher","DOI":"10.1145\/199688.199769"},{"key":"e_1_3_2_1_41_1","volume-title":"Retrieved","author":"Brian","year":"2024","unstructured":"Brian Seymour and contributors. 2023. Piston: A high performance general purpose code execution engine. Retrieved April 10, 2024 from https:\/\/github.com\/engineer-man\/piston"},{"key":"e_1_3_2_1_42_1","doi-asserted-by":"crossref","unstructured":"Auste Simkute Lev Tankelevitch Viktor Kewenig Ava\u00a0Elizabeth Scott Abigail Sellen and Sean Rintel. 2024. Ironies of Generative AI: Understanding and mitigating productivity loss in human-AI interactions. arxiv:2402.11364\u00a0[cs.HC]","DOI":"10.1080\/10447318.2024.2405782"},{"key":"e_1_3_2_1_43_1","volume-title":"On Compiler Error Messages: What They Say and What They Mean. Advances in Human-Computer Interaction 2010","author":"Traver Javier","year":"2010","unstructured":"V.\u00a0Javier Traver. 2010. On Compiler Error Messages: What They Say and What They Mean. Advances in Human-Computer Interaction 2010 (2010), 26\u00a0pages."},{"key":"e_1_3_2_1_44_1","doi-asserted-by":"publisher","DOI":"10.1145\/3491101.3519665"},{"key":"e_1_3_2_1_45_1","unstructured":"Jason Wei Yi Tay Rishi Bommasani Colin Raffel Barret Zoph Sebastian Borgeaud Dani Yogatama Maarten Bosma Denny Zhou Donald Metzler Ed\u00a0H. Chi Tatsunori Hashimoto Oriol Vinyals Percy Liang Jeff Dean and William Fedus. 2022. Emergent Abilities of Large Language Models. arxiv:2206.07682\u00a0[cs.CL]"},{"key":"e_1_3_2_1_46_1","doi-asserted-by":"publisher","DOI":"10.1145\/3570220"},{"key":"e_1_3_2_1_47_1","doi-asserted-by":"publisher","DOI":"10.5555\/800253.807695"},{"key":"e_1_3_2_1_48_1","doi-asserted-by":"crossref","unstructured":"Patricia Widjojo and Christoph Treude. 2023. Addressing Compiler Errors: Stack Overflow or Large Language Models?arxiv:2307.10793\u00a0[cs.SE]","DOI":"10.2139\/ssrn.4529345"}],"event":{"name":"UKICER 2024: The United Kingdom and Ireland Computing Education Research","location":"Manchester United Kingdom","acronym":"UKICER 2024"},"container-title":["Proceedings of the 2024 Conference on United Kingdom & Ireland Computing Education Research"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3689535.3689554","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3689535.3689554","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,8,23]],"date-time":"2025-08-23T02:13:49Z","timestamp":1755915229000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3689535.3689554"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,9,5]]},"references-count":48,"alternative-id":["10.1145\/3689535.3689554","10.1145\/3689535"],"URL":"https:\/\/doi.org\/10.1145\/3689535.3689554","relation":{},"subject":[],"published":{"date-parts":[[2024,9,5]]},"assertion":[{"value":"2024-11-06","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}