{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,13]],"date-time":"2026-04-13T10:11:10Z","timestamp":1776075070975,"version":"3.50.1"},"publisher-location":"New York, NY, USA","reference-count":63,"publisher":"ACM","license":[{"start":{"date-parts":[[2024,1,29]],"date-time":"2024-01-29T00:00:00Z","timestamp":1706486400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2024,1,29]]},"DOI":"10.1145\/3636243.3636245","type":"proceedings-article","created":{"date-parts":[[2024,1,2]],"date-time":"2024-01-02T23:20:13Z","timestamp":1704237613000},"page":"11-18","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":27,"title":["Decoding Logic Errors: A Comparative Study on Bug Detection by Students and Large Language Models"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-2781-6619","authenticated-orcid":false,"given":"Stephen","family":"Macneil","sequence":"first","affiliation":[{"name":"Temple University, United States"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-5150-9806","authenticated-orcid":false,"given":"Paul","family":"Denny","sequence":"additional","affiliation":[{"name":"The University of Auckland, New Zealand"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-0094-1113","authenticated-orcid":false,"given":"Andrew","family":"Tran","sequence":"additional","affiliation":[{"name":"Temple University, United States"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-6829-9449","authenticated-orcid":false,"given":"Juho","family":"Leinonen","sequence":"additional","affiliation":[{"name":"The University of Auckland, New Zealand"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-5767-1057","authenticated-orcid":false,"given":"Seth","family":"Bernstein","sequence":"additional","affiliation":[{"name":"Temple University, United States"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-6502-209X","authenticated-orcid":false,"given":"Arto","family":"Hellas","sequence":"additional","affiliation":[{"name":"Aalto University, Finland"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-7277-9282","authenticated-orcid":false,"given":"Sami","family":"Sarsa","sequence":"additional","affiliation":[{"name":"Aalto University, Finland"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-7646-2373","authenticated-orcid":false,"given":"Joanne","family":"Kim","sequence":"additional","affiliation":[{"name":"Temple University, United States"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2024,1,29]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"publisher","DOI":"10.1145\/3017680.3017761"},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.1145\/2676723.2677258"},{"key":"e_1_3_2_1_3_1","volume-title":"Ask Me Anything: A simple strategy for prompting language models. ArXiv abs\/2210.02441","author":"Arora Simran","year":"2022","unstructured":"Simran Arora, Avanika Narayan, Mayee\u00a0F. Chen, Laurel\u00a0J. Orr, Neel Guha, Kush\u00a0S Bhatia, Ines Chami, Frederic Sala, and Christopher R\u2019e. 2022. Ask Me Anything: A simple strategy for prompting language models. ArXiv abs\/2210.02441 (2022)."},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-39112-5_44"},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1145\/3344429.3372508"},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"publisher","DOI":"10.1145\/2632320.2632343"},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"publisher","DOI":"10.1145\/2994154"},{"key":"e_1_3_2_1_8_1","volume-title":"Language models are few-shot learners. Advances in neural information processing systems 33","author":"Brown Tom","year":"2020","unstructured":"Tom Brown, Benjamin Mann, Nick Ryder, Melanie Subbiah, Jared\u00a0D Kaplan, Prafulla Dhariwal, Arvind Neelakantan, Pranav Shyam, Girish Sastry, Amanda Askell, 2020. Language models are few-shot learners. Advances in neural information processing systems 33 (2020), 1877\u20131901."},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1145\/3545945.3569823"},{"key":"e_1_3_2_1_10_1","volume-title":"Promptly: Using Prompt Problems to Teach Learners How to Effectively Utilize AI Code Generators. arXiv preprint arXiv:2307.16364","author":"Denny Paul","year":"2023","unstructured":"Paul Denny, Juho Leinonen, James Prather, Andrew Luxton-Reilly, Thezyrie Amarouche, Brett\u00a0A Becker, and Brent\u00a0N Reeves. 2023. Promptly: Using Prompt Problems to Teach Learners How to Effectively Utilize AI Code Generators. arXiv preprint arXiv:2307.16364 (2023)."},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.1145\/2325296.2325318"},{"key":"e_1_3_2_1_12_1","volume-title":"Computing Education in the Era of Generative AI. arXiv preprint arXiv:2306.02608","author":"Denny Paul","year":"2023","unstructured":"Paul Denny, James Prather, Brett\u00a0A Becker, James Finnie-Ansley, Arto Hellas, Juho Leinonen, Andrew Luxton-Reilly, Brent\u00a0N Reeves, Eddie\u00a0Antonio Santos, and Sami Sarsa. 2023. Computing Education in the Era of Generative AI. arXiv preprint arXiv:2306.02608 (2023)."},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1145\/3411764.3445696"},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.1145\/3160489.3160493"},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE48619.2023.00128"},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.1145\/3511861.3511863"},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.1080\/08993400802114508"},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1145\/2699751"},{"key":"e_1_3_2_1_19_1","volume-title":"Synthesizing research on programmers","author":"Heinonen Ava","year":"2023","unstructured":"Ava Heinonen, Bettina Lehtel\u00e4, Arto Hellas, and Fabian Fagerholm. 2023. Synthesizing research on programmers\u2019 mental models of programs, tasks and concepts\u2014A systematic literature review. Information and Software Technology (2023), 107300."},{"key":"e_1_3_2_1_20_1","volume-title":"Help Requests. arXiv preprint arXiv:2306.05715","author":"Hellas Arto","year":"2023","unstructured":"Arto Hellas, Juho Leinonen, Sami Sarsa, Charles Koutcheme, Lilja Kujanp\u00e4\u00e4, and Juha Sorva. 2023. Exploring the Responses of Large Language Models to Beginner Programmers\u2019 Help Requests. arXiv preprint arXiv:2306.05715 (2023)."},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.48550\/arXiv.2311.04926"},{"key":"e_1_3_2_1_22_1","volume-title":"The Effects of Generative AI on Introductory Students\u2019 Help-Seeking Preferences. In Australasian Computing Education Conference(ACM ACE \u201924)","author":"Hou Irene","year":"2024","unstructured":"Irene Hou, Sophia Mettille, Owen Man, Zhuo Li, Cynthia Zastudil, and Stephen MacNeil. 2024. The Effects of Generative AI on Introductory Students\u2019 Help-Seeking Preferences. In Australasian Computing Education Conference(ACM ACE \u201924)."},{"key":"e_1_3_2_1_23_1","volume-title":"Impact of code language models on automated program repair. arXiv preprint arXiv:2302.05020","author":"Jiang Nan","year":"2023","unstructured":"Nan Jiang, Kevin Liu, Thibaud Lutellier, and Lin Tan. 2023. Impact of code language models on automated program repair. arXiv preprint arXiv:2302.05020 (2023)."},{"key":"e_1_3_2_1_24_1","volume-title":"Bug Catalogue: I. Technical Report","author":"Johnson Lewis","year":"1983","unstructured":"W.\u00a0Lewis Johnson, Elliot Soloway, Benjamin Cutler, and Steven Draper. 1983. Bug Catalogue: I. Technical Report. Yale University, YaleU\/CSD\/RR #286."},{"key":"e_1_3_2_1_25_1","volume-title":"Who Answers It Better? An In-Depth Analysis of ChatGPT and Stack Overflow Answers to Software Engineering Questions. arXiv preprint arXiv:2308.02312","author":"Kabir Samia","year":"2023","unstructured":"Samia Kabir, David\u00a0N Udo-Imeh, Bonan Kou, and Tianyi Zhang. 2023. Who Answers It Better? An In-Depth Analysis of ChatGPT and Stack Overflow Answers to Software Engineering Questions. arXiv preprint arXiv:2308.02312 (2023)."},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.1145\/3287324.3287381"},{"key":"e_1_3_2_1_27_1","volume-title":"Evaluating CodeClusters for Effectively Providing Feedback on Code Submissions. In 2022 IEEE Frontiers in Education Conf. (FIE). IEEE, 1\u20139.","author":"Koivisto Teemu","year":"2022","unstructured":"Teemu Koivisto and Arto Hellas. 2022. Evaluating CodeClusters for Effectively Providing Feedback on Code Submissions. In 2022 IEEE Frontiers in Education Conf. (FIE). IEEE, 1\u20139."},{"key":"e_1_3_2_1_28_1","volume-title":"Automated Program Repair Using Generative Models for Code Infilling. In Int. Conf. on Artificial Intelligence in Education. Springer, 798\u2013803","author":"Koutcheme Charles","year":"2023","unstructured":"Charles Koutcheme, Sami Sarsa, Juho Leinonen, Arto Hellas, and Paul Denny. 2023. Automated Program Repair Using Generative Models for Code Infilling. In Int. Conf. on Artificial Intelligence in Education. Springer, 798\u2013803."},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.1145\/3568813.3600138"},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"publisher","DOI":"10.1145\/3587102.3588785"},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"publisher","DOI":"10.1145\/3545945.3569770"},{"key":"e_1_3_2_1_32_1","doi-asserted-by":"publisher","DOI":"10.1145\/1041624.1041673"},{"key":"e_1_3_2_1_33_1","first-page":"161","article-title":"Further Evidence of a Relationship between Explaining","volume":"41","author":"Lister Raymond","year":"2009","unstructured":"Raymond Lister, Colin Fidge, and Donna Teague. 2009. Further Evidence of a Relationship between Explaining, Tracing and Writing Skills in Introductory Programming. SIGCSE Bull. 41, 3 (2009), 161\u2013165.","journal-title":"Tracing and Writing Skills in Introductory Programming. SIGCSE Bull."},{"key":"e_1_3_2_1_34_1","doi-asserted-by":"publisher","DOI":"10.1145\/3545947.3573358"},{"key":"e_1_3_2_1_35_1","doi-asserted-by":"publisher","DOI":"10.1145\/3545945.3569785"},{"key":"e_1_3_2_1_36_1","doi-asserted-by":"publisher","DOI":"10.1145\/3545947.3569630"},{"key":"e_1_3_2_1_37_1","doi-asserted-by":"publisher","DOI":"10.1145\/3334480.3383065"},{"key":"e_1_3_2_1_38_1","doi-asserted-by":"publisher","DOI":"10.1145\/3335814"},{"key":"e_1_3_2_1_39_1","volume-title":"Cary Laxer, Lynda Thomas, Ian Utting, and Tadeusz Wilusz.","author":"McCracken Michael","year":"2001","unstructured":"Michael McCracken, Vicki Almstrum, Danny Diaz, Mark Guzdial, Dianne Hagan, Yifat Ben-David Kolikant, Cary Laxer, Lynda Thomas, Ian Utting, and Tadeusz Wilusz. 2001. A multi-national, multi-institutional study of assessment of programming skills of first-year CS students. In Working group reports from ITiCSE on Innovation and technology in computer science education. 125\u2013180."},{"key":"e_1_3_2_1_40_1","volume-title":"Debugging: the good, the bad, and the quirky\u2013a qualitative analysis of novices","author":"Murphy Laurie","unstructured":"Laurie Murphy, Gary Lewandowski, Ren\u00e9e McCauley, Beth Simon, Lynda Thomas, and Carol Zander. 2008. Debugging: the good, the bad, and the quirky\u2013a qualitative analysis of novices\u2019 strategies. In ACM SIGCSE Bulletin, Vol.\u00a040. ACM."},{"key":"e_1_3_2_1_41_1","doi-asserted-by":"publisher","DOI":"10.1145\/3587103.3594206"},{"key":"e_1_3_2_1_42_1","volume-title":"The robots are here: Navigating the generative ai revolution in computing education. arXiv preprint arXiv:2310.00658","author":"Prather James","year":"2023","unstructured":"James Prather, Paul Denny, Juho Leinonen, Brett\u00a0A Becker, Ibrahim Albluwi, Michelle Craig, Hieke Keuning, Natalie Kiesler, Tobias Kohn, Andrew Luxton-Reilly, 2023. The robots are here: Navigating the generative ai revolution in computing education. arXiv preprint arXiv:2310.00658 (2023)."},{"key":"e_1_3_2_1_43_1","first-page":"37","article-title":"Github copilot in the classroom: learning to code with AI assistance","volume":"38","author":"Puryear Ben","year":"2022","unstructured":"Ben Puryear and Gina Sprint. 2022. Github copilot in the classroom: learning to code with AI assistance. J. of Computing Sciences in Colleges 38, 1 (2022), 37\u201347.","journal-title":"J. of Computing Sciences in Colleges"},{"key":"e_1_3_2_1_44_1","doi-asserted-by":"publisher","DOI":"10.1145\/3501385.3543957"},{"key":"e_1_3_2_1_45_1","doi-asserted-by":"publisher","DOI":"10.1145\/3568813.3600142"},{"key":"e_1_3_2_1_46_1","doi-asserted-by":"crossref","unstructured":"Jaromir Savelka Arav Agarwal Christopher Bogart and Majd Sakr. 2023. Large Language Models (GPT) Struggle to Answer Multiple-Choice Questions about Code. arxiv:2303.08033\u00a0[cs.CL]","DOI":"10.5220\/0011996900003470"},{"key":"e_1_3_2_1_47_1","volume-title":"Can Generative Pre-trained Transformers (GPT) Pass Assessments in Higher Education Programming Courses?arXiv preprint arXiv:2303.09325","author":"Savelka Jaromir","year":"2023","unstructured":"Jaromir Savelka, Arav Agarwal, Christopher Bogart, Yifan Song, and Majd Sakr. 2023. Can Generative Pre-trained Transformers (GPT) Pass Assessments in Higher Education Programming Courses?arXiv preprint arXiv:2303.09325 (2023)."},{"key":"e_1_3_2_1_48_1","doi-asserted-by":"publisher","DOI":"10.1145\/2828959.2828963"},{"key":"e_1_3_2_1_49_1","unstructured":"Chenglei Si Zhe Gan Zhengyuan Yang Shuohang Wang Jianfeng Wang Jordan Boyd-Graber and Lijuan Wang. 2023. Prompting GPT-3 To Be Reliable. arxiv:2210.09150\u00a0[cs.CL]"},{"key":"e_1_3_2_1_50_1","doi-asserted-by":"publisher","DOI":"10.1145\/3287324.3287394"},{"key":"e_1_3_2_1_51_1","doi-asserted-by":"publisher","DOI":"10.1145\/182.358436"},{"key":"e_1_3_2_1_52_1","volume-title":"What do novices know about programming? In Directions in Human\u2013Computer Interactions. Vol.\u00a06","author":"Soloway Elliot","unstructured":"Elliot Soloway, Kate Ehrlich, Jeffrey\u00a0G. Bonar, and Judith Greenspan. 1982. What do novices know about programming? In Directions in Human\u2013Computer Interactions. Vol.\u00a06. Ablex Publishing, 27\u201354."},{"key":"e_1_3_2_1_53_1","doi-asserted-by":"publisher","DOI":"10.1145\/6138.6145"},{"key":"e_1_3_2_1_54_1","volume-title":"Improved performance of ChatGPT-4 on the OKAP exam: A comparative study with ChatGPT-3.5. medRxiv","author":"Teebagy Sean","year":"2023","unstructured":"Sean Teebagy, Lauren Colwell, Emma Wood, Antonio Yaghy, and Misha Faustina. 2023. Improved performance of ChatGPT-4 on the OKAP exam: A comparative study with ChatGPT-3.5. medRxiv (2023), 2023\u201304."},{"key":"e_1_3_2_1_55_1","doi-asserted-by":"publisher","DOI":"10.1145\/3568812.3603482"},{"key":"e_1_3_2_1_56_1","doi-asserted-by":"publisher","DOI":"10.1145\/1269900.1268853"},{"key":"e_1_3_2_1_57_1","doi-asserted-by":"publisher","DOI":"10.1145\/3545945.3569830"},{"key":"e_1_3_2_1_58_1","volume-title":"The many ways of the BRACElet project. Bull. of Applied Computing and Information Technology","author":"Whalley Jacqueline","year":"2007","unstructured":"Jacqueline Whalley, Tony Clear, and RF Lister. 2007. The many ways of the BRACElet project. Bull. of Applied Computing and Information Technology (2007)."},{"key":"e_1_3_2_1_59_1","doi-asserted-by":"publisher","DOI":"10.1145\/3408877.3432374"},{"key":"e_1_3_2_1_60_1","volume-title":"Proc. of the 8th Australasian Conf. on Computing Education -","volume":"52","author":"Whalley L.","year":"2006","unstructured":"Jacqueline\u00a0L. Whalley, Raymond Lister, Errol Thompson, Tony Clear, Phil Robbins, P.\u00a0K.\u00a0Ajith Kumar, and Christine Prasad. 2006. An Australasian Study of Reading and Comprehension Skills in Novice Programmers, Using the Bloom and SOLO Taxonomies. In Proc. of the 8th Australasian Conf. on Computing Education - Volume 52. Australian Computer Society, Inc., AUS, 243\u2013252."},{"key":"e_1_3_2_1_61_1","doi-asserted-by":"publisher","DOI":"10.5555\/2615731.2615782"},{"key":"e_1_3_2_1_62_1","doi-asserted-by":"publisher","DOI":"10.48550\/arXiv.2308.04309"},{"key":"e_1_3_2_1_63_1","volume-title":"Calibrate Before Use: Improving Few-Shot Performance of Language Models. In Int. Conf. on Machine Learning.","author":"Zhao Tony","year":"2021","unstructured":"Tony Zhao, Eric Wallace, Shi Feng, Dan Klein, and Sameer Singh. 2021. Calibrate Before Use: Improving Few-Shot Performance of Language Models. In Int. Conf. on Machine Learning."}],"event":{"name":"ACE 2024: Australian Computing Education Conference","location":"Sydney NSW Australia","acronym":"ACE 2024"},"container-title":["Proceedings of the 26th Australasian Computing Education Conference"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3636243.3636245","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3636243.3636245","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T16:35:41Z","timestamp":1750178141000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3636243.3636245"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,1,29]]},"references-count":63,"alternative-id":["10.1145\/3636243.3636245","10.1145\/3636243"],"URL":"https:\/\/doi.org\/10.1145\/3636243.3636245","relation":{},"subject":[],"published":{"date-parts":[[2024,1,29]]},"assertion":[{"value":"2024-01-29","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}