{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,18]],"date-time":"2026-07-18T02:37:44Z","timestamp":1784342264817,"version":"3.55.0"},"publisher-location":"New York, NY, USA","reference-count":30,"publisher":"ACM","license":[{"start":{"date-parts":[[2020,6,15]],"date-time":"2020-06-15T00:00:00Z","timestamp":1592179200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2020,6,15]]},"DOI":"10.1145\/3341525.3387384","type":"proceedings-article","created":{"date-parts":[[2020,6,3]],"date-time":"2020-06-03T04:15:40Z","timestamp":1591157740000},"page":"480-486","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":45,"title":["Error Message Readability and Novice Debugging Performance"],"prefix":"10.1145","author":[{"given":"Paul","family":"Denny","sequence":"first","affiliation":[{"name":"University of Auckland, Auckland, New Zealand"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"James","family":"Prather","sequence":"additional","affiliation":[{"name":"Abilene Christian University, Abilene, TX, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Brett A.","family":"Becker","sequence":"additional","affiliation":[{"name":"University College Dublin, Dublin, Ireland"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2020,6,15]]},"reference":[{"key":"e_1_3_2_1_2_1","volume-title":"How Should Compilers Explain Problems to Developers? (ESEC\/FSE","author":"Barik Titus","year":"2018","unstructured":"Titus Barik , Denae Ford , Emerson Murphy-Hill , and Chris Parnin . 2018. How Should Compilers Explain Problems to Developers? (ESEC\/FSE 2018 ). ACM , New York, NY, USA , 633--643. https:\/\/doi.org\/10.1145\/3236024.3236040 10.1145\/3236024.3236040 Titus Barik, Denae Ford, Emerson Murphy-Hill, and Chris Parnin. 2018. How Should Compilers Explain Problems to Developers? (ESEC\/FSE 2018). ACM, New York, NY, USA, 633--643. https:\/\/doi.org\/10.1145\/3236024.3236040"},{"key":"e_1_3_2_1_3_1","volume-title":"Do Developers Read Compiler Error Messages? (ICSE '17)","author":"Barik Titus","year":"2017","unstructured":"Titus Barik , Justin Smith , Kevin Lubick , Elisabeth Holmes , Jing Feng , Emerson Murphy-Hill , and Chris Parnin . 2017. Do Developers Read Compiler Error Messages? (ICSE '17) . IEEE Press , 575--585. https:\/\/doi.org\/10.1109\/ICSE. 2017 .59 10.1109\/ICSE.2017.59 Titus Barik, Justin Smith, Kevin Lubick, Elisabeth Holmes, Jing Feng, Emerson Murphy-Hill, and Chris Parnin. 2017. Do Developers Read Compiler Error Messages? (ICSE '17). IEEE Press, 575--585. https:\/\/doi.org\/10.1109\/ICSE.2017.59"},{"key":"e_1_3_2_1_4_1","volume-title":"An Exploration Of The Effects Of Enhanced Compiler Error Messages For Computer Programming Novices. Masters Thesis","author":"Becker Brett A.","unstructured":"Brett A. Becker . 2015. An Exploration Of The Effects Of Enhanced Compiler Error Messages For Computer Programming Novices. Masters Thesis . Dublin Institute of Technology . https:\/\/doi.org\/10.13140\/RG.2.2.26637.13288 10.13140\/RG.2.2.26637.13288 Brett A. Becker. 2015. An Exploration Of The Effects Of Enhanced Compiler Error Messages For Computer Programming Novices. Masters Thesis. Dublin Institute of Technology. https:\/\/doi.org\/10.13140\/RG.2.2.26637.13288"},{"key":"e_1_3_2_1_5_1","volume-title":"An Effective Approach to Enhancing Compiler Error Messages (SIGCSE '16)","author":"Becker Brett A.","unstructured":"Brett A. Becker . 2016. An Effective Approach to Enhancing Compiler Error Messages (SIGCSE '16) . ACM , New York, NY, USA , 126--131. https:\/\/doi.org\/10.1145\/2839509.2844584 10.1145\/2839509.2844584 Brett A. Becker. 2016. An Effective Approach to Enhancing Compiler Error Messages (SIGCSE '16). ACM, New York, NY, USA, 126--131. https:\/\/doi.org\/10.1145\/2839509.2844584"},{"key":"e_1_3_2_1_6_1","volume-title":"Compiler Error Messages Considered Unhelpful: The Landscape of Text-Based Programming Error Message Research (ITiCSE-WGR '19)","author":"Becker Brett A.","unstructured":"Brett A. Becker , Paul Denny , Raymond Pettit , Durell Bouchard , Dennis J. Bouvier , Brian Harrington , Amir Kamil , Amey Karkare , Chris McDonald , Peter-Michael Osera , and et al. 2019 . Compiler Error Messages Considered Unhelpful: The Landscape of Text-Based Programming Error Message Research (ITiCSE-WGR '19) . ACM, New York, NY, USA, 177--210. https:\/\/doi.org\/10.1145\/3344429.3372508 10.1145\/3344429.3372508 Brett A. Becker, Paul Denny, Raymond Pettit, Durell Bouchard, Dennis J. Bouvier, Brian Harrington, Amir Kamil, Amey Karkare, Chris McDonald, Peter-Michael Osera, and et al. 2019. Compiler Error Messages Considered Unhelpful: The Landscape of Text-Based Programming Error Message Research (ITiCSE-WGR '19). ACM, New York, NY, USA, 177--210. https:\/\/doi.org\/10.1145\/3344429.3372508"},{"key":"e_1_3_2_1_7_1","volume-title":"The Effects of Enhanced Compiler Error Messages on a Syntax Error Debugging Test (SIGCSE '18)","author":"Becker Brett A.","unstructured":"Brett A. Becker , Kyle Goslin , and Graham Glanville . 2018a. The Effects of Enhanced Compiler Error Messages on a Syntax Error Debugging Test (SIGCSE '18) . ACM , New York, NY, USA , 640--645. https:\/\/doi.org\/10.1145\/3159450.3159461 10.1145\/3159450.3159461 Brett A. Becker, Kyle Goslin, and Graham Glanville. 2018a. The Effects of Enhanced Compiler Error Messages on a Syntax Error Debugging Test (SIGCSE '18). ACM, New York, NY, USA, 640--645. https:\/\/doi.org\/10.1145\/3159450.3159461"},{"key":"e_1_3_2_1_8_1","volume-title":"Ignore the Rest: Dealing with Multiple Compiler Error Messages (SIGCSE '18)","author":"Becker Brett A.","year":"2018","unstructured":"Brett A. Becker , Cormac Murray , Tianyi Tao , Changheng Song , Robert McCartney , and Kate Sanders . 2018 b. Fix the First , Ignore the Rest: Dealing with Multiple Compiler Error Messages (SIGCSE '18) . ACM, New York, NY, USA, 634--639. https:\/\/doi.org\/10.1145\/3159450.3159453 10.1145\/3159450.3159453 Brett A. Becker, Cormac Murray, Tianyi Tao, Changheng Song, Robert McCartney, and Kate Sanders. 2018b. Fix the First, Ignore the Rest: Dealing with Multiple Compiler Error Messages (SIGCSE '18). ACM, New York, NY, USA, 634--639. https:\/\/doi.org\/10.1145\/3159450.3159453"},{"key":"e_1_3_2_1_9_1","volume-title":"Becker and Keith Quille","author":"Brett","year":"2019","unstructured":"Brett A. Becker and Keith Quille . 2019 . 50 Years of CS1 at SIGCSE : A Review of the Evolution of Introductory Programming Education Research (SIGCSE '19). ACM, New York, NY, USA , 338--344. https:\/\/doi.org\/10.1145\/3287324.3287432 10.1145\/3287324.3287432 Brett A. Becker and Keith Quille. 2019. 50 Years of CS1 at SIGCSE: A Review of the Evolution of Introductory Programming Education Research (SIGCSE '19). ACM, New York, NY, USA, 338--344. https:\/\/doi.org\/10.1145\/3287324.3287432"},{"key":"e_1_3_2_1_10_1","volume-title":"Blackbox: A Large Scale Repository of Novice Programmers' Activity (SIGCSE '14)","author":"Charles Brown Neil Christopher","year":"2014","unstructured":"Neil Christopher Charles Brown , Michael K\u00f6lling , Davin McCall , and Ian Utting . 2014 . Blackbox: A Large Scale Repository of Novice Programmers' Activity (SIGCSE '14) . ACM , New York, NY, USA , 223--228. https:\/\/doi.org\/10.1145\/2538862.2538924 10.1145\/2538862.2538924 Neil Christopher Charles Brown, Michael K\u00f6lling, Davin McCall, and Ian Utting. 2014. Blackbox: A Large Scale Repository of Novice Programmers' Activity (SIGCSE '14). ACM, New York, NY, USA, 223--228. https:\/\/doi.org\/10.1145\/2538862.2538924"},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.1177\/016555158701300403"},{"key":"e_1_3_2_1_12_1","volume-title":"Student Difficulties with Pointer Concepts in C (ACSW '16)","author":"Craig Michelle","year":"2016","unstructured":"Michelle Craig and Andrew Petersen . 2016 . Student Difficulties with Pointer Concepts in C (ACSW '16) . ACM, New York, NY, USA, Article Article 8, 10 pages. https:\/\/doi.org\/10.1145\/2843043.2843348 10.1145\/2843043.2843348 Michelle Craig and Andrew Petersen. 2016. Student Difficulties with Pointer Concepts in C (ACSW '16). ACM, New York, NY, USA, Article Article 8, 10 pages. https:\/\/doi.org\/10.1145\/2843043.2843348"},{"key":"e_1_3_2_1_13_1","first-page":"1","article-title":"Five-Point Likert Items: t test versus Mann-Whitney-Wilcoxon","volume":"15","author":"de Winter Joost C. F.","year":"2012","unstructured":"Joost C. F. de Winter and Dimitra Dodou . 2012 . Five-Point Likert Items: t test versus Mann-Whitney-Wilcoxon . Practical Assessment, Research & Evaluation , Vol. 15 , 11 (2012), 1 -- 16 . Joost C. F. de Winter and Dimitra Dodou. 2012. Five-Point Likert Items: t test versus Mann-Whitney-Wilcoxon. Practical Assessment, Research & Evaluation, Vol. 15, 11 (2012), 1--16.","journal-title":"Practical Assessment, Research & Evaluation"},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.1147\/sj.214.0424"},{"key":"e_1_3_2_1_15_1","volume-title":"Enhancing Syntax Error Messages Appears Ineffectual (ITiCSE '14)","author":"Denny Paul","year":"2014","unstructured":"Paul Denny , Andrew Luxton-Reilly , and Dave Carpenter . 2014 . Enhancing Syntax Error Messages Appears Ineffectual (ITiCSE '14) . ACM, New York, NY, USA, 273--278. https:\/\/doi.org\/10.1145\/2591708.2591748 10.1145\/2591708.2591748 Paul Denny, Andrew Luxton-Reilly, and Dave Carpenter. 2014. Enhancing Syntax Error Messages Appears Ineffectual (ITiCSE '14). ACM, New York, NY, USA, 273--278. https:\/\/doi.org\/10.1145\/2591708.2591748"},{"key":"e_1_3_2_1_16_1","volume-title":"All Syntax Errors Are Not Equal (ITiCSE '12)","author":"Denny Paul","year":"2012","unstructured":"Paul Denny , Andrew Luxton-Reilly , and Ewan Tempero . 2012 . All Syntax Errors Are Not Equal (ITiCSE '12) . ACM, New York, NY, USA, 75--80. https:\/\/doi.org\/10.1145\/2325296.2325318 10.1145\/2325296.2325318 Paul Denny, Andrew Luxton-Reilly, and Ewan Tempero. 2012. All Syntax Errors Are Not Equal (ITiCSE '12). ACM, New York, NY, USA, 75--80. https:\/\/doi.org\/10.1145\/2325296.2325318"},{"key":"e_1_3_2_1_17_1","volume-title":"Understanding the Syntax Barrier for Novices (ITiCSE '11)","author":"Denny Paul","year":"1999","unstructured":"Paul Denny , Andrew Luxton-Reilly , Ewan Tempero , and Jacob Hendrickx . 2011. Understanding the Syntax Barrier for Novices (ITiCSE '11) . ACM , New York, NY, USA , 208--212. https:\/\/doi.org\/10.1145\/ 1999 747.1999807 10.1145\/1999747.1999807 Paul Denny, Andrew Luxton-Reilly, Ewan Tempero, and Jacob Hendrickx. 2011. Understanding the Syntax Barrier for Novices (ITiCSE '11). ACM, New York, NY, USA, 208--212. https:\/\/doi.org\/10.1145\/1999747.1999807"},{"key":"e_1_3_2_1_18_1","first-page":"3312978","article-title":"The Impact of ?Cosmetic","volume":"1145","author":"Dong Tao","year":"2019","unstructured":"Tao Dong and Kandarp Khandwala . 2019 . The Impact of ?Cosmetic \" Changes on the Usability of Error Messages (CHI EA '19). ACM, New York, NY, USA, Article Paper LBW0273, 6 pages. https:\/\/doi.org\/10. 1145\/3290607 . 3312978 10.1145\/3290607.3312978 Tao Dong and Kandarp Khandwala. 2019. The Impact of ?Cosmetic\" Changes on the Usability of Error Messages (CHI EA '19). ACM, New York, NY, USA, Article Paper LBW0273, 6 pages. https:\/\/doi.org\/10.1145\/3290607.3312978","journal-title":"ACM, New York, NY, USA, Article Paper LBW0273, 6 pages. https:\/\/doi.org\/10."},{"key":"e_1_3_2_1_19_1","volume-title":"A Mixed-Methods Study of Novice Programmer Interaction with Python Error Messages (WCCCE '19). ACM","author":"D'souza Rachel","unstructured":"Rachel D'souza , Mahima Bhayana , Marzieh Ahmadzadeh , and Brian Harrington . 2019. A Mixed-Methods Study of Novice Programmer Interaction with Python Error Messages (WCCCE '19). ACM , New York, NY, USA , Article Article 15, 2 pages. https:\/\/doi.org\/10.1145\/3314994.3325090 10.1145\/3314994.3325090 Rachel D'souza, Mahima Bhayana, Marzieh Ahmadzadeh, and Brian Harrington. 2019. A Mixed-Methods Study of Novice Programmer Interaction with Python Error Messages (WCCCE '19). ACM, New York, NY, USA, Article Article 15, 2 pages. https:\/\/doi.org\/10.1145\/3314994.3325090"},{"key":"#cr-split#-e_1_3_2_1_20_1.1","doi-asserted-by":"crossref","unstructured":"U. Engelmann and H. P. Meinzer. 1985. Rules for the Design of End User Languages. In Medical Informatics Europe 85 F. H. Roger P. Gr\u00f6 nroos R. Tervo-Pellikka and R. O'Moore (Eds.). Springer Berlin Heidelberg Helsinki Finland 240--245. https:\/\/doi.org\/10.1007\/978--3--642--93295--3_48 10.1007\/978--3--642--93295--3_48","DOI":"10.1007\/978-3-642-93295-3_48"},{"key":"#cr-split#-e_1_3_2_1_20_1.2","doi-asserted-by":"crossref","unstructured":"U. Engelmann and H. P. Meinzer. 1985. Rules for the Design of End User Languages. In Medical Informatics Europe 85 F. H. Roger P. Gr\u00f6 nroos R. Tervo-Pellikka and R. O'Moore (Eds.). Springer Berlin Heidelberg Helsinki Finland 240--245. https:\/\/doi.org\/10.1007\/978--3--642--93295--3_48","DOI":"10.1007\/978-3-642-93295-3_48"},{"key":"e_1_3_2_1_21_1","volume-title":"Examining the Effects of Enhanced Compilers on Student Productivity. Masters Thesis","author":"Harker Devon","unstructured":"Devon Harker . 2017. Examining the Effects of Enhanced Compilers on Student Productivity. Masters Thesis . University of Northern British Columbia . https:\/\/unbc.arcabc.ca\/islandora\/object\/unbc%3A58897 Devon Harker. 2017. Examining the Effects of Enhanced Compilers on Student Productivity. Masters Thesis. University of Northern British Columbia. https:\/\/unbc.arcabc.ca\/islandora\/object\/unbc%3A58897"},{"key":"e_1_3_2_1_22_1","volume-title":"Klemmer","author":"Hartmann Bj\u00f6rn","year":"2010","unstructured":"Bj\u00f6rn Hartmann , Daniel MacDougall , Joel Brandt , and Scott R . Klemmer . 2010 . What Would Other Programmers Do: Suggesting Solutions to Error Messages (CHI '10). ACM, New York, NY, USA , 1019--1028. https:\/\/doi.org\/10.1145\/1753326.1753478 10.1145\/1753326.1753478 Bj\u00f6rn Hartmann, Daniel MacDougall, Joel Brandt, and Scott R. Klemmer. 2010. What Would Other Programmers Do: Suggesting Solutions to Error Messages (CHI '10). ACM, New York, NY, USA, 1019--1028. https:\/\/doi.org\/10.1145\/1753326.1753478"},{"key":"e_1_3_2_1_23_1","volume-title":"Developing a Teaching Compiler for Students Learning the C Programming Language. Bachelor of Science Dissertation","author":"Hill Richard","unstructured":"Richard Hill . 2008. Developing a Teaching Compiler for Students Learning the C Programming Language. Bachelor of Science Dissertation . University of Bath. Richard Hill. 2008. Developing a Teaching Compiler for Students Learning the C Programming Language. Bachelor of Science Dissertation. University of Bath."},{"key":"e_1_3_2_1_24_1","volume-title":"Compiler Construction: An Advanced Course","author":"Horning James J","unstructured":"James J Horning . 1976. What the Compiler Should Tell the User . In Compiler Construction: An Advanced Course , G Goos and J Hartmanis (Eds.). Springer-Verlag , Berlin- Heidelberg , 525--548. James J Horning. 1976. What the Compiler Should Tell the User. In Compiler Construction: An Advanced Course, G Goos and J Hartmanis (Eds.). Springer-Verlag, Berlin-Heidelberg, 525--548."},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.1145\/3105759"},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE.2013.6606613"},{"key":"e_1_3_2_1_27_1","volume-title":"Becker","author":"Karvelas Ioannis","year":"2020","unstructured":"Ioannis Karvelas , Annie Li , and Brett A . Becker . 2020 . The Effects of Compilation Mechanisms and Error Message Presentation on Novice Programmer Behavior (SIGCSE '20). Association for Computing Machinery , New York, NY, USA, 759--765. https:\/\/doi.org\/10.1145\/3328778.3366882 10.1145\/3328778.3366882 Ioannis Karvelas, Annie Li, and Brett A. Becker. 2020. The Effects of Compilation Mechanisms and Error Message Presentation on Novice Programmer Behavior (SIGCSE '20). Association for Computing Machinery, New York, NY, USA, 759--765. https:\/\/doi.org\/10.1145\/3328778.3366882"},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.1145\/3287324.3287381"},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"publisher","DOI":"10.1076\/csed.13.4.249.17496"},{"key":"e_1_3_2_1_31_1","unstructured":"Emily Wilska. 2004. Non-Fatal Errors: Creating Usable Effective Error Messages. http:\/\/www.writersua.com\/articles\/message\/index.html.  Emily Wilska. 2004. Non-Fatal Errors: Creating Usable Effective Error Messages. http:\/\/www.writersua.com\/articles\/message\/index.html."}],"event":{"name":"ITiCSE '20: Innovation and Technology in Computer Science Education","location":"Trondheim Norway","acronym":"ITiCSE '20","sponsor":["SIGCSE ACM Special Interest Group on Computer Science Education"]},"container-title":["Proceedings of the 2020 ACM Conference on Innovation and Technology in Computer Science Education"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3341525.3387384","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3341525.3387384","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T23:12:58Z","timestamp":1750201978000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3341525.3387384"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2020,6,15]]},"references-count":30,"alternative-id":["10.1145\/3341525.3387384","10.1145\/3341525"],"URL":"https:\/\/doi.org\/10.1145\/3341525.3387384","relation":{},"subject":[],"published":{"date-parts":[[2020,6,15]]},"assertion":[{"value":"2020-06-15","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}