{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,13]],"date-time":"2026-04-13T17:11:06Z","timestamp":1776100266439,"version":"3.50.1"},"publisher-location":"New York, NY, USA","reference-count":19,"publisher":"ACM","license":[{"start":{"date-parts":[[2023,8,7]],"date-time":"2023-08-07T00:00:00Z","timestamp":1691366400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"DOI":"10.13039\/501100000781","name":"European Research Council","doi-asserted-by":"publisher","award":["ERC, TOPS, 101039090"],"award-info":[{"award-number":["ERC, TOPS, 101039090"]}],"id":[{"id":"10.13039\/501100000781","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2023,8,7]]},"DOI":"10.1145\/3568812.3603474","type":"proceedings-article","created":{"date-parts":[[2023,9,13]],"date-time":"2023-09-13T16:12:11Z","timestamp":1694621531000},"page":"14-15","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":17,"title":["Evaluating ChatGPT and GPT-4 for Visual Programming"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0001-9922-0668","authenticated-orcid":false,"given":"Adish","family":"Singla","sequence":"first","affiliation":[{"name":"MPI-SWS, Germany"}]}],"member":"320","published-online":{"date-parts":[[2023,9,13]]},"reference":[{"key":"e_1_3_2_1_1_1","unstructured":"Umair\u00a0Z. Ahmed Maria Christakis Aleksandr Efremov Nigel Fernandez Ahana Ghosh Abhik Roychoudhury and Adish Singla. 2020. Synthesizing Tasks for Block-based Programming. In NeurIPS."},{"key":"e_1_3_2_1_2_1","volume-title":"Sparks of Artificial General Intelligence: Early Experiments with GPT-4. CoRR abs\/2303.12712","author":"Bubeck S\u00e9bastien","year":"2023","unstructured":"S\u00e9bastien Bubeck, Varun Chandrasekaran, Ronen Eldan, Johannes Gehrke, Eric Horvitz, Ece Kamar, Peter Lee, Yin\u00a0Tat Lee, Yuanzhi Li, Scott\u00a0M. Lundberg, Harsha Nori, Hamid Palangi, Marco\u00a0T\u00falio Ribeiro, and Yi Zhang. 2023. Sparks of Artificial General Intelligence: Early Experiments with GPT-4. CoRR abs\/2303.12712 (2023)."},{"key":"e_1_3_2_1_3_1","unstructured":"CodeHS. 2012. Intro to Programming with Karel the Dog. https:\/\/codehs.com\/info\/curriculum\/introkarel."},{"key":"e_1_3_2_1_4_1","unstructured":"Code.org. 2013. Code.org: Learn Computer Science. https:\/\/code.org\/."},{"key":"e_1_3_2_1_5_1","unstructured":"Code.org. 2013. Hour of Code: Classic Maze Challenge. https:\/\/studio.code.org\/s\/hourofcode."},{"key":"e_1_3_2_1_6_1","unstructured":"Aleksandr Efremov Ahana Ghosh and Adish Singla. 2020. Zero-shot Learning of Hint Policy via Reinforcement Learning and Program Synthesis. In EDM."},{"key":"e_1_3_2_1_7_1","volume-title":"2021. Evaluating Large Language Models Trained on Code. CoRR abs\/2107-03374","author":"Mark\u00a0Chen","year":"2021","unstructured":"Mark\u00a0Chen et al.2021. Evaluating Large Language Models Trained on Code. CoRR abs\/2107-03374 (2021)."},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"crossref","unstructured":"James Finnie-Ansley Paul Denny Brett\u00a0A. Becker Andrew Luxton-Reilly and James Prather. 2022. The Robots Are Coming: Exploring the Implications of OpenAI Codex on Introductory Programming. In ACE.","DOI":"10.1145\/3511861.3511863"},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"crossref","unstructured":"Ahana Ghosh Sebastian Tschiatschek Sam Devlin and Adish Singla. 2022. Adaptive Scaffolding in Block-Based Programming via Synthesizing New Tasks as Pop Quizzes. In AIED.","DOI":"10.1007\/978-3-031-11644-5_3"},{"key":"e_1_3_2_1_10_1","unstructured":"Samiha Marwan Yang Shi Ian Menezes Min Chi Tiffany Barnes and Thomas\u00a0W. Price. 2021. Just a Few Expert Constraints Can Help: Humanizing Data-Driven Subgoal Detection for Novice Programming. In EDM."},{"key":"e_1_3_2_1_11_1","unstructured":"OpenAI. 2023. ChatGPT. https:\/\/openai.com\/blog\/chatgpt."},{"key":"e_1_3_2_1_13_1","volume-title":"Karel the Robot: A Gentle Introduction to the Art of Programming","author":"Pattis E","unstructured":"Richard\u00a0E Pattis, Jim Roberts, and Mark Stehlik. 1995. Karel the Robot: A Gentle Introduction to the Art of Programming. John Wiley & Sons, Inc."},{"key":"e_1_3_2_1_14_1","unstructured":"Tung Phung Jos\u00e9 Cambronero Sumit Gulwani Tobias Kohn Rupak Majumdar Adish Singla and Gustavo Soares. 2023. Generating High-Precision Feedback for Programming Syntax Errors using Large Language Models. In EDM."},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"crossref","unstructured":"Chris Piech Mehran Sahami Jonathan Huang and Leonidas\u00a0J. Guibas. 2015. Autonomously Generating Hints by Inferring Problem Solving Policies. In L@S.","DOI":"10.1145\/2724660.2724668"},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"crossref","unstructured":"Thomas\u00a0W. Price Rui Zhi and Tiffany Barnes. 2017. Hint Generation Under Uncertainty: The Effect of Hint Quality on Help-Seeking Behavior. In AIED.","DOI":"10.1007\/978-3-319-61425-0_26"},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.1145\/1592761.1592779"},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"crossref","unstructured":"Sami Sarsa Paul Denny Arto Hellas and Juho Leinonen. 2022. Automatic Generation of Programming Exercises and Code Explanations Using Large Language Models. In ICER.","DOI":"10.1145\/3501385.3543957"},{"key":"e_1_3_2_1_19_1","unstructured":"Adish Singla and Nikitas Theodoropoulos. 2022. From {Solution Synthesis} to {Student Attempt Synthesis} for Block-Based Visual Programming Tasks. In EDM."},{"key":"e_1_3_2_1_20_1","volume-title":"Repairing Bugs in Python Assignments Using Large Language Models. CoRR abs\/2209.14876","author":"Zhang Jialu","year":"2022","unstructured":"Jialu Zhang, Jos\u00e9 Cambronero, Sumit Gulwani, Vu Le, Ruzica Piskac, Gustavo Soares, and Gust Verbruggen. 2022. Repairing Bugs in Python Assignments Using Large Language Models. CoRR abs\/2209.14876 (2022)."}],"event":{"name":"ICER 2023: ACM Conference on International Computing Education Research","location":"Chicago IL USA","acronym":"ICER 2023","sponsor":["SIGCSE ACM Special Interest Group on Computer Science Education"]},"container-title":["Proceedings of the 2023 ACM Conference on International Computing Education Research - Volume 2"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3568812.3603474","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3568812.3603474","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,18]],"date-time":"2025-06-18T21:26:26Z","timestamp":1750281986000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3568812.3603474"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023,8,7]]},"references-count":19,"alternative-id":["10.1145\/3568812.3603474","10.1145\/3568812"],"URL":"https:\/\/doi.org\/10.1145\/3568812.3603474","relation":{},"subject":[],"published":{"date-parts":[[2023,8,7]]},"assertion":[{"value":"2023-09-13","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}