{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,2,14]],"date-time":"2026-02-14T05:09:44Z","timestamp":1771045784579,"version":"3.50.1"},"publisher-location":"New York, NY, USA","reference-count":36,"publisher":"ACM","funder":[{"DOI":"10.13039\/501100006374","name":"European Research Council","doi-asserted-by":"publisher","award":["ERC, TOPS, 101039090"],"award-info":[{"award-number":["ERC, TOPS, 101039090"]}],"id":[{"id":"10.13039\/501100006374","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2025,7,17]]},"DOI":"10.1145\/3698205.3733932","type":"proceedings-article","created":{"date-parts":[[2025,7,17]],"date-time":"2025-07-17T14:35:08Z","timestamp":1752762908000},"page":"241-245","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":1,"title":["Bridging Gaps Between Student and Expert Evaluations of AI-Generated Programming Hints"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0009-0005-2683-434X","authenticated-orcid":false,"given":"Tung","family":"Phung","sequence":"first","affiliation":[{"name":"MPI-SWS, Saarbr\u00fccken, Germany"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0009-0005-1528-0425","authenticated-orcid":false,"given":"Mengyan","family":"Wu","sequence":"additional","affiliation":[{"name":"University of Michigan, Ann Arbor, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-8955-8905","authenticated-orcid":false,"given":"Heeryung","family":"Choi","sequence":"additional","affiliation":[{"name":"University of Minnesota, Twin Cities, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-8061-9000","authenticated-orcid":false,"given":"Gustavo","family":"Soares","sequence":"additional","affiliation":[{"name":"Microsoft, Redmond, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-9226-9634","authenticated-orcid":false,"given":"Sumit","family":"Gulwani","sequence":"additional","affiliation":[{"name":"Microsoft, Redmond, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-9922-0668","authenticated-orcid":false,"given":"Adish","family":"Singla","sequence":"additional","affiliation":[{"name":"MPI-SWS, Saarbr\u00fccken, Germany"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-0875-0204","authenticated-orcid":false,"given":"Christopher","family":"Brooks","sequence":"additional","affiliation":[{"name":"University of Michigan, Ann Arbor, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2025,7,17]]},"reference":[{"key":"e_1_3_2_2_1_1","volume-title":"Goodman","author":"Andukuri Chinmaya","year":"2024","unstructured":"Chinmaya Andukuri, Jan-Philipp Fr\u00e4nken, Tobias Gerstenberg, and Noah D. Goodman. 2024. STaR-GATE: Teaching Language Models to Ask Clarifying Questions. CoRR, Vol. abs\/2403.19154 (2024)."},{"key":"e_1_3_2_2_2_1","doi-asserted-by":"publisher","DOI":"10.1145\/3649217.3653594"},{"key":"e_1_3_2_2_3_1","volume-title":"Samantha Bouwmeester, and JJ Asscher.","author":"Beishuizen JJ","year":"2001","unstructured":"JJ Beishuizen, Els Hof, CM Van Putten, Samantha Bouwmeester, and JJ Asscher. 2001. Studentstextquoteright and Teacherstextquoteright Cognitions about Good Teachers. British Journal of Educational Psychology, Vol. 71 (2001)."},{"key":"e_1_3_2_2_4_1","doi-asserted-by":"publisher","DOI":"10.1145\/3664646.3664778"},{"key":"e_1_3_2_2_5_1","volume-title":"Feedback in Higher and Professional Education","author":"Boud David","unstructured":"David Boud and Elizabeth Molloy. 2012. Feedback in Higher and Professional Education. Routledge."},{"key":"e_1_3_2_2_6_1","volume-title":"Proceedings of the Annual Conference on Neural Information Processing Systems (NeurIPS).","author":"Tom","unstructured":"Tom B. Brown et al. 2020. Language Models are Few-Shot Learners. In Proceedings of the Annual Conference on Neural Information Processing Systems (NeurIPS)."},{"key":"e_1_3_2_2_7_1","volume-title":"Exploring Interaction Patterns for Debugging: Enhancing Conversational Capabilities of AI-assistants. CoRR","author":"Chopra Bhavya","year":"2024","unstructured":"Bhavya Chopra, Yasharth Bajpai, Param Biyani, Gustavo Soares, Arjun Radhakrishna, Chris Parnin, and Sumit Gulwani. 2024. Exploring Interaction Patterns for Debugging: Enhancing Conversational Capabilities of AI-assistants. CoRR, Vol. abs\/2402.06229 (2024)."},{"key":"e_1_3_2_2_8_1","doi-asserted-by":"publisher","DOI":"10.1214\/aoms\/1177729380"},{"key":"e_1_3_2_2_9_1","doi-asserted-by":"publisher","DOI":"10.1080\/02602930903197883"},{"key":"e_1_3_2_2_10_1","volume-title":"Insert-Expansions for Tool-Enabled Conversational Agents. CoRR","author":"G\u00f6ldi Andreas","year":"2023","unstructured":"Andreas G\u00f6ldi and Roman Rietsche. 2023. Insert-Expansions for Tool-Enabled Conversational Agents. CoRR, Vol. abs\/2307.01644 (2023)."},{"key":"e_1_3_2_2_11_1","volume-title":"Leveraging Print Debugging to Improve Code Generation in Large Language Models. CoRR","author":"Hu Xueyu","year":"2024","unstructured":"Xueyu Hu, Kun Kuang, Jiankai Sun, Hongxia Yang, and Fei Wu. 2024. Leveraging Print Debugging to Improve Code Generation in Large Language Models. CoRR, Vol. abs\/2401.05319 (2024)."},{"key":"e_1_3_2_2_12_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.edurev.2007.05.002"},{"key":"e_1_3_2_2_13_1","volume-title":"Enhancing Programming Error Messages in Real Time with Generative AI. In Extended Abstracts of the CHI Conference on Human Factors in Computing Systems (CHI).","author":"Bailey","unstructured":"Bailey Kimmel and et al. 2024. Enhancing Programming Error Messages in Real Time with Generative AI. In Extended Abstracts of the CHI Conference on Human Factors in Computing Systems (CHI)."},{"key":"e_1_3_2_2_14_1","volume-title":"Urban Legends in Education. Educational Psychologist","volume":"48","author":"Kirschner Paul A","year":"2013","unstructured":"Paul A Kirschner and Jeroen JG van Merri\u00ebnboer. 2013. Do Learners Really Know Best? Urban Legends in Education. Educational Psychologist, Vol. 48 (2013)."},{"key":"e_1_3_2_2_15_1","volume-title":"Hints-in-browser: Benchmarking Language Models for Programming Feedback Generation. In Annual Conference on Neural Information Processing Systems (NeurIPS).","author":"Kotalwar Nachiket","year":"2024","unstructured":"Nachiket Kotalwar, Alkis Gotovos, and Adish Singla. 2024. Hints-in-browser: Benchmarking Language Models for Programming Feedback Generation. In Annual Conference on Neural Information Processing Systems (NeurIPS)."},{"key":"e_1_3_2_2_16_1","volume-title":"Who is the Expert? Construct and Criteria Validity of Student and Teacher Ratings of Instruction. Learning Environments Research","author":"Kunter Mareike","year":"2007","unstructured":"Mareike Kunter and J'' urgen Baumert. 2007. Who is the Expert? Construct and Criteria Validity of Student and Teacher Ratings of Instruction. Learning Environments Research (2007)."},{"key":"e_1_3_2_2_17_1","doi-asserted-by":"publisher","DOI":"10.1145\/3631802.3631830"},{"key":"e_1_3_2_2_18_1","unstructured":"Ying-Chun Lin and et al. 2024. Interpretable User Satisfaction Estimation for Conversational Systems with Large Language Models. CoRR Vol. abs\/2403.12388 (2024)."},{"key":"e_1_3_2_2_19_1","doi-asserted-by":"publisher","DOI":"10.1111\/jcal.13107"},{"key":"e_1_3_2_2_20_1","unstructured":"OpenAI. 2023. GPT-4 Technical Report. CoRR Vol. abs\/2303.08774 (2023)."},{"key":"e_1_3_2_2_21_1","doi-asserted-by":"publisher","DOI":"10.22318\/icls2024.230928"},{"key":"e_1_3_2_2_22_1","volume-title":"Large Language Models (GPT) for Automating Feedback on Programming Assignments. CoRR","author":"Pankiewicz Maciej","year":"2023","unstructured":"Maciej Pankiewicz and Ryan Shaun Baker. 2023. Large Language Models (GPT) for Automating Feedback on Programming Assignments. CoRR, Vol. abs\/2307.00150 (2023)."},{"key":"e_1_3_2_2_23_1","volume-title":"Proceedings of the International Conference on Educational Data Mining (EDM).","author":"Phung Tung","year":"2023","unstructured":"Tung Phung, Jos\u00e9 Cambronero, Sumit Gulwani, Tobias Kohn, Rupak Majumdar, Adish Singla, and Gustavo Soares. 2023. Generating High-Precision Feedback for Programming Syntax Errors using Large Language Models. In Proceedings of the International Conference on Educational Data Mining (EDM)."},{"key":"e_1_3_2_2_24_1","volume-title":"Proceedings of the Conference on International Computing Education Research (ICER) -","volume":"2","author":"Tung","unstructured":"Tung Phung and et al. 2023. Generative AI for Programming Education: Benchmarking ChatGPT, GPT-4, and Human Tutors. In Proceedings of the Conference on International Computing Education Research (ICER) - Volume 2."},{"key":"e_1_3_2_2_25_1","volume-title":"Proceedings of the International Learning Analytics and Knowledge Conference (LAK).","author":"Tung","unstructured":"Tung Phung and et al. 2024. Automating Human Tutor-Style Programming Feedback: Leveraging GPT-4 Tutor Model for Hint Generation and GPT-3.5 Student Model for Hint Validation. In Proceedings of the International Learning Analytics and Knowledge Conference (LAK)."},{"key":"e_1_3_2_2_26_1","volume-title":"Assessment& Evaluation in Higher Education","volume":"35","author":"Malini Reddy Y","year":"2010","unstructured":"Y Malini Reddy and Heidi Andrade. 2010. A Review of Rubric Use in Higher Education. Assessment& Evaluation in Higher Education, Vol. 35 (2010)."},{"key":"e_1_3_2_2_27_1","volume-title":"Yasmina Hobeika, and Francesca Bugiotti.","author":"Remadi Adel","year":"2024","unstructured":"Adel Remadi, Karim El Hage, Yasmina Hobeika, and Francesca Bugiotti. 2024. To Prompt or Not to Prompt: Navigating the Use of Large Language Models for Integrating and Modeling Heterogeneous Data. Data& Knowledge Engineering, Vol. 152 (2024)."},{"key":"e_1_3_2_2_28_1","doi-asserted-by":"publisher","DOI":"10.1145\/3636243.3636259"},{"key":"e_1_3_2_2_29_1","doi-asserted-by":"publisher","DOI":"10.1145\/3568812.3603474"},{"key":"e_1_3_2_2_30_1","article-title":"Teaching Critical Thinking and Problem Solving Skills","volume":"50","author":"Snyder Lisa Gueldenzoph","year":"2008","unstructured":"Lisa Gueldenzoph Snyder and Mark J Snyder. 2008. Teaching Critical Thinking and Problem Solving Skills. The Journal of Research in Business Education, Vol. 50 (2008).","journal-title":"The Journal of Research in Business Education"},{"key":"e_1_3_2_2_31_1","doi-asserted-by":"publisher","DOI":"10.1002\/2211-5463.12841"},{"key":"e_1_3_2_2_32_1","doi-asserted-by":"publisher","DOI":"10.1145\/3626252.3630764"},{"key":"e_1_3_2_2_33_1","volume-title":"Proceedings of the Annual Conference on Neural Information Processing Systems (NeurIPS).","author":"Wei Jason","year":"2022","unstructured":"Jason Wei, Xuezhi Wang, Dale Schuurmans, Maarten Bosma, Brian Ichter, Fei Xia, Ed H. Chi, Quoc V. Le, and Denny Zhou. 2022. Chain-of-Thought Prompting Elicits Reasoning in Large Language Models. In Proceedings of the Annual Conference on Neural Information Processing Systems (NeurIPS)."},{"key":"e_1_3_2_2_34_1","doi-asserted-by":"publisher","DOI":"10.1145\/3626252.3630773"},{"key":"e_1_3_2_2_35_1","volume-title":"NeurIPS'23 Workshop on Generative AI for Education (GAIED)","author":"Zamfirescu-Pereira J.D.","year":"2023","unstructured":"J.D. Zamfirescu-Pereira, Laryn Qi, Bjorn Hartmann, John DeNero, and Narges Norouzi. 2023. Conversational Programming with LLM-Powered Interactive Support in an Introductory Computer Science Course. NeurIPS'23 Workshop on Generative AI for Education (GAIED) (2023)."},{"key":"e_1_3_2_2_36_1","volume-title":"61A-Bot: AI Homework Assistance in CS1 is Fast and Cheap - But is It Helpful? CoRR","author":"Zamfirescu-Pereira J. D.","year":"2024","unstructured":"J. D. Zamfirescu-Pereira, Laryn Qi, Bj\u00f6rn Hartmann, John DeNero, and Narges Norouzi. 2024. 61A-Bot: AI Homework Assistance in CS1 is Fast and Cheap - But is It Helpful? CoRR, Vol. abs\/2406.05600 (2024)."}],"event":{"name":"L@S '25: Twelfth ACM Conference on Learning @ Scale","location":"Palermo Italy","acronym":"L@S '25"},"container-title":["Proceedings of the Twelfth ACM Conference on Learning @ Scale"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3698205.3733932","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,8,22]],"date-time":"2025-08-22T13:15:47Z","timestamp":1755868547000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3698205.3733932"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,7,17]]},"references-count":36,"alternative-id":["10.1145\/3698205.3733932","10.1145\/3698205"],"URL":"https:\/\/doi.org\/10.1145\/3698205.3733932","relation":{},"subject":[],"published":{"date-parts":[[2025,7,17]]},"assertion":[{"value":"2025-07-17","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}