{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,2,14]],"date-time":"2026-02-14T05:10:32Z","timestamp":1771045832818,"version":"3.50.1"},"publisher-location":"New York, NY, USA","reference-count":40,"publisher":"ACM","license":[{"start":{"date-parts":[[2024,11,12]],"date-time":"2024-11-12T00:00:00Z","timestamp":1731369600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2024,11,12]]},"DOI":"10.1145\/3699538.3699556","type":"proceedings-article","created":{"date-parts":[[2024,11,13]],"date-time":"2024-11-13T20:49:22Z","timestamp":1731530962000},"page":"1-12","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":17,"title":["One Step at a Time: Combining LLMs and Static Analysis to Generate Next-Step Hints for Programming Tasks"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-2269-8211","authenticated-orcid":false,"given":"Anastasiia","family":"Birillo","sequence":"first","affiliation":[{"name":"JetBrains Research, Belgrade, Serbia"}]},{"ORCID":"https:\/\/orcid.org\/0009-0007-4919-2698","authenticated-orcid":false,"given":"Elizaveta","family":"Artser","sequence":"additional","affiliation":[{"name":"JetBrains Research, Munich, Germany"}]},{"ORCID":"https:\/\/orcid.org\/0009-0004-4686-2412","authenticated-orcid":false,"given":"Anna","family":"Potriasaeva","sequence":"additional","affiliation":[{"name":"JetBrains Research, Belgrade, Serbia"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-2744-8345","authenticated-orcid":false,"given":"Ilya","family":"Vlasov","sequence":"additional","affiliation":[{"name":"JetBrains Research, Belgrade, Serbia"}]},{"ORCID":"https:\/\/orcid.org\/0009-0003-3269-3622","authenticated-orcid":false,"given":"Katsiaryna","family":"Dzialets","sequence":"additional","affiliation":[{"name":"JetBrains, Munich, Germany"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-7009-635X","authenticated-orcid":false,"given":"Yaroslav","family":"Golubev","sequence":"additional","affiliation":[{"name":"JetBrains Research, Belgrade, Serbia"}]},{"ORCID":"https:\/\/orcid.org\/0009-0006-1887-5317","authenticated-orcid":false,"given":"Igor","family":"Gerasimov","sequence":"additional","affiliation":[{"name":"JetBrains, Berlin, Germany"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-5778-7519","authenticated-orcid":false,"given":"Hieke","family":"Keuning","sequence":"additional","affiliation":[{"name":"Utrecht University, Utrecht, Netherlands"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-9022-3563","authenticated-orcid":false,"given":"Timofey","family":"Bryksin","sequence":"additional","affiliation":[{"name":"JetBrains Research, Limassol, Cyprus"}]}],"member":"320","published-online":{"date-parts":[[2024,11,13]]},"reference":[{"key":"e_1_3_3_1_2_2","volume-title":"CLion","year":"2024","unstructured":"2024. CLion. Retrieved September 26, 2024 from https:\/\/www.jetbrains.com\/clion\/"},{"key":"e_1_3_3_1_3_2","volume-title":"Figma","year":"2024","unstructured":"2024. Figma. Retrieved September 26, 2024 from https:\/\/www.figma.com\/"},{"key":"e_1_3_3_1_4_2","volume-title":"IntelliJ IDEA","year":"2024","unstructured":"2024. IntelliJ IDEA. Retrieved September 26, 2024 from https:\/\/www.jetbrains.com\/idea\/"},{"key":"e_1_3_3_1_5_2","volume-title":"JetBrains Academy Marketplace","year":"2024","unstructured":"2024. JetBrains Academy Marketplace. Retrieved September 26, 2024 from https:\/\/plugins.jetbrains.com\/education"},{"key":"e_1_3_3_1_6_2","volume-title":"JetBrains Academy Plugin","year":"2024","unstructured":"2024. JetBrains Academy Plugin. Retrieved September 26, 2024 from https:\/\/plugins.jetbrains.com\/plugin\/10081-jetbrains-academy"},{"key":"e_1_3_3_1_7_2","volume-title":"Kotlin Onboarding: Introduction","year":"2024","unstructured":"2024. Kotlin Onboarding: Introduction. Retrieved September 26, 2024 from https:\/\/plugins.jetbrains.com\/plugin\/21067-kotlin-onboarding-introduction"},{"key":"e_1_3_3_1_8_2","volume-title":"PyCharm","year":"2024","unstructured":"2024. PyCharm. Retrieved September 26, 2024 from https:\/\/www.jetbrains.com\/pycharm\/"},{"key":"e_1_3_3_1_9_2","volume-title":"Visual Studio Code","year":"2024","unstructured":"2024. Visual Studio Code. Retrieved September 26, 2024 from https:\/\/code.visualstudio.com\/"},{"key":"e_1_3_3_1_10_2","unstructured":"Kehinde Aruleba Ismaila\u00a0Temitayo Sanusi George Obaido and Blessing Ogbuokiri. 2023. Integrating ChatGPT in a Computer Science Course: Students Perceptions and Suggestions. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2402.01640 (2023)."},{"key":"e_1_3_3_1_11_2","volume-title":"Supplementary materials","author":"Birillo Anastasiia","year":"2024","unstructured":"Anastasiia Birillo, Elizaveta Artser, Anna Potriasaeva, Ilya Vlasov, Katsiaryna Dzialets, Yaroslav Golubev, Igor Gerasimov, Hieke Keuning, and Timofey Bryksin. 2024. Supplementary materials. Retrieved September 26, 2024 from https:\/\/zenodo.org\/records\/12584502"},{"key":"e_1_3_3_1_12_2","doi-asserted-by":"publisher","DOI":"10.1145\/3643796.3648454"},{"key":"e_1_3_3_1_13_2","doi-asserted-by":"crossref","unstructured":"Romana\u00a0Emilia Cramarenco Monica\u00a0Ioana Burc\u0103-Voicu and Dan-Cristian Dabija. 2023. Student Perceptions of Online Education and Digital Technologies During the COVID-19 Pandemic: A Systematic Review. Electronics 12 2 (2023) 319.","DOI":"10.3390\/electronics12020319"},{"key":"e_1_3_3_1_14_2","doi-asserted-by":"publisher","DOI":"10.1145\/3291279.3339405"},{"key":"e_1_3_3_1_15_2","doi-asserted-by":"crossref","unstructured":"Galina Deeva Daria Bogdanova Estefan\u00eda Serral Monique Snoeck and Jochen De\u00a0Weerdt. 2021. A Review of Automated Feedback Systems for Learners: Classification Framework Challenges and Opportunities. Computers & Education 162 (2021) 104094.","DOI":"10.1016\/j.compedu.2020.104094"},{"key":"e_1_3_3_1_16_2","doi-asserted-by":"crossref","unstructured":"Alex Gerdes Bastiaan Heeren Johan Jeuring and L\u00a0Thomas Van\u00a0Binsbergen. 2017. Ask-Elle: An Adaptable Programming Tutor for Haskell Giving Automated Feedback. International Journal of Artificial Intelligence in Education 27 (2017) 65\u2013100.","DOI":"10.1007\/s40593-015-0080-x"},{"key":"e_1_3_3_1_17_2","doi-asserted-by":"crossref","unstructured":"Irwanto Irwanto Dwi Wahyudiati Anip\u00a0Dwi Saputro and Isna\u00a0Rezkia Lukman. 2023. Massive Open Online Courses (MOOCs) in Higher Education: A Bibliometric Analysis (2012-2022). IJIET: International Journal of Information and Education Technology 13 2 (2023) 223\u2013231.","DOI":"10.18178\/ijiet.2023.13.2.1799"},{"key":"e_1_3_3_1_18_2","doi-asserted-by":"publisher","DOI":"10.1145\/3613904.3642773"},{"key":"e_1_3_3_1_19_2","doi-asserted-by":"publisher","DOI":"10.1145\/3059009.3059061"},{"key":"e_1_3_3_1_20_2","doi-asserted-by":"crossref","unstructured":"Hieke Keuning Johan Jeuring and Bastiaan Heeren. 2018. A Systematic Literature Review of Automated Feedback Generation for Programming Exercises. ACM Transactions on Computing Education (TOCE) 19 1 (2018) 1\u201343.","DOI":"10.1145\/3231711"},{"key":"e_1_3_3_1_21_2","doi-asserted-by":"publisher","DOI":"10.1109\/FIE58773.2023.10343457"},{"key":"e_1_3_3_1_22_2","doi-asserted-by":"crossref","unstructured":"Nils Knoth Antonia Tolzin Andreas Janson and Jan\u00a0Marco Leimeister. 2024. AI Literacy and Its Implications for Prompt Engineering Strategies. Computers and Education: Artificial Intelligence 6 (2024) 100225.","DOI":"10.1016\/j.caeai.2024.100225"},{"key":"e_1_3_3_1_23_2","unstructured":"Patrick Lewis Ethan Perez Aleksandra Piktus Fabio Petroni Vladimir Karpukhin Naman Goyal Heinrich K\u00fcttler Mike Lewis Wen-tau Yih Tim Rockt\u00e4schel et\u00a0al. 2020. Retrieval-Augmented Generation for Knowledge-Intensive NLP tasks. Advances in Neural Information Processing Systems 33 (2020) 9459\u20139474."},{"key":"e_1_3_3_1_24_2","doi-asserted-by":"publisher","DOI":"10.1145\/3631802.3631830"},{"key":"e_1_3_3_1_25_2","unstructured":"Fang Liu Yang Liu Lin Shi Houkun Huang Ruifeng Wang Zhen Yang and Li Zhang. 2024. Exploring and Evaluating Hallucinations in LLM-powered Code Generation. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2404.00971 (2024)."},{"key":"e_1_3_3_1_26_2","doi-asserted-by":"publisher","DOI":"10.1145\/3626252.3630938"},{"key":"e_1_3_3_1_27_2","doi-asserted-by":"crossref","unstructured":"Zhijie Liu Yutian Tang Xiapu Luo Yuming Zhou and Liang\u00a0Feng Zhang. 2024. No Need to Lift a Finger Anymore? Assessing the Quality of Code Generation by ChatGPT. IEEE Transactions on Software Engineering 50 6 (2024) 1548\u20131584.","DOI":"10.1109\/TSE.2024.3392499"},{"key":"e_1_3_3_1_28_2","doi-asserted-by":"publisher","DOI":"10.1145\/3408877.3432534"},{"key":"e_1_3_3_1_29_2","doi-asserted-by":"publisher","DOI":"10.1145\/3304221.3319756"},{"key":"e_1_3_3_1_30_2","doi-asserted-by":"crossref","unstructured":"Lauren\u00a0E Margulieux Briana\u00a0B Morrison Baker Franke and Harivololona Ramilison. 2020. Effect of Implementing Subgoals in Code.org\u2019s Intro to Programming Unit in Computer Science Principles. ACM Transactions on Computing Education (TOCE) 20 4 (2020) 1\u201324.","DOI":"10.1145\/3415594"},{"key":"e_1_3_3_1_31_2","first-page":"387","volume-title":"International Conference on Data Intelligence and Cognitive Informatics","author":"Marvin Ggaliwango","year":"2023","unstructured":"Ggaliwango Marvin, Nakayiza Hellen, Daudi Jjingo, and Joyce Nakatumba-Nabende. 2023. Prompt Engineering in Large Language Models. In International Conference on Data Intelligence and Cognitive Informatics. 387\u2013402."},{"key":"e_1_3_3_1_32_2","doi-asserted-by":"crossref","unstructured":"Timothy\u00a0R. McIntosh Tong Liu Teo Susnjak Paul Watters Alex Ng and Malka\u00a0N. Halgamuge. 2024. A Culturally Sensitive Test to Evaluate Nuanced GPT Hallucination. IEEE Transactions on Artificial Intelligence 5 6 (2024) 2739\u20132751.","DOI":"10.1109\/TAI.2023.3332837"},{"key":"e_1_3_3_1_33_2","doi-asserted-by":"publisher","DOI":"10.1145\/3623762.3633499"},{"key":"e_1_3_3_1_34_2","doi-asserted-by":"publisher","DOI":"10.1145\/3017680.3017762"},{"key":"e_1_3_3_1_35_2","first-page":"50","volume-title":"The First Workshop on AI-supported Education for Computer Science (AIEDCS 2013)","author":"Rivers Kelly","year":"2013","unstructured":"Kelly Rivers and Kenneth\u00a0R Koedinger. 2013. Automatic Generation of Programming Feedback: A Data-Driven Approach. In The First Workshop on AI-supported Education for Computer Science (AIEDCS 2013), Vol.\u00a050. 50\u201359."},{"key":"e_1_3_3_1_36_2","doi-asserted-by":"publisher","DOI":"10.1145\/3636243.3636259"},{"key":"e_1_3_3_1_37_2","unstructured":"Florian Tambon Arghavan\u00a0Moradi Dakhel Amin Nikanjam Foutse Khomh Michel\u00a0C Desmarais and Giuliano Antoniol. 2024. Bugs in Large Language Models Generated Code. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2403.08937 (2024)."},{"key":"e_1_3_3_1_38_2","doi-asserted-by":"publisher","DOI":"10.1109\/IPCC.2007.4464043"},{"key":"e_1_3_3_1_39_2","doi-asserted-by":"crossref","unstructured":"Albert Webson and Ellie Pavlick. 2021. Do Prompt-based Models Really Understand the Meaning of Their Prompts? arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2109.01247 (2021).","DOI":"10.18653\/v1\/2022.naacl-main.167"},{"key":"e_1_3_3_1_40_2","unstructured":"Jason Wei Xuezhi Wang Dale Schuurmans Maarten Bosma Fei Xia Ed Chi Quoc\u00a0V Le Denny Zhou et\u00a0al. 2022. Chain-of-Thought Prompting Elicits Reasoning in Large Language Models. Advances in neural information processing systems 35 (2022) 24824\u201324837."},{"key":"e_1_3_3_1_41_2","doi-asserted-by":"publisher","DOI":"10.1145\/3613905.3650937"}],"event":{"name":"Koli Calling '24: 24th Koli Calling International Conference on Computing Education Research","location":"Koli Finland","acronym":"Koli Calling '24"},"container-title":["Proceedings of the 24th Koli Calling International Conference on Computing Education Research"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3699538.3699556","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3699538.3699556","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,19]],"date-time":"2025-06-19T01:09:52Z","timestamp":1750295392000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3699538.3699556"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,11,12]]},"references-count":40,"alternative-id":["10.1145\/3699538.3699556","10.1145\/3699538"],"URL":"https:\/\/doi.org\/10.1145\/3699538.3699556","relation":{},"subject":[],"published":{"date-parts":[[2024,11,12]]},"assertion":[{"value":"2024-11-13","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}