{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,5,6]],"date-time":"2026-05-06T16:22:51Z","timestamp":1778084571472,"version":"3.51.4"},"publisher-location":"New York, NY, USA","reference-count":93,"publisher":"ACM","license":[{"start":{"date-parts":[[2024,8,12]],"date-time":"2024-08-12T00:00:00Z","timestamp":1723420800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"funder":[{"name":"Research Council of Finland","award":["356114"],"award-info":[{"award-number":["356114"]}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2024,8,12]]},"DOI":"10.1145\/3632620.3671103","type":"proceedings-article","created":{"date-parts":[[2024,8,6]],"date-time":"2024-08-06T12:47:29Z","timestamp":1722948449000},"page":"95-113","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":39,"title":["Evaluating Contextually Personalized Programming Exercises Created with Generative AI"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0009-0002-0962-9369","authenticated-orcid":false,"given":"Evanfiya","family":"Logacheva","sequence":"first","affiliation":[{"name":"Aalto University, Finland"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-6502-209X","authenticated-orcid":false,"given":"Arto","family":"Hellas","sequence":"additional","affiliation":[{"name":"Aalto University, Finland"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-2807-6042","authenticated-orcid":false,"given":"James","family":"Prather","sequence":"additional","affiliation":[{"name":"Abilene Christian University, United States of America"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-7277-9282","authenticated-orcid":false,"given":"Sami","family":"Sarsa","sequence":"additional","affiliation":[{"name":"University of Jyv\u00e4skyl\u00e4, Finland"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-6829-9449","authenticated-orcid":false,"given":"Juho","family":"Leinonen","sequence":"additional","affiliation":[{"name":"Aalto University, Finland"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2024,8,12]]},"reference":[{"key":"e_1_3_2_1_1_1","volume-title":"ICT Education: 47th Annual Conference of the","author":"Ade-Ibijola Abejide","year":"2018","unstructured":"Abejide Ade-Ibijola. 2019. Syntactic Generation of Practice Novice Programs in Python. In ICT Education: 47th Annual Conference of the Southern African Computer Lecturers\u2019 Association, SACLA 2018, Gordon\u2019s Bay, South Africa, June 18\u201320, 2018, Revised Selected Papers 47. Springer, 158\u2013172."},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.18260\/1-2--31231"},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1145\/3545945.3569759"},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10648-021-09615-8"},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1037\/edu0000250"},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"publisher","DOI":"10.1145\/3024906.3024912"},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"publisher","DOI":"10.1145\/3581754.3584111"},{"key":"e_1_3_2_1_8_1","volume-title":"Codet: Code generation with generated tests. arXiv preprint arXiv:2207.10397","author":"Chen Bei","year":"2022","unstructured":"Bei Chen, Fengji Zhang, Anh Nguyen, Daoguang Zan, Zeqi Lin, Jian-Guang Lou, and Weizhu Chen. 2022. Codet: Code generation with generated tests. arXiv preprint arXiv:2207.10397 (2022)."},{"key":"e_1_3_2_1_9_1","unstructured":"Mark Chen Jerry Tworek Heewoo Jun Qiming Yuan Henrique Ponde Jared Kaplan Harrison Edwards Yura Burda Nicholas Joseph Greg Brockman Alex Ray Raul Puri Gretchen Krueger Michael Petrov Heidy Khlaaf Girish Sastry Pamela Mishkin Brooke Chan Scott Gray Nick Ryder Mikhail Pavlov Alethea Power Lukasz Kaiser Mohammad Bavarian Clemens Winter Philippe Tillet Felipe\u00a0Petroski Such David\u00a0W. Cummings Matthias Plappert Fotios Chantzis Elizabeth Barnes Ariel Herbert-Voss William\u00a0H. Guss Alex Nichol Igor Babuschkin Suchir Balaji Shantanu Jain Andrew Carr Jan Leike Joshua Achiam Vedant Misra Evan Morikawa Alec Radford Matthew\u00a0M. Knight Miles Brundage Mira Murati Katie Mayer Peter Welinder Bob McGrew Dario Amodei Sam McCandlish Ilya Sutskever and Wojciech Zaremba. 2021. Evaluating Large Language Models Trained on Code. ArXiv abs\/2107.03374 (2021). https:\/\/api.semanticscholar.org\/CorpusID:235755472"},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.1017\/edp.2015.11"},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.1080\/08993408.2021.1920816"},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.1145\/3141880.3141898"},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1145\/3626252.3630863"},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.cedpsych.2020.101859"},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.1145\/3372782.3406259"},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.1007\/s11135-014-0003-1"},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.1016\/0895-4356(90)90158-L"},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.5210\/fm.v28i11.13346"},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","DOI":"10.1145\/3511861.3511863"},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.1145\/3576123.3576134"},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.1080\/08993408.2022.2079866"},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.stueduc.2019.07.005"},{"key":"e_1_3_2_1_23_1","volume-title":"Effects of experimenter bias and induced subject expectancy on hypnotic susceptibility.Journal of Personality and Social Psychology 18, 2","author":"Goebel A","year":"1971","unstructured":"Ronald\u00a0A Goebel and Charles\u00a0G Stewart. 1971. Effects of experimenter bias and induced subject expectancy on hypnotic susceptibility.Journal of Personality and Social Psychology 18, 2 (1971), 263."},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.stueduc.2006.10.003"},{"key":"e_1_3_2_1_25_1","unstructured":"Mark Guzdial. 2007. Contextualized Computing Education Increasing Retention by Making Computing Relevant. (2007)."},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.1145\/1869746.1869747"},{"key":"e_1_3_2_1_27_1","doi-asserted-by":"publisher","DOI":"10.1348\/000711006X126600"},{"key":"e_1_3_2_1_28_1","unstructured":"Kilem\u00a0L Gwet. 2014. Handbook of inter-rater reliability: The definitive guide to measuring the extent of agreement among raters. Advanced Analytics LLC."},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.3389\/fpsyg.2020.01134"},{"key":"e_1_3_2_1_30_1","volume-title":"Mediation of interpersonal expectancy effects: 31 meta-analyses.Psychological bulletin 97, 3","author":"Harris J","year":"1985","unstructured":"Monica\u00a0J Harris and Robert Rosenthal. 1985. Mediation of interpersonal expectancy effects: 31 meta-analyses.Psychological bulletin 97, 3 (1985), 363."},{"key":"e_1_3_2_1_31_1","volume-title":"Human-AI Co-Creation of Worked Examples for Programming Classes. arXiv preprint arXiv:2402.16235","author":"Hassany Mohammad","year":"2024","unstructured":"Mohammad Hassany, Peter Brusilovsky, Jiaze Ke, Kamil Akhuseyinoglu, and Arun Balajiee\u00a0Lekshmi Narayanan. 2024. Human-AI Co-Creation of Worked Examples for Programming Classes. arXiv preprint arXiv:2402.16235 (2024)."},{"key":"e_1_3_2_1_32_1","doi-asserted-by":"publisher","DOI":"10.1145\/3568813.3600139"},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"publisher","DOI":"10.1207\/s15326985ep4102_4"},{"key":"e_1_3_2_1_34_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.cedpsych.2015.03.006"},{"key":"e_1_3_2_1_35_1","doi-asserted-by":"publisher","DOI":"10.1080\/00220973.2016.1268085"},{"key":"e_1_3_2_1_36_1","doi-asserted-by":"publisher","DOI":"10.1145\/3626252.3630897"},{"key":"e_1_3_2_1_37_1","doi-asserted-by":"publisher","DOI":"10.1145\/3636243.3636252"},{"key":"e_1_3_2_1_38_1","doi-asserted-by":"publisher","DOI":"10.1109\/FIE58773.2023.10343457"},{"key":"e_1_3_2_1_39_1","doi-asserted-by":"publisher","DOI":"10.1177\/1745691612463704"},{"key":"e_1_3_2_1_40_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-36272-9_79"},{"key":"e_1_3_2_1_41_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-36272-9_74"},{"key":"e_1_3_2_1_42_1","doi-asserted-by":"publisher","DOI":"10.1145\/3478431.3499359"},{"key":"e_1_3_2_1_43_1","doi-asserted-by":"publisher","DOI":"10.1145\/3587102.3588785"},{"key":"e_1_3_2_1_44_1","doi-asserted-by":"publisher","DOI":"10.1145\/3441636.3442302"},{"key":"e_1_3_2_1_45_1","doi-asserted-by":"publisher","DOI":"10.1145\/3545945.3569770"},{"key":"e_1_3_2_1_46_1","doi-asserted-by":"publisher","DOI":"10.1126\/science.abq1158"},{"key":"e_1_3_2_1_47_1","doi-asserted-by":"publisher","DOI":"10.1145\/3631802.3631830"},{"key":"e_1_3_2_1_48_1","doi-asserted-by":"publisher","DOI":"10.1145\/3643864"},{"key":"e_1_3_2_1_49_1","doi-asserted-by":"publisher","DOI":"10.1145\/3017680.3017726"},{"key":"e_1_3_2_1_50_1","doi-asserted-by":"publisher","DOI":"10.1145\/3450329.3476855"},{"key":"e_1_3_2_1_51_1","doi-asserted-by":"publisher","DOI":"10.1145\/3545945.3569785"},{"key":"e_1_3_2_1_52_1","doi-asserted-by":"publisher","DOI":"10.1145\/3631802.3631810"},{"key":"e_1_3_2_1_53_1","doi-asserted-by":"publisher","DOI":"10.1111\/j.2044-8279.1976.tb02980.x"},{"key":"e_1_3_2_1_54_1","doi-asserted-by":"publisher","DOI":"10.1145\/3487054"},{"key":"e_1_3_2_1_55_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.jpdc.2017.01.015"},{"key":"e_1_3_2_1_56_1","volume-title":"Large Language Models (GPT) for automating feedback on programming assignments. arXiv preprint arXiv:2307.00150","author":"Pankiewicz Maciej","year":"2023","unstructured":"Maciej Pankiewicz and Ryan\u00a0S Baker. 2023. Large Language Models (GPT) for automating feedback on programming assignments. arXiv preprint arXiv:2307.00150 (2023)."},{"key":"e_1_3_2_1_57_1","unstructured":"Philipp Peess Annabell Brocker Rene Roepke and Ulrik Schroeder. 2023. A Grammar and Parameterization-Based Generator for Python Programming Exercises. (2023)."},{"key":"e_1_3_2_1_58_1","volume-title":"Ignore Previous Prompt: Attack Techniques For Language Models. In NeurIPS ML Safety Workshop.","author":"Perez F\u00e1bio","year":"2022","unstructured":"F\u00e1bio Perez and Ian Ribeiro. 2022. Ignore Previous Prompt: Attack Techniques For Language Models. In NeurIPS ML Safety Workshop."},{"key":"e_1_3_2_1_59_1","volume-title":"GPT-4, and Human Tutors. arXiv preprint arXiv:2306.17156","author":"Phung Tung","year":"2023","unstructured":"Tung Phung, Victor-Alexandru P\u0103durean, Jos\u00e9 Cambronero, Sumit Gulwani, Tobias Kohn, Rupak Majumdar, Adish Singla, and Gustavo Soares. 2023. Generative AI for Programming Education: Benchmarking ChatGPT, GPT-4, and Human Tutors. arXiv preprint arXiv:2306.17156 (2023)."},{"key":"e_1_3_2_1_60_1","volume-title":"Common method biases in behavioral research: a critical review of the literature and recommended remedies.Journal of applied psychology 88, 5","author":"Podsakoff M","year":"2003","unstructured":"Philip\u00a0M Podsakoff, Scott\u00a0B MacKenzie, Jeong-Yeon Lee, and Nathan\u00a0P Podsakoff. 2003. Common method biases in behavioral research: a critical review of the literature and recommended remedies.Journal of applied psychology 88, 5 (2003), 879."},{"key":"e_1_3_2_1_61_1","volume-title":"International conference on engineering and computer education: Icece, Vol.\u00a05.","author":"Prados Ferran","year":"2005","unstructured":"Ferran Prados, Imma Boada, Josep Soler, and Jordi Poch. 2005. Automatic generation and correction of technical exercises. In International conference on engineering and computer education: Icece, Vol.\u00a05."},{"key":"e_1_3_2_1_62_1","doi-asserted-by":"publisher","DOI":"10.1145\/3623762.3633499"},{"key":"e_1_3_2_1_63_1","article-title":"Github copilot in the classroom: learning to code with AI assistance","volume":"38","author":"Puryear Ben","year":"2022","unstructured":"Ben Puryear and Gina Sprint. 2022. Github copilot in the classroom: learning to code with AI assistance. J. Comput. Sci. Coll. 38, 1 (2022).","journal-title":"J. Comput. Sci. Coll."},{"key":"e_1_3_2_1_64_1","volume-title":"Automatic On-line Generation of Student\u2019s Exercises in Teaching Programming. In Central European Conference on Information and Intelligent Systems. Faculty of Organization and Informatics Varazdin, 87","author":"Rado\u0161evi\u0107 Danijel","year":"2010","unstructured":"Danijel Rado\u0161evi\u0107, Tihomir Orehavocki, and Zlatko Stapi\u0107. 2010. Automatic On-line Generation of Student\u2019s Exercises in Teaching Programming. In Central European Conference on Information and Intelligent Systems. Faculty of Organization and Informatics Varazdin, 87."},{"key":"e_1_3_2_1_65_1","doi-asserted-by":"publisher","DOI":"10.1017\/S0140525X00075506"},{"key":"e_1_3_2_1_66_1","doi-asserted-by":"publisher","DOI":"10.1080\/03055698.2011.598684"},{"key":"e_1_3_2_1_67_1","doi-asserted-by":"publisher","DOI":"10.1016\/S0191-491X(97)86215-3"},{"key":"e_1_3_2_1_68_1","doi-asserted-by":"publisher","DOI":"10.1145\/3105726.3106192"},{"key":"e_1_3_2_1_69_1","doi-asserted-by":"publisher","DOI":"10.1145\/3576882.3617909"},{"key":"e_1_3_2_1_70_1","doi-asserted-by":"publisher","DOI":"10.1145\/3501385.3543957"},{"key":"e_1_3_2_1_71_1","doi-asserted-by":"publisher","DOI":"10.1145\/3502718.3524752"},{"key":"e_1_3_2_1_72_1","doi-asserted-by":"publisher","DOI":"10.1111\/bjep.12653"},{"key":"e_1_3_2_1_73_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10212-022-00656-3"},{"key":"e_1_3_2_1_74_1","doi-asserted-by":"publisher","DOI":"10.1109\/TELE52840.2021.9482762"},{"key":"e_1_3_2_1_75_1","doi-asserted-by":"crossref","unstructured":"Sandro Speth Niklas Mei\u00dfner and Steffen Becker. 2023. Investigating the Use of AI-Generated Exercises for Beginner and Intermediate Programming Courses: A ChatGPT Case Study. In 2023 IEEE 35th International Conference on Software Engineering Education and Training (CSEE&T). IEEE 142\u2013146.","DOI":"10.1109\/CSEET58097.2023.00030"},{"key":"e_1_3_2_1_76_1","volume-title":"Students","author":"Struyven Katrien","year":"2003","unstructured":"Katrien Struyven, Filip Dochy, and Steven Janssens. 2003. Students\u2019 perceptions about new modes of assessment in higher education: A review. Optimising new modes of assessment: In search of qualities and standards (2003), 171\u2013223."},{"key":"e_1_3_2_1_77_1","doi-asserted-by":"publisher","DOI":"10.1145\/3408877.3432399"},{"key":"e_1_3_2_1_78_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10648-020-09570-w"},{"key":"e_1_3_2_1_79_1","doi-asserted-by":"publisher","DOI":"10.1109\/SMC.2018.00255"},{"key":"e_1_3_2_1_80_1","doi-asserted-by":"publisher","unstructured":"Haoye Tian Weiqi Lu Tsz\u00a0On Li Xunzhu Tang Shing-Chi Cheung Jacques Klein and Tegawend\u00e9\u00a0F. Bissyand\u00e9. 2023. Is ChatGPT the Ultimate Programming Assistant \u2013 How far is it?https:\/\/doi.org\/10.48550\/ARXIV.2304.11938","DOI":"10.48550\/ARXIV.2304.11938"},{"key":"e_1_3_2_1_81_1","doi-asserted-by":"publisher","DOI":"10.1080\/03055698.2014.898577"},{"key":"e_1_3_2_1_82_1","doi-asserted-by":"publisher","DOI":"10.1145\/3428029.3428058"},{"key":"e_1_3_2_1_83_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICIS.2015.7166637"},{"key":"e_1_3_2_1_84_1","doi-asserted-by":"publisher","DOI":"10.1109\/CSE-EUC-DCABES.2016.269"},{"key":"e_1_3_2_1_85_1","doi-asserted-by":"publisher","DOI":"10.1080\/00220973.2017.1380590"},{"key":"e_1_3_2_1_86_1","doi-asserted-by":"publisher","DOI":"10.1007\/s40593-018-0168-1"},{"key":"e_1_3_2_1_87_1","doi-asserted-by":"publisher","DOI":"10.1080\/15391523.2020.1747757"},{"key":"e_1_3_2_1_88_1","doi-asserted-by":"publisher","DOI":"10.1145\/3626252.3630764"},{"key":"e_1_3_2_1_89_1","doi-asserted-by":"publisher","DOI":"10.1145\/3545945.3569830"},{"key":"e_1_3_2_1_90_1","doi-asserted-by":"publisher","DOI":"10.1080\/08993408.2019.1565235"},{"key":"e_1_3_2_1_91_1","doi-asserted-by":"publisher","DOI":"10.1145\/3159450.3159608"},{"key":"e_1_3_2_1_92_1","unstructured":"Jun Zhao Zhihao Zhang Luhui Gao Qi Zhang Tao Gui and Xuanjing Huang. 2024. LLaMA Beyond English: An Empirical Study on Language Capability Transfer. arxiv:2401.01055\u00a0[cs.CL]"},{"key":"e_1_3_2_1_93_1","doi-asserted-by":"publisher","DOI":"10.1002\/bs.3830140503"}],"event":{"name":"ICER 2024: ACM Conference on International Computing Education Research","location":"Melbourne VIC Australia","acronym":"ICER 2024","sponsor":["SIGCSE ACM Special Interest Group on Computer Science Education"]},"container-title":["Proceedings of the 2024 ACM Conference on International Computing Education Research - Volume 1"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3632620.3671103","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3632620.3671103","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,8,23]],"date-time":"2025-08-23T00:33:16Z","timestamp":1755909196000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3632620.3671103"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,8,12]]},"references-count":93,"alternative-id":["10.1145\/3632620.3671103","10.1145\/3632620"],"URL":"https:\/\/doi.org\/10.1145\/3632620.3671103","relation":{},"subject":[],"published":{"date-parts":[[2024,8,12]]},"assertion":[{"value":"2024-08-12","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}