{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,13]],"date-time":"2026-04-13T17:11:03Z","timestamp":1776100263576,"version":"3.50.1"},"publisher-location":"New York, NY, USA","reference-count":43,"publisher":"ACM","license":[{"start":{"date-parts":[[2023,6,29]],"date-time":"2023-06-29T00:00:00Z","timestamp":1687996800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2023,6,29]]},"DOI":"10.1145\/3587102.3588805","type":"proceedings-article","created":{"date-parts":[[2023,6,30]],"date-time":"2023-06-30T22:17:59Z","timestamp":1688163479000},"page":"299-305","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":53,"title":["Evaluating the Performance of Code Generation Models for Solving Parsons Problems With Small Prompt Variations"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0001-5781-1136","authenticated-orcid":false,"given":"Brent","family":"Reeves","sequence":"first","affiliation":[{"name":"Abilene Christian University, Abilene, TX, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-7277-9282","authenticated-orcid":false,"given":"Sami","family":"Sarsa","sequence":"additional","affiliation":[{"name":"Aalto University, Helsinki, Finland"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-2807-6042","authenticated-orcid":false,"given":"James","family":"Prather","sequence":"additional","affiliation":[{"name":"Abilene Christian University, Abilene, TX, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-5150-9806","authenticated-orcid":false,"given":"Paul","family":"Denny","sequence":"additional","affiliation":[{"name":"University of Auckland, Auckland, New Zealand"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-1446-647X","authenticated-orcid":false,"given":"Brett A.","family":"Becker","sequence":"additional","affiliation":[{"name":"University College Dublin, Dublin, Ireland"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-6502-209X","authenticated-orcid":false,"given":"Arto","family":"Hellas","sequence":"additional","affiliation":[{"name":"Aalto University, Espoo, Finland"}]},{"ORCID":"https:\/\/orcid.org\/0009-0000-6655-0564","authenticated-orcid":false,"given":"Bailey","family":"Kimmel","sequence":"additional","affiliation":[{"name":"Abilene Christian University, Abilene, TX, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-3221-7015","authenticated-orcid":false,"given":"Garrett","family":"Powell","sequence":"additional","affiliation":[{"name":"Abilene Christian University, Abilene, TX, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-6829-9449","authenticated-orcid":false,"given":"Juho","family":"Leinonen","sequence":"additional","affiliation":[{"name":"University of Auckland, Auckland, New Zealand"}]}],"member":"320","published-online":{"date-parts":[[2023,6,30]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"publisher","DOI":"10.1145\/3545945.3569759"},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.1145\/3017680.3017704"},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","unstructured":"Arghavan Moradi Dakhel Vahid Majdinasab Amin Nikanjam Foutse Khomh Michel C. Desmarais et al. 2022. GitHub Copilot AI Pair Programmer: Asset or Liability? https:\/\/doi.org\/10.48550\/arXiv.2206.15331 arxiv: cs\/2206.15331","DOI":"10.48550\/arXiv.2206.15331"},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","unstructured":"Paul Denny Viraj Kumar and Nasser Giacaman. 2022a. Conversing with Copilot: Exploring Prompt Engineering for Solving CS1 Problems Using Natural Language. https:\/\/doi.org\/10.48550\/ARXIV.2210.15157","DOI":"10.48550\/ARXIV.2210.15157"},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1145\/1404520.1404532"},{"key":"e_1_3_2_1_6_1","volume-title":"Robosourcing Educational Resources--Leveraging Large Language Models for Learnersourcing. arXiv preprint arXiv:2211.04715","author":"Denny Paul","year":"2022","unstructured":"Paul Denny, Sami Sarsa, Arto Hellas, and Juho Leinonen. 2022b. Robosourcing Educational Resources--Leveraging Large Language Models for Learnersourcing. arXiv preprint arXiv:2211.04715 (2022)."},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"publisher","DOI":"10.1145\/3373165.3373187"},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.1145\/3502718.3524808"},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1145\/3571785.3574127"},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.1145\/3230977.3231000"},{"key":"e_1_3_2_1_11_1","volume-title":"Proc. of the 11th Annual Int. Conf. on Int. Computing Education Research. ACM, 169--178","author":"Ericson Barbara J.","unstructured":"Barbara J. Ericson, Mark J. Guzdial, and Briana B. Morrison. 2015. Analysis of Interactive Features Designed to Enhance Learning in an Ebook. In Proc. of the 11th Annual Int. Conf. on Int. Computing Education Research. ACM, 169--178."},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.1145\/3141880.3141895"},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1145\/3213586.3225235"},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.1145\/3511861.3511863"},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.1145\/3576123.3576134"},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.1145\/2632320.2632346"},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.1145\/3279720.3279746"},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1145\/2960310.2960314"},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","DOI":"10.1145\/3411764.3445292"},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.1145\/3564721.3564736"},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.1145\/2361276.2361300"},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.1145\/3501385.3543977"},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.28945\/1394"},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"publisher","DOI":"10.1145\/2401796.2401798"},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.1145\/3159450.3159576"},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.1145\/3304221.3319735"},{"key":"e_1_3_2_1_27_1","doi-asserted-by":"publisher","DOI":"10.1145\/3545945.3569770"},{"key":"e_1_3_2_1_28_1","first-page":"1","article-title":"Pre-train, prompt, and predict: A systematic survey of prompting methods in natural language processing","volume":"55","author":"Liu Pengfei","year":"2023","unstructured":"Pengfei Liu, Weizhe Yuan, Jinlan Fu, Zhengbao Jiang, Hiroaki Hayashi, et al. 2023. Pre-train, prompt, and predict: A systematic survey of prompting methods in natural language processing. Comput. Surveys, Vol. 55, 9 (2023), 1--35.","journal-title":"Comput. Surveys"},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.1145\/3545945.3569785"},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"crossref","unstructured":"Stephen MacNeil Andrew Tran Juho Leinonen Paul Denny Joanne Kim et al. 2022a. Automatically Generating CS Learning Materials with Large Language Models. arXiv preprint arXiv:2212.05113 (2022).","DOI":"10.1145\/3545947.3569630"},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"publisher","DOI":"10.1145\/3501709.3544280"},{"key":"e_1_3_2_1_32_1","doi-asserted-by":"publisher","DOI":"10.1145\/3446871.3469750"},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"publisher","DOI":"10.1145\/2839509.2844617"},{"key":"e_1_3_2_1_34_1","volume-title":"Proc. of the 8th Australasian Conf. on Computing Education -","volume":"52","author":"Parsons Dale","year":"2006","unstructured":"Dale Parsons and Patricia Haden. 2006. Parson's Programming Puzzles: A Fun and Effective Learning Tool for First Programming Courses. In Proc. of the 8th Australasian Conf. on Computing Education - Volume 52 (ACE '06). Australian Computer Society, Inc., AUS, 157--163."},{"key":"e_1_3_2_1_35_1","volume-title":"Proc. of the 2022 World Conf. on Computers in Education (WCCE '22)","author":"Prather James","year":"2022","unstructured":"James Prather, John Homer, Paul Denny, Brett Becker, John Marsden, et al. 2022. Scaffolding Task Planning Using Abstract Parsons Problems. In Proc. of the 2022 World Conf. on Computers in Education (WCCE '22). 1--10."},{"key":"e_1_3_2_1_36_1","doi-asserted-by":"publisher","DOI":"10.1145\/3501385.3543957"},{"key":"e_1_3_2_1_37_1","doi-asserted-by":"publisher","DOI":"10.1145\/2828959.2828963"},{"key":"e_1_3_2_1_38_1","volume-title":"CHI Conf. on Human Factors in Computing Systems Extended Abstracts. ACM, NY NY, USA, 1--7.","author":"Vaithilingam Priyan","unstructured":"Priyan Vaithilingam, Tianyi Zhang, and Elena L. Glassman. 2022. Expectation vs. Experience: Evaluating the Usability of Code Generation Tools Powered by Large Language Models. In CHI Conf. on Human Factors in Computing Systems Extended Abstracts. ACM, NY NY, USA, 1--7."},{"key":"e_1_3_2_1_39_1","doi-asserted-by":"publisher","DOI":"10.1145\/3328778.3372639"},{"key":"e_1_3_2_1_40_1","doi-asserted-by":"publisher","DOI":"10.1145\/3411764.3445228"},{"key":"e_1_3_2_1_41_1","doi-asserted-by":"publisher","DOI":"10.1145\/3570220"},{"key":"e_1_3_2_1_42_1","doi-asserted-by":"publisher","DOI":"10.1145\/3488042.3489968"},{"key":"e_1_3_2_1_43_1","volume-title":"Proc. of the 2019 ACM Conf. on Int. Computing Education Research. ACM, NY, NY, USA, 51--59","author":"Zhi Rui","unstructured":"Rui Zhi, Min Chi, Tiffany Barnes, and Thomas W. Price. 2019. Evaluating the Effectiveness of Parsons Problems for Block-Based Programming. In Proc. of the 2019 ACM Conf. on Int. Computing Education Research. ACM, NY, NY, USA, 51--59."}],"event":{"name":"ITiCSE 2023: Innovation and Technology in Computer Science Education","location":"Turku Finland","acronym":"ITiCSE 2023","sponsor":["SIGCSE ACM Special Interest Group on Computer Science Education"]},"container-title":["Proceedings of the 2023 Conference on Innovation and Technology in Computer Science Education V. 1"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3587102.3588805","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3587102.3588805","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T18:08:02Z","timestamp":1750183682000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3587102.3588805"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023,6,29]]},"references-count":43,"alternative-id":["10.1145\/3587102.3588805","10.1145\/3587102"],"URL":"https:\/\/doi.org\/10.1145\/3587102.3588805","relation":{},"subject":[],"published":{"date-parts":[[2023,6,29]]},"assertion":[{"value":"2023-06-30","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}