{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,6,6]],"date-time":"2026-06-06T16:36:09Z","timestamp":1780763769547,"version":"3.54.1"},"publisher-location":"New York, NY, USA","reference-count":74,"publisher":"ACM","license":[{"start":{"date-parts":[[2024,1,29]],"date-time":"2024-01-29T00:00:00Z","timestamp":1706486400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2024,1,29]]},"DOI":"10.1145\/3636243.3636247","type":"proceedings-article","created":{"date-parts":[[2024,1,2]],"date-time":"2024-01-02T23:20:13Z","timestamp":1704237613000},"page":"29-38","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":27,"title":["More Robots are Coming: Large Multimodal Models (ChatGPT) can Solve Visually Diverse Images of Parsons Problems"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0009-0008-0511-7685","authenticated-orcid":false,"given":"Irene","family":"Hou","sequence":"first","affiliation":[{"name":"Temple University, United States"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0003-0527-1395","authenticated-orcid":false,"given":"Owen","family":"Man","sequence":"additional","affiliation":[{"name":"Temple University, United States"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0009-9707-6311","authenticated-orcid":false,"given":"Sophia","family":"Mettille","sequence":"additional","affiliation":[{"name":"Temple University, United States"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0005-4844-692X","authenticated-orcid":false,"given":"Sebastian","family":"Gutierrez","sequence":"additional","affiliation":[{"name":"Temple University, United States"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0001-1597-3513","authenticated-orcid":false,"given":"Kenneth","family":"Angelikas","sequence":"additional","affiliation":[{"name":"Temple University, United States"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-2781-6619","authenticated-orcid":false,"given":"Stephen","family":"MacNeil","sequence":"additional","affiliation":[{"name":"Temple University, United States"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2024,1,29]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10710-019-09343-7"},{"key":"e_1_3_2_1_2_1","volume-title":"Grounded Copilot: How Programmers Interact with Code-Generating Models. arXiv preprint arXiv:2206.15000","author":"Barke Shraddha","year":"2022","unstructured":"Shraddha Barke, Michael\u00a0B James, and Nadia Polikarpova. 2022. Grounded Copilot: How Programmers Interact with Code-Generating Models. arXiv preprint arXiv:2206.15000 (2022)."},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1145\/3545945.3569759"},{"key":"e_1_3_2_1_4_1","volume-title":"Ungrading: Why rating students undermines learning (and what to do instead)","author":"Blum Susan\u00a0Debra","year":"2020","unstructured":"Susan\u00a0Debra Blum. 2020. Ungrading: Why rating students undermines learning (and what to do instead). West Virginia University Press."},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","unstructured":"Peter Brusilovsky Stephen Edwards Amruth Kumar Lauri Malmi Luciana Benotti Duane Buck Petri Ihantola Rikki Prince Teemu Sirki\u00e4 Sergey Sosnovsky Jaime Urquiza-Fuentes Arto Hellas and Michael Wollowski. 2014. Increasing Adoption of Smart Learning Content for Computer Science Education. ITiCSE-WGR 2014 - Working Group Reports of the 2014 Innovation and Technology in Computer Science Education Conference. https:\/\/doi.org\/10.1145\/2713609.2713611","DOI":"10.1145\/2713609.2713611"},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"publisher","DOI":"10.2139\/ssrn.3791771"},{"key":"e_1_3_2_1_7_1","volume-title":"Jared Kaplan, Harri Edwards, Yuri Burda","author":"Chen Mark","year":"2021","unstructured":"Mark Chen, Jerry Tworek, Heewoo Jun, Qiming Yuan, Henrique Ponde de\u00a0Oliveira Pinto, Jared Kaplan, Harri Edwards, Yuri Burda, Nicholas Joseph, Greg Brockman, 2021. Evaluating large language models trained on code. arXiv preprint arXiv:2107.03374 (2021)."},{"key":"e_1_3_2_1_8_1","volume-title":"Ungrading: Why rating students undermines learning (and what to do instead), Susan\u00a0Debra Blum (Ed.)","author":"Chu Gary","unstructured":"Gary Chu. 2020. The Point-less Classroom: A Math Teacher\u2019s Ironic Choice in not Calculating Grades. In Ungrading: Why rating students undermines learning (and what to do instead), Susan\u00a0Debra Blum (Ed.). West Virginia University Press, Chapter\u00a010, 161\u2013170."},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1007\/s13347-021-00476-1"},{"key":"e_1_3_2_1_10_1","volume-title":"Do racial and ethnic group differences in performance on the MCAT exam reflect test bias?Academic Medicine 88, 5","author":"Davis Dwight","year":"2013","unstructured":"Dwight Davis, J\u00a0Kevin Dorsey, Ronald\u00a0D Franks, Paul\u00a0R Sackett, Cynthia\u00a0A Searcy, and Xiaohui Zhao. 2013. Do racial and ethnic group differences in performance on the MCAT exam reflect test bias?Academic Medicine 88, 5 (2013), 593\u2013602."},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.1145\/3545945.3569823"},{"key":"e_1_3_2_1_12_1","volume-title":"Promptly: Using Prompt Problems to Teach Learners How to Effectively Utilize AI Code Generators. arxiv:2307.16364\u00a0[cs.HC]","author":"Denny Paul","year":"2023","unstructured":"Paul Denny, Juho Leinonen, James Prather, Andrew Luxton-Reilly, Thezyrie Amarouche, Brett\u00a0A. Becker, and Brent\u00a0N. Reeves. 2023. Promptly: Using Prompt Problems to Teach Learners How to Effectively Utilize AI Code Generators. arxiv:2307.16364\u00a0[cs.HC]"},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1145\/1404520.1404532"},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.1145\/3373165.3373187"},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","unstructured":"Barbara Ericson Mark Guzdial and Briana Morrison. 2015. Analysis of Interactive Features Designed to Enhance Learning in an Ebook. 169\u2013178. https:\/\/doi.org\/10.1145\/2787622.2787731","DOI":"10.1145\/2787622.2787731"},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.1145\/3571785.3574127"},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.1145\/2787622.2787731"},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1145\/3141880.3141895"},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","DOI":"10.1145\/3328778.3366950"},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.1145\/3587103.3594211"},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.1145\/2960310.2960335"},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE48619.2023.00128"},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.1145\/3511861.3511863"},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"publisher","unstructured":"Alessio Gaspar Dmytro Vitel and A.T.M. Bari. 2019. Lessons Learned from Available Parsons Puzzles Software. https:\/\/doi.org\/10.18260\/1-2\u201333060","DOI":"10.18260\/1-2"},{"key":"e_1_3_2_1_25_1","volume-title":"Multimodal-gpt: A vision and language model for dialogue with humans. arXiv preprint arXiv:2305.04790","author":"Gong Tao","year":"2023","unstructured":"Tao Gong, Chengqi Lyu, Shilong Zhang, Yudong Wang, Miao Zheng, Qian Zhao, Kuikun Liu, Wenwei Zhang, Ping Luo, and Kai Chen. 2023. Multimodal-gpt: A vision and language model for dialogue with humans. arXiv preprint arXiv:2305.04790 (2023)."},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.3390\/su12083488"},{"key":"e_1_3_2_1_27_1","doi-asserted-by":"publisher","DOI":"10.1145\/2960310.2960314"},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"publisher","DOI":"10.1145\/2361276.2361300"},{"key":"e_1_3_2_1_29_1","volume-title":"The Bad Test-Taker Identity. Teaching of psychology 48, 4","author":"Holmes D","year":"2021","unstructured":"Jeffrey\u00a0D Holmes. 2021. The Bad Test-Taker Identity. Teaching of psychology 48, 4 (2021), 293\u2013299."},{"key":"e_1_3_2_1_30_1","volume-title":"The Effects of Generative AI on Introductory Students\u2019 Help-Seeking Preferences. In Australasian Computing Education Conference(ACE \u201924)","author":"Hou Irene","year":"2024","unstructured":"Irene Hou, Sophia Mettille, Owen Man, Zhuo Li, Cynthia Zastudil, and Stephen MacNeil. 2024. The Effects of Generative AI on Introductory Students\u2019 Help-Seeking Preferences. In Australasian Computing Education Conference(ACE \u201924). ACM."},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"publisher","DOI":"10.1145\/2526968.2526974"},{"key":"e_1_3_2_1_32_1","doi-asserted-by":"publisher","DOI":"10.1145\/2526968.2526974"},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"publisher","DOI":"10.28945\/1394"},{"key":"e_1_3_2_1_34_1","doi-asserted-by":"publisher","DOI":"10.28945\/1394"},{"key":"e_1_3_2_1_35_1","volume-title":"Impact of code language models on automated program repair. arXiv:2302.05020","author":"Jiang Nan","year":"2023","unstructured":"Nan Jiang, Kevin Liu, Thibaud Lutellier, and Lin Tan. 2023. Impact of code language models on automated program repair. arXiv:2302.05020 (2023)."},{"key":"e_1_3_2_1_36_1","unstructured":"Ishika Joshi Ritvik Budhiraja Harshal Dev Jahnvi Kadia Muhammad Osama Ataullah Sayan Mitra and Dhruv Kumar. 2023. ChatGPT and the Future of Undergraduate Computer Science: Challenges Opportunities and Recommendations. https:\/\/api.semanticscholar.org\/CorpusID:258741417"},{"key":"e_1_3_2_1_37_1","volume-title":"Automated Program Repair Using Generative Models for Code Infilling. In International Conference on Artificial Intelligence in Education. Springer, 798\u2013803","author":"Koutcheme Charles","year":"2023","unstructured":"Charles Koutcheme, Sami Sarsa, Juho Leinonen, Arto Hellas, and Paul Denny. 2023. Automated Program Repair Using Generative Models for Code Infilling. In International Conference on Artificial Intelligence in Education. Springer, 798\u2013803."},{"key":"e_1_3_2_1_38_1","doi-asserted-by":"publisher","unstructured":"Amruth Kumar. 2017. The Effect of Providing Motivational Support in Parsons Puzzle Tutors. 528\u2013531. https:\/\/doi.org\/10.1007\/978-3-319-61425-0_56","DOI":"10.1007\/978-3-319-61425-0_56"},{"key":"e_1_3_2_1_39_1","doi-asserted-by":"publisher","DOI":"10.1145\/3159450.3159576"},{"key":"e_1_3_2_1_40_1","doi-asserted-by":"publisher","DOI":"10.1145\/2676723.2677240"},{"key":"e_1_3_2_1_41_1","doi-asserted-by":"publisher","DOI":"10.1145\/3568813.3600138"},{"key":"e_1_3_2_1_42_1","volume-title":"Comparing Code Explanations Created by Students and Large Language Models. arXiv preprint arXiv:2304.03938","author":"Leinonen Juho","year":"2023","unstructured":"Juho Leinonen, Paul Denny, Stephen MacNeil, Sami Sarsa, Seth Bernstein, Joanne Kim, Andrew Tran, and Arto Hellas. 2023. Comparing Code Explanations Created by Students and Large Language Models. arXiv preprint arXiv:2304.03938 (2023)."},{"key":"e_1_3_2_1_43_1","volume-title":"Visual instruction tuning. arXiv preprint arXiv:2304.08485","author":"Liu Haotian","year":"2023","unstructured":"Haotian Liu, Chunyuan Li, Qingyang Wu, and Yong\u00a0Jae Lee. 2023. Visual instruction tuning. arXiv preprint arXiv:2304.08485 (2023)."},{"key":"e_1_3_2_1_44_1","volume-title":"On the hidden mystery of ocr in large multimodal models. arXiv preprint arXiv:2305.07895","author":"Liu Yuliang","year":"2023","unstructured":"Yuliang Liu, Zhang Li, Hongliang Li, Wenwen Yu, Mingxin Huang, Dezhi Peng, Mingyu Liu, Mingrui Chen, Chunyuan Li, Lianwen Jin, 2023. On the hidden mystery of ocr in large multimodal models. arXiv preprint arXiv:2305.07895 (2023)."},{"key":"e_1_3_2_1_45_1","volume-title":"Decoding Logic Errors: A Comparative Study on Bug Detection by Students and Large Language Models. arXiv preprint arXiv:2311.16017","author":"MacNeil Stephen","year":"2023","unstructured":"Stephen MacNeil, Paul Denny, Andrew Tran, Juho Leinonen, Seth Bernstein, Arto Hellas, Sami Sarsa, and Joanne Kim. 2023. Decoding Logic Errors: A Comparative Study on Bug Detection by Students and Large Language Models. arXiv preprint arXiv:2311.16017 (2023)."},{"key":"e_1_3_2_1_46_1","doi-asserted-by":"publisher","DOI":"10.1145\/3545947.3573358"},{"key":"e_1_3_2_1_47_1","doi-asserted-by":"publisher","DOI":"10.1145\/2839509.2844577"},{"key":"e_1_3_2_1_48_1","doi-asserted-by":"publisher","DOI":"10.1145\/3545945.3569785"},{"key":"e_1_3_2_1_49_1","doi-asserted-by":"publisher","DOI":"10.1145\/3501709.3544280"},{"key":"e_1_3_2_1_50_1","volume-title":"Proceedings of the 8th Australasian Conference on Computing Education-Volume 52","author":"Parsons Dale","year":"2006","unstructured":"Dale Parsons and Patricia Haden. 2006. Parson\u2019s programming puzzles: a fun and effective learning tool for first programming courses. In Proceedings of the 8th Australasian Conference on Computing Education-Volume 52. 157\u2013163."},{"key":"e_1_3_2_1_51_1","doi-asserted-by":"publisher","DOI":"10.1145\/3587103.3594206"},{"key":"e_1_3_2_1_52_1","volume-title":"The Robots are Here: Navigating the Generative AI Revolution in Computing Education. arXiv preprint arXiv:2310.00658","author":"Prather James","year":"2023","unstructured":"James Prather, Paul Denny, Juho Leinonen, Brett\u00a0A Becker, Ibrahim Albluwi, Michelle Craig, Hieke Keuning, Natalie Kiesler, Tobias Kohn, Andrew Luxton-Reilly, 2023. The Robots are Here: Navigating the Generative AI Revolution in Computing Education. arXiv preprint arXiv:2310.00658 (2023)."},{"key":"e_1_3_2_1_53_1","doi-asserted-by":"publisher","DOI":"10.5555\/3575618.3575622"},{"key":"e_1_3_2_1_54_1","doi-asserted-by":"publisher","DOI":"10.1145\/3587102.3588805"},{"key":"e_1_3_2_1_55_1","volume-title":"ChatGPT: Bullshit spewer or the end of traditional assessments in higher education?Journal of Applied Learning and Teaching 6, 1","author":"Rudolph J\u00fcrgen","year":"2023","unstructured":"J\u00fcrgen Rudolph, Samson Tan, and Shannon Tan. 2023. ChatGPT: Bullshit spewer or the end of traditional assessments in higher education?Journal of Applied Learning and Teaching 6, 1 (2023)."},{"key":"e_1_3_2_1_56_1","volume-title":"Can ai-generated text be reliably detected?arXiv preprint arXiv:2303.11156","author":"Sadasivan Vinu\u00a0Sankar","year":"2023","unstructured":"Vinu\u00a0Sankar Sadasivan, Aounon Kumar, Sriram Balasubramanian, Wenxiao Wang, and Soheil Feizi. 2023. Can ai-generated text be reliably detected?arXiv preprint arXiv:2303.11156 (2023)."},{"key":"e_1_3_2_1_57_1","doi-asserted-by":"publisher","DOI":"10.1145\/3501385.3543957"},{"key":"e_1_3_2_1_58_1","doi-asserted-by":"publisher","DOI":"10.1145\/3568813.3600142"},{"key":"e_1_3_2_1_59_1","volume-title":"Large language models (gpt) struggle to answer multiple-choice questions about code. arXiv preprint arXiv:2303.08033","author":"Savelka Jaromir","year":"2023","unstructured":"Jaromir Savelka, Arav Agarwal, Christopher Bogart, and Majd Sakr. 2023. Large language models (gpt) struggle to answer multiple-choice questions about code. arXiv preprint arXiv:2303.08033 (2023)."},{"key":"e_1_3_2_1_60_1","doi-asserted-by":"crossref","unstructured":"Jaromir Savelka Arav Agarwal Christopher Bogart Yifan Song and Majd Sakr. 2023. Can Generative Pre-trained Transformers (GPT) Pass Assessments in Higher Education Programming Courses?arxiv:2303.09325","DOI":"10.1145\/3587102.3588792"},{"key":"e_1_3_2_1_61_1","volume-title":"Can Generative Pre-trained Transformers (GPT) Pass Assessments in Higher Education Programming Courses?arXiv preprint arXiv:2303.09325","author":"Savelka Jaromir","year":"2023","unstructured":"Jaromir Savelka, Arav Agarwal, Christopher Bogart, Yifan Song, and Majd Sakr. 2023. Can Generative Pre-trained Transformers (GPT) Pass Assessments in Higher Education Programming Courses?arXiv preprint arXiv:2303.09325 (2023)."},{"key":"e_1_3_2_1_62_1","doi-asserted-by":"publisher","DOI":"10.1080\/17439884.2014.924966"},{"key":"e_1_3_2_1_63_1","volume-title":"Prompt-Specific Poisoning Attacks on Text-to-Image Generative Models. arXiv preprint arXiv:2310.13828","author":"Shan Shawn","year":"2023","unstructured":"Shawn Shan, Wenxin Ding, Josephine Passananti, Haitao Zheng, and Ben\u00a0Y Zhao. 2023. Prompt-Specific Poisoning Attacks on Text-to-Image Generative Models. arXiv preprint arXiv:2310.13828 (2023)."},{"key":"e_1_3_2_1_64_1","doi-asserted-by":"publisher","DOI":"10.1145\/2999541.2999558"},{"key":"e_1_3_2_1_65_1","doi-asserted-by":"publisher","DOI":"10.1145\/3545945.3569747"},{"key":"e_1_3_2_1_66_1","volume-title":"Open coding. Social research methods: A reader","author":"Strauss L","year":"2004","unstructured":"Anselm\u00a0L Strauss and Juliet Corbin. 2004. Open coding. Social research methods: A reader (2004), 303\u2013306."},{"key":"e_1_3_2_1_67_1","unstructured":"Teo Susnjak. 2022. ChatGPT: The End of Online Exam Integrity?arxiv:2212.09292"},{"key":"e_1_3_2_1_68_1","doi-asserted-by":"crossref","unstructured":"Andrew Tran Kenneth Angelikas Egi Rama Chiku Okechukwu David\u00a0H Smith\u00a0IV and Stephen MacNeil. 2023. Generating Multiple Choice Questions for Computing Courses using Large Language Models. (2023).","DOI":"10.1109\/FIE58773.2023.10342898"},{"key":"e_1_3_2_1_69_1","volume-title":"Review of large vision models and visual prompt engineering. arXiv preprint arXiv:2307.00855","author":"Wang Jiaqi","year":"2023","unstructured":"Jiaqi Wang, Zhengliang Liu, Lin Zhao, Zihao Wu, Chong Ma, Sigang Yu, Haixing Dai, Qiushi Yang, Yiheng Liu, Songyao Zhang, 2023. Review of large vision models and visual prompt engineering. arXiv preprint arXiv:2307.00855 (2023)."},{"key":"e_1_3_2_1_70_1","doi-asserted-by":"crossref","unstructured":"Michel Wermelinger. 2023. Using GitHub Copilot to Solve Simple Programming Problems. (2023).","DOI":"10.1145\/3545945.3569830"},{"key":"e_1_3_2_1_71_1","doi-asserted-by":"publisher","DOI":"10.1145\/3430895.3460141"},{"key":"e_1_3_2_1_72_1","doi-asserted-by":"publisher","DOI":"10.1145\/3291279.3339411"},{"key":"e_1_3_2_1_73_1","doi-asserted-by":"publisher","DOI":"10.1145\/3544548.3581388"},{"key":"e_1_3_2_1_74_1","volume-title":"Generative AI in Computing Education: Perspectives of Students and Instructors. arXiv preprint arXiv:2308.04309","author":"Zastudil Cynthia","year":"2023","unstructured":"Cynthia Zastudil, Magdalena Rogalska, Christine Kapp, Jennifer Vaughn, and Stephen MacNeil. 2023. Generative AI in Computing Education: Perspectives of Students and Instructors. arXiv preprint arXiv:2308.04309 (2023)."}],"event":{"name":"ACE 2024: Australian Computing Education Conference","location":"Sydney NSW Australia","acronym":"ACE 2024"},"container-title":["Proceedings of the 26th Australasian Computing Education Conference"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3636243.3636247","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3636243.3636247","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T16:35:41Z","timestamp":1750178141000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3636243.3636247"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,1,29]]},"references-count":74,"alternative-id":["10.1145\/3636243.3636247","10.1145\/3636243"],"URL":"https:\/\/doi.org\/10.1145\/3636243.3636247","relation":{},"subject":[],"published":{"date-parts":[[2024,1,29]]},"assertion":[{"value":"2024-01-29","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}