{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,5,11]],"date-time":"2026-05-11T10:47:41Z","timestamp":1778496461039,"version":"3.51.4"},"publisher-location":"New York, NY, USA","reference-count":59,"publisher":"ACM","license":[{"start":{"date-parts":[[2024,1,29]],"date-time":"2024-01-29T00:00:00Z","timestamp":1706486400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2024,1,29]]},"DOI":"10.1145\/3636243.3636256","type":"proceedings-article","created":{"date-parts":[[2024,1,2]],"date-time":"2024-01-02T23:20:13Z","timestamp":1704237613000},"page":"114-123","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":92,"title":["A Comparative Study of AI-Generated (GPT-4) and Human-crafted MCQs in Programming Education"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0009-0008-5430-7282","authenticated-orcid":false,"given":"Jacob","family":"Doughty","sequence":"first","affiliation":[{"name":"Carnegie Mellon University, United States"}]},{"ORCID":"https:\/\/orcid.org\/0009-0002-5866-2376","authenticated-orcid":false,"given":"Zipiao","family":"Wan","sequence":"additional","affiliation":[{"name":"Carnegie Mellon University, United States"}]},{"ORCID":"https:\/\/orcid.org\/0009-0007-0420-3429","authenticated-orcid":false,"given":"Anishka","family":"Bompelli","sequence":"additional","affiliation":[{"name":"Carnegie Mellon University, United States"}]},{"ORCID":"https:\/\/orcid.org\/0009-0006-0956-447X","authenticated-orcid":false,"given":"Jubahed","family":"Qayum","sequence":"additional","affiliation":[{"name":"Carnegie Mellon University, United States"}]},{"ORCID":"https:\/\/orcid.org\/0009-0000-7945-0319","authenticated-orcid":false,"given":"Taozhi","family":"Wang","sequence":"additional","affiliation":[{"name":"Carnegie Mellon University, United States"}]},{"ORCID":"https:\/\/orcid.org\/0009-0003-8689-0357","authenticated-orcid":false,"given":"Juran","family":"Zhang","sequence":"additional","affiliation":[{"name":"Carnegie Mellon University, United States"}]},{"ORCID":"https:\/\/orcid.org\/0009-0001-6121-0621","authenticated-orcid":false,"given":"Yujia","family":"Zheng","sequence":"additional","affiliation":[{"name":"Carnegie Mellon University, United States"}]},{"ORCID":"https:\/\/orcid.org\/0009-0008-6260-7517","authenticated-orcid":false,"given":"Aidan","family":"Doyle","sequence":"additional","affiliation":[{"name":"Carnegie Mellon University - LTI, United States"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-2160-288X","authenticated-orcid":false,"given":"Pragnya","family":"Sridhar","sequence":"additional","affiliation":[{"name":"Carnegie Mellon University - LTI, United States"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-9848-1663","authenticated-orcid":false,"given":"Arav","family":"Agarwal","sequence":"additional","affiliation":[{"name":"Carnegie Mellon University, United States"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-8581-115X","authenticated-orcid":false,"given":"Christopher","family":"Bogart","sequence":"additional","affiliation":[{"name":"Carnegie Mellon University, United States"}]},{"ORCID":"https:\/\/orcid.org\/0009-0006-9782-658X","authenticated-orcid":false,"given":"Eric","family":"Keylor","sequence":"additional","affiliation":[{"name":"Carnegie Mellon University, United States"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-6427-4161","authenticated-orcid":false,"given":"Can","family":"Kultur","sequence":"additional","affiliation":[{"name":"Carnegie Mellon University, United States"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-3674-5456","authenticated-orcid":false,"given":"Jaromir","family":"Savelka","sequence":"additional","affiliation":[{"name":"Carnegie Mellon University, United States"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-3739-298X","authenticated-orcid":false,"given":"Majd","family":"Sakr","sequence":"additional","affiliation":[{"name":"Carnegie Mellon University, United States"}]}],"member":"320","published-online":{"date-parts":[[2024,1,29]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"crossref","unstructured":"Arav Agarwal Karthik Mittal Aidan Doyle Pragnya Sridhar Zipiao Wan Jacob Doughty Jaromir Savelka and Majd Sakr. 2023. Understanding the Role of Temperature in Diverse Question Generation by GPT-4.","DOI":"10.1145\/3626253.3635608"},{"key":"e_1_3_2_1_2_1","article-title":"The question answering systems: A survey","volume":"2","author":"Mohamed\u00a0Nabil Allam Ali","year":"2012","unstructured":"Ali Mohamed\u00a0Nabil Allam and Mohamed\u00a0Hassan Haggag. 2012. The question answering systems: A survey. International Journal of Research and Reviews in Information Sciences (IJRRIS) 2, 3 (2012).","journal-title":"International Journal of Research and Reviews in Information Sciences (IJRRIS)"},{"key":"e_1_3_2_1_3_1","volume-title":"Multiple-choice testing in education: Are the best practices for assessment also good for learning?Journal of Applied Research in Memory and Cognition 7, 3","author":"Butler C","year":"2018","unstructured":"Andrew\u00a0C Butler. 2018. Multiple-choice testing in education: Are the best practices for assessment also good for learning?Journal of Applied Research in Memory and Cognition 7, 3 (2018), 323\u2013331."},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.1109\/TLT.2018.2889100"},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D19-5821"},{"key":"e_1_3_2_1_6_1","volume-title":"Gordon Tin\u00a0Chun Wong, Elaine Yuen\u00a0Phin Lee, Dhananjay Kulkarni, Choon\u00a0Sheong Seow, Ruby Wong, and Michael Tiong\u00a0Hong Co.","author":"Ho\u00a0Hung Cheung Billy","year":"2023","unstructured":"Billy Ho\u00a0Hung Cheung, Gary Kui\u00a0Kai Lau, Gordon Tin\u00a0Chun Wong, Elaine Yuen\u00a0Phin Lee, Dhananjay Kulkarni, Choon\u00a0Sheong Seow, Ruby Wong, and Michael Tiong\u00a0Hong Co. 2023. ChatGPT versus human in generating medical graduate exam questions\u2013An international prospective study. medRxiv (2023), 2023\u201305."},{"key":"e_1_3_2_1_7_1","volume-title":"Contrastive multi-document question generation. arXiv preprint arXiv:1911.03047","author":"Cho Woon\u00a0Sang","year":"2019","unstructured":"Woon\u00a0Sang Cho, Yizhe Zhang, Sudha Rao, Asli Celikyilmaz, Chenyan Xiong, Jianfeng Gao, Mengdi Wang, and Bill Dolan. 2019. Contrastive multi-document question generation. arXiv preprint arXiv:1911.03047 (2019)."},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.1145\/1404520.1404526"},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"crossref","unstructured":"Paul Denny Viraj Kumar and Nasser Giacaman. 2022. Conversing with Copilot: Exploring Prompt Engineering for Solving CS1 Problems Using Natural Language. arxiv:2210.15157\u00a0[cs.HC]","DOI":"10.1145\/3545945.3569823"},{"key":"e_1_3_2_1_10_1","volume-title":"Bert: Pre-training of deep bidirectional transformers for language understanding. arXiv preprint arXiv:1810.04805","author":"Devlin Jacob","year":"2018","unstructured":"Jacob Devlin, Ming-Wei Chang, Kenton Lee, and Kristina Toutanova. 2018. Bert: Pre-training of deep bidirectional transformers for language understanding. arXiv preprint arXiv:1810.04805 (2018)."},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/P17-1123"},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D17-1090"},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v33i01.33016423"},{"key":"e_1_3_2_1_14_1","volume-title":"Automatic factual question generation from text. Ph.\u00a0D. Dissertation","author":"Heilman Michael","unstructured":"Michael Heilman. 2011. Automatic factual question generation from text. Ph.\u00a0D. Dissertation. Carnegie Mellon University."},{"key":"e_1_3_2_1_15_1","volume-title":"Long short-term memory. Neural computation 9, 8","author":"Hochreiter Sepp","year":"1997","unstructured":"Sepp Hochreiter and J\u00fcrgen Schmidhuber. 1997. Long short-term memory. Neural computation 9, 8 (1997), 1735\u20131780."},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/W17-5015"},{"key":"e_1_3_2_1_17_1","volume-title":"BERT-based distractor generation for Swedish reading comprehension questions using a small-scale dataset. arXiv preprint arXiv:2108.03973","author":"Kalpakchi Dmytro","year":"2021","unstructured":"Dmytro Kalpakchi and Johan Boye. 2021. BERT-based distractor generation for Swedish reading comprehension questions using a small-scale dataset. arXiv preprint arXiv:2108.03973 (2021)."},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1145\/3544548.3580919"},{"key":"e_1_3_2_1_19_1","volume-title":"A revision of Bloom\u2019s taxonomy: An overview. Theory into practice 41, 4","author":"Krathwohl R","year":"2002","unstructured":"David\u00a0R Krathwohl. 2002. A revision of Bloom\u2019s taxonomy: An overview. Theory into practice 41, 4 (2002), 212\u2013218."},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.eswa.2023.120022"},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.1007\/s40593-019-00186-y"},{"key":"e_1_3_2_1_22_1","unstructured":"Daniel Leiker Sara Finnigan Ashley\u00a0Ricker Gyllen and Mutlu Cukurova. 2023. Prototyping the use of Large Language Models (LLMs) for adult learning content creation at scale. In LLM@AIED. https:\/\/api.semanticscholar.org\/CorpusID:259076210"},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"crossref","unstructured":"Juho Leinonen Paul Denny Stephen MacNeil Sami Sarsa Seth Bernstein Joanne Kim Andrew Tran and Arto Hellas. 2023. Comparing Code Explanations Created by Students and Large Language Models. arxiv:2304.03938\u00a0[cs.CY]","DOI":"10.1145\/3587102.3588785"},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"publisher","DOI":"10.5281\/zenodo.6853191"},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/W18-0533"},{"key":"e_1_3_2_1_26_1","volume-title":"Codehelp: Using large language models with guardrails for scalable support in programming classes. arXiv preprint arXiv:2308.06921","author":"Liffiton Mark","year":"2023","unstructured":"Mark Liffiton, Brad Sheese, Jaromir Savelka, and Paul Denny. 2023. Codehelp: Using large language models with guardrails for scalable support in programming classes. arXiv preprint arXiv:2308.06921 (2023)."},{"key":"e_1_3_2_1_27_1","volume-title":"Jan Christian\u00a0Blaise Cruz, and Charibeth Cheng","author":"Lopez Luis\u00a0Enrico","year":"2020","unstructured":"Luis\u00a0Enrico Lopez, Diane\u00a0Kathryn Cruz, Jan Christian\u00a0Blaise Cruz, and Charibeth Cheng. 2020. Transformer-based end-to-end question generation. arXiv preprint arXiv:2005.01107 4 (2020)."},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"publisher","DOI":"10.1145\/3545945.3569785"},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.1145\/3501709.3544280"},{"key":"e_1_3_2_1_30_1","volume-title":"A comprehensive review on MCQ generation from text. Multimedia Tools and Applications","author":"Madri Vijaya\u00a0Raju","year":"2023","unstructured":"Vijaya\u00a0Raju Madri and Sreenivasulu Meruva. 2023. A comprehensive review on MCQ generation from text. Multimedia Tools and Applications (2023), 1\u201320."},{"key":"e_1_3_2_1_31_1","volume-title":"Using artificial intelligence to create biology multiple choice questions for higher education. Agricultural and Environmental Education 2, 1","author":"Nasution NEA","year":"2023","unstructured":"NEA Nasution. 2023. Using artificial intelligence to create biology multiple choice questions for higher education. Agricultural and Environmental Education 2, 1 (2023)."},{"key":"e_1_3_2_1_32_1","volume-title":"Better distractions: Transformer-based distractor generation and multiple choice question filtering. arXiv preprint arXiv:2010.09598","author":"Offerijns Jeroen","year":"2020","unstructured":"Jeroen Offerijns, Suzan Verberne, and Tessa Verhoef. 2020. Better distractions: Transformer-based distractor generation and multiple choice question filtering. arXiv preprint arXiv:2010.09598 (2020)."},{"key":"e_1_3_2_1_34_1","volume-title":"Generating High-Precision Feedback for Programming Syntax Errors using Large Language Models. ArXiv abs\/2302.04662","author":"Phung Tung","year":"2023","unstructured":"Tung Phung, Jos\u00e9\u00a0Pablo Cambronero, Sumit Gulwani, Tobias Kohn, Rupak Majumdar, Adish\u00a0Kumar Singla, and Gustavo Soares. 2023. Generating High-Precision Feedback for Programming Syntax Errors using Large Language Models. ArXiv abs\/2302.04662 (2023)."},{"key":"e_1_3_2_1_35_1","unstructured":"Stephen\u00a0R. Piccolo Paul Denny Andrew Luxton-Reilly Samuel Payne and Perry\u00a0G. Ridge. 2023. Many bioinformatics programming tasks can be automated with ChatGPT. arxiv:2303.13528\u00a0[q-bio.OT]"},{"key":"e_1_3_2_1_36_1","volume-title":"The robots are here: Navigating the generative ai revolution in computing education. arXiv preprint arXiv:2310.00658","author":"Prather James","year":"2023","unstructured":"James Prather, Paul Denny, Juho Leinonen, Brett\u00a0A Becker, Ibrahim Albluwi, Michelle Craig, Hieke Keuning, Natalie Kiesler, Tobias Kohn, Andrew Luxton-Reilly, 2023. The robots are here: Navigating the generative ai revolution in computing education. arXiv preprint arXiv:2310.00658 (2023)."},{"key":"e_1_3_2_1_37_1","doi-asserted-by":"publisher","DOI":"10.1109\/IALP54817.2021.9675153"},{"key":"e_1_3_2_1_38_1","unstructured":"Alec Radford Jeffrey Wu Rewon Child David Luan Dario Amodei and Ilya Sutskever. 2019. Language models are unsupervised multitask learners. (2019)."},{"key":"e_1_3_2_1_39_1","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v35i5.16559"},{"key":"e_1_3_2_1_40_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.eswa.2022.118258"},{"key":"e_1_3_2_1_41_1","doi-asserted-by":"publisher","unstructured":"Sami Sarsa Paul Denny Arto Hellas and Juho Leinonen. 2022. Automatic Generation of Programming Exercises and Code Explanations Using Large Language Models. ACM. https:\/\/doi.org\/10.1145\/3501385.3543957","DOI":"10.1145\/3501385.3543957"},{"key":"e_1_3_2_1_42_1","doi-asserted-by":"publisher","DOI":"10.1145\/3568813.3600142"},{"key":"e_1_3_2_1_43_1","volume-title":"From GPT-3 to GPT-4: On the Evolving Efficacy of LLMs to Answer Multiple-choice Questions for Programming Classes in Higher Education. arXiv preprint arXiv:2311.09518","author":"Savelka Jaromir","year":"2023","unstructured":"Jaromir Savelka, Arav Agarwal, Christopher Bogart, and Majd Sakr. 2023. From GPT-3 to GPT-4: On the Evolving Efficacy of LLMs to Answer Multiple-choice Questions for Programming Classes in Higher Education. arXiv preprint arXiv:2311.09518 (2023)."},{"key":"e_1_3_2_1_44_1","volume-title":"Large language models (gpt) struggle to answer multiple-choice questions about code. arXiv preprint arXiv:2303.08033","author":"Savelka Jaromir","year":"2023","unstructured":"Jaromir Savelka, Arav Agarwal, Christopher Bogart, and Majd Sakr. 2023. Large language models (gpt) struggle to answer multiple-choice questions about code. arXiv preprint arXiv:2303.08033 (2023)."},{"key":"e_1_3_2_1_45_1","doi-asserted-by":"publisher","DOI":"10.1145\/3587102.3588792"},{"key":"e_1_3_2_1_46_1","unstructured":"Jaromir Savelka Paul Denny Mark Liffiton and Brad Sheese. 2023. Efficient Classification of Student Help Requests in Programming Courses Using Large Language Models. arxiv:2310.20105\u00a0[cs.CY]"},{"key":"e_1_3_2_1_47_1","doi-asserted-by":"crossref","unstructured":"Brad Sheese Mark Liffiton Jaromir Savelka and Paul Denny. 2023. Patterns of Student Help-Seeking When Using a Large Language Model-Powered Programming Assistant. arxiv:2310.16984\u00a0[cs.CY]","DOI":"10.1145\/3636243.3636249"},{"key":"e_1_3_2_1_48_1","volume-title":"Multiple-choice item distractor development using topic modeling approaches. Frontiers in psychology 10","author":"Shin Jinnie","year":"2019","unstructured":"Jinnie Shin, Qi Guo, and Mark\u00a0J Gierl. 2019. Multiple-choice item distractor development using topic modeling approaches. Frontiers in psychology 10 (2019), 825."},{"key":"e_1_3_2_1_49_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.jksuci.2018.08.005"},{"key":"e_1_3_2_1_50_1","volume-title":"Harnessing llms in curricular design: Using gpt-4 to support authoring of learning objectives. arXiv preprint arXiv:2306.17459","author":"Sridhar Pragnya","year":"2023","unstructured":"Pragnya Sridhar, Aidan Doyle, Arav Agarwal, Christopher Bogart, Jaromir Savelka, and Majd Sakr. 2023. Harnessing llms in curricular design: Using gpt-4 to support authoring of learning objectives. arXiv preprint arXiv:2306.17459 (2023)."},{"key":"e_1_3_2_1_51_1","volume-title":"Question answering and question generation as dual tasks. arXiv preprint arXiv:1706.02027","author":"Tang Duyu","year":"2017","unstructured":"Duyu Tang, Nan Duan, Tao Qin, Zhao Yan, and Ming Zhou. 2017. Question answering and question generation as dual tasks. arXiv preprint arXiv:1706.02027 (2017)."},{"key":"e_1_3_2_1_52_1","doi-asserted-by":"publisher","DOI":"10.1021\/ed500076x"},{"key":"e_1_3_2_1_53_1","unstructured":"Andrew Tran Kenneth Angelikas Egi Rama Chiku Okechukwu David\u00a0H Smith\u00a0IV and Stephen MacNeil. [n. d.]. Generating Multiple Choice Questions for Computing Courses using Large Language Models. ([n. d.])."},{"key":"e_1_3_2_1_54_1","doi-asserted-by":"publisher","DOI":"10.1145\/1047124.1047502"},{"key":"e_1_3_2_1_55_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-99739-7_41"},{"key":"e_1_3_2_1_56_1","volume-title":"Attention is all you need. Advances in neural information processing systems 30","author":"Vaswani Ashish","year":"2017","unstructured":"Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, Llion Jones, Aidan\u00a0N Gomez, \u0141ukasz Kaiser, and Illia Polosukhin. 2017. Attention is all you need. Advances in neural information processing systems 30 (2017)."},{"key":"e_1_3_2_1_57_1","doi-asserted-by":"publisher","DOI":"10.1145\/3231644.3231654"},{"key":"e_1_3_2_1_58_1","doi-asserted-by":"publisher","unstructured":"N. Wongpakaran T. Wongpakaran D. Wedding and K. Gwet. 2013. A comparison of Cohen\u2019s Kappa and Gwet\u2019s AC1 when calculating inter-rater reliability coefficients: a study conducted with personality disorder samples.BMC Med Res Methodol (2013). https:\/\/doi.org\/10.1186\/1471-2288-13-61","DOI":"10.1186\/1471-2288-13-61"},{"key":"e_1_3_2_1_59_1","doi-asserted-by":"publisher","unstructured":"S. Zec N. Soriani R. Comoretto and I. Baldi. 2017. High Agreement and High Prevalence: The Paradox of Cohen\u2019s Kappa. The Open Nursing Journal (2017) 221\u2013218. https:\/\/doi.org\/10.2174\/1874434601711010211","DOI":"10.2174\/1874434601711010211"},{"key":"e_1_3_2_1_60_1","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v34i05.6522"}],"event":{"name":"ACE 2024: Australian Computing Education Conference","location":"Sydney NSW Australia","acronym":"ACE 2024"},"container-title":["Proceedings of the 26th Australasian Computing Education Conference"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3636243.3636256","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3636243.3636256","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T16:35:41Z","timestamp":1750178141000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3636243.3636256"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,1,29]]},"references-count":59,"alternative-id":["10.1145\/3636243.3636256","10.1145\/3636243"],"URL":"https:\/\/doi.org\/10.1145\/3636243.3636256","relation":{},"subject":[],"published":{"date-parts":[[2024,1,29]]},"assertion":[{"value":"2024-01-29","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}