{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,6,27]],"date-time":"2026-06-27T19:42:33Z","timestamp":1782589353187,"version":"3.54.5"},"publisher-location":"Cham","reference-count":28,"publisher":"Springer Nature Switzerland","isbn-type":[{"value":"9783031642982","type":"print"},{"value":"9783031642999","type":"electronic"}],"license":[{"start":{"date-parts":[[2024,1,1]],"date-time":"2024-01-01T00:00:00Z","timestamp":1704067200000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2024,1,1]],"date-time":"2024-01-01T00:00:00Z","timestamp":1704067200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2024]]},"DOI":"10.1007\/978-3-031-64299-9_3","type":"book-chapter","created":{"date-parts":[[2024,7,1]],"date-time":"2024-07-01T09:02:00Z","timestamp":1719824520000},"page":"31-46","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":15,"title":["An Automatic Question Usability Evaluation Toolkit"],"prefix":"10.1007","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-5256-0339","authenticated-orcid":false,"given":"Steven","family":"Moore","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-2775-6006","authenticated-orcid":false,"given":"Eamon","family":"Costello","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-1227-6173","authenticated-orcid":false,"given":"Huy A.","family":"Nguyen","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-2291-1468","authenticated-orcid":false,"given":"John","family":"Stamper","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2024,7,2]]},"reference":[{"issue":"4","key":"3_CR1","doi-asserted-by":"publisher","first-page":"322","DOI":"10.1108\/IJILT-02-2019-0023","volume":"36","author":"JM Azevedo","year":"2019","unstructured":"Azevedo, J.M., Oliveira, E.P., Beites, P.D.: Using learning analytics to evaluate the quality of multiple-choice questions: A perspective with classical test theory and item response theory. Int. J. Inf. Learn. Technol. 36(4), 322\u2013341 (2019)","journal-title":"Int. J. Inf. Learn. Technol."},{"key":"3_CR2","doi-asserted-by":"crossref","unstructured":"Bhowmick, A.K., Jagmohan, A., Vempaty, A., Dey, P., Hall, L., Hartman, J., Kokku, R., Maheshwari, H.: Automating Question Generation From Educational Text. In: Artificial Intelligence XL. pp. 437\u2013450 Springer Nature Switzerland, Cham (2023)","DOI":"10.1007\/978-3-031-47994-6_38"},{"key":"3_CR3","unstructured":"Bitew, S.K., Deleu, J., Develder, C., Demeester, T.: Distractor generation for multiple-choice questions with predictive prompting and large language models. In: RKDE2023, the 1st International Tutorial and Workshop on Responsible Knowledge Discovery in Education Side event at ECML-PKDD (2023)"},{"key":"3_CR4","doi-asserted-by":"publisher","unstructured":"Bulathwela, S., Muse, H., Yilmaz, E.: Scalable Educational Question Generation with Pre-trained Language Models. In: Wang, N., Rebolledo-Mendez, G., Matsuda, N., Santos, O.C., and Dimitrova, V. (eds.) Artificial Intelligence in Education. pp. 327\u2013339 Springer Nature Switzerland, Cham (2023). https:\/\/doi.org\/10.1007\/978-3-031-36272-9_27","DOI":"10.1007\/978-3-031-36272-9_27"},{"key":"3_CR5","doi-asserted-by":"publisher","DOI":"10.1186\/s13104-018-3959-4","author":"E Costello","year":"2018","unstructured":"Costello, E., Holland, J.C., Kirwan, C.: Evaluation of MCQs from MOOCs for common item writing flaws. BMC Res. (2018). https:\/\/doi.org\/10.1186\/s13104-018-3959-4","journal-title":"BMC Res."},{"key":"3_CR6","doi-asserted-by":"crossref","unstructured":"Doughty, J. et al.: A Comparative Study of AI-Generated (GPT-4) and Human-crafted MCQs in Programming Education. In: Proceedings of the 26th Australasian Computing Education Conference. pp. 114\u2013123 ACM, Sydney NSW Australia (2024)","DOI":"10.1145\/3636243.3636256"},{"key":"3_CR7","doi-asserted-by":"crossref","unstructured":"Elkins, S., Kochmar, E., Cheung, J.C.K., Serban, I.: How Teachers Can Use Large Language Models and Bloom\u2019s Taxonomy to Create Educational Quizzes. In: Proceedings of the AAAI Conference on Artificial Intelligence (2024)","DOI":"10.1609\/aaai.v38i21.30353"},{"key":"3_CR8","doi-asserted-by":"crossref","unstructured":"Faruqui, M., Das, D.: Identifying Well-formed Natural Language Questions. In: Proceedings of the 2018 Conference on Empirical Methods in Natural Language Processing. pp. 798\u2013803 (2018)","DOI":"10.18653\/v1\/D18-1091"},{"issue":"1","key":"3_CR9","first-page":"19","volume":"5","author":"D Ganda","year":"2018","unstructured":"Ganda, D., Buch, R.: A survey on multi label classification. Recent Trends Program. Lang. 5(1), 19\u201323 (2018)","journal-title":"Recent Trends Program. Lang."},{"key":"3_CR10","doi-asserted-by":"crossref","unstructured":"Kasneci, E., Se\u00dfler, K., K\u00fcchemann, S., Bannert, M., Dementieva, D., Fischer, F., Gasser, U., Groh, G., G\u00fcnnemann, S., H\u00fcllermeier, E.: ChatGPT for good? On opportunities and challenges of large language models for education. Learn. Individ. Differ. 102274 (2023)","DOI":"10.1016\/j.lindif.2023.102274"},{"key":"3_CR11","doi-asserted-by":"crossref","unstructured":"Kurdi, G., Leo, J., Parsia, B., Sattler, U., Al-Emari, S.: A systematic review of automatic question generation for educational purposes. Int. J. Artif. Intell. Educ. 30 (2020)","DOI":"10.1007\/s40593-019-00186-y"},{"key":"3_CR12","doi-asserted-by":"publisher","DOI":"10.1016\/j.csl.2020.101151","volume":"67","author":"C van der Lee","year":"2021","unstructured":"van der Lee, C., Gatt, A., van Miltenburg, E., Krahmer, E.: Human evaluation of automatically generated text: Current trends and best practice guidelines. Comput. Speech Lang. 67, 101151 (2021). https:\/\/doi.org\/10.1016\/j.csl.2020.101151","journal-title":"Comput. Speech Lang."},{"key":"3_CR13","doi-asserted-by":"crossref","unstructured":"Li, J., Galley, M., Brockett, C., Gao, J., Dolan, W.B.: A Diversity-Promoting Objective Function for Neural Conversation Models. In: Proceedings of the 2016 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies. pp. 110\u2013119 (2016)","DOI":"10.18653\/v1\/N16-1014"},{"key":"3_CR14","doi-asserted-by":"crossref","unstructured":"Lipton, Z.C., Elkan, C., Narayanaswamy, B.: Thresholding classifiers to maximize F1 score. stat. 1050, 14 (2014)","DOI":"10.1007\/978-3-662-44851-9_15"},{"key":"3_CR15","doi-asserted-by":"publisher","unstructured":"Lu, X., Fan, S., Houghton, J., Wang, L., Wang, X.: ReadingQuizMaker: A Human-NLP Collaborative System that Supports Instructors to Design High-Quality Reading Quiz Questions. In: Proceedings of the 2023 CHI Conference on Human Factors in Computing Systems. pp. 1\u201318 ACM, Hamburg Germany (2023). doi.org\/https:\/\/doi.org\/10.1145\/3544548.3580957","DOI":"10.1145\/3544548.3580957"},{"key":"3_CR16","doi-asserted-by":"crossref","unstructured":"Mathur, N., Baldwin, T., Cohn, T.: Tangled up in BLEU: Reevaluating the Evaluation of Automatic Machine Translation Evaluation Metrics. In: Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics. pp. 4984\u20134997 (2020)","DOI":"10.18653\/v1\/2020.acl-main.448"},{"key":"3_CR17","doi-asserted-by":"publisher","first-page":"575","DOI":"10.1080\/0142159X.2021.1879376","volume":"43","author":"SU Monrad","year":"2021","unstructured":"Monrad, S.U., et al.: What faculty write versus what students see? Perspectives on multiple-choice questions using Bloom\u2019s taxonomy. Med. Teach. 43, 575\u2013582 (2021)","journal-title":"Med. Teach."},{"key":"3_CR18","doi-asserted-by":"crossref","unstructured":"Moon, H., Yang, Y., Yu, H., Lee, S., Jeong, M., Park, J., Shin, J., Kim, M., Choi, S.: Evaluating the Knowledge Dependency of Questions. In: Proceedings of the 2022 Conference on Empirical Methods in Natural Language Processing (2022)","DOI":"10.18653\/v1\/2022.emnlp-main.718"},{"key":"3_CR19","doi-asserted-by":"crossref","unstructured":"Moore, S., Nguyen, H.A., Chen, T., Stamper, J.: Assessing the Quality of Multiple-Choice Questions Using GPT-4 and Rule-Based Methods. In: Responsive and Sustainable Educational Futures. pp. 229\u2013245 Springer Nature Switzerland, Cham (2023)","DOI":"10.1007\/978-3-031-42682-7_16"},{"key":"3_CR20","unstructured":"Morris, J.: Python Language Tool, github.com\/jxmorris12\/language_tool_python (2022)"},{"issue":"1","key":"3_CR21","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1007\/s13748-023-00295-9","volume":"12","author":"N Mulla","year":"2023","unstructured":"Mulla, N., Gharpure, P.: Automatic question generation: a review of methodologies, datasets, evaluation metrics, and applications. Prog. Artif. Intell. 12(1), 1\u201332 (2023)","journal-title":"Prog. Artif. Intell."},{"key":"3_CR22","first-page":"1","volume":"2","author":"NEA Nasution","year":"2023","unstructured":"Nasution, N.E.A.: Using artificial intelligence to create biology multiple choice questions for higher education. Agric. Environ. Educ. 2, 1 (2023)","journal-title":"Agric. Environ. Educ."},{"key":"3_CR23","doi-asserted-by":"publisher","first-page":"225","DOI":"10.15694\/mep.2018.0000225.1","volume":"7","author":"H Pham","year":"2018","unstructured":"Pham, H., Besanko, J., Devitt, P.: Examining the impact of specific types of item-writing flaws on student performance and psychometric properties of the multiple choice question. MedEdPublish. 7, 225 (2018)","journal-title":"MedEdPublish."},{"key":"3_CR24","unstructured":"Raina, V., Gales, M.: Multiple-Choice Question Generation: Towards an Automated Assessment Framework, http:\/\/arxiv.org\/abs\/2209.11830 (2022)"},{"key":"3_CR25","unstructured":"Scully, D.: Constructing multiple-choice items to measure higher-order thinking. Pract. Assess. Res. Eval. 22, 1, 4 (2019)"},{"issue":"8","key":"3_CR26","doi-asserted-by":"publisher","first-page":"662","DOI":"10.1016\/j.nedt.2006.07.006","volume":"26","author":"M Tarrant","year":"2006","unstructured":"Tarrant, M., Knierim, A., Hayes, S.K., Ware, J.: The frequency of item writing flaws in multiple-choice questions used in high stakes nursing assessments. Nurse Educ. Today 26(8), 662\u2013671 (2006)","journal-title":"Nurse Educ. Today"},{"key":"3_CR27","unstructured":"Wang, Z., Funakoshi, K., Okumura, M.: Automatic Answerability Evaluation for Question Generation, http:\/\/arxiv.org\/abs\/2309.12546 (2023)"},{"key":"3_CR28","doi-asserted-by":"publisher","first-page":"153","DOI":"10.1007\/978-3-031-11644-5_13","volume-title":"Artificial Intelligence in Education","author":"Z Wang","year":"2022","unstructured":"Wang, Z., Valdez, J., Basu Mallick, D., Baraniuk, R.G.: Towards Human-Like Educational Question Generation with Large Language Models. In: Rodrigo, M.M., Matsuda, N., Cristea, A.I., Dimitrova, V. (eds.) Artificial Intelligence in Education, pp. 153\u2013166. Springer International Publishing, Cham (2022)"}],"container-title":["Lecture Notes in Computer Science","Artificial Intelligence in Education"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-031-64299-9_3","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,7,1]],"date-time":"2024-07-01T09:05:18Z","timestamp":1719824718000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-3-031-64299-9_3"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024]]},"ISBN":["9783031642982","9783031642999"],"references-count":28,"URL":"https:\/\/doi.org\/10.1007\/978-3-031-64299-9_3","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"value":"0302-9743","type":"print"},{"value":"1611-3349","type":"electronic"}],"subject":[],"published":{"date-parts":[[2024]]},"assertion":[{"value":"2 July 2024","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"AIED","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"International Conference on Artificial Intelligence in Education","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Recife","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Brazil","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2024","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"8 July 2024","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"12 July 2024","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"25","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"aied2024","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"https:\/\/aied2024.cesar.school\/home","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}}]}}