{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,6,26]],"date-time":"2026-06-26T12:51:27Z","timestamp":1782478287499,"version":"3.54.5"},"publisher-location":"Cham","reference-count":28,"publisher":"Springer Nature Switzerland","isbn-type":[{"value":"9783032297433","type":"print"},{"value":"9783032297440","type":"electronic"}],"license":[{"start":{"date-parts":[[2026,6,27]],"date-time":"2026-06-27T00:00:00Z","timestamp":1782518400000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2026,6,27]],"date-time":"2026-06-27T00:00:00Z","timestamp":1782518400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2027]]},"DOI":"10.1007\/978-3-032-29744-0_37","type":"book-chapter","created":{"date-parts":[[2026,6,26]],"date-time":"2026-06-26T12:39:26Z","timestamp":1782477566000},"page":"555-570","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":0,"title":["Has Automated Essay Scoring Reached Sufficient Accuracy? Deriving Achievable QWK Ceilings from\u00a0Classical Test Theory"],"prefix":"10.1007","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-9330-5158","authenticated-orcid":false,"given":"Masaki","family":"Uto","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2026,6,27]]},"reference":[{"key":"37_CR1","doi-asserted-by":"crossref","unstructured":"Amorim, E., Can\u00e7ado, M., Veloso, A.: Automated essay scoring in the presence of biased ratings. In: Proceedings of the 2018 Conference of the North American Chapter of the Association for Computational Linguistics, pp. 229\u2013237 (2018)","DOI":"10.18653\/v1\/N18-1021"},{"issue":"4","key":"37_CR2","doi-asserted-by":"crossref","first-page":"1099","DOI":"10.1111\/j.1541-0420.2007.00797.x","volume":"63","author":"HX Barnhart","year":"2007","unstructured":"Barnhart, H.X., Haber, M.J., Lin, L.I.: Overall concordance correlation coefficient for evaluating agreement among multiple observers. Biometrics 63(4), 1099\u20131106 (2007)","journal-title":"Biometrics"},{"key":"37_CR3","unstructured":"Bishop, C.M.: Pattern Recognition and Machine Learning. Springer, New York (2006)"},{"key":"37_CR4","doi-asserted-by":"crossref","unstructured":"Brennan, R.L.: Generalizability Theory. Springer-Verlag, New York (2001)","DOI":"10.1007\/978-1-4757-3456-0"},{"key":"37_CR5","doi-asserted-by":"publisher","first-page":"60","DOI":"10.1007\/s40593-014-0026-8","volume":"25","author":"S Burrows","year":"2015","unstructured":"Burrows, S., Gurevych, I., Stein, B.: The eras and trends of automatic short answer grading. J. Artif. Intell. Educ. 25, 60\u2013117 (2015)","journal-title":"J. Artif. Intell. Educ."},{"key":"37_CR6","doi-asserted-by":"crossref","unstructured":"Chen, H., He, B.: Automated essay scoring by maximizing human-machine agreement. In: Proceedings of the 2013 Conference on Empirical Methods in Natural Language Processing, pp. 1741\u20131752 (2013)","DOI":"10.18653\/v1\/D13-1180"},{"key":"37_CR7","doi-asserted-by":"crossref","unstructured":"Chen, S., Lan, Y., Yuan, Z.: A multi-task automated assessment system for essay scoring. In: Proceedings of the International Conference on Artificial Intelligence in Education, pp. 276\u2013283 (2024)","DOI":"10.1007\/978-3-031-64299-9_22"},{"key":"37_CR8","doi-asserted-by":"crossref","unstructured":"Cozma, M., Butnaru, A., Ionescu, R.T.: Automated essay scoring with string kernels and word embeddings. In: Proceedings of the 56th Annual Meeting of the Association for Computational Linguistics, pp. 503\u2013509 (2018)","DOI":"10.18653\/v1\/P18-2080"},{"key":"37_CR9","doi-asserted-by":"crossref","unstructured":"Eckes, T.: Introduction to Many-Facet Rasch Measurement: Analyzing and Evaluating Rater-Mediated Assessments. Peter Lang Pub. Inc. (2023)","DOI":"10.3726\/b20875"},{"key":"37_CR10","doi-asserted-by":"crossref","unstructured":"ElMassry, A.M., Zaki, N., AlSheikh, N., Mediani, M.: A systematic review of pretrained models in automated essay scoring. IEEE Access (2025)","DOI":"10.1109\/ACCESS.2025.3584784"},{"issue":"3","key":"37_CR11","doi-asserted-by":"publisher","first-page":"317","DOI":"10.1177\/0265532210363144","volume":"27","author":"MK Enright","year":"2010","unstructured":"Enright, M.K., Quinlan, T.: Complementing human judgment of essays written by English language learners with e-rater scoring. Lang. Test. 27(3), 317\u2013334 (2010)","journal-title":"Lang. Test."},{"key":"37_CR12","doi-asserted-by":"crossref","unstructured":"Ke, Z., Ng, V.: Automated essay scoring: a survey of the state of the art. In: Proceedings of the International Joint Conference on Artificial Intelligence, pp. 6300\u20136308. (2019)","DOI":"10.24963\/ijcai.2019\/879"},{"key":"37_CR13","doi-asserted-by":"crossref","unstructured":"Lee, S., Cai, Y., Meng, D., Wang, Z., Wu, Y.: Unleashing large language models\u2019 proficiency in zero-shot essay scoring. In: Findings of the Association for Computational Linguistics, pp. 181\u2013198. (2024)","DOI":"10.18653\/v1\/2024.findings-emnlp.10"},{"key":"37_CR14","doi-asserted-by":"crossref","unstructured":"Mansour, W.A., Albatarni, S., Eltanbouly, S., Elsayed, T.: Can large language models automatically score proficiency of written essays? In: Proceedings of the 2024 Joint International Conference on Computational Linguistics, Language Resources and Evaluation, pp. 2777\u20132786 (2024)","DOI":"10.63317\/45fqsdi69jq5"},{"issue":"1","key":"37_CR15","doi-asserted-by":"publisher","first-page":"30","DOI":"10.1037\/1082-989X.1.1.30","volume":"1","author":"KO McGraw","year":"1996","unstructured":"McGraw, K.O., Wong, S.P.: Forming inferences about some intraclass correlation coefficients. Psychol. Methods 1(1), 30\u201346 (1996)","journal-title":"Psychol. Methods"},{"issue":"2","key":"37_CR16","doi-asserted-by":"publisher","first-page":"36","DOI":"10.1007\/s10462-024-11017-5","volume":"58","author":"H Misgna","year":"2024","unstructured":"Misgna, H., On, B.W., Lee, I., Choi, G.S.: A survey on deep learning-based automated essay scoring and feedback generation. Artif. Intell. Rev. 58(2), 36 (2024)","journal-title":"Artif. Intell. Rev."},{"issue":"3","key":"37_CR17","doi-asserted-by":"publisher","first-page":"2495","DOI":"10.1007\/s10462-021-10068-2","volume":"55","author":"D Ramesh","year":"2022","unstructured":"Ramesh, D., Sanampudi, S.K.: An automated essay scoring systems: a systematic literature review. Artif. Intell. Rev. 55(3), 2495\u20132527 (2022)","journal-title":"Artif. Intell. Rev."},{"issue":"1","key":"37_CR18","doi-asserted-by":"publisher","first-page":"25","DOI":"10.1016\/j.asw.2012.10.004","volume":"18","author":"C Ramineni","year":"2013","unstructured":"Ramineni, C., Williamson, D.M.: Automated essay scoring: psychometric guidelines and practices. Assess. Writ. 18(1), 25\u201339 (2013)","journal-title":"Assess. Writ."},{"key":"37_CR19","doi-asserted-by":"crossref","unstructured":"Shermis, M.D., Burstein, J.C.: Automated Essay Scoring: A Cross-disciplinary Perspective. Routledge (2003)","DOI":"10.4324\/9781410606860"},{"key":"37_CR20","doi-asserted-by":"publisher","first-page":"184792","DOI":"10.1109\/ACCESS.2025.3625589","volume":"13","author":"T Shibata","year":"2025","unstructured":"Shibata, T., Uto, M.: Cross-prompt automated essay scoring via reinforcement learning-based data valuation. IEEE Access 13, 184792\u2013184808 (2025)","journal-title":"IEEE Access"},{"key":"37_CR21","doi-asserted-by":"crossref","unstructured":"Taghipour, K., Ng, H.T.: A neural approach to automated essay scoring. In: Proceedings of the 2016 Conference on Empirical Methods in Natural Language Processing, pp. 1882\u20131891 (2016)","DOI":"10.18653\/v1\/D16-1193"},{"issue":"2","key":"37_CR22","doi-asserted-by":"publisher","first-page":"459","DOI":"10.1007\/s41237-021-00142-y","volume":"48","author":"M Uto","year":"2021","unstructured":"Uto, M.: A review of deep-neural automated essay scoring models. Behaviormetrika 48(2), 459\u2013484 (2021)","journal-title":"Behaviormetrika"},{"issue":"6","key":"37_CR23","doi-asserted-by":"publisher","first-page":"763","DOI":"10.1109\/TLT.2022.3145352","volume":"14","author":"M Uto","year":"2021","unstructured":"Uto, M., Okano, M.: Learning automated essay scoring models using item-response-theory-based scores to decrease effects of rater biases. IEEE Trans. Learn. Technol. 14(6), 763\u2013776 (2021)","journal-title":"IEEE Trans. Learn. Technol."},{"issue":"2","key":"37_CR24","doi-asserted-by":"publisher","first-page":"469","DOI":"10.1007\/s41237-020-00115-7","volume":"47","author":"M Uto","year":"2020","unstructured":"Uto, M., Ueno, M.: A generalized many-facet Rasch model and its Bayesian estimation using Hamiltonian Monte Carlo. Behaviormetrika 47(2), 469\u2013496 (2020)","journal-title":"Behaviormetrika"},{"issue":"1","key":"37_CR25","doi-asserted-by":"publisher","first-page":"27","DOI":"10.1080\/15305058.2017.1361426","volume":"18","author":"AA Wind","year":"2018","unstructured":"Wind, A.A., Wolfe, E.W., PeterFoltz, G.E., Jr., Rosenstein, M.: The influence of rater effects in training sets on the psychometric quality of automated scoring for writing assessments. Int. J. Test. 18(1), 27\u201349 (2018)","journal-title":"Int. J. Test."},{"key":"37_CR26","unstructured":"Xie, J., Cai, K., Kong, L., Zhou, J., Qu, W.: Automated essay scoring via pairwise contrastive regression. In: Proceedings of the 29th International Conference on Computational Linguistics, pp. 2724\u20132733 (2022)"},{"key":"37_CR27","doi-asserted-by":"crossref","unstructured":"Yamaura, M., Fukuda, I., Uto, M.: Neural automated essay scoring considering logical structure. In: Proceedings of the International Conference on Artificial Intelligence in Education, pp. 267\u2013278 (2023)","DOI":"10.1007\/978-3-031-36272-9_22"},{"key":"37_CR28","doi-asserted-by":"crossref","unstructured":"Yancey, K.P., Laflair, G., Verardi, A., Burstein, J.: Rating short L2 essays on the CEFR scale with GPT-4. In: Proceedings of the 18th Workshop on Innovative Use of NLP for Building Educational Applications, pp. 576\u2013584 (2023)","DOI":"10.18653\/v1\/2023.bea-1.49"}],"container-title":["Lecture Notes in Computer Science","Artificial Intelligence in Education"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-032-29744-0_37","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,6,26]],"date-time":"2026-06-26T12:39:41Z","timestamp":1782477581000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-3-032-29744-0_37"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026,6,27]]},"ISBN":["9783032297433","9783032297440"],"references-count":28,"URL":"https:\/\/doi.org\/10.1007\/978-3-032-29744-0_37","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"value":"0302-9743","type":"print"},{"value":"1611-3349","type":"electronic"}],"subject":[],"published":{"date-parts":[[2026,6,27]]},"assertion":[{"value":"27 June 2026","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"The authors have no competing interests to declare that are relevant to the content of this article.","order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Disclosure of Interests"}},{"value":"AIED","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"International Conference on Artificial Intelligence in Education","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Seoul","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Korea (Republic of)","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2026","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"29 June 2026","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"3 July 2026","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"27","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"aied2026","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"https:\/\/www.aied-conference.org\/2026","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}}]}}