{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,3,13]],"date-time":"2026-03-13T01:11:04Z","timestamp":1773364264454,"version":"3.50.1"},"reference-count":64,"publisher":"Elsevier BV","license":[{"start":{"date-parts":[[2025,12,1]],"date-time":"2025-12-01T00:00:00Z","timestamp":1764547200000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.elsevier.com\/tdm\/userlicense\/1.0\/"},{"start":{"date-parts":[[2025,12,1]],"date-time":"2025-12-01T00:00:00Z","timestamp":1764547200000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.elsevier.com\/legal\/tdmrep-license"},{"start":{"date-parts":[[2025,9,24]],"date-time":"2025-09-24T00:00:00Z","timestamp":1758672000000},"content-version":"vor","delay-in-days":0,"URL":"http:\/\/creativecommons.org\/licenses\/by-nc\/4.0\/"}],"funder":[{"DOI":"10.13039\/501100002703","name":"Jiangsu University","doi-asserted-by":"publisher","id":[{"id":"10.13039\/501100002703","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["elsevier.com","sciencedirect.com"],"crossmark-restriction":true},"short-container-title":["Computers and Education: Artificial Intelligence"],"published-print":{"date-parts":[[2025,12]]},"DOI":"10.1016\/j.caeai.2025.100481","type":"journal-article","created":{"date-parts":[[2025,9,25]],"date-time":"2025-09-25T08:51:22Z","timestamp":1758790282000},"page":"100481","update-policy":"https:\/\/doi.org\/10.1016\/elsevier_cm_policy","source":"Crossref","is-referenced-by-count":4,"special_numbering":"C","title":["Evaluating large language models as raters in large-scale writing assessments: A psychometric framework for reliability and validity"],"prefix":"10.1016","volume":"9","author":[{"given":"Yuehan","family":"Wang","sequence":"first","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0003-1641-6580","authenticated-orcid":false,"given":"Jinyan","family":"Huang","sequence":"additional","affiliation":[]},{"given":"Lun","family":"Du","sequence":"additional","affiliation":[]},{"given":"Yuxin","family":"Guo","sequence":"additional","affiliation":[]},{"given":"Ying","family":"Liu","sequence":"additional","affiliation":[]},{"given":"Rong","family":"Wang","sequence":"additional","affiliation":[]}],"member":"78","reference":[{"key":"10.1016\/j.caeai.2025.100481_bib1","first-page":"215","article-title":"Sample size for estimation of g and phi coefficients in generalizability theory","volume":"51","author":"Atilgan","year":"2013","journal-title":"European Journal of Educational Research"},{"key":"10.1016\/j.caeai.2025.100481_bib2","series-title":"Proceedings of the 2024 ACM conference on fairness, accountability, and transparency","article-title":"A critical analysis of the largest source for generative ai training data: Common crawl","author":"Baack","year":"2024"},{"key":"10.1016\/j.caeai.2025.100481_bib3","author":"Ben Hamner"},{"key":"10.1016\/j.caeai.2025.100481_bib4","series-title":"Explainability and semantics-bridging natural language flexibility and formal precision: Toward a semantic framework for Large Language Models","author":"Bilokon","year":"2024"},{"issue":"11","key":"10.1016\/j.caeai.2025.100481_bib5","doi-asserted-by":"crossref","first-page":"960","DOI":"10.3109\/0142159X.2012.703791","article-title":"Generalizability theory for the perplexed: A practical introduction and guide: AMEE guide No. 68","volume":"34","author":"Bloch","year":"2012","journal-title":"Medical Teacher"},{"issue":"4","key":"10.1016\/j.caeai.2025.100481_bib6","doi-asserted-by":"crossref","first-page":"27","DOI":"10.1111\/j.1745-3992.1992.tb00260.x","article-title":"Generalizability theory","volume":"11","author":"Brennan","year":"1992","journal-title":"Educational Measurement: Issues and Practice"},{"key":"10.1016\/j.caeai.2025.100481_bib7","series-title":"Findings of the association for computational linguistics: NAACL 2025","article-title":"Beyond excess and deficiency: Adaptive length bias mitigation in reward models for RLHF","author":"Bu","year":"2025"},{"key":"10.1016\/j.caeai.2025.100481_bib8","first-page":"1","article-title":"ChatGPT as an automated essay scoring tool in the writing classrooms: How it compares with human scoring","author":"Bui","year":"2024","journal-title":"Education and Information Technologies"},{"key":"10.1016\/j.caeai.2025.100481_bib9","series-title":"Proceedings of the 32nd ACM international conference on information and knowledge management","article-title":"Hallucination detection: Robustly discerning reliable answers in large language models","author":"Chen","year":"2023"},{"key":"10.1016\/j.caeai.2025.100481_bib10","doi-asserted-by":"crossref","first-page":"1058","DOI":"10.1162\/opmi_a_00160","article-title":"The limitations of large language models for understanding human language and cognition","volume":"8","author":"Cuskley","year":"2024","journal-title":"Open Mind"},{"key":"10.1016\/j.caeai.2025.100481_bib11","series-title":"The theory and practice of item response theory","author":"De Ayala","year":"2013"},{"issue":"4","key":"10.1016\/j.caeai.2025.100481_bib12","doi-asserted-by":"crossref","first-page":"997","DOI":"10.1177\/08944393231220483","article-title":"Performing an inductive thematic analysis of semi-structured interviews with a large language model: An exploration and provocation on the limits of the approach","volume":"42","author":"De Paoli","year":"2024","journal-title":"Social Science Computer Review"},{"issue":"2","key":"10.1016\/j.caeai.2025.100481_bib13","doi-asserted-by":"crossref","first-page":"93","DOI":"10.1111\/j.1745-3984.1994.tb00436.x","article-title":"Examining rater errors in the assessment of written composition with a many\u2010faceted Rasch model","volume":"31","author":"Engelhard","year":"1994","journal-title":"Journal of Educational Measurement"},{"issue":"1","key":"10.1016\/j.caeai.2025.100481_bib14","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1002\/ets2.12147","article-title":"Approaches to automated scoring of speaking for K\u201312 English language proficiency assessments","volume":"2017","author":"Evanini","year":"2017","journal-title":"ETS Research Report Series"},{"issue":"1","key":"10.1016\/j.caeai.2025.100481_bib15","doi-asserted-by":"crossref","first-page":"201","DOI":"10.1002\/berj.4069","article-title":"Grading exams using large language models: A comparison between human and AI grading of exams in higher education using ChatGPT","volume":"51","author":"Flod\u00e9n","year":"2025","journal-title":"British Educational Research Journal"},{"issue":"3","key":"10.1016\/j.caeai.2025.100481_bib16","doi-asserted-by":"crossref","first-page":"1097","DOI":"10.1162\/coli_a_00524","article-title":"Bias and fairness in large language models: A survey","volume":"50","author":"Gallegos","year":"2024","journal-title":"Computational Linguistics"},{"key":"10.1016\/j.caeai.2025.100481_bib17","doi-asserted-by":"crossref","first-page":"21","DOI":"10.1016\/j.asw.2016.07.004","article-title":"A Many-Facet Rasch analysis comparing essay rater behavior on an academic English reading\/writing test used for two purposes","volume":"30","author":"Goodwin","year":"2016","journal-title":"Assessing Writing"},{"issue":"1\u20132","key":"10.1016\/j.caeai.2025.100481_bib18","first-page":"7","article-title":"Validity and reliability issues in the direct assessment of writing","volume":"16","author":"Greenberg","year":"1992","journal-title":"WPA: Writing Program Administration"},{"issue":"2","key":"10.1016\/j.caeai.2025.100481_bib19","doi-asserted-by":"crossref","first-page":"16","DOI":"10.1111\/emip.12602","article-title":"Transforming assessment: The impacts and implications of large language models and generative AI","volume":"43","author":"Hao","year":"2024","journal-title":"Educational Measurement: Issues and Practice"},{"issue":"1","key":"10.1016\/j.caeai.2025.100481_bib20","doi-asserted-by":"crossref","first-page":"20","DOI":"10.1186\/s40468-022-00168-3","article-title":"Raters' perceptions of rating scales criteria and its effect on the process and outcome of their rating","volume":"12","author":"Heidari","year":"2022","journal-title":"Language Testing in Asia"},{"key":"10.1016\/j.caeai.2025.100481_bib21","series-title":"Proceedings of the eleventh ACM conference on learning@ scale","article-title":"Can large language models make the grade? An empirical study evaluating llms ability to mark short answer questions in k-12 education","author":"Henkel","year":"2024"},{"issue":"3","key":"10.1016\/j.caeai.2025.100481_bib22","doi-asserted-by":"crossref","first-page":"123","DOI":"10.1016\/j.asw.2011.12.003","article-title":"Using generalizability theory to examine the accuracy and validity of large-scale ESL writing assessment","volume":"17","author":"Huang","year":"2012","journal-title":"Assessing Writing"},{"key":"10.1016\/j.caeai.2025.100481_bib23","series-title":"An empirical study of llm-as-a-judge for llm evaluation: Fine-tuned judge models are task-specific classifiers","author":"Huang","year":"2024"},{"key":"10.1016\/j.caeai.2025.100481_bib24","series-title":"Proceedings of the 2024 ACM conference on fairness, accountability, and transparency","article-title":"Collective constitutional ai: Aligning a language model with public input","author":"Huang","year":"2024"},{"issue":"1","key":"10.1016\/j.caeai.2025.100481_bib25","doi-asserted-by":"crossref","first-page":"47","DOI":"10.1515\/ip-2012-0003","article-title":"Levels of pragmatic competence in an EFL academic context: A tool for assessment","volume":"9","author":"Ifantidou","year":"2012","journal-title":"Intercultural Pragmatics"},{"key":"10.1016\/j.caeai.2025.100481_bib26","series-title":"Large language models as partners in student essay evaluation","author":"Ishida","year":"2024"},{"key":"10.1016\/j.caeai.2025.100481_bib27","first-page":"113","article-title":"Measuring essay assessment: Intra-rater and inter-rater reliability","volume":"57","author":"Kayap\u0131nar","year":"2014","journal-title":"European Journal of Educational Research"},{"issue":"3","key":"10.1016\/j.caeai.2025.100481_bib28","first-page":"181","article-title":"Using generalizability theory software suite: GENOVA, urGENOVA, and mGENOVA","volume":"20","author":"Kim","year":"2022","journal-title":"Measurement: Interdisciplinary Research and Perspectives"},{"issue":"6","key":"10.1016\/j.caeai.2025.100481_bib29","doi-asserted-by":"crossref","first-page":"2971","DOI":"10.3390\/app15062971","article-title":"Evaluating creativity: Can LLMs Be good evaluators in creative writing tasks?","volume":"15","author":"Kim","year":"2025","journal-title":"Applied Sciences"},{"key":"10.1016\/j.caeai.2025.100481_bib30","series-title":"Understanding the effects of rlhf on llm generalisation and diversity","author":"Kirk","year":"2023"},{"key":"10.1016\/j.caeai.2025.100481_bib31","article-title":"C-LLM: Learn to check Chinese spelling errors character by character","author":"Li","year":"2024","journal-title":"arXiv preprint arXiv:2406.16536"},{"key":"10.1016\/j.caeai.2025.100481_bib32","article-title":"On the (in) effectiveness of large language models for Chinese text correction","author":"Li","year":"2023","journal-title":"arXiv preprint arXiv:2307.09007"},{"issue":"1","key":"10.1016\/j.caeai.2025.100481_bib33","first-page":"1","article-title":"Evaluating the role of ChatGPT in enhancing EFL writing assessments in classroom settings: A preliminary investigation","volume":"11","author":"Li","year":"2024","journal-title":"Humanities and Social Sciences Communications"},{"key":"10.1016\/j.caeai.2025.100481_bib34","series-title":"Many-faceted Rasch measurement","author":"Linacre","year":"1989"},{"key":"10.1016\/j.caeai.2025.100481_bib35","series-title":"Facets computer program for many-facet Rasch measurement, version 3.80. 0. Beaverton, Oregon: Winsteps. Com","author":"Linacre","year":"2017"},{"key":"10.1016\/j.caeai.2025.100481_bib36","series-title":"Proceedings of the 18th conference of the European chapter of the association for computational linguistics (Volume 1: Long papers)","article-title":"LLM comparative assessment: Zero-shot NLG evaluation through pairwise comparisons using Large Language Models","author":"Liusie","year":"2024"},{"key":"10.1016\/j.caeai.2025.100481_bib37","first-page":"1","article-title":"Human versus machine: The effectiveness of ChatGPT in automated essay scoring","author":"Manning","year":"2025","journal-title":"Innovations in Education & Teaching International"},{"key":"10.1016\/j.caeai.2025.100481_bib38","article-title":"The impact of rater background on score variability and reliability of EFL writing classroom assessment: A generalizability theory approach","volume":"10","author":"Mao","year":"2021","journal-title":"International Journal of TESOL and Learning"},{"issue":"2","key":"10.1016\/j.caeai.2025.100481_bib39","doi-asserted-by":"crossref","DOI":"10.1016\/j.rmal.2023.100050","article-title":"Exploring the potential of using an AI language model for automated essay scoring","volume":"2","author":"Mizumoto","year":"2023","journal-title":"Research Methods in Applied Linguistics"},{"issue":"2","key":"10.1016\/j.caeai.2025.100481_bib40","first-page":"189","article-title":"Detecting and measuring rater effects using many-facet Rasch measurement: Part II","volume":"5","author":"Myford","year":"2004","journal-title":"Journal of Applied Measurement"},{"key":"10.1016\/j.caeai.2025.100481_bib41","article-title":"Large language models and automated essay scoring of English language learner writing: Insights into validity and reliability","volume":"6","author":"Pack","year":"2024","journal-title":"Computers and Education: Artificial Intelligence"},{"key":"10.1016\/j.caeai.2025.100481_bib42","series-title":"Proceedings of the 51st annual meeting of the association for computational linguistics (volume 1: Long papers)","article-title":"Modeling thesis clarity in student essays","author":"Persing","year":"2013"},{"key":"10.1016\/j.caeai.2025.100481_bib43","series-title":"Better call claude: Can LLMs detect changes of writing style?","author":"R\u00f6misch","year":"2025"},{"issue":"3","key":"10.1016\/j.caeai.2025.100481_bib44","doi-asserted-by":"crossref","first-page":"2495","DOI":"10.1007\/s10462-021-10068-2","article-title":"An automated essay scoring systems: A systematic literature review","volume":"55","author":"Ramesh","year":"2022","journal-title":"Artificial Intelligence Review"},{"issue":"4","key":"10.1016\/j.caeai.2025.100481_bib45","doi-asserted-by":"crossref","first-page":"679","DOI":"10.1177\/02655322221086211","article-title":"Test review: The international English language testing system (IELTS)","volume":"39","author":"Read","year":"2022","journal-title":"Language Testing"},{"issue":"1","key":"10.1016\/j.caeai.2025.100481_bib46","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1002\/ets2.12341","article-title":"Designing efficient L2 writing assessment tasks for low\u2010proficiency learners of English","volume":"2021","author":"Sasayama","year":"2021","journal-title":"ETS Research Report Series"},{"key":"10.1016\/j.caeai.2025.100481_bib47","series-title":"Proceedings of the 15th international learning analytics and Knowledge conference","article-title":"Can AI grade your essays? A comparative analysis of large language models and teacher ratings in multidimensional essay scoring","author":"Se\u00dfler","year":"2025"},{"key":"10.1016\/j.caeai.2025.100481_bib48","series-title":"Llm-as-a-judge & reward model: What they can and cannot do","author":"Son","year":"2024"},{"key":"10.1016\/j.caeai.2025.100481_bib49","article-title":"Beyond human bias? The halo effect paradox and reversal in product design evaluation by Large Language Models","author":"Song","year":"2025","journal-title":"SSRN"},{"key":"10.1016\/j.caeai.2025.100481_bib50","first-page":"1","article-title":"Testing theory of mind in large language models and humans","author":"Strachan","year":"2024","journal-title":"Nature Human Behaviour"},{"key":"10.1016\/j.caeai.2025.100481_bib51","series-title":"Large language models are inconsistent and biased evaluators","author":"Stureborg","year":"2024"},{"key":"10.1016\/j.caeai.2025.100481_bib52","series-title":"Proceedings of the 2016 conference on empirical methods in natural language processing","article-title":"A neural approach to automated essay scoring","author":"Taghipour","year":"2016"},{"key":"10.1016\/j.caeai.2025.100481_bib53","article-title":"Can AI provide useful holistic essay scoring?","volume":"7","author":"Tate","year":"2024","journal-title":"Computers and Education: Artificial Intelligence"},{"key":"10.1016\/j.caeai.2025.100481_bib54","doi-asserted-by":"crossref","first-page":"291","DOI":"10.1007\/s10459-012-9370-3","article-title":"Exploring the impact of mental workload on rater-based assessments","volume":"18","author":"Tavares","year":"2013","journal-title":"Advances in Health Sciences Education"},{"issue":"6","key":"10.1016\/j.caeai.2025.100481_bib55","doi-asserted-by":"crossref","first-page":"308","DOI":"10.1080\/14739879.2014.11730760","article-title":"Impact of rating demands on rater-based assessments of clinical competence","volume":"25","author":"Tavares","year":"2014","journal-title":"Education for Primary Care"},{"issue":"1","key":"10.1016\/j.caeai.2025.100481_bib56","first-page":"81","article-title":"Black boxes revisited: Understanding GenAI responses to student writing across the curriculum","volume":"48","author":"Velez","year":"2025","journal-title":"Thresholds in Education"},{"issue":"1","key":"10.1016\/j.caeai.2025.100481_bib57","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1002\/ets2.12005","article-title":"Monitoring of scoring using the e\u2010rater\u00ae automated scoring system and human raters on a writing test","volume":"2014","author":"Wang","year":"2014","journal-title":"ETS Research Report Series"},{"key":"10.1016\/j.caeai.2025.100481_bib58","series-title":"Empirical study of large language models as automated essay scoring tools in English composition_taking toefl independent writing task for example","author":"Xia","year":"2024"},{"issue":"3","key":"10.1016\/j.caeai.2025.100481_bib59","doi-asserted-by":"crossref","DOI":"10.1016\/j.rmal.2024.100133","article-title":"An application of many-facet Rasch measurement to evaluate automated essay scoring: A case of ChatGPT-4.0","volume":"3","author":"Yamashita","year":"2024","journal-title":"Research Methods in Applied Linguistics"},{"issue":"1","key":"10.1016\/j.caeai.2025.100481_bib60","doi-asserted-by":"crossref","first-page":"150","DOI":"10.1111\/bjet.13494","article-title":"Utilizing large language models for EFL essay grading: An examination of reliability and validity in rubric\u2010based assessments","volume":"56","author":"Yavuz","year":"2025","journal-title":"British Journal of Educational Technology"},{"key":"10.1016\/j.caeai.2025.100481_bib61","article-title":"Large language model as attributed training data generator: A tale of diversity and bias","volume":"36","author":"Yu","year":"2024","journal-title":"Advances in Neural Information Processing Systems"},{"key":"10.1016\/j.caeai.2025.100481_bib62","first-page":"46595","article-title":"Judging llm-as-a-judge with mt-bench and chatbot arena","volume":"36","author":"Zheng","year":"2023","journal-title":"Advances in Neural Information Processing Systems"},{"key":"10.1016\/j.caeai.2025.100481_bib63","series-title":"Large language models are not robust multiple choice selectors. The Twelfth International Conference on Learning Representations","author":"Zheng","year":"2023"},{"issue":"1","key":"10.1016\/j.caeai.2025.100481_bib64","first-page":"62","article-title":"Automated versus human essay scoring: A comparative study","volume":"5","author":"Zribi","year":"2021","journal-title":"International Journal of Information Technology and Language Studies (IJITLS)"}],"container-title":["Computers and Education: Artificial Intelligence"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/api.elsevier.com\/content\/article\/PII:S2666920X25001213?httpAccept=text\/xml","content-type":"text\/xml","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/api.elsevier.com\/content\/article\/PII:S2666920X25001213?httpAccept=text\/plain","content-type":"text\/plain","content-version":"vor","intended-application":"text-mining"}],"deposited":{"date-parts":[[2025,12,17]],"date-time":"2025-12-17T10:03:11Z","timestamp":1765965791000},"score":1,"resource":{"primary":{"URL":"https:\/\/linkinghub.elsevier.com\/retrieve\/pii\/S2666920X25001213"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,12]]},"references-count":64,"alternative-id":["S2666920X25001213"],"URL":"https:\/\/doi.org\/10.1016\/j.caeai.2025.100481","relation":{},"ISSN":["2666-920X"],"issn-type":[{"value":"2666-920X","type":"print"}],"subject":[],"published":{"date-parts":[[2025,12]]},"assertion":[{"value":"Elsevier","name":"publisher","label":"This article is maintained by"},{"value":"Evaluating large language models as raters in large-scale writing assessments: A psychometric framework for reliability and validity","name":"articletitle","label":"Article Title"},{"value":"Computers and Education: Artificial Intelligence","name":"journaltitle","label":"Journal Title"},{"value":"https:\/\/doi.org\/10.1016\/j.caeai.2025.100481","name":"articlelink","label":"CrossRef DOI link to publisher maintained version"},{"value":"article","name":"content_type","label":"Content Type"},{"value":"\u00a9 2025 The Authors. Published by Elsevier Ltd.","name":"copyright","label":"Copyright"}],"article-number":"100481"}}