{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,13]],"date-time":"2026-07-13T21:10:11Z","timestamp":1783977011780,"version":"3.55.0"},"reference-count":56,"publisher":"Elsevier BV","license":[{"start":{"date-parts":[[2026,6,1]],"date-time":"2026-06-01T00:00:00Z","timestamp":1780272000000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.elsevier.com\/tdm\/userlicense\/1.0\/"},{"start":{"date-parts":[[2026,6,1]],"date-time":"2026-06-01T00:00:00Z","timestamp":1780272000000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.elsevier.com\/legal\/tdmrep-license"},{"start":{"date-parts":[[2026,2,27]],"date-time":"2026-02-27T00:00:00Z","timestamp":1772150400000},"content-version":"vor","delay-in-days":0,"URL":"http:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"funder":[{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["62507024"],"award-info":[{"award-number":["62507024"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100003819","name":"Natural Science Foundation of Hubei Province","doi-asserted-by":"publisher","award":["2025AFB197"],"award-info":[{"award-number":["2025AFB197"]}],"id":[{"id":"10.13039\/501100003819","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100004733","name":"University of Macau","doi-asserted-by":"publisher","award":["SRG2025-00059-FED"],"award-info":[{"award-number":["SRG2025-00059-FED"]}],"id":[{"id":"10.13039\/501100004733","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["elsevier.com","sciencedirect.com"],"crossmark-restriction":true},"short-container-title":["Computers and Education: Artificial Intelligence"],"published-print":{"date-parts":[[2026,6]]},"DOI":"10.1016\/j.caeai.2026.100565","type":"journal-article","created":{"date-parts":[[2026,3,7]],"date-time":"2026-03-07T23:33:18Z","timestamp":1772926398000},"page":"100565","update-policy":"https:\/\/doi.org\/10.1016\/elsevier_cm_policy","source":"Crossref","is-referenced-by-count":0,"special_numbering":"C","title":["A framework for evaluation of large language models in essay assessment: Reliability, alignment, and causal reasoning"],"prefix":"10.1016","volume":"10","author":[{"ORCID":"https:\/\/orcid.org\/0009-0009-0532-1050","authenticated-orcid":false,"given":"Tongxi","family":"Liu","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-0094-5760","authenticated-orcid":false,"given":"Luyao","family":"Ye","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Wei","family":"Yan","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"78","reference":[{"issue":"1","key":"10.1016\/j.caeai.2026.100565_bib1","doi-asserted-by":"crossref","DOI":"10.1038\/s41598-025-87862-3","article-title":"An LLM-based hybrid approach for enhanced automated essay scoring","volume":"15","author":"Atkinson","year":"2025","journal-title":"Scientific Reports"},{"issue":"3","key":"10.1016\/j.caeai.2026.100565_bib2","article-title":"Automated essay scoring with e-rater\u00ae v.2","volume":"4","author":"Attali","year":"2006","journal-title":"The Journal of Technology, Learning, and Assessment"},{"issue":"1","key":"10.1016\/j.caeai.2026.100565_bib3","doi-asserted-by":"crossref","first-page":"289","DOI":"10.1111\/j.2517-6161.1995.tb02031.x","article-title":"Controlling the false discovery rate: A practical and powerful approach to multiple testing","volume":"57","author":"Benjamini","year":"1995","journal-title":"Journal of the Royal Statistical Society: Series B"},{"key":"10.1016\/j.caeai.2026.100565_bib4","series-title":"How to assess higher-order thinking skills in your classroom","author":"Brookhart","year":"2010"},{"key":"10.1016\/j.caeai.2026.100565_bib5","first-page":"5796","article-title":"Rationale behind essay scores: Enhancing S-LLM\u2019s multitrait essay scoring with rationale generated by LLMs","author":"Chu","year":"2025","journal-title":"Findings of the Association for Computational Linguistics: NAACL 2025"},{"issue":"1","key":"10.1016\/j.caeai.2026.100565_bib6","doi-asserted-by":"crossref","first-page":"100","DOI":"10.1016\/j.asw.2012.11.001","article-title":"Large-scale assessment, locally developed measures, and automated scoring of essays: Fishing for red herrings?","volume":"18","author":"Condon","year":"2013","journal-title":"Assessing Writing"},{"key":"10.1016\/j.caeai.2026.100565_bib7","series-title":"International conference on artificial intelligence in education","first-page":"84","article-title":"Automated summarization evaluation (ASE) using natural language processing tools","author":"Crossley","year":"2019"},{"key":"10.1016\/j.caeai.2026.100565_bib8","series-title":"Proceedings of the 16th International Conference on Educational Data Mining (EDM 2023)","first-page":"103","article-title":"Evaluating quadratic weighted kappa as the standard performance metric for automated essay scoring","author":"Doewes","year":"2023"},{"key":"10.1016\/j.caeai.2026.100565_bib9","series-title":"An introduction to the bootstrap","author":"Efron","year":"1994"},{"issue":"10","key":"10.1016\/j.caeai.2026.100565_bib10","doi-asserted-by":"crossref","first-page":"5683","DOI":"10.3390\/app15105683","article-title":"Large language model-powered automated assessment: A systematic review","volume":"15","author":"Emirtekin","year":"2025","journal-title":"Applied Sciences"},{"issue":"21","key":"10.1016\/j.caeai.2026.100565_bib11","doi-asserted-by":"crossref","first-page":"3416","DOI":"10.3390\/math12213416","article-title":"Hybrid approach to automated essay scoring: Integrating deep learning embeddings with handcrafted linguistic features for improved accuracy","volume":"12","author":"Faseeh","year":"2024","journal-title":"Mathematics"},{"key":"10.1016\/j.caeai.2026.100565_bib12","series-title":"Proceedings of the 15th international learning analytics and knowledge conference","first-page":"93","article-title":"Automatic short answer grading in the LLM era: Does GPT-4 with prompt engineering beat traditional models?","author":"Ferreira Mello","year":"2025"},{"key":"10.1016\/j.caeai.2026.100565_bib13","series-title":"Assessing the reliability and validity of large language models for automated assessment of student essays in higher education","author":"Gaggioli","year":"2025"},{"issue":"1","key":"10.1016\/j.caeai.2026.100565_bib14","doi-asserted-by":"crossref","first-page":"139","DOI":"10.1002\/aaai.12149","article-title":"Building trustworthy NeuroSymbolic AI systems: Consistency, reliability, explainability, and safety","volume":"45","author":"Gaur","year":"2024","journal-title":"AI Magazine"},{"key":"10.1016\/j.caeai.2026.100565_bib15","series-title":"Do repetitions matter? Strengthening reliability in LLM evaluations","author":"Gonzalez","year":"2025"},{"key":"10.1016\/j.caeai.2026.100565_bib16","series-title":"Proceedings of the 62nd annual meeting of the association for computational linguistics (volume 1: Long papers)","first-page":"13806","article-title":"LLM-rubric: A multidimensional, calibrated approach to automated evaluation of natural language texts","author":"Hashemi","year":"2024"},{"key":"10.1016\/j.caeai.2026.100565_bib17","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1016\/j.asw.2019.03.001","article-title":"Developing and examining validity evidence for the Writing Rubric to Inform Teacher Educators (WRITE)","volume":"40","author":"Hodges","year":"2019","journal-title":"Assessing Writing"},{"key":"10.1016\/j.caeai.2026.100565_bib18","series-title":"Improve LLM-based automatic essay scoring with linguistic features","author":"Hou","year":"2025"},{"key":"10.1016\/j.caeai.2026.100565_bib19","doi-asserted-by":"crossref","DOI":"10.7717\/peerj-cs.208","article-title":"Automated language essay scoring systems: A literature review","volume":"5","author":"Hussein","year":"2019","journal-title":"PeerJ Computer Science"},{"key":"10.1016\/j.caeai.2026.100565_bib20","series-title":"Causal inference in statistics, social, and biomedical sciences","author":"Imbens","year":"2015"},{"issue":"3","key":"10.1016\/j.caeai.2026.100565_bib21","first-page":"275","article-title":"Comparison of dimension reduction methods for automated essay grading","volume":"11","author":"Kakkonen","year":"2008","journal-title":"Journal of Educational Technology & Society"},{"issue":"2","key":"10.1016\/j.caeai.2026.100565_bib22","doi-asserted-by":"crossref","first-page":"155","DOI":"10.1016\/j.jcm.2016.02.012","article-title":"A guideline of selecting and reporting intraclass correlation coefficients for reliability research","volume":"15","author":"Koo","year":"2016","journal-title":"Journal of Chiropractic Medicine"},{"issue":"1","key":"10.1016\/j.caeai.2026.100565_bib23","doi-asserted-by":"crossref","first-page":"143","DOI":"10.1609\/aaaiss.v3i1.31193","article-title":"LLMs in automated essay evaluation: A case study","volume":"3","author":"Kostic","year":"2024","journal-title":"Proceedings of the AAAI Symposium Series"},{"key":"10.1016\/j.caeai.2026.100565_bib24","series-title":"Are large language models good essay graders?","author":"Kundu","year":"2024"},{"issue":"1","key":"10.1016\/j.caeai.2026.100565_bib25","first-page":"1","article-title":"Applying large language models for automated essay scoring for non-native Japanese","volume":"11","author":"Li","year":"2024","journal-title":"Humanities and Social Sciences Communications"},{"issue":"3","key":"10.1016\/j.caeai.2026.100565_bib26","doi-asserted-by":"crossref","first-page":"1875","DOI":"10.47836\/pjst.29.3.27","article-title":"A comprehensive review of automated essay scoring (AES) research and development","volume":"29","author":"Lim","year":"2021","journal-title":"Pertanika Journal of Science & Technology"},{"key":"10.1016\/j.caeai.2026.100565_bib27","series-title":"Findings of the Association for Computational Linguistics: NAACL 2025","first-page":"4427","article-title":"A closer look into mixture-of-experts in large language models","author":"Lo","year":"2025"},{"key":"10.1016\/j.caeai.2026.100565_bib28","series-title":"Coming to terms: A theory of writing assessment","author":"Lynne","year":"2004"},{"issue":"1","key":"10.1016\/j.caeai.2026.100565_bib29","doi-asserted-by":"crossref","first-page":"30","DOI":"10.1037\/1082-989X.1.1.30","article-title":"Forming inferences about some intraclass correlation coefficients","volume":"1","author":"McGraw","year":"1996","journal-title":"Psychological Methods"},{"issue":"2","key":"10.1016\/j.caeai.2026.100565_bib30","doi-asserted-by":"crossref","first-page":"36","DOI":"10.1007\/s10462-024-11017-5","article-title":"A survey on deep learning-based automated essay scoring and feedback generation","volume":"58","author":"Misgna","year":"2024","journal-title":"Artificial Intelligence Review"},{"issue":"2","key":"10.1016\/j.caeai.2026.100565_bib31","doi-asserted-by":"crossref","DOI":"10.1016\/j.rmal.2023.100050","article-title":"Exploring the potential of using an AI language model for automated essay scoring","volume":"2","author":"Mizumoto","year":"2023","journal-title":"Research Methods in Applied Linguistics"},{"key":"10.1016\/j.caeai.2026.100565_bib32","series-title":"RDBE: Reasoning distillation-based evaluation enhances automatic essay scoring","author":"Mohammadkhani","year":"2024"},{"key":"10.1016\/j.caeai.2026.100565_bib33","series-title":"Proceedings of the 20th workshop on innovative use of NLP for building educational applications","first-page":"477","article-title":"Advances in auto-grading with large language models: A cross-disciplinary survey","author":"Nkoyo","year":"2025"},{"key":"10.1016\/j.caeai.2026.100565_bib34","doi-asserted-by":"crossref","first-page":"27730","DOI":"10.52202\/068431-2011","article-title":"Training language models to follow instructions with human feedback","volume":"35","author":"Ouyang","year":"2022","journal-title":"Advances in Neural Information Processing Systems"},{"key":"10.1016\/j.caeai.2026.100565_bib35","article-title":"Large language models and automated essay scoring of English language learner writing: Insights into validity and reliability","volume":"6","author":"Pack","year":"2024","journal-title":"Computers and Education: Artificial Intelligence"},{"key":"10.1016\/j.caeai.2026.100565_bib36","series-title":"Elements of causal inference: Foundations and learning algorithms","author":"Peters","year":"2017"},{"issue":"4","key":"10.1016\/j.caeai.2026.100565_bib37","doi-asserted-by":"crossref","DOI":"10.30935\/cedtech\/13740","article-title":"Exploring effective methods for automated essay scoring of non-native speakers","volume":"15","author":"Poonpon","year":"2023","journal-title":"Contemporary Educational Technology"},{"issue":"1","key":"10.1016\/j.caeai.2026.100565_bib38","doi-asserted-by":"crossref","first-page":"962","DOI":"10.1186\/s12909-024-05881-6","article-title":"Reliability of ChatGPT in automated essay scoring for dental undergraduate examinations","volume":"24","author":"Quah","year":"2024","journal-title":"BMC Medical Education"},{"issue":"4","key":"10.1016\/j.caeai.2026.100565_bib39","article-title":"An evaluation of IntelliMetric\u2122 essay scoring system","volume":"4","author":"Rudner","year":"2006","journal-title":"The Journal of Technology, Learning, and Assessment"},{"key":"10.1016\/j.caeai.2026.100565_bib40","series-title":"Proceedings of the 19th Workshop on Innovative Use of NLP for Building Educational Applications (BEA 2024)","first-page":"210","article-title":"Fairness in automated essay scoring: A comparative analysis of algorithms on German learner essays from secondary education","author":"Schaller","year":"2024"},{"key":"10.1016\/j.caeai.2026.100565_bib41","series-title":"Automated essay scoring: A cross-disciplinary perspective","author":"Shermis","year":"2003"},{"key":"10.1016\/j.caeai.2026.100565_bib42","series-title":"Exploring LLM autoscoring reliability in large-scale writing assessments using generalizability theory","author":"Song","year":"2025"},{"key":"10.1016\/j.caeai.2026.100565_bib43","series-title":"Proceedings of the 2025 conference of the north American chapter of the Association for computational linguistics: Human Language technologies (volume 1: Long papers)","first-page":"4195","article-title":"The good, the bad, and the greedy: Evaluation of LLMs should not ignore non-determinism","author":"Song","year":"2025"},{"key":"10.1016\/j.caeai.2026.100565_bib44","series-title":"Causation, prediction, and search","author":"Spirtes","year":"2000"},{"key":"10.1016\/j.caeai.2026.100565_bib45","series-title":"Proceedings of the 19th Workshop on Innovative Use of NLP for Building Educational Applications (BEA 2024)","first-page":"283","article-title":"Exploring LLM prompting strategies for joint essay scoring and feedback generation","author":"Stahl","year":"2024"},{"issue":"17","key":"10.1016\/j.caeai.2026.100565_bib46","doi-asserted-by":"crossref","DOI":"10.3390\/math13172828","article-title":"A comprehensive review on automated grading systems in STEM using AI techniques","volume":"13","author":"Tan","year":"2025","journal-title":"Mathematics"},{"issue":"14","key":"10.1016\/j.caeai.2026.100565_bib47","doi-asserted-by":"crossref","DOI":"10.1016\/j.heliyon.2024.e34262","article-title":"Harnessing LLMs for multi-dimensional writing assessment: Reliability and alignment with human judgments","volume":"10","author":"Tang","year":"2024","journal-title":"Heliyon"},{"issue":"10","key":"10.1016\/j.caeai.2026.100565_bib48","doi-asserted-by":"crossref","first-page":"4182","DOI":"10.3390\/app14104182","article-title":"Incorporating fine-grained linguistic features and explainable AI into multi-dimensional automated writing assessment","volume":"14","author":"Tang","year":"2024","journal-title":"Applied Sciences"},{"key":"10.1016\/j.caeai.2026.100565_bib49","series-title":"Gemini: A family of highly capable multimodal models","author":"Team Gemini","year":"2023"},{"key":"10.1016\/j.caeai.2026.100565_bib50","series-title":"LLaMA: Open and efficient foundation language models","author":"Touvron","year":"2023"},{"issue":"1","key":"10.1016\/j.caeai.2026.100565_bib51","doi-asserted-by":"crossref","first-page":"79","DOI":"10.1007\/s40593-017-0142-3","article-title":"Automated assessment of non-native learner essays: Investigating the role of linguistic features","volume":"28","author":"Vajjala","year":"2018","journal-title":"International Journal of Artificial Intelligence in Education"},{"key":"10.1016\/j.caeai.2026.100565_bib52","series-title":"Effective grading: A tool for learning and assessment in college","author":"Walvoord","year":"2011"},{"key":"10.1016\/j.caeai.2026.100565_bib53","series-title":"Findings of the Association for Computational Linguistics: EMNLP 2024","first-page":"8906","article-title":"Beyond agreement: Diagnosing the rationale alignment of automated essay scoring methods based on linguistically informed counterfactuals","author":"Wang","year":"2024"},{"key":"10.1016\/j.caeai.2026.100565_bib54","article-title":"Evaluating large language models as raters in large-scale writing assessments: A psychometric framework for reliability and validity","volume":"9","author":"Wang","year":"2025","journal-title":"Computers and Education: Artificial Intelligence"},{"key":"10.1016\/j.caeai.2026.100565_bib55","series-title":"International conference on artificial intelligence in education","first-page":"60","article-title":"Do we need a detailed rubric for automated essay scoring using large language models?","author":"Yoshida","year":"2025"},{"key":"10.1016\/j.caeai.2026.100565_bib56","series-title":"Findings of the Association for Computational Linguistics: EMNLP 2024","first-page":"1950","article-title":"ProSA: Assessing and understanding the prompt sensitivity of LLMs","author":"Zhuo","year":"2024"}],"container-title":["Computers and Education: Artificial Intelligence"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/api.elsevier.com\/content\/article\/PII:S2666920X26000275?httpAccept=text\/xml","content-type":"text\/xml","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/api.elsevier.com\/content\/article\/PII:S2666920X26000275?httpAccept=text\/plain","content-type":"text\/plain","content-version":"vor","intended-application":"text-mining"}],"deposited":{"date-parts":[[2026,7,13]],"date-time":"2026-07-13T20:15:52Z","timestamp":1783973752000},"score":1,"resource":{"primary":{"URL":"https:\/\/linkinghub.elsevier.com\/retrieve\/pii\/S2666920X26000275"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026,6]]},"references-count":56,"alternative-id":["S2666920X26000275"],"URL":"https:\/\/doi.org\/10.1016\/j.caeai.2026.100565","relation":{},"ISSN":["2666-920X"],"issn-type":[{"value":"2666-920X","type":"print"}],"subject":[],"published":{"date-parts":[[2026,6]]},"assertion":[{"value":"Elsevier","name":"publisher","label":"This article is maintained by"},{"value":"A framework for evaluation of large language models in essay assessment: Reliability, alignment, and causal reasoning","name":"articletitle","label":"Article Title"},{"value":"Computers and Education: Artificial Intelligence","name":"journaltitle","label":"Journal Title"},{"value":"https:\/\/doi.org\/10.1016\/j.caeai.2026.100565","name":"articlelink","label":"CrossRef DOI link to publisher maintained version"},{"value":"article","name":"content_type","label":"Content Type"},{"value":"\u00a9 2026 The Authors. Published by Elsevier Ltd.","name":"copyright","label":"Copyright"}],"article-number":"100565"}}