{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,6,26]],"date-time":"2026-06-26T13:46:49Z","timestamp":1782481609322,"version":"3.54.5"},"publisher-location":"Cham","reference-count":36,"publisher":"Springer Nature Switzerland","isbn-type":[{"value":"9783032297433","type":"print"},{"value":"9783032297440","type":"electronic"}],"license":[{"start":{"date-parts":[[2026,6,27]],"date-time":"2026-06-27T00:00:00Z","timestamp":1782518400000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2026,6,27]],"date-time":"2026-06-27T00:00:00Z","timestamp":1782518400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2027]]},"DOI":"10.1007\/978-3-032-29744-0_26","type":"book-chapter","created":{"date-parts":[[2026,6,26]],"date-time":"2026-06-26T12:54:04Z","timestamp":1782478444000},"page":"385-399","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":0,"title":["When Can We Trust LLM Graders? Calibrating Confidence for\u00a0Automated Assessment"],"prefix":"10.1007","author":[{"ORCID":"https:\/\/orcid.org\/0009-0003-7829-327X","authenticated-orcid":false,"given":"Robinson","family":"Vasquez Ferrer","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-8635-7198","authenticated-orcid":false,"given":"Damla","family":"Turgut","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-3324-5330","authenticated-orcid":false,"given":"Zhongzhou","family":"Chen","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-4409-1341","authenticated-orcid":false,"given":"Shashank","family":"Sonkar","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2026,6,27]]},"reference":[{"key":"26_CR1","doi-asserted-by":"crossref","unstructured":"Benjamini, Y., Hochberg, Y.: Controlling the false discovery rate: a practical and powerful approach to multiple testing. J. Royal Stat. Soc. Series B (Methodological) 57(1), 289\u2013300 (1995)","DOI":"10.1111\/j.2517-6161.1995.tb02031.x"},{"key":"26_CR2","doi-asserted-by":"crossref","unstructured":"Brier, G.W.: Verification of forecasts expressed in terms of probability. Mon. Weather Rev. 78(1), 1\u20133 (1950). https:\/\/doi.org\/10.1175\/1520-0493(1950)078$$<$$0001:VOFEIT$$>$$2.0.CO;2","DOI":"10.1175\/1520-0493(1950)078<0001:VOFEIT>2.0.CO;2"},{"key":"26_CR3","doi-asserted-by":"publisher","first-page":"60","DOI":"10.1007\/s40593-014-0026-8","volume":"25","author":"S Burrows","year":"2015","unstructured":"Burrows, S., Gurevych, I., Stein, B.: The eras and trends of automatic short answer grading. Int. J. Artif. Intell. Educ. 25, 60\u2013117 (2015)","journal-title":"Int. J. Artif. Intell. Educ."},{"issue":"2","key":"26_CR4","doi-asserted-by":"publisher","first-page":"282","DOI":"10.1002\/sim.5544","volume":"32","author":"JE Dalton","year":"2013","unstructured":"Dalton, J.E.: Flexible recalibration of binary clinical prediction models. Stat. Med. 32(2), 282\u2013289 (2013). https:\/\/doi.org\/10.1002\/sim.5544","journal-title":"Stat. Med."},{"key":"26_CR5","unstructured":"Dzikovska, M., Nielsen, R., Brew, C., Leacock, C., Giampiccolo, D., Bentivogli, L., Clark, P., Dagan, I., Dang, H.T.: SemEval-2013 task 7: The joint student response analysis and 8th recognizing textual entailment challenge. In: Proc. SemEval. pp. 263\u2013274 (2013)"},{"key":"26_CR6","unstructured":"Dzikovska, M.O., Nielsen, R.D., Brew, C.: Towards effective tutorial feedback for explanation questions: a dataset and baselines. In: Proceedings of NAACL-HL, pp. 200\u2013210 (2012)"},{"key":"26_CR7","doi-asserted-by":"crossref","unstructured":"Efron, B., Tibshirani, R.J.: An Introduction to the Bootstrap. Chapman & Hall\/CRC, New York (1994)","DOI":"10.1201\/9780429246593"},{"issue":"8","key":"26_CR8","doi-asserted-by":"publisher","first-page":"861","DOI":"10.1016\/j.patrec.2005.10.010","volume":"27","author":"T Fawcett","year":"2006","unstructured":"Fawcett, T.: An introduction to ROC analysis. Pattern Recogn. Lett. 27(8), 861\u2013874 (2006). https:\/\/doi.org\/10.1016\/j.patrec.2005.10.010","journal-title":"Pattern Recogn. Lett."},{"issue":"3","key":"26_CR9","doi-asserted-by":"publisher","first-page":"369","DOI":"10.1111\/j.1467-9868.2007.00593.x","volume":"69","author":"CA Field","year":"2007","unstructured":"Field, C.A., Welsh, A.H.: Bootstrapping clustered data. J. Royal Stati. Soc. Ser. B (Statist. Methodo.) 69(3), 369\u2013390 (2007). https:\/\/doi.org\/10.1111\/j.1467-9868.2007.00593.x","journal-title":"J. Royal Stati. Soc. Ser. B (Statist. Methodo.)"},{"key":"26_CR10","unstructured":"Fielding, A., Goldstein, H.: Cross-classified and multiple membership structures in multilevel models: An introduction and review, vol.\u00a0791. Department for Education and Skills London (2006)"},{"key":"26_CR11","unstructured":"Gal, Y., Ghahramani, Z.: Dropout as a Bayesian approximation: representing model uncertainty in deep learning. In: International Conference on Machine Learning (ICML) (2016)"},{"key":"26_CR12","doi-asserted-by":"publisher","DOI":"10.1007\/s40593-024-00403-3","author":"A Gandolfi","year":"2025","unstructured":"Gandolfi, A.: GPT-4 in education: evaluating aptness, reliability, and loss of coherence in solving calculus problems and grading submissions. Int. J. Artif. Intell. Educ. (2025). https:\/\/doi.org\/10.1007\/s40593-024-00403-3","journal-title":"Int. J. Artif. Intell. Educ."},{"key":"26_CR13","doi-asserted-by":"crossref","unstructured":"Geng, J., Cai, F., Wang, Y., Koeppl, H., Nakov, P., Gurevych, I.: A survey of confidence estimation and calibration in large language models. In: Proceedings of the 2024 Conference of the North American Chapter of the Association for Computational Linguistics (NAACL) (2024)","DOI":"10.18653\/v1\/2024.naacl-long.366"},{"key":"26_CR14","unstructured":"Guo, C., Pleiss, G., Sun, Y., Weinberger, K.Q.: On calibration of modern neural networks. In: Proceedings of ICML, pp. 1321\u20131330. JMLR.org (2017)"},{"key":"26_CR15","unstructured":"Hendrycks, D., Gimpel, K.: A baseline for detecting misclassified and out-of-distribution examples in neural networks. In: Proceedings of ICLR (2017)"},{"key":"26_CR16","unstructured":"Holtzman, A., Buys, J., Du, L., Forbes, M., Choi, Y.: The curious case of neural text degeneration. In: Proceedings of ICLR (2020)"},{"key":"26_CR17","doi-asserted-by":"crossref","unstructured":"Jiang, L., Bosch, N.: Short answer scoring with GPT-4. In: Proceedings of the 11th ACM Conference on Learning @ Scale, pp. 438\u2013442. ACM (2024)","DOI":"10.1145\/3657604.3664685"},{"key":"26_CR18","unstructured":"Kadavath, S., et\u00a0al.: Language models (mostly) know what they know. arXiv preprint arXiv:2207.05221 (2022). anthropic"},{"key":"26_CR19","unstructured":"Koehn, P.: Statistical significance tests for machine translation evaluation. In: Proceedings of the 2004 Conference on Empirical Methods in Natural Language Processing (EMNLP). Association for Computational Linguistics (2004)"},{"issue":"1","key":"26_CR20","doi-asserted-by":"publisher","first-page":"143","DOI":"10.1609\/aaaiss.v3i1.31193","volume":"3","author":"M Kostic","year":"2024","unstructured":"Kostic, M., Witschel, H.F., Hinkelmann, K., Spahic-Bogdanovic, M.: LLMs in automated essay evaluation: a case study. Proc. AAAI Sympos. Ser. 3(1), 143\u2013147 (2024). https:\/\/doi.org\/10.1609\/aaaiss.v3i1.31193","journal-title":"Proc. AAAI Sympos. Ser."},{"key":"26_CR21","unstructured":"Kuhn, L., Gal, Y., Farquhar, S.: Semantic uncertainty: Linguistic invariances for uncertainty estimation in natural language generation. In: International Conference on Learning Representations (ICLR) (2023)"},{"key":"26_CR22","unstructured":"Kwon, W., et al.: vLLM: Easy, fast, and cheap LLM serving with Paged Attention. In: Proceedings of the ACM SIGOPS 29th Symposium on Operating Systems Principles (2023)"},{"key":"26_CR23","unstructured":"Lakshminarayanan, B., Pritzel, A., Blundell, C.: Simple and scalable predictive uncertainty estimation using deep ensembles. In: Neural Information Processing Systems (NeurIPS) (2017)"},{"key":"26_CR24","doi-asserted-by":"publisher","DOI":"10.3102\/10769986028004369","author":"W Van den Noortgate","year":"2003","unstructured":"Van den Noortgate, W., De Boeck, P., Meulders, M.: Cross-classification multilevel logistic models in psychometrics. J. Educat. Behav. Statist. (2003). https:\/\/doi.org\/10.3102\/10769986028004369","journal-title":"J. Educat. Behav. Statist."},{"key":"26_CR25","unstructured":"OpenAI: GPT-OSS-120B & GPT-OSS-20B model card. arXiv preprint arXiv:2508.10925 (2025)"},{"key":"26_CR26","doi-asserted-by":"publisher","unstructured":"Pakdaman\u00a0Naeini, M., Cooper, G., Hauskrecht, M.: Obtaining well calibrated probabilities using Bayesian binning. Proc. AAAI Conf. Artif. Intell. 29 (2015). https:\/\/doi.org\/10.1609\/aaai.v29i1.9602","DOI":"10.1609\/aaai.v29i1.9602"},{"key":"26_CR27","doi-asserted-by":"crossref","unstructured":"Platt, J.C.: Probabilistic outputs for support vector machines and comparisons to regularized likelihood methods. In: Advances in Large Margin Classifiers, pp. 61\u201374. Cambridge, MA (1999)","DOI":"10.7551\/mitpress\/1113.003.0008"},{"key":"26_CR28","doi-asserted-by":"crossref","unstructured":"Sonkar, S., Ni, K., Tran\u00a0Lu, L., Kincaid, K., Hutchinson, J.S., Baraniuk, R.G.: Automated long answer grading with Ricechem dataset. In: Artificial Intelligence in Education, pp. 163\u2013176 (2024)","DOI":"10.1007\/978-3-031-64302-6_12"},{"key":"26_CR29","doi-asserted-by":"publisher","unstructured":"Team, L.: The llama 3 herd of models. CoRR. abs\/2407.21783 (2024). https:\/\/doi.org\/10.48550\/ARXIV.2407.21783","DOI":"10.48550\/ARXIV.2407.21783"},{"key":"26_CR30","unstructured":"Team, Q.: Qwen3 technical report (2025). https:\/\/arxiv.org\/abs\/2505.09388"},{"key":"26_CR31","doi-asserted-by":"crossref","unstructured":"Tian, K., et al.: Just ask for calibration: strategies for eliciting calibrated confidence scores from language models fine-tuned with human feedback. In: Proceedings of the 2023 Conference on Empirical Methods in Natural Language Processing (EMNLP) (2023)","DOI":"10.18653\/v1\/2023.emnlp-main.330"},{"key":"26_CR32","doi-asserted-by":"publisher","unstructured":"Tian, X., Mannekote, A., Others: Examining LLM prompting strategies for automatic evaluation of learner-created computational artifacts. In: Paa\u00c3Yen, B., Epp, C.D. (eds.) Proceedings of the 17th International Conference on Educational Data Mining, pp. 698\u2013706 (2024). https:\/\/doi.org\/10.5281\/zenodo.12729922","DOI":"10.5281\/zenodo.12729922"},{"key":"26_CR33","doi-asserted-by":"publisher","unstructured":"Usher, M., Faraon, M.: Who grades best? comparing ChatGPT, peer, and instructor evaluations across varying levels of student project quality. Assessment & Evaluation in Higher Education 0(0), 1\u201320 (2025). https:\/\/doi.org\/10.1080\/02602938.2025.2588682","DOI":"10.1080\/02602938.2025.2588682"},{"key":"26_CR34","unstructured":"Wang, X., et al.: Self-consistency improves chain of thought reasoning in language models. In: Proceedings of ICLR (2023)"},{"key":"26_CR35","unstructured":"Xiong, M., Hu, Z., Lu, X., Li, Y., Fu, J., He, J., Hooi, B.: Can LLMs express their uncertainty? An empirical evaluation of confidence elicitation in LLMs. In: International Conference on Learning Representations (ICLR) (2024)"},{"key":"26_CR36","doi-asserted-by":"crossref","unstructured":"Zadrozny, B., Elkan, C.: Transforming classifier scores into accurate multiclass probability estimates. In: Proceedings of the Eighth ACM SIGKDD International Conference on Knowledge Discovery and Data Mining. ACM (2002)","DOI":"10.1145\/775047.775151"}],"container-title":["Lecture Notes in Computer Science","Artificial Intelligence in Education"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-032-29744-0_26","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,6,26]],"date-time":"2026-06-26T12:54:33Z","timestamp":1782478473000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-3-032-29744-0_26"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026,6,27]]},"ISBN":["9783032297433","9783032297440"],"references-count":36,"URL":"https:\/\/doi.org\/10.1007\/978-3-032-29744-0_26","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"value":"0302-9743","type":"print"},{"value":"1611-3349","type":"electronic"}],"subject":[],"published":{"date-parts":[[2026,6,27]]},"assertion":[{"value":"27 June 2026","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"AIED","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"International Conference on Artificial Intelligence in Education","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Seoul","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Korea (Republic of)","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2026","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"29 June 2026","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"3 July 2026","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"27","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"aied2026","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"https:\/\/www.aied-conference.org\/2026","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}}]}}