{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,6,24]],"date-time":"2026-06-24T15:56:10Z","timestamp":1782316570916,"version":"3.54.5"},"publisher-location":"Cham","reference-count":40,"publisher":"Springer Nature Switzerland","isbn-type":[{"value":"9783032297549","type":"print"},{"value":"9783032297556","type":"electronic"}],"license":[{"start":{"date-parts":[[2026,6,25]],"date-time":"2026-06-25T00:00:00Z","timestamp":1782345600000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2026,6,25]],"date-time":"2026-06-25T00:00:00Z","timestamp":1782345600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2027]]},"DOI":"10.1007\/978-3-032-29755-6_8","type":"book-chapter","created":{"date-parts":[[2026,6,24]],"date-time":"2026-06-24T15:39:03Z","timestamp":1782315543000},"page":"110-123","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":0,"title":["Confidence Estimation in\u00a0Automatic Short Answer Grading with\u00a0LLMs"],"prefix":"10.1007","author":[{"ORCID":"https:\/\/orcid.org\/0009-0008-2568-4556","authenticated-orcid":false,"given":"Longwei","family":"Cong","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-5461-6383","authenticated-orcid":false,"given":"Sonja","family":"Hahn","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-5598-9547","authenticated-orcid":false,"given":"Sebastian","family":"Gombert","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0008-8871-2657","authenticated-orcid":false,"given":"Leon","family":"Camus","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-8407-5314","authenticated-orcid":false,"given":"Hendrik","family":"Drachsler","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-0412-169X","authenticated-orcid":false,"given":"Ulf","family":"Kroehne","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2026,6,25]]},"reference":[{"key":"8_CR1","unstructured":"Agarwal, S., et\u00a0al.: gpt-oss-120b & gpt-oss-20b model card. arXiv preprint arXiv:2508.10925 (2025)"},{"key":"8_CR2","doi-asserted-by":"crossref","unstructured":"Aggarwal, D., Sil, P., Raman, B., Bhattacharyya, P.: \u201ci understand why i got this grade\u201d: Automatic short answer grading (asag) with feedback. In: International Conference on Artificial Intelligence in Education, pp. 304\u2013318. Springer (2025)","DOI":"10.1007\/978-3-031-98420-4_22"},{"key":"8_CR3","unstructured":"Attali, Y., Burstein, J.: Automated essay scoring with e-rater\u00ae v. 2. The Journal of Technology, Learning and Assessment 4(3), (2006)"},{"key":"8_CR4","unstructured":"Bereska, L., Gavves, E.: Mechanistic interpretability for ai safety-a review, (2024). arXiv:2404.14082 arXiv preprint"},{"key":"8_CR5","unstructured":"Bexte, M., Horbach, A., Sch\u00fctzler, L., Christ, O., Zesch, T.: Scoring with confidence?-exploring high-confidence scoring for saving manual grading effort. In: Proceedings of the 19th Workshop on Innovative Use of NLP for Building Educational Applications (BEA 2024), pp. 119\u2013124. (2024)"},{"issue":"7","key":"8_CR6","doi-asserted-by":"publisher","first-page":"1145","DOI":"10.1016\/S0031-3203(96)00142-2","volume":"30","author":"AP Bradley","year":"1997","unstructured":"Bradley, A.P.: The use of the area under the roc curve in the evaluation of machine learning algorithms. Pattern Recogn. 30(7), 1145\u20131159 (1997)","journal-title":"Pattern Recogn."},{"issue":"1","key":"8_CR7","doi-asserted-by":"publisher","first-page":"60","DOI":"10.1007\/s40593-014-0026-8","volume":"25","author":"S Burrows","year":"2015","unstructured":"Burrows, S., Gurevych, I., Stein, B.: The eras and trends of automatic short answer grading. Int. J. Artif. Intell. Educ. 25(1), 60\u2013117 (2015)","journal-title":"Int. J. Artif. Intell. Educ."},{"key":"8_CR8","doi-asserted-by":"publisher","unstructured":"Camus, L., Filighera, A.: Investigating transformers for automatic short answer grading. In: Artificial Intelligence in Education: 21st International Conference, AIED 2020, Ifrane, Morocco, July 6\u201310, 2020, Proceedings, Part II, pp. 43\u201348. Springer, Heidelberg (2020). https:\/\/doi.org\/10.1007\/978-3-030-52240-7_8","DOI":"10.1007\/978-3-030-52240-7_8"},{"key":"8_CR9","doi-asserted-by":"publisher","unstructured":"Cong, L., Hammerla, L., Hahn, S., Gombert, S., Drachsler, H., Kroehne, U.: Automatic short answer grading with LLMs: From memorization to reasoning. In: Proceedings of the 16th International Learning Analytics and Knowledge Conference. Association for Computing Machinery, New York (2026). https:\/\/doi.org\/10.1145\/3785022.3785031","DOI":"10.1145\/3785022.3785031"},{"key":"8_CR10","unstructured":"Dzikovska, M., et al.: SemEval-2013 task 7: The joint student response analysis and 8th recognizing textual entailment challenge. In: Manandhar, S., Yuret, D. (eds.) Second Joint Conference on Lexical and Computational Semantics (SEM), Volume 2: Proceedings of the Seventh International Workshop on Semantic Evaluation (SemEval 2013), pp. 263\u2013274. Association for Computational Linguistics, Atlanta, Georgia, USA (Jun 2013), https:\/\/aclanthology.org\/S13-2045"},{"issue":"8017","key":"8_CR11","doi-asserted-by":"publisher","first-page":"625","DOI":"10.1038\/s41586-024-07421-0","volume":"630","author":"S Farquhar","year":"2024","unstructured":"Farquhar, S., Kossen, J., Kuhn, L., Gal, Y.: Detecting hallucinations in large language models using semantic entropy. Nature 630(8017), 625\u2013630 (2024)","journal-title":"Nature"},{"key":"8_CR12","doi-asserted-by":"crossref","unstructured":"Ferreira Mello, R., et al.: Automatic short answer grading in the llm era: does gpt-4 with prompt engineering beat traditional models? In: Proceedings of the 15th international learning analytics and knowledge conference, pp. 93\u2013103. (2025)","DOI":"10.1145\/3706468.3706481"},{"key":"8_CR13","doi-asserted-by":"crossref","unstructured":"Frohn, S., Burleigh, T., Chen, J.: Automated scoring of short answer questions with large language models: impacts of model, item, and rubric design. In: International Conference on Artificial Intelligence in Education, pp. 44\u201351. Springer (2025)","DOI":"10.1007\/978-3-031-98465-5_6"},{"key":"8_CR14","doi-asserted-by":"crossref","unstructured":"Funayama, H., Sato, T., Matsubayashi, Y., Mizumoto, T., Suzuki, J., Inui, K.: Balancing cost and quality: an exploration of human-in-the-loop frameworks for automated short answer scoring. In: International Conference on Artificial Intelligence in Education, pp. 465\u2013476. Springer (2022)","DOI":"10.1007\/978-3-031-11644-5_38"},{"key":"8_CR15","doi-asserted-by":"crossref","unstructured":"Geng, J., Cai, F., Wang, Y., Koeppl, H., Nakov, P., Gurevych, I.: A survey of confidence estimation and calibration in large language models. In: Proceedings of the 2024 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies (Volume 1: Long Papers), pp. 6577\u20136595 (2024)","DOI":"10.18653\/v1\/2024.naacl-long.366"},{"key":"8_CR16","doi-asserted-by":"crossref","unstructured":"Ghandeharioun, A., Eoff, B., Jou, B., Picard, R.: Characterizing sources of uncertainty to proxy calibration and disambiguate annotator and data bias. In: IEEE\/CVF International Conference on Computer Vision Workshop (ICCVW), pp. 4202\u20134206. IEEE (2019)","DOI":"10.1109\/ICCVW.2019.00517"},{"issue":"3","key":"8_CR17","doi-asserted-by":"publisher","first-page":"767","DOI":"10.1111\/jcal.12767","volume":"39","author":"S Gombert","year":"2023","unstructured":"Gombert, S., Di Mitri, D., Karademir, O., Kubsch, M., Kolbe, H., Tautz, S., Grimm, A., Bohm, I., Neumann, K., Drachsler, H.: Coding energy knowledge in constructed responses with explainable nlp models. J. Comput. Assist. Learn. 39(3), 767\u2013786 (2023)","journal-title":"J. Comput. Assist. Learn."},{"key":"8_CR18","unstructured":"Haller, S., Aldea, A., Seifert, C., Strisciuglio, N.: Survey on automated short answer grading with deep learning: from word embeddings to transformers (2022). arXiv:2204.03503 arXiv preprint"},{"key":"8_CR19","doi-asserted-by":"crossref","unstructured":"Horbach, A., Zesch, T.: The influence of variance in learner answers on automatic content scoring. In: Frontiers in education, vol.\u00a04, p.\u00a028. Frontiers Media SA (2019)","DOI":"10.3389\/feduc.2019.00028"},{"issue":"3","key":"8_CR20","doi-asserted-by":"publisher","first-page":"457","DOI":"10.1007\/s10994-021-05946-3","volume":"110","author":"E H\u00fcllermeier","year":"2021","unstructured":"H\u00fcllermeier, E., Waegeman, W.: Aleatoric and epistemic uncertainty in machine learning: an introduction to concepts and methods. Mach. Learn. 110(3), 457\u2013506 (2021)","journal-title":"Mach. Learn."},{"key":"8_CR21","unstructured":"Jiang, H., Kim, B., Guan, M., Gupta, M.: To trust or not to trust a classifier. In: Advances in neural information processing systems, p. 31 (2018)"},{"key":"8_CR22","unstructured":"Kadavath, S., et al.: Language models (mostly) know what they know (2022). https:\/\/arxiv.org\/abs\/2207.05221"},{"key":"8_CR23","unstructured":"Livingston, S.A.: Constructed-response test questions: Why we use them; how we score them. r&d connections. number 11. Educational Testing Service (2009)"},{"key":"8_CR24","doi-asserted-by":"crossref","unstructured":"Lord, F.M.: Applications of item response theory to practical testing problems. Routledge (2012)","DOI":"10.4324\/9780203056615"},{"key":"8_CR25","unstructured":"Magooda, A.E., Zahran, M.A., Rashwan, M.A., Raafat, H.M., Fayek, M.B.: Vector based techniques for short answer grading. FLAIRS, pp. 238\u2013243 (2016)"},{"key":"8_CR26","unstructured":"Nadeem, M.S.A., Zucker, J.D., Hanczar, B.: Accuracy-rejection curves (arcs) for comparing classification methods with a reject option. In: Machine Learning in Systems Biology, pp. 65\u201381 (2009) PMLR"},{"key":"8_CR27","first-page":"100","volume":"6","author":"A Petukhova","year":"2025","unstructured":"Petukhova, A., Matos-Carvalho, J.P., Fachada, N.: Text clustering with large language model embeddings. Int. J. Cognit. Comput. Eng. 6, 100\u2013108 (2025)","journal-title":"Int. J. Cognit. Comput. Eng."},{"key":"8_CR28","doi-asserted-by":"crossref","unstructured":"Shorinwa, O., Mei, Z., Lidard, J., Ren, A.Z., Majumdar, A.: A survey on uncertainty quantification of large language models: Taxonomy, open research challenges, and future directions, ACM Computing Surveys (2025)","DOI":"10.1145\/3744238"},{"issue":"9","key":"8_CR29","doi-asserted-by":"publisher","first-page":"3211","DOI":"10.1007\/s10994-023-06336-7","volume":"112","author":"T Silva Filho","year":"2023","unstructured":"Silva Filho, T., Song, H., Perello-Nieto, M., Santos-Rodriguez, R., Kull, M., Flach, P.: Classifier calibration: a survey on how to assess and improve predicted class probabilities. Mach. Learn. 112(9), 3211\u20133260 (2023)","journal-title":"Mach. Learn."},{"key":"8_CR30","doi-asserted-by":"crossref","unstructured":"Tian, K., et al.: Just ask for calibration: Strategies for eliciting calibrated confidence scores from language models fine-tuned with human feedback (2023). arXiv:2305.14975 arXiv preprint","DOI":"10.18653\/v1\/2023.emnlp-main.330"},{"key":"8_CR31","doi-asserted-by":"publisher","first-page":"1385","DOI":"10.1613\/jair.1.12752","volume":"72","author":"AN Uma","year":"2021","unstructured":"Uma, A.N., Fornaciari, T., Hovy, D., Paun, S., Plank, B., Poesio, M.: Learning from disagreement: a survey. J. Artif. Intell. Res. 72, 1385\u20131470 (2021)","journal-title":"J. Artif. Intell. Res."},{"key":"8_CR32","unstructured":"Wang, W., Wei, F., Dong, L., Bao, H., Yang, N., Zhou, M.: Minilm: deep self-attention distillation for task-agnostic compression of pre-trained transformers. In: Proceedings of the 34th International Conference on Neural Information Processing Systems, NIPS \u201920. Curran Associates Inc, Red Hook (2020)"},{"issue":"301","key":"8_CR33","doi-asserted-by":"publisher","first-page":"236","DOI":"10.1080\/01621459.1963.10500845","volume":"58","author":"JH Ward Jr","year":"1963","unstructured":"Ward, J.H., Jr.: Hierarchical grouping to optimize an objective function. J. Am. Stat. Assoc. 58(301), 236\u2013244 (1963)","journal-title":"J. Am. Stat. Assoc."},{"key":"8_CR34","unstructured":"Wei, T., et\u00a0al.: Agentic reasoning for large language models. arXiv preprint arXiv:2601.12538 (2026)"},{"key":"8_CR35","doi-asserted-by":"publisher","unstructured":"Xia, Z., Xu, J., Zhang, Y., Liu, H.: A survey of uncertainty estimation methods on large language models. In: Che, W., Nabende, J., Shutova, E., Pilehvar, M.T. (eds.) Findings of the Association for Computational Linguistics: ACL 2025, pp. 21381\u201321396. Association for Computational Linguistics, Vienna, Austria (2025). https:\/\/doi.org\/10.18653\/v1\/2025.findings-acl.1101","DOI":"10.18653\/v1\/2025.findings-acl.1101"},{"key":"8_CR36","unstructured":"Xiong, M., Hu, Z., Lu, X., Li, Y., Fu, J., He, J., Hooi, B.: Can llms express their uncertainty? an empirical evaluation of confidence elicitation in llms, (2023). arXiv:2306.13063 arXiv preprint"},{"issue":"1","key":"8_CR37","doi-asserted-by":"publisher","first-page":"32","DOI":"10.1002\/1097-0142(1950)3:1<32::AID-CNCR2820030106>3.0.CO;2-3","volume":"3","author":"WJ Youden","year":"1950","unstructured":"Youden, W.J.: Index for rating diagnostic tests. Cancer 3(1), 32\u201335 (1950)","journal-title":"Cancer"},{"issue":"2","key":"8_CR38","doi-asserted-by":"publisher","first-page":"280","DOI":"10.1177\/0013164415590022","volume":"76","author":"F Zehner","year":"2016","unstructured":"Zehner, F., S\u00e4lzer, C., Goldhammer, F.: Automatic coding of short text responses via clustering in educational assessment. Educ. Psychol. Measur. 76(2), 280\u2013303 (2016)","journal-title":"Educ. Psychol. Measur."},{"key":"8_CR39","doi-asserted-by":"publisher","unstructured":"Zehner, F., et al.: Down the cascades of omethi: Hierarchical automatic scoring in large-scale assessments. In: Proceedings of the 20th Workshop on Innovative Use of NLP for Building Educational Applications (BEA 2025), pp. 660\u2013671. Association for Computational Linguistics, Vienna, Austria (2025). https:\/\/doi.org\/10.18653\/v1\/2025.bea-1.47","DOI":"10.18653\/v1\/2025.bea-1.47"},{"key":"8_CR40","unstructured":"Zhao, W.X., et\u00a0al.: A survey of large language models. arXiv preprint arXiv:2303.18223 (2023)"}],"container-title":["Lecture Notes in Computer Science","Artificial Intelligence in Education"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-032-29755-6_8","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,6,24]],"date-time":"2026-06-24T15:39:40Z","timestamp":1782315580000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-3-032-29755-6_8"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026,6,25]]},"ISBN":["9783032297549","9783032297556"],"references-count":40,"URL":"https:\/\/doi.org\/10.1007\/978-3-032-29755-6_8","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"value":"0302-9743","type":"print"},{"value":"1611-3349","type":"electronic"}],"subject":[],"published":{"date-parts":[[2026,6,25]]},"assertion":[{"value":"25 June 2026","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"AIED","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"International Conference on Artificial Intelligence in Education","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Seoul","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Korea (Republic of)","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2026","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"29 June 2026","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"3 July 2026","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"27","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"aied2026","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"https:\/\/www.aied-conference.org\/2026","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}}]}}