{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,6,26]],"date-time":"2026-06-26T12:49:32Z","timestamp":1782478172499,"version":"3.54.5"},"publisher-location":"Cham","reference-count":53,"publisher":"Springer Nature Switzerland","isbn-type":[{"value":"9783032297723","type":"print"},{"value":"9783032297730","type":"electronic"}],"license":[{"start":{"date-parts":[[2026,6,27]],"date-time":"2026-06-27T00:00:00Z","timestamp":1782518400000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2026,6,27]],"date-time":"2026-06-27T00:00:00Z","timestamp":1782518400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2027]]},"DOI":"10.1007\/978-3-032-29773-0_9","type":"book-chapter","created":{"date-parts":[[2026,6,26]],"date-time":"2026-06-26T12:21:55Z","timestamp":1782476515000},"page":"117-131","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":0,"title":["Modernizing Ground Truth: Four Shifts Toward Improving Reliability and\u00a0Validity in\u00a0AI in\u00a0Education"],"prefix":"10.1007","author":[{"ORCID":"https:\/\/orcid.org\/0000-0001-8196-3252","authenticated-orcid":false,"given":"Danielle R.","family":"Thomas","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-3437-8979","authenticated-orcid":false,"given":"Conrad","family":"Borchers","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-0673-5721","authenticated-orcid":false,"given":"Kirk P.","family":"Vanacore","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-5850-4768","authenticated-orcid":false,"given":"Kenneth R.","family":"Koedinger","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-6283-5546","authenticated-orcid":false,"given":"Ren\u00e9 F.","family":"Kizilcec","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2026,6,27]]},"reference":[{"key":"9_CR1","doi-asserted-by":"crossref","unstructured":"Ahtisham, B., Vanacore, K., Lee, J., Zhou, Z., Pietrzak, D., Kizilcec, R.F.: AI annotation orchestration: evaluating LLM verifiers to improve the quality of LLM annotations in learning analytics. arXiv preprint arXiv:2511.09785 (2025)","DOI":"10.1145\/3785022.3785095"},{"key":"9_CR2","doi-asserted-by":"publisher","unstructured":"Aleven, V., et\u00a0al.: Towards the future of AI-augmented human tutoring in math learning. In: International Conference on Artificial Intelligence in Education, pp. 26\u201331. Springer, Heidelberg (2023). https:\/\/doi.org\/10.1007\/978-3-031-36336-8_3","DOI":"10.1007\/978-3-031-36336-8_3"},{"key":"9_CR3","unstructured":"Ando, R.K., Zhang, T., Bartlett, P.: A framework for learning predictive structures from multiple tasks and unlabeled data. J. Mach. Learn. 6(11) (2005)"},{"issue":"4","key":"9_CR4","doi-asserted-by":"publisher","first-page":"1052","DOI":"10.1007\/s40593-021-00285-9","volume":"32","author":"RS Baker","year":"2022","unstructured":"Baker, R.S., Hawn, A.: Algorithmic bias in education. Int. J. Artif. Intell. Educ. 32(4), 1052\u20131092 (2022)","journal-title":"Int. J. Artif. Intell. Educ."},{"key":"9_CR5","doi-asserted-by":"publisher","unstructured":"Barany, A., et\u00a0al.: Chatgpt for education research: exploring the potential of large language models for qualitative codebook development. In: International Conference on Artificial Intelligence in Education, pp. 134\u2013149. Springer, Heidelberg (2024). https:\/\/doi.org\/10.1007\/978-3-031-64299-9_10","DOI":"10.1007\/978-3-031-64299-9_10"},{"key":"9_CR6","doi-asserted-by":"crossref","unstructured":"Bond, M., et al.: A meta systematic review of artificial intelligence in higher education. Educ. Technol. High. Educ. 21(1), 4 (2024)","DOI":"10.1186\/s41239-023-00436-z"},{"issue":"2","key":"9_CR7","doi-asserted-by":"publisher","first-page":"77","DOI":"10.1191\/1478088706qp063oa","volume":"3","author":"V Braun","year":"2006","unstructured":"Braun, V., Clarke, V.: Using thematic analysis in psychology. Qual. Res. Psychol. 3(2), 77\u2013101 (2006)","journal-title":"Qual. Res. Psychol."},{"key":"9_CR8","doi-asserted-by":"crossref","unstructured":"Calderon, N., Reichart, R., Dror, R.: The alternative annotator test for LLM-as-a-judge: how to statistically justify replacing human annotators with LLMs. In: Association for Computational Linguistics, pp. 16051\u201316081 (2025)","DOI":"10.18653\/v1\/2025.acl-long.782"},{"key":"9_CR9","series-title":"Lecture Notes in Computer Science (Lecture Notes in Artificial Intelligence)","doi-asserted-by":"publisher","first-page":"55","DOI":"10.1007\/978-3-030-52237-7_5","volume-title":"Artificial Intelligence in Education","author":"D Carpenter","year":"2020","unstructured":"Carpenter, D., et al.: Detecting off-task behavior from student dialogue in game-based collaborative learning. In: Bittencourt, I.I., Cukurova, M., Muldner, K., Luckin, R., Mill\u00e1n, E. (eds.) AIED 2020. LNCS (LNAI), vol. 12163, pp. 55\u201366. Springer, Cham (2020). https:\/\/doi.org\/10.1007\/978-3-030-52237-7_5"},{"key":"9_CR10","doi-asserted-by":"crossref","unstructured":"Chen, G.H., Chen, S., Liu, Z., Jiang, F., Wang, B.: Humans or llms as the judge? a study on judgement biases. arXiv preprint arXiv:2402.10669 (2024)","DOI":"10.18653\/v1\/2024.emnlp-main.474"},{"issue":"4","key":"9_CR11","doi-asserted-by":"publisher","first-page":"1944","DOI":"10.1177\/00491241231156971","volume":"53","author":"R Cole","year":"2024","unstructured":"Cole, R.: Inter-rater reliability methods in qualitative case study research. Sociol. Methods Res. 53(4), 1944\u20131975 (2024)","journal-title":"Sociol. Methods Res."},{"issue":"5","key":"9_CR12","doi-asserted-by":"publisher","first-page":"1433","DOI":"10.1111\/isj.12504","volume":"34","author":"RM Davison","year":"2024","unstructured":"Davison, R.M., Chughtai, H., Nielsen, P., Trenz, M., Techatassanasoontorn, A.A., Andrade, A.D., et al.: The ethics of using generative ai for qualitative data analysis. Inf. Syst. J. 34(5), 1433\u20131439 (2024)","journal-title":"Inf. Syst. J."},{"key":"9_CR13","doi-asserted-by":"crossref","unstructured":"Demszky, D., et al.: Measuring conversational uptake: a case study on student-teacher interactions. In: Proceedings of Computational Linguistics, pp. 1638\u20131653 (2021)","DOI":"10.18653\/v1\/2021.acl-long.130"},{"key":"9_CR14","doi-asserted-by":"crossref","unstructured":"Foster, J.K., Youngs, P., van Aswegen, R., Watson, G.S., Acton, S.T.: Automated classification of elementary instructional activities. J. Learn. Anal. (2024)","DOI":"10.18608\/jla.2024.8323"},{"key":"9_CR15","unstructured":"Gwet, K.L.: Handbook of inter-rater reliability: The definitive guide to measuring the extent of agreement among raters. Advanced Analytics, LLC (2014)"},{"key":"9_CR16","unstructured":"Henkel, O., Roberts, B., Jaffe, D., Holt, L.: Seeing the big picture: evaluating multimodal LLMs\u2019 ability to interpret and grade. arXiv:2510.05538 (2025)"},{"key":"9_CR17","unstructured":"Henkel, O., Vanacore, K., Roberts, B.: When humans can\u2019t agree, neither can machines. In: Proceedings of the AI in Measurement and Education (2025)"},{"issue":"2","key":"9_CR18","doi-asserted-by":"publisher","first-page":"211","DOI":"10.1037\/a0025940","volume":"138","author":"M Hilbert","year":"2012","unstructured":"Hilbert, M.: Toward a synthesis of cognitive biases: how noisy information processing can bias human decision making. Psychol. Bull. 138(2), 211 (2012)","journal-title":"Psychol. Bull."},{"issue":"3","key":"9_CR19","doi-asserted-by":"publisher","first-page":"837","DOI":"10.3758\/s13428-014-0506-7","volume":"47","author":"H Holle","year":"2015","unstructured":"Holle, H., Rein, R.: Easydiag: a tool for easy determination of interrater agreement. Behav. Res. Methods 47(3), 837\u2013847 (2015)","journal-title":"Behav. Res. Methods"},{"key":"9_CR20","doi-asserted-by":"crossref","unstructured":"Hou, C., Zhu, G., Zheng, J., Du, H., Ker, C.L.: Prompt-based and fine-tuned GPT models for context-dependent and-independent deductive coding. In: Proceedings of the 14th Learning Analytics and Knowledge Conference, pp. 518\u2013528 (2024)","DOI":"10.1145\/3636555.3636910"},{"key":"9_CR21","doi-asserted-by":"crossref","unstructured":"Jowsey, T., Braun, V., Clarke, V., Lupton, D., Fine, M.: We reject the use of generative ai for reflexive qualitative research. Qual. Inquiry (2025)","DOI":"10.1177\/10778004251401851"},{"key":"9_CR22","doi-asserted-by":"crossref","unstructured":"Kahneman, D., Sibony, O., Sunstein, C.R.: Noise: a flaw in human judgment. Hachette UK (2021)","DOI":"10.53776\/playbooks-judgment"},{"key":"9_CR23","doi-asserted-by":"crossref","unstructured":"Kizilcec, R.F., Lee, H.: Algorithmic fairness in education. In: The Ethics of Artificial Intelligence in Education, pp. 174\u2013202. Routledge (2022)","DOI":"10.4324\/9780429329067-10"},{"issue":"5","key":"9_CR24","doi-asserted-by":"publisher","first-page":"757","DOI":"10.1111\/j.1551-6709.2012.01245.x","volume":"36","author":"KR Koedinger","year":"2012","unstructured":"Koedinger, K.R., Corbett, A.T., Perfetti, C.: The knowledge-learning-instruction framework. Cogn. Sci. 36(5), 757\u2013798 (2012)","journal-title":"Cogn. Sci."},{"key":"9_CR25","doi-asserted-by":"crossref","unstructured":"Landis, J.R., Koch, G.G.: An application of hierarchical kappa-type statistics in the assessment of majority agreement among multiple observers. Biometrics (1977)","DOI":"10.2307\/2529786"},{"key":"9_CR26","unstructured":"Li, H., et al.: LLMs-as-judges: a comprehensive survey on LLM-based evaluation methods. arXiv:2412.05579 (2024)"},{"key":"9_CR27","doi-asserted-by":"publisher","unstructured":"Li, Z., Akemoglu, Y., Lyu, J., Zheng, Q., Xiong, J.: ASD-HI: a parent-child interaction dataset for automated assessment of home intervention. In: International Conference on Artificial Intelligence in Education, pp. 48\u201362. Springer, Heidelberg (2025). https:\/\/doi.org\/10.1007\/978-3-031-98414-3_4","DOI":"10.1007\/978-3-031-98414-3_4"},{"key":"9_CR28","unstructured":"McHugh, M.L.: Irr: the kappa statistic. Biochemia medica 22(3) (2012)"},{"key":"9_CR29","doi-asserted-by":"crossref","unstructured":"Messer, M., Brown, N.C., K\u00f6lling, M., Shi, M.: How consistent are humans when grading programming assignments? arXiv preprint arXiv:2409.12967 (2024)","DOI":"10.35542\/osf.io\/nd6qy"},{"key":"9_CR30","doi-asserted-by":"publisher","first-page":"142733","DOI":"10.1109\/ACCESS.2024.3468368","volume":"12","author":"U Mittal","year":"2024","unstructured":"Mittal, U., Sai, S., Chamola, V., Sangwan, D.: A comprehensive review on generative AI for education. IEEE Access 12, 142733\u2013142759 (2024)","journal-title":"IEEE Access"},{"key":"9_CR31","doi-asserted-by":"crossref","unstructured":"Na, Y., Feng, S.: LLM-assisted automated deductive coding of dialogue data: leveraging dialogue-specific characteristics to enhance contextual understanding. In: International Conference on Artificial Intelligence in Education, pp. 248\u2013262 (2025)","DOI":"10.1007\/978-3-031-98459-4_18"},{"key":"9_CR32","doi-asserted-by":"crossref","unstructured":"Nahum, O., Calderon, N., Keller, O., Szpektor, I., Reichart, R.: Are LLMs better than reported? In: Proceedings of the 2025 Empirical Methods in Natural Language Processing, pp. 26770\u201326797 (2025)","DOI":"10.18653\/v1\/2025.emnlp-main.1360"},{"key":"9_CR33","doi-asserted-by":"crossref","unstructured":"Nickow, A., Oreopoulos, P., Quan, V.: The impressive effects of tutoring on prek-12 learning: a systematic review of the experimental evidence (2020)","DOI":"10.3386\/w27476"},{"key":"9_CR34","unstructured":"Ocumpaugh, J., et\u00a0al.: Baker rodrigo ocumpaugh monitoring protocol (bromp) 2.0 technical and training manual. Teachers College (2015)"},{"key":"9_CR35","doi-asserted-by":"crossref","unstructured":"Petukhova, K., Kochmar, E.: Intent matters: enhancing AI tutoring with fine-grained pedagogical intent annotation. arXiv preprint arXiv:2506.07626 (2025)","DOI":"10.18653\/v1\/2025.bea-1.63"},{"key":"9_CR36","unstructured":"Rachatasumrit, N., Carvalho, P., Koedinger, K.: Beyond accuracy: embracing meaningful parameters in educational data mining. In: Proceedings of the 17th International Conference on Educational Data Mining, pp. 203\u2013210 (2024)"},{"issue":"2","key":"9_CR37","first-page":"279","volume":"40","author":"B Rosenshine","year":"1970","unstructured":"Rosenshine, B.: 5: Evaluation of classroom instruction. Rev. Educ. Res. 40(2), 279\u2013300 (1970)","journal-title":"Rev. Educ. Res."},{"issue":"4","key":"9_CR38","first-page":"3","volume":"1","author":"J Self","year":"1990","unstructured":"Self, J.: Theoretical foundations for intelligent tutoring systems. J. Artif. Intell. Educ. 1(4), 3\u201314 (1990)","journal-title":"J. Artif. Intell. Educ."},{"key":"9_CR39","doi-asserted-by":"crossref","unstructured":"Tajik, E., et al.: Disagreement as data: reasoning trace analytics in multi-agent systems. arXiv preprint arXiv:2601.12618 (2026)","DOI":"10.1145\/3785022.3785101"},{"key":"9_CR40","doi-asserted-by":"crossref","unstructured":"Tan, Z., et al.: Large language models for data annotation and synthesis: a survey. In: Proceedings of the 2024 Conference on Empirical Methods in Natural Language Processing, pp. 930\u2013957 (2024)","DOI":"10.18653\/v1\/2024.emnlp-main.54"},{"key":"9_CR41","unstructured":"Thakur, A.S., Choudhary, K., Ramayapally, V.S., Vaidyanathan, S., Hupkes, D.: Judging the judges: evaluating alignment and vulnerabilities in LLMs-as-judges. In: Proceedings of the Fourth Workshop on Generation, Evaluation and Metrics (GEM$$^2$$), pp. 404\u2013430 (2025)"},{"key":"9_CR42","doi-asserted-by":"publisher","unstructured":"Thomas, D.R., et al.: Improving open-response assessment with learnlm. In: International Conference on Artificial Intelligence in Education, pp. 220\u2013227. Springer, Heidelberg (2025). https:\/\/doi.org\/10.1007\/978-3-031-98462-4_28","DOI":"10.1007\/978-3-031-98462-4_28"},{"key":"9_CR43","unstructured":"Thomas, D.R., Borchers, C., Koedinger, K.: Beyond agreement: rethinking ground truth in educational AI annotation. In: Proceedings of the Artificial Intelligence in Measurement and Education Conference (AIME-Con), pp. 345\u2013351 (2025)"},{"key":"9_CR44","doi-asserted-by":"crossref","unstructured":"Thomas, D.R., et al.: Does multiple choice have a future in the age of genai? In: Proceedings of the Inter. Learning Analytics and Knowledge Conference, pp. 494\u2013504 (2025)","DOI":"10.1145\/3706468.3706530"},{"key":"9_CR45","doi-asserted-by":"crossref","unstructured":"Thomas, D.R., et al.: Advancing the science of teaching with tutoring data: a collaborative workshop with the national tutoring observatory. In: Proceedings of the ACM Conference on Learning@Scale, pp. 404\u2013406 (2025)","DOI":"10.1145\/3698205.3733961"},{"key":"9_CR46","unstructured":"Trochim, W.M., Donnelly, J.P., Arora, K.: Research methods: the essential knowledge base (2016)"},{"key":"9_CR47","doi-asserted-by":"crossref","unstructured":"Vanacore, K., Kizilcec, R.F.: How well do large language models recognize instructional moves? Establishing baselines for foundation models in educational discourse. arXiv preprint arXiv:2512.19903 (2025)","DOI":"10.1145\/3774398.3811605"},{"key":"9_CR48","doi-asserted-by":"publisher","unstructured":"Vitale, J., Kocabagli, K.B., Sanghi, S., Van\u00a0Camp, A., Miller, S., Coker, B.: From noisy classroom transcripts to actionable feedback: fine-tuning gpt-4o to detect teachers\u2019 opportunities to respond. In: International Conference on Artificial Intelligence in Education, pp. 149\u2013162. Springer, Heidelberg (2025). https:\/\/doi.org\/10.1007\/978-3-031-98417-4_11","DOI":"10.1007\/978-3-031-98417-4_11"},{"key":"9_CR49","doi-asserted-by":"crossref","unstructured":"Wang, R.E., Ribeiro, A.T., Robinson, C.D., Loeb, S., Demszky, D.: Tutor copilot: a human-AI approach for scaling real-time expertise. arXiv:2410.03017 (2024)","DOI":"10.21203\/rs.3.rs-5363154\/v1"},{"key":"9_CR50","doi-asserted-by":"crossref","unstructured":"Wang, X., Kim, H., Rahman, S., Mitra, K., Miao, Z.: Human-LLM collaborative annotation through effective verification of LLM labels. In: Proceedings of the 2024 CHI Conference on Human Factors in Computing Systems, pp. 1\u201321 (2024)","DOI":"10.1145\/3613904.3641960"},{"key":"9_CR51","doi-asserted-by":"publisher","unstructured":"Wong, K., Wu, B., Bulathwela, S., Cukurova, M.: Rethinking the potential of multimodality in collaborative problem solving diagnosis with large language models. In: International Conference on Artificial Intelligence in Education. Springer, Heidelberg (2025). https:\/\/doi.org\/10.1007\/978-3-031-98417-4_2","DOI":"10.1007\/978-3-031-98417-4_2"},{"key":"9_CR52","doi-asserted-by":"crossref","unstructured":"Zhao, C., Silva, M., Poulsen, S.: Autograding mathematical induction proofs with natural language processing. J. Artif. Intell. Educ. (2025)","DOI":"10.1007\/s40593-025-00498-2"},{"key":"9_CR53","doi-asserted-by":"crossref","unstructured":"Zhuang, Y., Zhang, S.: Integrating chatgpt in mathematics teacher education: AI- based simulation role-playing to support practice-based teaching. Artif. Intell. Educ. 35 (2025)","DOI":"10.1007\/s40593-025-00519-0"}],"container-title":["Lecture Notes in Computer Science","Artificial Intelligence in Education"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-032-29773-0_9","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,6,26]],"date-time":"2026-06-26T12:22:32Z","timestamp":1782476552000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-3-032-29773-0_9"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026,6,27]]},"ISBN":["9783032297723","9783032297730"],"references-count":53,"URL":"https:\/\/doi.org\/10.1007\/978-3-032-29773-0_9","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"value":"0302-9743","type":"print"},{"value":"1611-3349","type":"electronic"}],"subject":[],"published":{"date-parts":[[2026,6,27]]},"assertion":[{"value":"27 June 2026","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"AIED","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"International Conference on Artificial Intelligence in Education","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Seoul","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Korea (Republic of)","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2026","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"29 June 2026","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"3 July 2026","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"27","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"aied2026","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"https:\/\/www.aied-conference.org\/2026","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}}]}}