{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,13]],"date-time":"2026-04-13T22:17:15Z","timestamp":1776118635718,"version":"3.50.1"},"publisher-location":"Cham","reference-count":47,"publisher":"Springer Nature Switzerland","isbn-type":[{"value":"9783031938634","type":"print"},{"value":"9783031938641","type":"electronic"}],"license":[{"start":{"date-parts":[[2025,1,1]],"date-time":"2025-01-01T00:00:00Z","timestamp":1735689600000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2025,1,1]],"date-time":"2025-01-01T00:00:00Z","timestamp":1735689600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2025]]},"DOI":"10.1007\/978-3-031-93864-1_12","type":"book-chapter","created":{"date-parts":[[2025,6,3]],"date-time":"2025-06-03T09:14:59Z","timestamp":1748942099000},"page":"169-186","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":3,"title":["Human-in-the-Loop Annotation for\u00a0Image-Based Engagement Estimation: Assessing the\u00a0Impact of\u00a0Model Reliability on\u00a0Annotation Accuracy"],"prefix":"10.1007","author":[{"ORCID":"https:\/\/orcid.org\/0009-0002-9504-7479","authenticated-orcid":false,"given":"Sahana","family":"Yadnakudige Subramanya","sequence":"first","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0003-0252-1785","authenticated-orcid":false,"given":"Ko","family":"Watanabe","sequence":"additional","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0002-6100-8255","authenticated-orcid":false,"given":"Andreas","family":"Dengel","sequence":"additional","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0002-5374-1510","authenticated-orcid":false,"given":"Shoya","family":"Ishimaru","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2025,6,4]]},"reference":[{"key":"12_CR1","unstructured":"Ali, S., Tanweer, S., Khalid, S.S., Rao, N.: Mel frequency cepstral coefficient: a review. In: ICIDSSD (2020)"},{"issue":"4","key":"12_CR2","first-page":"105","volume":"35","author":"S Amershi","year":"2014","unstructured":"Amershi, S., Cakmak, M., Knox, W.B., Kulesza, T.: Power to the people: the role of humans in interactive machine learning. AI Mag. 35(4), 105\u2013120 (2014)","journal-title":"AI Mag."},{"issue":"3","key":"12_CR3","first-page":"277","volume":"11","author":"T Barbu","year":"2010","unstructured":"Barbu, T.: Gabor filter-based face recognition technique. Proc. Rom. Acad. 11(3), 277\u2013283 (2010)","journal-title":"Proc. Rom. Acad."},{"key":"12_CR4","doi-asserted-by":"crossref","unstructured":"Berger, V.W., Zhou, Y.: Kolmogorov-smirnov test: Overview. Statistics reference online, Wiley statsref (2014)","DOI":"10.1002\/9781118445112.stat06558"},{"issue":"3","key":"12_CR5","doi-asserted-by":"publisher","first-page":"323","DOI":"10.1016\/j.jkss.2012.10.002","volume":"42","author":"W Bergsma","year":"2013","unstructured":"Bergsma, W.: A bias-correction for cram\u00e9r\u2019s v and tschuprow\u2019s t. J. Korean Stat. Soc. 42(3), 323\u2013328 (2013)","journal-title":"J. Korean Stat. Soc."},{"key":"12_CR6","doi-asserted-by":"publisher","first-page":"192219","DOI":"10.1109\/ACCESS.2024.3515838","volume":"12","author":"A Bhatt","year":"2024","unstructured":"Bhatt, A., Watanabe, K., Santhosh, J., Dengel, A., Ishimaru, S.: Estimating self-confidence in video-based learning using eye-tracking and deep neural networks. IEEE Access 12, 192219\u2013192229 (2024). https:\/\/doi.org\/10.1109\/ACCESS.2024.3515838","journal-title":"IEEE Access"},{"key":"12_CR7","doi-asserted-by":"crossref","unstructured":"Bosch, N., et al.: Automatic detection of learning-centered affective states in the wild. In: Proceedings of the 20th International Conference on Intelligent User Interfaces, pp. 379\u2013388 (2015)","DOI":"10.1145\/2678025.2701397"},{"key":"12_CR8","doi-asserted-by":"crossref","unstructured":"Chen, C., Arakawa, Y., Watanabe, K., Ishimaru, S.: Quantitative evaluation system for online meetings based on multimodal microbehavior analysis. Sens. Mater. 34 (2022)","DOI":"10.18494\/SAM3959"},{"issue":"1","key":"12_CR9","doi-asserted-by":"publisher","first-page":"6991","DOI":"10.1038\/s41598-022-11173-0","volume":"12","author":"T Debnath","year":"2022","unstructured":"Debnath, T., Reza, M.M., Rahman, A., Beheshti, A., Band, S.S., Alinejad-Rokny, H.: Four-layer convnet to facial emotion recognition with minimal epochs and the significance of data diversity. Sci. Rep. 12(1), 6991 (2022)","journal-title":"Sci. Rep."},{"key":"12_CR10","doi-asserted-by":"crossref","unstructured":"D\u2019mello, S., Graesser, A.: Autotutor and affective autotutor: learning by talking with cognitively and emotionally intelligent computers that talk back. ACM Trans. Interact. Intell. Syst. (TiiS) 2(4), 1\u201339 (2013)","DOI":"10.1145\/2395123.2395128"},{"issue":"3","key":"12_CR11","doi-asserted-by":"publisher","first-page":"572","DOI":"10.1016\/j.patcog.2010.09.020","volume":"44","author":"M El Ayadi","year":"2011","unstructured":"El Ayadi, M., Kamel, M.S., Karray, F.: Survey on speech emotion recognition: features, classification schemes, and databases. Pattern Recogn. 44(3), 572\u2013587 (2011)","journal-title":"Pattern Recogn."},{"issue":"2","key":"12_CR12","doi-asserted-by":"publisher","first-page":"203","DOI":"10.1037\/0033-2909.128.2.203","volume":"128","author":"HA Elfenbein","year":"2002","unstructured":"Elfenbein, H.A., Ambady, N.: On the universality and cultural specificity of emotion recognition: a meta-analysis. Psychol. Bull. 128(2), 203 (2002)","journal-title":"Psychol. Bull."},{"key":"12_CR13","doi-asserted-by":"crossref","unstructured":"Erat, K., \u015eahin, E.B., Do\u011fan, F., Merdano\u011flu, N., Akcakaya, A., Durdu, P.O.: Emotion recognition with eeg-based brain-computer interfaces: a systematic literature review. Multimedia Tools Appl. 1\u201348 (2024)","DOI":"10.1007\/s11042-024-18259-z"},{"key":"12_CR14","doi-asserted-by":"crossref","unstructured":"Gong, X., Chen, C.P., Zhang, T.: Cross-cultural emotion recognition with eeg and eye movement signals based on multiple stacked broad learning system. IEEE Trans. Comput. Soc. Syst. (2023)","DOI":"10.1109\/TCSS.2023.3298324"},{"key":"12_CR15","unstructured":"Gupta, A., D\u2019Cunha, A., Awasthi, K., Balasubramanian, V.: Daisee: towards user engagement recognition in the wild. arXiv preprint arXiv:1609.01885 (2016)"},{"key":"12_CR16","doi-asserted-by":"publisher","first-page":"36","DOI":"10.1016\/j.compedu.2015.09.005","volume":"90","author":"CR Henrie","year":"2015","unstructured":"Henrie, C.R., Halverson, L.R., Graham, C.R.: Measuring student engagement in technology-mediated learning: a review. Comput. Educ. 90, 36\u201353 (2015)","journal-title":"Comput. Educ."},{"key":"12_CR17","doi-asserted-by":"crossref","unstructured":"Holstein, K., Wortman\u00a0Vaughan, J., Daum\u00e9\u00a0III, H., Dudik, M., Wallach, H.: Improving fairness in machine learning systems: what do industry practitioners need? In: Proceedings of the 2019 CHI Conference on Human Factors in Computing Systems, pp. 1\u201316 (2019)","DOI":"10.1145\/3290605.3300830"},{"key":"12_CR18","doi-asserted-by":"publisher","unstructured":"John, B.: Pupil diameter as a measure of emotion and sickness in vr. In: Proceedings of the 11th ACM Symposium on Eye Tracking Research & Applications. ETRA 2019, Association for Computing Machinery, New York, NY, USA (2019). https:\/\/doi.org\/10.1145\/3314111.3322868","DOI":"10.1145\/3314111.3322868"},{"issue":"2","key":"12_CR19","doi-asserted-by":"publisher","first-page":"449","DOI":"10.1007\/s10758-021-09547-w","volume":"27","author":"FG Karaoglan Yilmaz","year":"2022","unstructured":"Karaoglan Yilmaz, F.G., Yilmaz, R.: Learning analytics intervention improves students\u2019 engagement in online learning. Technol. Knowl. Learn. 27(2), 449\u2013460 (2022)","journal-title":"Technol. Knowl. Learn."},{"key":"12_CR20","doi-asserted-by":"crossref","unstructured":"Lim, W., Jang, D., Lee, T.: Speech emotion recognition using convolutional and recurrent neural networks. In: 2016 Asia-Pacific Signal and Information Processing Association Annual Summit and Conference (APSIPA), pp.\u00a01\u20134. IEEE (2016)","DOI":"10.1109\/APSIPA.2016.7820699"},{"issue":"1","key":"12_CR21","doi-asserted-by":"publisher","first-page":"8414","DOI":"10.1038\/s41598-023-34932-z","volume":"13","author":"M Lukac","year":"2023","unstructured":"Lukac, M., Zhambulova, G., Abdiyeva, K., Lewis, M.: Study on emotion recognition bias in different regional groups. Sci. Rep. 13(1), 8414 (2023)","journal-title":"Sci. Rep."},{"key":"12_CR22","doi-asserted-by":"publisher","unstructured":"Luo, Y., Yu, J., Liang, M., Wan, Y., Zhu, K., Santosa, S.S.: Emotion embodied: unveiling the expressive potential of single-hand gestures. In: Proceedings of the 2024 CHI Conference on Human Factors in Computing Systems, CHI 2024, Association for Computing Machinery, New York, NY, USA (2024). https:\/\/doi.org\/10.1145\/3613904.3642255","DOI":"10.1145\/3613904.3642255"},{"key":"12_CR23","unstructured":"Markus, H.R., Kitayama, S.: Culture and the self: implications for cognition, emotion, and motivation. In: College Student Development and Academic Life, pp. 264\u2013293. Routledge (2014)"},{"issue":"11","key":"12_CR24","doi-asserted-by":"publisher","first-page":"3978","DOI":"10.3390\/s18113978","volume":"18","author":"Y Matsuda","year":"2018","unstructured":"Matsuda, Y., Fedotov, D., Takahashi, Y., Arakawa, Y., Yasumoto, K., Minker, W.: Emotour: estimating emotion and satisfaction of users based on behavioral cues and audiovisual data. Sensors 18(11), 3978 (2018)","journal-title":"Sensors"},{"issue":"2","key":"12_CR25","doi-asserted-by":"publisher","first-page":"143","DOI":"10.11613\/BM.2013.018","volume":"23","author":"ML McHugh","year":"2013","unstructured":"McHugh, M.L.: The chi-square test of independence. Biochemia medica 23(2), 143\u2013149 (2013)","journal-title":"Biochemia medica"},{"key":"12_CR26","doi-asserted-by":"crossref","unstructured":"McKight, P.E., Najab, J.: Kruskal-wallis test. The corsini encyclopedia of psychology pp.\u00a01\u20131 (2010)","DOI":"10.1002\/9780470479216.corpsy0491"},{"key":"12_CR27","doi-asserted-by":"publisher","first-page":"555913","DOI":"10.3389\/frobt.2021.555913","volume":"8","author":"C Oertel","year":"2021","unstructured":"Oertel, C., Jonell, P., Kontogiorgos, D., Mora, K.F., Odobez, J.M., Gustafson, J.: Towards an engagement-aware attentive artificial listener for multi-party interactions. Front. Robot. AI 8, 555913 (2021)","journal-title":"Front. Robot. AI"},{"issue":"1","key":"12_CR28","doi-asserted-by":"publisher","first-page":"22","DOI":"10.1080\/00220973.2012.745471","volume":"82","author":"C Peng","year":"2014","unstructured":"Peng, C., Chen, L.T.: Beyond cohen\u2019s d: alternative effect size measures for between-subject designs. J. Exp. Educ. 82(1), 22\u201350 (2014)","journal-title":"J. Exp. Educ."},{"key":"12_CR29","unstructured":"Salam, H., Celiktutan, O., Gunes, H., Chetouani, M.: Automatic context-driven inference of engagement in hmi: a survey. arXiv preprint arXiv:2209.15370 (2022)"},{"key":"12_CR30","doi-asserted-by":"crossref","unstructured":"Sharma, K., Jermann, P., Dillenbourg, P.: Displaying teacher\u2019s gaze in a mooc: effects on students\u2019 video navigation patterns. In: Design for Teaching and Learning in a Networked World: 10th European Conference on Technology Enhanced Learning, EC-TEL 2015, Toledo, Spain, 15\u201318 September 2015, Proceedings 10, pp. 325\u2013338. Springer (2015)","DOI":"10.1007\/978-3-319-24258-3_24"},{"issue":"7","key":"12_CR31","doi-asserted-by":"publisher","first-page":"2074","DOI":"10.3390\/s18072074","volume":"18","author":"L Shu","year":"2018","unstructured":"Shu, L., et al.: A review of emotion recognition using physiological signals. Sensors 18(7), 2074 (2018)","journal-title":"Sensors"},{"issue":"4","key":"12_CR32","doi-asserted-by":"publisher","first-page":"259","DOI":"10.1016\/0169-7439(89)80095-4","volume":"6","author":"L St","year":"1989","unstructured":"St, L., Wold, S., et al.: Analysis of variance (anova). Chemom. Intell. Lab. Syst. 6(4), 259\u2013272 (1989)","journal-title":"Chemom. Intell. Lab. Syst."},{"key":"12_CR33","doi-asserted-by":"crossref","unstructured":"Taherisadr, M., Al\u00a0Faruque, M.A., Elmalaki, S.: Erudite: human-in-the-loop IoT for an adaptive personalized learning system. IEEE Internet Things J. (2023)","DOI":"10.1109\/JIOT.2023.3343462"},{"key":"12_CR34","doi-asserted-by":"crossref","unstructured":"Tanaka, N., Watanabe, K., Ishimaru, S., Dengel, A., Ata, S., Fujimoto, M.: Concentration estimation in online video lecture using multimodal sensors. In: Companion of the 2024 on ACM International Joint Conference on Pervasive and Ubiquitous Computing, pp. 71\u201375 (2024)","DOI":"10.1145\/3675094.3677587"},{"key":"12_CR35","doi-asserted-by":"crossref","unstructured":"Vairamani, A.D.: Advancements in multimodal emotion recognition: integrating facial expressions and physiological signals. In: Affective Computing for Social Good: Enhancing Well-being, Empathy, and Equity, pp. 217\u2013240. Springer (2024)","DOI":"10.1007\/978-3-031-63821-3_12"},{"key":"12_CR36","doi-asserted-by":"publisher","first-page":"251","DOI":"10.1016\/j.neucom.2018.05.107","volume":"312","author":"SJ Wang","year":"2018","unstructured":"Wang, S.J., et al.: Micro-expression recognition with small sample size by transferring long-term convolutional neural network. Neurocomputing 312, 251\u2013262 (2018)","journal-title":"Neurocomputing"},{"key":"12_CR37","doi-asserted-by":"publisher","first-page":"1126994","DOI":"10.3389\/fpsyg.2023.1126994","volume":"14","author":"X Wang","year":"2023","unstructured":"Wang, X., Ren, Y., Luo, Z., He, W., Hong, J., Huang, Y.: Deep learning-based eeg emotion recognition: current trends and future perspectives. Front. Psychol. 14, 1126994 (2023)","journal-title":"Front. Psychol."},{"key":"12_CR38","doi-asserted-by":"crossref","unstructured":"Watanabe, K., Dengel, A., Ishimaru, S.: Metacognition-engauge: real-time augmentation of self-and-group engagement levels understanding by gauge interface in online meetings. In: Proceedings of the Augmented Humans International Conference 2024, pp. 301\u2013303 (2024)","DOI":"10.1145\/3652920.3653054"},{"key":"12_CR39","first-page":"52886","volume":"11","author":"K Watanabe","year":"2023","unstructured":"Watanabe, K., Sathyanarayana, T., Dengel, A., Ishimaru, S.: Engauge: engagement gauge of meeting participants estimated by facial expression and deep neural network. IEEE Access 11, 52886\u201352898 (2023)","journal-title":"IEEE Access"},{"issue":"17","key":"12_CR40","doi-asserted-by":"publisher","first-page":"5719","DOI":"10.3390\/s21175719","volume":"21","author":"K Watanabe","year":"2021","unstructured":"Watanabe, K., et al.: Discaas: micro behavior analysis on discussion by camera as a sensor. Sensors 21(17), 5719 (2021)","journal-title":"Sensors"},{"issue":"6","key":"12_CR41","doi-asserted-by":"publisher","first-page":"1304","DOI":"10.1111\/bjet.12324","volume":"47","author":"CH Wu","year":"2016","unstructured":"Wu, C.H., Huang, Y.M., Hwang, J.P.: Review of affective computing in education\/learning: trends and challenges. Br. J. Edu. Technol. 47(6), 1304\u20131323 (2016)","journal-title":"Br. J. Edu. Technol."},{"key":"12_CR42","doi-asserted-by":"publisher","first-page":"364","DOI":"10.1016\/j.future.2022.05.014","volume":"135","author":"X Wu","year":"2022","unstructured":"Wu, X., Xiao, L., Sun, Y., Zhang, J., Ma, T., He, L.: A survey of human-in-the-loop for machine learning. Futur. Gener. Comput. Syst. 135, 364\u2013381 (2022)","journal-title":"Futur. Gener. Comput. Syst."},{"key":"12_CR43","doi-asserted-by":"publisher","first-page":"108906","DOI":"10.1109\/ACCESS.2019.2930359","volume":"7","author":"K Yan","year":"2019","unstructured":"Yan, K., et al.: Cross-domain facial expression recognition based on transductive deep transfer learning. IEEE Access 7, 108906\u2013108915 (2019)","journal-title":"IEEE Access"},{"key":"12_CR44","doi-asserted-by":"crossref","unstructured":"Yigitbas, E., Karakaya, K., Jovanovikj, I., Engels, G.: Enhancing human-in-the-loop adaptive systems through digital twins and vr interfaces. In: 2021 International Symposium on Software Engineering for Adaptive and Self-Managing Systems (SEAMS), pp. 30\u201340. IEEE (2021)","DOI":"10.1109\/SEAMS51251.2021.00015"},{"key":"12_CR45","unstructured":"Younis, E.M., Mohsen, S., Houssein, E.H., Ibrahim, O.A.S.: Machine learning for human emotion recognition: a comprehensive review. Neural Comput. Appl. 1\u201347 (2024)"},{"key":"12_CR46","doi-asserted-by":"crossref","unstructured":"Zeng, Z., Pantic, M., Roisman, G.I., Huang, T.S.: A survey of affect recognition methods: audio, visual and spontaneous expressions. In: Proceedings of the 9th international conference on Multimodal interfaces, pp. 126\u2013133 (2007)","DOI":"10.1145\/1322192.1322216"},{"key":"12_CR47","doi-asserted-by":"crossref","unstructured":"Zhang, X., Xuan, X., Dima, A., Sexton, T., Ma, K.L.: Labelvizier: interactive validation and relabeling for technical text annotations. In: 2023 IEEE 16th Pacific Visualization Symposium (PacificVis), pp. 167\u2013176. IEEE (2023)","DOI":"10.1109\/PacificVis56936.2023.00026"}],"container-title":["Lecture Notes in Computer Science","Human-Computer Interaction"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-031-93864-1_12","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,3]],"date-time":"2025-06-03T09:15:20Z","timestamp":1748942120000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-3-031-93864-1_12"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025]]},"ISBN":["9783031938634","9783031938641"],"references-count":47,"URL":"https:\/\/doi.org\/10.1007\/978-3-031-93864-1_12","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"value":"0302-9743","type":"print"},{"value":"1611-3349","type":"electronic"}],"subject":[],"published":{"date-parts":[[2025]]},"assertion":[{"value":"4 June 2025","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"HCII","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"International Conference on Human-Computer Interaction","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Gothenburg","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Sweden","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2025","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"22 June 2025","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"27 June 2025","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"27","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"hcii2025","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"https:\/\/2025.hci.international\/","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}}]}}