{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,9,11]],"date-time":"2025-09-11T19:18:24Z","timestamp":1757618304791,"version":"3.44.0"},"publisher-location":"Cham","reference-count":38,"publisher":"Springer Nature Switzerland","isbn-type":[{"type":"print","value":"9783031941528"},{"type":"electronic","value":"9783031941535"}],"license":[{"start":{"date-parts":[[2025,1,1]],"date-time":"2025-01-01T00:00:00Z","timestamp":1735689600000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2025,1,1]],"date-time":"2025-01-01T00:00:00Z","timestamp":1735689600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2025]]},"DOI":"10.1007\/978-3-031-94153-5_45","type":"book-chapter","created":{"date-parts":[[2025,6,7]],"date-time":"2025-06-07T09:39:14Z","timestamp":1749289154000},"page":"454-464","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":0,"title":["Multimodal Situational Awareness: Neuro-Symbolic AI for\u00a0Real-Time HCI in\u00a0the\u00a0Classroom"],"prefix":"10.1007","author":[{"ORCID":"https:\/\/orcid.org\/0009-0000-6597-7275","authenticated-orcid":false,"given":"Yifan","family":"Zhu","sequence":"first","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0003-2870-7019","authenticated-orcid":false,"given":"Kenneth","family":"Lai","sequence":"additional","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0009-0009-4374-7263","authenticated-orcid":false,"given":"Ibrahim","family":"Khebour","sequence":"additional","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0002-2284-8163","authenticated-orcid":false,"given":"Marc","family":"Verhagen","sequence":"additional","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0001-7878-7227","authenticated-orcid":false,"given":"Nikhil","family":"Krishnaswamy","sequence":"additional","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0003-2233-9761","authenticated-orcid":false,"given":"James","family":"Pustejovsky","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2025,5,30]]},"reference":[{"issue":"3","key":"45_CR1","doi-asserted-by":"publisher","first-page":"389","DOI":"10.1080\/0309877X.2017.1281887","volume":"42","author":"PT Balwant","year":"2018","unstructured":"Balwant, P.T.: The meaning of student engagement and disengagement in the classroom context: lessons from organisational behaviour. J. Furth. High. Educ. 42(3), 389\u2013401 (2018)","journal-title":"J. Furth. High. Educ."},{"issue":"1","key":"45_CR2","doi-asserted-by":"publisher","first-page":"106","DOI":"10.1016\/j.apal.2013.07.007","volume":"165","author":"J van Benthem","year":"2014","unstructured":"van Benthem, J., Fern\u00e1ndez-Duque, D., Pacuit, E.: Evidence and plausibility in neighborhood structures. Ann. Pure Appl. Logic 165(1), 106\u2013133 (2014)","journal-title":"Ann. Pure Appl. Logic"},{"key":"45_CR3","doi-asserted-by":"publisher","DOI":"10.1016\/j.lindif.2024.102601","volume":"118","author":"A Bewersdorff","year":"2025","unstructured":"Bewersdorff, A., et al.: Taking the next step with generative artificial intelligence: the transformative role of multimodal large language models in science education. Learn. Individ. Differ. 118, 102601 (2025)","journal-title":"Learn. Individ. Differ."},{"issue":"4","key":"45_CR4","doi-asserted-by":"publisher","first-page":"588","DOI":"10.1109\/TASE.2008.2004965","volume":"6","author":"O Brdiczka","year":"2009","unstructured":"Brdiczka, O., Langet, M., Maisonnasse, J., Crowley, J.L.: Detecting human behavior models from multimodal observation in a smart home. IEEE Trans. Autom. Sci. Eng. 6(4), 588\u2013597 (2009). https:\/\/doi.org\/10.1109\/TASE.2008.2004965","journal-title":"IEEE Trans. Autom. Sci. Eng."},{"key":"45_CR5","doi-asserted-by":"crossref","unstructured":"Bredin, H., et al.: Pyannote. audio: neural building blocks for speaker diarization. In: ICASSP 2020-2020 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP), pp. 7124\u20137128. IEEE (2020)","DOI":"10.1109\/ICASSP40776.2020.9052974"},{"key":"45_CR6","doi-asserted-by":"publisher","unstructured":"B\u00f6ck, R., et al.: Intraindividual and interindividual multimodal emotion analyses in human-machine-interaction. In: 2012 IEEE International Multi-Disciplinary Conference on Cognitive Methods in Situation Awareness and Decision Support, pp. 59\u201364 (2012). https:\/\/doi.org\/10.1109\/CogSIMA.2012.6188409","DOI":"10.1109\/CogSIMA.2012.6188409"},{"issue":"3","key":"45_CR7","doi-asserted-by":"publisher","first-page":"2551","DOI":"10.1109\/TIV.2022.3200826","volume":"8","author":"M Capallera","year":"2023","unstructured":"Capallera, M., Angelini, L., Meteier, Q., Khaled, O.A., Mugellini, E.: Human-vehicle interaction to support driver\u2019s situation awareness in automated vehicles: a systematic review. IEEE Trans. Intell. Veh. 8(3), 2551\u20132567 (2023). https:\/\/doi.org\/10.1109\/TIV.2022.3200826","journal-title":"IEEE Trans. Intell. Veh."},{"key":"45_CR8","doi-asserted-by":"crossref","unstructured":"Chen, J., Seng, K.P., Smith, J., Ang, L.M.: Situation awareness in AI-based technologies and multimodal systems: architectures, challenges and applications. IEEE Access (2024)","DOI":"10.1109\/ACCESS.2024.3416370"},{"key":"45_CR9","doi-asserted-by":"publisher","unstructured":"DAniello, G., Gravina, R., Gaeta, M., Fortino, G.: Situation-aware sensor-based wearable computing systems: a reference architecture-driven review. IEEE Sens. J. 22(14), 13853\u201313863 (2022). https:\/\/doi.org\/10.1109\/JSEN.2022.3180902","DOI":"10.1109\/JSEN.2022.3180902"},{"key":"45_CR10","doi-asserted-by":"publisher","first-page":"32","DOI":"10.1016\/j.learninstruc.2016.12.001","volume":"49","author":"SR Earl","year":"2017","unstructured":"Earl, S.R., Taylor, I.M., Meijen, C., Passfield, L.: Autonomy and competence frustration in young adolescent classrooms: different associations with active and passive disengagement. Learn. Instr. 49, 32\u201340 (2017)","journal-title":"Learn. Instr."},{"key":"45_CR11","doi-asserted-by":"publisher","unstructured":"Endsley, M.R.: A systematic review and meta-analysis of direct objective measures of situation awareness: a comparison of SAGAT and SPAM. Hum. Fact. 63(1), 124\u2013150 (2021). https:\/\/doi.org\/10.1177\/0018720819875376, pMID: 31560575","DOI":"10.1177\/0018720819875376"},{"key":"45_CR12","doi-asserted-by":"publisher","DOI":"10.1016\/j.artint.2024.104111","volume":"331","author":"S Goel","year":"2024","unstructured":"Goel, S., et al.: A neurosymbolic cognitive architecture framework for handling novelties in open worlds. Artif. Intell. 331, 104111 (2024)","journal-title":"Artif. Intell."},{"issue":"4","key":"45_CR13","doi-asserted-by":"publisher","first-page":"645","DOI":"10.1109\/TMM.2009.2017631","volume":"11","author":"RM Hegde","year":"2009","unstructured":"Hegde, R.M., Kurniawan, J., Rao, B.D.: On the design and prototype implementation of a multimodal situation aware system. IEEE Trans. Multimedia 11(4), 645\u2013657 (2009). https:\/\/doi.org\/10.1109\/TMM.2009.2017631","journal-title":"IEEE Trans. Multimedia"},{"key":"45_CR14","doi-asserted-by":"publisher","first-page":"59800","DOI":"10.1109\/ACCESS.2021.3070212","volume":"9","author":"G Joshi","year":"2021","unstructured":"Joshi, G., Walambe, R., Kotecha, K.: A review on explainability in multimodal deep neural nets. IEEE Access 9, 59800\u201359821 (2021). https:\/\/doi.org\/10.1109\/ACCESS.2021.3070212","journal-title":"IEEE Access"},{"issue":"4","key":"45_CR15","doi-asserted-by":"publisher","first-page":"3281","DOI":"10.1109\/TITS.2020.3035374","volume":"23","author":"U Ju","year":"2022","unstructured":"Ju, U., Chuang, L.L., Wallraven, C.: Acoustic cues increase situational awareness in accident situations: a VR car-driving study. IEEE Trans. Intell. Transp. Syst. 23(4), 3281\u20133291 (2022). https:\/\/doi.org\/10.1109\/TITS.2020.3035374","journal-title":"IEEE Trans. Intell. Transp. Syst."},{"key":"45_CR16","doi-asserted-by":"publisher","DOI":"10.1016\/j.ress.2023.109793","volume":"242","author":"Y Keshun","year":"2024","unstructured":"Keshun, Y., Guangqi, Q., Yingkui, G.: Optimizing prior distribution parameters for probabilistic prediction of remaining useful life using deep learning. Reliab. Eng. Syst. Saf. 242, 109793 (2024)","journal-title":"Reliab. Eng. Syst. Saf."},{"key":"45_CR17","unstructured":"Khebour, I., et\u00a0al.: Common ground tracking in multimodal dialogue. arXiv preprint arXiv:2403.17284 (2024)"},{"key":"45_CR18","doi-asserted-by":"crossref","unstructured":"Krishnaswamy, N., et al.: Diana\u2019s world: a situated multimodal interactive agent. In: Proceedings of the AAAI Conference on Artificial Intelligence, vol.\u00a034, pp. 13618\u201313619 (2020)","DOI":"10.1609\/aaai.v34i09.7096"},{"issue":"6","key":"45_CR19","doi-asserted-by":"publisher","first-page":"6040","DOI":"10.1007\/s11227-022-04882-w","volume":"79","author":"CE Lee","year":"2023","unstructured":"Lee, C.E., Baek, J., Son, J., Ha, Y.G.: Deep AI military staff: cooperative battlefield situation awareness for commanders decision making. J. Supercomput. 79(6), 6040\u20136069 (2023)","journal-title":"J. Supercomput."},{"key":"45_CR20","doi-asserted-by":"publisher","DOI":"10.1016\/j.knosys.2022.110179","volume":"260","author":"Q Li","year":"2023","unstructured":"Li, Q., Ng, K.K., Simon, C., Yiu, C.Y., Lyu, M.: Recognising situation awareness associated with different workloads using EEG and eye-tracking features in air traffic control tasks. Knowl.-Based Syst. 260, 110179 (2023)","journal-title":"Knowl.-Based Syst."},{"issue":"5","key":"45_CR21","doi-asserted-by":"publisher","first-page":"053103","DOI":"10.1117\/1.OE.59.5.053103","volume":"59","author":"S Liu","year":"2020","unstructured":"Liu, S., Liu, H., John, V., Liu, Z., Blasch, E.: Enhanced situation awareness through CNN-based deep multimodal image fusion. Opt. Eng. 59(5), 053103\u2013053103 (2020)","journal-title":"Opt. Eng."},{"key":"45_CR22","unstructured":"Lugaresi, C., et\u00a0al.: MediaPipe: a framework for building perception pipelines. arXiv preprint arXiv:1906.08172 (2019)"},{"issue":"1","key":"45_CR23","doi-asserted-by":"publisher","first-page":"320","DOI":"10.1109\/TAFFC.2020.3000510","volume":"13","author":"S Mai","year":"2022","unstructured":"Mai, S., Hu, H., Xu, J., Xing, S.: Multi-fusion residual memory network for multimodal human sentiment comprehension. IEEE Trans. Affect. Comput. 13(1), 320\u2013334 (2022). https:\/\/doi.org\/10.1109\/TAFFC.2020.3000510","journal-title":"IEEE Trans. Affect. Comput."},{"key":"45_CR24","doi-asserted-by":"crossref","unstructured":"Munir, A., Aved, A., Blasch, E.: Situational awareness: techniques, challenges, and prospects. AI 3(1), 55\u201377 (2022)","DOI":"10.3390\/ai3010005"},{"key":"45_CR25","unstructured":"Nelson, L.M.: Collaborative problem solving. In: Instructional-Design Theories and Models, pp. 241\u2013267. Routledge (2013)"},{"key":"45_CR26","doi-asserted-by":"crossref","unstructured":"Pacuit, E.: Neighborhood Semantics for Modal Logic. Springer (2017)","DOI":"10.1007\/978-3-319-67149-9"},{"key":"45_CR27","doi-asserted-by":"publisher","unstructured":"Paul, P.P., Gavrilova, M., Klimenko, S.: Situation awareness through multimodal biometric template security in real-time environments. In: 2013 International Conference on Cyberworlds, pp. 82\u201388 (2013). https:\/\/doi.org\/10.1109\/CW.2013.80","DOI":"10.1109\/CW.2013.80"},{"issue":"3","key":"45_CR28","doi-asserted-by":"publisher","first-page":"307","DOI":"10.1007\/s13218-021-00727-5","volume":"35","author":"J Pustejovsky","year":"2021","unstructured":"Pustejovsky, J., Krishnaswamy, N.: Embodied human computer interaction. KI-K\u00fcnstliche Intelligenz 35(3), 307\u2013327 (2021)","journal-title":"KI-K\u00fcnstliche Intelligenz"},{"key":"45_CR29","doi-asserted-by":"crossref","unstructured":"Pustejovsky, J., Krishnaswamy, N.: Multimodal semantics for affordances and actions. In: International Conference on Human-Computer Interaction, pp. 137\u2013160. Springer (2022)","DOI":"10.1007\/978-3-031-05311-5_9"},{"key":"45_CR30","doi-asserted-by":"crossref","unstructured":"Pustejovsky, J., Zhu, Y.: Lexical event models for multimodal dialogues. In: International Conference on Human-Computer Interaction, pp. 174\u2013192. Springer (2025)","DOI":"10.1007\/978-3-031-76803-3_10"},{"key":"45_CR31","unstructured":"Radford, A., Kim, J.W., Xu, T., Brockman, G., McLeavey, C., Sutskever, I.: Robust speech recognition via large-scale weak supervision. In: International Conference on Machine Learning, pp. 28492\u201328518. PMLR (2023)"},{"key":"45_CR32","doi-asserted-by":"crossref","unstructured":"Rim, K., Tu, J., Ye, B., Verhagen, M., Holderness, E., Pustejovsky, J.: The coreference under transformation labeling dataset: entity tracking in procedural texts using event models. In: Findings of the Association for Computational Linguistics: ACL 2023, pp. 12448\u201312460 (2023)","DOI":"10.18653\/v1\/2023.findings-acl.788"},{"key":"45_CR33","doi-asserted-by":"publisher","unstructured":"Saad, A., Fouad, H., Mohamed, A.A.: Situation-aware recommendation system for personalized healthcare applications. J. Ambient. Intell. Humaniz. Comput., 1\u201315 (2021). https:\/\/doi.org\/10.1007\/s12652-021-02927-1","DOI":"10.1007\/s12652-021-02927-1"},{"key":"45_CR34","unstructured":"Southwell, R., et al.: Challenges and feasibility of automatic speech recognition for modeling student collaborative discourse in classrooms. Int. Educ. Data Mining Soc. (2022)"},{"key":"45_CR35","doi-asserted-by":"publisher","first-page":"1479905","DOI":"10.3389\/frai.2024.1479905","volume":"7","author":"J Tu","year":"2024","unstructured":"Tu, J., Rim, K., Ye, B., Lai, K., Pustejovsky, J.: Dense paraphrasing for multimodal dialogue interpretation. Front. Artif. Intell. 7, 1479905 (2024)","journal-title":"Front. Artif. Intell."},{"issue":"5","key":"45_CR36","doi-asserted-by":"publisher","first-page":"4999","DOI":"10.1109\/TIE.2023.3288182","volume":"71","author":"H Yang","year":"2024","unstructured":"Yang, H., Wu, J., Hu, Z., Lv, C.: Real-time driver cognitive workload recognition: attention-enabled learning with multimodal information fusion. IEEE Trans. Industr. Electron. 71(5), 4999\u20135009 (2024). https:\/\/doi.org\/10.1109\/TIE.2023.3288182","journal-title":"IEEE Trans. Industr. Electron."},{"issue":"2","key":"45_CR37","doi-asserted-by":"publisher","first-page":"270","DOI":"10.1109\/THMS.2023.3234429","volume":"53","author":"J Yang","year":"2023","unstructured":"Yang, J., et al.: Multimodal sensing and computational intelligence for situation awareness classification in autonomous driving. IEEE Trans. Hum.-Mach. Syst. 53(2), 270\u2013281 (2023). https:\/\/doi.org\/10.1109\/THMS.2023.3234429","journal-title":"IEEE Trans. Hum.-Mach. Syst."},{"key":"45_CR38","doi-asserted-by":"crossref","unstructured":"Zhu, Y., et al.: Modeling theory of mind in multimodal HCI. In: International Conference on Human-Computer Interaction, pp. 205\u2013225. Springer (2024)","DOI":"10.1007\/978-3-031-60405-8_14"}],"container-title":["Communications in Computer and Information Science","HCI International 2025 Posters"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-031-94153-5_45","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,9,6]],"date-time":"2025-09-06T18:17:15Z","timestamp":1757182635000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-3-031-94153-5_45"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025]]},"ISBN":["9783031941528","9783031941535"],"references-count":38,"URL":"https:\/\/doi.org\/10.1007\/978-3-031-94153-5_45","relation":{},"ISSN":["1865-0929","1865-0937"],"issn-type":[{"type":"print","value":"1865-0929"},{"type":"electronic","value":"1865-0937"}],"subject":[],"published":{"date-parts":[[2025]]},"assertion":[{"value":"30 May 2025","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"HCII","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"International Conference on Human-Computer Interaction","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Gothenburg","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Sweden","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2025","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"22 June 2025","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"27 June 2025","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"27","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"hcii2025","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"https:\/\/2025.hci.international\/","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}}]}}