{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,11]],"date-time":"2026-07-11T11:12:03Z","timestamp":1783768323333,"version":"3.55.0"},"reference-count":37,"publisher":"Springer Science and Business Media LLC","issue":"4","license":[{"start":{"date-parts":[[2026,5,6]],"date-time":"2026-05-06T00:00:00Z","timestamp":1778025600000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0"},{"start":{"date-parts":[[2026,5,6]],"date-time":"2026-05-06T00:00:00Z","timestamp":1778025600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0"}],"funder":[{"name":"he National Key Research and Development Program of China","award":["61906051"],"award-info":[{"award-number":["61906051"]}]},{"name":"the Key Laboratory of Education Blockchain and Intelligent Technology, Ministry of Education","award":["EBME24-06"],"award-info":[{"award-number":["EBME24-06"]}]}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Multimedia Systems"],"published-print":{"date-parts":[[2026,6]]},"DOI":"10.1007\/s00530-026-02299-6","type":"journal-article","created":{"date-parts":[[2026,5,6]],"date-time":"2026-05-06T05:51:49Z","timestamp":1778046709000},"update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":0,"title":["Exploiting multimodal video semantic hierarchy for emotion recognition in E-learning"],"prefix":"10.1007","volume":"32","author":[{"given":"Xingbing","family":"Li","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Luyao","family":"Huang","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Xiaomei","family":"Tao","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Tao","family":"Chen","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Yuting","family":"Zhang","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Yanling","family":"Gan","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Faliang","family":"Huang","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2026,5,6]]},"reference":[{"issue":"2","key":"2299_CR1","doi-asserted-by":"publisher","first-page":"173","DOI":"10.1109\/TLT.2021.3072159","volume":"14","author":"K Mangaroska","year":"2021","unstructured":"Mangaroska, K., Vesin, B., Kostakos, V., Brusilovsky, P., Giannakos, M.N.: Architecting analytics across multiple E-learning systems to enhance learning design. IEEE Trans. Learn. Technol. 14(2), 173\u2013188 (2021)","journal-title":"IEEE Trans. Learn. Technol."},{"issue":"3","key":"2299_CR2","doi-asserted-by":"publisher","first-page":"313","DOI":"10.1109\/TLT.2021.3083180","volume":"14","author":"X Tian","year":"2021","unstructured":"Tian, X., Liu, F.: Capacity tracing-enhanced course recommendation in MOOCs. IEEE Trans. Learn. Technol. 14(3), 313\u2013321 (2021)","journal-title":"IEEE Trans. Learn. Technol."},{"key":"2299_CR3","doi-asserted-by":"publisher","first-page":"364","DOI":"10.1109\/TLT.2023.3314399","volume":"17","author":"Y Wang","year":"2024","unstructured":"Wang, Y., Ma, D., Ma, J., Jin, Q.: HGCR: a heterogeneous graph-enhanced interactive course recommendation scheme for online learning. IEEE Trans. Learn. Technol. 17, 364\u2013374 (2024)","journal-title":"IEEE Trans. Learn. Technol."},{"issue":"5","key":"2299_CR4","doi-asserted-by":"publisher","first-page":"2883","DOI":"10.1007\/s11571-024-10123-y","volume":"18","author":"Y Yin","year":"2024","unstructured":"Yin, Y., Kong, W., Tang, J., Li, J., Babiloni, F.: PSPN: pseudo-SIAMESE pyramid network for multimodal emotion analysis. Cogn. Neurodyn. 18(5), 2883\u20132896 (2024). https:\/\/doi.org\/10.1007\/s11571-024-10123-y","journal-title":"Cogn. Neurodyn."},{"issue":"1","key":"2299_CR5","doi-asserted-by":"publisher","first-page":"171","DOI":"10.1109\/TCSS.2022.3221128","volume":"11","author":"J Bao","year":"2024","unstructured":"Bao, J., Tao, X., Zhou, Y.: An emotion recognition method based on eye movement and audiovisual features in MOOC learning environment. IEEE Trans. Comput. Soc. Syst. 11(1), 171\u2013183 (2024)","journal-title":"IEEE Trans. Comput. Soc. Syst."},{"key":"2299_CR6","unstructured":"Zhang, Y., Tao, X., Ai, H., Chen, T., Gan, Y.:  MultimodalEmotion Recognition by Fusing Video Semantics in Video LearningScenarios, in Neural Information Processing, M. Mahmud, M.Doborjeh, K. Wong, A. C. S. Leung, Z. Doborjeh, and M. Tanveer,Eds. Singapore: Springer Nature Singapore, pp. 1-15(2025)"},{"key":"2299_CR7","doi-asserted-by":"publisher","first-page":"12142","DOI":"10.1109\/TNNLS.2023.3252359","volume":"35","author":"X Sun","year":"2023","unstructured":"Sun, X., Yao, F., Ding, C.: Modeling high-order relationships: brain-inspired hypergraph-induced multimodal-multitask framework for semantic comprehension. IEEE Trans. Neural Netw. Learn. Syst. 35, 12142\u201312156 (2023)","journal-title":"IEEE Trans. Neural Netw. Learn. Syst."},{"key":"2299_CR8","doi-asserted-by":"crossref","unstructured":"Dey, A., Dasgupta, K.: Emotion recognition using deep learning in pandemic with real-time email alert. In: Proceedings of Third International Conference on Communication, Computing and Electronics Systems: ICCCES 2021, pp. 175\u2013190. Springer (2022)","DOI":"10.1007\/978-981-16-8862-1_13"},{"issue":"14","key":"2299_CR9","doi-asserted-by":"publisher","first-page":"4913","DOI":"10.3390\/s21144913","volume":"21","author":"B Xie","year":"2021","unstructured":"Xie, B., Sidulova, M., Park, C.H.: Robust multimodal emotion recognition from conversation with transformer-based crossmodality fusion. Sensors 21(14), 4913 (2021)","journal-title":"Sensors"},{"issue":"3","key":"2299_CR10","doi-asserted-by":"publisher","first-page":"142","DOI":"10.1007\/s00530-024-01345-5","volume":"30","author":"L Zhao","year":"2024","unstructured":"Zhao, L., Yang, Y., Ning, T.: A Three-stage multimodal emotion recognition network based on text low-rank fusion. Multimedia Syst. 30(3), 142 (2024). https:\/\/doi.org\/10.1007\/s00530-024-01345-5","journal-title":"Multimedia Syst."},{"issue":"11","key":"2299_CR11","doi-asserted-by":"publisher","first-page":"688","DOI":"10.3390\/bioengineering9110688","volume":"9","author":"Z Ahmad","year":"2022","unstructured":"Ahmad, Z., Khan, N.: A survey on physiological signal-based emotion recognition. Bioengineering 9(11), 688 (2022)","journal-title":"Bioengineering"},{"issue":"07","key":"2299_CR12","doi-asserted-by":"publisher","first-page":"2350125","DOI":"10.1142\/S0218126623501256","volume":"32","author":"Y Hu","year":"2023","unstructured":"Hu, Y., Wang, F.: Multi-modal emotion recognition combining face image and EEG signal. J. Circuits Syst. Comput. 32(07), 2350125 (2023)","journal-title":"J. Circuits Syst. Comput."},{"key":"2299_CR13","doi-asserted-by":"publisher","DOI":"10.1016\/j.inffus.2023.102220","volume":"105","author":"J Li","year":"2024","unstructured":"Li, J., Chen, N., Zhu, H., Li, G., Xu, Z., Chen, D.: Incongruity-aware multimodal physiology signals fusion for emotion recognition. Inf. Fusion 105, 102220 (2024)","journal-title":"Inf. Fusion"},{"key":"2299_CR14","doi-asserted-by":"publisher","first-page":"106353","DOI":"10.1016\/j.bspc.2024.106353","volume":"94","author":"N Gahlan","year":"2024","unstructured":"Gahlan, N., Sethia, D.: AFLEMP: attention-based federated learning for emotion recognition using multi-modal physiological data. Biomed. Signal Process. Control 94, 106353 (2024)","journal-title":"Biomed. Signal Process. Control"},{"issue":"2","key":"2299_CR15","doi-asserted-by":"publisher","first-page":"1709","DOI":"10.32604\/csse.2023.039186","volume":"47","author":"H Ye","year":"2023","unstructured":"Ye, H., Zhou, Y., Tao, X.: A method of multimodal emotion recognition in video learning based on knowledge enhancement. Comput. Syst. Sci. Eng. 47(2), 1709\u20131732 (2023)","journal-title":"Comput. Syst. Sci. Eng."},{"issue":"5","key":"2299_CR16","doi-asserted-by":"publisher","first-page":"1475","DOI":"10.18280\/ts.390503","volume":"39","author":"C Zhu","year":"2022","unstructured":"Zhu, C., Ding, T., Min, X.: Emotion recognition of college students based on audio and video image. Traitement du Signal 39(5), 1475\u20131481 (2022). https:\/\/doi.org\/10.18280\/ts.390503","journal-title":"Traitement du Signal"},{"issue":"5s","key":"2299_CR17","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1145\/3583690","volume":"19","author":"H Liu","year":"2023","unstructured":"Liu, H., Yang, X., Xu, C.: Counterfactual scenario-relevant knowledge-enriched multi-modal emotion reasoning. ACM Trans. Multimed. Comput. Commun. Appl. 19(5s), 1\u201325 (2023). https:\/\/doi.org\/10.1145\/3583690","journal-title":"ACM Trans. Multimed. Comput. Commun. Appl."},{"key":"2299_CR18","doi-asserted-by":"publisher","first-page":"5933","DOI":"10.1109\/TIP.2021.3090521","volume":"30","author":"Y Hu","year":"2021","unstructured":"Hu, Y., Nie, L., Liu, M., Wang, K., Wang, Y., Hua, X.-S.: Coarse-to-fine semantic alignment for cross-modal moment localization. IEEE Trans. Image Process. 30, 5933\u20135943 (2021)","journal-title":"IEEE Trans. Image Process."},{"issue":"5","key":"2299_CR19","doi-asserted-by":"publisher","first-page":"2715","DOI":"10.1007\/s00530-023-01122-w","volume":"29","author":"Y Li","year":"2023","unstructured":"Li, Y., Xi, M., Jiang, D.: Cross-view adaptive graph attention network for dynamic facial expression recognition. Multimedia Syst. 29(5), 2715\u20132728 (2023)","journal-title":"Multimedia Syst."},{"issue":"5","key":"2299_CR20","doi-asserted-by":"publisher","first-page":"3663","DOI":"10.1109\/TCSVT.2023.3317447","volume":"34","author":"J Liu","year":"2024","unstructured":"Liu, J., Wang, G., Xie, J., Zhou, F., Xu, H.: Video question answering with semantic disentanglement and reasoning. IEEE Trans. Circuits Syst. Video Technol. 34(5), 3663\u20133673 (2024)","journal-title":"IEEE Trans. Circuits Syst. Video Technol."},{"issue":"28","key":"2299_CR21","doi-asserted-by":"publisher","first-page":"72113","DOI":"10.1007\/s11042-024-18372-z","volume":"83","author":"X Yao","year":"2024","unstructured":"Yao, X., Zeng, Y., Gu, M., Yuan, R., Li, J., Ge, J.: Multi-level video captioning method based on semantic space. Multimedia Tools Appl. 83(28), 72113\u201372130 (2024)","journal-title":"Multimedia Tools Appl."},{"key":"2299_CR22","volume-title":"E-learning and the Science of Instruction: Proven Guidelines for Consumers and Designers of Multimedia Learning","author":"RC Clark","year":"2023","unstructured":"Clark, R.C., Mayer, R.E.: E-learning and the Science of Instruction: Proven Guidelines for Consumers and Designers of Multimedia Learning. Wiley (2023)"},{"key":"2299_CR23","doi-asserted-by":"crossref","unstructured":"Haviv, A., Berant, J., Globerson, A.: BERTese: learning to speak to BERT. arXiv preprint arXiv:2103.05327 (2021)","DOI":"10.18653\/v1\/2021.eacl-main.316"},{"issue":"8","key":"2299_CR24","doi-asserted-by":"publisher","first-page":"1735","DOI":"10.1162\/neco.1997.9.8.1735","volume":"9","author":"S Hochreiter","year":"1997","unstructured":"Hochreiter, S., Schmidhuber, J.: Long short-term memory. Neural Comput. 9(8), 1735\u20131780 (1997)","journal-title":"Neural Comput."},{"issue":"11","key":"2299_CR25","doi-asserted-by":"publisher","first-page":"2278","DOI":"10.1109\/5.726791","volume":"86","author":"Y Lecun","year":"1998","unstructured":"Lecun, Y., Bottou, L., Bengio, Y., Haffner, P.: Gradient-based learning applied to document recognition. Proc. IEEE 86(11), 2278\u20132324 (1998)","journal-title":"Proc. IEEE"},{"key":"2299_CR26","unstructured":"Ye, Q., Xu, H., Xu, G., Ye, J., Yan, M., Zhou, Y., Wang, J., Hu, A., Shi, P., Shi, Y., Li, C., Xu, Y., Chen, H., Tian, J., Qian, Q., Zhang, J., Huang, F., Zhou, J.: mPLUG-Owl: modularization empowers large language models with multimodality (2023), version number 3. arXiv:2304.14178"},{"key":"2299_CR27","unstructured":"Kwon, T., Gopalan, A.: CO-STAR: conceptualisation of stereotypes for analysis and reasoning (2021), version Number: 1. arXiv:2112.00819"},{"issue":"1","key":"2299_CR28","doi-asserted-by":"publisher","first-page":"42","DOI":"10.1109\/T-AFFC.2011.25","volume":"3","author":"M Soleymani","year":"2012","unstructured":"Soleymani, M., Lichtenauer, J., Pun, T., Pantic, M.: A multimodal database for affect recognition and implicit tagging. IEEE Trans. Affect. Comput. 3(1), 42\u201355 (2012)","journal-title":"IEEE Trans. Affect. Comput."},{"issue":"1","key":"2299_CR29","doi-asserted-by":"publisher","first-page":"18","DOI":"10.1109\/T-AFFC.2011.15","volume":"3","author":"S Koelstra","year":"2012","unstructured":"Koelstra, S., Muhl, C., Soleymani, M., Lee, J.-S., Yazdani, A., Ebrahimi, T., Pun, T., Nijholt, A., Patras, I.: DEAP: a database for emotion analysis; using physiological signals. IEEE Trans. Affect. Comput. 3(1), 18\u201331 (2012)","journal-title":"IEEE Trans. Affect. Comput."},{"key":"2299_CR30","doi-asserted-by":"publisher","unstructured":"Akiba, T., Sano, S., Yanase, T., Ohta, T., Koyama, M.: Optuna: a next-generation hyperparameter optimization framework. In: Proceedings of the 25th ACM SIGKDD International Conference on Knowledge Discovery & Data Mining. ACM, Anchorage AK USA, pp. 2623\u20132631 (2019). https:\/\/doi.org\/10.1145\/3292500.3330701","DOI":"10.1145\/3292500.3330701"},{"issue":"3","key":"2299_CR31","doi-asserted-by":"publisher","first-page":"273","DOI":"10.1007\/BF00994018","volume":"20","author":"C Cortes","year":"1995","unstructured":"Cortes, C., Vapnik, V.: Support-vector networks. Mach. Learn. 20(3), 273\u2013297 (1995). https:\/\/doi.org\/10.1007\/BF00994018","journal-title":"Mach. Learn."},{"key":"2299_CR32","doi-asserted-by":"crossref","unstructured":"He, K., Zhang, X., Ren, S., Sun, J.: Deep residual learning for image recognition. In: 2016 IEEE Conference on Computer Vision and Pattern Recognition (CVPR). IEEE, pp. 770\u2013778","DOI":"10.1109\/CVPR.2016.90"},{"issue":"4","key":"2299_CR33","doi-asserted-by":"publisher","first-page":"541","DOI":"10.1162\/neco.1989.1.4.541","volume":"1","author":"Y LeCun","year":"1989","unstructured":"LeCun, Y., Boser, B., Denker, J.S., Henderson, D., Howard, R.E., Hubbard, W., Jackel, L.D.: Backpropagation applied to handwritten zip code recognition. Neural Comput. 1(4), 541\u2013551 (1989)","journal-title":"Neural Comput."},{"issue":"1","key":"2299_CR34","doi-asserted-by":"publisher","first-page":"96","DOI":"10.1109\/TAFFC.2019.2916015","volume":"13","author":"R Siddharth","year":"2022","unstructured":"Siddharth, R., Jung, T.-P., Sejnowski, T.J.: Utilizing deep learning towards multi-modal bio-sensing and vision-based affective computing. IEEE Trans. Affect. Comput. 13(1), 96\u2013107 (2022)","journal-title":"IEEE Trans. Affect. Comput."},{"issue":"2","key":"2299_CR35","doi-asserted-by":"publisher","DOI":"10.1088\/1741-2552\/ad3987","volume":"21","author":"Y Zhang","year":"2024","unstructured":"Zhang, Y., Liu, H., Wang, D., Zhang, D., Lou, T., Zheng, Q., Quek, C.: Cross-modal credibility modelling for EEG-based multimodal emotion recognition. J. Neural Eng. 21(2), 026040 (2024). https:\/\/doi.org\/10.1088\/1741-2552\/ad3987","journal-title":"J. Neural Eng."},{"issue":"5","key":"2299_CR36","doi-asserted-by":"publisher","first-page":"977","DOI":"10.3390\/diagnostics13050977","volume":"13","author":"F Muhammad","year":"2023","unstructured":"Muhammad, F., Hussain, M., Aboalsamh, H.: A bimodal emotion recognition approach through the fusion of electroencephalography and facial sequences. Diagnostics 13(5), 977 (2023)","journal-title":"Diagnostics"},{"key":"2299_CR37","doi-asserted-by":"crossref","unstructured":"Ai, H., Tao, X., Li, X., Gan, Y.: Modeling high-order relationships between human and video for emotion recognition in video learning. In: International Conference on Multimedia Modeling, pp. 3\u201316. Springer (2024)","DOI":"10.1007\/978-981-96-2064-7_1"}],"container-title":["Multimedia Systems"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s00530-026-02299-6.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s00530-026-02299-6","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s00530-026-02299-6.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,7,11]],"date-time":"2026-07-11T10:18:07Z","timestamp":1783765087000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s00530-026-02299-6"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026,5,6]]},"references-count":37,"journal-issue":{"issue":"4","published-print":{"date-parts":[[2026,6]]}},"alternative-id":["2299"],"URL":"https:\/\/doi.org\/10.1007\/s00530-026-02299-6","relation":{},"ISSN":["0942-4962","1432-1882"],"issn-type":[{"value":"0942-4962","type":"print"},{"value":"1432-1882","type":"electronic"}],"subject":[],"published":{"date-parts":[[2026,5,6]]},"assertion":[{"value":"4 September 2025","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"15 February 2026","order":2,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"6 May 2026","order":3,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Declarations"}},{"value":"The authors declare no competing interests.","order":2,"name":"Ethics","group":{"name":"EthicsHeading","label":"Conflict of interest"}}],"article-number":"270"}}