{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,6,5]],"date-time":"2026-06-05T16:27:08Z","timestamp":1780676828363,"version":"3.54.1"},"reference-count":86,"publisher":"Springer Science and Business Media LLC","issue":"4","license":[{"start":{"date-parts":[[2025,5,22]],"date-time":"2025-05-22T00:00:00Z","timestamp":1747872000000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2025,5,22]],"date-time":"2025-05-22T00:00:00Z","timestamp":1747872000000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"funder":[{"DOI":"10.13039\/501100003787","name":"Natural Science Foundation of Hebei Province","doi-asserted-by":"publisher","award":["D2024203008"],"award-info":[{"award-number":["D2024203008"]}],"id":[{"id":"10.13039\/501100003787","id-type":"DOI","asserted-by":"publisher"}]},{"name":"Provincial Key Laboratory Performance Subsidy Project","award":["22567612H"],"award-info":[{"award-number":["22567612H"]}]},{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["62106214"],"award-info":[{"award-number":["62106214"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]},{"name":"Science Research Project of Hebei Education Department","award":["CXY2024024"],"award-info":[{"award-number":["CXY2024024"]}]}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Multimedia Systems"],"published-print":{"date-parts":[[2025,8]]},"DOI":"10.1007\/s00530-025-01845-y","type":"journal-article","created":{"date-parts":[[2025,5,22]],"date-time":"2025-05-22T04:04:05Z","timestamp":1747886645000},"update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":5,"title":["EAVFormer: an end-to-end audio and visual emotion recognition network based on transformers"],"prefix":"10.1007","volume":"31","author":[{"given":"Haoran","family":"Liu","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Zijian","family":"Sun","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Haibin","family":"Li","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Yaqian","family":"Li","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Wenming","family":"Zhang","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Tao","family":"Song","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2025,5,22]]},"reference":[{"issue":"4","key":"1845_CR1","first-page":"529","volume":"10","author":"K Aghajani","year":"2022","unstructured":"Aghajani, K.: Audio-visual emotion recognition based on a deep convolutional neural network. J. AI Data Mining 10(4), 529\u2013537 (2022)","journal-title":"J. AI Data Mining"},{"key":"1845_CR2","doi-asserted-by":"crossref","unstructured":"Antoniadis, P., Pikoulis, I., Filntisis, P.P., Maragos, Petros: An audiovisual and contextual approach for categorical and continuous emotion recognition in-the-wild. In: Proceedings of the IEEE\/CVF International Conference on Computer Vision, pp. 3645-3651. 2021","DOI":"10.1109\/ICCVW54120.2021.00407"},{"key":"1845_CR3","doi-asserted-by":"publisher","DOI":"10.1016\/j.apacoust.2021.108260","volume":"182","author":"O Atila","year":"2021","unstructured":"Atila, O., \u015eeng\u00fcr, A.: Attention guided 3D CNN-LSTM model for accurate speech based emotion recognition. Appl. Acoust. 182, 108260 (2021)","journal-title":"Appl. Acoust."},{"key":"1845_CR4","unstructured":"Ba, J.L., Kiros, J.R., Hinton, G.E.: Layer normalization. arXiv:1607.06450 (2016)"},{"key":"1845_CR5","doi-asserted-by":"crossref","unstructured":"Baltru\u0161aitis, T., Robinson, P., Morency, L.-P.: Openface: an open source facial behavior analysis toolkit. In: 2016 IEEE winter conference on applications of computer vision (WACV), pp. 1\u201310. IEEE, (2016)","DOI":"10.1109\/WACV.2016.7477553"},{"issue":"4","key":"1845_CR6","doi-asserted-by":"publisher","first-page":"377","DOI":"10.1109\/TAFFC.2014.2336244","volume":"5","author":"H Cao","year":"2014","unstructured":"Cao, H., Cooper, D.G., Keutmann, M.K., Gur, R.C., Nenkova, A., Verma, R.: Crema-d: Crowd-sourced emotional multimodal actors dataset. IEEE Trans. Affect. Comput. 5(4), 377\u2013390 (2014)","journal-title":"IEEE Trans. Affect. Comput."},{"issue":"16","key":"1845_CR7","doi-asserted-by":"publisher","first-page":"5452","DOI":"10.3390\/s21165452","volume":"21","author":"X Chang","year":"2021","unstructured":"Chang, X., Skarbek, W.: Multi-modal residual perceptron network for audio-video emotion recognition. Sensors 21(16), 5452 (2021)","journal-title":"Sensors"},{"key":"1845_CR8","doi-asserted-by":"crossref","unstructured":"Chen, G., Jiao, S.: Speech-Visual Emotion Recognition by Fusing Shared and Specific Features. IEEE Signal Processing Lett. (2023)","DOI":"10.1109\/LSP.2023.3279689"},{"key":"1845_CR9","doi-asserted-by":"crossref","unstructured":"Chen, M., Li, X.: Swafn: Sentimental words aware fusion network for multimodal sentiment analysis. In: Proceedings of the 28th international conference on computational linguistics, pp. 1067-1077. 2020","DOI":"10.18653\/v1\/2020.coling-main.93"},{"issue":"12","key":"1845_CR10","doi-asserted-by":"publisher","first-page":"8703","DOI":"10.1109\/TCSVT.2022.3197420","volume":"32","author":"R Chen","year":"2022","unstructured":"Chen, R., Zhou, W., Li, Y., Zhou, H.: Video-based cross-modal auxiliary network for multimodal sentiment analysis. IEEE Trans. Circuits Syst. Video Technol. 32(12), 8703\u20138716 (2022)","journal-title":"IEEE Trans. Circuits Syst. Video Technol."},{"key":"1845_CR11","doi-asserted-by":"crossref","unstructured":"Chumachenko, K., Iosifidis, A., Gabbouj, M.: Self-attention fusion for audiovisual emotion recognition with incomplete data. In: 2022 26th International Conference on Pattern Recognition (ICPR), pp. 2822-2828. IEEE (2022)","DOI":"10.1109\/ICPR56361.2022.9956592"},{"key":"1845_CR12","doi-asserted-by":"crossref","unstructured":"Cioroiu, G., Radoi, A.: Multimodal Emotion Recognition with Attention. In: 2023 International Symposium on Signals, Circuits and Systems (ISSCS), pp. 1\u20134. IEEE (2023)","DOI":"10.1109\/ISSCS58449.2023.10190872"},{"key":"1845_CR13","doi-asserted-by":"crossref","unstructured":"Cohen, O., Hazan, G., Gannot, S.: Multi-Microphone Speech Emotion Recognition Using the Hierarchical Token-Semantic Audio Transformer Architecture. In: ICASSP 2025-2025 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP), pp. 1-5. IEEE (2025)","DOI":"10.1109\/ICASSP49660.2025.10887873"},{"key":"1845_CR14","doi-asserted-by":"crossref","unstructured":"Fan, R., Liu, H., Li, Y., Guo, P., Wang, G., Wang, T.: AttA-NET: Attention Aggregation Network for Audio-Visual Emotion Recognition. In: ICASSP 2024\u20132024 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP), pp. 8030\u20138034. IEEE (2024)","DOI":"10.1109\/ICASSP48485.2024.10447640"},{"key":"1845_CR15","doi-asserted-by":"crossref","unstructured":"Fan, H., Xiong, B., Mangalam, K., Li, Y., Yan, Z., Malik, J., Feichtenhofer, C.: Multiscale vision transformers. In: Proceedings of the IEEE\/CVF international conference on computer vision, pp. 6824-6835. 2021","DOI":"10.1109\/ICCV48922.2021.00675"},{"key":"1845_CR16","doi-asserted-by":"publisher","first-page":"92","DOI":"10.1016\/j.specom.2020.12.001","volume":"127","author":"Z Farhoudi","year":"2021","unstructured":"Farhoudi, Z., Setayeshi, S.: Fusion of deep learning features with mixture of brain emotional learning for audio-visual emotion recognition. Speech Communication 127, 92\u2013103 (2021)","journal-title":"Speech Communication"},{"key":"1845_CR17","unstructured":"Fu, Z., Liu, F., Wang, H., Qi, J., Fu, X., Zhou, A., Li, Z.: A cross-modal fusion network based on self-attention and residual structure for multimodal emotion recognition. arXiv:2111.02172 (2021)"},{"key":"1845_CR18","doi-asserted-by":"crossref","unstructured":"Ge, S., Jiang, Z., Cheng, Z., Wang, C., Yin, Y., Qing, G.: Learning Robust Multi-Modal Representation for Multi-Label Emotion Recognition via Adversarial Masking and Perturbation. In: Proceedings of the ACM Web Conference 2023, 1510\u20131518 (2023)","DOI":"10.1145\/3543507.3583258"},{"issue":"8","key":"1845_CR19","doi-asserted-by":"publisher","first-page":"11239","DOI":"10.1007\/s11042-022-13557-w","volume":"82","author":"E Ghaleb","year":"2023","unstructured":"Ghaleb, E., Niehues, J., Asteriadis, S.: Joint modelling of audio-visual cues using attention mechanisms for emotion recognition. Multimed. Tools Appl. 82(8), 11239\u201311264 (2023)","journal-title":"Multimed. Tools Appl."},{"key":"1845_CR20","doi-asserted-by":"crossref","unstructured":"Ghosal, D., Akhtar, M.S., Chauhan, D., Poria, S., Ekbal, A., Bhattacharyya, P.: Contextual inter-modal attention for multi-modal sentiment analysis. In: proceedings of the 2018 conference on empirical methods in natural language processing, pp. 3454\u20133466 (2018)","DOI":"10.18653\/v1\/D18-1382"},{"key":"1845_CR21","doi-asserted-by":"crossref","unstructured":"Goncalves, L., Busso, C.: AuxFormer: Robust approach to audiovisual emotion recognition. In: ICASSP 2022-2022 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP), pp. 7357\u20137361. IEEE (2022)","DOI":"10.1109\/ICASSP43922.2022.9747157"},{"key":"1845_CR22","unstructured":"Goncalves, L., Leem, S.-G., Lin, W.-C., Sisman, B., Busso, C.: Versatile Audio-Visual Learning for Handling Single and Multi Modalities in Emotion Regression and Classification Tasks. arXiv:2305.07216 (2023)"},{"key":"1845_CR23","doi-asserted-by":"publisher","first-page":"2437","DOI":"10.1109\/LSP.2022.3224688","volume":"29","author":"Y Gong","year":"2022","unstructured":"Gong, Y., Liu, A.H., Rouditchenko, A., Glass, J.: Uavm: Towards unifying audio and visual models. IEEE Signal Processing Letters 29, 2437\u20132441 (2022)","journal-title":"IEEE Signal Processing Letters"},{"key":"1845_CR24","doi-asserted-by":"crossref","unstructured":"Guo, P., Chen, Z., Li, Y., Liu, H.: Audio-visual fusion network based on conformer for multimodal emotion recognition. In: CAAI International Conference on Artificial Intelligence, pp. 315-326. Cham: Springer Nature Switzerland (2022)","DOI":"10.1007\/978-3-031-20500-2_26"},{"key":"1845_CR25","doi-asserted-by":"crossref","unstructured":"Haddad, S., Daassi, O., Belghith, S.: Emotion Recognition from Audio-Visual Information based on Convolutional Neural Network. In: 2023 International Conference on Control, Automation and Diagnosis (ICCAD), pp. 1-5. IEEE, (2023)","DOI":"10.1109\/ICCAD57653.2023.10152451"},{"key":"1845_CR26","doi-asserted-by":"crossref","unstructured":"Han, W., Chen, H., Gelbukh, A., Zadeh, A., Morency, L.-P., Poria, S.: Bi-bimodal modality fusion for correlation-controlled multimodal sentiment analysis. In: Proceedings of the 2021 International Conference on Multimodal Interaction, pp. 6-15. 2021","DOI":"10.1145\/3462244.3479919"},{"key":"1845_CR27","unstructured":"Han, B.-J., Rho, S., Dannenberg, R.B., Hwang, E.: SMERS: Music Emotion Recognition Using Support Vector Regression. In: ISMIR, pp. 651\u2013656 (2009)"},{"key":"1845_CR28","doi-asserted-by":"crossref","unstructured":"Hara, K., Kataoka, H., Satoh, Y.: Can spatiotemporal 3d cnns retrace the history of 2d cnns and imagenet?. In: Proceedings of the IEEE conference on Computer Vision and Pattern Recognition, pp. 6546-6555. 2018","DOI":"10.1109\/CVPR.2018.00685"},{"key":"1845_CR29","unstructured":"He, X., Zhao, D., Dong, Y., Shen, G., Yang, X., Zeng, Y.: Enhancing audio-visual spiking neural networks through semantic-alignment and cross-modal residual learning. arXiv:2502.12488 (2025)"},{"key":"1845_CR30","doi-asserted-by":"publisher","DOI":"10.1109\/TCSVT.2023.3276075","volume-title":"Multimodal Mutual Attention-Based Sentiment Analysis Framework Adapted to Complicated Contexts","author":"L He","year":"2023","unstructured":"He, L., Wang, Z., Wang, L., Li, F.: Multimodal Mutual Attention-Based Sentiment Analysis Framework Adapted to Complicated Contexts. IEEE Trans. Circuits Syst, Video Technol (2023)"},{"key":"1845_CR31","doi-asserted-by":"crossref","unstructured":"Hsu, J.-H., Wu, C.-H.: Applying Segment-Level Attention on Bi-modal Transformer Encoder for Audio-Visual Emotion Recognition. IEEE Trans. Affect. Comput. (2023)","DOI":"10.1109\/TAFFC.2023.3258900"},{"key":"1845_CR32","unstructured":"Hua, Cong, Xu, Q., Bao, S., Yang, Z., Huang, Q.: ReconBoost: Boosting Can Achieve Modality Reconcilement. arXiv:2405.09321 (2024)"},{"key":"1845_CR33","unstructured":"Ioffe, S., Szegedy, C.: Batch normalization: Accelerating deep network training by reducing internal covariate shift. In: International conference on machine learning, pp. 448\u2013456. pmlr, 2015"},{"key":"1845_CR34","doi-asserted-by":"crossref","unstructured":"Karas, V., Tellamekala, M.K., Mallol-Ragolta, A., Valstar, M., Schuller, B.W.: Time-continuous audiovisual fusion with recurrence vs attention for in-the-wild affect recognition. In: Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition, pp. 2382-2391. 2022","DOI":"10.1109\/CVPRW56347.2022.00266"},{"key":"1845_CR35","doi-asserted-by":"crossref","unstructured":"Khare, A., Parthasarathy, S., Sundaram, S.: Self-supervised learning with cross-modal transformers for emotion recognition. In: 2021 IEEE Spoken Language Technology Workshop (SLT), pp. 381\u2013388 IEEE (2021)","DOI":"10.1109\/SLT48900.2021.9383618"},{"key":"1845_CR36","first-page":"1755","volume":"10","author":"DE King","year":"2009","unstructured":"King, D.E.: Dlib-ml: A machine learning toolkit. J. Mach. Learn. Res. 10, 1755\u20131758 (2009)","journal-title":"J. Mach. Learn. Res."},{"key":"1845_CR37","unstructured":"Krizhevsky, A., Sutskever, I., Hinton, G.E.: Imagenet classification with deep convolutional neural networks. Adv. Neural Inf. Process. Syst. 25 (2012)"},{"key":"1845_CR38","doi-asserted-by":"crossref","unstructured":"Lei, Y., Cao, H.: Audio-visual emotion recognition with preference learning based on intended and multi-modal perceived labels. IEEE Transactions on Affective Computing (2023)","DOI":"10.1109\/TAFFC.2023.3234777"},{"key":"1845_CR39","doi-asserted-by":"crossref","unstructured":"Li, H., Guo, A., Li, Y.: CCMA: CapsNet for audio-video sentiment analysis using cross-modal attention. Visual Computer 1\u201312 (2024)","DOI":"10.1007\/s00371-024-03453-9"},{"key":"1845_CR40","doi-asserted-by":"crossref","unstructured":"Li, K., Yali, W., Zhang, J., Gao, P., Song, G., Liu, Y., Li, H., Qiao, Y.: Uniformer: Unifying convolution and self-attention for visual recognition. IEEE Trans. Pattern Anal. Mach. Intell. (2023)","DOI":"10.1109\/TPAMI.2023.3282631"},{"key":"1845_CR41","doi-asserted-by":"publisher","DOI":"10.1016\/j.eswa.2024.126274","volume":"269","author":"X Li","year":"2025","unstructured":"Li, X., Zhang, H., Dong, Z., Cheng, X., Liu, Y., Zhang, X.: Learning fine-grained representation with token-level alignment for multimodal sentiment analysis. Expert Syst. Appl. 269, 126274 (2025)","journal-title":"Expert Syst. Appl."},{"key":"1845_CR42","doi-asserted-by":"publisher","first-page":"50","DOI":"10.1109\/TMM.2021.3120873","volume":"25","author":"X Lin","year":"2021","unstructured":"Lin, X., Sun, S., Huang, W., Sheng, B., Li, P., Feng, D.D.: EAPT: efficient attention pyramid transformer for image processing. IEEE Trans. Multimed. 25, 50\u201361 (2021)","journal-title":"IEEE Trans. Multimed."},{"key":"1845_CR43","doi-asserted-by":"crossref","unstructured":"Liu, Z., Mao, H., Wu, C.-Y., Feichtenhofer, C., Darrell, T., Xie, S.: A convnet for the 2020s. In: Proceedings of the IEEE\/CVF conference on computer vision and pattern recognition, pp. 11976-11986. 2022","DOI":"10.1109\/CVPR52688.2022.01167"},{"key":"1845_CR44","doi-asserted-by":"crossref","unstructured":"Liu, Z., Ning, J., Cao, Y., Wei, Y., Zhang, Z., Lin, S., Hu, H.: Video swin transformer. In: Proceedings of the IEEE\/CVF conference on computer vision and pattern recognition, pp. 3202-3211. 2022","DOI":"10.1109\/CVPR52688.2022.00320"},{"key":"1845_CR45","unstructured":"Liu, X., Xia, N., Zhou, J., Li, Z., Guo, D.: Towards Energy-efficient Audio-Visual Classification via Multimodal Interactive Spiking Neural Network. ACM Trans. Multimed. Comput. Commun. Appl"},{"issue":"5","key":"1845_CR46","doi-asserted-by":"publisher","DOI":"10.1371\/journal.pone.0196391","volume":"13","author":"SR Livingstone","year":"2018","unstructured":"Livingstone, S.R., Russo, F.A.: The Ryerson Audio-Visual Database of Emotional Speech and Song (RAVDESS): A dynamic, multimodal set of facial and vocal expressions in North American English. PLoS One 13(5), e0196391 (2018)","journal-title":"PLoS One"},{"key":"1845_CR47","unstructured":"Loshchilov, I., Hutter, F.: Decoupled weight decay regularization. arXiv:1711.05101 (2017)"},{"issue":"22","key":"1845_CR48","doi-asserted-by":"publisher","first-page":"7665","DOI":"10.3390\/s21227665","volume":"21","author":"C Luna-Jimnez","year":"2021","unstructured":"Luna-Jimnez, C., Griol, D., Callejas, Z., Kleinlein, R., Montero, J.M., Fernndez-Martnez, F.: Multimodal emotion recognition on RAVDESS dataset using transfer learning. Sensors 21(22), 7665 (2021)","journal-title":"Sensors"},{"issue":"1","key":"1845_CR49","doi-asserted-by":"publisher","first-page":"527","DOI":"10.3390\/app12010527","volume":"12","author":"F Ma","year":"2022","unstructured":"Ma, F., Li, Y., Ni, S., Huang, S.-L., Zhang, L.: Data augmentation for audio-visual emotion recognition with an efficient multimodal conditional GAN. Appl. Sci. 12(1), 527 (2022)","journal-title":"Appl. Sci."},{"key":"1845_CR50","doi-asserted-by":"crossref","unstructured":"Middya, A.I., Nag, B., Roy, S.: Deep learning based multimodal emotion recognition using model-level fusion of audio-visual modalities. Knowl.-Based Syst. 244, 108580 (2022)","DOI":"10.1016\/j.knosys.2022.108580"},{"key":"1845_CR51","doi-asserted-by":"crossref","unstructured":"Mocanu, B., Tapu, R.: Audio-video fusion with double attention for multimodal emotion recognition. In: 2022 IEEE 14th Image, Video, and Multidimensional Signal Processing Workshop (IVMSP), pp. 1-5. IEEE, 2022","DOI":"10.1109\/IVMSP54334.2022.9816349"},{"key":"1845_CR52","doi-asserted-by":"publisher","DOI":"10.1016\/j.imavis.2023.104676","volume":"133","author":"B Mocanu","year":"2023","unstructured":"Mocanu, B., Tapu, R., Zaharia, T.: Multimodal emotion recognition using cross modal audio-video fusion with attention and deep metric learning. Image and Vision Computing 133, 104676 (2023)","journal-title":"Image and Vision Computing"},{"issue":"1","key":"1845_CR53","doi-asserted-by":"publisher","first-page":"60","DOI":"10.1109\/TAFFC.2017.2713783","volume":"10","author":"F Noroozi","year":"2017","unstructured":"Noroozi, F., Marjanovic, M., Njegus, A., Escalera, S., Anbarjafari, G.: Audio-visual emotion recognition in video clips. IEEE Trans. Affect. Comput. 10(1), 60\u201375 (2017)","journal-title":"IEEE Trans. Affect. Comput."},{"key":"1845_CR54","doi-asserted-by":"crossref","unstructured":"Orjesek, R., Jarina, R., Chmulik, M., Kuba, M.: DNN based music emotion recognition from raw audio signal. In: 2019 29th International Conference Radioelektronika (RADIOELEKTRONIKA), pp. 1\u20134. IEEE (2019)","DOI":"10.1109\/RADIOELEK.2019.8733572"},{"key":"1845_CR55","doi-asserted-by":"crossref","unstructured":"Poria, S., Cambria, E., Hazarika, D., Majumder, N., Zadeh, A., Morency, L.-P.: Context-dependent sentiment analysis in user-generated videos. In: Proceedings of the 55th annual meeting of the association for computational linguistics (volume 1: Long papers), pp. 873-883. 2017","DOI":"10.18653\/v1\/P17-1081"},{"key":"1845_CR56","doi-asserted-by":"publisher","DOI":"10.1109\/TBIOM.2022.3233083","volume-title":"Audio-visual fusion for emotion recognition in the valence-arousal space using joint cross-attention","author":"RG Praveen","year":"2023","unstructured":"Praveen, R.G., Cardinal, P., Granger, E.: Audio-visual fusion for emotion recognition in the valence-arousal space using joint cross-attention. Behavior, and Identity Science, IEEE Transactions on Biometrics (2023)"},{"key":"1845_CR57","unstructured":"Qian, C., Han, K., Wang, J., Yuan, Z., Qian, R., Lyu, C., Chen, J., Liu, Z.: Dyncim: Dynamic curriculum for imbalanced multimodal learning. arXiv:2503.06456 (2025)"},{"key":"1845_CR58","doi-asserted-by":"publisher","first-page":"15273","DOI":"10.1109\/ACCESS.2020.2967224","volume":"8","author":"MK Reddy","year":"2020","unstructured":"Reddy, M.K., Alku, P., Rao, K.S.: Detection of specific language impairment in children using glottal source features. IEEE Access 8, 15273\u201315279 (2020)","journal-title":"IEEE Access"},{"issue":"01","key":"1845_CR59","doi-asserted-by":"publisher","first-page":"5","DOI":"10.1142\/S0219622019300052","volume":"19","author":"F Ren","year":"2020","unstructured":"Ren, F., Bao, Y.: A review on human-computer interaction and intelligent robots. Int. J. Inf. Technol. Decision Making 19(01), 5\u201347 (2020)","journal-title":"Int. J. Inf. Technol. Decision Making"},{"key":"1845_CR60","doi-asserted-by":"crossref","unstructured":"Schroff, F., Kalenichenko, D., Philbin, J.: Facenet: A unified embedding for face recognition and clustering. In: 2015 IEEE Conference on Computer Vision and Pattern Recognition (CVPR), pages 815-823, June 2015. 3","DOI":"10.1109\/CVPR.2015.7298682"},{"issue":"7","key":"1845_CR61","doi-asserted-by":"publisher","first-page":"6662","DOI":"10.1109\/TCYB.2021.3079311","volume":"52","author":"B Sheng","year":"2021","unstructured":"Sheng, B., Li, P., Ali, R., Philip Chen, C.L.: Improving video temporal consistency via broad learning system. IEEE Transactions on Cybernetics 52(7), 6662\u20136675 (2021)","journal-title":"IEEE Transactions on Cybernetics"},{"key":"1845_CR62","doi-asserted-by":"crossref","unstructured":"Shi, Zhihao, Xu, Xiangyu, Liu, Xiaohong, Chen, Jun, Yang, Ming-Hsuan: Video frame interpolation transformer. In: Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition, pp. 17482-17491 (2022)","DOI":"10.1109\/CVPR52688.2022.01696"},{"key":"1845_CR63","doi-asserted-by":"crossref","unstructured":"Shi, T., Ge, X., Jose, J.M., Pugeault, N., Henderson, P.: Detail-Enhanced Intra-and Inter-modal Interaction for Audio-Visual Emotion Recognition. In: International Conference on Pattern Recognition, pp. 451\u2013465. Springer, Cham (2025)","DOI":"10.1007\/978-3-031-78305-0_29"},{"key":"1845_CR64","first-page":"23495","volume":"35","author":"C Si","year":"2022","unstructured":"Si, C., Weihao, Y., Zhou, P., Zhou, Y., Wang, X., Yan, S.: Inception transformer. Adv. Neural Inf. Process. Syst. 35, 23495\u201323509 (2022)","journal-title":"Adv. Neural Inf. Process. Syst."},{"key":"1845_CR65","doi-asserted-by":"crossref","unstructured":"Song, Y., Cai, Y., Tan, L.: Video-audio emotion recognition based on feature fusion deep learning method. In: 2021 IEEE International Midwest Symposium on Circuits and Systems (MWSCAS), pp. 611\u2013616. IEEE (2021)","DOI":"10.1109\/MWSCAS47672.2021.9531812"},{"key":"1845_CR66","doi-asserted-by":"publisher","first-page":"1927","DOI":"10.1109\/TIP.2023.3256763","volume":"32","author":"Y Song","year":"2023","unstructured":"Song, Y., He, Z., Qian, H., Xin, D.: Vision transformers for single image dehazing. IEEE Trans. Image Process. 32, 1927\u20131941 (2023)","journal-title":"IEEE Trans. Image Process."},{"key":"1845_CR67","unstructured":"Su, L., Hu, C., Li, G., Cao, D.: Msaf: Multimodal split attention fusion. arXiv:2012.07175 (2020)"},{"key":"1845_CR68","doi-asserted-by":"publisher","first-page":"93","DOI":"10.1007\/s10772-018-9491-z","volume":"21","author":"M Swain","year":"2018","unstructured":"Swain, M., Routray, A., Kabisatpathy, P.: Databases, features and classifiers for speech emotion recognition: a review. Int. J. Speech Technol. 21, 93\u2013120 (2018)","journal-title":"Int. J. Speech Technol."},{"key":"1845_CR69","doi-asserted-by":"crossref","unstructured":"Tran, D., Bourdev, L.D., Fergus, R., Torresani, L., Paluri, M.: Learning spatiotemporal features with 3d convolutional networks. 2015 IEEE International Conference on Computer Vision (ICCV), pp. 4489-4497, 2015","DOI":"10.1109\/ICCV.2015.510"},{"key":"1845_CR70","doi-asserted-by":"crossref","unstructured":"Tran, M., Soleymani, M.: A pre-trained audio-visual transformer for emotion recognition. In: ICASSP 2022-2022 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP), pp. 4698\u20134702. IEEE (2022)","DOI":"10.1109\/ICASSP43922.2022.9747278"},{"key":"1845_CR71","doi-asserted-by":"crossref","unstructured":"Tsai, Y.-H., Hubert, S.B., Pu Liang, P., Zico Kolter, J., Morency, L.-P., Salakhutdinov, R.: Multimodal transformer for unaligned multimodal language sequences. In: Proceedings of the conference. Association for Computational Linguistics. Meeting, vol. 2019, p. 6558. NIH Public Access (2019)","DOI":"10.18653\/v1\/P19-1656"},{"key":"1845_CR72","unstructured":"Wang, Y., Li, Y., Cui, Z.: Incomplete multimodality-diffused emotion recognition. Advances in Neural Inf. Process. Syst. 36, (2024)"},{"key":"1845_CR73","doi-asserted-by":"publisher","DOI":"10.1016\/j.bspc.2019.101756","volume":"58","author":"C Wei","year":"2020","unstructured":"Wei, C., Chen, L.-L., Song, Z.-Z., Lou, X.-G., Li, D.-D.: EEG-based emotion recognition using simple recurrent units network and ensemble learning. Biomed. Signal Process. Control 58, 101756 (2020)","journal-title":"Biomed. Signal Process. Control"},{"issue":"1","key":"1845_CR74","doi-asserted-by":"publisher","first-page":"10","DOI":"10.1109\/TBC.2022.3215245","volume":"69","author":"Q Wei","year":"2022","unstructured":"Wei, Q., Huang, X., Zhang, Y.: FV2ES: A Fully End2End Multimodal System for Fast Yet Effective Video Emotion Recognition Inference. IEEE Trans. Broadcast. 69(1), 10\u201320 (2022)","journal-title":"IEEE Trans. Broadcast."},{"key":"1845_CR75","doi-asserted-by":"crossref","unstructured":"Woo, S., Park, J., Lee, J.-Y., Kweon, I.S.: Cbam: Convolutional block attention module. In: Proceedings of the European conference on computer vision (ECCV), pp. 3\u201319 (2018)","DOI":"10.1007\/978-3-030-01234-2_1"},{"key":"1845_CR76","doi-asserted-by":"crossref","unstructured":"Wu, L., Bai, L., Cheng, W., Cheng, Z., Chen, G.: Adaptive Alignment and Time Aggregation Network for Speech-Visual Emotion Recognition. IEEE Signal Process. Lett. (2025)","DOI":"10.1109\/LSP.2025.3550007"},{"key":"1845_CR77","doi-asserted-by":"crossref","unstructured":"Wu, K., Peng, H., Chen, M., Fu, J., Chao, H.: Rethinking and improving relative position encoding for vision transformer. In: Proceedings of the IEEE\/CVF international conference on computer vision, pp. 10033\u201310041 (2021)","DOI":"10.1109\/ICCV48922.2021.00988"},{"key":"1845_CR78","doi-asserted-by":"crossref","unstructured":"Wu, Y., Zhang, Z., Peng, P., Zhao, Y., Qin, B.: Leveraging multi-modal interactions among the intermediate representations of deep transformers for emotion recognition. In: Proceedings of the 3rd International on Multimodal Sentiment Analysis Workshop and Challenge, pp. 101\u2013109 (2022)","DOI":"10.1145\/3551876.3554813"},{"key":"1845_CR79","doi-asserted-by":"crossref","unstructured":"Xie, S., Girshick, R., Dollar, P., Tu, Z., He, K.: Aggregated residual transformations for deep neural networks. In: 2017 IEEE Conference on Computer Vision and Pattern Recognition (CVPR), pages 5987-5995, 2017","DOI":"10.1109\/CVPR.2017.634"},{"key":"1845_CR80","doi-asserted-by":"publisher","DOI":"10.1016\/j.apacoust.2023.109752","volume":"216","author":"L Yu","year":"2024","unstructured":"Yu, L., Fengjun, X., Yundong, Q., Zhou, K.: Speech emotion recognition based on multi-dimensional feature extraction and multi-scale feature fusion. Appl. Acoust. 216, 109752 (2024)","journal-title":"Appl. Acoust."},{"key":"1845_CR81","unstructured":"Zadeh, A., Bagher, L., Pu, P., Soujanya, P., Erik, C., Morency, L.-P.: Multimodal language analysis in the wild: Cmu-mosei dataset and interpretable dynamic fusion graph. In: Proceedings of the 56th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers), pp. 2236\u20132246. 2018"},{"key":"1845_CR82","doi-asserted-by":"crossref","unstructured":"Zhang, S., Ding, Y., Wei, Z., Guan, C.: Continuous emotion recognition with audio-visual leader-follower attentive fusion. In: Proceedings of the IEEE\/CVF International Conference on Computer Vision, pp. 3567\u20133574 (2021)","DOI":"10.1109\/ICCVW54120.2021.00397"},{"issue":"10","key":"1845_CR83","doi-asserted-by":"publisher","first-page":"1499","DOI":"10.1109\/LSP.2016.2603342","volume":"23","author":"K Zhang","year":"2016","unstructured":"Zhang, K., Zhang, Z., Li, Z., Qiao, Y.: Joint face detection and alignment using multitask cascaded convolutional networks. IEEE Signal Process. Lett. 23(10), 1499\u20131503 (2016)","journal-title":"IEEE Signal Process. Lett."},{"issue":"10","key":"1845_CR84","doi-asserted-by":"publisher","first-page":"3030","DOI":"10.1109\/TCSVT.2017.2719043","volume":"28","author":"S Zhang","year":"2017","unstructured":"Zhang, S., Zhang, S., Huang, T., Gao, W., Tian, Q.: Learning affective features with a hybrid deep model for audio-visual emotion recognition. IEEE Transactions on Circuits and Systems for Video Technology 28(10), 3030\u20133043 (2017)","journal-title":"IEEE Transactions on Circuits and Systems for Video Technology"},{"key":"1845_CR85","doi-asserted-by":"publisher","first-page":"2617","DOI":"10.1109\/TASLP.2021.3096037","volume":"29","author":"H Zhou","year":"2021","unstructured":"Zhou, H., Jun, D., Zhang, Y., Wang, Q., Liu, Q.-F., Lee, C.-H.: Information fusion in attention networks using adaptive and multi-level factorized bilinear pooling for audio-visual motion recognition. IEEE\/ACM Trans. Audio Speech Lang Process. 29, 2617\u20132629 (2021)","journal-title":"IEEE\/ACM Trans. Audio Speech Lang Process."},{"issue":"3","key":"1845_CR86","doi-asserted-by":"publisher","first-page":"1943","DOI":"10.1007\/s11063-021-10713-5","volume":"54","author":"X Zhuang","year":"2022","unstructured":"Zhuang, X., Liu, F., Hou, J., Hao, J., Cai, X.: Transformer-based interactive multi-modal attention network for video sentiment detection. Neural Process. Lett. 54(3), 1943\u20131960 (2022)","journal-title":"Neural Process. Lett."}],"container-title":["Multimedia Systems"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s00530-025-01845-y.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s00530-025-01845-y\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s00530-025-01845-y.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,9,15]],"date-time":"2025-09-15T09:02:17Z","timestamp":1757926937000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s00530-025-01845-y"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,5,22]]},"references-count":86,"journal-issue":{"issue":"4","published-print":{"date-parts":[[2025,8]]}},"alternative-id":["1845"],"URL":"https:\/\/doi.org\/10.1007\/s00530-025-01845-y","relation":{},"ISSN":["0942-4962","1432-1882"],"issn-type":[{"value":"0942-4962","type":"print"},{"value":"1432-1882","type":"electronic"}],"subject":[],"published":{"date-parts":[[2025,5,22]]},"assertion":[{"value":"22 October 2024","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"4 May 2025","order":2,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"22 May 2025","order":3,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Declarations"}},{"value":"The authors declare that they have no known competing financial interests or personal relationships that could have appeared to influence the work reported in this paper.","order":2,"name":"Ethics","group":{"name":"EthicsHeading","label":"Conflict of interest"}},{"value":"This study does not involve any human participant or animal. All the data used in this article are sourced from open and publicly accessible platforms. No proprietary, confidential, or private data has been used.","order":3,"name":"Ethics","group":{"name":"EthicsHeading","label":"Ethical Compliance"}}],"article-number":"262"}}