{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,5,20]],"date-time":"2026-05-20T09:09:23Z","timestamp":1779268163153,"version":"3.51.4"},"reference-count":88,"publisher":"Springer Science and Business Media LLC","issue":"5","license":[{"start":{"date-parts":[[2026,2,28]],"date-time":"2026-02-28T00:00:00Z","timestamp":1772236800000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by-nc-nd\/4.0"},{"start":{"date-parts":[[2026,5,20]],"date-time":"2026-05-20T00:00:00Z","timestamp":1779235200000},"content-version":"vor","delay-in-days":81,"URL":"https:\/\/creativecommons.org\/licenses\/by-nc-nd\/4.0"}],"funder":[{"DOI":"10.13039\/501100003787","name":"Natural Science Foundation of Hebei Province","doi-asserted-by":"publisher","award":["D2024203008"],"award-info":[{"award-number":["D2024203008"]}],"id":[{"id":"10.13039\/501100003787","id-type":"DOI","asserted-by":"publisher"}]},{"name":"Science Research Project of Hebei Education Department","award":["CXY2024024"],"award-info":[{"award-number":["CXY2024024"]}]}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["J. King Saud Univ. Comput. Inf. Sci."],"published-print":{"date-parts":[[2026,7]]},"DOI":"10.1007\/s44443-026-00610-z","type":"journal-article","created":{"date-parts":[[2026,2,28]],"date-time":"2026-02-28T12:15:18Z","timestamp":1772280918000},"update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":0,"title":["FMII: a unified intra- and inter-modal interaction framework for robust audio-visual emotion recognition"],"prefix":"10.1007","volume":"38","author":[{"given":"Zijian","family":"Sun","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Haoran","family":"Liu","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Yaqian","family":"Li","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Haibin","family":"Li","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Wenming","family":"Zhang","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Tao","family":"Song","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2026,2,28]]},"reference":[{"issue":"26","key":"610_CR1","doi-asserted-by":"publisher","first-page":"40375","DOI":"10.1007\/s11042-023-15066-w","volume":"82","author":"FM Alamgir","year":"2023","unstructured":"Alamgir FM, Alam MS (2023) Hybrid multi-modal emotion recognition framework based on inceptionv3densenet. Multimed Tools Appl 82(26):40375\u201340402","journal-title":"Multimed Tools Appl"},{"key":"610_CR2","unstructured":"Alemi AA, Fischer I, Dillon JV, Murphy K (2016) Deep variational information bottleneck. arXiv:1612.00410"},{"key":"610_CR3","unstructured":"Belghazi MI, Baratin A, Rajeshwar S, Ozair S, Bengio Y, Courville A, Hjelm D (2018) Mutual information neural estimation. In: International conference on machine learning. PMLR, pp 531\u2013540"},{"key":"610_CR4","doi-asserted-by":"publisher","DOI":"10.1016\/j.engappai.2023.107708","volume":"130","author":"U Bilotti","year":"2024","unstructured":"Bilotti U, Bisogni C, De Marsico M, Tramonte S (2024) Multimodal emotion recognition via convolutional neural networks: comparison of different strategies on two multimodal datasets. Eng Appl Artif Intell 130:107708","journal-title":"Eng Appl Artif Intell"},{"issue":"4","key":"610_CR5","doi-asserted-by":"publisher","first-page":"377","DOI":"10.1109\/TAFFC.2014.2336244","volume":"5","author":"H Cao","year":"2014","unstructured":"Cao H, Cooper DG, Keutmann MK, Gur RC, Nenkova A, Verma R (2014) CREMA-D: crowd-sourced emotional multimodal actors dataset. IEEE Trans Affect Comput 5(4):377\u2013390","journal-title":"IEEE Trans Affect Comput"},{"key":"610_CR6","doi-asserted-by":"crossref","unstructured":"Chen L, Zhou X, Chen H (2024) Audio scanning network: bridging time and frequency domains for audio classification. In: Proceedings of the AAAI conference on artificial intelligence. Vancouver, Canada, vol 38, pp 11355\u201311363","DOI":"10.1609\/aaai.v38i10.29015"},{"key":"610_CR7","unstructured":"Cheng P, Hao W, Dai S, Liu J, Gan Z, Carin L (2020) CLUB: a contrastive log-ratio upper bound of mutual information. In: Proceedings of the international conference on machine learning. PMLR, pp 1779\u20131788"},{"key":"610_CR8","doi-asserted-by":"publisher","first-page":"2689","DOI":"10.1109\/TASLP.2022.3192728","volume":"30","author":"Q Chen","year":"2022","unstructured":"Chen Q, Huang G, Wang Y (2022) The weighted cross-modal attention mechanism with sentiment prediction auxiliary task for multimodal sentiment analysis. IEEE\/ACM Trans Audio Speech Lang Process 30:2689\u20132695","journal-title":"IEEE\/ACM Trans Audio Speech Lang Process"},{"issue":"13s","key":"610_CR9","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1145\/3586075","volume":"55","author":"R Das","year":"2023","unstructured":"Das R, Singh TD (2023) Multimodal sentiment analysis: a survey of methods, trends, and challenges. ACM Comput Surv 55(13s):1\u201338","journal-title":"ACM Comput Surv"},{"key":"610_CR10","doi-asserted-by":"crossref","unstructured":"Deng Z, Huang G, Zhong G, Yuan X, Huang L, Pun C-M (2024) Cross-modality disentangled information bottleneck strategy for multimodal sentiment analysis. In: Proceedings of the IEEE international conference on systems, man, and cybernetics (SMC). IEEE, Kuching, Sarawak, Malaysia, pp 2267\u20132274","DOI":"10.1109\/SMC54092.2024.10831481"},{"key":"610_CR11","doi-asserted-by":"crossref","unstructured":"Ding SY, Tang TB, Lu C-K (2025) Lightweight spatio-temporal convolutional neural network for audio-visual emotion recognition. IEEE Trans Affect Comput","DOI":"10.1109\/TAFFC.2025.3566773"},{"key":"610_CR12","doi-asserted-by":"crossref","unstructured":"Fan Y, Xu W, Wang H, Wang J, Guo S (2023) PMR: prototypical modal rebalance for multimodal learning. In: Proceedings of the IEEE\/CVF conference on computer vision and pattern recognition. IEEE, Vancouver, Canada, pp 20029\u201320038","DOI":"10.1109\/CVPR52729.2023.01918"},{"key":"610_CR13","doi-asserted-by":"crossref","unstructured":"Feng Y, Ye X, Manoharan S (2024) Multi-modal emotion recognition. In: Proceedings of the IEEE international conference on computing (ICOCO). IEEE, Kuala Lumpur, Malaysia, pp 422\u2013427","DOI":"10.1109\/ICOCO62848.2024.10928263"},{"key":"610_CR14","doi-asserted-by":"publisher","DOI":"10.1016\/j.neucom.2023.127201","volume":"571","author":"Y Fu","year":"2024","unstructured":"Fu Y, Zhang Z, Yang R, Yao C (2024) Hybrid cross-modal interaction learning for multimodal sentiment analysis. Neurocomputing 571:127201","journal-title":"Neurocomputing"},{"key":"610_CR15","doi-asserted-by":"crossref","unstructured":"Gao Z, Hu D, Jiang X, Lu H, Shen HT, Xu X (2024) Enhanced experts with uncertainty-aware routing for multimodal sentiment analysis. In: Proceedings of the 32nd ACM International conference on multimedia. ACM, Melbourne, Australia. Melbourne Convention and Exhibition Centre (MCEC), pp 9650\u20139659","DOI":"10.1145\/3664647.3680949"},{"key":"610_CR16","doi-asserted-by":"publisher","first-page":"102218","DOI":"10.1016\/j.inffus.2023.102218","volume":"105","author":"AV Geetha","year":"2024","unstructured":"Geetha AV, Mala T, Priyanka D, Uma E (2024) Multimodal emotion recognition with deep learning: advancements, challenges, and future directions. Inf Fusion 105:102218","journal-title":"Inf Fusion"},{"key":"610_CR17","doi-asserted-by":"crossref","unstructured":"Goncalves L, Busso C (2022) AuxFormer: robust approach to audiovisual emotion recognition. In: Proceedings of the IEEE international conference on acoustics, speech and signal processing (ICASSP). IEEE, Singapore, pp 7357\u20137361","DOI":"10.1109\/ICASSP43922.2022.9747157"},{"key":"610_CR18","doi-asserted-by":"crossref","unstructured":"Goncalves L, Leem S-G, Lin W-C, Sisman B, Busso C (2024) Versatile audio-visual learning for emotion recognition. IEEE Trans Affect Comput","DOI":"10.1109\/TAFFC.2024.3433386"},{"issue":"6","key":"610_CR19","doi-asserted-by":"publisher","first-page":"669","DOI":"10.1007\/s42979-024-03020-y","volume":"5","author":"S Haddad","year":"2024","unstructured":"Haddad S, Daassi O, Belghith S (2024) Single modality and joint fusion for emotion recognition on RAVDESS dataset. SN Comput Sci 5(6):669","journal-title":"SN Comput Sci"},{"key":"610_CR20","doi-asserted-by":"crossref","unstructured":"Han W, Chen H, Poria S (2021) Improving multimodal fusion with hierarchical mutual information maximization for multimodal sentiment analysis. arXiv:2109.00412","DOI":"10.18653\/v1\/2021.emnlp-main.723"},{"key":"610_CR21","doi-asserted-by":"crossref","unstructured":"Hazarika D, Zimmermann R, Poria S (2020) MISA: modality-invariant and -specific representations for multimodal sentiment analysis. In: Proceedings of the 28th ACM international conference on multimedia. ACM, Virtual Event \/ Seattle, WA, USA, pp 1122\u20131131","DOI":"10.1145\/3394171.3413678"},{"key":"610_CR22","doi-asserted-by":"publisher","DOI":"10.1016\/j.engappai.2024.108339","volume":"133","author":"S Hazmoune","year":"2024","unstructured":"Hazmoune S, Bougamouza F (2024) Using transformers for multimodal emotion recognition: taxonomies and state of the art review. Eng Appl Artif Intell 133:108339","journal-title":"Eng Appl Artif Intell"},{"issue":"4","key":"610_CR23","doi-asserted-by":"publisher","first-page":"3231","DOI":"10.1109\/TAFFC.2023.3258900","volume":"14","author":"J-H Hsu","year":"2023","unstructured":"Hsu J-H, Wu C-H (2023) Applying segment-level attention on bi-modal transformer encoder for audio-visual emotion recognition. IEEE Trans Affect Comput 14(4):3231\u20133243","journal-title":"IEEE Trans Affect Comput"},{"issue":"1","key":"610_CR24","doi-asserted-by":"publisher","first-page":"31","DOI":"10.1007\/s11227-024-06582-z","volume":"81","author":"Z Jin","year":"2025","unstructured":"Jin Z, Zai W (2025) Audiovisual emotion recognition based on bi-layer LSTM and multi-head attention mechanism on RAVDESS dataset. J Supercomput 81(1):31","journal-title":"J Supercomput"},{"key":"610_CR25","doi-asserted-by":"crossref","unstructured":"John V, Kawanishi Y (2022) Audio and video-based emotion recognition using multimodal transformers. In: Proceedings of the 26th international conference on pattern recognition (ICPR). IEEE, Montr\u00e9al, QC, Canada, pp 2582\u20132588","DOI":"10.1109\/ICPR56361.2022.9956730"},{"key":"610_CR26","unstructured":"Joze HRV, Shaban A, Iuzzolino ML, Koishida K (2020) MMTM: multimodal transfer module for CNN fusion. In: Proceedings of the IEEE\/CVF conference on computer vision and pattern recognition, pp 13289\u201313299"},{"key":"610_CR27","unstructured":"Kingma DP, Welling M (2013) Auto-encoding variational Bayes. arXiv:1312.6114"},{"key":"610_CR28","doi-asserted-by":"publisher","first-page":"2880","DOI":"10.1109\/TASLP.2020.3030497","volume":"28","author":"Q Kong","year":"2020","unstructured":"Kong Q, Cao Y, Iqbal T, Wang Y, Wang W, Plumbley MD (2020) PANNs: large-scale pretrained audio neural networks for audio pattern recognition. IEEE\/ACM Trans Audio Speech Lang Process 28:2880\u20132894","journal-title":"IEEE\/ACM Trans Audio Speech Lang Process"},{"key":"610_CR29","doi-asserted-by":"publisher","first-page":"94557","DOI":"10.1109\/ACCESS.2021.3092735","volume":"9","author":"S Lee","year":"2021","unstructured":"Lee S, Han DK, Ko H (2021) Multimodal emotion recognition fusion analysis adapting BERT with heterogeneous feature unification. IEEE Access 9:94557\u201394572","journal-title":"IEEE Access"},{"key":"610_CR30","doi-asserted-by":"crossref","unstructured":"Li H, Li X, Hu P, Lei Y, Li C, Zhou Y (2023) Boosting multi-modal model performance with adaptive gradient modulation. In: Proceedings of the IEEE\/CVF international conference on computer vision. IEEE, Paris, France, pp 22214\u201322224","DOI":"10.1109\/ICCV51070.2023.02030"},{"issue":"7","key":"610_CR31","first-page":"8419","volume":"45","author":"Z Lian","year":"2023","unstructured":"Lian Z, Chen L, Sun L, Liu B, Tao J (2023) GCNet: graph completion network for incomplete multimodal learning in conversation. IEEE Trans Pattern Anal Mach Intell 45(7):8419\u20138432","journal-title":"IEEE Trans Pattern Anal Mach Intell"},{"issue":"3","key":"610_CR32","doi-asserted-by":"publisher","first-page":"262","DOI":"10.1007\/s00530-025-01845-y","volume":"31","author":"H Liu","year":"2025","unstructured":"Liu H, Sun Z, Li H, Li Y, Zhang W, Song T (2025) EAVFormer: an end-to-end audio and visual emotion recognition network based on transformers. Multimedia Syst 31(3):262","journal-title":"Multimedia Syst"},{"issue":"5","key":"610_CR33","doi-asserted-by":"publisher","first-page":"0196391","DOI":"10.1371\/journal.pone.0196391","volume":"13","author":"SR Livingstone","year":"2018","unstructured":"Livingstone SR, Russo FA (2018) The Ryerson audio-visual database of emotional speech and song (RAVDESS): a dynamic, multimodal set of facial and vocal expressions in north american english. PLoS ONE 13(5):0196391","journal-title":"PLoS ONE"},{"issue":"6","key":"610_CR34","doi-asserted-by":"publisher","first-page":"3159","DOI":"10.1109\/TCSS.2022.3219825","volume":"10","author":"C Lu","year":"2022","unstructured":"Lu C, Zheng W, Lian H, Zong Y, Tang C, Li S, Zhao Y (2022) Speech emotion recognition via an attentive time-frequency neural network. IEEE Trans Comput Soc Syst 10(6):3159\u20133168","journal-title":"IEEE Trans Comput Soc Syst"},{"key":"610_CR35","doi-asserted-by":"publisher","first-page":"4121","DOI":"10.1109\/TMM.2022.3171679","volume":"25","author":"S Mai","year":"2022","unstructured":"Mai S, Zeng Y, Hu H (2022) Multimodal information bottleneck: learning minimal sufficient unimodal and multimodal representations. IEEE Trans Multimedia 25:4121\u20134134","journal-title":"IEEE Trans Multimedia"},{"key":"610_CR36","doi-asserted-by":"publisher","DOI":"10.1016\/j.knosys.2022.108580","volume":"244","author":"AI Middya","year":"2022","unstructured":"Middya AI, Nag B, Roy S (2022) Deep learning based multimodal emotion recognition using model-level fusion of audio-visual modalities. Knowl-Based Syst 244:108580","journal-title":"Knowl-Based Syst"},{"key":"610_CR37","doi-asserted-by":"crossref","unstructured":"Pan Z, Niu L, Zhang J, Zhang L (2021) Disentangled information bottleneck. In: Proceedings of the AAAI Conference on Artificial Intelligence, vol 35, pp 9285\u20139293","DOI":"10.1609\/aaai.v35i10.17120"},{"key":"610_CR38","unstructured":"Poole B, Ozair S, Oord A, Alemi AA, Tucker G (2018) On variational lower bounds of mutual information. In: NeurIPS workshop on bayesian deep learning, Montr\u00e9al, QC, Canada"},{"key":"610_CR39","doi-asserted-by":"crossref","unstructured":"Praveen RG, Melo W, Ullah N, Aslam H, Zeeshan O, Denorme T, Pedersoli M et al (2022) A joint cross-attention model for audio-visual fusion in dimensional emotion recognition. In: Proceedings of the IEEE\/CVF conference on computer vision and pattern recognition. IEEE, New Orleans, LA, USA, pp 2486\u20132495","DOI":"10.1109\/CVPRW56347.2022.00278"},{"issue":"3","key":"610_CR40","doi-asserted-by":"publisher","first-page":"444","DOI":"10.1109\/JSTSP.2024.3422823","volume":"18","author":"RG Praveen","year":"2024","unstructured":"Praveen RG, Alam J (2024) Incongruity-aware cross-modal attention for audio-visual fusion in dimensional emotion recognition. IEEE J Sel Top Signal Process 18(3):444\u2013458","journal-title":"IEEE J Sel Top Signal Process"},{"key":"610_CR41","doi-asserted-by":"crossref","unstructured":"Ryumina E, Markitantov M, Ryumin D, Kaya H, Karpov A (2024) Zero-shot audio-visual compound expression recognition method based on emotion probability fusion. In: Proceedings of the IEEE\/CVF conference on computer vision and pattern recognition. IEEE, Seattle, WA, USA, pp 4752\u20134760","DOI":"10.1109\/CVPRW63382.2024.00478"},{"key":"610_CR42","doi-asserted-by":"publisher","DOI":"10.1016\/j.cviu.2025.104362","volume":"257","author":"S Sadok","year":"2025","unstructured":"Sadok S, Leglaive S, S\u00e9guier R (2025) A vector quantized masked autoencoder for audiovisual speech emotion recognition. Comput Vis Image Underst 257:104362","journal-title":"Comput Vis Image Underst"},{"key":"610_CR43","doi-asserted-by":"publisher","DOI":"10.1016\/j.eswa.2023.119750","volume":"222","author":"D Salvati","year":"2023","unstructured":"Salvati D, Drioli C, Foresti GL (2023) A late fusion deep neural network for robust speaker identification using raw waveforms and gammatone cepstral coefficients. Expert Syst Appl 222:119750","journal-title":"Expert Syst Appl"},{"key":"610_CR44","unstructured":"Shen S, Chen CL, Zhang T (2025) AIM: adaptive intra-network modulation for balanced multimodal learning. arXiv:2508.19769"},{"key":"610_CR45","doi-asserted-by":"crossref","unstructured":"Shi T, Ge X, Jose JM, Pugeault N, Henderson P (2024) Detail-enhanced intra- and inter-modal interaction for audio-visual emotion recognition. In: International conference on pattern recognition. Springer, Cham, pp 451\u2013465","DOI":"10.1007\/978-3-031-78305-0_29"},{"key":"610_CR46","doi-asserted-by":"crossref","unstructured":"Shu Y, Li W, Ruan Y-P, Liu W, Zhang Y, Guan J, Zhou S (2025) Ensuring pre-fusion modality consistency: a new approach to multimodal sentiment detection. ACM Trans Intell Syst Technol","DOI":"10.1145\/3748658"},{"key":"610_CR47","doi-asserted-by":"crossref","unstructured":"Song Y, Cho S (2025) Leveraging CLIP encoder for multimodal emotion recognition. In: Proceedings of the IEEE\/CVF winter conference on applications of computer vision (WACV). IEEE, Tucson, AZ, USA. JW Marriott Starr Pass Resort, pp 6115\u20136124","DOI":"10.1109\/WACV61041.2025.00596"},{"key":"610_CR48","doi-asserted-by":"crossref","unstructured":"Srihari M, Bhaskaran V (2024) A multimodal fusion approach: emotion identification from audio and video using convolutional neural network. In: Proceedings of the IEEE international conference on computer vision and machine intelligence (CVMI). IEEE, Prayagraj, India, pp 1\u20135","DOI":"10.1109\/CVMI61877.2024.10782687"},{"key":"610_CR49","doi-asserted-by":"crossref","unstructured":"Srinivas PVVS, Mishra P (2022) Human emotion recognition by integrating facial and speech features: an implementation of multimodal framework using CNN. Int J Adv Comput Sci Appl 13(1)","DOI":"10.14569\/IJACSA.2022.0130172"},{"key":"610_CR50","doi-asserted-by":"crossref","unstructured":"Sun H, Niu Z, Wang H, Yu X, Liu J, Chen Y-W, Lin L (2025) Multimodal sentiment analysis with mutual information-based disentangled representation learning. IEEE Trans Affect Comput","DOI":"10.1109\/TAFFC.2025.3529732"},{"key":"610_CR51","doi-asserted-by":"publisher","first-page":"102382","DOI":"10.1016\/j.inffus.2024.102382","volume":"108","author":"L Sun","year":"2024","unstructured":"Sun L, Lian Z, Liu B, Tao J (2024) HICMAE: hierarchical contrastive masked autoencoder for self-supervised audio-visual emotion recognition. Inf Fusion 108:102382","journal-title":"Inf Fusion"},{"key":"610_CR52","doi-asserted-by":"publisher","first-page":"504","DOI":"10.1016\/j.inffus.2022.10.031","volume":"91","author":"H Sun","year":"2023","unstructured":"Sun H, Liu J, Chen Y-W, Lin L (2023) Modality-invariant temporal representation learning for multimodal sentiment classification. Inf Fusion 91:504\u2013514","journal-title":"Inf Fusion"},{"key":"610_CR53","doi-asserted-by":"publisher","first-page":"105081","DOI":"10.1016\/j.dsp.2025.105081","volume":"161","author":"Z Sun","year":"2025","unstructured":"Sun Z, Liu H, Li H, Li Y, Zhang W (2025) AVERFormer: end-to-end audio-visual emotion recognition transformer framework with balanced modal contributions. Digital Signal Process 161:105081","journal-title":"Digital Signal Process"},{"key":"610_CR54","unstructured":"Tishby N, Pereira FC, Bialek W (2000) The information bottleneck method. arXiv:physics\/0004057"},{"key":"610_CR55","doi-asserted-by":"crossref","unstructured":"Tran M, Soleymani M (2022)A pre-trained audio-visual transformer for emotion recognition. In: Proceedings of the IEEE international conference on acoustics, speech and signal processing (ICASSP). IEEE, Singapore, pp 4698\u20134702","DOI":"10.1109\/ICASSP43922.2022.9747278"},{"key":"610_CR56","unstructured":"Vaswani A, Shazeer N, Parmar N, Uszkoreit J, Jones L, Gomez AN, Kaiser \u0141, Polosukhin I (2017) Attention is all you need. Adv Neural Inf Process Syst 30"},{"key":"610_CR57","unstructured":"Venkatraman S, Sharma V, Malarvannan S, Narendra M (2024) Multimodal emotion recognition using audio-video transformer fusion with cross attention. arXiv:2407.18552"},{"key":"610_CR58","doi-asserted-by":"crossref","unstructured":"Wang Y, Cui Z, Li Y (2023) Distribution-consistent modal recovering for incomplete multimodal learning. In: Proceedings of the IEEE\/CVF international conference on computer vision. IEEE, Paris, France, pp 22025\u201322034","DOI":"10.1109\/ICCV51070.2023.02013"},{"key":"610_CR59","doi-asserted-by":"crossref","unstructured":"Wang Y, Li Y, Cui Z (2023) Incomplete multimodality-diffused emotion recognition. In: Advances in neural information processing systems, vol 36, pp 17117\u201317128","DOI":"10.52202\/075280-0748"},{"key":"610_CR60","doi-asserted-by":"crossref","unstructured":"Wang J, Wang C, Guo L, Zhao S, Wang D, Zhang S, Zhao X et al (2025) MDKAT: multimodal decoupling with knowledge aggregation and transfer for video emotion recognition. IEEE Trans Circ Syst Video Technol","DOI":"10.1109\/TCSVT.2025.3571534"},{"key":"610_CR61","doi-asserted-by":"crossref","unstructured":"Wang S (2025) AGCL: attention aggregation contrastive learning for multimodal emotion analysis. In: Proceedings of the 4th international symposium on computer applications and information technology (ISCAIT). IEEE, Xi\u2019an, China, pp 123\u2013127","DOI":"10.1109\/ISCAIT64916.2025.11010588"},{"issue":"3","key":"610_CR62","doi-asserted-by":"publisher","first-page":"597","DOI":"10.1109\/TMM.2012.2189550","volume":"14","author":"Y Wang","year":"2012","unstructured":"Wang Y, Guan L, Venetsanopoulos AN (2012) Kernel cross-modal factor analysis for information fusion with application to bimodal emotion recognition. IEEE Trans Multimedia 14(3):597\u2013607","journal-title":"IEEE Trans Multimedia"},{"issue":"3","key":"610_CR63","doi-asserted-by":"publisher","first-page":"103675","DOI":"10.1016\/j.ipm.2024.103675","volume":"61","author":"L Wang","year":"2024","unstructured":"Wang L, Peng J, Zheng C, Zhao T, Li\u2019an Z (2024) A cross modal hierarchical fusion multimodal sentiment analysis method based on multi-task learning. Inf Process Manag 61(3):103675","journal-title":"Inf Process Manag"},{"key":"610_CR64","doi-asserted-by":"crossref","unstructured":"Wei Y, Hu D, Du H, Wen J-R (2024) On-the-fly modulation for balanced multimodal learning. IEEE Trans Pattern Anal Mach Intell","DOI":"10.1109\/TPAMI.2024.3468315"},{"key":"610_CR65","unstructured":"Wei Y, Hu D (2024) MMPareto: boosting multimodal learning with innocent unimodal assistance. arXiv:2405.17730"},{"key":"610_CR66","doi-asserted-by":"crossref","unstructured":"Wei Y, Li S, Feng R, Hu D (2024) Diagnosing and re-learning for balanced multimodal learning. In: European conference on computer vision. Springer, Cham, pp 71\u201386","DOI":"10.1007\/978-3-031-73039-9_5"},{"key":"610_CR67","doi-asserted-by":"crossref","unstructured":"Wei S, Luo C, Luo Y (2025) Improving multimodal learning via imbalanced learning. In: Proceedings of the IEEE\/CVF international conference on computer vision. IEEE, Honolulu, HI, USA. Hawaii Convention Center, pp 2250\u20132259","DOI":"10.1109\/ICCV51701.2025.00217"},{"key":"610_CR68","doi-asserted-by":"crossref","unstructured":"Wu T, Liu A, Bijith P, Ravi V (2025) Innovative multimodal emotion detection: developing dual algorithms for audio-video integration in learning disability support. In: Proceedings of the IEEE 6th international conference on pattern recognition and machine learning (PRML). IEEE, Chongqing, China, pp 54\u201359","DOI":"10.1109\/PRML66062.2025.11159827"},{"key":"610_CR69","doi-asserted-by":"crossref","unstructured":"Xie Y, Zhu Z, Lu X, Huang Z, Xiong H (2024) InfoEnh: towards multimodal sentiment analysis via information bottleneck filter and optimal transport alignment. In: Proceedings of the joint international conference on computational linguistics, language resources and evaluation (LREC-COLING 2024), pp 9073\u20139083","DOI":"10.63317\/2a4kxypqhzvh"},{"key":"610_CR70","doi-asserted-by":"crossref","unstructured":"Xiong A, Zeng Y, Hu H (2024) Hierarchical knowledge stripping for multimodal sentiment analysis. IEEE Trans Affect Comput","DOI":"10.1109\/TAFFC.2024.3456117"},{"key":"610_CR71","doi-asserted-by":"crossref","unstructured":"Xu Q, Gu Y, Li C, Zhang H, Lin H-X, Liu L (2025) TDMER: a task-driven method for multimodal emotion recognition. In: Proceedings of the IEEE international conference on acoustics, speech and signal processing (ICASSP). IEEE, Hyderabad, India, pp 1\u20135","DOI":"10.1109\/ICASSP49660.2025.10889666"},{"key":"610_CR72","unstructured":"Xu G, Xue J, Liu Y, Wang Z, Zhang M, Song Z, Zhang Z (2024) Semantic-guided multimodal sentiment decoding with adversarial temporal-invariant learning. arXiv:2409.00143"},{"key":"610_CR73","doi-asserted-by":"crossref","unstructured":"Yang X, Bao C, Zhou J, Chenx X (2024) Target speaker extraction by directly exploiting contextual information in the time-frequency domain. In: Proceedings of the IEEE international conference on acoustics, speech and signal Processing (ICASSP). IEEE, Seoul, Korea, pp 10476\u201310480","DOI":"10.1109\/ICASSP48485.2024.10447529"},{"key":"610_CR74","doi-asserted-by":"crossref","unstructured":"Yang D, Huang S., Kuang H, Du Yx, Zhang L (2022) Disentangled representation learning for multimodal emotion recognition. In: Proceedings of the 30th ACM international conference on multimedia. ACM, Lisbon, Portugal, pp 1642\u20131651","DOI":"10.1145\/3503161.3547754"},{"issue":"1","key":"610_CR75","doi-asserted-by":"publisher","first-page":"206","DOI":"10.5755\/j01.itc.53.1.35060","volume":"53","author":"Z Yang","year":"2024","unstructured":"Yang Z, Li Z, Zhu D, Zhou Y (2024) TRI-CLT: learning tri-modal representations with contrastive learning and transformer for multimodal sentiment recognition. Inf Technol Control 53(1):206\u2013219","journal-title":"Inf Technol Control"},{"key":"610_CR76","doi-asserted-by":"crossref","unstructured":"Ying N, Mei T, Jiang Y, Hou M, Lin X (2025) Multimodal emotion recognition with dual-branch bidirectional information fusion for low frame rate scenarios. Available at SSRN 5381088","DOI":"10.2139\/ssrn.5381088"},{"issue":"22","key":"610_CR77","doi-asserted-by":"publisher","first-page":"35812","DOI":"10.1109\/JIOT.2024.3363176","volume":"11","author":"N Ying","year":"2024","unstructured":"Ying N, Jiang Y, Guo C, Zhou D, Zhao J (2024) A multimodal driver emotion recognition algorithm based on the audio and video signals in internet of vehicles platform. IEEE Internet Things J 11(22):35812\u201335824","journal-title":"IEEE Internet Things J"},{"key":"610_CR78","doi-asserted-by":"crossref","unstructured":"Yu J, Zhu W, Zhu J, Cai Z, Zhao G, Zhang Z, Xie G, Wei Z, Liu Q, Liang J (2024) Efficient feature extraction and late fusion strategy for audiovisual emotional mimicry intensity estimation. In: Proceedings of the IEEE\/CVF conference on computer vision and pattern recognition. IEEE, Seattle, WA, USA, pp 4866\u20134872","DOI":"10.1109\/CVPRW63382.2024.00490"},{"key":"610_CR79","doi-asserted-by":"crossref","unstructured":"Zadeh AB, Liang PP, Poria S, Cambria E, Morency L-P (2018) Multimodal language analysis in the wild: CMU-MOSEI dataset and interpretable dynamic fusion graph. In: Proceedings of the 56th annual meeting of the association for computational linguistics (Volume 1: Long Papers), Melbourne, Australia, pp 2236\u20132246","DOI":"10.18653\/v1\/P18-1208"},{"key":"610_CR80","doi-asserted-by":"crossref","unstructured":"Zhang X, Yoon J, Bansal M, Yao H (2024) Multimodal representation learning by alternating unimodal adaptation. In: Proceedings of the IEEE\/CVF conference on computer vision and pattern recognition. IEEE, Seattle, WA, USA, pp 27456\u201327466","DOI":"10.1109\/CVPR52733.2024.02592"},{"issue":"6","key":"610_CR81","doi-asserted-by":"publisher","first-page":"378","DOI":"10.1007\/s00530-024-01551-1","volume":"30","author":"Z Zhang","year":"2024","unstructured":"Zhang Z, Guo Q, Lu S, Su J, Tang T (2024) Integrating gating and learned queries in audiovisual emotion recognition. Multimedia Syst 30(6):378","journal-title":"Multimedia Syst"},{"key":"610_CR82","doi-asserted-by":"publisher","DOI":"10.1016\/j.eswa.2023.121692","volume":"237","author":"S Zhang","year":"2024","unstructured":"Zhang S, Yang Y, Chen C, Zhang X, Leng Q, Zhao X (2024) Deep learning-based multimodal emotion recognition from audio, visual, and text modalities: a systematic review of recent advancements and future prospects. Expert Syst Appl 237:121692","journal-title":"Expert Syst Appl"},{"key":"610_CR83","unstructured":"Zheng J, Zhang S, Wang X, Zeng Z (2022) Multimodal representations learning based on mutual information maximization and minimization and identity embedding for multimodal sentiment analysis. arXiv:2201.03969"},{"key":"610_CR84","doi-asserted-by":"crossref","unstructured":"Zhou Y, Liang X, Xu Y, Gao B (2025) Sample-level self-paced learning to tackle multimodal imbalance problem. In: Proceedings of the IEEE international conference on acoustics, speech and signal processing (ICASSP). IEEE, Hyderabad, India, pp 1\u20135","DOI":"10.1109\/ICASSP49660.2025.10888208"},{"key":"610_CR85","doi-asserted-by":"publisher","first-page":"2617","DOI":"10.1109\/TASLP.2021.3096037","volume":"29","author":"H Zhou","year":"2021","unstructured":"Zhou H, Du J, Zhang Y, Wang Q, Liu Q-F, Lee C-H (2021) Information fusion in attention networks using adaptive and multi-level factorized bilinear pooling for audio-visual emotion recognition. IEEE\/ACM Trans Audio Speech Lang Process 29:2617\u20132629","journal-title":"IEEE\/ACM Trans Audio Speech Lang Process"},{"key":"610_CR86","doi-asserted-by":"crossref","unstructured":"Zhu W, Li X (2022) Speech emotion recognition with global-aware fusion on multi-scale feature representation. In: Proceedings of the IEEE international conference on acoustics, speech and signal processing (ICASSP). IEEE, Singapore, pp 6437\u20136441","DOI":"10.1109\/ICASSP43922.2022.9747517"},{"key":"610_CR87","doi-asserted-by":"crossref","unstructured":"Zhu B, Wang C, Liu F, Lei J, Huang Z, Peng Y, Li F (2018) Learning environmental sounds with multi-scale convolutional neural network. In: Proceedings of the international joint conference on neural networks (IJCNN). IEEE, Rio de Janeiro, Brazil, pp 1\u20138","DOI":"10.1109\/IJCNN.2018.8489641"},{"key":"610_CR88","doi-asserted-by":"crossref","unstructured":"Zuo H, Liu R, Zhao J, Gao G, Li H (2023) Exploiting modality-invariant feature for robust multimodal emotion recognition with missing modalities. In: Proceedings of the IEEE international conference on acoustics, speech and signal processing (ICASSP). IEEE, Rhodes Island, Greece, pp 1\u20135","DOI":"10.1109\/ICASSP49357.2023.10095836"}],"container-title":["Journal of King Saud University Computer and Information Sciences"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s44443-026-00610-z","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s44443-026-00610-z.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s44443-026-00610-z.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,5,20]],"date-time":"2026-05-20T08:40:42Z","timestamp":1779266442000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s44443-026-00610-z"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026,2,28]]},"references-count":88,"journal-issue":{"issue":"5","published-print":{"date-parts":[[2026,7]]}},"alternative-id":["610"],"URL":"https:\/\/doi.org\/10.1007\/s44443-026-00610-z","relation":{},"ISSN":["1319-1578","2213-1248"],"issn-type":[{"value":"1319-1578","type":"print"},{"value":"2213-1248","type":"electronic"}],"subject":[],"published":{"date-parts":[[2026,2,28]]},"assertion":[{"value":"6 January 2026","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"23 February 2026","order":2,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"28 February 2026","order":3,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Declarations"}},{"value":"The authors declare no competing interests.","order":2,"name":"Ethics","group":{"name":"EthicsHeading","label":"Competing Interests"}},{"value":"Not applicable. This study used only publicly available benchmark datasets. No new data were collected from human participants by the authors, and no identifiable private information was involved.","order":3,"name":"Ethics","group":{"name":"EthicsHeading","label":"Ethics declaration"}}],"article-number":"180"}}