{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,16]],"date-time":"2026-04-16T21:17:00Z","timestamp":1776374220866,"version":"3.51.2"},"reference-count":36,"publisher":"Springer Science and Business Media LLC","issue":"19","license":[{"start":{"date-parts":[[2024,3,30]],"date-time":"2024-03-30T00:00:00Z","timestamp":1711756800000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2024,3,30]],"date-time":"2024-03-30T00:00:00Z","timestamp":1711756800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Neural Comput & Applic"],"published-print":{"date-parts":[[2024,7]]},"DOI":"10.1007\/s00521-024-09634-w","type":"journal-article","created":{"date-parts":[[2024,3,30]],"date-time":"2024-03-30T05:01:56Z","timestamp":1711774916000},"page":"11121-11135","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":7,"title":["TSCL-FHFN: two-stage contrastive learning and feature hierarchical fusion network for multimodal sentiment analysis"],"prefix":"10.1007","volume":"36","author":[{"given":"Yuqiang","family":"Li","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Wenxuan","family":"Weng","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Chun","family":"Liu","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2024,3,30]]},"reference":[{"issue":"2","key":"9634_CR1","doi-asserted-by":"publisher","first-page":"828","DOI":"10.1109\/TKDE.2020.2983360","volume":"34","author":"J Li","year":"2020","unstructured":"Li J, Chiu B, Shang S, Shao L (2020) Neural text segmentation and its application to sentiment analysis. IEEE Trans Knowl Data Eng 34(2):828\u2013842","journal-title":"IEEE Trans Knowl Data Eng"},{"issue":"4","key":"9634_CR2","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1145\/3503044","volume":"55","author":"G Brauwers","year":"2022","unstructured":"Brauwers G, Frasincar F (2022) A survey on aspect-based sentiment classification. ACM Comput Surv 55(4):1\u201337","journal-title":"ACM Comput Surv"},{"key":"9634_CR3","doi-asserted-by":"crossref","unstructured":"Yang J, She D, Lai Y-K, Yang M-H (2018) Retrieving and classifying affective images via deep metric learning. In: Proceedings of the AAAI conference on artificial intelligence 32:491\u2013498","DOI":"10.1609\/aaai.v32i1.11275"},{"key":"9634_CR4","first-page":"1","volume":"2022","author":"S Yang","year":"2022","unstructured":"Yang S, Xing L, Chang Z, Li Y et al (2022) Attention-based sentiment region importance and relationship analysis for image sentiment recognition. Comput Intell Neurosci 2022:1\u201314","journal-title":"Comput Intell Neurosci"},{"issue":"10","key":"9634_CR5","doi-asserted-by":"publisher","first-page":"1163","DOI":"10.3390\/electronics10101163","volume":"10","author":"E Lieskovsk\u00e1","year":"2021","unstructured":"Lieskovsk\u00e1 E, Jakubec M, Jarina R, Chmul\u00edk M (2021) A review on speech emotion recognition using deep learning and attention mechanism. Electronics 10(10):1163","journal-title":"Electronics"},{"key":"9634_CR6","doi-asserted-by":"crossref","unstructured":"Zou H, Si Y, Chen C, Rajan D, Chng ES (2022) Speech emotion recognition with co-attention based multi-level acoustic information. In: ICASSP 2022\u20132022 IEEE international conference on acoustics, speech and signal processing (ICASSP). IEEE, pp 7367\u20137371","DOI":"10.1109\/ICASSP43922.2022.9747095"},{"key":"9634_CR7","doi-asserted-by":"publisher","first-page":"424","DOI":"10.1016\/j.inffus.2022.09.025","volume":"91","author":"A Gandhi","year":"2022","unstructured":"Gandhi A, Adhvaryu K, Poria S, Cambria E, Hussain A (2022) Multimodal sentiment analysis: a systematic review of history, datasets, multimodal fusion methods, applications, challenges and future directions. Inf Fus 91:424\u2013444","journal-title":"Inf Fus"},{"issue":"27","key":"9634_CR8","doi-asserted-by":"publisher","first-page":"411","DOI":"10.1016\/j.ifacol.2019.12.696","volume":"52","author":"A Rozanska","year":"2019","unstructured":"Rozanska A, Podpora M (2019) Multimodal sentiment analysis applied to interaction between patients and a humanoid robot pepper. IFAC-PapersOnLine 52(27):411\u2013414","journal-title":"IFAC-PapersOnLine"},{"key":"9634_CR9","doi-asserted-by":"crossref","unstructured":"Tumasjan A, Sprenger T, Sandner P, Welpe I (2010) Predicting elections with twitter: What 140 characters reveal about political sentiment. In: Proceedings of the international AAAI conference on web and social media vol 4, pp 178\u2013185","DOI":"10.1609\/icwsm.v4i1.14009"},{"key":"9634_CR10","doi-asserted-by":"publisher","first-page":"2601","DOI":"10.1007\/s12652-020-02423-y","volume":"12","author":"P Chauhan","year":"2021","unstructured":"Chauhan P, Sharma N, Sikka G (2021) The emergence of social media data and sentiment analysis in election prediction. J Ambient Intell Humaniz Comput 12:2601\u20132627","journal-title":"J Ambient Intell Humaniz Comput"},{"key":"9634_CR11","doi-asserted-by":"crossref","unstructured":"Mai S, Hu H, Xing S (2020) Modality to modality translation: An adversarial representation learning and graph fusion network for multimodal fusion. In: Proceedings of the AAAI Conference on Artificial Intelligence vol 34, pp 164\u2013172","DOI":"10.1609\/aaai.v34i01.5347"},{"key":"9634_CR12","doi-asserted-by":"crossref","unstructured":"Hu J, Liu Y, Zhao J, Jin Q (2021) MMGCN: multimodal fusion via deep graph convolution network for emotion recognition in conversation. In: Proceedings of the 59th annual meeting of the association for computational linguistics and the 11th international joint conference on natural language processing (Long Papers), vol 1, pp 5666\u20135675","DOI":"10.18653\/v1\/2021.acl-long.440"},{"issue":"2","key":"9634_CR13","doi-asserted-by":"publisher","first-page":"103229","DOI":"10.1016\/j.ipm.2022.103229","volume":"60","author":"H Lin","year":"2023","unstructured":"Lin H, Zhang P, Ling J, Yang Z, Lee LK, Liu W (2023) Ps-mixer: a polar-vector and strength-vector mixer model for multimodal sentiment analysis. Inf Process Manag 60(2):103229\u2013103245","journal-title":"Inf Process Manag"},{"key":"9634_CR14","doi-asserted-by":"publisher","first-page":"5455","DOI":"10.1007\/s10462-020-09825-6","volume":"53","author":"A Khan","year":"2020","unstructured":"Khan A, Sohail A, Zahoora U, Qureshi AS (2020) A survey of the recent architectures of deep convolutional neural networks. Artif Intell Rev 53:5455\u20135516","journal-title":"Artif Intell Rev"},{"key":"9634_CR15","doi-asserted-by":"crossref","unstructured":"Li Z, Liu F, Yang W, Peng S, Zhou J (2021) A survey of convolutional neural networks: analysis, applications, and prospects. In: IEEE transactions on neural networks and learning systems 33(12):6999\u20137019","DOI":"10.1109\/TNNLS.2021.3084827"},{"key":"9634_CR16","doi-asserted-by":"publisher","first-page":"5929","DOI":"10.1007\/s10462-020-09838-1","volume":"53","author":"G Van Houdt","year":"2020","unstructured":"Van Houdt G, Mosquera C, N\u00e1poles G (2020) A review on the long short-term memory model. Artif Intell Rev 53:5929\u20135955","journal-title":"Artif Intell Rev"},{"key":"9634_CR17","doi-asserted-by":"publisher","DOI":"10.1016\/j.knosys.2021.107657","volume":"235","author":"IC Kaadoud","year":"2022","unstructured":"Kaadoud IC, Rougier NP, Alexandre F (2022) Knowledge extraction from the learning of sequences in a long short term memory (LSTM) architecture. Knowl-Based Syst 235:107657","journal-title":"Knowl-Based Syst"},{"key":"9634_CR18","doi-asserted-by":"crossref","unstructured":"Yu W, Xu H, Yuan Z, Wu J (2021) Learning modality-specific representations with self-supervised multi-task learning for multimodal sentiment analysis. In: Proceedings of the AAAI conference on artificial intelligence, vol 35, pp 10790\u201310797","DOI":"10.1609\/aaai.v35i12.17289"},{"key":"9634_CR19","doi-asserted-by":"crossref","unstructured":"Karpathy A, Toderici G, Shetty S, Leung T, Sukthankar R, Fei-Fei L (2014) Large-scale video classification with convolutional neural networks. In: Proceedings of the IEEE conference on computer vision and pattern recognition, pp 1725\u20131732","DOI":"10.1109\/CVPR.2014.223"},{"key":"9634_CR20","doi-asserted-by":"crossref","unstructured":"Hazarika D, Zimmermann R, Poria S (2020) Misa: Modality-invariant and-specific representations for multimodal sentiment analysis. In: Proceedings of the 28th ACM international conference on multimedia, pp 1122\u20131131","DOI":"10.1145\/3394171.3413678"},{"key":"9634_CR21","doi-asserted-by":"publisher","first-page":"4121","DOI":"10.1109\/TMM.2022.3171679","volume":"25","author":"S Mai","year":"2023","unstructured":"Mai S, Zeng Y, Hu H (2023) Multimodal information bottleneck: learning minimal sufficient unimodal and multimodal representations. IEEE Trans Multimedia 25:4121\u20134134","journal-title":"IEEE Trans Multimedia"},{"key":"9634_CR22","doi-asserted-by":"crossref","unstructured":"Zadeh A, Chen M, Poria S, Cambria E, Morency L-P (2017) Tensor fusion network for multimodal sentiment analysis. In: Proceedings of the 2017 conference on empirical methods in natural language processing, pp 1103\u22121114","DOI":"10.18653\/v1\/D17-1115"},{"key":"9634_CR23","doi-asserted-by":"crossref","unstructured":"Verma S, Wang J, Ge Z, Shen R, Jin F, Wang Y, Chen F, Liu W (2020) Deep-HOSeq: deep higher order sequence fusion for multimodal sentiment analysis. In: 2020 IEEE international conference on data mining (ICDM), pp. 561\u2013570. IEEE","DOI":"10.1109\/ICDM50108.2020.00065"},{"key":"9634_CR24","doi-asserted-by":"crossref","unstructured":"Tsai Y-HH, Bai S, Liang PP, Kolter JZ, Morency L-P, Salakhutdinov R (2019) Multimodal transformer for unaligned multimodal language sequences. In: Proceedings of the conference association for computational linguistics meeting, vol 2019, pp 6558\u20136569","DOI":"10.18653\/v1\/P19-1656"},{"key":"9634_CR25","unstructured":"Ma L, Yao Y, Liang T, Liu T (2022) Multi-scale cooperative multimodal transformers for multimodal sentiment analysis in videos. arXiv preprint arXiv:2206.07981"},{"key":"9634_CR26","doi-asserted-by":"publisher","first-page":"2015","DOI":"10.1109\/TASLP.2022.3178204","volume":"30","author":"B Yang","year":"2022","unstructured":"Yang B, Wu L, Zhu J, Shao B, Lin X, Liu T-Y (2022) Multimodal sentiment analysis with two-phase multi-task learning. IEEE\/ACM Trans Audio Speech Lang Process 30:2015\u20132024","journal-title":"IEEE\/ACM Trans Audio Speech Lang Process"},{"key":"9634_CR27","doi-asserted-by":"crossref","unstructured":"Yang J, Wang Y, Yi R, Zhu Y, Rehman A, Zadeh A, Poria S, Morency L-P (2021) MTAG: modal-temporal attention graph for unaligned human multimodal language sequences. In: Proceedings of the 2021 conference of the North American chapter of the association for computational linguistics: human language technologies, pp 1009\u20131021","DOI":"10.18653\/v1\/2021.naacl-main.79"},{"key":"9634_CR28","doi-asserted-by":"publisher","first-page":"542","DOI":"10.1016\/j.inffus.2022.11.003","volume":"91","author":"S Mai","year":"2023","unstructured":"Mai S, Sun Y, Zeng Y, Hu H (2023) Excavating multimodal correlation for representation learning. Inf Fus 91:542\u2013555","journal-title":"Inf Fus"},{"key":"9634_CR29","doi-asserted-by":"crossref","unstructured":"Pan T, Song Y, Yang T, Jiang W, Liu W (2021) Videomoco: contrastive video representation learning with temporally adversarial examples. In: Proceedings of the IEEE\/CVF conference on computer vision and pattern recognition, pp 11205\u201311214","DOI":"10.1109\/CVPR46437.2021.01105"},{"key":"9634_CR30","unstructured":"Kendall A, Gal Y, Cipolla R (2018) Multi-task learning using uncertainty to weigh losses for scene geometry and semantics. In: Proceedings of the IEEE conference on computer vision and pattern recognition, pp 7482\u20137491"},{"issue":"6","key":"9634_CR31","doi-asserted-by":"publisher","first-page":"82","DOI":"10.1109\/MIS.2016.94","volume":"31","author":"A Zadeh","year":"2016","unstructured":"Zadeh A, Zellers R, Pincus E, Morency L-P (2016) Multimodal sentiment intensity analysis in videos: facial gestures and verbal messages. IEEE Intell Syst 31(6):82\u201388","journal-title":"IEEE Intell Syst"},{"key":"9634_CR32","unstructured":"Zadeh A.B, Liang P.P, Poria S, Cambria E, Morency L-P (2018) Multimodal language analysis in the wild: CMU-MOSEI dataset and interpretable dynamic fusion graph. In: Proceedings of the 56th annual meeting of the association for computational linguistics (long papers), vol 1, pp 2236\u20132246"},{"issue":"5","key":"9634_CR33","doi-asserted-by":"publisher","first-page":"3878","DOI":"10.1121\/1.2935783","volume":"123","author":"J Yuan","year":"2008","unstructured":"Yuan J, Liberman M et al (2008) Speaker identification on the SCOTUS corpus. J Acoust Soc Am 123(5):3878","journal-title":"J Acoust Soc Am"},{"key":"9634_CR34","doi-asserted-by":"crossref","unstructured":"Degottex G, Kane J, Drugman T, Raitio T, Scherer S (2014) Covarepa collaborative voice analysis repository for speech technologies. In: 2014 IEEE international conference on acoustics, speech and signal processing (ICASSP). IEEE, pp 960\u2013964","DOI":"10.1109\/ICASSP.2014.6853739"},{"issue":"1","key":"9634_CR35","doi-asserted-by":"publisher","first-page":"296","DOI":"10.1016\/j.inffus.2022.07.006","volume":"88","author":"F Zhang","year":"2022","unstructured":"Zhang F, Li X-C, Lim CP, Hua Q, Dong C-R, Zhai J-H (2022) Deep emotional arousal network for multimodal sentiment analysis and emotion recognition. Inf Fus 88(1):296\u2013304","journal-title":"Inf Fus"},{"key":"9634_CR36","doi-asserted-by":"crossref","unstructured":"Ma Y, Ma B (2022) Multimodal sentiment analysis on unaligned sequences via holographic embedding. In: ICASSP 2022-2022 IEEE international conference on acoustics, speech and signal processing (ICASSP). IEEE, pp 8547\u20138551","DOI":"10.1109\/ICASSP43922.2022.9747646"}],"container-title":["Neural Computing and Applications"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s00521-024-09634-w.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s00521-024-09634-w\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s00521-024-09634-w.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,6,25]],"date-time":"2024-06-25T11:09:53Z","timestamp":1719313793000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s00521-024-09634-w"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,3,30]]},"references-count":36,"journal-issue":{"issue":"19","published-print":{"date-parts":[[2024,7]]}},"alternative-id":["9634"],"URL":"https:\/\/doi.org\/10.1007\/s00521-024-09634-w","relation":{},"ISSN":["0941-0643","1433-3058"],"issn-type":[{"value":"0941-0643","type":"print"},{"value":"1433-3058","type":"electronic"}],"subject":[],"published":{"date-parts":[[2024,3,30]]},"assertion":[{"value":"16 August 2023","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"21 February 2024","order":2,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"30 March 2024","order":3,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Declarations"}},{"value":"The author(s) declared no potential conflicts of interest with respect to the research, authorship, and\/or publication of this article.","order":2,"name":"Ethics","group":{"name":"EthicsHeading","label":"Conflict of interest"}}]}}