{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,16]],"date-time":"2026-04-16T21:16:59Z","timestamp":1776374219683,"version":"3.51.2"},"reference-count":30,"publisher":"Springer Science and Business Media LLC","issue":"24","license":[{"start":{"date-parts":[[2023,11,18]],"date-time":"2023-11-18T00:00:00Z","timestamp":1700265600000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2023,11,18]],"date-time":"2023-11-18T00:00:00Z","timestamp":1700265600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"funder":[{"name":"the Open Project Program of Shanghai Key Laboratory of Data Science","award":["No. 2020090600004"],"award-info":[{"award-number":["No. 2020090600004"]}]},{"name":"Shanghai Engineering Research Center of Intelligent Computing System","award":["No. 19DZ2252600"],"award-info":[{"award-number":["No. 19DZ2252600"]}]}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Appl Intell"],"published-print":{"date-parts":[[2023,12]]},"DOI":"10.1007\/s10489-023-05151-w","type":"journal-article","created":{"date-parts":[[2023,11,18]],"date-time":"2023-11-18T12:02:13Z","timestamp":1700308933000},"page":"30455-30468","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":22,"title":["A graph convolution-based heterogeneous fusion network for multimodal sentiment analysis"],"prefix":"10.1007","volume":"53","author":[{"given":"Tong","family":"Zhao","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-0143-9862","authenticated-orcid":false,"given":"Junjie","family":"Peng","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Yansong","family":"Huang","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Lan","family":"Wang","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Huiran","family":"Zhang","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Zesu","family":"Cai","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2023,11,18]]},"reference":[{"key":"5151_CR1","doi-asserted-by":"publisher","unstructured":"Zadeh A, Chen M, Poria S, Cambria E, Morency L (2017) Tensor fusion network for multimodal sentiment analysis. In: Palmer M, Hwa R, Riedel S (eds) Proceedings of the 2017 conference on empirical methods in natural language processing, EMNLP 2017, Copenhagen, Denmark, September 9-11, 2017, pp 1103\u20131114. https:\/\/doi.org\/10.18653\/v1\/d17-1115","DOI":"10.18653\/v1\/d17-1115"},{"key":"5151_CR2","doi-asserted-by":"publisher","unstructured":"Liu Z, Shen Y, Lakshminarasimhan VB, Liang PP, Zadeh A, Morency L (2018) Efficient low-rank multimodal fusion with modality-specific factors. In: Gurevych I, Miyao Y (eds) Proceedings of the 56th annual meeting of the association for computational linguistics, ACL 2018, Melbourne, Australia, July 15-20, 2018, vol 1: Long papers, pp 2247\u20132256. https:\/\/doi.org\/10.18653\/v1\/P18-1209","DOI":"10.18653\/v1\/P18-1209"},{"key":"5151_CR3","unstructured":"Vaswani A, Shazeer N, Parmar N, Uszkoreit J, Jones L, Gomez AN, Kaiser L, Polosukhin I (2017) Attention is all you need. In: Advances in neural information processing systems 30: annual conference on neural information processing systems 2017, December 4-9, 2017, Long Beach, CA, USA, pp 5998\u20136008. https:\/\/proceedings.neurips.cc\/paper\/2017\/hash\/3f5ee243547dee91fbd053c1c4a845aa-Abstract.html"},{"key":"5151_CR4","doi-asserted-by":"publisher","unstructured":"Wang Y, Shen Y, Liu Z, Liang PP, Zadeh A, Morency L (2019) Words can shift: dynamically adjusting word representations using nonverbal behaviors. In: The thirty-third AAAI conference on artificial intelligence, AAAI 2019, the thirty-first innovative applications of artificial intelligence conference, IAAI 2019, the ninth AAAI symposium on educational advances in artificial intelligence, EAAI 2019, Honolulu, Hawaii, USA, January 27 - February 1, 2019. AAAI Press, pp 7216\u20137223. https:\/\/doi.org\/10.1609\/aaai.v33i01.33017216","DOI":"10.1609\/aaai.v33i01.33017216"},{"key":"5151_CR5","doi-asserted-by":"publisher","unstructured":"Akhtar MS, Chauhan DS, Ghosal D, Poria S, Ekbal A, Bhattacharyya P (2019) Multi-task learning for multi-modal emotion recognition and sentiment analysis. In: Burstein J, Doran C, Solorio T (eds) Proceedings of the 2019 conference of the North American chapter of the association for computational linguistics: human language technologies, NAACL-HLT 2019, Minneapolis, MN, USA, June 2-7, 2019, vol 1 (Long and Short Papers). Association for computational linguistics, pp 370\u2013379. https:\/\/doi.org\/10.18653\/v1\/n19-1034","DOI":"10.18653\/v1\/n19-1034"},{"issue":"2","key":"5151_CR6","doi-asserted-by":"publisher","first-page":"423","DOI":"10.1109\/TPAMI.2018.2798607","volume":"41","author":"T Baltrusaitis","year":"2019","unstructured":"Baltrusaitis T, Ahuja C, Morency L (2019) Multimodal machine learning: a survey and taxonomy. IEEE Trans Pattern Anal Mach Intell 41(2):423\u2013443. https:\/\/doi.org\/10.1109\/TPAMI.2018.2798607","journal-title":"IEEE Trans Pattern Anal Mach Intell"},{"key":"5151_CR7","doi-asserted-by":"publisher","first-page":"184","DOI":"10.1016\/j.inffus.2020.09.005","volume":"66","author":"D Gkoumas","year":"2021","unstructured":"Gkoumas D, Li Q, Lioma C, Yu Y, Song D (2021) What makes the difference? an empirical comparison of fusion strategies for multimodal language analysis. Inf Fusion 66:184\u2013197. https:\/\/doi.org\/10.1016\/j.inffus.2020.09.005","journal-title":"Inf Fusion"},{"key":"5151_CR8","doi-asserted-by":"publisher","first-page":"204","DOI":"10.1016\/j.inffus.2021.06.003","volume":"76","author":"SA Abdu","year":"2021","unstructured":"Abdu SA, Yousef AH, Salem A (2021) Multimodal video sentiment analysis using deep learning approaches, a survey. Inf Fusion 76:204\u2013226. https:\/\/doi.org\/10.1016\/j.inffus.2021.06.003","journal-title":"Inf Fusion"},{"key":"5151_CR9","doi-asserted-by":"publisher","unstructured":"Zadeh A, Liang PP, Mazumder N, Poria S, Cambria E, Morency L-P (2018) Memory fusion network for multi-view sequential learning. In: Proceedings of the AAAI conference on artificial intelligence, vol 32. https:\/\/doi.org\/10.1609\/aaai.v32i1.12021","DOI":"10.1609\/aaai.v32i1.12021"},{"issue":"1","key":"5151_CR10","doi-asserted-by":"publisher","first-page":"320","DOI":"10.1109\/TAFFC.2020.3000510","volume":"13","author":"S Mai","year":"2020","unstructured":"Mai S, Hu H, Xu J, Xing S (2020) Multi-fusion residual memory network for multimodal human sentiment comprehension. IEEE Trans Affect Comput 13(1):320\u2013334. https:\/\/doi.org\/10.1109\/TAFFC.2020.3000510","journal-title":"IEEE Trans Affect Comput"},{"key":"5151_CR11","doi-asserted-by":"publisher","first-page":"279","DOI":"10.1016\/j.future.2020.08.005","volume":"115","author":"ME Basiri","year":"2021","unstructured":"Basiri ME, Nemati S, Abdar M, Cambria E, Acharya UR (2021) Abcdm: an attention-based bidirectional cnn-rnn deep model for sentiment analysis. Futur Gener Comput Syst 115:279\u2013294. https:\/\/doi.org\/10.1016\/j.future.2020.08.005","journal-title":"Futur Gener Comput Syst"},{"key":"5151_CR12","doi-asserted-by":"publisher","DOI":"10.1016\/j.knosys.2021.107676","volume":"235","author":"T Wu","year":"2022","unstructured":"Wu T, Peng J, Zhang W, Zhang H, Tan S, Yi F, Ma C, Huang Y (2022) Video sentiment analysis with bimodal information-augmented multi-head attention. Knowl Based Syst 235:107676. https:\/\/doi.org\/10.1016\/j.knosys.2021.107676","journal-title":"Knowl Based Syst"},{"key":"5151_CR13","doi-asserted-by":"publisher","DOI":"10.1016\/j.patcog.2022.109259","volume":"136","author":"D Wang","year":"2023","unstructured":"Wang D, Guo X, Tian Y, Liu J, He L, Luo X (2023) TETFN: a text enhanced transformer fusion network for multimodal sentiment analysis. Pattern Recognit 136:109259. https:\/\/doi.org\/10.1016\/j.patcog.2022.109259","journal-title":"Pattern Recognit"},{"issue":"5","key":"5151_CR14","doi-asserted-by":"publisher","first-page":"5105","DOI":"10.1109\/TKDE.2022.3155290","volume":"35","author":"X Xue","year":"2023","unstructured":"Xue X, Zhang C, Niu Z, Wu X (2023) Multi-level attention map network for multimodal sentiment analysis. IEEE Trans Knowl Data Eng 35(5):5105\u20135118. https:\/\/doi.org\/10.1109\/TKDE.2022.3155290","journal-title":"IEEE Trans Knowl Data Eng"},{"key":"5151_CR15","doi-asserted-by":"publisher","first-page":"3375","DOI":"10.1109\/TMM.2022.3160060","volume":"25","author":"T Zhu","year":"2023","unstructured":"Zhu T, Li L, Yang J, Zhao S, Liu H, Qian J (2023) Multimodal sentiment analysis with image-text interaction network. IEEE Trans Multim 25:3375\u20133385. https:\/\/doi.org\/10.1109\/TMM.2022.3160060","journal-title":"IEEE Trans Multim"},{"issue":"10","key":"5151_CR16","doi-asserted-by":"publisher","first-page":"12179","DOI":"10.1007\/s10489-022-03998-z","volume":"53","author":"X Zhang","year":"2023","unstructured":"Zhang X, Chen Y, He L (2023) Information block multi-head subspace based long short-term memory networks for sentiment analysis. Appl Intell 53(10):12179\u201312197. https:\/\/doi.org\/10.1007\/s10489-022-03998-z","journal-title":"Appl Intell"},{"key":"5151_CR17","doi-asserted-by":"publisher","DOI":"10.1016\/j.eswa.2023.119721","volume":"221","author":"J Peng","year":"2023","unstructured":"Peng J, Wu T, Zhang W, Cheng F, Tan S, Yi F, Huang Y (2023) A fine-grained modal label-based multi-stage network for multimodal sentiment analysis. Expert Syst Appl 221:119721. https:\/\/doi.org\/10.1016\/j.eswa.2023.119721","journal-title":"Expert Syst Appl"},{"key":"5151_CR18","doi-asserted-by":"publisher","first-page":"2689","DOI":"10.1109\/TASLP.2022.3192728","volume":"30","author":"Q Chen","year":"2022","unstructured":"Chen Q, Huang G, Wang Y (2022) The weighted cross-modal attention mechanism with sentiment prediction auxiliary task for multimodal sentiment analysis. IEEE ACM Trans Audio Speech Lang Process 30:2689\u20132695. https:\/\/doi.org\/10.1109\/TASLP.2022.3192728","journal-title":"IEEE ACM Trans Audio Speech Lang Process"},{"key":"5151_CR19","doi-asserted-by":"publisher","unstructured":"Wu J, Mai S, Hu H (2021) Graph capsule aggregation for unaligned multimodal sequences. In: Proceedings of the 2021 international conference on multimodal interaction, pp 521\u2013529. https:\/\/doi.org\/10.1145\/3462244.3479931","DOI":"10.1145\/3462244.3479931"},{"key":"5151_CR20","unstructured":"Sabour S, Frosst N, Hinton GE (2017) Dynamic routing between capsules. In: Advances in neural information processing systems 30: annual conference on neural information processing systems 2017, December 4-9, 2017, Long Beach, CA, USA, pp 3856\u20133866. https:\/\/proceedings.neurips.cc\/paper\/2017\/hash\/2cad8fa47bbef282badbb8de5374b894-Abstract.html"},{"key":"5151_CR21","doi-asserted-by":"publisher","unstructured":"Yang J, Wang Y, Yi R, Zhu Y, Rehman A, Zadeh A, Poria S, Morency L-P (2021) Mtag: modal-temporal attention graph for unaligned human multimodal language sequences. In: Proceedings of the 2021 conference of the North American chapter of the association for computational linguistics: human language technologies, pp 1009\u20131021. https:\/\/doi.org\/10.18653\/v1\/2021.naacl-main.79","DOI":"10.18653\/v1\/2021.naacl-main.79"},{"key":"5151_CR22","doi-asserted-by":"publisher","unstructured":"Yang X, Feng S, Zhang Y, Wang D (2021) Multimodal sentiment detection based on multi-channel graph neural networks. In: Zong C, Xia F, Li W, Navigli R (eds) Proceedings of the 59th annual meeting of the association for computational linguistics and the 11th international joint conference on natural language processing, ACL\/IJCNLP 2021, (vol 1: Long Papers), Virtual Event, August 1-6, 2021. Association for computational linguistics, pp 328\u2013339. https:\/\/doi.org\/10.18653\/v1\/2021.acl-long.28","DOI":"10.18653\/v1\/2021.acl-long.28"},{"key":"5151_CR23","doi-asserted-by":"publisher","unstructured":"Zeng Z, Sun S, Li Q (2023) Multimodal negative sentiment recognition of online public opinion on public health emergencies based on\u00a0graph convolutional networks and ensemble learning. Inf Process Manag 60(4):103378. https:\/\/doi.org\/10.1016\/j.ipm.2023.103378","DOI":"10.1016\/j.ipm.2023.103378"},{"issue":"8","key":"5151_CR24","doi-asserted-by":"publisher","first-page":"8542","DOI":"10.1109\/TITS.2022.3205477","volume":"24","author":"Y Zhang","year":"2023","unstructured":"Zhang Y, Tiwari P, Zheng Q, El-Saddik A, Hossain MS (2023) A multimodal coupled graph attention network for joint traffic event detection and sentiment classification. IEEE Trans Intell Transp Syst 24(8):8542\u20138554. https:\/\/doi.org\/10.1109\/TITS.2022.3205477","journal-title":"IEEE Trans Intell Transp Syst"},{"issue":"7","key":"5151_CR25","doi-asserted-by":"publisher","first-page":"4408","DOI":"10.1007\/s10489-020-02095-3","volume":"51","author":"Q Lu","year":"2021","unstructured":"Lu Q, Zhu Z, Zhang G, Kang S, Liu P (2021) Aspect-gated graph convolutional networks for aspect-based sentiment analysis. Appl Intell 51(7):4408\u20134419. https:\/\/doi.org\/10.1007\/s10489-020-02095-3","journal-title":"Appl Intell"},{"key":"5151_CR26","doi-asserted-by":"publisher","unstructured":"Xu Q, Peng J, Zheng C, Tan S, Yi F, Cheng F (2023) Short text classification of chinese with label information assisting. ACM Transactions on Asian and Low-Resource Language Information Processing, 1\u201318. https:\/\/doi.org\/10.1145\/3582301","DOI":"10.1145\/3582301"},{"issue":"6","key":"5151_CR27","doi-asserted-by":"publisher","first-page":"82","DOI":"10.1109\/MIS.2016.94","volume":"31","author":"A Zadeh","year":"2016","unstructured":"Zadeh A, Zellers R, Pincus E, Morency L-P (2016) Multimodal sentiment intensity analysis in videos: facial gestures and verbal messages. IEEE Intell Syst 31(6):82\u201388. https:\/\/doi.org\/10.1109\/MIS.2016.94","journal-title":"IEEE Intell Syst"},{"key":"5151_CR28","doi-asserted-by":"publisher","unstructured":"Zadeh AB, Liang PP, Poria S, Cambria E, Morency L-P (2018) Multimodal language analysis in the wild: cmu-mosei dataset and interpretable dynamic fusion graph. In: Proceedings of the 56th annual meeting of the association for computational linguistics (vol 1: Long Papers), pp 2236\u20132246. https:\/\/doi.org\/10.18653\/v1\/P18-1208","DOI":"10.18653\/v1\/P18-1208"},{"key":"5151_CR29","doi-asserted-by":"publisher","unstructured":"Tsai Y-HH, Bai S, Liang PP, Kolter JZ, Morency L-P, Salakhutdinov R (2019) Multimodal transformer for unaligned multimodal language sequences. In: Proceedings of the conference. Association for computational linguistics. Meeting, vol 2019, p 6558. https:\/\/doi.org\/10.18653\/v1\/p19-1656","DOI":"10.18653\/v1\/p19-1656"},{"key":"5151_CR30","doi-asserted-by":"publisher","unstructured":"Hazarika D, Zimmermann R, Poria S (2020) MISA: modality-invariant and -specific representations for multimodal sentiment analysis. In: Chen CW, Cucchiara R, Hua X, Qi G, Ricci E, Zhang Z, Zimmermann R (eds) MM \u201920: the 28th ACM international conference on multimedia, Virtual Event \/ Seattle, WA, USA, October 12-16, 2020. ACM, pp 1122\u20131131. https:\/\/doi.org\/10.1145\/3394171.3413678","DOI":"10.1145\/3394171.3413678"}],"container-title":["Applied Intelligence"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10489-023-05151-w.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s10489-023-05151-w\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10489-023-05151-w.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2023,12,28]],"date-time":"2023-12-28T06:23:57Z","timestamp":1703744637000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s10489-023-05151-w"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023,11,18]]},"references-count":30,"journal-issue":{"issue":"24","published-print":{"date-parts":[[2023,12]]}},"alternative-id":["5151"],"URL":"https:\/\/doi.org\/10.1007\/s10489-023-05151-w","relation":{},"ISSN":["0924-669X","1573-7497"],"issn-type":[{"value":"0924-669X","type":"print"},{"value":"1573-7497","type":"electronic"}],"subject":[],"published":{"date-parts":[[2023,11,18]]},"assertion":[{"value":"1 November 2023","order":1,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"18 November 2023","order":2,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Declarations"}},{"value":"This article has never been submitted to more than one journal for simultaneous consideration. This article is original.","order":2,"name":"Ethics","group":{"name":"EthicsHeading","label":"Ethics approval"}},{"value":"The authors have approved this article before submission, including the names and order of authors.","order":3,"name":"Ethics","group":{"name":"EthicsHeading","label":"Consent to participate"}},{"value":"The authors agreed with the content and gave explicit consent to submit.","order":4,"name":"Ethics","group":{"name":"EthicsHeading","label":"Consent for publication"}},{"value":"The authors declared that they have no conflict of interest to this article.","order":5,"name":"Ethics","group":{"name":"EthicsHeading","label":"Competing interests"}}]}}