{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,17]],"date-time":"2026-07-17T10:27:05Z","timestamp":1784284025143,"version":"3.55.0"},"reference-count":84,"publisher":"Institute of Electrical and Electronics Engineers (IEEE)","license":[{"start":{"date-parts":[[2021,1,1]],"date-time":"2021-01-01T00:00:00Z","timestamp":1609459200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/legalcode"}],"funder":[{"DOI":"10.13039\/501100003662","name":"Korea Evaluation Institute of Industrial Technology","doi-asserted-by":"publisher","award":["10073162"],"award-info":[{"award-number":["10073162"]}],"id":[{"id":"10.13039\/501100003662","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["IEEE Access"],"published-print":{"date-parts":[[2021]]},"DOI":"10.1109\/access.2021.3092735","type":"journal-article","created":{"date-parts":[[2021,6,28]],"date-time":"2021-06-28T21:23:08Z","timestamp":1624915388000},"page":"94557-94572","source":"Crossref","is-referenced-by-count":84,"title":["Multimodal Emotion Recognition Fusion Analysis Adapting BERT With Heterogeneous Feature Unification"],"prefix":"10.1109","volume":"9","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-1107-0122","authenticated-orcid":false,"given":"Sanghyun","family":"Lee","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-5055-5408","authenticated-orcid":false,"given":"David K.","family":"Han","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-8744-4514","authenticated-orcid":false,"given":"Hanseok","family":"Ko","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"263","reference":[{"key":"ref73","first-page":"1","article-title":"OpenFace: An open source facial behavior analysis toolkit","author":"baltrusaitis","year":"2016","journal-title":"Proc IEEE Winter Conf Appl Comput Vis (WACV)"},{"key":"ref72","doi-asserted-by":"publisher","DOI":"10.1109\/ICASSP.2017.7952261"},{"key":"ref71","doi-asserted-by":"publisher","DOI":"10.1109\/ICASSP40776.2020.9053648"},{"key":"ref70","article-title":"Investigation of multimodal features, classifiers and fusion methods for emotion recognition","author":"lian","year":"2018","journal-title":"arXiv 1809 06225"},{"key":"ref76","author":"ekman","year":"1978","journal-title":"Facial Action Coding System Investigator’s Guide"},{"key":"ref77","doi-asserted-by":"publisher","DOI":"10.5244\/C.29.41"},{"key":"ref74","article-title":"Very deep convolutional networks for large-scale image recognition","author":"simonyan","year":"2014","journal-title":"arXiv 1409 1556"},{"key":"ref39","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/P18-1208"},{"key":"ref75","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2016.90"},{"key":"ref38","article-title":"MOSI: Multimodal corpus of sentiment intensity and subjectivity analysis in online opinion videos","author":"zadeh","year":"2016","journal-title":"arXiv 1606 06259"},{"key":"ref78","article-title":"Google’s neural machine translation system: Bridging the gap between human and machine translation","author":"wu","year":"2016","journal-title":"arXiv 1609 08144"},{"key":"ref79","article-title":"Attention is all you need","author":"vaswani","year":"2017","journal-title":"arXiv 1706 03762"},{"key":"ref33","doi-asserted-by":"publisher","DOI":"10.1109\/TAFFC.2020.3000510"},{"key":"ref32","doi-asserted-by":"crossref","first-page":"6558","DOI":"10.18653\/v1\/P19-1656","article-title":"Multimodal transformer for unaligned multimodal language sequences","author":"tsai","year":"2019","journal-title":"Proc Annual Meeting of the Assoc Computational Linguistics"},{"key":"ref31","doi-asserted-by":"publisher","DOI":"10.1145\/3234695.3240986"},{"key":"ref30","doi-asserted-by":"publisher","DOI":"10.1109\/TPAMI.2016.2515606"},{"key":"ref37","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v33i01.33016892"},{"key":"ref36","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D18-1382"},{"key":"ref35","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v34i05.6431"},{"key":"ref34","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v33i01.33017216"},{"key":"ref60","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/P18-1209"},{"key":"ref62","doi-asserted-by":"publisher","DOI":"10.1109\/BigDataSecurity-HPSC-IDS.2019.00027"},{"key":"ref61","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/P17-1081"},{"key":"ref63","doi-asserted-by":"publisher","DOI":"10.1109\/ICASSP.2019.8682823"},{"key":"ref28","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2015.341"},{"key":"ref64","doi-asserted-by":"crossref","first-page":"9","DOI":"10.3389\/fcomp.2020.00009","article-title":"A review of generalizable transfer learning in automatic emotion recognition","volume":"2","author":"feng","year":"2020","journal-title":"Frontiers Comput Sci"},{"key":"ref27","first-page":"6499","article-title":"HGFM: A hierarchical grained and feature model for acoustic emotion recognition","author":"xu","year":"2020","journal-title":"Proc IEEE Int Conf Acoust Speech Signal Process (ICASSP)"},{"key":"ref65","article-title":"BERT: Pre-training of deep bidirectional transformers for language understanding","author":"devlin","year":"2018","journal-title":"arXiv 1810 04805"},{"key":"ref66","doi-asserted-by":"publisher","DOI":"10.17763\/haer.47.3.8840364413869005"},{"key":"ref29","doi-asserted-by":"publisher","DOI":"10.1109\/ACIIAsia.2018.8470326"},{"key":"ref67","first-page":"1","article-title":"The INTERSPEECH 2010 paralinguistic challenge","author":"schuller","year":"2010","journal-title":"Proc 11th Annu Conf Int Speech Commun Assoc"},{"key":"ref68","article-title":"SoundNet: Learning sound representations from unlabeled video","author":"aytar","year":"2016","journal-title":"arXiv 1610 09001"},{"key":"ref69","doi-asserted-by":"publisher","DOI":"10.1109\/ICASSP.2017.7952132"},{"key":"ref2","doi-asserted-by":"publisher","DOI":"10.21437\/Interspeech.2018-1858"},{"key":"ref1","doi-asserted-by":"publisher","DOI":"10.1016\/j.cortex.2019.03.020"},{"key":"ref20","article-title":"Distributed representations of words and phrases and their compositionality","author":"mikolov","year":"2013","journal-title":"arXiv 1310 4546"},{"key":"ref22","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/P17-1067"},{"key":"ref21","article-title":"A sensitivity analysis of (and practitioners’ guide to) convolutional neural networks for sentence classification","author":"zhang","year":"2015","journal-title":"arXiv 1510 03820"},{"key":"ref24","doi-asserted-by":"publisher","DOI":"10.24963\/ijcai.2019\/752"},{"key":"ref23","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D19-1015"},{"key":"ref26","doi-asserted-by":"publisher","DOI":"10.3390\/s20226688"},{"key":"ref25","article-title":"HiGRU: Hierarchical gated recurrent units for utterance-level emotion recognition","author":"jiao","year":"2019","journal-title":"arXiv 1904 04446"},{"key":"ref50","doi-asserted-by":"publisher","DOI":"10.1109\/ICME.2017.8019301"},{"key":"ref51","article-title":"Investigating audio, visual, and text fusion methods for end-to-end automatic personality prediction","author":"kampman","year":"2018","journal-title":"arXiv 1805 00705"},{"key":"ref59","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D18-1014"},{"key":"ref58","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D17-1115"},{"key":"ref57","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v33i01.3301371"},{"key":"ref56","article-title":"Multimodal classification for analysing social media","author":"duong","year":"2017","journal-title":"arXiv 1708 02099"},{"key":"ref55","first-page":"1","article-title":"Inferring emotion from conversational voice data: A semi-supervised multi-path generative neural network approach","volume":"32","author":"zhou","year":"2018","journal-title":"Proc AAAI Conf Artif Intell"},{"key":"ref54","first-page":"649","article-title":"Jointly discovering visual objects and spoken words from raw sensory input","author":"harwath","year":"2018","journal-title":"Proc Eur Conf Comput Vis (ECCV)"},{"key":"ref53","doi-asserted-by":"publisher","DOI":"10.5772\/54002"},{"key":"ref52","doi-asserted-by":"publisher","DOI":"10.1145\/2993148.2993176"},{"key":"ref10","doi-asserted-by":"publisher","DOI":"10.1109\/FG.2015.7284869"},{"key":"ref11","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2016.602"},{"key":"ref40","doi-asserted-by":"publisher","DOI":"10.1007\/s10579-008-9076-6"},{"key":"ref12","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2017.361"},{"key":"ref13","article-title":"Deep learning using linear support vector machines","author":"tang","year":"2013","journal-title":"arXiv 1306 0239"},{"key":"ref14","first-page":"543","article-title":"Combining modality specific deep neural networks for emotion recognition in video","author":"kahou","year":"2013","journal-title":"Proc 15th ACM Int Conf Multimodal Interact"},{"key":"ref15","doi-asserted-by":"publisher","DOI":"10.1109\/FG.2018.00051"},{"key":"ref82","article-title":"Adam: A method for stochastic optimization","author":"kingma","year":"2014","journal-title":"arXiv 1412 6980"},{"key":"ref16","doi-asserted-by":"publisher","DOI":"10.1109\/TIP.2017.2689999"},{"key":"ref81","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2015.11"},{"key":"ref17","first-page":"761","article-title":"Just how mad are you? finding strong and weak opinion clauses","volume":"4","author":"wilson","year":"2004","journal-title":"Proc AAAI"},{"key":"ref84","doi-asserted-by":"publisher","DOI":"10.1109\/MIS.2016.94"},{"key":"ref18","doi-asserted-by":"publisher","DOI":"10.1109\/WI.2007.51"},{"key":"ref83","first-page":"1","article-title":"Memory fusion network for multi-view sequential learning","volume":"32","author":"zadeh","year":"2018","journal-title":"Proc AAAI Conf Artif Intell"},{"key":"ref19","doi-asserted-by":"publisher","DOI":"10.3115\/v1\/D14-1162"},{"key":"ref80","article-title":"Layer normalization","author":"ba","year":"2016","journal-title":"arXiv 1607 06450"},{"key":"ref4","doi-asserted-by":"publisher","DOI":"10.1109\/ACII.2017.8273599"},{"key":"ref3","doi-asserted-by":"publisher","DOI":"10.1145\/2502081.2502224"},{"key":"ref6","doi-asserted-by":"publisher","DOI":"10.1109\/ICASSP.2016.7472669"},{"key":"ref5","first-page":"1","article-title":"Speech emotion recognition using deep neural network and extreme learning machine","author":"han","year":"2014","journal-title":"Proc 15th Annu Conf Int Speech Commun Assoc"},{"key":"ref8","doi-asserted-by":"publisher","DOI":"10.21437\/Interspeech.2019-1649"},{"key":"ref7","doi-asserted-by":"publisher","DOI":"10.1145\/3319921.3319963"},{"key":"ref49","first-page":"260","article-title":"Speech intention classification with multimodal deep learning","author":"gu","year":"2017","journal-title":"Proc 9th Canadian Conf Artificial Intell"},{"key":"ref9","doi-asserted-by":"publisher","DOI":"10.1109\/CVPRW.2006.85"},{"key":"ref46","doi-asserted-by":"crossref","first-page":"1863","DOI":"10.3390\/s19081863","article-title":"A review on automatic facial expression recognition systems assisted by multimodal sensor data","volume":"19","author":"samadiani","year":"2019","journal-title":"SENSORS"},{"key":"ref45","doi-asserted-by":"publisher","DOI":"10.1145\/3340555.3355720"},{"key":"ref48","first-page":"1","article-title":"Multimodal deep learning","author":"ngiam","year":"2011","journal-title":"Proc ICML"},{"key":"ref47","doi-asserted-by":"publisher","DOI":"10.3115\/v1\/N15-1016"},{"key":"ref42","doi-asserted-by":"publisher","DOI":"10.1109\/TPAMI.2018.2798607"},{"key":"ref41","article-title":"Learning factorized multimodal representations","author":"tsai","year":"2018","journal-title":"arXiv 1806 06176"},{"key":"ref44","doi-asserted-by":"publisher","DOI":"10.1016\/j.imavis.2017.08.003"},{"key":"ref43","doi-asserted-by":"publisher","DOI":"10.1109\/ACCESS.2019.2916887"}],"container-title":["IEEE Access"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx7\/6287639\/9312710\/09466122.pdf?arnumber=9466122","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2022,1,27]],"date-time":"2022-01-27T21:30:59Z","timestamp":1643319059000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/9466122\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2021]]},"references-count":84,"URL":"https:\/\/doi.org\/10.1109\/access.2021.3092735","relation":{},"ISSN":["2169-3536"],"issn-type":[{"value":"2169-3536","type":"electronic"}],"subject":[],"published":{"date-parts":[[2021]]}}}