{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,11,6]],"date-time":"2026-11-06T12:05:10Z","timestamp":1793966710160,"version":"4.8.0"},"reference-count":38,"publisher":"Institute of Electronics, Information and Communications Engineers (IEICE)","issue":"3","content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["IEICE Trans. Inf. & Syst."],"published-print":{"date-parts":[[2024,3,1]]},"DOI":"10.1587\/transinf.2023hcp0006","type":"journal-article","created":{"date-parts":[[2024,2,29]],"date-time":"2024-02-29T22:25:15Z","timestamp":1709245515000},"page":"342-353","source":"Crossref","is-referenced-by-count":5,"title":["An Intra- and Inter-Emotion Transformer-Based Fusion Model with Homogeneous and Diverse Constraints Using Multi-Emotional Audiovisual Features for Depression Detection"],"prefix":"10.1587","volume":"E107.D","author":[{"given":"Shiyu","family":"TENG","sequence":"first","affiliation":[{"name":"College of Information Science and Engineering, Ritsumeikan University"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Jiaqing","family":"LIU","sequence":"additional","affiliation":[{"name":"College of Information Science and Engineering, Ritsumeikan University"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Yue","family":"HUANG","sequence":"additional","affiliation":[{"name":"Affiliated Kangning Hospital of Wenzhou Medical University, Zhejiang Provincial Clinical Research Center for Mental Disorder"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Shurong","family":"CHAI","sequence":"additional","affiliation":[{"name":"College of Information Science and Engineering, Ritsumeikan University"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Tomoko","family":"TATEYAMA","sequence":"additional","affiliation":[{"name":"Department of Intelligent Information Engineering, Fujita Health University"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Xinyin","family":"HUANG","sequence":"additional","affiliation":[{"name":"School of Education, Soochow University"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Lanfen","family":"LIN","sequence":"additional","affiliation":[{"name":"College of Computer Science and Technology, Zhejiang University"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Yen-Wei","family":"CHEN","sequence":"additional","affiliation":[{"name":"College of Information Science and Engineering, Ritsumeikan University"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"532","reference":[{"key":"1","doi-asserted-by":"publisher","unstructured":"[1] D.F. Santomauro, A.M. Mantilla Herrera, J. Shadid, P. Zheng, C. Ashbaugh, D.M. Pigott, C. Abbafati, C. Adolph, J.O. Amlag, A.Y. Aravkin, B.L. Bang-Jensen, G.J. Bertolacci, S.S. Bloom, R.Castellano, E. Castro, S. Chakrabarti, J. Chattopadhyay, R.M. Cogen, J.K. Collins, X. Dai, W.J. Dangel, C. Dapper, A. Deen, M. Erickson, S.B. Ewald, A.D. Flaxman, J.J. Frostad, N. Fullman, J.R. Giles, A.Z. Giref, G. Guo, J. He, M. Helak, E.N. Hulland, B. Idrisov, A. Lindstrom, E. Linebarger, P.A. Lotufo, R. Lozano, B. Magistro, D.C. Malta, J.C. Mansson, F. Marinho, A.H. Mokdad, L. Monasta, P. Naik, S. Nomura, J.K. O'Halloran, S.M. Ostroff, M. Pasovic, L. Penberthy, R.C. Reiner Jr, G. Reinke, A.L.P. Ribeiro, A. Sholokhov, R.J.D. Sorensen, E. Varavikova, A.T. Vo, R. Walcott, S. Watson, C.S. Wiysonge, B. Zigler, S.I. Hay, T. Vos, C.J.L. Murray, H.A. Whiteford, and A.J. Ferrari, \u201cGlobal prevalence and burden of depressive and anxiety disorders in 204 countries and territories in 2020 due to the covid-19 pandemic,\u201d The Lancet, vol.398, no.10312, pp.1700-1712, 2021. 10.1016\/s0140-6736(21)02143-7","DOI":"10.1016\/S0140-6736(21)02143-7"},{"key":"2","doi-asserted-by":"crossref","unstructured":"[2] A.S. Zigmond and R.P. Snaith, \u201cThe hospital anxiety and depression scale,\u201d Acta psychiatrica scandinavica, vol.67, no.6, pp.361-370, 1983. 10.1111\/j.1600-0447.1983.tb09716.x","DOI":"10.1111\/j.1600-0447.1983.tb09716.x"},{"key":"3","doi-asserted-by":"publisher","unstructured":"[3] K. Kroenke, R.L. Spitzer, and J.B.W. Williams, \u201cThe phq-9: validity of a brief depression severity measure,\u201d Journal of general internal medicine, vol.16, no.9, pp.606-613, 2001. 10.1046\/j.1525-1497.2001.016009606.x","DOI":"10.1046\/j.1525-1497.2001.016009606.x"},{"key":"4","doi-asserted-by":"crossref","unstructured":"[4] A.T. Beck, R.A. Steer, and G. Brown, \u201cBeck depression inventory-ii,\u201d Psychological assessment, 1996. 10.1037\/t00742-000","DOI":"10.1037\/t00742-000"},{"key":"5","doi-asserted-by":"crossref","unstructured":"[5] S. Teng, S. Chai, J. Liu, T. Tomoko, X. Huang, and Y.-W. Chen, \u201cA transformer-based multimodal network for audiovisual depression prediction,\u201d 2022 IEEE 11th Global Conference on Consumer Electronics (GCCE), pp.761-764, IEEE, 2022. 10.1109\/gcce56475.2022.10014157","DOI":"10.1109\/GCCE56475.2022.10014157"},{"key":"6","doi-asserted-by":"crossref","unstructured":"[6] H. Sun, H. Wang, J. Liu, Y.-W. Chen, and L. Lin, \u201cCubemlp: An mlp-based model for multimodal sentiment analysis and depression estimation,\u201d Proc. 30th ACM International Conference on Multimedia, pp.3722-3729, 2022. 10.1145\/3503161.3548025","DOI":"10.1145\/3503161.3548025"},{"key":"7","doi-asserted-by":"publisher","unstructured":"[7] H. Sun, Y.-W. Chen, and L. Lin, \u201cTensorformer: A tensor-based multimodal transformer for multimodal sentiment analysis and depression detection,\u201d IEEE Transactions on Affective Computing, vol.14, no.4, pp.2776-2786, 2022. 10.1109\/taffc.2022.3233070","DOI":"10.1109\/TAFFC.2022.3233070"},{"key":"8","doi-asserted-by":"crossref","unstructured":"[8] J. Liu, Y. Huang, S. Chai, H. Sun, X. Huang, L. Lin, and Y.-W. Chen, \u201cComputer-aided detection of depressive severity using multimodal behavioral data,\u201d Handbook of Artificial Intelligence in Healthcare, pp.353-371, Springer, 2022. 10.1007\/978-3-030-79161-2_14","DOI":"10.1007\/978-3-030-79161-2_14"},{"key":"9","doi-asserted-by":"crossref","unstructured":"[9] H. Sun, J. Liu, S. Chai, Z. Qiu, L. Lin, X. Huang, and Y. Chen, \u201cMulti-modal adaptive fusion transformer network for the estimation of depression level,\u201d Sensors, vol.21, no.14, p.4764, 2021. 10.3390\/s21144764","DOI":"10.3390\/s21144764"},{"key":"10","doi-asserted-by":"publisher","unstructured":"[10] M. Fang, S. Peng, Y. Liang, C.-C. Hung, and S. Liu, \u201cA multimodal fusion model with multi-level attention mechanism for depression detection,\u201d Biomedical Signal Processing and Control, vol.82, p.104561, 2023. 10.1016\/j.bspc.2022.104561","DOI":"10.1016\/j.bspc.2022.104561"},{"key":"11","doi-asserted-by":"crossref","unstructured":"[11] Y. Kan, M. Mimura, K. Kamijima, and M. Kawamura, \u201cRecognition of emotion from moving facial and prosodic stimuli in depressed patients,\u201d Journal of Neurology, Neurosurgery & Psychiatry, vol.75, no.12, pp.1667-1671, 2004.","DOI":"10.1136\/jnnp.2004.036079"},{"key":"12","doi-asserted-by":"publisher","unstructured":"[12] C. Naranjo, C. Kornreich, S. Campanella, X. No\u00ebl, Y. Vandriette, B. Gillain, X. De Longueville, B. Delatte, P. Verbanck, and E. Constant, \u201cMajor depression is associated with impaired processing of emotion in music as well as in facial and vocal stimuli,\u201d Journal of affective disorders, vol.128, no.3, pp.243-251, 2011. 10.1016\/j.jad.2010.06.039","DOI":"10.1016\/j.jad.2010.06.039"},{"key":"13","doi-asserted-by":"publisher","unstructured":"[13] H. Cai, Z. Qu, Z. Li, Y. Zhang, X. Hu, and B. Hu, \u201cFeature-level fusion approaches based on multimodal eeg data for depression recognition,\u201d Information Fusion, vol.59, pp.127-138, 2020. 10.1016\/j.inffus.2020.01.008","DOI":"10.1016\/j.inffus.2020.01.008"},{"key":"14","doi-asserted-by":"crossref","unstructured":"[14] W. Wu, M. Wu, and K. Yu, \u201cClimate and weather: Inspecting depression detection via emotion recognition,\u201d ICASSP 2022-2022 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP), pp.6262-6266, IEEE, 2022. 10.1109\/icassp43922.2022.9746634","DOI":"10.1109\/ICASSP43922.2022.9746634"},{"key":"15","unstructured":"[15] A. Vaswani, N. Shazeer, N. Parmar, J. Uszkoreit, L. Jones, A.N. Gomez, \u0141. Kaiser, and I. Polosukhin, \u201cAttention is all you need,\u201d Advances in neural information processing systems, vol.30, 2017."},{"key":"16","doi-asserted-by":"publisher","unstructured":"[16] J. Joshi, R. Goecke, S. Alghowinem, A. Dhall, M. Wagner, J. Epps, G. Parker, and M. Breakspear, \u201cMultimodal assistive technologies for depression diagnosis and monitoring,\u201d Journal on Multimodal User Interfaces, vol.7, no.3, pp.217-228, 2013. 10.1007\/s12193-013-0123-2","DOI":"10.1007\/s12193-013-0123-2"},{"key":"17","doi-asserted-by":"crossref","unstructured":"[17] H. Kaya, D. Fedotov, D. Dresvyanskiy, M. Doyran, D. Mamontov, M. Markitantov, A.A. Akdag Salah, E. Kavcar, A. Karpov, and A.A. Salah, \u201cPredicting depression and emotions in the cross-roads of cultures, para-linguistics, and non-linguistics,\u201d Proc. 9th International on Audio\/Visual Emotion Challenge and Workshop, pp.27-35, 2019. 10.1145\/3347320.3357691","DOI":"10.1145\/3347320.3357691"},{"key":"18","doi-asserted-by":"crossref","unstructured":"[18] M. Rodrigues Makiuchi, T. Warnita, K. Uto, and K. Shinoda, \u201cMultimodal fusion of bert-cnn and gated cnn representations for depression detection,\u201d Proc. 9th International on Audio\/Visual Emotion Challenge and Workshop, pp.55-63, 2019. 10.1145\/3347320.3357694","DOI":"10.1145\/3347320.3357694"},{"key":"19","doi-asserted-by":"publisher","unstructured":"[19] B. Cui, J. Wang, H. Lin, Y. Zhang, L. Yang, and B. Xu, \u201cEmotion-based reinforcement attention network for depression detection on social media: Algorithm development and validation,\u201d JMIR Medical Informatics, vol.10, no.8, p.e37818, 2022. 10.2196\/37818","DOI":"10.2196\/37818"},{"key":"20","doi-asserted-by":"crossref","unstructured":"[20] K.M. Valencia-Segura, H.J. Escalante, and L. Villase\u00f1or-Pineda, \u201cLeveraging multiple characterizations of social media users for depression detection using data fusion,\u201d Pattern Recognition, ed. O.O. Vergara-Villegas, V.G. Cruz-S\u00e1nchez, J.H. Sossa-Azuela, J.A. Carrasco-Ochoa, J.F. Mart\u00ednez-Trinidad, and J.A. Olvera-L\u00f3pez, Cham, pp.215-224, Springer International Publishing, 2022. 10.1007\/978-3-031-07750-0_20","DOI":"10.1007\/978-3-031-07750-0_20"},{"key":"21","doi-asserted-by":"publisher","unstructured":"[21] L. Ren, H. Lin, B. Xu, S. Zhang, L. Yang, and S. Sun, \u201cDepression detection on reddit with an emotion-based attention network: algorithm development and validation,\u201d JMIR Medical Informatics, vol.9, no.7, p.e28754, 2021. 10.2196\/28754","DOI":"10.2196\/28754"},{"key":"22","doi-asserted-by":"crossref","unstructured":"[22] D. Hazarika, R. Zimmermann, and S. Poria, \u201cMisa: Modality-invariant and-specific representations for multimodal sentiment analysis,\u201d Proc. 28th ACM international conference on multimedia, pp.1122-1131, 2020. 10.1145\/3394171.3413678","DOI":"10.1145\/3394171.3413678"},{"key":"23","doi-asserted-by":"crossref","unstructured":"[23] D. Yang, S. Huang, H. Kuang, Y. Du, and L. Zhang, \u201cDisentangled representation learning for multimodal emotion recognition,\u201d Proc. 30th ACM International Conference on Multimedia, pp.1642-1651, 2022. 10.1145\/3503161.3547754","DOI":"10.1145\/3503161.3547754"},{"key":"24","doi-asserted-by":"publisher","unstructured":"[24] H. Sun, J. Liu, Y.-W. Chen, and L. Lin, \u201cModality-invariant temporal representation learning for multimodal sentiment classification,\u201d Information Fusion, vol.91, pp.504-514, 2023. 10.1016\/j.inffus.2022.10.031","DOI":"10.1016\/j.inffus.2022.10.031"},{"key":"25","doi-asserted-by":"crossref","unstructured":"[25] F. Ringeval, B. Schuller, M. Valstar, N. Cummins, R. Cowie, L. Tavabi, M. Schmitt, S. Alisamir, S. Amiriparian, E.-M. Messner, S. Song, S. Liu, Z. Zhao, A. Mallol-Ragolta, Z. Ren, M. Soleymani, and M. Pantic, \u201cAvec 2019 workshop and challenge: state-of-mind, detecting depression with ai, and cross-cultural affect recognition,\u201d Proc. 9th International on Audio\/visual Emotion Challenge and Workshop, pp.3-12, 2019. 10.1145\/3347320.3357688","DOI":"10.1145\/3347320.3357688"},{"key":"26","doi-asserted-by":"crossref","unstructured":"[26] F. Eyben, M. W\u00f6llmer, and B. Schuller, \u201cOpensmile: the munich versatile and fast open-source audio feature extractor,\u201d Proc. 18th ACM international conference on Multimedia, pp.1459-1462, 2010. 10.1145\/1873951.1874246","DOI":"10.1145\/1873951.1874246"},{"key":"27","doi-asserted-by":"crossref","unstructured":"[27] F. Eyben, K.R. Scherer, B.W. Schuller, J. Sundberg, E. Andr\u00e9, C. Busso, L.Y. Devillers, J. Epps, P. Laukka, S.S. Narayanan, and K.P. Truong, \u201cThe geneva minimalistic acoustic parameter set (gemaps) for voice research and affective computing,\u201d IEEE transactions on affective computing, vol.7, no.2, pp.190-202, 2015. 10.1109\/taffc.2015.2457417","DOI":"10.1109\/TAFFC.2015.2457417"},{"key":"28","unstructured":"[28] M. Schmitt and B. Schuller, \u201cOpenxbow: introducing the passau open-source crossmodal bag-of-words toolkit,\u201d Journal of Machine Learning Research, vol.18, no.96, pp.1-5, 2017."},{"key":"29","doi-asserted-by":"crossref","unstructured":"[29] J. Deng, W. Dong, R. Socher, L.-J. Li, K. Li, and L. Fei-Fei, \u201cImagenet: A large-scale hierarchical image database,\u201d 2009 IEEE conference on computer vision and pattern recognition, pp.248-255, Ieee, 2009. 10.1109\/cvpr.2009.5206848","DOI":"10.1109\/CVPR.2009.5206848"},{"key":"30","unstructured":"[30] K. Simonyan and A. Zisserman, \u201cVery deep convolutional networks for large-scale image recognition,\u201d arXiv preprint arXiv:1409.1556, 2014."},{"key":"31","doi-asserted-by":"crossref","unstructured":"[31] G. Huang, Z. Liu, L. Van Der Maaten, and K.Q. Weinberger, \u201cDensely connected convolutional networks,\u201d Proc. IEEE conference on computer vision and pattern recognition, pp.4700-4708, 2017. 10.1109\/cvpr.2017.243","DOI":"10.1109\/CVPR.2017.243"},{"key":"32","doi-asserted-by":"crossref","unstructured":"[32] T. Baltrusaitis, A. Zadeh, Y.C. Lim, and L.P. Morency, \u201cOpenface 2.0: Facial behavior analysis toolkit,\u201d 2018 13th IEEE international conference on automatic face & gesture recognition (FG 2018), pp.59-66, IEEE, 2018.","DOI":"10.1109\/FG.2018.00019"},{"key":"33","doi-asserted-by":"crossref","unstructured":"[33] K. He, X. Zhang, S. Ren, and J. Sun, \u201cDeep residual learning for image recognition,\u201d Proc. IEEE conference on computer vision and pattern recognition, pp.770-778, 2016. 10.1109\/cvpr.2016.90","DOI":"10.1109\/CVPR.2016.90"},{"key":"34","doi-asserted-by":"publisher","unstructured":"[34] D. Kollias, P. Tzirakis, M.A. Nicolaou, A. Papaioannou, G. Zhao, B. Schuller, I. Kotsia, and S. Zafeiriou, \u201cDeep affect prediction in-the-wild: Aff-wild database and challenge, deep architectures, and beyond,\u201d International Journal of Computer Vision, vol.127, no.6-7, pp.907-929, 2019. 10.1007\/s11263-019-01158-4","DOI":"10.1007\/s11263-019-01158-4"},{"key":"35","unstructured":"[35] W. Zellinger, T. Grubinger, E. Lughofer, T. Natschl\u00e4ger, and S. Saminger-Platz, \u201cCentral moment discrepancy (CMD) for domain-invariant representation learning,\u201d International Conference on Learning Representations, 2017."},{"key":"36","doi-asserted-by":"publisher","unstructured":"[36] Y.-P. Wang and C. Gorenstein, \u201cPsychometric properties of the beck depression inventory-ii: a comprehensive review,\u201d Brazilian Journal of Psychiatry, vol.35, no.4, pp.416-431, 2013. 10.1590\/1516-4446-2012-1048","DOI":"10.1590\/1516-4446-2012-1048"},{"key":"37","doi-asserted-by":"crossref","unstructured":"[37] L.I.-K. Lin, \u201cA concordance correlation coefficient to evaluate reproducibility,\u201d Biometrics, vol.45, no.1, pp.255-268, 1989. 10.2307\/2532051","DOI":"10.2307\/2532051"},{"key":"38","unstructured":"[38] L. Van der Maaten and G. Hinton, \u201cVisualizing data using t-sne.,\u201d Journal of machine learning research, vol.9, no.11, 2008."}],"container-title":["IEICE Transactions on Information and Systems"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/www.jstage.jst.go.jp\/article\/transinf\/E107.D\/3\/E107.D_2023HCP0006\/_pdf","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,3,2]],"date-time":"2024-03-02T04:24:41Z","timestamp":1709353481000},"score":1,"resource":{"primary":{"URL":"https:\/\/www.jstage.jst.go.jp\/article\/transinf\/E107.D\/3\/E107.D_2023HCP0006\/_article"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,3,1]]},"references-count":38,"journal-issue":{"issue":"3","published-print":{"date-parts":[[2024]]}},"URL":"https:\/\/doi.org\/10.1587\/transinf.2023hcp0006","relation":{},"ISSN":["0916-8532","1745-1361"],"issn-type":[{"value":"0916-8532","type":"print"},{"value":"1745-1361","type":"electronic"}],"subject":[],"published":{"date-parts":[[2024,3,1]]},"article-number":"2023HCP0006"}}