{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,13]],"date-time":"2026-04-13T19:04:19Z","timestamp":1776107059966,"version":"3.50.1"},"publisher-location":"New York, NY, USA","reference-count":36,"publisher":"ACM","license":[{"start":{"date-parts":[[2019,10,14]],"date-time":"2019-10-14T00:00:00Z","timestamp":1571011200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2019,10,14]]},"DOI":"10.1145\/3340555.3353750","type":"proceedings-article","created":{"date-parts":[[2019,10,17]],"date-time":"2019-10-17T12:49:48Z","timestamp":1571316588000},"page":"95-104","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":19,"title":["Multimodal Learning for Identifying Opportunities for Empathetic Responses"],"prefix":"10.1145","author":[{"given":"Leili","family":"Tavabi","sequence":"first","affiliation":[{"name":"University of Southern California, Institute for Creative Technologies, Los Angeles, CA, USA"}]},{"given":"Kalin","family":"Stefanov","sequence":"additional","affiliation":[{"name":"University of Southern California, Institute for Creative Technologies, Los Angeles, CA, USA"}]},{"given":"Setareh","family":"Nasihati Gilani","sequence":"additional","affiliation":[{"name":"University of Southern California, Institute for Creative Technologies, Los Angeles, CA, USA"}]},{"given":"David","family":"Traum","sequence":"additional","affiliation":[{"name":"University of Southern California, Institute for Creative Technologies, Los Angeles, CA, USA"}]},{"given":"Mohammad","family":"Soleymani","sequence":"additional","affiliation":[{"name":"University of Southern California, Institute for Creative Technologies, Los Angeles, CA, USA"}]}],"member":"320","published-online":{"date-parts":[[2019,10,14]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.csl.2017.12.003"},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.1109\/WACV.2016.7477553"},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1109\/FG.2018.00019"},{"key":"e_1_3_2_1_4_1","volume-title":"International Conference on Affective Computing and Intelligent Interaction. Springer, 466\u2013473","author":"Becker C.","unstructured":"C. Becker , H. Prendinger , M. Ishizuka , and I. Wachsmuth . 2005. Evaluating affective feedback of the 3D agent max in a competitive cards game . In International Conference on Affective Computing and Intelligent Interaction. Springer, 466\u2013473 . C. Becker, H. Prendinger, M. Ishizuka, and I. Wachsmuth. 2005. Evaluating affective feedback of the 3D agent max in a competitive cards game. In International Conference on Affective Computing and Intelligent Interaction. Springer, 466\u2013473."},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"crossref","unstructured":"S. Brave C. Nass and K. Hutchinson. 2005. Computers that care: investigating the effects of orientation of emotion exhibited by an embodied computer agent. International journal of human-computer studies 62 2 (2005) 161\u2013178.  S. Brave C. Nass and K. Hutchinson. 2005. Computers that care: investigating the effects of orientation of emotion exhibited by an embodied computer agent. International journal of human-computer studies 62 2 (2005) 161\u2013178.","DOI":"10.1016\/j.ijhcs.2004.11.002"},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"crossref","unstructured":"E. Cambria I. Hupont A. Hussain E. Cerezo and S. Baldassarri. 2011. Sentic avatar: Multimodal affective conversational agent with common sense. In Toward Autonomous Adaptive and Context-Aware Multimodal Interfaces. Theoretical and Practical Issues. Springer 81\u201395.  E. Cambria I. Hupont A. Hussain E. Cerezo and S. Baldassarri. 2011. Sentic avatar: Multimodal affective conversational agent with common sense. In Toward Autonomous Adaptive and Context-Aware Multimodal Interfaces. Theoretical and Practical Issues. Springer 81\u201395.","DOI":"10.1007\/978-3-642-18184-9_8"},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"crossref","unstructured":"E. Cambria B. Schuller Y. Xia and C. Havasi. 2013. New avenues in opinion mining and sentiment analysis. IEEE Intelligent systems 28 2 (2013) 15\u201321.  E. Cambria B. Schuller Y. Xia and C. Havasi. 2013. New avenues in opinion mining and sentiment analysis. IEEE Intelligent systems 28 2 (2013) 15\u201321.","DOI":"10.1109\/MIS.2013.30"},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1145\/3136755.3136801"},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.1109\/TAFFC.2015.2444846"},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.1561\/2300000056"},{"key":"e_1_3_2_1_12_1","volume-title":"2009 IEEE conference on computer vision and pattern recognition. IEEE, 248\u2013255","author":"Deng J.","unstructured":"J. Deng , W. Dong , R. Socher , L.-J. Li , K. Li , and L. Fei-Fei . 2009. Imagenet: A large-scale hierarchical image database . In 2009 IEEE conference on computer vision and pattern recognition. IEEE, 248\u2013255 . J. Deng, W. Dong, R. Socher, L.-J. Li, K. Li, and L. Fei-Fei. 2009. Imagenet: A large-scale hierarchical image database. In 2009 IEEE conference on computer vision and pattern recognition. IEEE, 248\u2013255."},{"key":"e_1_3_2_1_13_1","volume-title":"Proceedings of the 2014 international conference on Autonomous agents and multi-agent systems. International Foundation for Autonomous Agents and Multiagent Systems, 1061\u20131068","author":"DeVault D.","year":"2014","unstructured":"D. DeVault , R. Artstein , G. Benn , T. Dey , E. Fast , A. Gainer , K. Georgila , J. Gratch , A. Hartholt , M. Lhommet , 2014 . SimSensei Kiosk: A virtual human interviewer for healthcare decision support . In Proceedings of the 2014 international conference on Autonomous agents and multi-agent systems. International Foundation for Autonomous Agents and Multiagent Systems, 1061\u20131068 . D. DeVault, R. Artstein, G. Benn, T. Dey, E. Fast, A. Gainer, K. Georgila, J. Gratch, A. Hartholt, M. Lhommet, 2014. SimSensei Kiosk: A virtual human interviewer for healthcare decision support. In Proceedings of the 2014 international conference on Autonomous agents and multi-agent systems. International Foundation for Autonomous Agents and Multiagent Systems, 1061\u20131068."},{"key":"e_1_3_2_1_14_1","volume-title":"Bert: Pre-training of deep bidirectional transformers for language understanding. arXiv preprint arXiv:1810.04805(2018).","author":"Devlin J.","year":"2018","unstructured":"J. Devlin , M.-W. Chang , K. Lee , and K. Toutanova . 2018 . Bert: Pre-training of deep bidirectional transformers for language understanding. arXiv preprint arXiv:1810.04805(2018). J. Devlin, M.-W. Chang, K. Lee, and K. Toutanova. 2018. Bert: Pre-training of deep bidirectional transformers for language understanding. arXiv preprint arXiv:1810.04805(2018)."},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"crossref","unstructured":"P. Ekman and W. Friesen. 1978. The Facial Action Coding System (FACS). Consulting Psychologists Press Stanford University Palo Alto.  P. Ekman and W. Friesen. 1978. The Facial Action Coding System (FACS). Consulting Psychologists Press Stanford University Palo Alto.","DOI":"10.1037\/t27734-000"},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.1109\/TAFFC.2015.2457417"},{"key":"e_1_3_2_1_17_1","volume-title":"Proceedings of the 21st ACM international conference on Multimedia - MM \u201913","author":"Eyben F.","unstructured":"F. Eyben , F. Weninger , F. Gross , and B. Schuller . 2013. Recent developments in openSMILE, the munich open-source multimedia feature extractor . In Proceedings of the 21st ACM international conference on Multimedia - MM \u201913 . ACM Press, New York, New York, USA, 835\u2013838. F. Eyben, F. Weninger, F. Gross, and B. Schuller. 2013. Recent developments in openSMILE, the munich open-source multimedia feature extractor. In Proceedings of the 21st ACM international conference on Multimedia - MM \u201913. ACM Press, New York, New York, USA, 835\u2013838."},{"key":"e_1_3_2_1_18_1","volume-title":"Proceedings of the 18th ACM International Conference on Multimedia(MM \u201910)","author":"Eyben F.","unstructured":"F. Eyben , M. W\u00f6llmer , and B. Schuller . 2010. OpenSMILE: The Munich Versatile and Fast Open-source Audio Feature Extractor . In Proceedings of the 18th ACM International Conference on Multimedia(MM \u201910) . ACM, New York, NY, USA, 1459\u20131462. F. Eyben, M. W\u00f6llmer, and B. Schuller. 2010. OpenSMILE: The Munich Versatile and Fast Open-source Audio Feature Extractor. In Proceedings of the 18th ACM International Conference on Multimedia(MM \u201910). ACM, New York, NY, USA, 1459\u20131462."},{"key":"e_1_3_2_1_19_1","unstructured":"J. Gratch R. Artstein G.\u00a0M. Lucas G. Stratou S. Scherer A. Nazarian R. Wood J. Boberg D. DeVault S. Marsella 2014. The distress analysis interview corpus of human and computer interviews.. In LREC. Citeseer 3123\u20133128.  J. Gratch R. Artstein G.\u00a0M. Lucas G. Stratou S. Scherer A. Nazarian R. Wood J. Boberg D. DeVault S. Marsella 2014. The distress analysis interview corpus of human and computer interviews.. In LREC. Citeseer 3123\u20133128."},{"key":"e_1_3_2_1_20_1","volume-title":"Proceedings of the 2018 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies","volume":"1","author":"Hazarika D.","unstructured":"D. Hazarika , S. Poria , A. Zadeh , E. Cambria , L.-P. Morency , and R. Zimmermann . 2018. Conversational memory network for emotion recognition in dyadic dialogue videos . In Proceedings of the 2018 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies , Volume 1 (Long Papers). 2122\u20132132. D. Hazarika, S. Poria, A. Zadeh, E. Cambria, L.-P. Morency, and R. Zimmermann. 2018. Conversational memory network for emotion recognition in dyadic dialogue videos. In Proceedings of the 2018 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, Volume 1 (Long Papers). 2122\u20132132."},{"key":"e_1_3_2_1_21_1","volume-title":"Proceedings of the IEEE conference on computer vision and pattern recognition. 770\u2013778","author":"He K.","unstructured":"K. He , X. Zhang , S. Ren , and J. Sun . 2016. Deep residual learning for image recognition . In Proceedings of the IEEE conference on computer vision and pattern recognition. 770\u2013778 . K. He, X. Zhang, S. Ren, and J. Sun. 2016. Deep residual learning for image recognition. In Proceedings of the IEEE conference on computer vision and pattern recognition. 770\u2013778."},{"key":"e_1_3_2_1_22_1","volume-title":"Eighth international AAAI conference on weblogs and social media.","author":"Hutto J.","unstructured":"C.\u00a0 J. Hutto and E. Gilbert . 2014. Vader: A parsimonious rule-based model for sentiment analysis of social media text . In Eighth international AAAI conference on weblogs and social media. C.\u00a0J. Hutto and E. Gilbert. 2014. Vader: A parsimonious rule-based model for sentiment analysis of social media text. In Eighth international AAAI conference on weblogs and social media."},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"crossref","unstructured":"I. Leite A. Pereira S. Mascarenhas C. Martinho R. Prada and A. Paiva. 2013. The influence of empathy in human\u2013robot relations. International journal of human-computer studies 71 3 (2013) 250\u2013260.  I. Leite A. Pereira S. Mascarenhas C. Martinho R. Prada and A. Paiva. 2013. The influence of empathy in human\u2013robot relations. International journal of human-computer studies 71 3 (2013) 250\u2013260.","DOI":"10.1016\/j.ijhcs.2012.09.005"},{"key":"e_1_3_2_1_24_1","volume-title":"Dialoguernn: An attentive rnn for emotion detection in conversations. arXiv preprint arXiv:1811.00405(2018).","author":"Majumder N.","year":"2018","unstructured":"N. Majumder , S. Poria , D. Hazarika , R. Mihalcea , A. Gelbukh , and E. Cambria . 2018 . Dialoguernn: An attentive rnn for emotion detection in conversations. arXiv preprint arXiv:1811.00405(2018). N. Majumder, S. Poria, D. Hazarika, R. Mihalcea, A. Gelbukh, and E. Cambria. 2018. Dialoguernn: An attentive rnn for emotion detection in conversations. arXiv preprint arXiv:1811.00405(2018)."},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.imavis.2012.08.018"},{"key":"e_1_3_2_1_26_1","first-page":"2001","article-title":"Linguistic inquiry and word count: LIWC 2001. Mahway","volume":"71","author":"Pennebaker W.","year":"2001","unstructured":"J.\u00a0 W. Pennebaker , M.\u00a0 E. Francis , and R.\u00a0 J. Booth . 2001 . Linguistic inquiry and word count: LIWC 2001. Mahway : Lawrence Erlbaum Associates 71 , 2001 (2001), 2001 . J.\u00a0W. Pennebaker, M.\u00a0E. Francis, and R.\u00a0J. Booth. 2001. Linguistic inquiry and word count: LIWC 2001. Mahway: Lawrence Erlbaum Associates 71, 2001 (2001), 2001.","journal-title":"Lawrence Erlbaum Associates"},{"key":"e_1_3_2_1_27_1","volume-title":"Proceedings of the 51st Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers), Vol.\u00a01. 973\u2013982","author":"P\u00e9rez-Rosas V.","year":"2013","unstructured":"V. P\u00e9rez-Rosas , R. Mihalcea , and L.-P. Morency . 2013 . Utterance-level multimodal sentiment analysis . In Proceedings of the 51st Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers), Vol.\u00a01. 973\u2013982 . V. P\u00e9rez-Rosas, R. Mihalcea, and L.-P. Morency. 2013. Utterance-level multimodal sentiment analysis. In Proceedings of the 51st Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers), Vol.\u00a01. 973\u2013982."},{"key":"e_1_3_2_1_28_1","volume-title":"Proceedings of the 2015 conference on empirical methods in natural language processing. 2539\u20132544","author":"Poria S.","unstructured":"S. Poria , E. Cambria , and A. Gelbukh . 2015. Deep convolutional neural network textual features and multiple kernel learning for utterance-level multimodal sentiment analysis . In Proceedings of the 2015 conference on empirical methods in natural language processing. 2539\u20132544 . S. Poria, E. Cambria, and A. Gelbukh. 2015. Deep convolutional neural network textual features and multiple kernel learning for utterance-level multimodal sentiment analysis. In Proceedings of the 2015 conference on empirical methods in natural language processing. 2539\u20132544."},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.neucom.2015.01.095"},{"key":"e_1_3_2_1_30_1","volume-title":"2016 IEEE 16th international conference on data mining (ICDM). IEEE, 439\u2013448","author":"Poria S.","unstructured":"S. Poria , I. Chaturvedi , E. Cambria , and A. Hussain . 2016. Convolutional MKL based multimodal emotion recognition and sentiment analysis . In 2016 IEEE 16th international conference on data mining (ICDM). IEEE, 439\u2013448 . S. Poria, I. Chaturvedi, E. Cambria, and A. Hussain. 2016. Convolutional MKL based multimodal emotion recognition and sentiment analysis. In 2016 IEEE 16th international conference on data mining (ICDM). IEEE, 439\u2013448."},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"publisher","DOI":"10.1145\/3173574.3174127"},{"key":"e_1_3_2_1_32_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.imavis.2017.08.003"},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"publisher","DOI":"10.21437\/Interspeech.2010-646"},{"key":"e_1_3_2_1_34_1","volume-title":"Proceedings of The 2012 Asia Pacific Signal and Information Processing Association Annual Summit and Conference. IEEE, 1\u20134.","author":"Xiao B.","year":"2012","unstructured":"B. Xiao , D. Can , P.\u00a0 G. Georgiou , D. Atkins , and S.\u00a0 S. Narayanan . 2012 . Analyzing the language of therapist empathy in motivational interview based psychotherapy . In Proceedings of The 2012 Asia Pacific Signal and Information Processing Association Annual Summit and Conference. IEEE, 1\u20134. B. Xiao, D. Can, P.\u00a0G. Georgiou, D. Atkins, and S.\u00a0S. Narayanan. 2012. Analyzing the language of therapist empathy in motivational interview based psychotherapy. In Proceedings of The 2012 Asia Pacific Signal and Information Processing Association Annual Summit and Conference. IEEE, 1\u20134."},{"key":"e_1_3_2_1_35_1","first-page":"e0143055","volume":"10","author":"Xiao B.","year":"2015","unstructured":"B. Xiao , Z.\u00a0 E. Imel , P.\u00a0 G. Georgiou , D.\u00a0 C. Atkins , and S.\u00a0 S. Narayanan . 2015 . \u201d Rate My Therapist\u201d: Automated Detection of Empathy in Drug and Alcohol Counseling via Speech and Language Processing. PloS one 10 , 12 (2015), e0143055 . B. Xiao, Z.\u00a0E. Imel, P.\u00a0G. Georgiou, D.\u00a0C. Atkins, and S.\u00a0S. Narayanan. 2015. \u201d Rate My Therapist\u201d: Automated Detection of Empathy in Drug and Alcohol Counseling via Speech and Language Processing. PloS one 10, 12 (2015), e0143055.","journal-title":"Automated Detection of Empathy in Drug and Alcohol Counseling via Speech and Language Processing. PloS one"},{"key":"e_1_3_2_1_36_1","unstructured":"H. Xiao. 2018. bert-as-service. https:\/\/github.com\/hanxiao\/bert-as-service.  H. Xiao. 2018. bert-as-service. https:\/\/github.com\/hanxiao\/bert-as-service."},{"key":"e_1_3_2_1_37_1","doi-asserted-by":"crossref","unstructured":"A. Zadeh M. Chen S. Poria E. Cambria and L.-P. Morency. 2017. Tensor fusion network for multimodal sentiment analysis. arXiv preprint arXiv:1707.07250(2017).  A. Zadeh M. Chen S. Poria E. Cambria and L.-P. Morency. 2017. Tensor fusion network for multimodal sentiment analysis. arXiv preprint arXiv:1707.07250(2017).","DOI":"10.18653\/v1\/D17-1115"}],"event":{"name":"ICMI '19: INTERNATIONAL CONFERENCE ON MULTIMODAL INTERACTION","location":"Suzhou China","acronym":"ICMI '19"},"container-title":["2019 International Conference on Multimodal Interaction"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3340555.3353750","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3340555.3353750","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T23:13:28Z","timestamp":1750202008000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3340555.3353750"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2019,10,14]]},"references-count":36,"alternative-id":["10.1145\/3340555.3353750","10.1145\/3340555"],"URL":"https:\/\/doi.org\/10.1145\/3340555.3353750","relation":{},"subject":[],"published":{"date-parts":[[2019,10,14]]},"assertion":[{"value":"2019-10-14","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}