{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,1,2]],"date-time":"2026-01-02T07:51:49Z","timestamp":1767340309239,"version":"3.40.3"},"publisher-location":"Cham","reference-count":48,"publisher":"Springer Nature Switzerland","isbn-type":[{"type":"print","value":"9783031816871"},{"type":"electronic","value":"9783031816888"}],"license":[{"start":{"date-parts":[[2025,1,1]],"date-time":"2025-01-01T00:00:00Z","timestamp":1735689600000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2025,1,1]],"date-time":"2025-01-01T00:00:00Z","timestamp":1735689600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2025]]},"DOI":"10.1007\/978-3-031-81688-8_18","type":"book-chapter","created":{"date-parts":[[2025,2,25]],"date-time":"2025-02-25T07:22:55Z","timestamp":1740468175000},"page":"240-254","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":3,"title":["Intelligent Multimodal Artificial Agents that\u00a0Talk and\u00a0Express Emotions"],"prefix":"10.1007","author":[{"given":"Niyati","family":"Rawal","sequence":"first","affiliation":[]},{"given":"Rahul Singh","family":"Maharjan","sequence":"additional","affiliation":[]},{"given":"Marta","family":"Romeo","sequence":"additional","affiliation":[]},{"given":"Roberto","family":"Bigazzi","sequence":"additional","affiliation":[]},{"given":"Lorenzo","family":"Baraldi","sequence":"additional","affiliation":[]},{"given":"Rita","family":"Cucchiara","sequence":"additional","affiliation":[]},{"given":"Angelo","family":"Cangelosi","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2025,2,26]]},"reference":[{"key":"18_CR1","first-page":"23716","volume":"35","author":"JB Alayrac","year":"2022","unstructured":"Alayrac, J.B., et al.: Flamingo: a visual language model for few-shot learning. Adv. Neural. Inf. Process. Syst. 35, 23716\u201323736 (2022)","journal-title":"Adv. Neural. Inf. Process. Syst."},{"issue":"6","key":"18_CR2","doi-asserted-by":"publisher","first-page":"1229","DOI":"10.5465\/amj.2006.23478695","volume":"49","author":"PB Barger","year":"2006","unstructured":"Barger, P.B., Grandey, A.A.: Service with a smile and encounter satisfaction: emotional contagion and appraisal mechanisms. Acad. Manag. J. 49(6), 1229\u20131238 (2006)","journal-title":"Acad. Manag. J."},{"issue":"2","key":"18_CR3","doi-asserted-by":"publisher","first-page":"157","DOI":"10.1109\/72.279181","volume":"5","author":"Y Bengio","year":"1994","unstructured":"Bengio, Y., Simard, P., Frasconi, P.: Learning long-term dependencies with gradient descent is difficult. IEEE Trans. Neural Netw. 5(2), 157\u2013166 (1994)","journal-title":"IEEE Trans. Neural Netw."},{"key":"18_CR4","unstructured":"Bordes, F., et\u00a0al.: An introduction to vision-language modeling. arXiv preprint arXiv:2405.17247 (2024)"},{"key":"18_CR5","first-page":"15920","volume":"33","author":"P Buzzega","year":"2020","unstructured":"Buzzega, P., Boschini, M., Porrello, A., Abati, D., Calderara, S.: Dark experience for general continual learning: a strong, simple baseline. Adv. Neural. Inf. Process. Syst. 33, 15920\u201315930 (2020)","journal-title":"Adv. Neural. Inf. Process. Syst."},{"key":"18_CR6","doi-asserted-by":"crossref","unstructured":"Cao, Q., Shen, L., Xie, W., Parkhi, O.M., Zisserman, A.: VGGFace2: a dataset for recognising faces across pose and age. In: 2018 13th IEEE International Conference on Automatic Face and Gesture Recognition (FG 2018), pp. 67\u201374. IEEE (2018)","DOI":"10.1109\/FG.2018.00020"},{"key":"18_CR7","doi-asserted-by":"crossref","unstructured":"Castillo, J.C., Castro-Gonz\u00e1lez, \u00c1., Alonso-Mart\u00edn, F., Fern\u00e1ndez-Caballero, A., Salichs, M.\u00c1.: Emotion detection and regulation from personal assistant robot in smart environment. Personal assistants: emerging computational technologies, pp. 179\u2013195 (2018)","DOI":"10.1007\/978-3-319-62530-0_10"},{"key":"18_CR8","doi-asserted-by":"publisher","first-page":"351","DOI":"10.3758\/BRM.42.1.351","volume":"42","author":"NC Ebner","year":"2010","unstructured":"Ebner, N.C., Riediger, M., Lindenberger, U.: Faces-a database of facial expressions in young, middle-aged, and older women and men: development and validation. Behav. Res. Methods 42, 351\u2013362 (2010)","journal-title":"Behav. Res. Methods"},{"key":"18_CR9","unstructured":"Goodfellow, I.J., et\u00a0al.: Challenges in representation learning: a report on three machine learning contests. In: Neural Information Processing: 20th International Conference, ICONIP 2013, Daegu, Korea, November 3-7, 2013. Proceedings, Part III 20, pp. 117\u2013124. Springer (2013)"},{"key":"18_CR10","doi-asserted-by":"crossref","unstructured":"Gunson, N., et al.: A visually-aware conversational robot receptionist. Association for Computational Linguistics (ACL) (2022)","DOI":"10.18653\/v1\/2022.sigdial-1.61"},{"key":"18_CR11","doi-asserted-by":"crossref","unstructured":"H\u00e4ring, M., Bee, N., Andr\u00e9, E.: Creation and evaluation of emotion expression with body movement, sound and eye color for humanoid robots. In: 2011 RO-MAN, pp. 204\u2013209. IEEE (2011)","DOI":"10.1109\/ROMAN.2011.6005263"},{"key":"18_CR12","doi-asserted-by":"crossref","unstructured":"He, K., Zhang, X., Ren, S., Sun, J.: Deep residual learning for image recognition. In: Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition, pp. 770\u2013778 (2016)","DOI":"10.1109\/CVPR.2016.90"},{"key":"18_CR13","doi-asserted-by":"publisher","first-page":"20727","DOI":"10.1109\/ACCESS.2022.3149214","volume":"10","author":"J Heredia","year":"2022","unstructured":"Heredia, J., et al.: Adaptive multimodal emotion detection architecture for social robots. IEEE Access 10, 20727\u201320744 (2022)","journal-title":"IEEE Access"},{"issue":"8","key":"18_CR14","doi-asserted-by":"publisher","first-page":"1735","DOI":"10.1162\/neco.1997.9.8.1735","volume":"9","author":"S Hochreiter","year":"1997","unstructured":"Hochreiter, S., Schmidhuber, J.: Long short-term memory. Neural Comput. 9(8), 1735\u20131780 (1997)","journal-title":"Neural Comput."},{"issue":"1535","key":"18_CR15","doi-asserted-by":"publisher","first-page":"3585","DOI":"10.1098\/rstb.2009.0202","volume":"364","author":"K H\u00f6\u00f6k","year":"2009","unstructured":"H\u00f6\u00f6k, K.: Affective loop experiences: designing for interactional embodiment. Philos. Trans. R. Soc. B Biol. Sci. 364(1535), 3585\u20133595 (2009)","journal-title":"Philos. Trans. R. Soc. B Biol. Sci."},{"key":"18_CR16","unstructured":"Kwon, G., Cai, Z., Ravichandran, A., Bas, E., Bhotika, R., Soatto, S.: Masked vision and language modeling for multi-modal representation learning. arXiv preprint arXiv:2208.02131 (2022)"},{"issue":"3","key":"18_CR17","doi-asserted-by":"publisher","first-page":"250","DOI":"10.1016\/j.ijhcs.2012.09.005","volume":"71","author":"I Leite","year":"2013","unstructured":"Leite, I., Pereira, A., Mascarenhas, S., Martinho, C., Prada, R., Paiva, A.: The influence of empathy in human-robot relations. Int. J. Hum Comput Stud. 71(3), 250\u2013260 (2013)","journal-title":"Int. J. Hum Comput Stud."},{"issue":"2","key":"18_CR18","doi-asserted-by":"publisher","first-page":"125","DOI":"10.1007\/s12369-010-0071-x","volume":"3","author":"J Li","year":"2011","unstructured":"Li, J., Chignell, M.: Communication of emotion in social robots through simple head and arm movements. Int. J. Soc. Robot. 3(2), 125\u2013142 (2011)","journal-title":"Int. J. Soc. Robot."},{"issue":"3","key":"18_CR19","doi-asserted-by":"publisher","first-page":"1195","DOI":"10.1109\/TAFFC.2020.2981446","volume":"13","author":"S Li","year":"2020","unstructured":"Li, S., Deng, W.: Deep facial expression recognition: a survey. IEEE Trans. Affect. Comput. 13(3), 1195\u20131215 (2020)","journal-title":"IEEE Trans. Affect. Comput."},{"key":"18_CR20","doi-asserted-by":"crossref","unstructured":"Li, S., Deng, W., Du, J.: Reliable crowdsourcing and deep locality-preserving learning for expression recognition in the wild. In: Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition, pp. 2852\u20132861 (2017)","DOI":"10.1109\/CVPR.2017.277"},{"key":"18_CR21","unstructured":"Liu, H., Li, C., Wu, Q., Lee, Y.J.: Visual instruction tuning. In: Advances in Neural Information Processing Systems, vol. 36 (2024)"},{"key":"18_CR22","doi-asserted-by":"crossref","unstructured":"Lundqvist, D., Flykt, A., \u00d6hman, A.: Karolinska directed emotional faces. PsycTESTS Dataset 91, 630 (1998)","DOI":"10.1037\/t27732-000"},{"key":"18_CR23","doi-asserted-by":"crossref","unstructured":"Maharjan, R.S., Romeo, M., Cangelosi, A.: Faces are domains: domain incremental learning for expression recognition. In: 2023 International Joint Conference on Neural Networks (IJCNN), pp. 1\u20138. IEEE (2023)","DOI":"10.1109\/IJCNN54540.2023.10191542"},{"issue":"1","key":"18_CR24","doi-asserted-by":"publisher","first-page":"18","DOI":"10.1109\/TAFFC.2017.2740923","volume":"10","author":"A Mollahosseini","year":"2017","unstructured":"Mollahosseini, A., Hasani, B., Mahoor, M.H.: AffectNet: a database for facial expression, valence, and arousal computing in the wild. IEEE Trans. Affect. Comput. 10(1), 18\u201331 (2017)","journal-title":"IEEE Trans. Affect. Comput."},{"key":"18_CR25","unstructured":"OpenAI: ChatGPT (2023). https:\/\/openai.com\/blog\/chatgpt\/"},{"key":"18_CR26","unstructured":"van Otterdijk, M., Laeng, B., Lindblom, D.S., Torresen, J.: The effect of expressive robot behavior on users\u2019 mental effort: a pupillometry study. IEEE Trans. Cogn. Dev. Syst. 10(1), 18\u201331 (2024)"},{"issue":"1","key":"18_CR27","doi-asserted-by":"publisher","first-page":"2299075","DOI":"10.1080\/23311975.2023.2299075","volume":"11","author":"RY Pai","year":"2024","unstructured":"Pai, R.Y., Shetty, A., Dinesh, T.K., Shetty, A.D., Pillai, N.: Effectiveness of social robots as a tutoring and learning companion: a bibliometric analysis. Cogent Bus. Manag. 11(1), 2299075 (2024)","journal-title":"Cogent Bus. Manag."},{"key":"18_CR28","unstructured":"Radford, A., et\u00a0al.: Learning transferable visual models from natural language supervision. In: International Conference on Machine Learning, pp. 8748\u20138763. PMLR (2021)"},{"key":"18_CR29","unstructured":"Radford, A., Kim, J.W., Xu, T., Brockman, G., McLeavey, C., Sutskever, I.: Robust speech recognition via large-scale weak supervision. In: International Conference on Machine Learning, pp. 28492\u201328518. PMLR (2023)"},{"key":"18_CR30","unstructured":"Radford, A., Narasimhan, K., Salimans, T., Sutskever, I., et\u00a0al.: Improving language understanding by generative pre-training. OpenAI (2018)"},{"key":"18_CR31","doi-asserted-by":"publisher","first-page":"730317","DOI":"10.3389\/frobt.2021.730317","volume":"8","author":"N Rawal","year":"2022","unstructured":"Rawal, N., Koert, D., Turan, C., Kersting, K., Peters, J., Stock-Homburg, R.: ExGenNet: learning to generate robotic facial expression using facial expression recognition. Front. Rob. AI 8, 730317 (2022)","journal-title":"Front. Rob. AI"},{"issue":"7","key":"18_CR32","doi-asserted-by":"publisher","first-page":"1583","DOI":"10.1007\/s12369-022-00867-0","volume":"14","author":"N Rawal","year":"2022","unstructured":"Rawal, N., Stock-Homburg, R.M.: Facial emotion expressions in human-robot interaction: a survey. Int. J. Soc. Robot. 14(7), 1583\u20131604 (2022)","journal-title":"Int. J. Soc. Robot."},{"key":"18_CR33","doi-asserted-by":"crossref","unstructured":"Rombach, R., Blattmann, A., Lorenz, D., Esser, P., Ommer, B.: High-resolution image synthesis with latent diffusion models. In: Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition, pp. 10684\u201310695 (2022)","DOI":"10.1109\/CVPR52688.2022.01042"},{"issue":"6088","key":"18_CR34","doi-asserted-by":"publisher","first-page":"533","DOI":"10.1038\/323533a0","volume":"323","author":"DE Rumelhart","year":"1986","unstructured":"Rumelhart, D.E., Hinton, G.E., Williams, R.J.: Learning representations by back-propagating errors. Nature 323(6088), 533\u2013536 (1986)","journal-title":"Nature"},{"key":"18_CR35","first-page":"36479","volume":"35","author":"C Saharia","year":"2022","unstructured":"Saharia, C., et al.: Photorealistic text-to-image diffusion models with deep language understanding. Adv. Neural. Inf. Process. Syst. 35, 36479\u201336494 (2022)","journal-title":"Adv. Neural. Inf. Process. Syst."},{"key":"18_CR36","doi-asserted-by":"publisher","first-page":"201","DOI":"10.1007\/s12369-011-0124-9","volume":"4","author":"M Salem","year":"2012","unstructured":"Salem, M., Kopp, S., Wachsmuth, I., Rohlfing, K., Joublin, F.: Generation and evaluation of communicative robot gesture. Int. J. Soc. Robot. 4, 201\u2013217 (2012)","journal-title":"Int. J. Soc. Robot."},{"key":"18_CR37","doi-asserted-by":"crossref","unstructured":"Sawik, B., et al.: Robots for elderly care: review, multi-criteria optimization model and qualitative case study. In: Healthcare, vol.\u00a011, p. 1286. MDPI (2023)","DOI":"10.3390\/healthcare11091286"},{"key":"18_CR38","doi-asserted-by":"crossref","unstructured":"Tapaswi, M., Zhu, Y., Stiefelhagen, R., Torralba, A., Urtasun, R., Fidler, S.: MovieQA: understanding stories in movies through question-answering. In: Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition, pp. 4631\u20134640 (2016)","DOI":"10.1109\/CVPR.2016.501"},{"key":"18_CR39","unstructured":"Team, C.: Chameleon: mixed-modal early-fusion foundation models. arXiv preprint arXiv:2405.09818 (2024)"},{"key":"18_CR40","unstructured":"Touvron, H., et\u00a0al.: LLaMA: open and efficient foundation language models. arXiv preprint arXiv:2302.13971 (2023)"},{"key":"18_CR41","first-page":"200","volume":"34","author":"M Tsimpoukelli","year":"2021","unstructured":"Tsimpoukelli, M., Menick, J.L., Cabi, S., Eslami, S., Vinyals, O., Hill, F.: Multimodal few-shot learning with frozen language models. Adv. Neural. Inf. Process. Syst. 34, 200\u2013212 (2021)","journal-title":"Adv. Neural. Inf. Process. Syst."},{"key":"18_CR42","doi-asserted-by":"publisher","first-page":"555","DOI":"10.1007\/s12369-019-00524-z","volume":"11","author":"C Tsiourti","year":"2019","unstructured":"Tsiourti, C., Weiss, A., Wac, K., Vincze, M.: Multimodal integration of emotional signals from voice, body, and context: effects of (in) congruence on emotion recognition and attitudes towards robots. Int. J. Soc. Robot. 11, 555\u2013573 (2019)","journal-title":"Int. J. Soc. Robot."},{"key":"18_CR43","unstructured":"Vaswani, A., et al.: Attention is all you need. In: Advances in Neural Information Processing Systems, vol. 30 (2017)"},{"key":"18_CR44","doi-asserted-by":"crossref","unstructured":"Xu, R., Xiong, C., Chen, W., Corso, J.: Jointly modeling deep video and compositional text to bridge vision and language in a unified framework. In: Proceedings of the AAAI Conference on Artificial Intelligence, vol.\u00a029 (2015)","DOI":"10.1609\/aaai.v29i1.9512"},{"key":"18_CR45","doi-asserted-by":"crossref","unstructured":"Yang, D., Sato, W., Liu, Q., Minato, T., Namba, S., Nishida, S.: Optimizing facial expressions of an android robot effectively: a bayesian optimization approach. In: 2022 IEEE-RAS 21st International Conference on Humanoid Robots (Humanoids), pp. 542\u2013549. IEEE (2022)","DOI":"10.1109\/Humanoids53995.2022.10000154"},{"key":"18_CR46","unstructured":"Yenamandra, S., et al.: HomeRobot: open vocab mobile manipulation (2023). https:\/\/aihabitat.org\/static\/challenge\/home_robot_ovmm_2023\/OVMM.pdf"},{"key":"18_CR47","doi-asserted-by":"crossref","unstructured":"Yoon, Y., Ko, W.R., Jang, M., Lee, J., Kim, J., Lee, G.: Robots learn social skills: end-to-end learning of co-speech gesture generation for humanoid robots. In: 2019 International Conference on Robotics and Automation (ICRA), pp. 4303\u20134309. IEEE (2019)","DOI":"10.1109\/ICRA.2019.8793720"},{"key":"18_CR48","unstructured":"Yu, J., Wang, Z., Vasudevan, V., Yeung, L., Seyedhosseini, M., Wu, Y.: CoCa: contrastive captioners are image-text foundation models. arXiv preprint arXiv:2205.01917 (2022)"}],"container-title":["Springer Proceedings in Advanced Robotics","Human-Friendly Robotics 2024"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-031-81688-8_18","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,2,25]],"date-time":"2025-02-25T07:23:14Z","timestamp":1740468194000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-3-031-81688-8_18"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025]]},"ISBN":["9783031816871","9783031816888"],"references-count":48,"URL":"https:\/\/doi.org\/10.1007\/978-3-031-81688-8_18","relation":{},"ISSN":["2511-1256","2511-1264"],"issn-type":[{"type":"print","value":"2511-1256"},{"type":"electronic","value":"2511-1264"}],"subject":[],"published":{"date-parts":[[2025]]},"assertion":[{"value":"26 February 2025","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"HFR","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"International Workshop on Human-Friendly Robotics","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Lugano","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Switzerland","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2024","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"30 September 2024","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"1 October 2024","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"17","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"hfr2024","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"https:\/\/sites.google.com\/view\/hfr2024\/home","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}}]}}