{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,5,12]],"date-time":"2026-05-12T21:58:43Z","timestamp":1778623123593,"version":"3.51.4"},"reference-count":54,"publisher":"Springer Science and Business Media LLC","issue":"5","license":[{"start":{"date-parts":[[2019,9,28]],"date-time":"2019-09-28T00:00:00Z","timestamp":1569628800000},"content-version":"tdm","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"},{"start":{"date-parts":[[2019,9,28]],"date-time":"2019-09-28T00:00:00Z","timestamp":1569628800000},"content-version":"vor","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Int J of Soc Robotics"],"published-print":{"date-parts":[[2019,12]]},"DOI":"10.1007\/s12369-019-00591-2","type":"journal-article","created":{"date-parts":[[2019,9,28]],"date-time":"2019-09-28T18:30:10Z","timestamp":1569695410000},"page":"815-828","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":43,"title":["On-the-Fly Detection of User Engagement Decrease in Spontaneous Human\u2013Robot Interaction Using Recurrent and Deep Neural Networks"],"prefix":"10.1007","volume":"11","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-8396-001X","authenticated-orcid":false,"given":"Atef","family":"Ben-Youssef","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Giovanna","family":"Varni","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Slim","family":"Essid","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Chlo\u00e9","family":"Clavel","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2019,9,28]]},"reference":[{"key":"591_CR1","doi-asserted-by":"publisher","first-page":"293","DOI":"10.1007\/978-3-319-70022-9_29","volume-title":"Social Robotics","author":"Sean Andrist","year":"2017","unstructured":"Andrist S, Bohus D, Kamar E, Horvitz E (2017) What went wrong and why? Diagnosing situated interaction failures in the wild. In: 9th international conference on social robotics (ICSR), Tsukuba, Japan"},{"key":"591_CR2","unstructured":"Anzalone SM, Varni G, Zibetti E, Ivaldi S, Chetouani M (2015) Automated prediction of extraversion during human\u2013robot interaction. In: Finzi A, Alberto and Mastrogiovanni, Fulvio and Orlandini, Andrea and Sgorbissa (ed) AIRO@AI*IA, vol 1544, pp 29\u201339"},{"key":"591_CR3","doi-asserted-by":"crossref","unstructured":"Baltrusaitis T, Mahmoud M, Robinson P (2015) Cross-dataset learning and person-specific normalisation for automatic Action Unit detection. In: 2015 11th IEEE international conference and workshops on automatic face and gesture recognition (FG). IEEE, pp 1\u20136","DOI":"10.1109\/FG.2015.7284869"},{"key":"591_CR4","doi-asserted-by":"crossref","unstructured":"Baltrusaitis T, Zadeh A, Lim YC, Morency LP (2018) OpenFace 2.0: facial behavior analysis toolkit. In: 2018 13th IEEE international conference on automatic face and gesture recognition (FG 2018). IEEE, pp 59\u201366","DOI":"10.1109\/FG.2018.00019"},{"key":"591_CR5","doi-asserted-by":"publisher","DOI":"10.1109\/TAFFC.2019.2898399","author":"A Ben-Youssef","year":"2019","unstructured":"Ben-Youssef A, Clavel C, Essid S (2019) Early detection of user engagement breakdown in spontaneous human\u2013humanoid interaction. IEEE Trans Affect Comput. \nhttps:\/\/doi.org\/10.1109\/TAFFC.2019.2898399","journal-title":"IEEE Trans Affect Comput"},{"key":"591_CR6","doi-asserted-by":"crossref","unstructured":"Ben-Youssef A, Clavel C, Essid S, Bilac M, Chamoux M, Lim A (2017) UE-HRI: a new dataset for the study of user engagement in spontaneous human\u2013robot interactions. In: Proceedings of the 19th ACM international conference on multimodal interaction, ICMI 2017. ACM, New York, pp 464\u2013472","DOI":"10.1145\/3136755.3136814"},{"issue":"1","key":"591_CR7","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1561\/2200000006","volume":"2","author":"Y Bengio","year":"2009","unstructured":"Bengio Y (2009) Learning deep architectures for AI. Found Trends Mach Learn 2(1):1\u2013127","journal-title":"Found Trends Mach Learn"},{"key":"591_CR8","doi-asserted-by":"crossref","unstructured":"Bohus D, Horvitz E (2009) Learning to predict engagement with a spoken dialog system in open-world settings. In: Proceedings of the SIGDIAL 2009 conference on the 10th annual meeting of the special interest group on discourse and dialogue\u2014SIGDIAL \u201909, September, pp 244\u2013252","DOI":"10.3115\/1708376.1708411"},{"key":"591_CR9","doi-asserted-by":"crossref","unstructured":"Bohus D, Horvitz E (2009) Models for multiparty engagement in open-world dialog. In: Proceedings of the SIGDIAL 2009 conference: the 10th annual meeting of the special interest group on discourse and dialogue, SIGDIAL \u201909. Association for Computational Linguistics, Stroudsburg, pp 225\u2013234","DOI":"10.3115\/1708376.1708409"},{"key":"591_CR10","unstructured":"Bohus D, Horvitz E (2009) Open-world dialog: challenges, directions, and a prototype. In: Proceedings of the IJCAI\u20192009 workshop on knowledge and reasoning in practical dialogue systems, Pasadena, California, USA, pp 34\u201345"},{"key":"591_CR11","unstructured":"Bohus D, Horvitz E (2014) Managing human\u2013robot engagement with forecasts and...um...hesitations. In: Proceedings of the 16th international conference on multimodal interaction\u2014ICMI \u201914. ACM Press, New York, pp 2\u20139"},{"issue":"1","key":"591_CR12","doi-asserted-by":"publisher","first-page":"181","DOI":"10.1007\/s40593-015-0069-5","volume":"27","author":"N Bosch","year":"2015","unstructured":"Bosch N, D\u2019Mello S (2015) The affective experience of novice computer programmers. Int J Artif Intell Educ 27(1):181\u2013206","journal-title":"Int J Artif Intell Educ"},{"issue":"7","key":"591_CR13","doi-asserted-by":"publisher","first-page":"1145","DOI":"10.1016\/S0031-3203(96)00142-2","volume":"30","author":"AP Bradley","year":"1997","unstructured":"Bradley AP (1997) The use of the area under the ROC curve in the evaluation of machine learning algorithms. Pattern Recognit 30(7):1145\u20131159","journal-title":"Pattern Recognit"},{"key":"591_CR14","doi-asserted-by":"crossref","unstructured":"Castellano G, Leite I, Pereira A, Martinho C, Paiva A, McOwan PW (2012) Detecting engagement in HRI: an exploration of social and task-based context. In: 2012 international conference on privacy, security, risk and trust and 2012 international conference on social computing. IEEE, pp 421\u2013428","DOI":"10.1109\/SocialCom-PASSAT.2012.51"},{"key":"591_CR15","unstructured":"Celiktutan O, Skordos E, Gunes H (2017) Multimodal human\u2013human\u2013robot interactions (MHHRI) dataset for studying personality and engagement. IEEE Trans Affect Comput"},{"key":"591_CR16","doi-asserted-by":"crossref","unstructured":"Cho K, van Merrienboer B, Gulcehre C, Bahdanau D, Bougares F, Schwenk H, Bengio Y (2014) Learning phrase representations using RNN encoder\u2013decoder for statistical machine translation","DOI":"10.3115\/v1\/D14-1179"},{"key":"591_CR17","unstructured":"Chollet F (2015) keras. \nhttps:\/\/github.com\/fchollet\/keras\n\n. Accessed 05 Feb 2018"},{"key":"591_CR18","first-page":"93","volume-title":"Fostering user engagement in face-to-face human\u2013agent interactions: a survey","author":"C Clavel","year":"2016","unstructured":"Clavel C, Cafaro A, Campano S, Pelachaud C (2016) Fostering user engagement in face-to-face human\u2013agent interactions: a survey. Springer, Cham, pp 93\u2013120"},{"key":"591_CR19","first-page":"29","volume-title":"Engagement perception and generation for social robots and virtual agents","author":"LJ Corrigan","year":"2016","unstructured":"Corrigan LJ, Peters C, K\u00fcster D, Castellano G (2016) Engagement perception and generation for social robots and virtual agents. Springer, Cham, pp 29\u201351"},{"issue":"2","key":"591_CR20","doi-asserted-by":"publisher","first-page":"145","DOI":"10.1016\/j.learninstruc.2011.10.001","volume":"22","author":"S D\u2019Mello","year":"2012","unstructured":"D\u2019Mello S, Graesser A (2012) Dynamics of affective states during complex learning. Learn Instr 22(2):145\u2013157","journal-title":"Learn Instr"},{"key":"591_CR21","doi-asserted-by":"crossref","unstructured":"Dominey P, Metta G, Nori F, Natale L (2008) Anticipation and initiative in human-humanoid interaction. In: Humanoids 2008\u20148th IEEE-RAS international conference on humanoid robots. IEEE, pp 693\u2013699","DOI":"10.1109\/ICHR.2008.4755974"},{"key":"591_CR22","doi-asserted-by":"crossref","unstructured":"Eyben F, W\u00f6llmer M, Schuller B (2010) Opensmile: the Munich versatile and fast open-source audio feature extractor. In: Proceedings of the international conference on multimedia\u2014MM \u201910. ACM Press, New York, pp 1459\u20131462","DOI":"10.1145\/1873951.1874246"},{"key":"591_CR23","doi-asserted-by":"crossref","unstructured":"Feil-Seifer D, Mataric M (2005) Defining socially assistive robotics. In: 9th international conference on rehabilitation robotics, 2005. ICORR 2005. IEEE, pp 465\u2013468","DOI":"10.1109\/ICORR.2005.1501143"},{"key":"591_CR24","first-page":"753","volume-title":"The MuMMER project: engaging human\u2013robot interaction in real-world public spaces","author":"ME Foster","year":"2016","unstructured":"Foster ME, Alami R, Gestranius O, Lemon O, Niemel\u00e4 M, Odobez JM, Pandey AK (2016) The MuMMER project: engaging human\u2013robot interaction in real-world public spaces. Springer, Cham, pp 753\u2013763"},{"issue":"5","key":"591_CR25","doi-asserted-by":"publisher","first-page":"659","DOI":"10.1007\/s12369-017-0414-y","volume":"9","author":"ME Foster","year":"2017","unstructured":"Foster ME, Gaschler A, Giuliani M (2017) Automatically classifying user engagement for dynamic multi-party human-robot interaction. Int J Soc Robot 9(5):659\u2013674","journal-title":"Int J Soc Robot"},{"key":"591_CR26","doi-asserted-by":"crossref","unstructured":"Gehle R, Pitsch K, Dankert T, Wrede S (2017) How to open an interaction between robot and museum visitor? Strategies to establish a focused encounter in HRI. In: Proceedings of the 2017 ACM\/IEEE international conference on human\u2013robot interaction\u2014HRI \u201917. ACM Press, New York, pp 187\u2013195","DOI":"10.1145\/2909824.3020219"},{"key":"591_CR27","unstructured":"Glas N, Pelachaud C (2015) User engagement and preferences in information-giving chat with virtual agents, pp 33\u201340"},{"issue":"3","key":"591_CR28","doi-asserted-by":"publisher","first-page":"392","DOI":"10.1016\/j.robot.2013.09.012","volume":"62","author":"J Hall","year":"2014","unstructured":"Hall J, Tritton T, Rowe A, Pipe A, Melhuish C, Leonards U (2014) Perception of own and robot engagement in human\u2013robot interactions and their dependence on robotics knowledge. Robot Autonom Syst 62(3):392\u2013399","journal-title":"Robot Autonom Syst"},{"key":"591_CR29","doi-asserted-by":"crossref","unstructured":"Hayashi K, Sakamoto D, Kanda T, Shiomi M, KoizumiS, Ishiguro H, Ogasawara T, Hagita N (2007) Humanoid robots as a passive-social medium. In: Proceedings of the ACM\/IEEE international conference on human\u2013robot interaction\u2014HRI \u201907. ACM Press, New York, p 137","DOI":"10.1145\/1228716.1228735"},{"issue":"8","key":"591_CR30","doi-asserted-by":"publisher","first-page":"1735","DOI":"10.1162\/neco.1997.9.8.1735","volume":"9","author":"S Hochreiter","year":"1997","unstructured":"Hochreiter S, Schmidhuber J (1997) Long short-term memory. Neural Comput 9(8):1735\u201380","journal-title":"Neural Comput"},{"key":"591_CR31","unstructured":"Human Vision Components (HVC-P2) B5T-007001 Command Specifications. Technical report, OMRON Corporation Electronic and Mechanical Components Company, Japan (2016)"},{"issue":"1","key":"591_CR32","doi-asserted-by":"publisher","first-page":"63","DOI":"10.1007\/s12369-016-0357-8","volume":"9","author":"S Ivaldi","year":"2017","unstructured":"Ivaldi S, Lefort S, Peters J, Chetouani M, Provasi J, Zibetti E (2017) Towards engagement models that consider individual factors in HRI: on the relation of extroversion and negative attitude towards robots to gaze and speech during a human-robot assembly task. Int J Soc Robot 9(1):63\u201386","journal-title":"Int J Soc Robot"},{"issue":"1","key":"591_CR33","doi-asserted-by":"publisher","first-page":"174","DOI":"10.1109\/TASL.2008.2007613","volume":"17","author":"C Joder","year":"2009","unstructured":"Joder C, Essid S, Richard G (2009) Temporal integration for audio classification with application to musical instrument classification. IEEE Trans Audio Speech Lang Process 17(1):174\u2013186","journal-title":"IEEE Trans Audio Speech Lang Process"},{"key":"591_CR34","doi-asserted-by":"crossref","unstructured":"Kanda T, Shiomi M, Miyashita Z, Ishiguro H, Hagita N (2009) An affective guide robot in a shopping mall. In: Proceedings of the 4th ACM\/IEEE international conference on Human robot interaction\u2014HRI \u201909. ACM Press, New York, p 173","DOI":"10.1145\/1514095.1514127"},{"key":"591_CR35","doi-asserted-by":"publisher","first-page":"22","DOI":"10.1016\/0001-6918(67)90005-4","volume":"26","author":"A Kendon","year":"1967","unstructured":"Kendon A (1967) Some functions of gaze-direction in social interaction. Acta Psychol 26:22\u201363","journal-title":"Acta Psychol"},{"key":"591_CR36","doi-asserted-by":"crossref","unstructured":"Leite I, McCoy M, Ullman D, Salomons N, Scassellati B (2015) Comparing models of disengagement in individual and group interactions. In: Proceedings of the tenth annual ACM\/IEEE international conference on human\u2013robot interaction\u2014HRI \u201915. ACM Press, New York, pp 99\u2013105","DOI":"10.1145\/2696454.2696466"},{"key":"591_CR37","doi-asserted-by":"crossref","unstructured":"Li L, Xu Q, Tan YK (2012) Attention-based addressee selection for service and social robots to interact with multiple persons. In: Proceedings of the workshop at SIGGRAPH Asia, WASA \u201912. ACM, New York, pp 131\u2013136","DOI":"10.1145\/2425296.2425319"},{"key":"591_CR38","unstructured":"Liu T, Kappas A (2018) Predicting engagement breakdown in HRI using thin-slices of facial expressions. In: Workshops at the thirty-second AAAI conference on artificial intelligence, pp 37\u201343"},{"key":"591_CR39","unstructured":"Martinovski B, Traum D (2003) The error is the clue: breakdown in human\u2013machine interaction. In: Proceedings of the ISCA workshop on error handling in spoken dialogue systems, pp 11\u201317"},{"key":"591_CR40","doi-asserted-by":"crossref","unstructured":"Miller RB (1968) Response time in man-computer conversational transactions. In: Proceedings of the December 9\u201311, 1968, fall joint computer conference, part I on\u2013AFIPS \u201968 (Fall, part I). ACM Press, New York, p 267","DOI":"10.1145\/1476589.1476628"},{"key":"591_CR41","first-page":"2825","volume":"12","author":"F Pedregosa","year":"2011","unstructured":"Pedregosa F, Varoquaux G, Gramfort A, Michel V, Thirion B, Grisel O, Blondel M, Prettenhofer P, Weiss R, Dubourg V, Vanderplas J, Passos A, Cournapeau D, Brucher M, Perrot M, Duchesnay \u00c9 (2011) Scikit-learn: machine learning in Python. J Mach Learn Res 12:2825\u20132830","journal-title":"J Mach Learn Res"},{"key":"591_CR42","doi-asserted-by":"crossref","unstructured":"Pitsch K, Kuzuoka H, Suzuki Y, Sussenbach L, Luff P, Heath C (2009) \u201cThe first five seconds\u201d: contingent stepwise entry into an interaction as a means to secure sustained engagement in HRI. In: RO-MAN 2009\u2014the 18th IEEE international symposium on robot and human interactive communication. IEEE, Toyama, pp 985\u2013991","DOI":"10.1109\/ROMAN.2009.5326167"},{"key":"591_CR43","volume-title":"Mind, hands, face and body: a goal and belief view of multimodal communication","author":"I Poggi","year":"2007","unstructured":"Poggi I (2007) Mind, hands, face and body: a goal and belief view of multimodal communication. Weidler Buchverlag, Berlin"},{"issue":"11","key":"591_CR44","doi-asserted-by":"publisher","first-page":"3098","DOI":"10.1109\/TKDE.2016.2592527","volume":"28","author":"R Rawassizadeh","year":"2016","unstructured":"Rawassizadeh R, Momeni E, Dobbins C, Gharibshah J, Pazzani M (2016) Scalable daily human behavioral pattern mining from multivariate temporal data. IEEE Trans Knowl Data Eng 28(11):3098\u20133112","journal-title":"IEEE Trans Knowl Data Eng"},{"key":"591_CR45","unstructured":"Rich C, Ponsler B, Holroyd A, Sidner CL (2010) Recognizing engagement in human\u2013robot interaction. In: 2010 5th ACM\/IEEE international conference on human\u2013robot interaction (HRI). IEEE, pp 375\u2013382"},{"key":"591_CR46","unstructured":"Robots in public spaces (2013) towards multi-party, short-term, dynamic human-robot interaction. In: Giuliani M, Petrick R (eds) International conference on social robotics (ICSR 2013), Bristol, UK"},{"key":"591_CR47","unstructured":"Schuller B, Ganascia JG, Devillers L (2016) Multimodal sentiment analysis in the wild: ethical considerations on data collection, annotation, and exploitation. In: Actes du workshop on ethics in corpus collection, annotation and application (ETHI-CA2), LREC, Portoroz, Slov\u00e9nie"},{"key":"591_CR48","doi-asserted-by":"crossref","unstructured":"Schuller B, M\u00fceller R, H\u00f6ernler B, H\u00f6ethker A, Konosu H, Rigoll G (2007) Audiovisual recognition of spontaneous interest within conversations. In: Proceedings of the ninth international conference on multimodal interfaces\u2014ICMI \u201907. ACM Press, New York, p\u00a030","DOI":"10.1145\/1322192.1322201"},{"issue":"1\u20132","key":"591_CR49","doi-asserted-by":"publisher","first-page":"140","DOI":"10.1016\/j.artint.2005.03.005","volume":"166","author":"CL Sidner","year":"2005","unstructured":"Sidner CL, Lee C, Kidd CD, Lesh N, Rich C (2005) Explorations in engagement for humans and robots. Artif Intell 166(1\u20132):140\u2013164","journal-title":"Artif Intell"},{"key":"591_CR50","unstructured":"Tapus A Mataric MJ (2008) Socially assistive robots: the link between personality, empathy, physiological signals, and task performance. Undefined"},{"key":"591_CR51","doi-asserted-by":"crossref","unstructured":"Trung P, Giuliani M, Miksch M, Stollnberger G, Stadler S, Mirnig N, Tscheligi M (2017) Head and shoulders: automatic error detection in human\u2013robot interaction. In: Proceedings of the 19th ACM international conference on multimodal interaction\u2014ICMI 2017. ACM Press, New York, pp 181\u2013188","DOI":"10.1145\/3136755.3136785"},{"key":"591_CR52","doi-asserted-by":"publisher","first-page":"4","DOI":"10.1016\/j.robot.2015.01.004","volume":"75","author":"D Vaufreydaz","year":"2016","unstructured":"Vaufreydaz D, Johal W, Combe C (2016) Starting engagement detection towards a companion robot using multimodal features. Robot Autonom Syst 75:4\u201316","journal-title":"Robot Autonom Syst"},{"key":"591_CR53","unstructured":"Wittenburg P, Brugman H, Russel A, Klassmann A, Sloetjes H (2006) ELAN: a professional framework for multimodality research. In: LREC 2006, pp 1556\u20131559"},{"key":"591_CR54","doi-asserted-by":"crossref","unstructured":"Wood E, Baltruaitis T, Zhang X, Sugano Y, Robinson P, Bulling A (2015) Rendering of eyes for eye-shape registration and gaze estimation. In: 2015 IEEE international conference on computer vision (ICCV). IEEE, pp 3756\u20133764","DOI":"10.1109\/ICCV.2015.428"}],"container-title":["International Journal of Social Robotics"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s12369-019-00591-2.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/article\/10.1007\/s12369-019-00591-2\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s12369-019-00591-2.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2020,9,26]],"date-time":"2020-09-26T23:18:09Z","timestamp":1601162289000},"score":1,"resource":{"primary":{"URL":"http:\/\/link.springer.com\/10.1007\/s12369-019-00591-2"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2019,9,28]]},"references-count":54,"journal-issue":{"issue":"5","published-print":{"date-parts":[[2019,12]]}},"alternative-id":["591"],"URL":"https:\/\/doi.org\/10.1007\/s12369-019-00591-2","relation":{},"ISSN":["1875-4791","1875-4805"],"issn-type":[{"value":"1875-4791","type":"print"},{"value":"1875-4805","type":"electronic"}],"subject":[],"published":{"date-parts":[[2019,9,28]]},"assertion":[{"value":"11 September 2019","order":1,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"28 September 2019","order":2,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Compliance with ethical standards"}},{"value":"The authors declare that they have no conflict of interest.","order":2,"name":"Ethics","group":{"name":"EthicsHeading","label":"Conflict of interest"}}]}}