{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,13]],"date-time":"2026-04-13T19:37:03Z","timestamp":1776109023693,"version":"3.50.1"},"publisher-location":"New York, NY, USA","reference-count":40,"publisher":"ACM","license":[{"start":{"date-parts":[[2023,3,19]],"date-time":"2023-03-19T00:00:00Z","timestamp":1679184000000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"funder":[{"name":"NWO","award":["Hybrid Intelligence Center"],"award-info":[{"award-number":["Hybrid Intelligence Center"]}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2023,3,19]]},"DOI":"10.1145\/3576840.3578319","type":"proceedings-article","created":{"date-parts":[[2023,3,20]],"date-time":"2023-03-20T17:43:05Z","timestamp":1679334185000},"page":"258-268","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":7,"title":["Weakly Supervised Turn-level Engagingness Evaluator for Dialogues"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0001-6842-6583","authenticated-orcid":false,"given":"Shaojie","family":"Jiang","sequence":"first","affiliation":[{"name":"University of Amsterdam, Netherlands"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-5278-8886","authenticated-orcid":false,"given":"Svitlana","family":"Vakulenko","sequence":"additional","affiliation":[{"name":"Amazon, Spain"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-1086-0202","authenticated-orcid":false,"given":"Maarten","family":"de Rijke","sequence":"additional","affiliation":[{"name":"University of Amsterdam, Netherlands"}]}],"member":"320","published-online":{"date-parts":[[2023,3,20]]},"reference":[{"key":"e_1_3_2_1_1_1","volume-title":"BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding. arXiv preprint arXiv:1810.04805(2018).","author":"Devlin Jacob","year":"2018","unstructured":"Jacob Devlin, Ming-Wei Chang, Kenton Lee, and Kristina Toutanova. 2018. BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding. arXiv preprint arXiv:1810.04805(2018)."},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"crossref","unstructured":"Emily Dinan Varvara Logacheva Valentin Malykh Alexander\u00a0H. Miller Kurt Shuster Jack Urbanek Douwe Kiela Arthur Szlam Iulian Serban Ryan Lowe Shrimai Prabhumoye Alan\u00a0W. Black Alexander\u00a0I. Rudnicky Jason Williams Joelle Pineau Mikhail\u00a0S. Burtsev and Jason Weston. 2019. The Second Conversational Intelligence Challenge (ConvAI2). CoRR abs\/1902.00098(2019).","DOI":"10.1007\/978-3-030-29135-8_7"},{"key":"e_1_3_2_1_3_1","unstructured":"Emily Dinan Stephen Roller Kurt Shuster Angela Fan Michael Auli and Jason Weston. 2018. Wizard of Wikipedia: Knowledge-powered Conversational Agents. arXiv preprint arXiv:1811.01241(2018)."},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-71292-1_26"},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.emnlp-main.28"},{"key":"e_1_3_2_1_6_1","volume-title":"Predictive Engagement: An Efficient Metric for Automatic Evaluation of Open-Domain Dialogue Systems. In The Thirty-Fourth AAAI Conference on Artificial Intelligence, AAAI","author":"Ghazarian Sarik","year":"2020","unstructured":"Sarik Ghazarian, Ralph\u00a0M. Weischedel, Aram Galstyan, and Nanyun Peng. 2020. Predictive Engagement: An Efficient Metric for Automatic Evaluation of Open-Domain Dialogue Systems. In The Thirty-Fourth AAAI Conference on Artificial Intelligence, AAAI 2020, The Thirty-Second Innovative Applications of Artificial Intelligence Conference, IAAI 2020, The Tenth AAAI Symposium on Educational Advances in Artificial Intelligence, EAAI 2020, New York, NY, USA, February 7-12, 2020. AAAI Press, 7789\u20137796."},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/W19-5944"},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.1609\/aimag.v39i3.2810"},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.2200\/S00605ED1V01Y201410ICR038"},{"key":"e_1_3_2_1_10_1","volume-title":"Acute-eval: Improved dialogue evaluation with optimized questions and multi-turn comparisons. arXiv preprint arXiv:1909.03087(2019).","author":"Li Margaret","year":"2019","unstructured":"Margaret Li, Jason Weston, and Stephen Roller. 2019. Acute-eval: Improved dialogue evaluation with optimized questions and multi-turn comparisons. arXiv preprint arXiv:1909.03087(2019)."},{"key":"e_1_3_2_1_11_1","volume-title":"Proceedings of the Eighth International Joint Conference on Natural Language Processing, IJCNLP 2017","volume":"995","author":"Li Yanran","year":"2017","unstructured":"Yanran Li, Hui Su, Xiaoyu Shen, Wenjie Li, Ziqiang Cao, and Shuzi Niu. 2017. DailyDialog: A Manually Labelled Multi-turn Dialogue Dataset. In Proceedings of the Eighth International Joint Conference on Natural Language Processing, IJCNLP 2017, Taipei, Taiwan, November 27-December 1, 2017 - Volume 1: Long Papers, Greg Kondrak and Taro Watanabe (Eds.). Asian Federation of Natural Language Processing, 986\u2013995."},{"key":"e_1_3_2_1_12_1","volume-title":"Proceedings of the 59th Annual Meeting of the Association for Computational Linguistics.","author":"Li Zekang","year":"2021","unstructured":"Zekang Li, Jinchao Zhang, Zhengcong Fei, Yang Feng, and Jie Zhou. 2021. Conversations are not flat: Modeling the intrinsic information flow between dialogue utterances. In Proceedings of the 59th Annual Meeting of the Association for Computational Linguistics."},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2021.acl-long.283"},{"key":"e_1_3_2_1_14_1","volume-title":"The NIPS\u201917 Competition: Building Intelligent Systems","author":"Logacheva Varvara","unstructured":"Varvara Logacheva, Mikhail Burtsev, Valentin Malykh, Vadim Polulyakh, and Aleksandr Seliverstov. 2018. ConvAI Dataset of Topic-oriented Human-to-chatbot Dialogues. In The NIPS\u201917 Competition: Building Intelligent Systems. Springer, 47\u201357."},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/P17-1103"},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.24963\/ijcai.2018\/809"},{"key":"e_1_3_2_1_17_1","volume-title":"Proceedings of the 21th Annual Meeting of the Special Interest Group on Discourse and Dialogue, SIGdial 2020, 1st virtual meeting","author":"Mehri Shikib","year":"2020","unstructured":"Shikib Mehri and Maxine Esk\u00e9nazi. 2020. Unsupervised Evaluation of Interactive Dialog with DialoGPT. In Proceedings of the 21th Annual Meeting of the Special Interest Group on Discourse and Dialogue, SIGdial 2020, 1st virtual meeting, July 1-3, 2020, Olivier Pietquin, Smaranda Muresan, Vivian Chen, Casey Kennington, David Vandyke, Nina Dethlefs, Koji Inoue, Erik Ekstedt, and Stefan Ultes (Eds.). Association for Computational Linguistics, 225\u2013235."},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.acl-main.64"},{"key":"e_1_3_2_1_19_1","volume-title":"Why Engagement Matters: Cross-Disciplinary Perspectives and Innovations on User Engagement with Digital Media","author":"O\u2019Brien L.","unstructured":"Heather\u00a0L. O\u2019Brien. 2016. Theoretical Perspectives on User Engagement. In Why Engagement Matters: Cross-Disciplinary Perspectives and Innovations on User Engagement with Digital Media. Springer, 1\u201326."},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.ijhcs.2018.01.004"},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.acl-main.333"},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.1145\/3498366.3505809"},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.coling-main.368"},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"publisher","DOI":"10.1145\/3020165.3020183"},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/P19-1534"},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2021.eacl-main.24"},{"key":"e_1_3_2_1_27_1","volume-title":"Proceedings of the 2019 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, NAACL-HLT 2019","volume":"1","author":"Roller Stephen","year":"2019","unstructured":"Abigail See, Stephen Roller, Douwe Kiela, and Jason Weston. 2019. What Makes a Good Conversation? How Controllable Attributes Affect Human Judgments. In Proceedings of the 2019 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, NAACL-HLT 2019, Minneapolis, MN, USA, June 2-7, 2019, Volume 1 (Long and Short Papers), Jill Burstein, Christy Doran, and Thamar Solorio (Eds.). Association for Computational Linguistics, 1702\u20131723."},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.acl-main.220"},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.1145\/3477495.3531798"},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.acl-main.183"},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"publisher","DOI":"10.1145\/3466796"},{"key":"e_1_3_2_1_32_1","unstructured":"Anu Venkatesh Chandra Khatri Ashwin Ram Fenfei Guo Raefer Gabriel Ashish Nagar Rohit Prasad Ming Cheng Behnam Hedayatnia Angeliki Metallinou Rahul Goel Shaohua Yang and Anirudh Raju. 2018. On Evaluating and Comparing Open Domain Dialog Systems. arXiv preprint arXiv:1801.03625(2018)."},{"key":"e_1_3_2_1_33_1","unstructured":"Falcon William and The PyTorch\u00a0Lightning team. 2019. PyTorch Lightning. https:\/\/github.com\/PyTorchLightning\/pytorch-lightning."},{"key":"e_1_3_2_1_34_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.emnlp-demos.6"},{"key":"e_1_3_2_1_35_1","unstructured":"Omry Yadan. 2019. Hydra - A Framework for Elegantly Configuring Complex Applications. https:\/\/github.com\/facebookresearch\/hydra."},{"key":"e_1_3_2_1_36_1","unstructured":"Yi-Ting Yeh Maxine Esk\u00e9nazi and Shikib Mehri. 2021. A Comprehensive Assessment of Dialog Evaluation Metrics. CoRR abs\/2106.03706(2021)."},{"key":"e_1_3_2_1_37_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/W19-8608"},{"key":"e_1_3_2_1_38_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2021.acl-long.441"},{"key":"e_1_3_2_1_39_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/P18-1205"},{"key":"e_1_3_2_1_40_1","volume-title":"Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics: System Demonstrations, ACL 2020","author":"Zhang Yizhe","year":"2020","unstructured":"Yizhe Zhang, Siqi Sun, Michel Galley, Yen-Chun Chen, Chris Brockett, Xiang Gao, Jianfeng Gao, Jingjing Liu, and Bill Dolan. 2020. DIALOGPT: Large-Scale Generative Pre-training for Conversational Response Generation. In Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics: System Demonstrations, ACL 2020, Online, July 5-10, 2020, Asli \u00c7elikyilmaz and Tsung-Hsien Wen (Eds.). Association for Computational Linguistics, 270\u2013278."}],"event":{"name":"CHIIR '23: ACM SIGIR Conference on Human Information Interaction and Retrieval","location":"Austin TX USA","acronym":"CHIIR '23","sponsor":["SIGIR ACM Special Interest Group on Information Retrieval","SIGCHI ACM Special Interest Group on Computer-Human Interaction"]},"container-title":["Proceedings of the 2023 Conference on Human Information Interaction and Retrieval"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3576840.3578319","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3576840.3578319","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T16:47:27Z","timestamp":1750178847000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3576840.3578319"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023,3,19]]},"references-count":40,"alternative-id":["10.1145\/3576840.3578319","10.1145\/3576840"],"URL":"https:\/\/doi.org\/10.1145\/3576840.3578319","relation":{},"subject":[],"published":{"date-parts":[[2023,3,19]]},"assertion":[{"value":"2023-03-20","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}