{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,2,2]],"date-time":"2026-02-02T21:03:13Z","timestamp":1770066193412,"version":"3.49.0"},"publisher-location":"New York, NY, USA","reference-count":27,"publisher":"ACM","license":[{"start":{"date-parts":[[2022,7,6]],"date-time":"2022-07-06T00:00:00Z","timestamp":1657065600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"name":"Huawei Finland"},{"name":"Hybrid Intelligence Center"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2022,7,6]]},"DOI":"10.1145\/3477495.3531798","type":"proceedings-article","created":{"date-parts":[[2022,7,7]],"date-time":"2022-07-07T15:12:13Z","timestamp":1657206733000},"page":"2018-2023","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":12,"title":["Understanding User Satisfaction with Task-oriented Dialogue Systems"],"prefix":"10.1145","author":[{"given":"Clemencia","family":"Siro","sequence":"first","affiliation":[{"name":"University of Amsterdam, Amsterdam, Netherlands"}]},{"given":"Mohammad","family":"Aliannejadi","sequence":"additional","affiliation":[{"name":"University of Amsterdam, Amsterdam, Netherlands"}]},{"given":"Maarten","family":"de Rijke","sequence":"additional","affiliation":[{"name":"University of Amsterdam, Amsterdam, Netherlands"}]}],"member":"320","published-online":{"date-parts":[[2022,7,7]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"publisher","DOI":"10.5555\/1814508.1814521"},{"key":"e_1_3_2_1_2_1","volume-title":"Predicting User Intents and Satisfaction with Dialogue-Based Conversational Recommendations","author":"Cai Wanling","unstructured":"Wanling Cai and Li Chen . 2020. Predicting User Intents and Satisfaction with Dialogue-Based Conversational Recommendations . Association for Computing Machinery , New York, NY, USA , 33--42. https:\/\/doi.org\/10.1145\/3340631.3394856 10.1145\/3340631.3394856 Wanling Cai and Li Chen. 2020. Predicting User Intents and Satisfaction with Dialogue-Based Conversational Recommendations .Association for Computing Machinery, New York, NY, USA, 33--42. https:\/\/doi.org\/10.1145\/3340631.3394856"},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10462-020-09866-x"},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/N19-1381"},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.specom.2008.04.002"},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.aiopen.2021.06.002"},{"key":"e_1_3_2_1_7_1","first-page":"1891","article-title":"Topical-Chat","volume":"2019","author":"Gopalakrishnan Karthik","year":"2019","unstructured":"Karthik Gopalakrishnan , Behnam Hedayatnia , Qinlang Chen , Anna Gottardi , Sanjeev Kwatra , Anu Venkatesh , Raefer Gabriel , and Dilek Hakkani-T\u00fcr . 2019 . Topical-Chat : Towards Knowledge-Grounded Open-Domain Conversations. In Proc. Interspeech 2019. 1891 -- 1895 . https:\/\/doi.org\/10.21437\/Interspeech.2019--3079 10.21437\/Interspeech.2019--3079 Karthik Gopalakrishnan, Behnam Hedayatnia, Qinlang Chen, Anna Gottardi, Sanjeev Kwatra, Anu Venkatesh, Raefer Gabriel, and Dilek Hakkani-T\u00fcr. 2019. Topical-Chat: Towards Knowledge-Grounded Open-Domain Conversations. In Proc. Interspeech 2019. 1891--1895. https:\/\/doi.org\/10.21437\/Interspeech.2019--3079","journal-title":"Towards Knowledge-Grounded Open-Domain Conversations. In Proc. Interspeech"},{"key":"e_1_3_2_1_8_1","volume-title":"Proceedings of the 27th ACM International Conference on Information and Knowledge Management","author":"Hashemi Seyyed Hadi","year":"1802","unstructured":"Seyyed Hadi Hashemi , Kyle Williams , Ahmed El Kholy , Imed Zitouni , and Paul A. Crook . 2018. Measuring User Satisfaction on Smart Speaker Intelligent Assistants Using Intent Sensitive Query Embeddings . In Proceedings of the 27th ACM International Conference on Information and Knowledge Management ( Torino, Italy) (CIKM '18). Association for Computing Machinery, New York, NY, USA, 1183--1192. https:\/\/doi.org\/10.1145\/3269206.327 1802 10.1145\/3269206.3271802 Seyyed Hadi Hashemi, Kyle Williams, Ahmed El Kholy, Imed Zitouni, and Paul A. Crook. 2018. Measuring User Satisfaction on Smart Speaker Intelligent Assistants Using Intent Sensitive Query Embeddings. In Proceedings of the 27th ACM International Conference on Information and Knowledge Management (Torino, Italy) (CIKM '18). Association for Computing Machinery, New York, NY, USA, 1183--1192. https:\/\/doi.org\/10.1145\/3269206.3271802"},{"key":"e_1_3_2_1_9_1","volume-title":"Proceedings of the Eighth ACM International Conference on Web Search and Data Mining","author":"Jiang Jiepu","unstructured":"Jiepu Jiang , Ahmed Hassan Awadallah , Xiaolin Shi , and Ryen W. White . 2015. Understanding and Predicting Graded Search Satisfaction . In Proceedings of the Eighth ACM International Conference on Web Search and Data Mining ( Shanghai, China) (WSDM '15). Association for Computing Machinery, New York, NY, USA, 57--66. https:\/\/doi.org\/10.1145\/2684822.2685319 10.1145\/2684822.2685319 Jiepu Jiang, Ahmed Hassan Awadallah, Xiaolin Shi, and Ryen W. White. 2015. Understanding and Predicting Graded Search Satisfaction. In Proceedings of the Eighth ACM International Conference on Web Search and Data Mining (Shanghai, China) (WSDM '15). Association for Computing Machinery, New York, NY, USA, 57--66. https:\/\/doi.org\/10.1145\/2684822.2685319"},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.1561\/1500000012"},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.1145\/2556195.2556220"},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.1145\/2911451.2911521"},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1145\/2854946.2854961"},{"key":"e_1_3_2_1_14_1","volume-title":"ACUTE-EVAL: Improved Dialogue Evaluation with Optimized Questions and Multi-turn Comparisons. ArXiv","author":"Li Margaret","year":"2019","unstructured":"Margaret Li , Jason Weston , and Stephen Roller . 2019. ACUTE-EVAL: Improved Dialogue Evaluation with Optimized Questions and Multi-turn Comparisons. ArXiv , Vol. abs\/ 1909 .03087 ( 2019 ). Margaret Li, Jason Weston, and Stephen Roller. 2019. ACUTE-EVAL: Improved Dialogue Evaluation with Optimized Questions and Multi-turn Comparisons. ArXiv , Vol. abs\/1909.03087 (2019)."},{"key":"e_1_3_2_1_15_1","volume-title":"Hannes Schulz, Vincent Michalski, Laurent Charlin, and Chris Pal.","author":"Li Raymond","year":"2018","unstructured":"Raymond Li , Samira Ebrahimi Kahou , Hannes Schulz, Vincent Michalski, Laurent Charlin, and Chris Pal. 2018 . Towards Deep Conversational Recommendations. In Advances in Neural Information Processing Systems 31 (NIPS 2018) . Raymond Li, Samira Ebrahimi Kahou, Hannes Schulz, Vincent Michalski, Laurent Charlin, and Chris Pal. 2018. Towards Deep Conversational Recommendations. In Advances in Neural Information Processing Systems 31 (NIPS 2018) ."},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D16-1230"},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.1145\/3397271.3401085"},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.acl-main.64"},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v34i05.6394"},{"key":"e_1_3_2_1_20_1","volume-title":"Proceedings of the 2019 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies","volume":"1","author":"Roller Stephen","year":"2019","unstructured":"Abigail See, Stephen Roller , Douwe Kiela , and Jason Weston . 2019 . What makes a good conversation? How controllable attributes affect human judgments . In Proceedings of the 2019 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies , Volume 1 (Long and Short Papers). Association for Computational Linguistics, Minneapolis, Minnesota, 1702--1723. https:\/\/doi.org\/10. 18653\/v1\/N19--1170 10.18653\/v1 Abigail See, Stephen Roller, Douwe Kiela, and Jason Weston. 2019. What makes a good conversation? How controllable attributes affect human judgments. In Proceedings of the 2019 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, Volume 1 (Long and Short Papers). Association for Computational Linguistics, Minneapolis, Minnesota, 1702--1723. https:\/\/doi.org\/10.18653\/v1\/N19--1170"},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICASSP40776.2020.9054379"},{"key":"e_1_3_2_1_22_1","volume-title":"Proceedings of the 2021 Conference of the North American","author":"Sun Kai","year":"1865","unstructured":"Kai Sun , Seungwhan Moon , Paul Crook , Stephen Roller , Becka Silvert , Bing Liu , Zhiguang Wang , Honglei Liu , Eunjoon Cho , and Claire Cardie . 2021 a. Adding Chit-Chat to Enhance Task-Oriented Dialogues . In Proceedings of the 2021 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies. Association for Computational Linguistics , Online , 1570--1583. https:\/\/doi.org\/10. 1865 3\/v1\/2021.naacl-main.124 10.18653\/v1 Kai Sun, Seungwhan Moon, Paul Crook, Stephen Roller, Becka Silvert, Bing Liu, Zhiguang Wang, Honglei Liu, Eunjoon Cho, and Claire Cardie. 2021 a. Adding Chit-Chat to Enhance Task-Oriented Dialogues. In Proceedings of the 2021 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies. Association for Computational Linguistics, Online, 1570--1583. https:\/\/doi.org\/10.18653\/v1\/2021.naacl-main.124"},{"key":"e_1_3_2_1_23_1","volume-title":"2021 b. Simulating User Satisfaction for the Evaluation of Task-Oriented Dialogue Systems","author":"Sun Weiwei","unstructured":"Weiwei Sun , Shuo Zhang , Krisztian Balog , Zhaochun Ren , Pengjie Ren , Zhumin Chen , and Maarten de Rijke . 2021 b. Simulating User Satisfaction for the Evaluation of Task-Oriented Dialogue Systems . Association for Computing Machinery , New York, NY, USA , 2499--2506. https:\/\/doi.org\/10.1145\/3404835.3463241 10.1145\/3404835.3463241 Weiwei Sun, Shuo Zhang, Krisztian Balog, Zhaochun Ren, Pengjie Ren, Zhumin Chen, and Maarten de Rijke. 2021 b. Simulating User Satisfaction for the Evaluation of Task-Oriented Dialogue Systems .Association for Computing Machinery, New York, NY, USA, 2499--2506. https:\/\/doi.org\/10.1145\/3404835.3463241"},{"key":"e_1_3_2_1_24_1","volume-title":"On Evaluating and Comparing Open Domain Dialog Systems. arXiv: Computation and Language","author":"Venkatesh Anu","year":"2018","unstructured":"Anu Venkatesh , Chandra Khatri , Ashwin Ram , Fenfei Guo , Raefer Gabriel , Ashish Nagar , Rohit Prasad , Ming Cheng , Behnam Hedayatnia , Angeliki Metallinou , Rahul Goel , Shaohua Yang , and Anirudh Raju . 2018. On Evaluating and Comparing Open Domain Dialog Systems. arXiv: Computation and Language ( 2018 ). Anu Venkatesh, Chandra Khatri, Ashwin Ram, Fenfei Guo, Raefer Gabriel, Ashish Nagar, Rohit Prasad, Ming Cheng, Behnam Hedayatnia, Angeliki Metallinou, Rahul Goel, Shaohua Yang, and Anirudh Raju. 2018. On Evaluating and Comparing Open Domain Dialog Systems. arXiv: Computation and Language (2018)."},{"key":"e_1_3_2_1_25_1","volume-title":"Proceedings of the 35th Annual Meeting of the Association for Computational Linguistics and Eighth Conference of the European Chapter of the Association for Computational Linguistics","author":"Walker Marilyn A.","year":"1997","unstructured":"Marilyn A. Walker , Diane J. Litman , Candace A. Kamm , and Alicia Abella . 1997 . PARADISE: A Framework for Evaluating Spoken Dialogue Agents . In Proceedings of the 35th Annual Meeting of the Association for Computational Linguistics and Eighth Conference of the European Chapter of the Association for Computational Linguistics ( Madrid, Spain) (ACL '98\/EACL '98). Association for Computational Linguistics, USA, 271--280. https:\/\/doi.org\/10.3115\/976909.979652 10.3115\/976909.979652 Marilyn A. Walker, Diane J. Litman, Candace A. Kamm, and Alicia Abella. 1997. PARADISE: A Framework for Evaluating Spoken Dialogue Agents. In Proceedings of the 35th Annual Meeting of the Association for Computational Linguistics and Eighth Conference of the European Chapter of the Association for Computational Linguistics (Madrid, Spain) (ACL '98\/EACL '98). Association for Computational Linguistics, USA, 271--280. https:\/\/doi.org\/10.3115\/976909.979652"},{"key":"e_1_3_2_1_26_1","volume-title":"Evaluating Conversational Recommender Systems via User Simulation","author":"Zhang Shuo","unstructured":"Shuo Zhang and Krisztian Balog . 2020. Evaluating Conversational Recommender Systems via User Simulation . Association for Computing Machinery , New York, NY, USA , 1512--1520. https:\/\/doi.org\/10.1145\/3394486.3403202 10.1145\/3394486.3403202 Shuo Zhang and Krisztian Balog. 2020. Evaluating Conversational Recommender Systems via User Simulation .Association for Computing Machinery, New York, NY, USA, 1512--1520. https:\/\/doi.org\/10.1145\/3394486.3403202"},{"key":"e_1_3_2_1_27_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/P18-1205"}],"event":{"name":"SIGIR '22: The 45th International ACM SIGIR Conference on Research and Development in Information Retrieval","location":"Madrid Spain","acronym":"SIGIR '22","sponsor":["SIGIR ACM Special Interest Group on Information Retrieval"]},"container-title":["Proceedings of the 45th International ACM SIGIR Conference on Research and Development in Information Retrieval"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3477495.3531798","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3477495.3531798","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T18:10:25Z","timestamp":1750183825000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3477495.3531798"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2022,7,6]]},"references-count":27,"alternative-id":["10.1145\/3477495.3531798","10.1145\/3477495"],"URL":"https:\/\/doi.org\/10.1145\/3477495.3531798","relation":{},"subject":[],"published":{"date-parts":[[2022,7,6]]},"assertion":[{"value":"2022-07-07","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}