{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,16]],"date-time":"2026-07-16T05:04:37Z","timestamp":1784178277858,"version":"3.55.0"},"publisher-location":"New York, NY, USA","reference-count":27,"publisher":"ACM","license":[{"start":{"date-parts":[[2021,7,11]],"date-time":"2021-07-11T00:00:00Z","timestamp":1625961600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2021,7,11]]},"DOI":"10.1145\/3404835.3463090","type":"proceedings-article","created":{"date-parts":[[2021,7,12]],"date-time":"2021-07-12T03:08:41Z","timestamp":1626059321000},"page":"1935-1939","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":9,"title":["Hierarchical Dependence-aware Evaluation Measures for Conversational Search"],"prefix":"10.1145","author":[{"given":"Guglielmo","family":"Faggioli","sequence":"first","affiliation":[{"name":"University of Padova, Padova, Italy"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Marco","family":"Ferrante","sequence":"additional","affiliation":[{"name":"University of Padova, Padova, Italy"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Nicola","family":"Ferro","sequence":"additional","affiliation":[{"name":"University of Padova, Padova, Italy"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Raffaele","family":"Perego","sequence":"additional","affiliation":[{"name":"National Research Council, Pisa, Italy"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Nicola","family":"Tonellotto","sequence":"additional","affiliation":[{"name":"University of Pisa, Pisa, Italy"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2021,7,11]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"publisher","DOI":"10.1145\/3451964.3451967"},{"key":"e_1_3_2_1_2_1","volume-title":"Semi-formal Evaluation of Conversational Characters","author":"Artstein Ron","unstructured":"Ron Artstein, Sudeep Gandhe, Jillian Gerten, Anton Leuski, and David Traum. 2009. Semi-formal Evaluation of Conversational Characters .Springer Berlin Heidelberg, Berlin, Heidelberg, 22--35."},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1109\/TASL.2008.2001102"},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.1023\/A:1009984519381"},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1145\/2009916.2010037"},{"key":"e_1_3_2_1_6_1","volume-title":"Yejin Choi, Percy Liang, and Luke Zettlemoyer.","author":"Choi Eunsol","year":"2018","unstructured":"Eunsol Choi, He He, Mohit Iyyer, Mark Yatskar, Wen tau Yih, Yejin Choi, Percy Liang, and Luke Zettlemoyer. 2018. QuAC : Question Answering in Context. arxiv: cs.CL\/1808.07036"},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"publisher","DOI":"10.1145\/3397271.3401206"},{"key":"e_1_3_2_1_8_1","volume-title":"Survey on evaluation methods for dialogue systems. Artificial Intelligence Review","author":"Deriu Jan","year":"2020","unstructured":"Jan Deriu, Alvaro Rodrigo, Arantxa Otegi, Guillermo Echegoyen, Sophie Rosset, Eneko Agirre, and Mark Cieliebak. 2020. Survey on evaluation methods for dialogue systems. Artificial Intelligence Review (2020), 1--56."},{"key":"e_1_3_2_1_9_1","unstructured":"Mateusz Dubiel Martin Halvey Leif Azzopardi and Sylvain Daronnat. 2018. Investigating how conversational search agents affect user's behaviour performance and search experience. In The second international workshop on conversational approaches to information retrieval ."},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"crossref","unstructured":"N. Ferro and G. Silvello. 2018. Toward an anatomy of IR system component performances. jasist Vol. 69 2 (2018) 187--200.","DOI":"10.1002\/asi.23910"},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.1007\/s41233-020-00033-2"},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.1145\/3190580.3190586"},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1109\/TASLP.2019.2955290"},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.1145\/3290607.3312979"},{"key":"e_1_3_2_1_15_1","volume-title":"Methods for evaluating interactive information retrieval systems with users. Foundations and trends in Information Retrieval","author":"Kelly Diane","year":"2009","unstructured":"Diane Kelly. 2009. Methods for evaluating interactive information retrieval systems with users. Foundations and trends in Information Retrieval, Vol. 3, 1-2 (2009), 1--224."},{"key":"e_1_3_2_1_16_1","unstructured":"Chia-Wei Liu Ryan Lowe Iulian V. Serban Michael Noseworthy Laurent Charlin and Joelle Pineau. 2017. How NOT To Evaluate Your Dialogue System: An Empirical Study of Unsupervised Evaluation Metrics for Dialogue Response Generation. arxiv: cs.CL\/1603.08023"},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.1145\/2407085.2407092"},{"key":"e_1_3_2_1_18_1","volume-title":"Introducing MANtIS: a novel multi-domain information seeking dialogues dataset. arXiv preprint arXiv:1912.04639","author":"Penha Gustavo","year":"2019","unstructured":"Gustavo Penha, Alexandru Balan, and Claudia Hauff. 2019. Introducing MANtIS: a novel multi-domain information seeking dialogues dataset. arXiv preprint arXiv:1912.04639 (2019)."},{"key":"e_1_3_2_1_19_1","unstructured":"Gustavo Penha and C. Hauff. 2020. Challenges in the Evaluation of Conversational Search Systems. In Converse@KDD."},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.1162\/tacl_a_00266"},{"key":"e_1_3_2_1_21_1","volume-title":"Introducing ANOVA and ANCOVA: a GLM approach","author":"Rutherford Andrew","unstructured":"Andrew Rutherford. 2001. Introducing ANOVA and ANCOVA: a GLM approach. Sage."},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.1145\/3451964.3451976"},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.1145\/3289600.3290985"},{"key":"e_1_3_2_1_24_1","volume-title":"Comparing individual means in the analysis of variance. Biometrics","author":"Tukey John W","year":"1949","unstructured":"John W Tukey. 1949. Comparing individual means in the analysis of variance. Biometrics (1949), 99--114."},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/P17-1046"},{"key":"e_1_3_2_1_26_1","first-page":"5520","article-title":"Chitty-Chitty-Chat Bot\": Deep Learning for Conversational AI","volume":"18","author":"Yan Rui","year":"2018","unstructured":"Rui Yan. 2018. \"Chitty-Chitty-Chat Bot\": Deep Learning for Conversational AI.. In IJCAI, Vol. 18. 5520--5526.","journal-title":"IJCAI"},{"key":"e_1_3_2_1_27_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/W16-3649"}],"event":{"name":"SIGIR '21: The 44th International ACM SIGIR Conference on Research and Development in Information Retrieval","location":"Virtual Event Canada","acronym":"SIGIR '21","sponsor":["SIGIR ACM Special Interest Group on Information Retrieval"]},"container-title":["Proceedings of the 44th International ACM SIGIR Conference on Research and Development in Information Retrieval"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3404835.3463090","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3404835.3463090","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T20:18:30Z","timestamp":1750191510000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3404835.3463090"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2021,7,11]]},"references-count":27,"alternative-id":["10.1145\/3404835.3463090","10.1145\/3404835"],"URL":"https:\/\/doi.org\/10.1145\/3404835.3463090","relation":{},"subject":[],"published":{"date-parts":[[2021,7,11]]},"assertion":[{"value":"2021-07-11","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}