{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,3,27]],"date-time":"2025-03-27T20:12:04Z","timestamp":1743106324941,"version":"3.40.3"},"publisher-location":"Cham","reference-count":44,"publisher":"Springer International Publishing","isbn-type":[{"type":"print","value":"9783030721121"},{"type":"electronic","value":"9783030721138"}],"license":[{"start":{"date-parts":[[2021,1,1]],"date-time":"2021-01-01T00:00:00Z","timestamp":1609459200000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"},{"start":{"date-parts":[[2021,1,1]],"date-time":"2021-01-01T00:00:00Z","timestamp":1609459200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2021]]},"DOI":"10.1007\/978-3-030-72113-8_29","type":"book-chapter","created":{"date-parts":[[2021,3,26]],"date-time":"2021-03-26T12:03:02Z","timestamp":1616760182000},"page":"433-450","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":1,"title":["Predicting User Engagement Status for\u00a0Online Evaluation of Intelligent Assistants"],"prefix":"10.1007","author":[{"ORCID":"https:\/\/orcid.org\/0000-0001-5583-4924","authenticated-orcid":false,"given":"Rui","family":"Meng","sequence":"first","affiliation":[]},{"given":"Zhen","family":"Yue","sequence":"additional","affiliation":[]},{"given":"Alyssa","family":"Glass","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2021,3,27]]},"reference":[{"key":"29_CR1","doi-asserted-by":"crossref","unstructured":"Aliannejadi, M., Zamani, H., Crestani, F., Croft, W.B.: Asking clarifying questions in open-domain information-seeking conversations. In: Proceedings of the 42nd International ACM SIGIR Conference on Research and Development in Information Retrieval, pp. 475\u2013484 (2019)","DOI":"10.1145\/3331184.3331265"},{"issue":"5","key":"29_CR2","doi-asserted-by":"publisher","first-page":"502","DOI":"10.1111\/opo.12131","volume":"34","author":"RA Armstrong","year":"2014","unstructured":"Armstrong, R.A.: When to use the bonferroni correction. Ophthalmic Physiol. Optics 34(5), 502\u2013508 (2014)","journal-title":"Ophthalmic Physiol. Optics"},{"issue":"7","key":"29_CR3","doi-asserted-by":"publisher","first-page":"1249","DOI":"10.1109\/TASL.2008.2001102","volume":"16","author":"S Bangalore","year":"2008","unstructured":"Bangalore, S., Di Fabbrizio, G., Stent, A.: Learning the structure of task-driven human-human dialogs. IEEE Trans. Audio, Speech Lang. Process. 16(7), 1249\u20131259 (2008)","journal-title":"IEEE Trans. Audio, Speech Lang. Process."},{"key":"29_CR4","doi-asserted-by":"publisher","first-page":"2910","DOI":"10.21437\/Interspeech.2016-859","volume":"2016","author":"SA Chowdhury","year":"2016","unstructured":"Chowdhury, S.A., Stepanov, E.A., Riccardi, G.: Predicting user satisfaction from turn-taking in spoken conversations. Interspeech 2016, 2910\u20132914 (2016)","journal-title":"Interspeech"},{"key":"29_CR5","first-page":"2493","volume":"12","author":"R Collobert","year":"2011","unstructured":"Collobert, R., Weston, J., Bottou, L., Karlen, M., Kavukcuoglu, K., Kuksa, P.: Natural language processing (almost) from scratch. J. Mach. Learn. Res. 12, 2493\u20132537 (2011)","journal-title":"J. Mach. Learn. Res."},{"key":"29_CR6","doi-asserted-by":"crossref","unstructured":"Deng, A., Shi, X.: Data-driven metric development for online controlled experiments: seven lessons learned. In: Proceedings of the 22nd ACM SIGKDD International Conference on Knowledge Discovery and Data Mining, pp. 77\u201386 (2016)","DOI":"10.1145\/2939672.2939700"},{"key":"29_CR7","unstructured":"Devlin, J., Chang, M.W., Lee, K., Toutanova, K.: Bert: pre-training of deep bidirectional transformers for language understanding. In: Proceedings of the 2019 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, Volume 1 (Long and Short Papers), pp. 4171\u20134186 (2019)"},{"key":"29_CR8","doi-asserted-by":"crossref","unstructured":"Diriye, A., White, R., Buscher, G., Dumais, S.: Leaving so soon?: understanding and predicting web search abandonment rationales. In: Proceedings of the 21st ACM International Conference on Information and Knowledge Management, pp. 1025\u20131034. ACM (2012)","DOI":"10.1145\/2396761.2398399"},{"key":"29_CR9","unstructured":"Graepel, T., Candela, J.Q., Borchert, T., Herbrich, R.: Web-scale Bayesian click-through rate prediction for sponsored search advertising in microsoft\u2019s bing search engine. In: Proceedings of the 27th International Conference on International Conference on Machine Learning, pp. 13\u201320 (2010)"},{"key":"29_CR10","doi-asserted-by":"publisher","first-page":"44","DOI":"10.1155\/2016\/8402127","volume":"2016","author":"D Griol","year":"2016","unstructured":"Griol, D., Callejas, Z.: A neural network approach to intention modeling for user-adapted conversational agents. Comput. Intell. Neurosci. 2016, 44 (2016)","journal-title":"Comput. Intell. Neurosci."},{"key":"29_CR11","unstructured":"Hara, S., Kitaoka, N., Takeda, K.: Estimation method of user satisfaction using n-gram-based dialog history model for spoken dialog system. In: Proceedings of the Seventh International Conference on Language Resources and Evaluation (LREC 2010) (2010)"},{"key":"29_CR12","doi-asserted-by":"crossref","unstructured":"Hashemi, S.H., Williams, K., El Kholy, A., Zitouni, I., Crook, P.A.: Measuring user satisfaction on smart speaker intelligent assistants using intent sensitive query embeddings. In: Proceedings of the 27th ACM International Conference on Information and Knowledge Management, pp. 1183\u20131192 (2018)","DOI":"10.1145\/3269206.3271802"},{"key":"29_CR13","doi-asserted-by":"crossref","unstructured":"Hassan, A., Jones, R., Klinkner, K.L.: Beyond DCG: user behavior as a predictor of a successful search. In: Proceedings of the Third ACM International Conference on Web Search and Data Mining, pp. 221\u2013230. ACM (2010)","DOI":"10.1145\/1718487.1718515"},{"key":"29_CR14","doi-asserted-by":"crossref","unstructured":"Hassan, A., Shi, X., Craswell, N., Ramsey, B.: Beyond clicks: query reformulation as a predictor of search satisfaction. In: Proceedings of the 22nd ACM International Conference on Conference on Information & Knowledge Management, pp. 2019\u20132028. ACM (2013)","DOI":"10.1145\/2505515.2505682"},{"key":"29_CR15","doi-asserted-by":"crossref","unstructured":"Hassan, A., Song, Y., He, L.W.: A task level metric for measuring web search satisfaction and its application on improving relevance estimation. In: Proceedings of the 20th ACM International Conference on Information and Knowledge Management, pp. 125\u2013134. ACM (2011)","DOI":"10.1145\/2063576.2063599"},{"key":"29_CR16","doi-asserted-by":"crossref","unstructured":"Henderson, M., Thomson, B., Williams, J.D.: The second dialog state tracking challenge. In: Proceedings of the 15th Annual Meeting of the Special Interest Group on Discourse and Dialogue (SIGDIAL), pp. 263\u2013272 (2014)","DOI":"10.3115\/v1\/W14-4337"},{"key":"29_CR17","doi-asserted-by":"crossref","unstructured":"Henderson, M., Thomson, B., Williams, J.D.: The third dialog state tracking challenge. In: 2014 IEEE Spoken Language Technology Workshop (SLT), pp. 324\u2013329. IEEE (2014)","DOI":"10.1109\/SLT.2014.7078595"},{"key":"29_CR18","unstructured":"Higashinaka, R., Funakoshi, K., Kobayashi, Y., Inaba, M.: The dialogue breakdown detection challenge: task description, datasets, and evaluation metrics. In: Proceedings of the Tenth International Conference on Language Resources and Evaluation (LREC 2016), pp. 3146\u20133150 (2016)"},{"key":"29_CR19","doi-asserted-by":"crossref","unstructured":"Hill, F., Cho, K., Korhonen, A.: Learning distributed representations of sentences from unlabelled data. In: 15th Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, NAACL HLT 2016, pp. 1367\u20131377. Association for Computational Linguistics (ACL) (2016)","DOI":"10.18653\/v1\/N16-1162"},{"key":"29_CR20","doi-asserted-by":"crossref","unstructured":"Jiang, J.E.A.: Automatic online evaluation of intelligent assistants. In: Proceedings of the 24th WWW, pp. 506\u2013516. International World Wide Web Conferences Steering Committee (2015)","DOI":"10.1145\/2736277.2741669"},{"issue":"22","key":"29_CR21","doi-asserted-by":"publisher","first-page":"10031","DOI":"10.1073\/pnas.92.22.10031","volume":"92","author":"C Kamm","year":"1995","unstructured":"Kamm, C.: User interfaces for voice applications. Proc. Natl. Acad. Sci. 92(22), 10031\u201310037 (1995)","journal-title":"Proc. Natl. Acad. Sci."},{"key":"29_CR22","unstructured":"Kim, S.N., Cavedon, L., Baldwin, T.: Classifying dialogue acts in one-on-one live chats. In: Proceedings of the 2010 Conference on EMNLP, pp. 862\u2013871. Association for Computational Linguistics (2010)"},{"key":"29_CR23","doi-asserted-by":"crossref","unstructured":"Kim, Y.: Convolutional neural networks for sentence classification. In: Proceedings of the 2014 Conference on Empirical Methods in Natural Language Processing (EMNLP), pp. 1746\u20131751 (2014)","DOI":"10.3115\/v1\/D14-1181"},{"key":"29_CR24","unstructured":"Kiros, R.E.A.: Skip-thought vectors. In: Advances in Neural Information Processing Systems, pp. 3294\u20133302 (2015)"},{"key":"29_CR25","doi-asserted-by":"crossref","unstructured":"Kiseleva, J., Williams, K., Hassan Awadallah, A., Crook, A.C., Zitouni, I., Anastasakos, T.: Predicting user satisfaction with intelligent assistants. In: Proceedings of the 39th International ACM SIGIR Conference on Research and Development in Information Retrieval, pp. 45\u201354. ACM (2016)","DOI":"10.1145\/2911451.2911521"},{"issue":"1","key":"29_CR26","doi-asserted-by":"publisher","first-page":"19","DOI":"10.1023\/A:1009648614566","volume":"4","author":"E Krahmer","year":"2001","unstructured":"Krahmer, E., Swerts, M., Theune, M., Weegels, M.: Error detection in spoken human-machine interaction. Int. J. Speech Technol. 4(1), 19\u201330 (2001)","journal-title":"Int. J. Speech Technol."},{"key":"29_CR27","unstructured":"Le, Q., Mikolov, T.: Distributed representations of sentences and documents. In: Proceedings of the 31st International Conference on Machine Learning (ICML-14), pp. 1188\u20131196 (2014)"},{"key":"29_CR28","doi-asserted-by":"crossref","unstructured":"Li, J., Galley, M., Brockett, C., Gao, J., Dolan, B.: A diversity-promoting objective function for neural conversation models. In: Proceedings of the 2016 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, pp. 110\u2013119 (2016)","DOI":"10.18653\/v1\/N16-1014"},{"key":"29_CR29","doi-asserted-by":"crossref","unstructured":"Liu, C.W., Lowe, R., Serban, I.V., Noseworthy, M., Charlin, L., Pineau, J.: How not to evaluate your dialogue system: an empirical study of unsupervised evaluation metrics for dialogue response generation. In: Proceedings of the 2016 Conference on Empirical Methods in Natural Language Processing, pp. 2122\u20132132 (2016)","DOI":"10.18653\/v1\/D16-1230"},{"key":"29_CR30","doi-asserted-by":"crossref","unstructured":"Lowe, R., Noseworthy, M., Serban, I.V., Angelard-Gontier, N., Bengio, Y., Pineau, J.: Towards an automatic turing test: learning to evaluate dialogue responses. In: Proceedings of the 55th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers), pp. 1116\u20131126 (2017)","DOI":"10.18653\/v1\/P17-1103"},{"key":"29_CR31","doi-asserted-by":"crossref","unstructured":"Meena, R., Lopes, J., Skantze, G., Gustafson, J.: Automatic detection of miscommunication in spoken dialogue systems. In: Proceedings of the 16th Annual Meeting of the Special Interest Group on Discourse and Dialogue, pp. 354\u2013363 (2015)","DOI":"10.18653\/v1\/W15-4647"},{"key":"29_CR32","unstructured":"Mikolov, T., Sutskever, I., Chen, K., Corrado, G.S., Dean, J.: Distributed representations of words and phrases and their compositionality. In: Advances in Neural Information Processing Systems, pp. 3111\u20133119 (2013)"},{"key":"29_CR33","unstructured":"Ohtake, K.: Unsupervised approach for dialogue act classification. In: Proceedings of the 22nd Pacific Asia Conference on Language, Information and Computation, pp. 445\u2013451 (2008)"},{"key":"29_CR34","doi-asserted-by":"crossref","unstructured":"Papineni, K., Roukos, S., Ward, T., Zhu, W.J.: Bleu: a method for automatic evaluation of machine translation. In: Proceedings of the 40th Annual Meeting of the Association for Computational Linguistics, pp. 311\u2013318 (2002)","DOI":"10.3115\/1073083.1073135"},{"key":"29_CR35","doi-asserted-by":"crossref","unstructured":"Polifroni, J., Hirschman, L., Seneff, S., Zue, V.: Experiments in evaluating interactive spoken language systems. In: Proceedings of the workshop on Speech and Natural Language, pp. 28\u201333. Association for Computational Linguistics (1992)","DOI":"10.3115\/1075527.1075533"},{"key":"29_CR36","unstructured":"Ritter, A., Cherry, C., Dolan, W.B.: Data-driven response generation in social media. In: Proceedings of the 2011 Conference on Empirical Methods in Natural Language Processing, pp. 583\u2013593 (2011)"},{"issue":"3","key":"29_CR37","doi-asserted-by":"publisher","first-page":"317","DOI":"10.1023\/A:1009752403260","volume":"1","author":"SL Salzberg","year":"1997","unstructured":"Salzberg, S.L.: On comparing classifiers: pitfalls to avoid and a recommended approach. Data Mining Knowl. Discov. 1(3), 317\u2013328 (1997)","journal-title":"Data Mining Knowl. Discov."},{"key":"29_CR38","doi-asserted-by":"crossref","unstructured":"Shriberg, E., Wade, E., Price, P.: Human-machine problem solving using spoken language systems (sls): factors affecting performance and user satisfaction. In: Proceedings of the Workshop on Speech and Natural Language, pp. 49\u201354. Association for Computational Linguistics (1992)","DOI":"10.3115\/1075527.1075538"},{"key":"29_CR39","doi-asserted-by":"crossref","unstructured":"Sordoni, A., et al.: A neural network approach to context-sensitive generation of conversational responses. In: Proceedings of the 2015 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, pp. 196\u2013205 (2015)","DOI":"10.3115\/v1\/N15-1020"},{"key":"29_CR40","unstructured":"Vinyals, O., Le, Q.V.: A neural conversational model. In: ICML Deep Learning Workshop (2015). http:\/\/arxiv.org\/pdf\/1506.05869v3.pdf"},{"key":"29_CR41","doi-asserted-by":"crossref","unstructured":"Walker, M.A., Litman, D.J., Kamm, C.A., Abella, A.: Paradise: a framework for evaluating spoken dialogue agents. In: Proceedings of the 35th Annual Meeting of the Association for Computational Linguistics and Eighth Conference of the European Chapter of the Association for Computational Linguistics, pp. 271\u2013280 (1997)","DOI":"10.3115\/976909.979652"},{"key":"29_CR42","doi-asserted-by":"crossref","unstructured":"Yang, Z., Li, B., Zhu, Y., King, I., Levow, G., Meng, H.: Collaborative filtering model for user satisfaction prediction in spoken dialog system evaluation. In: Spoken Language Technology Workshop (SLT), 2010 IEEE, pp. 472\u2013477. IEEE (2010)","DOI":"10.1109\/SLT.2010.5700898"},{"key":"29_CR43","doi-asserted-by":"crossref","unstructured":"Yi, X., Hong, L., Zhong, E., Liu, N.N., Rajan, S.: Beyond clicks: dwell time for personalization. In: Proceedings of the 8th ACM Conference on Recommender Systems, pp. 113\u2013120. ACM (2014)","DOI":"10.1145\/2645710.2645724"},{"key":"29_CR44","doi-asserted-by":"crossref","unstructured":"Yin, W., Sch\u00fctze, H.: Convolutional neural network for paraphrase identification. In: Proceedings of the 2015 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, pp. 901\u2013911 (2015)","DOI":"10.3115\/v1\/N15-1091"}],"container-title":["Lecture Notes in Computer Science","Advances in Information Retrieval"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-030-72113-8_29","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,3,12]],"date-time":"2024-03-12T11:58:07Z","timestamp":1710244687000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-3-030-72113-8_29"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2021]]},"ISBN":["9783030721121","9783030721138"],"references-count":44,"URL":"https:\/\/doi.org\/10.1007\/978-3-030-72113-8_29","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"type":"print","value":"0302-9743"},{"type":"electronic","value":"1611-3349"}],"subject":[],"published":{"date-parts":[[2021]]},"assertion":[{"value":"27 March 2021","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"ECIR","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"European Conference on Information Retrieval","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2021","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"28 March 2021","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"1 April 2021","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"43","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"ecir2021","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"https:\/\/www.ecir2021.eu\/","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Double-blind","order":1,"name":"type","label":"Type","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"EasyChair","order":2,"name":"conference_management_system","label":"Conference Management System","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"436","order":3,"name":"number_of_submissions_sent_for_review","label":"Number of Submissions Sent for Review","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"50","order":4,"name":"number_of_full_papers_accepted","label":"Number of Full Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"39","order":5,"name":"number_of_short_papers_accepted","label":"Number of Short Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"11% - The value is computed by the equation \"Number of Full Papers Accepted \/ Number of Submissions Sent for Review * 100\" and then rounded to a whole number.","order":6,"name":"acceptance_rate_of_full_papers","label":"Acceptance Rate of Full Papers","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"3","order":7,"name":"average_number_of_reviews_per_paper","label":"Average Number of Reviews per Paper","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"3","order":8,"name":"average_number_of_papers_per_reviewer","label":"Average Number of Papers per Reviewer","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"Yes","order":9,"name":"external_reviewers_involved","label":"External Reviewers Involved","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}}]}}