{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,3,14]],"date-time":"2025-03-14T08:40:03Z","timestamp":1741941603555,"version":"3.38.0"},"reference-count":21,"publisher":"China Science Publishing & Media Ltd.","issue":"2","license":[{"start":{"date-parts":[[2021,3,6]],"date-time":"2021-03-06T00:00:00Z","timestamp":1614988800000},"content-version":"vor","delay-in-days":64,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"content-domain":{"domain":["direct.mit.edu"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2021,6,2]]},"abstract":"<jats:title>Abstract<\/jats:title>\n               <jats:p>There is a growing interest in developing human-computer dialogue systems which is an important branch in the field of artificial intelligence (AI). However, the evaluation of large-scale Chinese human-computer dialogues is still a challenging task. To attract more attention to dialogue evaluation work, we held the fourth Evaluation of Chinese Human-Computer Dialogue Technology (ECDT). It consists of few-shot learning in spoken language understanding (SLU) (Task 1) and knowledge-driven multi-turn dialogue competition (Task 2), the data sets of which are provided by Harbin Institute of Technology and Tsinghua University. In this paper, we will introduce the evaluation tasks and data sets in detail. Meanwhile, we will also analyze the evaluation results and the existing problems in the evaluation.<\/jats:p>","DOI":"10.1162\/dint_a_00090","type":"journal-article","created":{"date-parts":[[2021,2,10]],"date-time":"2021-02-10T17:11:22Z","timestamp":1612977082000},"page":"274-286","update-policy":"https:\/\/doi.org\/10.1162\/mitpressjournals.corrections.policy","source":"Crossref","is-referenced-by-count":1,"title":["An Evaluation of Chinese Human-Computer Dialogue\n                    Technology"],"prefix":"10.3724","volume":"3","author":[{"given":"Zixian","family":"Feng","sequence":"first","affiliation":[{"name":"Research Center for Social Computing and Information Retrieval, Harbin Institute of Technology, Harbin 150001, China"}]},{"given":"Caihai","family":"Zhu","sequence":"additional","affiliation":[{"name":"Research Center for Social Computing and Information Retrieval, Harbin Institute of Technology, Harbin 150001, China"}]},{"given":"Weinan","family":"Zhang","sequence":"additional","affiliation":[{"name":"Research Center for Social Computing and Information Retrieval, Harbin Institute of Technology, Harbin 150001, China"}]},{"given":"Zhigang","family":"Chen","sequence":"additional","affiliation":[{"name":"AI Research Institute, IFLYTEK CO., LTD., Hefei 230088, China"}]},{"given":"Wanxiang","family":"Che","sequence":"additional","affiliation":[{"name":"Research Center for Social Computing and Information Retrieval, Harbin Institute of Technology, Harbin 150001, China"}]},{"given":"Minlie","family":"Huang","sequence":"additional","affiliation":[{"name":"Department of Computer Science and Technology, Tsinghua University, Beijing 100084, China"}]},{"given":"Linlin","family":"Li","sequence":"additional","affiliation":[{"name":"Consumer BG, HUAWEI Technologies CO., LTD., Nanjing 210012, China"}]}],"member":"2026","published-online":{"date-parts":[[2021,6,2]]},"reference":[{"key":"2021091617271734700_ref1","first-page":"13","article-title":"A Chinese intelligent conversational robot","volume-title":"Proceedings of the 55th Annual Meeting of the Association for\n                        Computational Linguistics-System Demonstrations","author":"Zhang","year":"2017"},{"volume-title":"A deep reinforcement learning chatbot","year":"2017","author":"Serban","key":"2021091617271734700_ref2"},{"volume-title":"The first evaluation of Chinese human-computer dialogue\n                        technology","year":"2017","author":"Zhang","key":"2021091617271734700_ref3"},{"issue":"236","key":"2021091617271734700_ref4","doi-asserted-by":"crossref","first-page":"433","DOI":"10.1093\/mind\/LIX.236.433","article-title":"Computing machinery and intelligence","volume":"59","author":"Turing","journal-title":"Mind"},{"issue":"4","key":"2021091617271734700_ref5","doi-asserted-by":"crossref","first-page":"303","DOI":"10.1016\/j.trit.2016.12.004","article-title":"Recent advances on human-computer dialogue","volume":"1","author":"Wang","journal-title":"CAAI Transactions on Intelligence Technology"},{"key":"2021091617271734700_ref6","doi-asserted-by":"crossref","DOI":"10.1145\/3166054.3166058","volume-title":"A survey on dialogue systems: Recent advances and new\n                        frontiers","author":"Chen","year":"2017"},{"issue":"8","key":"2021091617271734700_ref7","doi-asserted-by":"crossref","first-page":"953","DOI":"10.1360\/N112017-00125","article-title":"Survey of evaluation methods for dialogue systems (in\n                        Chinese)","volume":"47","author":"Zhang","year":"2017","journal-title":"SCIE NTIA SINICA Informationis"},{"issue":"3","key":"2021091617271734700_ref8","doi-asserted-by":"crossref","first-page":"530","DOI":"10.1109\/TASLP.2014.2383614","article-title":"Using recurrent neural networks for slot filling in spoken\n                        language understanding","volume":"23","author":"Mesnil","journal-title":"IEEE\/ACM Transactions on\n                        Audio Speech Language Processing"},{"key":"2021091617271734700_ref9","doi-asserted-by":"crossref","first-page":"2574","DOI":"10.1145\/3219819.3220045","article-title":"Coupled context modeling for deep chit-chat: Towards\n                        conversations between human and computer","volume-title":"Proceedings of the 24th ACM SIGKDD International Conference on\n                        Knowledge Discovery Data Mining (KDD '18)","author":"Yan","year":"2018"},{"key":"2021091617271734700_ref10","doi-asserted-by":"crossref","first-page":"1427","DOI":"10.1007\/s11280-018-0598-6","article-title":"Neural personalized response generation as domain\n                        adaptation","volume":"22","author":"Zhang","journal-title":"World Wide Web"},{"key":"2021091617271734700_ref11","doi-asserted-by":"crossref","DOI":"10.18653\/v1\/2020.acl-main.128","volume-title":"Few-shot slot tagging with collapsed dependency transfer and\n                        label-enhanced task-adaptive projection network","author":"Hou","year":"2020"},{"key":"2021091617271734700_ref12","doi-asserted-by":"crossref","first-page":"7098","DOI":"10.18653\/v1\/2020.acl-main.635","article-title":"KdConv: A Chinese multi-domain dialogue data set towards\n                        multi-turn knowledge-driven conversation","volume-title":"Proceedings of the 58th Annual Meeting of the Association for\n                        Computational Linguistics","author":"Zhou","year":"2020"},{"volume-title":"Chinese human-computer dialogue technology dataset","author":"Feng","key":"2021091617271734700_ref13"},{"volume-title":"Toward optimal feature selection in NaiveBayes for text\n                        categorization","year":"2016","author":"Tang","key":"2021091617271734700_ref14"},{"volume-title":"A diversity-promoting objective function for neural conversation\n                        models","year":"2015","author":"Li","key":"2021091617271734700_ref15"},{"key":"2021091617271734700_ref16","first-page":"311","article-title":"BLEU: A method for automatic evaluation of machine\n                        translation","volume-title":"Proceedings of the 40th Annual\n                        Meeting of the Association for Computational Linguistics","author":"Papineni","year":"2002"},{"volume-title":"BERT: Pre-training of deep bidirectional transformers for language\n                        understanding","year":"2018","author":"Devlin","key":"2021091617271734700_ref17"},{"volume-title":"ERNIE: Enhanced representation through knowledge\n                        integration","year":"2019","author":"Sun","key":"2021091617271734700_ref18"},{"volume-title":"Pre-training with whole word masking for Chinese BERT","year":"2019","author":"Cui","key":"2021091617271734700_ref19"},{"volume-title":"BERT\n                        for joint intent classification and slot filling","year":"2019","author":"Chen","key":"2021091617271734700_ref20"},{"volume-title":"Vector projection network for few-shot slot tagging in natural\n                        language understanding","year":"2020","author":"Zhu","key":"2021091617271734700_ref21"}],"container-title":["Data Intelligence"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/direct.mit.edu\/dint\/article-pdf\/3\/2\/274\/1963473\/dint_a_00090.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"syndication"},{"URL":"http:\/\/direct.mit.edu\/dint\/article-pdf\/3\/2\/274\/1963473\/dint_a_00090.pdf","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,3,14]],"date-time":"2025-03-14T07:41:10Z","timestamp":1741938070000},"score":1,"resource":{"primary":{"URL":"https:\/\/www.sciengine.com\/doi\/10.1162\/dint_a_00090"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2021]]},"references-count":21,"journal-issue":{"issue":"2","published-print":{"date-parts":[[2021,6,2]]}},"URL":"https:\/\/doi.org\/10.1162\/dint_a_00090","relation":{},"ISSN":["2641-435X"],"issn-type":[{"type":"electronic","value":"2641-435X"}],"subject":[],"published-other":{"date-parts":[[2021]]},"published":{"date-parts":[[2021]]}}}