{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,2,17]],"date-time":"2026-02-17T03:19:00Z","timestamp":1771298340591,"version":"3.50.1"},"publisher-location":"New York, NY, USA","reference-count":39,"publisher":"ACM","license":[{"start":{"date-parts":[[2022,4,25]],"date-time":"2022-04-25T00:00:00Z","timestamp":1650844800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"name":"R&D programme of Madrid","award":["P2018\/TCS-4314"],"award-info":[{"award-number":["P2018\/TCS-4314"]}]},{"name":"Spanish Ministry of Science","award":["RTI2018-095255-B-I00"],"award-info":[{"award-number":["RTI2018-095255-B-I00"]}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2022,4,25]]},"DOI":"10.1145\/3477314.3507255","type":"proceedings-article","created":{"date-parts":[[2022,5,7]],"date-time":"2022-05-07T00:37:36Z","timestamp":1651883856000},"page":"1491-1498","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":10,"title":["Automating the measurement of heterogeneous chatbot designs"],"prefix":"10.1145","author":[{"given":"Pablo C.","family":"Ca\u00f1izares","sequence":"first","affiliation":[{"name":"Universidad Aut\u00f3noma de Madrid, Madrid, Spain"}]},{"given":"Sara","family":"P\u00e9rez-Soler","sequence":"additional","affiliation":[{"name":"Universidad Aut\u00f3noma de Madrid, Madrid, Spain"}]},{"given":"Esther","family":"Guerra","sequence":"additional","affiliation":[{"name":"Universidad Aut\u00f3noma de Madrid, Madrid, Spain"}]},{"given":"Juan","family":"de Lara","sequence":"additional","affiliation":[{"name":"Universidad Aut\u00f3noma de Madrid, Madrid, Spain"}]}],"member":"320","published-online":{"date-parts":[[2022,5,6]]},"reference":[{"key":"e_1_3_2_1_1_1","unstructured":"ISO 9241-11. 1998. Ergonomic requirements for office work with visual display terminals (VDTs). Part II guidance on usability."},{"key":"e_1_3_2_1_2_1","volume-title":"Tensorflow: A system for large-scale machine learning","author":"Abadi M.","year":"2016","unstructured":"M. Abadi, P. Barham, J. Chen, Z. Chen, A. Davis, J. Dean, M. Devin, S. Ghemawat, G. Irving, M. Isard, M. Kudlur, J. Levenberg, R. Monga, S. Moore, D. Gordon Murray, B. Steiner, P. A. Tucker, V. Vasudevan, P. Warden, M. Wicke, Y. Yu, and X. Zheng. 2016. Tensorflow: A system for large-scale machine learning. In OSDI. USENIX Association, 265--283."},{"key":"e_1_3_2_1_3_1","volume-title":"d.]. https:\/\/www.botium.ai\/. last access","year":"2021","unstructured":"Botium. [n. d.]. https:\/\/www.botium.ai\/. last access in 2021."},{"key":"e_1_3_2_1_4_1","volume-title":"ICTSS (LNCS","volume":"55","author":"Bozic J.","unstructured":"J. Bozic and F. Wotawa. 2019. Testing chatbots using metamorphic relations. In ICTSS (LNCS, Vol. 11812). Springer, 41--55."},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-58793-2_34"},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.jml.2019.104047"},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"crossref","unstructured":"J. Cabot L. Burgue\u00f1o R. Claris\u00f3 G. Daniel J. Perianez-Pascual and R. Rodr\u00edguez-Echeverr\u00eda. 2021. Testing challenges for NLP-intensive bots. In BotSE. IEEE 31--34.","DOI":"10.1109\/BotSE52550.2021.00014"},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"crossref","unstructured":"D. Cer Y. Yang S.-Y. Kong N. Hua N. Limtiaco R. St. John N. Constant M. Guajardo-C\u00e9spedes S. Yuan C. Tar Y.-H. Sung B. Strope and R. Kurzweil. 2018. Universal sentence encoder. arXiv preprint arXiv:1803.11175 (2018) 7.","DOI":"10.18653\/v1\/D18-2029"},{"key":"e_1_3_2_1_9_1","volume-title":"d.]. https:\/\/chatbottest.com\/. last access","year":"2021","unstructured":"Chatbottest. [n. d.]. https:\/\/chatbottest.com\/. last access in 2021."},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.1515\/text-2014-0018"},{"key":"e_1_3_2_1_11_1","volume-title":"d.]. https:\/\/djl.ai\/. last access","year":"2021","unstructured":"Deep Java Library. [n. d.]. https:\/\/djl.ai\/. last access in 2021."},{"key":"e_1_3_2_1_12_1","volume-title":"d.]. https:\/\/dialogflow.com\/. last access","year":"2021","unstructured":"Dialogflow. [n. d.]. https:\/\/dialogflow.com\/. last access in 2021."},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"crossref","unstructured":"M. B. dos Santos A. P. C. C. Furtado S. C. Nogueira and D. D. Moreira. 2020. OggyBug: A test automation tool in chatbots. In SAST. ACM 79--87.","DOI":"10.1145\/3425174.3425230"},{"key":"e_1_3_2_1_14_1","unstructured":"N. E. Fenton and S. Lawrence Pfleeger. 1996. Software metrics - a practical and rigorous approach (2. ed.). International Thomson."},{"key":"e_1_3_2_1_15_1","volume-title":"d.]. https:\/\/flowxo.com\/. last access","author":"XO.","year":"2021","unstructured":"FlowXO. [n. d.]. https:\/\/flowxo.com\/. last access in 2021."},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"crossref","unstructured":"J. Jiang and N. Ahuja. 2020. Response quality in human-chatbot collaborative systems. In SIGIR. ACM 1545--1548.","DOI":"10.1145\/3397271.3401234"},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.1109\/MS.2017.4541027"},{"key":"e_1_3_2_1_18_1","volume-title":"d.]. https:\/\/aws.amazon.com\/en\/lex\/. last access","year":"2021","unstructured":"Lex. [n. d.]. https:\/\/aws.amazon.com\/en\/lex\/. last access in 2021."},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"crossref","unstructured":"C.-W. Liu R. Lowe I. Serban M. Noseworthy L. Charlin and J. Pineau. 2016. How NOT to evaluate your dialogue system: An empirical study of unsupervised evaluation metrics for dialogue response generation. In EMNLP. ACL 2122--2132.","DOI":"10.18653\/v1\/D16-1230"},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"crossref","unstructured":"C. D. Manning M. Surdeanu J. Bauer J. Finkel S. J. Bethard and D. McClosky. 2014. The Stanford CoreNLP natural language processing toolkit. In ACL: System Demonstrations. 55--60.","DOI":"10.3115\/v1\/P14-5010"},{"key":"e_1_3_2_1_21_1","volume-title":"d.]. https:\/\/dev.botframework.com\/. last access","year":"2021","unstructured":"Microsoft Bot Framework. [n. d.]. https:\/\/dev.botframework.com\/. last access in 2021."},{"key":"e_1_3_2_1_22_1","volume-title":"Memo: Towards automatic usability evaluation of spoken dialogue services by user error simulations. In ICSLP. ISCA.","author":"M\u00f6ller S.","year":"2006","unstructured":"S. M\u00f6ller, R. Englert, K.-P. Engelbrecht, V. V. Hafner, A. Jameson, A. Oulasvirta, A. Raake, and N. Reithinger. 2006. Memo: Towards automatic usability evaluation of spoken dialogue services by user error simulations. In ICSLP. ISCA."},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"crossref","unstructured":"R. J. Moore and R. Arar. 2018. Conversational UX Design: An Introduction. In Studies in Conversational UX Design. Springer 1--16.","DOI":"10.1007\/978-3-319-95579-7_1"},{"key":"e_1_3_2_1_24_1","unstructured":"R. J. Moore and R. Arar. 2019. Conversational UX Design: A Practitioner's Guide to the Natural Conversation Framework. ACM New York NY USA."},{"key":"e_1_3_2_1_25_1","first-page":"1","article-title":"Design systems for conversational UX","volume":"45","author":"Moore R. J.","year":"2020","unstructured":"R. J. Moore, E. Young Liu, S. Mishra, and G.-J. Ren. 2020. Design systems for conversational UX. In CUI. ACM, 45:1--45:4.","journal-title":"CUI. ACM"},{"key":"e_1_3_2_1_26_1","volume-title":"d.]. https:\/\/home.pandorabots.com\/. last access","year":"2021","unstructured":"Pandorabots. [n. d.]. https:\/\/home.pandorabots.com\/. last access in 2021."},{"key":"e_1_3_2_1_27_1","volume-title":"Chatbot evaluation metrics: Review paper","author":"Peras D.","unstructured":"D. Peras. 2018. Chatbot evaluation metrics: Review paper. In ESD. Varazdin Development and Entrepreneurship Agency, 89--97."},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-62522-1_15"},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.1109\/MS.2020.3030198"},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"crossref","unstructured":"E. Pitler and A. Nenkova. 2008. Revisiting readability: A unified framework for predicting text quality. In EMNLP. ACL 186--195.","DOI":"10.3115\/1613715.1613742"},{"key":"e_1_3_2_1_31_1","unstructured":"N. M. Radziwill and M. C. Benton. 2017. Evaluating quality of chatbots and intelligent conversational agents. (2017). http:\/\/arxiv.org\/abs\/1704.04579"},{"key":"e_1_3_2_1_32_1","volume-title":"d.]. https:\/\/rasa.com\/. last access","year":"2021","unstructured":"Rasa. [n. d.]. https:\/\/rasa.com\/. last access in 2021."},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"publisher","DOI":"10.1142\/S0218194019400163"},{"key":"e_1_3_2_1_34_1","volume-title":"Sequence Organization in Interaction","author":"Schegloff E. A.","unstructured":"E. A. Schegloff. 2007. Sequence Organization in Interaction. Cambridge University Press."},{"key":"e_1_3_2_1_35_1","volume-title":"Chateval: A tool for chatbot evaluation. In NAACL-HLT (Demonstrations). ACL, 60--65.","author":"Sedoc J.","year":"2019","unstructured":"J. Sedoc, D. Ippolito, A. Kirubarajan, J. Thirani, L. Ungar, and C. Callison-Burch. 2019. Chateval: A tool for chatbot evaluation. In NAACL-HLT (Demonstrations). ACL, 60--65."},{"key":"e_1_3_2_1_36_1","unstructured":"A. Shevat. 2017. Designing bots: Creating conversational experiences. O'Reilly."},{"key":"e_1_3_2_1_37_1","volume-title":"PARADISE: A frame-work for evaluating spoken dialogue agents","author":"Walker M. A.","year":"1997","unstructured":"M. A. Walker, D. J. Litman, C. A. Kamm, and A. Abella. 1997. PARADISE: A frame-work for evaluating spoken dialogue agents. In ACL\/EACL. Morgan Kaufmann Publishers \/ ACL, 271--280."},{"key":"e_1_3_2_1_38_1","volume-title":"d.]. https:\/\/www.ibm.com\/cloud\/watson-assistant\/. last access","year":"2021","unstructured":"Watson. [n. d.]. https:\/\/www.ibm.com\/cloud\/watson-assistant\/. last access in 2021."},{"key":"e_1_3_2_1_39_1","doi-asserted-by":"publisher","DOI":"10.1145\/365153.365168"}],"event":{"name":"SAC '22: The 37th ACM\/SIGAPP Symposium on Applied Computing","location":"Virtual Event","acronym":"SAC '22","sponsor":["SIGAPP ACM Special Interest Group on Applied Computing"]},"container-title":["Proceedings of the 37th ACM\/SIGAPP Symposium on Applied Computing"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3477314.3507255","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3477314.3507255","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T19:30:29Z","timestamp":1750188629000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3477314.3507255"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2022,4,25]]},"references-count":39,"alternative-id":["10.1145\/3477314.3507255","10.1145\/3477314"],"URL":"https:\/\/doi.org\/10.1145\/3477314.3507255","relation":{},"subject":[],"published":{"date-parts":[[2022,4,25]]},"assertion":[{"value":"2022-05-06","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}