{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,8,22]],"date-time":"2025-08-22T18:40:09Z","timestamp":1755888009241,"version":"3.44.0"},"publisher-location":"New York, NY, USA","reference-count":52,"publisher":"ACM","license":[{"start":{"date-parts":[[2024,7,8]],"date-time":"2024-07-08T00:00:00Z","timestamp":1720396800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2024,7,8]]},"DOI":"10.1145\/3640794.3665553","type":"proceedings-article","created":{"date-parts":[[2024,7,7]],"date-time":"2024-07-07T06:24:56Z","timestamp":1720333496000},"page":"1-15","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":2,"title":["Towards Interactive Guidance for Writing Training Utterances for Conversational Agents"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-6740-4902","authenticated-orcid":false,"given":"David","family":"Piorkowski","sequence":"first","affiliation":[{"name":"IBM Research, United States"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-5491-7656","authenticated-orcid":false,"given":"Rachel","family":"Ostrand","sequence":"additional","affiliation":[{"name":"IBM Research, United States"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-5616-9567","authenticated-orcid":false,"given":"Kristina","family":"Brimijoin","sequence":"additional","affiliation":[{"name":"IBM Research, United States"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-2368-0099","authenticated-orcid":false,"given":"Jessica","family":"He","sequence":"additional","affiliation":[{"name":"IBM Research, United States"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0009-0006-9782-7734","authenticated-orcid":false,"given":"Erica","family":"Albert","sequence":"additional","affiliation":[{"name":"IBM, United States"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-0246-2183","authenticated-orcid":false,"given":"Stephanie","family":"Houde","sequence":"additional","affiliation":[{"name":"IBM Research, United States"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2024,7,8]]},"reference":[{"key":"e_1_3_2_2_1_1","doi-asserted-by":"publisher","DOI":"10.1609\/aimag.v35i4.2513"},{"key":"e_1_3_2_2_2_1","unstructured":"Ashley Belanger. [n. d.]. Air Canada must honor refund policy invented by airline\u2019s chatbot. ArsTechnica ([n. d.]). https:\/\/arstechnica.com\/tech-policy\/2024\/02\/air-canada-must-honor-refund-policy-invented-by-airlines-chatbot\/"},{"key":"e_1_3_2_2_3_1","unstructured":"Adam Benvie Eric Wayne and Arnold. Matthew. 2020. Watson Assistant Continuous Improvement Best Practices. https:\/\/www.ibm.com\/downloads\/cas\/V0XQ0ZRE Accessed: 2023-09-04."},{"key":"e_1_3_2_2_4_1","volume-title":"Selection of relevant features and examples in machine learning. Artificial intelligence 97, 1-2","author":"Blum L","year":"1997","unstructured":"Avrim\u00a0L Blum and Pat Langley. 1997. Selection of relevant features and examples in machine learning. Artificial intelligence 97, 1-2 (1997), 245\u2013271."},{"key":"e_1_3_2_2_5_1","doi-asserted-by":"publisher","DOI":"10.1145\/3555768"},{"key":"e_1_3_2_2_6_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.nlp4convai-1.5"},{"key":"e_1_3_2_2_7_1","volume-title":"Large language models for text classification: From zero-shot learning to fine-tuning","author":"Chae Youngjin","year":"2023","unstructured":"Youngjin Chae and Thomas Davidson. 2023. Large language models for text classification: From zero-shot learning to fine-tuning. Open Science Foundation (2023)."},{"key":"e_1_3_2_2_8_1","doi-asserted-by":"publisher","DOI":"10.1109\/SLT.2016.7846288"},{"key":"e_1_3_2_2_9_1","doi-asserted-by":"publisher","DOI":"10.1515\/text-2014-0018"},{"key":"e_1_3_2_2_10_1","doi-asserted-by":"publisher","DOI":"10.48550\/arXiv.2302.13007"},{"key":"e_1_3_2_2_11_1","doi-asserted-by":"publisher","DOI":"10.1002\/jocb.349"},{"key":"e_1_3_2_2_12_1","doi-asserted-by":"crossref","unstructured":"James Glass Eugene Weinstein Scott Cyphers Joseph Polifroni Grace Chung and Mikio Nakano. 2005. A framework for developing conversational user interfaces. In Computer-Aided Design of User Interfaces IV: Proceedings of the Fifth International Conference on Computer-Aided Design of User Interfaces CADUI\u20192004 Sponsored by ACM and jointly organised with the Eight ACM International Conference on Intelligent User Interfaces IUI\u20192004 13\u201316 January 2004 Funchal Isle of Madeira. Springer 349\u2013360.","DOI":"10.1007\/1-4020-3304-4_28"},{"key":"e_1_3_2_2_13_1","first-page":"195","article-title":"A black-box approach for response quality evaluation of conversational agent systems","volume":"3","author":"Goh Ong\u00a0Sing","year":"2007","unstructured":"Ong\u00a0Sing Goh, Cemal Ardil, Wilson Wong, and Chun\u00a0Che Fung. 2007. A black-box approach for response quality evaluation of conversational agent systems. International Journal of Computational Intelligence 3, 3 (2007), 195\u2013203.","journal-title":"International Journal of Computational Intelligence"},{"key":"e_1_3_2_2_14_1","volume-title":"Fast and scalable expansion of natural language understanding functionality for intelligent agents. arXiv preprint arXiv:1805.01542","author":"Goyal Anuj","year":"2018","unstructured":"Anuj Goyal, Angeliki Metallinou, and Spyros Matsoukas. 2018. Fast and scalable expansion of natural language understanding functionality for intelligent agents. arXiv preprint arXiv:1805.01542 (2018)."},{"key":"e_1_3_2_2_15_1","doi-asserted-by":"publisher","DOI":"10.1145\/3411764.3445569"},{"key":"e_1_3_2_2_16_1","volume-title":"Learning from dialogue after deployment: Feed yourself, chatbot!arXiv preprint arXiv:1901.05415","author":"Hancock Braden","year":"2019","unstructured":"Braden Hancock, Antoine Bordes, Pierre-Emmanuel Mazare, and Jason Weston. 2019. Learning from dialogue after deployment: Feed yourself, chatbot!arXiv preprint arXiv:1901.05415 (2019)."},{"key":"e_1_3_2_2_17_1","volume-title":"ConveRT: Efficient and accurate conversational representations from transformers. arXiv preprint arXiv:1911.03688","author":"Henderson Matthew","year":"2019","unstructured":"Matthew Henderson, I\u00f1igo Casanueva, Nikola Mrk\u0161i\u0107, Pei-Hao Su, Tsung-Hsien Wen, and Ivan Vuli\u0107. 2019. ConveRT: Efficient and accurate conversational representations from transformers. arXiv preprint arXiv:1911.03688 (2019)."},{"key":"e_1_3_2_2_18_1","doi-asserted-by":"publisher","DOI":"10.1145\/3313831.3376177"},{"key":"e_1_3_2_2_19_1","doi-asserted-by":"publisher","DOI":"10.1145\/3491102.3502116"},{"key":"e_1_3_2_2_20_1","volume-title":"Selective in-context data augmentation for intent detection using pointwise v-information. arXiv preprint arXiv:2302.05096","author":"Lin Yen-Ting","year":"2023","unstructured":"Yen-Ting Lin, Alexandros Papangelis, Seokhwan Kim, Sungjin Lee, Devamanyu Hazarika, Mahdi Namazifar, Di Jin, Yang Liu, and Dilek Hakkani-Tur. 2023. Selective in-context data augmentation for intent detection using pointwise v-information. arXiv preprint arXiv:2302.05096 (2023)."},{"key":"e_1_3_2_2_21_1","doi-asserted-by":"publisher","DOI":"10.1145\/3560815"},{"key":"e_1_3_2_2_22_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSSSM.2007.4280175"},{"key":"e_1_3_2_2_23_1","doi-asserted-by":"publisher","DOI":"10.1145\/3604237.3626891"},{"key":"e_1_3_2_2_24_1","doi-asserted-by":"publisher","DOI":"10.1145\/3429360.3468203"},{"key":"e_1_3_2_2_25_1","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v33i01.33019528"},{"key":"e_1_3_2_2_26_1","volume-title":"Comparison of four primary methods for coordinating the interruption of people in human-computer interaction. Human-computer interaction 17, 1","author":"McFarlane C","year":"2002","unstructured":"Daniel\u00a0C McFarlane. 2002. Comparison of four primary methods for coordinating the interruption of people in human-computer interaction. Human-computer interaction 17, 1 (2002), 63\u2013139."},{"key":"e_1_3_2_2_27_1","volume-title":"Proceedings of the world congress on engineering, Vol.\u00a01. WCE London, 1\u20138.","author":"Meira O","year":"2015","unstructured":"M\u00a0de\u00a0O Meira and AM\u00a0de\u00a0P Canuto. 2015. Evaluation of emotional agents\u2019 architectures: an approach based on quality metrics and the influence of emotions on users. In Proceedings of the world congress on engineering, Vol.\u00a01. WCE London, 1\u20138."},{"key":"e_1_3_2_2_28_1","volume-title":"Joint Proceedings of the ACM IUI Workshops.","author":"Moilanen Joonas","year":"2022","unstructured":"Joonas Moilanen, Aku Visuri, Elina Kuosmanen, Andy Alorwu, and Simo Hosio. 2022. Designing personalities for mental health conversational agents. In Joint Proceedings of the ACM IUI Workshops."},{"key":"e_1_3_2_2_29_1","volume-title":"Is a prompt and a few samples all you need? Using GPT-4 for data augmentation in low-resource classification tasks. arXiv preprint arXiv:2304.13861","author":"M\u00f8ller Anders\u00a0Giovanni","year":"2023","unstructured":"Anders\u00a0Giovanni M\u00f8ller, Jacob\u00a0Aarup Dalsgaard, Arianna Pera, and Luca\u00a0Maria Aiello. 2023. Is a prompt and a few samples all you need? Using GPT-4 for data augmentation in low-resource classification tasks. arXiv preprint arXiv:2304.13861 (2023)."},{"key":"e_1_3_2_2_30_1","volume-title":"Gpt-3 models are poor few-shot learners in the biomedical domain. arXiv preprint arXiv:2109.02555","author":"Moradi Milad","year":"2021","unstructured":"Milad Moradi, Kathrin Blagec, Florian Haberl, and Matthias Samwald. 2021. Gpt-3 models are poor few-shot learners in the biomedical domain. arXiv preprint arXiv:2109.02555 (2021)."},{"key":"e_1_3_2_2_31_1","unstructured":"Kyle Orland. [n. d.]. NYC\u2019s government chatbot is lying about city laws and regulations. ArsTechnica ([n. d.]). https:\/\/arstechnica.com\/ai\/2024\/03\/nycs-government-chatbot-is-lying-about-city-laws-and-regulations\/"},{"key":"e_1_3_2_2_32_1","volume-title":"Training language models to follow instructions with human feedback. Advances in neural information processing systems 35","author":"Ouyang Long","year":"2022","unstructured":"Long Ouyang, Jeffrey Wu, Xu Jiang, Diogo Almeida, Carroll Wainwright, Pamela Mishkin, Chong Zhang, Sandhini Agarwal, Katarina Slama, Alex Ray, 2022. Training language models to follow instructions with human feedback. Advances in neural information processing systems 35 (2022), 27730\u201327744."},{"key":"e_1_3_2_2_33_1","doi-asserted-by":"publisher","DOI":"10.1109\/MS.2020.3030198"},{"key":"e_1_3_2_2_34_1","volume-title":"Zero-shot text classification with generative language models. arXiv preprint arXiv:1912.10165","author":"Puri Raul","year":"2019","unstructured":"Raul Puri and Bryan Catanzaro. 2019. Zero-shot text classification with generative language models. arXiv preprint arXiv:1912.10165 (2019)."},{"key":"e_1_3_2_2_35_1","doi-asserted-by":"publisher","DOI":"10.48550\/arXiv.1704.04579"},{"key":"e_1_3_2_2_36_1","article-title":"Performance, energy consumption, and costs: a comparative analysis of automatic text classification approaches in the Legal domain","volume":"13","author":"Rigutini Leonardo","year":"2024","unstructured":"Leonardo Rigutini, Achille Globo, Marco Stefanelli, Andrea Zugarini, Sinan Gultekin, and Marco Ernandes. 2024. Performance, energy consumption, and costs: a comparative analysis of automatic text classification approaches in the Legal domain. International Journal on Natural Language Computing (IJNLC) 13, 1 (2024).","journal-title":"International Journal on Natural Language Computing (IJNLC)"},{"key":"e_1_3_2_2_37_1","doi-asserted-by":"publisher","DOI":"10.1145\/985692.985729"},{"key":"e_1_3_2_2_38_1","unstructured":"Daniel Schlo\u00df. 2023. Towards Designing a NLU Model Improvement System for Customer Service Chatbots. (2023)."},{"key":"e_1_3_2_2_39_1","doi-asserted-by":"publisher","DOI":"10.1145\/3313831.3376708"},{"key":"e_1_3_2_2_40_1","doi-asserted-by":"publisher","DOI":"10.1145\/3491102.3501972"},{"key":"e_1_3_2_2_41_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D18-1072"},{"key":"e_1_3_2_2_42_1","doi-asserted-by":"publisher","DOI":"10.1145\/2675133.2675239"},{"volume-title":"Shaping conversations: Investigating how conversational agents are designed and developed. Ph.\u00a0D. Dissertation","author":"Sillard Annetta","key":"e_1_3_2_2_43_1","unstructured":"Annetta Sillard. 2022. Shaping conversations: Investigating how conversational agents are designed and developed. Ph.\u00a0D. Dissertation. Kth Royal Institute of Technology. https:\/\/urn.kb.se\/resolve?urn=urn:nbn:se:kth:diva-321528"},{"key":"e_1_3_2_2_44_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2022.bigscience-1.3"},{"key":"e_1_3_2_2_45_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.specom.2004.08.002"},{"key":"e_1_3_2_2_46_1","volume-title":"Quality aspects of bots. Software quality and software testing in internet times","author":"Vetter Michael","year":"2002","unstructured":"Michael Vetter. 2002. Quality aspects of bots. Software quality and software testing in internet times (2002), 165\u2013184."},{"key":"e_1_3_2_2_47_1","doi-asserted-by":"publisher","DOI":"10.1109\/SLT.2012.6424200"},{"key":"e_1_3_2_2_48_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.artint.2023.103953"},{"key":"e_1_3_2_2_49_1","volume-title":"Mouni Reddy, and Geoff Zweig.","author":"Williams D","year":"2015","unstructured":"Jason\u00a0D Williams, Nobal\u00a0B Niraula, Pradeep Dasigi, Aparna Lakshmiratan, Carlos Garcia\u00a0Jurado Suarez, Mouni Reddy, and Geoff Zweig. 2015. Rapidly scaling dialog systems with interactive learning. Natural language dialog systems and intelligent assistants (2015), 1\u201313."},{"key":"e_1_3_2_2_50_1","doi-asserted-by":"publisher","DOI":"10.1145\/642611.642665"},{"key":"e_1_3_2_2_51_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2022.naacl-main.399"},{"key":"e_1_3_2_2_52_1","volume-title":"A study of incorrect paraphrases in crowdsourced user utterances. NAACL\u201919","author":"Yaghoubzadeh Mohammadali","year":"2019","unstructured":"Mohammadali Yaghoubzadeh, Boualem Benatallah, M Chai\u00a0Barush, and Shayan Zamanirad. 2019. A study of incorrect paraphrases in crowdsourced user utterances. NAACL\u201919 (2019)."}],"event":{"name":"CUI '24: ACM Conversational User Interfaces 2024","sponsor":["SIGCHI ACM Special Interest Group on Computer-Human Interaction"],"location":"Luxembourg Luxembourg","acronym":"CUI '24"},"container-title":["ACM Conversational User Interfaces 2024"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3640794.3665553","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3640794.3665553","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,8,22]],"date-time":"2025-08-22T18:02:12Z","timestamp":1755885732000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3640794.3665553"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,7,8]]},"references-count":52,"alternative-id":["10.1145\/3640794.3665553","10.1145\/3640794"],"URL":"https:\/\/doi.org\/10.1145\/3640794.3665553","relation":{},"subject":[],"published":{"date-parts":[[2024,7,8]]},"assertion":[{"value":"2024-07-08","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}