{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,12,2]],"date-time":"2025-12-02T15:08:31Z","timestamp":1764688111558,"version":"3.41.0"},"publisher-location":"New York, NY, USA","reference-count":34,"publisher":"ACM","license":[{"start":{"date-parts":[[2024,4,15]],"date-time":"2024-04-15T00:00:00Z","timestamp":1713139200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"name":"Spanish MICINN","award":["PID2021-122270OB-I00","TED2021-129381B-C21","RED2022-134647-T"],"award-info":[{"award-number":["PID2021-122270OB-I00","TED2021-129381B-C21","RED2022-134647-T"]}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2024,4,15]]},"DOI":"10.1145\/3644032.3644456","type":"proceedings-article","created":{"date-parts":[[2024,6,10]],"date-time":"2024-06-10T15:00:25Z","timestamp":1718031625000},"page":"23-33","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":6,"title":["Coverage-based Strategies for the Automated Synthesis of Test Scenarios for Conversational Agents"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-2084-1558","authenticated-orcid":false,"given":"Pablo C.","family":"Canizares","sequence":"first","affiliation":[{"name":"Departamento de Ingenier\u00eda Inform\u00e1tica, Universidad Aut\u00f3noma de Madrid, Madrid, Spain"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-7371-6104","authenticated-orcid":false,"given":"Daniel","family":"\u00c1vila","sequence":"additional","affiliation":[{"name":"Departamento de Ingenier\u00eda Inform\u00e1tica, Universidad Aut\u00f3noma de Madrid, Madrid, Spain"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-4558-7111","authenticated-orcid":false,"given":"Sara","family":"Perez-Soler","sequence":"additional","affiliation":[{"name":"Departamento de Ingenier\u00eda Inform\u00e1tica, Universidad Aut\u00f3noma de Madrid, Madrid, Spain"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-2818-2278","authenticated-orcid":false,"given":"Esther","family":"Guerra","sequence":"additional","affiliation":[{"name":"Departamento de Ingenier\u00eda Inform\u00e1tica, Universidad Aut\u00f3noma de Madrid, Madrid, Spain"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-9425-6362","authenticated-orcid":false,"given":"Juan","family":"De Lara","sequence":"additional","affiliation":[{"name":"Departamento de Ingenier\u00eda Inform\u00e1tica, Universidad Autonoma de Madrid, Madrid, Spain"}]}],"member":"320","published-online":{"date-parts":[[2024,6,10]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.mlwa.2020.100006"},{"key":"e_1_3_2_1_2_1","volume-title":"Artificial Hallucinations in Chat-GPT: Implications in Scientific Writing. Cur\u0113us 15, 2","author":"Alkaissi Hussam","year":"2023","unstructured":"Hussam Alkaissi and Samy I McFarlane. 2023. Artificial Hallucinations in Chat-GPT: Implications in Scientific Writing. Cur\u0113us 15, 2 (2023), 4 pages."},{"key":"e_1_3_2_1_3_1","volume-title":"Testing Non-Testable Programs Using Association Rules. In 3rd ACM\/IEEE International Conference on Automation of Software Test. ACM\/IEEE, 87--91","author":"Bertolino Antonia","year":"2022","unstructured":"Antonia Bertolino, Emilio Cruciani, Breno Miranda, and Roberto Verdecchia. 2022. Testing Non-Testable Programs Using Association Rules. In 3rd ACM\/IEEE International Conference on Automation of Software Test. ACM\/IEEE, 87--91."},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.1007\/s11219-020-09544-9"},{"key":"e_1_3_2_1_5_1","volume-title":"Chatbot Testing Using AI Planning. In 2019 IEEE International Conference on Artificial Intelligence Testing (AITest). IEEE, 37--44","author":"Bozic Josip","year":"2019","unstructured":"Josip Bozic, Oliver A. Tazl, and Franz Wotawa. 2019. Chatbot Testing Using AI Planning. In 2019 IEEE International Conference on Artificial Intelligence Testing (AITest). IEEE, 37--44."},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-31280-0_3"},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"crossref","unstructured":"M. Brambilla J. Cabot and M. Wimmer. 2017. Model-Driven Software Engineering in Practice Second Edition. Morgan & Claypool Publishers.","DOI":"10.1007\/978-3-031-02549-5"},{"key":"e_1_3_2_1_8_1","volume-title":"Testing Chatbots with Charm. In 13th Int. Conf. on Quality of Information and Communications Technology (CCIS","volume":"438","author":"Bravo-Santos Sergio","year":"2020","unstructured":"Sergio Bravo-Santos, Esther Guerra, and Juan de Lara. 2020. Testing Chatbots with Charm. In 13th Int. Conf. on Quality of Information and Communications Technology (CCIS, Vol. 1266). Springer, 426--438."},{"key":"e_1_3_2_1_9_1","volume-title":"Testing Challenges for NLP-intensive Bots. In 3rd IEEE\/ACM International Workshop on Bots in Software Engineering (BotSE@ICSE). IEEE, 31--34","author":"Cabot Jordi","year":"2021","unstructured":"Jordi Cabot, Loli Burgue\u00f1o, Robert Claris\u00f3, Gwendal Daniel, Jorge Perianez-Pascual, and Roberto Rodr\u00edguez-Echeverr\u00eda. 2021. Testing Challenges for NLP-intensive Bots. In 3rd IEEE\/ACM International Workshop on Bots in Software Engineering (BotSE@ICSE). IEEE, 31--34."},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","unstructured":"P. C. Ca\u00f1izares J. M. L\u00f3pez-Morales S. P\u00e9rez-Soler E. Guerra and J. de Lara. 2023. Measuring and clustering heterogeneous chatbot designs. ACM Trans. Softw. Eng. Methodol. (2023) 42 pages. 10.1145\/3637228","DOI":"10.1145\/3637228"},{"key":"e_1_3_2_1_11_1","first-page":"34","article-title":"Hints on Test Data Selection","volume":"11","author":"DeMillo Richard A.","year":"1978","unstructured":"Richard A. DeMillo, Richard J. Lipton, and Frederick G. Sayward. 1978. Hints on Test Data Selection: Help for the Practicing Programmer. Computer 11, 4 (1978), 34--41.","journal-title":"Help for the Practicing Programmer. Computer"},{"volume-title":"Web Intelligence","author":"Doherty Dana","key":"e_1_3_2_1_12_1","unstructured":"Dana Doherty and Kevin Curran. 2019. Chatbots for Online Banking Services. In Web Intelligence, Vol. 17. IOS Press, 327--342."},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1007\/s41233-021-00046-5"},{"key":"e_1_3_2_1_14_1","unstructured":"European Committee for Electrotechnical Standardization. 2001. EN 50128: Railway Applications-Communication Signalling and Processing Systems-Software for Railway Control and Protection Systems. Standard."},{"key":"e_1_3_2_1_16_1","volume-title":"Democratizing Chatbot Debugging: A Computational Framework for Evaluating and Explaining Inappropriate Chatbot Responses. In 5th International Conference on Conversational User Interfaces. ACM","author":"Han Xu","year":"2023","unstructured":"Xu Han, Michelle Zhou, Yichen Wang, Wenxi Chen, and Tom Yeh. 2023. Democratizing Chatbot Debugging: A Computational Framework for Evaluating and Explaining Inappropriate Chatbot Responses. In 5th International Conference on Conversational User Interfaces. ACM, New York, NY, USA, Article 39, 7 pages."},{"volume-title":"AEON: A Method for Automatic Evaluation of NLP Test Cases. In 31th ACM SIGSOFT International Symposium on Software Testing and Analysis (ISSTA). ACM","author":"Zhang Jianping","key":"e_1_3_2_1_17_1","unstructured":"Jen-tse Huang, Jianping Zhang, Wenxuan Wang, Pinjia He, Yuxin Su, and Michael R. Lyu. 2022. AEON: A Method for Automatic Evaluation of NLP Test Cases. In 31th ACM SIGSOFT International Symposium on Software Testing and Analysis (ISSTA). ACM, New York, NY, USA, 202--214."},{"key":"e_1_3_2_1_18_1","first-page":"11","article-title":"DO-178B: Software Considerations in Airborne Systems and Equipment Certification","volume":"199","author":"Johnson Leslie A.","year":"1998","unstructured":"Leslie A. Johnson. 1998. DO-178B: Software Considerations in Airborne Systems and Equipment Certification. CrossTalk 199 (1998), 11--20.","journal-title":"CrossTalk"},{"key":"e_1_3_2_1_19_1","volume-title":"DialTest: Automated Testing for Recurrent-Neural-Network-Driven Dialogue Systems. In 30th ACM SIGSOFT International Symposium on Software Testing and Analysis (ISSTA). ACM, 115--126","author":"Liu Zixi","year":"2021","unstructured":"Zixi Liu, Yang Feng, and Zhenyu Chen. 2021. DialTest: Automated Testing for Recurrent-Neural-Network-Driven Dialogue Systems. In 30th ACM SIGSOFT International Symposium on Software Testing and Analysis (ISSTA). ACM, 115--126."},{"key":"e_1_3_2_1_20_1","volume-title":"Asymob: A Platform for Measuring and Clustering Chatbots. In 44th IEEE\/ACM International Conference on Software Engineering: Companion Proceedings (ICSE Companion). ACM\/IEEE, 16--20","author":"L\u00f3pez-Morales Jose Mar\u00eda","year":"2022","unstructured":"Jose Mar\u00eda L\u00f3pez-Morales, Pablo C. Ca\u00f1izares, Sara P\u00e9rez-Soler, Esther Guerra, and Juan de Lara. 2022. Asymob: A Platform for Measuring and Clustering Chatbots. In 44th IEEE\/ACM International Conference on Software Engineering: Companion Proceedings (ICSE Companion). ACM\/IEEE, 16--20."},{"key":"e_1_3_2_1_21_1","volume-title":"Sapienz: Multi-objective Automated Testing for Android Applications. In 25th International Symposium on Software Testing and Analysis. ACM, 94--105","author":"Mao Ke","year":"2016","unstructured":"Ke Mao, Mark Harman, and Yue Jia. 2016. Sapienz: Multi-objective Automated Testing for Android Applications. In 25th International Symposium on Software Testing and Analysis. ACM, 94--105. See also: https:\/\/engineering.fb.com\/2018\/05\/02\/developer-tools\/sapienz-intelligent-automated-software-testing-at-scale\/."},{"key":"e_1_3_2_1_22_1","volume-title":"Taming Google-Scale Continuous Testing. In 39th International Conference on Software Engineering: Software Engineering in Practice Track (ICSE-SEIP). IEEE, 233--242","author":"Memon Atif M.","year":"2017","unstructured":"Atif M. Memon, Zebao Gao, Bao N. Nguyen, Sanjeev Dhanda, Eric Nickell, Rob Siemborski, and John Micco. 2017. Taming Google-Scale Continuous Testing. In 39th International Conference on Software Engineering: Software Engineering in Practice Track (ICSE-SEIP). IEEE, 233--242."},{"key":"e_1_3_2_1_23_1","volume-title":"Article 11 (feb","author":"Nie Changhai","year":"2011","unstructured":"Changhai Nie and Hareton Leung. 2011. A survey of combinatorial testing. ACM Comput. Surv. 43, 2, Article 11 (feb 2011), 29 pages."},{"key":"e_1_3_2_1_24_1","unstructured":"OpenAI. (last accessed in 2023). https:\/\/openai.com\/research\/gpt-4#limitations."},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.1049\/cp.2011.0251"},{"key":"e_1_3_2_1_26_1","volume-title":"Model-Driven Chatbot Development. In 39th International Conference on Conceptual Modeling (ER) (LNCS","volume":"222","author":"P\u00e9rez-Soler Sara","year":"2020","unstructured":"Sara P\u00e9rez-Soler, Esther Guerra, and Juan de Lara. 2020. Model-Driven Chatbot Development. In 39th International Conference on Conceptual Modeling (ER) (LNCS, Vol. 12400). Springer, 207--222."},{"key":"e_1_3_2_1_27_1","volume-title":"Creating and Migrating Chatbots with Conga. In 43rd IEEE\/ACM International Conference on Software Engineering: Companion Proceedings (ICSE Companion). IEEE, 37--40","author":"P\u00e9rez-Soler Sara","year":"2021","unstructured":"Sara P\u00e9rez-Soler, Esther Guerra, and Juan de Lara. 2021. Creating and Migrating Chatbots with Conga. In 43rd IEEE\/ACM International Conference on Software Engineering: Companion Proceedings (ICSE Companion). IEEE, 37--40."},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"publisher","DOI":"10.1142\/S0218194019400163"},{"key":"e_1_3_2_1_29_1","volume-title":"Chateval: A Tool for Chatbot Evaluation. In 2019 Conference of the North American","author":"Sedoc Jo\u00e3o","year":"2019","unstructured":"Jo\u00e3o Sedoc, Daphne Ippolito, Arun Kirubarajan, Jai Thirani, Lyle H. Ungar, and Chris Callison-Burch. 2019. Chateval: A Tool for Chatbot Evaluation. In 2019 Conference of the North American Chapter of the Association for Computational Linguistics (Demonstrations). Association for Computational Linguistics, 60--65."},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"publisher","DOI":"10.3897\/jucs.91311"},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.infsof.2022.107145"},{"key":"e_1_3_2_1_32_1","doi-asserted-by":"publisher","DOI":"10.1145\/3160504.3160584"},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"publisher","DOI":"10.1109\/AST52587.2021.00023"},{"key":"e_1_3_2_1_34_1","volume-title":"AI-Driven Conversational Bot Test Automation Using Industry Specific Data Cartridges. In IEEE\/ACM 1st International Conference on Automation of Software Test (AST). ACM","author":"Yalla Muralidhar","year":"2020","unstructured":"Muralidhar Yalla and Asha Sunil. 2020. AI-Driven Conversational Bot Test Automation Using Industry Specific Data Cartridges. In IEEE\/ACM 1st International Conference on Automation of Software Test (AST). ACM, New York, NY, USA, 105--107."},{"key":"e_1_3_2_1_35_1","doi-asserted-by":"publisher","DOI":"10.1145\/3563657.3596138"}],"event":{"name":"AST '24: 5th ACM\/IEEE International Conference on Automation of Software Test (AST 2024)","sponsor":["SIGSOFT ACM Special Interest Group on Software Engineering","IEEE TCSE"],"location":"Lisbon Portugal","acronym":"AST '24"},"container-title":["Proceedings of the 5th ACM\/IEEE International Conference on Automation of Software Test (AST 2024)"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3644032.3644456","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3644032.3644456","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,18]],"date-time":"2025-06-18T23:56:58Z","timestamp":1750291018000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3644032.3644456"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,4,15]]},"references-count":34,"alternative-id":["10.1145\/3644032.3644456","10.1145\/3644032"],"URL":"https:\/\/doi.org\/10.1145\/3644032.3644456","relation":{},"subject":[],"published":{"date-parts":[[2024,4,15]]},"assertion":[{"value":"2024-06-10","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}