{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,3,16]],"date-time":"2026-03-16T22:34:42Z","timestamp":1773700482013,"version":"3.50.1"},"publisher-location":"New York, NY, USA","reference-count":50,"publisher":"ACM","license":[{"start":{"date-parts":[[2022,5,16]],"date-time":"2022-05-16T00:00:00Z","timestamp":1652659200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2022,5,16]]},"DOI":"10.1145\/3522664.3528592","type":"proceedings-article","created":{"date-parts":[[2022,10,17]],"date-time":"2022-10-17T16:30:14Z","timestamp":1666024214000},"page":"22-32","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":2,"title":["Quality assurance of generative dialog models in an evolving conversational agent used for Swedish language practice"],"prefix":"10.1145","author":[{"given":"Markus","family":"Borg","sequence":"first","affiliation":[{"name":"RISE Research Institutes of Sweden, Lund, Sweden"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Johan","family":"Bengtsson","sequence":"additional","affiliation":[{"name":"Lund University, Lund, Sweden"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Harald","family":"\u00d6sterling","sequence":"additional","affiliation":[{"name":"Lund University, Lund, Sweden"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Alexander","family":"Hagelborn","sequence":"additional","affiliation":[{"name":"NordAxon AB, Malm\u00f6, Sweden"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Isabella","family":"Gagner","sequence":"additional","affiliation":[{"name":"NordAxon AB, Malm\u00f6, Sweden"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Piotr","family":"Tomaszewski","sequence":"additional","affiliation":[{"name":"RISE Research Institutes of Sweden, Lund, Sweden"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2022,10,17]]},"reference":[{"key":"e_1_3_2_1_1_1","unstructured":"2022. Emely Testing Repo. https:\/\/github.com\/JoohanBengtsson\/Emely-testing\/"},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.gloenvcha.2018.12.003"},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1080\/09528139508953799"},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.chb.2018.03.051"},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2014.2372785"},{"key":"e_1_3_2_1_6_1","volume-title":"Natural language processing with Python: Analyzing text with the natural language toolkit","author":"Bird Steven","unstructured":"Steven Bird, Ewan Klein, and Edward Loper. 2009. Natural language processing with Python: Analyzing text with the natural language toolkit. O'Reilly Media."},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.infsof.2016.03.008"},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.1080\/13670050.2017.1294557"},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-65854-0_6"},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-94238-0_1"},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.2991\/jase.d.190131.001"},{"key":"e_1_3_2_1_12_1","unstructured":"Tom B Brown Benjamin Mann Nick Ryder Melanie Subbiah et al. 2020. Language models are few-shot learners. arXiv preprint arXiv:2005.14165 (2020)."},{"key":"e_1_3_2_1_13_1","volume-title":"Aspects of the Theory of Syntax","author":"Chomsky Noam","unstructured":"Noam Chomsky. 2014. Aspects of the Theory of Syntax. Vol. 11. MIT press."},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2021.emnlp-main.599"},{"key":"e_1_3_2_1_15_1","volume-title":"Bert: Pre-training of deep bidirectional transformers for language understanding. arXiv preprint arXiv:1810.04805","author":"Devlin Jacob","year":"2018","unstructured":"Jacob Devlin, Ming-Wei Chang, Kenton Lee, and Kristina Toutanova. 2018. Bert: Pre-training of deep bidirectional transformers for language understanding. arXiv preprint arXiv:1810.04805 (2018)."},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.3389\/frobt.2019.00153"},{"key":"e_1_3_2_1_17_1","volume-title":"Contemporary Empirical Methods in Softw. Engineering","author":"Garousi Vahid","unstructured":"Vahid Garousi, Michael Felderer, Mika V M\u00e4ntyl\u00e4, and Austen Rainer. 2020. Benefitting from the grey literature in software engineering research. In Contemporary Empirical Methods in Softw. Engineering. Springer, 385--413."},{"key":"e_1_3_2_1_18_1","unstructured":"Laura Hanu and Unitary team. 2020. Detoxify. Github. https:\/\/github.com\/unitaryai\/detoxify."},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","DOI":"10.1145\/3463274.3463361"},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-15035-8_93"},{"key":"e_1_3_2_1_21_1","first-page":"2020","article-title":"Software and Systems Engineering --- Software Testing --- Part 1: Concepts and Definitions","volume":"29119","author":"IEEE.","year":"2013","unstructured":"ISO\/IEC\/IEEE. 2013. Software and Systems Engineering --- Software Testing --- Part 1: Concepts and Definitions. Technical Report ISO\/IEC\/IEEE CD 29119-1:2020. International Organization for Standardization.","journal-title":"Technical Report ISO\/IEC\/IEEE CD"},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE.2019.00108"},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.1109\/AST52587.2021.00017"},{"key":"e_1_3_2_1_24_1","volume-title":"Looking ahead: Future directions in, and future research into, second language acquisition. Foreign language annals 51, 1","author":"Larsen-Freeman Diane","year":"2018","unstructured":"Diane Larsen-Freeman. 2018. Looking ahead: Future directions in, and future research into, second language acquisition. Foreign language annals 51, 1 (2018), 55--72."},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.1145\/3460319.3464829"},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.1017\/S0958344019000065"},{"key":"e_1_3_2_1_27_1","unstructured":"Jes\u00fas Mart\u00edn Carlos Mu\u00f1oz-Romero and Nieves \u00c1balos. 2021. Chatbottest - The Free Guide For You to Understand What is Your Chatbot Doing Wrong. https:\/\/chatbottest.com\/. Accessed: 2021-12-27."},{"key":"e_1_3_2_1_28_1","first-page":"2019","article-title":"English language education for asylum seekers and refugees in Scotland: Provision and governance. Edinburgh","volume":"6","author":"Meer Nasar","year":"2019","unstructured":"Nasar Meer, Timothy Peace, and Emma Hill. 2019. English language education for asylum seekers and refugees in Scotland: Provision and governance. Edinburgh: GLIMMER Project 6 (2019), 2019.","journal-title":"GLIMMER Project"},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.1093\/jrs\/fez023"},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"publisher","DOI":"10.1111\/imre.12048"},{"key":"e_1_3_2_1_32_1","doi-asserted-by":"publisher","DOI":"10.1145\/3291078.3291115"},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D16-1264"},{"key":"e_1_3_2_1_34_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D19-1410"},{"key":"e_1_3_2_1_35_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10664-020-09881-0"},{"key":"e_1_3_2_1_36_1","doi-asserted-by":"crossref","unstructured":"Stephen Roller Emily Dinan Naman Goyal Da Ju Mary Williamson Yinhan Liu Jing Xu Myle Ott Kurt Shuster Eric M Smith et al. 2020. Recipes for building an open-domain chatbot. arXiv preprint arXiv:2004.13637 (2020).","DOI":"10.18653\/v1\/2021.eacl-main.24"},{"key":"e_1_3_2_1_37_1","unstructured":"Richard Sagor. 2000. Guiding school improvement with action research. Ascd."},{"key":"e_1_3_2_1_38_1","volume-title":"Turing test: 50 years later. Minds and machines 10, 4","author":"Saygin Ayse Pinar","year":"2000","unstructured":"Ayse Pinar Saygin, Ilyas Cicekli, and Varol Akman. 2000. Turing test: 50 years later. Minds and machines 10, 4 (2000), 463--518."},{"key":"e_1_3_2_1_39_1","volume-title":"Proc. of the 28th Int'l Conf. on Neural Information Proc. Systems. 2503--2511","author":"D. Sculley","year":"2015","unstructured":"D. Sculley et al. 2015. Hidden Technical Debt in Machine Learning Systems. In Proc. of the 28th Int'l Conf. on Neural Information Proc. Systems. 2503--2511."},{"key":"e_1_3_2_1_40_1","volume-title":"Gamification in mobile-assisted language learning: A systematic review of Duolingo literature from public release of 2012 to early","author":"Shortt Mitchell","year":"2020","unstructured":"Mitchell Shortt, Shantanu Tilak, Irina Kuznetcova, Bethany Martens, and Babatunde Akinkuolie. 2021. Gamification in mobile-assisted language learning: A systematic review of Duolingo literature from public release of 2012 to early 2020. Computer Assisted Language Learning (2021), 1--38."},{"key":"e_1_3_2_1_41_1","volume-title":"Action Research in Software Engineering","author":"Staron Miroslaw","unstructured":"Miroslaw Staron. 2020. Action research as research methodology in software engineering. In Action Research in Software Engineering. Springer, 15--36."},{"key":"e_1_3_2_1_42_1","volume-title":"Quality Assurance: Consistency in the Face of Complexity and Change","author":"Walkinshaw Neil","unstructured":"Neil Walkinshaw. 2017. Softw. Quality Assurance: Consistency in the Face of Complexity and Change. Springer."},{"key":"e_1_3_2_1_43_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.acl-main.450"},{"key":"e_1_3_2_1_44_1","doi-asserted-by":"publisher","DOI":"10.1080\/0952813X.2014.921734"},{"key":"e_1_3_2_1_45_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.infsof.2021.106678"},{"key":"e_1_3_2_1_46_1","doi-asserted-by":"publisher","DOI":"10.29297\/orbit.v1i2.49"},{"key":"e_1_3_2_1_47_1","doi-asserted-by":"publisher","DOI":"10.33182\/ml.v12i3.273"},{"key":"e_1_3_2_1_48_1","volume-title":"A comprehensive assessment of dialog evaluation metrics. arXiv preprint arXiv:2106.03706","author":"Yeh Yi-Ting","year":"2021","unstructured":"Yi-Ting Yeh, Maxine Eskenazi, and Shikib Mehri. 2021. A comprehensive assessment of dialog evaluation metrics. arXiv preprint arXiv:2106.03706 (2021)."},{"key":"e_1_3_2_1_49_1","volume-title":"Proc. of the Int'l. Workshop on Testing for Deep Learning and Deep Learning for Testing. https:\/\/deeptestconf.github.io\/2020\/index.html","author":"Yoo Shin","year":"2020","unstructured":"Shin Yoo. 2020. Searching for cost effective test inputs for DNN testing (Keynote address). In Proc. of the Int'l. Workshop on Testing for Deep Learning and Deep Learning for Testing. https:\/\/deeptestconf.github.io\/2020\/index.html"},{"key":"e_1_3_2_1_50_1","volume-title":"Proc. of the 35th Conference Neural Information Processing Systems.","author":"Yuan Weizhe","year":"2021","unstructured":"Weizhe Yuan, Graham Neubig, and Pengfei Liu. 2021. BARTScore: Evaluating generated text as text generation. Proc. of the 35th Conference Neural Information Processing Systems."},{"key":"e_1_3_2_1_51_1","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2019.2962027"}],"event":{"name":"CAIN '22: 1st Conference on AI Engineering - Software Engineering for AI","location":"Pittsburgh Pennsylvania","acronym":"CAIN '22","sponsor":["SIGSOFT ACM Special Interest Group on Software Engineering","IEEE TCSC IEEE Technical Committee on Scalable Computing"]},"container-title":["Proceedings of the 1st International Conference on AI Engineering: Software Engineering for AI"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3522664.3528592","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3522664.3528592","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T18:09:34Z","timestamp":1750183774000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3522664.3528592"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2022,5,16]]},"references-count":50,"alternative-id":["10.1145\/3522664.3528592","10.1145\/3522664"],"URL":"https:\/\/doi.org\/10.1145\/3522664.3528592","relation":{},"subject":[],"published":{"date-parts":[[2022,5,16]]},"assertion":[{"value":"2022-10-17","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}