{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,12,2]],"date-time":"2025-12-02T15:03:59Z","timestamp":1764687839119,"version":"3.40.3"},"publisher-location":"Cham","reference-count":29,"publisher":"Springer International Publishing","isbn-type":[{"type":"print","value":"9783030312794"},{"type":"electronic","value":"9783030312800"}],"license":[{"start":{"date-parts":[[2019,1,1]],"date-time":"2019-01-01T00:00:00Z","timestamp":1546300800000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2019,1,1]],"date-time":"2019-01-01T00:00:00Z","timestamp":1546300800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2019]]},"DOI":"10.1007\/978-3-030-31280-0_3","type":"book-chapter","created":{"date-parts":[[2019,10,7]],"date-time":"2019-10-07T23:06:50Z","timestamp":1570489610000},"page":"41-55","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":22,"title":["Testing Chatbots Using Metamorphic Relations"],"prefix":"10.1007","author":[{"ORCID":"https:\/\/orcid.org\/0000-0001-6086-8846","authenticated-orcid":false,"given":"Josip","family":"Bozic","sequence":"first","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0002-0462-2283","authenticated-orcid":false,"given":"Franz","family":"Wotawa","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2019,10,8]]},"reference":[{"unstructured":"Dialogflow. https:\/\/dialogflow.com\/. Accessed 11 Dec 2018","key":"3_CR1"},{"unstructured":"Gartner Top Strategic Predictions for 2018 and Beyond. https:\/\/www.gartner.com\/smarterwithgartner\/gartner-top-strategic-predictions-for-2018-and-beyond\/. Accessed 07 May 2018","key":"3_CR2"},{"doi-asserted-by":"crossref","unstructured":"Grammar-solver. https:\/\/github.com\/bd21\/Grammar-Solver. Accessed 13 July 2018","key":"3_CR3","DOI":"10.1163\/9789004344884_003"},{"doi-asserted-by":"crossref","unstructured":"Bozic, J., Tazl, O.A., Wotawa, F.: Chatbot testing using AI planning. In: Proceedings of the International Conference on Artificial Intelligence Testing (AITest) (2019)","key":"3_CR4","DOI":"10.1109\/AITest.2019.00-10"},{"doi-asserted-by":"crossref","unstructured":"Brandtz\u00e6g, P.B., F\u00f8lstad, A.: Why people use chatbots. In: Proceedings of the 4th International Conference on Internet Science (INSCI 2017) (2017)","key":"3_CR5","DOI":"10.1007\/978-3-319-70284-1_30"},{"issue":"2","key":"3_CR6","doi-asserted-by":"publisher","first-page":"e0212476","DOI":"10.1371\/journal.pone.0212476","volume":"14","author":"J Chen","year":"2019","unstructured":"Chen, J., Wang, Y., Guo, Y., Jiang, M.: A metamorphic testing approach for event sequences. PLoS ONE 14(2), e0212476 (2019)","journal-title":"PLoS ONE"},{"unstructured":"Chen, T.Y., Cheung, S.C., Yiu, S.M.: Metamorphic testing: a new approach for generating next test cases. Technical report HKUST-CS98-01, Department of Computer Science, Hong Kong University of Science and Technology, Hong Kong (1998)","key":"3_CR7"},{"issue":"1","key":"3_CR8","doi-asserted-by":"publisher","first-page":"4","DOI":"10.1145\/3143561","volume":"51","author":"TY Chen","year":"2018","unstructured":"Chen, T.Y., et al.: Metamorphic testing: a review of challenges and opportunities. ACM Comput. Surv. (CSUR) 51(1), 4 (2018)","journal-title":"ACM Comput. Surv. (CSUR)"},{"issue":"6","key":"3_CR9","doi-asserted-by":"publisher","first-page":"48","DOI":"10.1109\/MC.2016.176","volume":"49","author":"TY Chen","year":"2016","unstructured":"Chen, T.Y., et al.: Metamorphic testing for cybersecurity. Computer 49(6), 48\u201355 (2016)","journal-title":"Computer"},{"doi-asserted-by":"crossref","unstructured":"Dwarakanath, A., et al.: Identifying implementation bugs in machine learning based image classifiers using metamorphic testing. In: Proceedings of the 27th ACM SIGSOFT International Symposium on Software Testing and Analysis (ISSTA 2018) (2018)","key":"3_CR10","DOI":"10.1145\/3213846.3213858"},{"doi-asserted-by":"crossref","unstructured":"Guichard, J., Ruane, E., Smith, R., Bean, D., Ventresque, A.: Assessing the robustness of conversational agents using paraphrases. IEEE, University College Dublin (2019)","key":"3_CR11","DOI":"10.1109\/AITest.2019.000-7"},{"doi-asserted-by":"crossref","unstructured":"Lindvall, M., Porter, A., Magnusson, G., Schulze, C.: Metamorphic model-based testing of autonomous systems. In: Proceedings of the 2nd International Workshop on Metamorphic Testing (MET 2017) (2017)","key":"3_CR12","DOI":"10.1109\/MET.2017.6"},{"unstructured":"Mauldin, M.L.: ChatterBots, TinyMuds and the turing test: entering the loebner prize competition. In: AAAI 1994 Proceedings of the Twelfth National Conference on Artificial Intelligence, vol. 1, pp. 16\u201321 (1994)","key":"3_CR13"},{"unstructured":"Ruane, E., Faure, T., Smith, R., Bean, D., Carson-Berndsen, J., Ventresque, A.: BoTest: a framework to test the quality of conversational agents using divergent input examples. In: Proceedings of the 23rd International Conference on Intelligent User Interfaces Companion (IUI 2018 Companion) (2018)","key":"3_CR14"},{"doi-asserted-by":"crossref","unstructured":"Saha, P., Kanewala, U.: Fault detection effectiveness of metamorphic relations developed for testing supervised classifiers. In: Proceedings of the International Conference on Artificial Intelligence Testing (AITest) (2019)","key":"3_CR15","DOI":"10.1109\/AITest.2019.00019"},{"issue":"2","key":"3_CR16","doi-asserted-by":"publisher","first-page":"138","DOI":"10.1002\/stvr.1566","volume":"25","author":"S Segura","year":"2015","unstructured":"Segura, S., Dur\u00e1n, A., S\u00e1nchez, A.B., Le Berre, D., Lonca, E., Ruiz-Cort\u00e9s, A.: Automated metamorphic testing of variability analysis tools. Softw. Test. Verif. Reliab. 25(2), 138\u2013163 (2015)","journal-title":"Softw. Test. Verif. Reliab."},{"issue":"9","key":"3_CR17","doi-asserted-by":"publisher","first-page":"805","DOI":"10.1109\/TSE.2016.2532875","volume":"42","author":"S Segura","year":"2016","unstructured":"Segura, S., Fraser, G., S\u00e1nchez, A.B., Ruiz-Cort\u00e9s, A.: A survey on metamorphic testing. IEEE Trans. Softw. Eng. 42(9), 805\u2013824 (2016)","journal-title":"IEEE Trans. Softw. Eng."},{"doi-asserted-by":"crossref","unstructured":"Segura, S., Hierons, R.M., Benavides, D., Ruiz-Cort\u00e9s, A.: Automated test data generation on the analyses of feature models: a metamorphic testing approach. In: Proceedings of the 2010 Third International Conference on Software Testing, Verification and Validation (2010)","key":"3_CR18","DOI":"10.1109\/ICST.2010.20"},{"issue":"11","key":"3_CR19","doi-asserted-by":"publisher","first-page":"1083","DOI":"10.1109\/TSE.2017.2764464","volume":"44","author":"S Segura","year":"2018","unstructured":"Segura, S., Parejo, J.A., Troya, J., Ruiz-Cort\u00e9s, A.: Metamorphic testing of RESTful web APIs. IEEE Trans. Softw. Eng. 44(11), 1083\u20131099 (2018)","journal-title":"IEEE Trans. Softw. Eng."},{"key":"3_CR20","doi-asserted-by":"publisher","first-page":"489","DOI":"10.1075\/ijcl.10.4.06sha","volume":"10","author":"BA Shawar","year":"2005","unstructured":"Shawar, B.A., Atwell, E.: Using corpora in machine-learning chatbot systems. Int. J. Corpus Linguist. 10, 489\u2013516 (2005)","journal-title":"Int. J. Corpus Linguist."},{"doi-asserted-by":"crossref","unstructured":"Tian, Y., Pei, K., Jana, S., Ray, B.: DeepTest: automated testing of deep-neural-network-driven autonomous cars. In: Proceedings of the 40th International Conference on Software Engineering (2018)","key":"3_CR21","DOI":"10.1145\/3180155.3180220"},{"doi-asserted-by":"crossref","unstructured":"Vasconcelos, M., Candello, H., Pinhanez, C., dos Santos, T.: Bottester: testing conversational systems with simulated users. In: Proceedings of the XVI Brazilian Symposium on Human Factors in Computing Systems (IHC 2017) (2017)","key":"3_CR22","DOI":"10.1145\/3160504.3160584"},{"doi-asserted-by":"crossref","unstructured":"Walker, M.A., Litman, D.J., Kamm, C.A., Abella, A.: PARADISE: a framework for evaluating spoken dialogue agents. In: Proceedings of the 35th Annual General Meeting of the Association for Computational Linguistics, ACL\/EACL 1997 (1997)","key":"3_CR23","DOI":"10.3115\/976909.979652"},{"unstructured":"Wallace, R.S.: The elements of AIML style. In: ALICE A.I. Foundation (2003)","key":"3_CR24"},{"issue":"4","key":"3_CR25","doi-asserted-by":"publisher","first-page":"465","DOI":"10.1093\/comjnl\/25.4.465","volume":"25","author":"E Weyuker","year":"1982","unstructured":"Weyuker, E.: On testing non-testable programs. Comput. J. 25(4), 465\u2013470 (1982)","journal-title":"Comput. J."},{"issue":"4","key":"3_CR26","doi-asserted-by":"publisher","first-page":"544","DOI":"10.1016\/j.jss.2010.11.920","volume":"84","author":"X Xie","year":"2011","unstructured":"Xie, X., Ho, J.W.K., Murphy, C., Kaiser, G., Xu, B., Chen, T.Y.: Testing and validating machine learning classifiers by metamorphic testing. J. Syst. Softw. 84(4), 544\u2013558 (2011)","journal-title":"J. Syst. Softw."},{"doi-asserted-by":"crossref","unstructured":"Yan, B., Yecies, B., Zhou, Z.Q.: Metamorphic relations for data validation: a case study of translated text messages. In: Proceedings of the 4th International Workshop on Metamorphic Testing (MET 2019) (2019)","key":"3_CR27","DOI":"10.1109\/MET.2019.00018"},{"doi-asserted-by":"crossref","unstructured":"Zhang, M., Zhang, Y., Zhang, L., Liu, C., Khurshid, S.: DeepRoad: GAN-based metamorphic testing and input validation framework for autonomous driving systems. In: Proceedings of the 33rd ACM\/IEEE International Conference on Automated Software Engineering (ASE 2018) (2018)","key":"3_CR28","DOI":"10.1145\/3238147.3238187"},{"issue":"3","key":"3_CR29","doi-asserted-by":"publisher","first-page":"61","DOI":"10.1145\/3241979","volume":"62","author":"ZQ Zhou","year":"2019","unstructured":"Zhou, Z.Q., Sun, L.: Metamorphic testing of driverless cars. Commun. ACM 62(3), 61\u201367 (2019)","journal-title":"Commun. ACM"}],"container-title":["Lecture Notes in Computer Science","Testing Software and Systems"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-030-31280-0_3","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,3,13]],"date-time":"2024-03-13T20:58:17Z","timestamp":1710363497000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-3-030-31280-0_3"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2019]]},"ISBN":["9783030312794","9783030312800"],"references-count":29,"URL":"https:\/\/doi.org\/10.1007\/978-3-030-31280-0_3","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"type":"print","value":"0302-9743"},{"type":"electronic","value":"1611-3349"}],"subject":[],"published":{"date-parts":[[2019]]},"assertion":[{"value":"8 October 2019","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"ICTSS","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"IFIP International Conference on Testing Software and Systems","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Paris","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"France","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2019","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"15 October 2019","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"17 October 2019","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"31","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"pts2019","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"http:\/\/ictss2019.centralesupelec.fr\/","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Single-blind","order":1,"name":"type","label":"Type","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"EasyChair","order":2,"name":"conference_management_system","label":"Conference Management System","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"30","order":3,"name":"number_of_submissions_sent_for_review","label":"Number of Submissions Sent for Review","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"14","order":4,"name":"number_of_full_papers_accepted","label":"Number of Full Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"3","order":5,"name":"number_of_short_papers_accepted","label":"Number of Short Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"47% - The value is computed by the equation \"Number of Full Papers Accepted \/ Number of Submissions Sent for Review * 100\" and then rounded to a whole number.","order":6,"name":"acceptance_rate_of_full_papers","label":"Acceptance Rate of Full Papers","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"3","order":7,"name":"average_number_of_reviews_per_paper","label":"Average Number of Reviews per Paper","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"2,1","order":8,"name":"average_number_of_papers_per_reviewer","label":"Average Number of Papers per Reviewer","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"Yes","order":9,"name":"external_reviewers_involved","label":"External Reviewers Involved","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"This content has been made available to all.","name":"free","label":"Free to read"}]}}