{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,1,30]],"date-time":"2026-01-30T05:20:58Z","timestamp":1769750458647,"version":"3.49.0"},"publisher-location":"New York, NY, USA","reference-count":50,"publisher":"ACM","license":[{"start":{"date-parts":[[2024,4,12]],"date-time":"2024-04-12T00:00:00Z","timestamp":1712880000000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["62172202"],"award-info":[{"award-number":["62172202"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["62272221"],"award-info":[{"award-number":["62272221"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["62172205"],"award-info":[{"award-number":["62172205"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["61972082"],"award-info":[{"award-number":["61972082"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]},{"name":"the Major Program of the Natural Science Foundation of Jiangsu Higher Education Institutions of China","award":["22KJA520008"],"award-info":[{"award-number":["22KJA520008"]}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2024,4,12]]},"DOI":"10.1145\/3597503.3639109","type":"proceedings-article","created":{"date-parts":[[2024,4,12]],"date-time":"2024-04-12T16:43:26Z","timestamp":1712940206000},"page":"1-13","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":2,"title":["Knowledge Graph Driven Inference Testing for Question Answering Software"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0009-0001-5491-0443","authenticated-orcid":false,"given":"Jun","family":"Wang","sequence":"first","affiliation":[{"name":"State Key Laboratory for Novel Software Technology, Nanjing University, Nanjing, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-2282-7175","authenticated-orcid":false,"given":"Yanhui","family":"Li","sequence":"additional","affiliation":[{"name":"State Key Laboratory for Novel Software Technology, Nanjing University, Nanjing, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-3475-6979","authenticated-orcid":false,"given":"Zhifei","family":"Chen","sequence":"additional","affiliation":[{"name":"School of Computer Science and Engineering, Nanjing University of Science and Technology, Nanjing, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-2352-2226","authenticated-orcid":false,"given":"Lin","family":"Chen","sequence":"additional","affiliation":[{"name":"State Key Laboratory for Novel Software Technology, Nanjing University, Nanjing, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-8667-0456","authenticated-orcid":false,"given":"Xiaofang","family":"Zhang","sequence":"additional","affiliation":[{"name":"School of Computer Science and Technology, Soochow University, Suzhou, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-4645-2526","authenticated-orcid":false,"given":"Yuming","family":"Zhou","sequence":"additional","affiliation":[{"name":"State Key Laboratory for Novel Software Technology, Nanjing University, Nanjing, China"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2024,4,12]]},"reference":[{"key":"e_1_3_2_1_1_1","unstructured":"2023. Amazon Alexa. https:\/\/alexa.amazon.com."},{"key":"e_1_3_2_1_2_1","unstructured":"2023. Apple Siri. https:\/\/www.apple.com\/siri\/."},{"key":"e_1_3_2_1_3_1","unstructured":"2023. Microsoft Cortana. https:\/\/www.microsoft.com\/en-us\/cortana."},{"key":"e_1_3_2_1_4_1","volume-title":"Introduction to description logic","author":"Baader Franz","unstructured":"Franz Baader, Ian Horrocks, Carsten Lutz, and Uli Sattler. 2017. Introduction to description logic. Cambridge University Press."},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1145\/1376616.1376746"},{"key":"e_1_3_2_1_6_1","volume-title":"Weinberger (Eds.)","volume":"26","author":"Bordes Antoine","year":"2013","unstructured":"Antoine Bordes, Nicolas Usunier, Alberto Garcia-Duran, Jason Weston, and Oksana Yakhnenko. 2013. Translating Embeddings for Modeling Multirelational Data. In Advances in Neural Information Processing Systems, C.J. Burges, L. Bottou, M. Welling, Z. Ghahramani, and K.Q. Weinberger (Eds.), Vol. 26. Curran Associates, Inc. https:\/\/proceedings.neurips.cc\/paper\/2013\/file\/1cecc7a77928ca8133fa24680a88d2f9-Paper.pdf"},{"key":"e_1_3_2_1_7_1","volume-title":"ACM SIGIR Forum","volume":"54","author":"Cambazoglu B Barla","year":"2021","unstructured":"B Barla Cambazoglu, Mark Sanderson, Falk Scholer, and Bruce Croft. 2021. A review of public datasets in question answering research. In ACM SIGIR Forum, Vol. 54. ACM New York, NY, USA, 1--23."},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.1109\/ASE51524.2021.9678670"},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1145\/3468264.3468569"},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","unstructured":"Hyung Won Chung Le Hou Shayne Longpre Barret Zoph Yi Tay William Fedus Eric Li Xuezhi Wang Mostafa Dehghani Siddhartha Brahma Albert Webson Shixiang Shane Gu Zhuyun Dai Mirac Suzgun Xinyun Chen Aakanksha Chowdhery Sharan Narang Gaurav Mishra Adams Yu Vincent Y. Zhao Yanping Huang Andrew M. Dai Hongkun Yu Slav Petrov Ed H. Chi Jeff Dean Jacob Devlin Adam Roberts Denny Zhou Quoc V. Le and Jason Wei. 2022. Scaling Instruction-Finetuned Language Models. CoRR abs\/2210.11416 (2022). arXiv:2210.11416 10.48550\/arXiv.2210.11416","DOI":"10.48550\/arXiv.2210.11416"},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/n19-1300"},{"key":"e_1_3_2_1_12_1","unstructured":"Ryan Daws. 2020. Medical chatbot using OpenAI's GPT-3 told a fake patient to kill themselves. https:\/\/www.artificialintelligence-news.com\/2020\/10\/28\/medical-chatbot-openai-gpt3-patient-kill-themselves\/."},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1145\/2844544"},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.1145\/3368089.3409756"},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.1145\/3377811.3380339"},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE43902.2021.00047"},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.3115\/v1\/P14-2029"},{"key":"e_1_3_2_1_18_1","volume-title":"Advances in natural language processing. Science 349, 6245","author":"Hirschberg Julia","year":"2015","unstructured":"Julia Hirschberg and Christopher D Manning. 2015. Advances in natural language processing. Science 349, 6245 (2015), 261--266."},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","unstructured":"Aidan Hogan Eva Blomqvist Michael Cochez Claudia D'amato Gerard De Melo Claudio Gutierrez Sabrina Kirrane Jos\u00e9 Emilio Labra Gayo Roberto Navigli Sebastian Neumaier Axel-Cyrille Ngonga Ngomo Axel Polleres Sabbir M. Rashid Anisa Rula Lukas Schmelzeisen Juan Sequeda Steffen Staab and Antoine Zimmermann. 2021. Knowledge Graphs. ACM Comput. Surv. 54 4 Article 71 (jul 2021) 37 pages. 10.1145\/3447772","DOI":"10.1145\/3447772"},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.1145\/3533767.3534391"},{"key":"e_1_3_2_1_21_1","volume-title":"A survey on knowledge graphs: Representation, acquisition, and applications","author":"Ji Shaoxiong","year":"2021","unstructured":"Shaoxiong Ji, Shirui Pan, Erik Cambria, Pekka Marttinen, and S Yu Philip. 2021. A survey on knowledge graphs: Representation, acquisition, and applications. IEEE transactions on neural networks and learning systems 33, 2 (2021), 494--514."},{"key":"e_1_3_2_1_22_1","volume-title":"Next-generation of virtual personal assistants (microsoft cortana, apple siri, amazon alexa and google home). In 2018 IEEE 8th annual computing and communication workshop and conference (CCWC)","author":"Kepuska Veton","unstructured":"Veton Kepuska and Gamal Bohouta. 2018. Next-generation of virtual personal assistants (microsoft cortana, apple siri, amazon alexa and google home). In 2018 IEEE 8th annual computing and communication workshop and conference (CCWC). IEEE, 99--103."},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/N18-1023"},{"key":"e_1_3_2_1_24_1","volume-title":"UnifiedQA-v2: Stronger Generalization via Broader Cross-Format Training. CoRR abs\/2202.12359","author":"Khashabi Daniel","year":"2022","unstructured":"Daniel Khashabi, Yeganeh Kordi, and Hannaneh Hajishirzi. 2022. UnifiedQA-v2: Stronger Generalization via Broader Cross-Format Training. CoRR abs\/2202.12359 (2022). arXiv:2202.12359 https:\/\/arxiv.org\/abs\/2202.12359"},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.1162\/tacl_a_00023"},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.3233\/SW-140134"},{"key":"e_1_3_2_1_27_1","doi-asserted-by":"publisher","DOI":"10.1109\/TBDATA.2017.2735442"},{"key":"e_1_3_2_1_28_1","volume-title":"QATest: A Uniform Fuzzing Framework for Question Answering Systems. In 37th IEEE\/ACM International Conference on Automated Software Engineering. 1--12","author":"Liu Zixi","year":"2022","unstructured":"Zixi Liu, Yang Feng, Yining Yin, Jingyu Sun, Zhenyu Chen, and Baowen Xu. 2022. QATest: A Uniform Fuzzing Framework for Question Answering Systems. In 37th IEEE\/ACM International Conference on Automated Software Engineering. 1--12."},{"key":"e_1_3_2_1_29_1","volume-title":"Decoupled Weight Decay Regularization. In 7th International Conference on Learning Representations, ICLR 2019","author":"Loshchilov Ilya","year":"2019","unstructured":"Ilya Loshchilov and Frank Hutter. 2019. Decoupled Weight Decay Regularization. In 7th International Conference on Learning Representations, ICLR 2019, New Orleans, LA, USA, May 6--9, 2019. OpenReview.net. https:\/\/openreview.net\/forum?id=Bkg6RiCqY7"},{"key":"e_1_3_2_1_30_1","volume-title":"International Conference on Machine Learning. PMLR, 6905--6916","author":"Miller John","year":"2020","unstructured":"John Miller, Karl Krauth, Benjamin Recht, and Ludwig Schmidt. 2020. The effect of natural distribution shift on question answering models. In International Conference on Machine Learning. PMLR, 6905--6916."},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D15-1299"},{"key":"e_1_3_2_1_32_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/E17-2037"},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"publisher","DOI":"10.1145\/3193977.3193980"},{"key":"e_1_3_2_1_34_1","doi-asserted-by":"publisher","unstructured":"Peter Szemraj. 2022. flan-t5-large-grammar-synthesis (Revision d0b5ae2). 10.57967\/hf\/0138","DOI":"10.57967\/hf\/0138"},{"key":"e_1_3_2_1_35_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/P18-2124"},{"key":"e_1_3_2_1_36_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D16-1264"},{"key":"e_1_3_2_1_37_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D19-1410"},{"key":"e_1_3_2_1_38_1","volume-title":"Beyond accuracy: Behavioral testing of NLP models with CheckList. arXiv preprint arXiv:2005.04118","author":"Ribeiro Marco Tulio","year":"2020","unstructured":"Marco Tulio Ribeiro, Tongshuang Wu, Carlos Guestrin, and Sameer Singh. 2020. Beyond accuracy: Behavioral testing of NLP models with CheckList. arXiv preprint arXiv:2005.04118 (2020)."},{"key":"e_1_3_2_1_39_1","doi-asserted-by":"publisher","DOI":"10.1145\/3551349.3556953"},{"key":"e_1_3_2_1_40_1","doi-asserted-by":"publisher","DOI":"10.1145\/3324884.3416621"},{"key":"e_1_3_2_1_41_1","volume-title":"Axel-Cyrille Ngonga Ngomo, and Jens Lehmann","author":"Singh Kuldeep","year":"2019","unstructured":"Kuldeep Singh, Muhammad Saleem, Abhishek Nadgeri, Felix Conrads, Jeff Z Pan, Axel-Cyrille Ngonga Ngomo, and Jens Lehmann. 2019. Qaldgen: Towards microbenchmarking of question answering systems over knowledge graphs. In The Semantic Web-ISWC 2019: 18th International Semantic Web Conference, Auckland, New Zealand, October 26--30, 2019, Proceedings, Part II 18. Springer, 277--292."},{"key":"e_1_3_2_1_42_1","doi-asserted-by":"publisher","DOI":"10.1145\/1242572.1242667"},{"key":"e_1_3_2_1_43_1","doi-asserted-by":"publisher","DOI":"10.1145\/3377811.3380420"},{"key":"e_1_3_2_1_44_1","doi-asserted-by":"publisher","DOI":"10.1145\/3510003.3510206"},{"key":"e_1_3_2_1_45_1","doi-asserted-by":"publisher","DOI":"10.1145\/2872427.2874809"},{"key":"e_1_3_2_1_46_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-49461-2_34"},{"key":"e_1_3_2_1_47_1","doi-asserted-by":"publisher","DOI":"10.1145\/3180155.3180220"},{"key":"e_1_3_2_1_48_1","doi-asserted-by":"publisher","DOI":"10.1109\/FUZZY.2010.5584447"},{"key":"e_1_3_2_1_49_1","doi-asserted-by":"publisher","DOI":"10.1145\/2629489"},{"key":"e_1_3_2_1_50_1","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2019.2962027"}],"event":{"name":"ICSE '24: IEEE\/ACM 46th International Conference on Software Engineering","location":"Lisbon Portugal","acronym":"ICSE '24","sponsor":["SIGSOFT ACM Special Interest Group on Software Engineering","IEEE CS","Faculty of Engineering of University of Porto"]},"container-title":["Proceedings of the IEEE\/ACM 46th International Conference on Software Engineering"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3597503.3639109","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3597503.3639109","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,18]],"date-time":"2025-06-18T22:49:11Z","timestamp":1750286951000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3597503.3639109"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,4,12]]},"references-count":50,"alternative-id":["10.1145\/3597503.3639109","10.1145\/3597503"],"URL":"https:\/\/doi.org\/10.1145\/3597503.3639109","relation":{},"subject":[],"published":{"date-parts":[[2024,4,12]]},"assertion":[{"value":"2024-04-12","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}