{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,21]],"date-time":"2026-04-21T14:48:39Z","timestamp":1776782919150,"version":"3.51.2"},"publisher-location":"New York, NY, USA","reference-count":58,"publisher":"ACM","license":[{"start":{"date-parts":[[2022,10,10]],"date-time":"2022-10-10T00:00:00Z","timestamp":1665360000000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2022,10,10]]},"DOI":"10.1145\/3551349.3556929","type":"proceedings-article","created":{"date-parts":[[2023,1,5]],"date-time":"2023-01-05T20:43:54Z","timestamp":1672951434000},"page":"1-12","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":14,"title":["QATest: A Uniform Fuzzing Framework for Question Answering Systems"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-9592-7022","authenticated-orcid":false,"given":"Zixi","family":"Liu","sequence":"first","affiliation":[{"name":"Nanjing University, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Yang","family":"Feng","sequence":"additional","affiliation":[{"name":"Nanjing University, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Yining","family":"Yin","sequence":"additional","affiliation":[{"name":"Nanjing University, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Jingyu","family":"Sun","sequence":"additional","affiliation":[{"name":"Nanjing University, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Zhenyu","family":"Chen","sequence":"additional","affiliation":[{"name":"Nanjing University, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Baowen","family":"Xu","sequence":"additional","affiliation":[{"name":"Nanjing University, China"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2023,1,5]]},"reference":[{"key":"e_1_3_2_1_1_1","unstructured":"[n.d.]. Amazon promises fix for creepy Alexa laugh - BBC News. https:\/\/www.bbc.com\/news\/technology-43325230. (Accessed on 05\/05\/2022)."},{"key":"e_1_3_2_1_2_1","unstructured":"[n.d.]. Python Release Python 3.6.0 | Python.org. https:\/\/www.python.org\/downloads\/release\/python-360\/. (Accessed on 05\/05\/2022)."},{"key":"e_1_3_2_1_3_1","unstructured":"[n.d.]. PyTorch. https:\/\/pytorch.org\/. (Accessed on 05\/05\/2022)."},{"key":"e_1_3_2_1_4_1","unstructured":"[n.d.]. The Stanford Question Answering Dataset. https:\/\/rajpurkar.github.io\/SQuAD-explorer\/. (Accessed on 05\/07\/2022)."},{"key":"e_1_3_2_1_5_1","unstructured":"[n.d.]. TagMe - TagMe API. https:\/\/services.d4science.org\/web\/tagme\/tagme-help. (Accessed on 04\/30\/2022)."},{"key":"e_1_3_2_1_6_1","unstructured":"[n.d.]. Tay: Microsoft issues apology over racist chatbot fiasco - BBC News. https:\/\/www.bbc.com\/news\/technology-35902104. (Accessed on 05\/05\/2022)."},{"key":"e_1_3_2_1_7_1","unstructured":"Razieh Baradaran Razieh Ghiasi and Hossein Amirkhani. 2020. A survey on machine reading comprehension systems. Natural Language Engineering(2020) 1\u201350."},{"key":"e_1_3_2_1_8_1","volume-title":"A Question-Entailment Approach to Question Answering. BMC Bioinform. 20, 1","author":"Abacha Asma Ben","year":"2019","unstructured":"Asma Ben Abacha and Dina Demner-Fushman. 2019. A Question-Entailment Approach to Question Answering. BMC Bioinform. 20, 1 (2019), 511:1\u2013511:23. https:\/\/bmcbioinformatics.biomedcentral.com\/articles\/10.1186\/s12859-019-3119-4"},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D13-1160"},{"key":"e_1_3_2_1_10_1","volume-title":"Proceedings of SDAIR-94","author":"Cavnar B","year":"1994","unstructured":"William\u00a0B Cavnar, John\u00a0M Trenkle, 1994. N-gram-based text categorization. In Proceedings of SDAIR-94, 3rd annual symposium on document analysis and information retrieval, Vol.\u00a0161175. Citeseer."},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"crossref","unstructured":"Danqi Chen Adam Fisch Jason Weston and Antoine Bordes. 2017. Reading wikipedia to answer open-domain questions. arXiv preprint arXiv:1704.00051(2017).","DOI":"10.18653\/v1\/P17-1171"},{"key":"e_1_3_2_1_12_1","unstructured":"Junjie Chen Ming Yan Zan Wang Yuning Kang and Zhuo Wu. 2020. Deep neural network test coverage: How far are we?arXiv preprint arXiv:2010.04946(2020)."},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1109\/ASE51524.2021.9678670"},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.1145\/3468264.3468569"},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"crossref","unstructured":"KR1442 Chowdhary. 2020. Natural language processing. Fundamentals of artificial intelligence(2020) 603\u2013649.","DOI":"10.1007\/978-81-322-3972-7_19"},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-02154-1"},{"key":"e_1_3_2_1_17_1","unstructured":"Christopher Clark Kenton Lee Ming-Wei Chang Tom Kwiatkowski Michael Collins and Kristina Toutanova. 2019. BoolQ: Exploring the Surprising Difficulty of Natural Yes\/No Questions. In NAACL."},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"crossref","unstructured":"Nicola De\u00a0Cao Wilker Aziz and Ivan Titov. 2018. Question answering by reasoning across documents with graph convolutional networks. arXiv preprint arXiv:1808.09920(2018).","DOI":"10.18653\/v1\/N19-1240"},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","unstructured":"Jan Deriu Alvaro Rodrigo Arantxa Otegi Guillermo Echegoyen Sophie Rosset Eneko Agirre and Mark Cieliebak. 2020. Survey on evaluation methods for dialogue systems. Artificial Intelligence Review(2020) 1\u201356. https:\/\/doi.org\/10.1007\/s10462-020-09866-x","DOI":"10.1007\/s10462-020-09866-x"},{"key":"e_1_3_2_1_20_1","volume-title":"Bert: Pre-training of deep bidirectional transformers for language understanding. arXiv preprint arXiv:1810.04805(2018).","author":"Devlin Jacob","year":"2018","unstructured":"Jacob Devlin, Ming-Wei Chang, Kenton Lee, and Kristina Toutanova. 2018. Bert: Pre-training of deep bidirectional transformers for language understanding. arXiv preprint arXiv:1810.04805(2018)."},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.1145\/3395363.3397357"},{"key":"e_1_3_2_1_22_1","volume-title":"Deep learning","author":"Goodfellow Ian","unstructured":"Ian Goodfellow, Yoshua Bengio, and Aaron Courville. 2016. Deep learning. MIT press."},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.1145\/3442381.3449992"},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"publisher","DOI":"10.1145\/3377811.3380339"},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.24963\/ijcai.2020\/509"},{"key":"e_1_3_2_1_26_1","unstructured":"Dan Jurafsky. 2000. Speech & language processing. Pearson Education India."},{"key":"e_1_3_2_1_27_1","doi-asserted-by":"publisher","DOI":"10.1109\/CCWC.2018.8301638"},{"key":"e_1_3_2_1_28_1","unstructured":"Daniel Keysers Nathanael Sch\u00e4rli Nathan Scales Hylke Buisman Daniel Furrer Sergii Kashubin Nikola Momchev Danila Sinopalnikov Lukasz Stafiniak Tibor Tihon 2019. Measuring compositional generalization: A comprehensive method on realistic data. arXiv preprint arXiv:1912.09713(2019)."},{"key":"e_1_3_2_1_29_1","volume-title":"Unifiedqa: Crossing format boundaries with a single qa system. arXiv preprint arXiv:2005.00700(2020).","author":"Khashabi Daniel","year":"2020","unstructured":"Daniel Khashabi, Sewon Min, Tushar Khot, Ashish Sabharwal, Oyvind Tafjord, Peter Clark, and Hannaneh Hajishirzi. 2020. Unifiedqa: Crossing format boundaries with a single qa system. arXiv preprint arXiv:2005.00700(2020)."},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE.2019.00108"},{"key":"e_1_3_2_1_31_1","volume-title":"Race: Large-scale reading comprehension dataset from examinations. arXiv preprint arXiv:1704.04683(2017).","author":"Lai Guokun","year":"2017","unstructured":"Guokun Lai, Qizhe Xie, Hanxiao Liu, Yiming Yang, and Eduard Hovy. 2017. Race: Large-scale reading comprehension dataset from examinations. arXiv preprint arXiv:1704.04683(2017)."},{"key":"e_1_3_2_1_32_1","unstructured":"Yunshi Lan Gaole He Jinhao Jiang Jing Jiang Wayne\u00a0Xin Zhao and Ji-Rong Wen. 2021. Complex Knowledge Base Question Answering: A Survey. arXiv preprint arXiv:2108.06688(2021)."},{"key":"e_1_3_2_1_33_1","unstructured":"Yunshi Lan Gaole He Jinhao Jiang Jing Jiang Wayne\u00a0Xin Zhao and Ji-Rong Wen. 2021. A survey on complex knowledge base question answering: Methods challenges and solutions. arXiv preprint arXiv:2105.11644(2021)."},{"key":"e_1_3_2_1_34_1","volume-title":"Albert: A lite bert for self-supervised learning of language representations. arXiv preprint arXiv:1909.11942(2019).","author":"Lan Zhenzhong","year":"2019","unstructured":"Zhenzhong Lan, Mingda Chen, Sebastian Goodman, Kevin Gimpel, Piyush Sharma, and Radu Soricut. 2019. Albert: A lite bert for self-supervised learning of language representations. arXiv preprint arXiv:1909.11942(2019)."},{"key":"e_1_3_2_1_35_1","volume-title":"ROUGE: A Package for Automatic Evaluation of Summaries. In Text Summarization Branches Out","author":"Lin Chin-Yew","year":"2004","unstructured":"Chin-Yew Lin. 2004. ROUGE: A Package for Automatic Evaluation of Summaries. In Text Summarization Branches Out. Association for Computational Linguistics, Barcelona, Spain, 74\u201381. https:\/\/aclanthology.org\/W04-1013"},{"key":"e_1_3_2_1_36_1","doi-asserted-by":"publisher","DOI":"10.1145\/3460319.3464829"},{"key":"e_1_3_2_1_37_1","unstructured":"Edward Ma. 2019. NLP Augmentation. https:\/\/github.com\/makcedward\/nlpaug."},{"key":"e_1_3_2_1_38_1","doi-asserted-by":"publisher","DOI":"10.1145\/3238147.3238202"},{"key":"e_1_3_2_1_39_1","doi-asserted-by":"publisher","DOI":"10.1145\/219717.219748"},{"key":"e_1_3_2_1_40_1","volume-title":"International Conference on Machine Learning. PMLR, 6905\u20136916","author":"Miller John","year":"2020","unstructured":"John Miller, Karl Krauth, Benjamin Recht, and Ludwig Schmidt. 2020. The effect of natural distribution shift on question answering models. In International Conference on Machine Learning. PMLR, 6905\u20136916."},{"key":"e_1_3_2_1_41_1","doi-asserted-by":"publisher","DOI":"10.5555\/3524938.3525579"},{"key":"e_1_3_2_1_42_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.jksuci.2014.10.007"},{"key":"e_1_3_2_1_43_1","doi-asserted-by":"publisher","DOI":"10.3390\/app11125456"},{"key":"e_1_3_2_1_44_1","doi-asserted-by":"publisher","unstructured":"J.\u00a0R. Norris. 1997. Markov Chains. Cambridge University Press. https:\/\/doi.org\/10.1017\/CBO9780511810633","DOI":"10.1017\/CBO9780511810633"},{"key":"e_1_3_2_1_45_1","volume-title":"International Conference on Machine Learning. PMLR, 4901\u20134911","author":"Odena Augustus","year":"2019","unstructured":"Augustus Odena, Catherine Olsson, David Andersen, and Ian Goodfellow. 2019. Tensorfuzz: Debugging neural networks with coverage-guided fuzzing. In International Conference on Machine Learning. PMLR, 4901\u20134911."},{"key":"e_1_3_2_1_46_1","first-page":"1","article-title":"Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer","volume":"21","author":"Raffel Colin","year":"2020","unstructured":"Colin Raffel, Noam Shazeer, Adam Roberts, Katherine Lee, Sharan Narang, Michael Matena, Yanqi Zhou, Wei Li, and Peter\u00a0J. Liu. 2020. Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer. Journal of Machine Learning Research 21, 140 (2020), 1\u201367. http:\/\/jmlr.org\/papers\/v21\/20-074.html","journal-title":"Journal of Machine Learning Research"},{"key":"e_1_3_2_1_47_1","doi-asserted-by":"crossref","unstructured":"Pranav Rajpurkar Robin Jia and Percy Liang. 2018. Know what you don\u2019t know: Unanswerable questions for SQuAD. arXiv preprint arXiv:1806.03822(2018).","DOI":"10.18653\/v1\/P18-2124"},{"key":"e_1_3_2_1_48_1","doi-asserted-by":"crossref","unstructured":"Pranav Rajpurkar Jian Zhang Konstantin Lopyrev and Percy Liang. 2016. Squad: 100 000+ questions for machine comprehension of text. arXiv preprint arXiv:1606.05250(2016).","DOI":"10.18653\/v1\/D16-1264"},{"key":"e_1_3_2_1_49_1","doi-asserted-by":"crossref","unstructured":"Amrita Saha Vardaan Pahuja Mitesh\u00a0M. Khapra Karthik Sankaranarayanan and Sarath Chandar. 2018. Complex Sequential Question Answering: Towards Learning to Converse over Linked Question Answer Pairs with a Knowledge Graph. In Proceedings of the Thirty-Second AAAI Conference on Artificial Intelligence and Thirtieth Innovative Applications of Artificial Intelligence Conference and Eighth AAAI Symposium on Educational Advances in Artificial Intelligence (New Orleans Louisiana USA) (AAAI\u201918\/IAAI\u201918\/EAAI\u201918). AAAI Press Article 87 9\u00a0pages.","DOI":"10.1609\/aaai.v32i1.11332"},{"key":"e_1_3_2_1_50_1","volume-title":"Deep Learning Framework Fuzzing Based on Model Mutation. In 2021 IEEE Sixth International Conference on Data Science in Cyberspace (DSC). IEEE, 375\u2013380","author":"Shen Xiangzhong","year":"2021","unstructured":"Xiangzhong Shen, Jieyi Zhang, Xiaonan Wang, Hongfang Yu, and Gang Sun. 2021. Deep Learning Framework Fuzzing Based on Model Mutation. In 2021 IEEE Sixth International Conference on Data Science in Cyberspace (DSC). IEEE, 375\u2013380."},{"key":"e_1_3_2_1_51_1","doi-asserted-by":"publisher","DOI":"10.1145\/3180155.3180220"},{"key":"e_1_3_2_1_52_1","volume-title":"Newsqa: A machine comprehension dataset. arXiv preprint arXiv:1611.09830(2016).","author":"Trischler Adam","year":"2016","unstructured":"Adam Trischler, Tong Wang, Xingdi Yuan, Justin Harris, Alessandro Sordoni, Philip Bachman, and Kaheer Suleman. 2016. Newsqa: A machine comprehension dataset. arXiv preprint arXiv:1611.09830(2016)."},{"key":"e_1_3_2_1_53_1","doi-asserted-by":"publisher","DOI":"10.1145\/3293882.3330579"},{"key":"e_1_3_2_1_54_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D15-1237"},{"key":"e_1_3_2_1_55_1","unstructured":"Munazza Zaib Wei\u00a0Emma Zhang Quan\u00a0Z Sheng Adnan Mahmood and Yang Zhang. 2021. Conversational question answering: A survey. arXiv preprint arXiv:2106.00874(2021)."},{"key":"e_1_3_2_1_56_1","doi-asserted-by":"publisher","DOI":"10.1109\/TR.2021.3107165"},{"key":"e_1_3_2_1_57_1","unstructured":"Xin Zhang An Yang Sujian Li and Yizhong Wang. 2019. Machine reading comprehension: a literature review. arXiv preprint arXiv:1907.01686(2019)."},{"key":"e_1_3_2_1_58_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE-Companion.2019.00131"}],"event":{"name":"ASE '22: 37th IEEE\/ACM International Conference on Automated Software Engineering","location":"Rochester MI USA","acronym":"ASE '22"},"container-title":["Proceedings of the 37th IEEE\/ACM International Conference on Automated Software Engineering"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3551349.3556929","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3551349.3556929","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,8,22]],"date-time":"2025-08-22T08:31:52Z","timestamp":1755851512000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3551349.3556929"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2022,10,10]]},"references-count":58,"alternative-id":["10.1145\/3551349.3556929","10.1145\/3551349"],"URL":"https:\/\/doi.org\/10.1145\/3551349.3556929","relation":{},"subject":[],"published":{"date-parts":[[2022,10,10]]},"assertion":[{"value":"2023-01-05","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}