{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,3,17]],"date-time":"2026-03-17T00:54:30Z","timestamp":1773708870697,"version":"3.50.1"},"publisher-location":"New York, NY, USA","reference-count":39,"publisher":"ACM","license":[{"start":{"date-parts":[[2023,11,30]],"date-time":"2023-11-30T00:00:00Z","timestamp":1701302400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["No. 62102340"],"award-info":[{"award-number":["No. 62102340"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2023,11,30]]},"DOI":"10.1145\/3611643.3616295","type":"proceedings-article","created":{"date-parts":[[2023,11,30]],"date-time":"2023-11-30T23:14:38Z","timestamp":1701386078000},"page":"883-894","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":7,"title":["Automated Testing and Improvement of Named Entity Recognition Systems"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0001-5213-7189","authenticated-orcid":false,"given":"Boxi","family":"Yu","sequence":"first","affiliation":[{"name":"The Chinese University of Hong Kong, Shenzhen, Shenzhen, China"}]},{"ORCID":"https:\/\/orcid.org\/0009-0001-3906-1738","authenticated-orcid":false,"given":"Yiyan","family":"Hu","sequence":"additional","affiliation":[{"name":"The Chinese University of Hong Kong, Shenzhen, Shenzhen, China"}]},{"ORCID":"https:\/\/orcid.org\/0009-0000-4462-8527","authenticated-orcid":false,"given":"Qiuyang","family":"Mang","sequence":"additional","affiliation":[{"name":"The Chinese University of Hong Kong, Shenzhen, Shenzhen, China"}]},{"ORCID":"https:\/\/orcid.org\/0009-0003-9503-9230","authenticated-orcid":false,"given":"Wenhan","family":"Hu","sequence":"additional","affiliation":[{"name":"The Chinese University of Hong Kong, Shenzhen, Shenzhen, China"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-3377-8129","authenticated-orcid":false,"given":"Pinjia","family":"He","sequence":"additional","affiliation":[{"name":"The Chinese University of Hong Kong, Shenzhen, Shenzhen, China"}]}],"member":"320","published-online":{"date-parts":[[2023,11,30]]},"reference":[{"key":"e_1_3_2_2_1_1","unstructured":"2020. Stanford CoreNLP. https:\/\/stanfordnlp.github.io\/CoreNLP"},{"key":"e_1_3_2_2_2_1","unstructured":"2022. Flair: A very simple framework for state-of-the-art NLP. https:\/\/github.com\/flairNLP\/flair"},{"key":"e_1_3_2_2_3_1","unstructured":"2022. NLTK:. https:\/\/www.nltk.org\/"},{"key":"e_1_3_2_2_4_1","unstructured":"2022. Wikipedia:Size of Wikipedia. https:\/\/en.wikipedia.org\/wiki\/Wikipedia:Size_of_Wikipedia##cite_note-2"},{"key":"e_1_3_2_2_5_1","unstructured":"2023. Context-free grammar. https:\/\/en.wikipedia.org\/wiki\/Context-free_grammar"},{"key":"e_1_3_2_2_6_1","unstructured":"2023. TestNER: A toolkit for testing and improving named entity recognition. https:\/\/github.com\/RobustNLP\/TestNER"},{"key":"e_1_3_2_2_7_1","doi-asserted-by":"publisher","DOI":"10.1145\/3290353"},{"key":"e_1_3_2_2_8_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D18-1316"},{"key":"e_1_3_2_2_9_1","unstructured":"Isaac Caswell Onkur Sen and Allen Nie. 2015. Exploring adversarial learning on neural network models for text classification."},{"key":"e_1_3_2_2_10_1","volume-title":"Neural Reading Comprehension and Beyond. Ph. D. Dissertation","author":"Chen Danqi","unstructured":"Danqi Chen. 2018. Neural Reading Comprehension and Beyond. Ph. D. Dissertation. Stanford University."},{"key":"e_1_3_2_2_11_1","volume-title":"Proceedings of the 54th Annual Meeting of the Association for Computational Linguistics (ACL).","author":"Chen Danqi","unstructured":"Danqi Chen, Jason Bolton, and Christopher D. Manning. 2016. A Thorough Examination of the CNN\/Daily Mail Reading Comprehension Task. In Proceedings of the 54th Annual Meeting of the Association for Computational Linguistics (ACL)."},{"key":"e_1_3_2_2_12_1","volume-title":"Bert: Pre-training of deep bidirectional transformers for language understanding. arXiv preprint arXiv:1810.04805.","author":"Devlin Jacob","year":"2018","unstructured":"Jacob Devlin, Ming-Wei Chang, Kenton Lee, and Kristina Toutanova. 2018. Bert: Pre-training of deep bidirectional transformers for language understanding. arXiv preprint arXiv:1810.04805."},{"key":"e_1_3_2_2_13_1","volume-title":"Proceedings of the 28th ACM Joint Meeting on European Software Engineering Conference and Symposium on the Foundations of Software Engineering. 863\u2013875","author":"Gupta Shashij","year":"2020","unstructured":"Shashij Gupta, Pinjia He, Clara Meister, and Zhendong Su. 2020. Machine translation testing via pathological invariance. In Proceedings of the 28th ACM Joint Meeting on European Software Engineering Conference and Symposium on the Foundations of Software Engineering. 863\u2013875."},{"key":"e_1_3_2_2_14_1","doi-asserted-by":"publisher","DOI":"10.1145\/3377811.3380339"},{"key":"e_1_3_2_2_15_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE43902.2021.00047"},{"key":"e_1_3_2_2_16_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/P16-1195"},{"key":"e_1_3_2_2_17_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/N18-1170"},{"key":"e_1_3_2_2_18_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D17-1215"},{"key":"e_1_3_2_2_19_1","doi-asserted-by":"publisher","DOI":"10.1145\/2594291.2594334"},{"key":"e_1_3_2_2_20_1","unstructured":"Dianqi Li Yizhe Zhang Hao Peng Liqun Chen Chris Brockett Ming-Ting Sun and Bill Dolan. 2020. Contextualized perturbation for textual adversarial attack. arXiv preprint arXiv:2009.07502."},{"key":"e_1_3_2_2_21_1","doi-asserted-by":"publisher","DOI":"10.14722\/ndss.2019.23138"},{"key":"e_1_3_2_2_22_1","volume-title":"Bert-attack: Adversarial attack against bert using bert. arXiv preprint arXiv:2004.09984.","author":"Li Linyang","year":"2020","unstructured":"Linyang Li, Ruotian Ma, Qipeng Guo, Xiangyang Xue, and Xipeng Qiu. 2020. Bert-attack: Adversarial attack against bert using bert. arXiv preprint arXiv:2004.09984."},{"key":"e_1_3_2_2_23_1","volume-title":"Many-Core Compiler Fuzzing. In ACM SIGPLAN Conference on Programming Language Design and Implementation (PLDI).","author":"Lidbury Christopher","unstructured":"Christopher Lidbury, Andrei Lascu, Nathan Chong, and Alastair F. Donaldson. 2015. Many-Core Compiler Fuzzing. In ACM SIGPLAN Conference on Programming Language Design and Implementation (PLDI)."},{"key":"e_1_3_2_2_24_1","unstructured":"Bill Yuchen Lin Wenyang Gao Jun Yan Ryan Moreno and Xiang Ren. 2021. RockNER: A simple method to create adversarial examples for evaluating the robustness of named entity recognition models. arXiv preprint arXiv:2109.05620."},{"key":"e_1_3_2_2_25_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE.2015.348"},{"key":"e_1_3_2_2_26_1","unstructured":"Shubhanshu Mishra Sijun He and Luca Belli. 2020. Assessing demographic bias in named entity recognition. arXiv preprint arXiv:2008.03415."},{"key":"e_1_3_2_2_27_1","volume-title":"Proceedings of the 5th International Conference on Learning Representations (ICLR).","author":"Miyato Takeru","year":"2017","unstructured":"Takeru Miyato, Andrew M Dai, and Ian Goodfellow. 2017. Adversarial Training Methods for Semi-Supervised Text Classification. In Proceedings of the 5th International Conference on Learning Representations (ICLR)."},{"key":"e_1_3_2_2_28_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/P18-1176"},{"key":"e_1_3_2_2_29_1","doi-asserted-by":"publisher","DOI":"10.1145\/3276517"},{"key":"e_1_3_2_2_30_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/P18-1079"},{"key":"e_1_3_2_2_31_1","unstructured":"Erik F Sang and Fien De Meulder. 2003. Introduction to the CoNLL-2003 shared task: Language-independent named entity recognition. arXiv preprint cs\/0306050."},{"key":"e_1_3_2_2_32_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2021.emnlp-demo.35"},{"key":"e_1_3_2_2_33_1","doi-asserted-by":"publisher","DOI":"10.1145\/3377811.3380420"},{"key":"e_1_3_2_2_34_1","doi-asserted-by":"publisher","DOI":"10.1145\/3510003.3510206"},{"key":"e_1_3_2_2_35_1","unstructured":"Andrew Trask Phil Michalak and John Liu. 2015. sense2vec-a fast and accurate method for word sense disambiguation in neural word embeddings. arXiv preprint arXiv:1511.06388."},{"key":"e_1_3_2_2_36_1","volume-title":"Ontonotes release 5.0 ldc2013t19","author":"Weischedel Ralph","year":"2013","unstructured":"Ralph Weischedel, Martha Palmer, Mitchell Marcus, Eduard Hovy, Sameer Pradhan, Lance Ramshaw, Nianwen Xue, Ann Taylor, Jeff Kaufman, and Michelle Franchini. 2013. Ontonotes release 5.0 ldc2013t19. Linguistic Data Consortium, Philadelphia, PA, 23 (2013)."},{"key":"e_1_3_2_2_37_1","volume-title":"Xiaoxia Zhang, Jing Wang, and Mingyue Jiang.","author":"Xu Yezi","year":"2022","unstructured":"Yezi Xu, Zhi Quan Zhou, Xiaoxia Zhang, Jing Wang, and Mingyue Jiang. 2022. Metamorphic testing of named entity recognition systems: A case study. IET Software."},{"key":"e_1_3_2_2_38_1","doi-asserted-by":"publisher","DOI":"10.1145\/3533767.3534389"},{"key":"e_1_3_2_2_39_1","unstructured":"Xingmeng Zhao Ali Niazi and Anthony Rios. 2022. A Comprehensive Study of Gender Bias in Chemical Named Entity Recognition Models. arXiv preprint arXiv:2212.12799."}],"event":{"name":"ESEC\/FSE '23: 31st ACM Joint European Software Engineering Conference and Symposium on the Foundations of Software Engineering","location":"San Francisco CA USA","acronym":"ESEC\/FSE '23","sponsor":["SIGSOFT ACM Special Interest Group on Software Engineering"]},"container-title":["Proceedings of the 31st ACM Joint European Software Engineering Conference and Symposium on the Foundations of Software Engineering"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3611643.3616295","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3611643.3616295","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T16:36:04Z","timestamp":1750178164000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3611643.3616295"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023,11,30]]},"references-count":39,"alternative-id":["10.1145\/3611643.3616295","10.1145\/3611643"],"URL":"https:\/\/doi.org\/10.1145\/3611643.3616295","relation":{},"subject":[],"published":{"date-parts":[[2023,11,30]]},"assertion":[{"value":"2023-11-30","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}