{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,3,14]],"date-time":"2026-03-14T09:53:40Z","timestamp":1773482020493,"version":"3.50.1"},"publisher-location":"New York, NY, USA","reference-count":75,"publisher":"ACM","license":[{"start":{"date-parts":[[2023,11,30]],"date-time":"2023-11-30T00:00:00Z","timestamp":1701302400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"name":"National Science Foundation of China","award":["61932021, 62141210, 62002125"],"award-info":[{"award-number":["61932021, 62141210, 62002125"]}]},{"name":"the Hong Kong Research Grant Council\/General Research Fund","award":["16205722"],"award-info":[{"award-number":["16205722"]}]},{"name":"the Hong Kong Research Grant Council\/Research Impact Fund","award":["R5034-18"],"award-info":[{"award-number":["R5034-18"]}]},{"name":"the Hong Kong Innovation and Technology Fund","award":["MHP\/055\/19"],"award-info":[{"award-number":["MHP\/055\/19"]}]},{"name":"the Young Elite Scientists Sponsorship Program by CAST","award":["2021QNRC001"],"award-info":[{"award-number":["2021QNRC001"]}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2023,11,30]]},"DOI":"10.1145\/3611643.3616258","type":"proceedings-article","created":{"date-parts":[[2023,11,30]],"date-time":"2023-11-30T23:14:38Z","timestamp":1701386078000},"page":"107-119","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":3,"title":["Testing Coreference Resolution Systems without Labeled Test Sets"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-4892-6294","authenticated-orcid":false,"given":"Jialun","family":"Cao","sequence":"first","affiliation":[{"name":"Hong Kong University of Science and Technology, Hong Kong, China \/ Guangzhou HKUST Fok Ying Tung Research Institute, Guangzhou, China"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-5842-7715","authenticated-orcid":false,"given":"Yaojie","family":"Lu","sequence":"additional","affiliation":[{"name":"Chinese Institute of Software at Chinese Academy of Sciences, Beijing, China"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-5588-9618","authenticated-orcid":false,"given":"Ming","family":"Wen","sequence":"additional","affiliation":[{"name":"Huazhong University of Science and Technology, Hubei, China"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-3508-7172","authenticated-orcid":false,"given":"Shing-Chi","family":"Cheung","sequence":"additional","affiliation":[{"name":"Hong Kong University of Science and Technology, Hong Kong, China \/ Guangzhou HKUST Fok Ying Tung Research Institute, Hong Kong, China"}]}],"member":"320","published-online":{"date-parts":[[2023,11,30]]},"reference":[{"key":"e_1_3_2_2_1_1","doi-asserted-by":"publisher","DOI":"10.1145\/3290353"},{"key":"e_1_3_2_2_2_1","unstructured":"Anonymous. 2023. CREST. https:\/\/anonymous.4open.science\/r\/Crest_FSE2023\/"},{"key":"e_1_3_2_2_3_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1"},{"key":"e_1_3_2_2_4_1","doi-asserted-by":"publisher","DOI":"10.5555\/1608810.1608825"},{"key":"e_1_3_2_2_5_1","doi-asserted-by":"publisher","DOI":"10.3115\/1613715.1613756"},{"key":"e_1_3_2_2_6_1","doi-asserted-by":"publisher","DOI":"10.1145\/3490488"},{"key":"e_1_3_2_2_7_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1"},{"key":"e_1_3_2_2_8_1","doi-asserted-by":"publisher","DOI":"10.1145\/3468264.3468569"},{"key":"e_1_3_2_2_9_1","volume-title":"Metamorphic Testing: A New Approach for Generating Next Test Cases. In Technical Report HKUST-CS98-01. CoRR, abs\/2002.12543, 11. arXiv:2002.12543. arxiv:2002.12543","author":"Chen Tsong Yueh","year":"2020","unstructured":"Tsong Yueh Chen, S. C. Cheung, and Siu-Ming Yiu. 2020. Metamorphic Testing: A New Approach for Generating Next Test Cases. In Technical Report HKUST-CS98-01. CoRR, abs\/2002.12543, 11. arXiv:2002.12543. arxiv:2002.12543"},{"key":"e_1_3_2_2_10_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1"},{"key":"e_1_3_2_2_11_1","doi-asserted-by":"publisher","DOI":"10.3115\/v1"},{"key":"e_1_3_2_2_12_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1"},{"key":"e_1_3_2_2_13_1","volume-title":"Proceedings of the 54th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers). Association for Computational Linguistics","author":"Clark Kevin","unstructured":"Kevin Clark and Christopher D. Manning. 2016. Improving Coreference Resolution by Learning Entity-Level Distributed Representations. In Proceedings of the 54th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers). Association for Computational Linguistics, Berlin, Germany. 643\u2013653."},{"key":"e_1_3_2_2_14_1","volume-title":"BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding. In Proceedings of the 2019 Conference of the North American","author":"Devlin Jacob","year":"2019","unstructured":"Jacob Devlin, Ming-Wei Chang, Kenton Lee, and Kristina Toutanova. 2019. BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding. In Proceedings of the 2019 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, NAACL-HLT. Association for Computational Linguistics, Online. 4171\u20134186."},{"key":"e_1_3_2_2_15_1","first-page":"13","volume-title":"Proceedings of the 2013 Conference on Empirical Methods in Natural Language Processing. Association for Computational Linguistics","author":"Durrett Greg","year":"2013","unstructured":"Greg Durrett and Dan Klein. 2013. Easy Victories and Uphill Battles in Coreference Resolution. In Proceedings of the 2013 Conference on Empirical Methods in Natural Language Processing. Association for Computational Linguistics, Seattle, Washington, USA. 1971\u20131982. https:\/\/aclanthology.org\/D13-1203"},{"key":"e_1_3_2_2_16_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1"},{"key":"e_1_3_2_2_17_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1"},{"key":"e_1_3_2_2_18_1","first-page":"12","article-title":"Coreference resolution: A survey. University of Wisconsin, Madison","volume":"1","author":"Elango Pradheep","year":"2005","unstructured":"Pradheep Elango. 2005. Coreference resolution: A survey. University of Wisconsin, Madison, WI, 1, 12 (2005), 12.","journal-title":"WI"},{"key":"e_1_3_2_2_19_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1"},{"key":"e_1_3_2_2_20_1","doi-asserted-by":"publisher","DOI":"10.1145\/3368089.3409756"},{"key":"e_1_3_2_2_21_1","volume-title":"Entity-Centered Model. In Human Language Technologies: The 2010 Annual Conference of the North American","author":"Haghighi Aria","unstructured":"Aria Haghighi and Dan Klein. 2010. Coreference Resolution in a Modular, Entity-Centered Model. In Human Language Technologies: The 2010 Annual Conference of the North American Chapter of the Association for Computational Linguistics. Association for Computational Linguistics, USA. 385\u2013393."},{"key":"e_1_3_2_2_22_1","doi-asserted-by":"publisher","DOI":"10.1145\/3377811.3380339"},{"key":"e_1_3_2_2_23_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE43902.2021.00047"},{"key":"e_1_3_2_2_24_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1"},{"key":"e_1_3_2_2_25_1","volume-title":"Seventh Message Understanding Conference (MUC-7): Proceedings of a Conference Held in Fairfax","author":"Hirschman Lynette","year":"1998","unstructured":"Lynette Hirschman and Nancy Chinchor. 1998. Appendix F: MUC-7 Coreference Task Definition (version 3.0). In Seventh Message Understanding Conference (MUC-7): Proceedings of a Conference Held in Fairfax, Virginia, April 29 - May 1, 1998. Proceedings of a Conference Held in Fairfax, Virginia, Fairfax, Virginia. 17. https:\/\/aclanthology.org\/M98-1029"},{"key":"e_1_3_2_2_26_1","volume-title":"Resolving Pronoun References","author":"Hobbs J","unstructured":"J Hobbs. 1986. Resolving Pronoun References. Morgan Kaufmann Publishers Inc., San Francisco, CA, USA. 339\u2013352. isbn:0934613117"},{"key":"e_1_3_2_2_27_1","doi-asserted-by":"publisher","DOI":"10.5281\/zenodo.1212303"},{"key":"e_1_3_2_2_28_1","doi-asserted-by":"publisher","DOI":"10.1145\/3533767.3534394"},{"key":"e_1_3_2_2_29_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/P16-1195"},{"key":"e_1_3_2_2_30_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1"},{"key":"e_1_3_2_2_31_1","volume-title":"Proceedings of the 2016 Text Analysis Conference, TAC 2016","author":"Ji Heng","year":"2016","unstructured":"Heng Ji and Joel Nothman. 2016. Overview of TAC-KBP2016 Tri-lingual EDL and Its Impact on End-to-End KBP. In Proceedings of the 2016 Text Analysis Conference, TAC 2016, Gaithersburg, Maryland, USA, November 14-15, 2016. NIST, USA. 15."},{"key":"e_1_3_2_2_32_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1"},{"key":"e_1_3_2_2_33_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-01267-0_10"},{"key":"e_1_3_2_2_34_1","volume-title":"Proceedings of the 2013 Conference on Empirical Methods in Natural Language Processing. Association for Computational Linguistics","author":"Jonathan","unstructured":"Jonathan K. Kummerfeld and Dan Klein. 2013. Error-Driven Analysis of Challenges in Coreference Resolution. In Proceedings of the 2013 Conference on Empirical Methods in Natural Language Processing. Association for Computational Linguistics, Seattle, Washington, USA. 265\u2013277."},{"key":"e_1_3_2_2_35_1","volume-title":"Proceedings of the Fifteenth Conference on Computational Natural Language Learning: Shared Task. Association for Computational Linguistics","author":"Lee Heeyoung","year":"2011","unstructured":"Heeyoung Lee, Yves Peirsman, Angel Chang, Nathanael Chambers, Mihai Surdeanu, and Dan Jurafsky. 2011. Stanford\u2019s Multi-Pass Sieve Coreference Resolution System at the CoNLL-2011 Shared Task. In Proceedings of the Fifteenth Conference on Computational Natural Language Learning: Shared Task. Association for Computational Linguistics, Portland, Oregon, USA. 28\u201334. https:\/\/aclanthology.org\/W11-1902"},{"key":"e_1_3_2_2_36_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1"},{"key":"e_1_3_2_2_37_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1"},{"key":"e_1_3_2_2_38_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2021.sigdial-1.53"},{"key":"e_1_3_2_2_39_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1"},{"key":"e_1_3_2_2_40_1","doi-asserted-by":"publisher","DOI":"10.3115\/v1"},{"key":"e_1_3_2_2_41_1","doi-asserted-by":"publisher","DOI":"10.1145\/3487043"},{"key":"e_1_3_2_2_42_1","doi-asserted-by":"publisher","DOI":"10.1145\/219717.219748"},{"key":"e_1_3_2_2_43_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.emnlp-demos.16"},{"key":"e_1_3_2_2_44_1","doi-asserted-by":"publisher","DOI":"10.5555\/1608810.1608826"},{"key":"e_1_3_2_2_45_1","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v31i1.11149"},{"key":"e_1_3_2_2_46_1","doi-asserted-by":"publisher","DOI":"10.3115\/1073083.1073102"},{"key":"e_1_3_2_2_47_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2019.00684"},{"key":"e_1_3_2_2_48_1","doi-asserted-by":"publisher","DOI":"10.1145\/3193977.3193980"},{"key":"e_1_3_2_2_49_1","doi-asserted-by":"publisher","DOI":"10.1145\/3276517"},{"key":"e_1_3_2_2_50_1","volume-title":"Proceedings of the Seventeenth Conference on Computational Natural Language Learning. Association for Computational Linguistics","author":"Pradhan Sameer","year":"2013","unstructured":"Sameer Pradhan, Alessandro Moschitti, Nianwen Xue, Hwee Tou Ng, Anders Bj\u00f6rkelund, Olga Uryupina, Yuchen Zhang, and Zhi Zhong. 2013. Towards Robust Linguistic Analysis using OntoNotes. In Proceedings of the Seventeenth Conference on Computational Natural Language Learning. Association for Computational Linguistics, Seattle, Washington, USA. 143\u2013152."},{"key":"e_1_3_2_2_51_1","volume-title":"CoNLL-2012 Shared Task: Modeling Multilingual Unrestricted Coreference in OntoNotes. In Joint Conference on EMNLP and CoNLL - Shared Task (CoNLL \u201912)","author":"Pradhan Sameer","year":"2012","unstructured":"Sameer Pradhan, Alessandro Moschitti, Nianwen Xue, Olga Uryupina, and Yuchen Zhang. 2012. CoNLL-2012 Shared Task: Modeling Multilingual Unrestricted Coreference in OntoNotes. In Joint Conference on EMNLP and CoNLL - Shared Task (CoNLL \u201912). Association for Computational Linguistics, USA. 1\u201340."},{"key":"e_1_3_2_2_52_1","doi-asserted-by":"publisher","DOI":"10.5555\/1870658.1870706"},{"key":"e_1_3_2_2_53_1","volume-title":"Proceedings of the 2013 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies. Association for Computational Linguistics","author":"Recasens Marta","year":"2013","unstructured":"Marta Recasens, Marie-Catherine de Marneffe, and Christopher Potts. 2013. The Life and Death of Discourse Entities: Identifying Singleton Mentions. In Proceedings of the 2013 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies. Association for Computational Linguistics, Atlanta, Georgia. 627\u2013633. https:\/\/aclanthology.org\/N13-1071"},{"key":"e_1_3_2_2_54_1","doi-asserted-by":"publisher","DOI":"10.1017\/S135132491000029X"},{"key":"e_1_3_2_2_55_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D19-1410"},{"key":"e_1_3_2_2_56_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1"},{"key":"e_1_3_2_2_57_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1"},{"key":"e_1_3_2_2_58_1","volume-title":"Proceedings of the 2021 Conference on Empirical Methods in Natural Language Processing: System Demonstrations. Association for Computational Linguistics, Online. 308\u2013318","author":"Simoncini Walter","year":"2021","unstructured":"Walter Simoncini and Gerasimos Spanakis. 2021. SeqAttack: On adversarial attacks for named entity recognition. In Proceedings of the 2021 Conference on Empirical Methods in Natural Language Processing: System Demonstrations. Association for Computational Linguistics, Online. 308\u2013318."},{"key":"e_1_3_2_2_59_1","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2022.3141758"},{"key":"e_1_3_2_2_60_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1"},{"key":"e_1_3_2_2_61_1","doi-asserted-by":"publisher","DOI":"10.1145\/3377811.3380420"},{"key":"e_1_3_2_2_62_1","doi-asserted-by":"publisher","DOI":"10.1145\/3510003.3510206"},{"key":"e_1_3_2_2_63_1","volume-title":"International Conference on Learning Representations. International Conference on Learning Representations","author":"Szegedy Christian","year":"2014","unstructured":"Christian Szegedy, Wojciech Zaremba, Ilya Sutskever, Joan Bruna, Dumitru Erhan, Ian J. Goodfellow, and Rob Fergus. 2014. Intriguing properties of neural networks. In International Conference on Learning Representations. International Conference on Learning Representations, Banff, Canada. 10."},{"key":"e_1_3_2_2_64_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1"},{"key":"e_1_3_2_2_65_1","doi-asserted-by":"publisher","DOI":"10.1109\/TKDE.2021.3117608"},{"key":"e_1_3_2_2_66_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1"},{"key":"e_1_3_2_2_67_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1"},{"key":"e_1_3_2_2_68_1","doi-asserted-by":"publisher","DOI":"10.1007\/s11633-019-1211-x"},{"key":"e_1_3_2_2_69_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1"},{"key":"e_1_3_2_2_70_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1"},{"key":"e_1_3_2_2_71_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1"},{"key":"e_1_3_2_2_72_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1"},{"key":"e_1_3_2_2_73_1","doi-asserted-by":"publisher","DOI":"10.1145\/3374217"},{"key":"e_1_3_2_2_74_1","volume-title":"Proceedings of the 25th Australasian Software Engineering Conference (ASWEC). IEEE Computer Society, Adelaide, SA, Australia. 96\u2013100","author":"Zhou Zhi Quan","year":"2018","unstructured":"Zhi Quan Zhou and Liqun Sun. 2018. Metamorphic Testing for Machine Translations: MT4MT. In Proceedings of the 25th Australasian Software Engineering Conference (ASWEC). IEEE Computer Society, Adelaide, SA, Australia. 96\u2013100."},{"key":"e_1_3_2_2_75_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1"}],"event":{"name":"ESEC\/FSE '23: 31st ACM Joint European Software Engineering Conference and Symposium on the Foundations of Software Engineering","location":"San Francisco CA USA","acronym":"ESEC\/FSE '23","sponsor":["SIGSOFT ACM Special Interest Group on Software Engineering"]},"container-title":["Proceedings of the 31st ACM Joint European Software Engineering Conference and Symposium on the Foundations of Software Engineering"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3611643.3616258","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3611643.3616258","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T16:36:03Z","timestamp":1750178163000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3611643.3616258"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023,11,30]]},"references-count":75,"alternative-id":["10.1145\/3611643.3616258","10.1145\/3611643"],"URL":"https:\/\/doi.org\/10.1145\/3611643.3616258","relation":{},"subject":[],"published":{"date-parts":[[2023,11,30]]},"assertion":[{"value":"2023-11-30","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}