{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,2,5]],"date-time":"2026-02-05T11:58:45Z","timestamp":1770292725814,"version":"3.49.0"},"publisher-location":"New York, NY, USA","reference-count":85,"publisher":"ACM","license":[{"start":{"date-parts":[[2022,7,18]],"date-time":"2022-07-18T00:00:00Z","timestamp":1658102400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2022,7,18]]},"DOI":"10.1145\/3533767.3534394","type":"proceedings-article","created":{"date-parts":[[2022,7,15]],"date-time":"2022-07-15T14:28:50Z","timestamp":1657895330000},"page":"202-214","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":17,"title":["AEON: a method for automatic evaluation of NLP test cases"],"prefix":"10.1145","author":[{"given":"Jen-tse","family":"Huang","sequence":"first","affiliation":[{"name":"Chinese University of Hong Kong, China"}]},{"given":"Jianping","family":"Zhang","sequence":"additional","affiliation":[{"name":"Chinese University of Hong Kong, China"}]},{"given":"Wenxuan","family":"Wang","sequence":"additional","affiliation":[{"name":"Chinese University of Hong Kong, China"}]},{"given":"Pinjia","family":"He","sequence":"additional","affiliation":[{"name":"Chinese University of Hong Kong at Shenzhen, China"}]},{"given":"Yuxin","family":"Su","sequence":"additional","affiliation":[{"name":"Sun Yat-sen University, China"}]},{"given":"Michael R.","family":"Lyu","sequence":"additional","affiliation":[{"name":"Chinese University of Hong Kong, China"}]}],"member":"320","published-online":{"date-parts":[[2022,7,18]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"publisher","DOI":"10.1145\/3290353"},{"key":"e_1_3_2_1_2_1","volume-title":"Proceedings of the 35th International Conference on Machine Learning (ICML). 80","author":"Athalye Anish","unstructured":"Anish Athalye , Nicholas Carlini , and David A. Wagner . 2018. Obfuscated Gradients Give a False Sense of Security: Circumventing Defenses to Adversarial Examples . In Proceedings of the 35th International Conference on Machine Learning (ICML). 80 , 274\u2013283. Anish Athalye, Nicholas Carlini, and David A. Wagner. 2018. Obfuscated Gradients Give a False Sense of Security: Circumventing Defenses to Adversarial Examples. In Proceedings of the 35th International Conference on Machine Learning (ICML). 80, 274\u2013283."},{"key":"e_1_3_2_1_3_1","volume-title":"Proceedings of the 2015 Conference on Empirical Methods in Natural Language Processing (EMNLP). 632\u2013642","author":"Bowman Samuel R.","unstructured":"Samuel R. Bowman , Gabor Angeli , Christopher Potts , and Christopher D. Manning . 2015. A large annotated corpus for learning natural language inference . In Proceedings of the 2015 Conference on Empirical Methods in Natural Language Processing (EMNLP). 632\u2013642 . Samuel R. Bowman, Gabor Angeli, Christopher Potts, and Christopher D. Manning. 2015. A large annotated corpus for learning natural language inference. In Proceedings of the 2015 Conference on Empirical Methods in Natural Language Processing (EMNLP). 632\u2013642."},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.5555\/3241094.3241135"},{"key":"e_1_3_2_1_5_1","volume-title":"Towards Evaluating the Robustness of Neural Networks. In 2017 IEEE Symposium on Security and Privacy (SP). 39\u201357","author":"Carlini Nicholas","unstructured":"Nicholas Carlini and David A. Wagner . 2017 . Towards Evaluating the Robustness of Neural Networks. In 2017 IEEE Symposium on Security and Privacy (SP). 39\u201357 . Nicholas Carlini and David A. Wagner. 2017. Towards Evaluating the Robustness of Neural Networks. In 2017 IEEE Symposium on Security and Privacy (SP). 39\u201357."},{"key":"e_1_3_2_1_6_1","volume-title":"Evaluation of Text Generation: A Survey. CoRR, abs\/2006.14799","author":"Celikyilmaz Asli","year":"2020","unstructured":"Asli Celikyilmaz , Elizabeth Clark , and Jianfeng Gao . 2020. Evaluation of Text Generation: A Survey. CoRR, abs\/2006.14799 ( 2020 ), arxiv:2006.14799. Asli Celikyilmaz, Elizabeth Clark, and Jianfeng Gao. 2020. Evaluation of Text Generation: A Survey. CoRR, abs\/2006.14799 (2020), arxiv:2006.14799."},{"key":"e_1_3_2_1_7_1","volume-title":"Evolution of Semantic Similarity - A Survey. ACM Comput. Surv., 54, 2","author":"Chandrasekaran Dhivya","year":"2021","unstructured":"Dhivya Chandrasekaran and Vijay Mago . 2021. Evolution of Semantic Similarity - A Survey. ACM Comput. Surv., 54, 2 ( 2021 ), 41:1\u201341:37. Dhivya Chandrasekaran and Vijay Mago. 2021. Evolution of Semantic Similarity - A Survey. ACM Comput. Surv., 54, 2 (2021), 41:1\u201341:37."},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.1109\/24.963124"},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1109\/ASE51524.2021.9678670"},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.1145\/3468264.3468569"},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D17-1070"},{"key":"e_1_3_2_1_12_1","volume-title":"Proceedings of the Ninth Workshop on Statistical Machine Translation (WMT@ACL). 376\u2013380","author":"Michael","unstructured":"Michael J. Denkowski and Alon Lavie. 2014. Meteor Universal: Language Specific Translation Evaluation for Any Target Language . In Proceedings of the Ninth Workshop on Statistical Machine Translation (WMT@ACL). 376\u2013380 . Michael J. Denkowski and Alon Lavie. 2014. Meteor Universal: Language Specific Translation Evaluation for Any Target Language. In Proceedings of the Ninth Workshop on Statistical Machine Translation (WMT@ACL). 376\u2013380."},{"key":"e_1_3_2_1_13_1","volume-title":"Proceedings of the 2019 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies (NAACL-HLT)","volume":"1","author":"Devlin Jacob","year":"2019","unstructured":"Jacob Devlin , Ming-Wei Chang , Kenton Lee , and Kristina Toutanova . 2019 . BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding . In Proceedings of the 2019 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies (NAACL-HLT) , Volume 1 (Long and Short Papers). 4171\u20134186. Jacob Devlin, Ming-Wei Chang, Kenton Lee, and Kristina Toutanova. 2019. BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding. In Proceedings of the 2019 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies (NAACL-HLT), Volume 1 (Long and Short Papers). 4171\u20134186."},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.5555\/1944284.1944298"},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.1145\/3320269.3384733"},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.1145\/3338906.3338954"},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.1145\/3293882.3330566"},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2021.emnlp-main.552"},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.emnlp-main.498"},{"key":"e_1_3_2_1_20_1","volume-title":"Proceedings of the 3rd International Conference on Learning Representations (ICLR).","author":"Goodfellow Ian J.","year":"2015","unstructured":"Ian J. Goodfellow , Jonathon Shlens , and Christian Szegedy . 2015 . Explaining and Harnessing Adversarial Examples . In Proceedings of the 3rd International Conference on Learning Representations (ICLR). Ian J. Goodfellow, Jonathon Shlens, and Christian Szegedy. 2015. Explaining and Harnessing Adversarial Examples. In Proceedings of the 3rd International Conference on Learning Representations (ICLR)."},{"key":"e_1_3_2_1_21_1","volume-title":"Towards Variable-Length Textual Adversarial Attacks. CoRR, abs\/2104.08139","author":"Guo Junliang","year":"2021","unstructured":"Junliang Guo , Zhirui Zhang , Linlin Zhang , Linli Xu , Boxing Chen , Enhong Chen , and Weihua Luo . 2021. Towards Variable-Length Textual Adversarial Attacks. CoRR, abs\/2104.08139 ( 2021 ). Junliang Guo, Zhirui Zhang, Linlin Zhang, Linli Xu, Boxing Chen, Enhong Chen, and Weihua Luo. 2021. Towards Variable-Length Textual Adversarial Attacks. CoRR, abs\/2104.08139 (2021)."},{"key":"e_1_3_2_1_22_1","volume-title":"RNN-Test: Adversarial Testing Framework for Recurrent Neural Network Systems. CoRR, abs\/1911.06155","author":"Guo Jianmin","year":"2019","unstructured":"Jianmin Guo , Yue Zhao , Xueying Han , Yu Jiang , and Jiaguang Sun . 2019. RNN-Test: Adversarial Testing Framework for Recurrent Neural Network Systems. CoRR, abs\/1911.06155 ( 2019 ). Jianmin Guo, Yue Zhao, Xueying Han, Yu Jiang, and Jiaguang Sun. 2019. RNN-Test: Adversarial Testing Framework for Recurrent Neural Network Systems. CoRR, abs\/1911.06155 (2019)."},{"key":"e_1_3_2_1_23_1","volume-title":"Proceedings of the 28th ACM Joint European Software Engineering Conference and Symposium on the Foundations of Software Engineering (ESEC\/FSE). 863\u2013875","author":"Gupta Shashij","year":"2020","unstructured":"Shashij Gupta , Pinjia He , Clara Meister , and Zhendong Su . 2020 . Machine translation testing via pathological invariance . In Proceedings of the 28th ACM Joint European Software Engineering Conference and Symposium on the Foundations of Software Engineering (ESEC\/FSE). 863\u2013875 . Shashij Gupta, Pinjia He, Clara Meister, and Zhendong Su. 2020. Machine translation testing via pathological invariance. In Proceedings of the 28th ACM Joint European Software Engineering Conference and Symposium on the Foundations of Software Engineering (ESEC\/FSE). 863\u2013875."},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"publisher","DOI":"10.1145\/3377811.3380339"},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE43902.2021.00047"},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.1109\/AITest.2019.00-12"},{"key":"e_1_3_2_1_27_1","doi-asserted-by":"publisher","DOI":"10.1109\/ASE.2019.00126"},{"key":"e_1_3_2_1_28_1","volume-title":"Coverage Guided Testing for Recurrent Neural Networks. CoRR, abs\/1911.01952","author":"Huang Wei","year":"2019","unstructured":"Wei Huang , Youcheng Sun , Xingyu Zhao , James Sharp , Wenjie Ruan , Jie Meng , and Xiaowei Huang . 2019. Coverage Guided Testing for Recurrent Neural Networks. CoRR, abs\/1911.01952 ( 2019 ). Wei Huang, Youcheng Sun, Xingyu Zhao, James Sharp, Wenjie Ruan, Jie Meng, and Xiaowei Huang. 2019. Coverage Guided Testing for Recurrent Neural Networks. CoRR, abs\/1911.01952 (2019)."},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/P16-1195"},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"publisher","DOI":"10.1121\/1.2016299"},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D19-1423"},{"key":"e_1_3_2_1_32_1","volume-title":"Joey Tianyi Zhou, and Peter Szolovits","author":"Jin Di","year":"2020","unstructured":"Di Jin , Zhijing Jin , Joey Tianyi Zhou, and Peter Szolovits . 2020 . Is BERT Really Robust? A Strong Baseline for Natural Language Attack on Text Classification and Entailment. In The Thirty-Fourth AAAI Conference on Artificial Intelligence (AAAI), The Thirty-Second Innovative Applications of Artificial Intelligence Conference (IAAI), The Tenth AAAI Symposium on Educational Advances in Artificial Intelligence (EAAI) . 8018\u20138025. Di Jin, Zhijing Jin, Joey Tianyi Zhou, and Peter Szolovits. 2020. Is BERT Really Robust? A Strong Baseline for Natural Language Attack on Text Classification and Entailment. In The Thirty-Fourth AAAI Conference on Artificial Intelligence (AAAI), The Thirty-Second Innovative Applications of Artificial Intelligence Conference (IAAI), The Tenth AAAI Symposium on Educational Advances in Artificial Intelligence (EAAI). 8018\u20138025."},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.acl-main.245"},{"key":"e_1_3_2_1_34_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICNN.1995.488968"},{"key":"e_1_3_2_1_35_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE.2019.00108"},{"key":"e_1_3_2_1_36_1","volume-title":"Proceedings of the 8th International Conference on Learning Representations (ICLR).","author":"Lan Zhenzhong","year":"2020","unstructured":"Zhenzhong Lan , Mingda Chen , Sebastian Goodman , Kevin Gimpel , Piyush Sharma , and Radu Soricut . 2020 . ALBERT: A Lite BERT for Self-supervised Learning of Language Representations . In Proceedings of the 8th International Conference on Learning Representations (ICLR). Zhenzhong Lan, Mingda Chen, Sebastian Goodman, Kevin Gimpel, Piyush Sharma, and Radu Soricut. 2020. ALBERT: A Lite BERT for Self-supervised Learning of Language Representations. In Proceedings of the 8th International Conference on Learning Representations (ICLR)."},{"key":"e_1_3_2_1_37_1","first-page":"707","article-title":"Binary codes capable of correcting deletions, insertions, and reversals","volume":"10","author":"Levenshtein Vladimir I.","year":"1966","unstructured":"Vladimir I. Levenshtein . 1966 . Binary codes capable of correcting deletions, insertions, and reversals . Cybernetics and Control Theory , 10 , 8 (1966), 707 \u2013 710 . Vladimir I. Levenshtein. 1966. Binary codes capable of correcting deletions, insertions, and reversals. Cybernetics and Control Theory, 10, 8 (1966), 707\u2013710.","journal-title":"Cybernetics and Control Theory"},{"key":"e_1_3_2_1_38_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2021.naacl-main.400"},{"key":"e_1_3_2_1_39_1","volume-title":"TextBugger: Generating Adversarial Text Against Real-world Applications. In 26th Annual Network and Distributed System Security Symposium (NDSS).","author":"Li Jinfeng","year":"2019","unstructured":"Jinfeng Li , Shouling Ji , Tianyu Du , Bo Li , and Ting Wang . 2019 . TextBugger: Generating Adversarial Text Against Real-world Applications. In 26th Annual Network and Distributed System Security Symposium (NDSS). Jinfeng Li, Shouling Ji, Tianyu Du, Bo Li, and Ting Wang. 2019. TextBugger: Generating Adversarial Text Against Real-world Applications. In 26th Annual Network and Distributed System Security Symposium (NDSS)."},{"key":"e_1_3_2_1_40_1","doi-asserted-by":"publisher","DOI":"10.24963\/ijcai.2018\/585"},{"key":"e_1_3_2_1_41_1","volume-title":"RoBERTa: A Robustly Optimized BERT Pretraining Approach. CoRR, abs\/1907.11692","author":"Liu Yinhan","year":"2019","unstructured":"Yinhan Liu , Myle Ott , Naman Goyal , Jingfei Du , Mandar Joshi , Danqi Chen , Omer Levy , Mike Lewis , Luke Zettlemoyer , and Veselin Stoyanov . 2019. RoBERTa: A Robustly Optimized BERT Pretraining Approach. CoRR, abs\/1907.11692 ( 2019 ), arxiv:1907.11692. Yinhan Liu, Myle Ott, Naman Goyal, Jingfei Du, Mandar Joshi, Danqi Chen, Omer Levy, Mike Lewis, Luke Zettlemoyer, and Veselin Stoyanov. 2019. RoBERTa: A Robustly Optimized BERT Pretraining Approach. CoRR, abs\/1907.11692 (2019), arxiv:1907.11692."},{"key":"e_1_3_2_1_42_1","doi-asserted-by":"publisher","DOI":"10.1145\/3238147.3238202"},{"key":"e_1_3_2_1_43_1","doi-asserted-by":"publisher","DOI":"10.1109\/ISSRE.2018.00021"},{"key":"e_1_3_2_1_44_1","doi-asserted-by":"publisher","DOI":"10.1145\/3236024.3236082"},{"key":"e_1_3_2_1_45_1","doi-asserted-by":"publisher","DOI":"10.14722\/ndss.2019.23415"},{"key":"e_1_3_2_1_46_1","unstructured":"Edoardo Maggio. 2018. Apple says that 500 million customers use Siri. Online Available:. https:\/\/www.businessinsider.com\/apple-says-siri-has-500-million-users-2018-1. \t\t\t\t\t  Edoardo Maggio. 2018. Apple says that 500 million customers use Siri. Online Available:. https:\/\/www.businessinsider.com\/apple-says-siri-has-500-million-users-2018-1."},{"key":"e_1_3_2_1_47_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/N19-1314"},{"key":"e_1_3_2_1_48_1","doi-asserted-by":"publisher","DOI":"10.1145\/219717.219748"},{"key":"e_1_3_2_1_49_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.findings-emnlp.341"},{"key":"e_1_3_2_1_50_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.emnlp-demos.16"},{"key":"e_1_3_2_1_51_1","unstructured":"Arika Okrent. 2016. 9 Little Translation Mistakes That Caused Big Problems. Online Available:. https:\/\/www.mentalfloss.com\/article\/48795\/9-little-translation-mistakes-caused-big-problems. \t\t\t\t\t  Arika Okrent. 2016. 9 Little Translation Mistakes That Caused Big Problems. Online Available:. https:\/\/www.mentalfloss.com\/article\/48795\/9-little-translation-mistakes-caused-big-problems."},{"key":"e_1_3_2_1_52_1","unstructured":"Ahuy Ong. 2017. Facebook apologizes after wrong translation sees Palestinian man arrested for posting \u2018good morning\u2019. Online Available:. https:\/\/www.theverge.com\/us-world\/2017\/10\/24\/16533496\/facebook-apology-wrong-translation-palestinian-arrested-post-good-morning. \t\t\t\t\t  Ahuy Ong. 2017. Facebook apologizes after wrong translation sees Palestinian man arrested for posting \u2018good morning\u2019. Online Available:. https:\/\/www.theverge.com\/us-world\/2017\/10\/24\/16533496\/facebook-apology-wrong-translation-palestinian-arrested-post-good-morning."},{"key":"e_1_3_2_1_53_1","doi-asserted-by":"publisher","DOI":"10.3115\/1219840.1219855"},{"key":"e_1_3_2_1_54_1","volume-title":"Proceedings of the 40th Annual Meeting of the Association for Computational Linguistics (ACL). 311\u2013318","author":"Papineni Kishore","year":"2002","unstructured":"Kishore Papineni , Salim Roukos , Todd Ward , and Wei-Jing Zhu . 2002 . Bleu: a Method for Automatic Evaluation of Machine Translation . In Proceedings of the 40th Annual Meeting of the Association for Computational Linguistics (ACL). 311\u2013318 . Kishore Papineni, Salim Roukos, Todd Ward, and Wei-Jing Zhu. 2002. Bleu: a Method for Automatic Evaluation of Machine Translation. In Proceedings of the 40th Annual Meeting of the Association for Computational Linguistics (ACL). 311\u2013318."},{"key":"e_1_3_2_1_55_1","doi-asserted-by":"publisher","DOI":"10.1145\/3132747.3132785"},{"key":"e_1_3_2_1_56_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE.2019.00107"},{"key":"e_1_3_2_1_57_1","volume-title":"Deep Learning: Algorithms, Techniques, and Applications. ACM Comput. Surv., 51, 5","author":"Pouyanfar Samira","year":"2019","unstructured":"Samira Pouyanfar , Saad Sadiq , Yilin Yan , Haiman Tian , Yudong Tao , Maria E. Presa Reyes , Mei-Ling Shyu , Shu-Ching Chen , and S. S. Iyengar . 2019 . A Survey on Deep Learning: Algorithms, Techniques, and Applications. ACM Comput. Surv., 51, 5 (2019), 92:1\u201392:36. Samira Pouyanfar, Saad Sadiq, Yilin Yan, Haiman Tian, Yudong Tao, Maria E. Presa Reyes, Mei-Ling Shyu, Shu-Ching Chen, and S. S. Iyengar. 2019. A Survey on Deep Learning: Algorithms, Techniques, and Applications. ACM Comput. Surv., 51, 5 (2019), 92:1\u201392:36."},{"key":"e_1_3_2_1_58_1","doi-asserted-by":"publisher","DOI":"10.1145\/3276517"},{"key":"e_1_3_2_1_59_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D19-1410"},{"key":"e_1_3_2_1_60_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/P19-1103"},{"key":"e_1_3_2_1_61_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.acl-main.442"},{"key":"e_1_3_2_1_62_1","volume-title":"Incredible Amazon Alexa Statistics You Need to Know","year":"2021","unstructured":"SafeAtLast. 2021. Incredible Amazon Alexa Statistics You Need to Know in 2021 . Online , Available :. https:\/\/safeatlast.co\/blog\/amazon-alexa-statistics\/gref. SafeAtLast. 2021. Incredible Amazon Alexa Statistics You Need to Know in 2021. Online, Available:. https:\/\/safeatlast.co\/blog\/amazon-alexa-statistics\/gref."},{"key":"e_1_3_2_1_63_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/P16-1162"},{"key":"e_1_3_2_1_64_1","doi-asserted-by":"publisher","DOI":"10.1145\/3377811.3380420"},{"key":"e_1_3_2_1_65_1","volume-title":"Proceedings of Advances in Neural Information Processing Systems 31: Annual Conference on Neural Information Processing Systems 2018 (NeurIPS). 7728\u20137739","author":"Tao Guanhong","year":"2018","unstructured":"Guanhong Tao , Shiqing Ma , Yingqi Liu , and Xiangyu Zhang . 2018 . Attacks Meet Interpretability: Attribute-steered Detection of Adversarial Samples . In Proceedings of Advances in Neural Information Processing Systems 31: Annual Conference on Neural Information Processing Systems 2018 (NeurIPS). 7728\u20137739 . Guanhong Tao, Shiqing Ma, Yingqi Liu, and Xiangyu Zhang. 2018. Attacks Meet Interpretability: Attribute-steered Detection of Adversarial Samples. In Proceedings of Advances in Neural Information Processing Systems 31: Annual Conference on Neural Information Processing Systems 2018 (NeurIPS). 7728\u20137739."},{"key":"e_1_3_2_1_66_1","doi-asserted-by":"publisher","DOI":"10.1145\/3180155.3180220"},{"key":"e_1_3_2_1_67_1","unstructured":"Barak Turovsky. 2016. Ten years of Google Translate. Online Available:. https:\/\/blog.google\/products\/translate\/ten-years-of-google-translate. \t\t\t\t\t  Barak Turovsky. 2016. Ten years of Google Translate. Online Available:. https:\/\/blog.google\/products\/translate\/ten-years-of-google-translate."},{"key":"e_1_3_2_1_68_1","volume-title":"Advances in Neural Information Processing Systems 30: Annual Conference on Neural Information Processing Systems 2017 (NIPS). 5998\u20136008","author":"Vaswani Ashish","year":"2017","unstructured":"Ashish Vaswani , Noam Shazeer , Niki Parmar , Jakob Uszkoreit , Llion Jones , Aidan N. Gomez , Lukasz Kaiser , and Illia Polosukhin . 2017 . Attention is All you Need . In Advances in Neural Information Processing Systems 30: Annual Conference on Neural Information Processing Systems 2017 (NIPS). 5998\u20136008 . Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, Llion Jones, Aidan N. Gomez, Lukasz Kaiser, and Illia Polosukhin. 2017. Attention is All you Need. In Advances in Neural Information Processing Systems 30: Annual Conference on Neural Information Processing Systems 2017 (NIPS). 5998\u20136008."},{"key":"e_1_3_2_1_69_1","volume-title":"Proceedings of the 7th International Conference on Learning Representations (ICLR).","author":"Wang Alex","unstructured":"Alex Wang , Amanpreet Singh , Julian Michael , Felix Hill , Omer Levy , and Samuel R. Bowman . 2019. GLUE: A Multi-Task Benchmark and Analysis Platform for Natural Language Understanding . In Proceedings of the 7th International Conference on Learning Representations (ICLR). Alex Wang, Amanpreet Singh, Julian Michael, Felix Hill, Omer Levy, and Samuel R. Bowman. 2019. GLUE: A Multi-Task Benchmark and Analysis Platform for Natural Language Understanding. In Proceedings of the 7th International Conference on Learning Representations (ICLR)."},{"key":"e_1_3_2_1_70_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE.2019.00126"},{"key":"e_1_3_2_1_71_1","volume-title":"Proceedings of the 2018 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies (NAACL-HLT)","volume":"1","author":"Williams Adina","unstructured":"Adina Williams , Nikita Nangia , and Samuel R. Bowman . 2018. A Broad-Coverage Challenge Corpus for Sentence Understanding through Inference . In Proceedings of the 2018 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies (NAACL-HLT) , Volume 1 (Long Papers). 1112\u20131122. Adina Williams, Nikita Nangia, and Samuel R. Bowman. 2018. A Broad-Coverage Challenge Corpus for Sentence Understanding through Inference. In Proceedings of the 2018 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies (NAACL-HLT), Volume 1 (Long Papers). 1112\u20131122."},{"key":"e_1_3_2_1_72_1","volume-title":"Towards Global Explanations of Convolutional Neural Networks With Concept Attribution. In 2020 IEEE\/CVF Conference on Computer Vision and Pattern Recognition (CVPR). 8649\u20138658","author":"Wu Weibin","year":"2020","unstructured":"Weibin Wu , Yuxin Su , Xixian Chen , Shenglin Zhao , Irwin King , Michael R. Lyu , and Yu-Wing Tai . 2020 . Towards Global Explanations of Convolutional Neural Networks With Concept Attribution. In 2020 IEEE\/CVF Conference on Computer Vision and Pattern Recognition (CVPR). 8649\u20138658 . Weibin Wu, Yuxin Su, Xixian Chen, Shenglin Zhao, Irwin King, Michael R. Lyu, and Yu-Wing Tai. 2020. Towards Global Explanations of Convolutional Neural Networks With Concept Attribution. In 2020 IEEE\/CVF Conference on Computer Vision and Pattern Recognition (CVPR). 8649\u20138658."},{"key":"e_1_3_2_1_73_1","volume-title":"Improving the Transferability of Adversarial Samples With Adversarial Transformations. In IEEE Conference on Computer Vision and Pattern Recognition (CVPR). 9024\u20139033","author":"Wu Weibin","year":"2021","unstructured":"Weibin Wu , Yuxin Su , Michael R. Lyu , and Irwin King . 2021 . Improving the Transferability of Adversarial Samples With Adversarial Transformations. In IEEE Conference on Computer Vision and Pattern Recognition (CVPR). 9024\u20139033 . Weibin Wu, Yuxin Su, Michael R. Lyu, and Irwin King. 2021. Improving the Transferability of Adversarial Samples With Adversarial Transformations. In IEEE Conference on Computer Vision and Pattern Recognition (CVPR). 9024\u20139033."},{"key":"e_1_3_2_1_74_1","volume-title":"Deep Validation: Toward Detecting Real-World Corner Cases for Deep Neural Networks. In 49th Annual IEEE\/IFIP International Conference on Dependable Systems and Networks (DSN). 125\u2013137","author":"Wu Weibin","year":"2019","unstructured":"Weibin Wu , Hui Xu , Sanqiang Zhong , Michael R. Lyu , and Irwin King . 2019 . Deep Validation: Toward Detecting Real-World Corner Cases for Deep Neural Networks. In 49th Annual IEEE\/IFIP International Conference on Dependable Systems and Networks (DSN). 125\u2013137 . Weibin Wu, Hui Xu, Sanqiang Zhong, Michael R. Lyu, and Irwin King. 2019. Deep Validation: Toward Detecting Real-World Corner Cases for Deep Neural Networks. In 49th Annual IEEE\/IFIP International Conference on Dependable Systems and Networks (DSN). 125\u2013137."},{"key":"e_1_3_2_1_75_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2019.00935"},{"key":"e_1_3_2_1_76_1","doi-asserted-by":"publisher","DOI":"10.1145\/3293882.3330579"},{"key":"e_1_3_2_1_77_1","doi-asserted-by":"publisher","DOI":"10.14722\/ndss.2018.23198"},{"key":"e_1_3_2_1_78_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.acl-main.540"},{"key":"e_1_3_2_1_79_1","doi-asserted-by":"publisher","DOI":"10.1145\/3368089.3409750"},{"key":"e_1_3_2_1_80_1","volume-title":"Lyu","author":"Zhang Jianping","year":"2022","unstructured":"Jianping Zhang , Weibin Wu , Jen-tse Huang, Yizhan Huang , Wenxuan Wang , Yuxin Su , and Michael R . Lyu . 2022 . Improving Adversarial Transferability via Neuron Attribution-Based Attacks. CoRR , abs\/2204.00008 (2022). Jianping Zhang, Weibin Wu, Jen-tse Huang, Yizhan Huang, Wenxuan Wang, Yuxin Su, and Michael R. Lyu. 2022. Improving Adversarial Transferability via Neuron Attribution-Based Attacks. CoRR, abs\/2204.00008 (2022)."},{"key":"e_1_3_2_1_81_1","article-title":"Machine Learning Testing: Survey, Landscapes and Horizons","author":"Zhang Jie M.","year":"2020","unstructured":"Jie M. Zhang , Mark Harman , Lei Ma , and Yang Liu . 2020 . Machine Learning Testing: Survey, Landscapes and Horizons . IEEE Trans. Software Eng.. Jie M. Zhang, Mark Harman, Lei Ma, and Yang Liu. 2020. Machine Learning Testing: Survey, Landscapes and Horizons. IEEE Trans. Software Eng..","journal-title":"IEEE Trans. Software Eng.."},{"key":"e_1_3_2_1_82_1","doi-asserted-by":"publisher","DOI":"10.1145\/3238147.3238187"},{"key":"e_1_3_2_1_83_1","volume-title":"Proceedings of the 8th International Conference on Learning Representations (ICLR).","author":"Zhang Tianyi","year":"2020","unstructured":"Tianyi Zhang , Varsha Kishore , Felix Wu , Kilian Q. Weinberger , and Yoav Artzi . 2020 . BERTScore: Evaluating Text Generation with BERT . In Proceedings of the 8th International Conference on Learning Representations (ICLR). Tianyi Zhang, Varsha Kishore, Felix Wu, Kilian Q. Weinberger, and Yoav Artzi. 2020. BERTScore: Evaluating Text Generation with BERT. In Proceedings of the 8th International Conference on Learning Representations (ICLR)."},{"key":"e_1_3_2_1_84_1","doi-asserted-by":"publisher","DOI":"10.1145\/3374217"},{"key":"e_1_3_2_1_85_1","volume-title":"Character-level Convolutional Networks for Text Classification. In Advances in Neural Information Processing Systems 28: Annual Conference on Neural Information Processing Systems 2015 (NIPS). 649\u2013657","author":"Zhang Xiang","year":"2015","unstructured":"Xiang Zhang , Junbo Jake Zhao , and Yann LeCun . 2015 . Character-level Convolutional Networks for Text Classification. In Advances in Neural Information Processing Systems 28: Annual Conference on Neural Information Processing Systems 2015 (NIPS). 649\u2013657 . Xiang Zhang, Junbo Jake Zhao, and Yann LeCun. 2015. Character-level Convolutional Networks for Text Classification. In Advances in Neural Information Processing Systems 28: Annual Conference on Neural Information Processing Systems 2015 (NIPS). 649\u2013657."}],"event":{"name":"ISSTA '22: 31st ACM SIGSOFT International Symposium on Software Testing and Analysis","location":"Virtual South Korea","acronym":"ISSTA '22","sponsor":["SIGSOFT ACM Special Interest Group on Software Engineering"]},"container-title":["Proceedings of the 31st ACM SIGSOFT International Symposium on Software Testing and Analysis"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3533767.3534394","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3533767.3534394","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,18]],"date-time":"2025-06-18T18:43:41Z","timestamp":1750272221000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3533767.3534394"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2022,7,18]]},"references-count":85,"alternative-id":["10.1145\/3533767.3534394","10.1145\/3533767"],"URL":"https:\/\/doi.org\/10.1145\/3533767.3534394","relation":{},"subject":[],"published":{"date-parts":[[2022,7,18]]},"assertion":[{"value":"2022-07-18","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}