{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,3,17]],"date-time":"2026-03-17T00:53:29Z","timestamp":1773708809078,"version":"3.50.1"},"publisher-location":"New York, NY, USA","reference-count":98,"publisher":"ACM","license":[{"start":{"date-parts":[[2020,6,27]],"date-time":"2020-06-27T00:00:00Z","timestamp":1593216000000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2020,6,27]]},"DOI":"10.1145\/3377811.3380339","type":"proceedings-article","created":{"date-parts":[[2020,10,1]],"date-time":"2020-10-01T18:25:38Z","timestamp":1601576738000},"page":"961-973","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":66,"title":["Structure-invariant testing for machine translation"],"prefix":"10.1145","author":[{"given":"Pinjia","family":"He","sequence":"first","affiliation":[{"name":"ETH Zurich, Switzerland"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Clara","family":"Meister","sequence":"additional","affiliation":[{"name":"ETH Zurich, Switzerland"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Zhendong","family":"Su","sequence":"additional","affiliation":[{"name":"ETH Zurich, Switzerland"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2020,10]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"publisher","DOI":"10.1145\/3290353"},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D18-1316"},{"key":"e_1_3_2_1_3_1","volume-title":"Proceedings of the 35th International Conference on Machine Learning (ICML).","author":"Athalye Anish","year":"2018","unstructured":"Anish Athalye, Nicholas Carlini, and David Wagner. 2018. Obfuscated Gradients Give a False Sense of Security: Circumventing Defenses to Adversarial Examples. In Proceedings of the 35th International Conference on Machine Learning (ICML)."},{"key":"e_1_3_2_1_4_1","volume-title":"Proceedings of the 6th International Conference on Learning Representations (ICLR).","author":"Belinkov Yonatan","year":"2018","unstructured":"Yonatan Belinkov and Yonatan Bisk. 2018. Synthetic and Natural Noise Both Break Neural Machine Translation. In Proceedings of the 6th International Conference on Learning Representations (ICLR)."},{"key":"e_1_3_2_1_5_1","unstructured":"Bing. [n.d.]. Bing Microsoft Translator. https:\/\/www.bing.com\/translator"},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"publisher","DOI":"10.5555\/3241094.3241135"},{"key":"e_1_3_2_1_7_1","volume-title":"Towards Evaluating the Robustness of Neural Networks. In IEEE Symposium on Security and Privacy.","author":"Carlini Nicholas","year":"2017","unstructured":"Nicholas Carlini and David Wagner. 2017. Towards Evaluating the Robustness of Neural Networks. In IEEE Symposium on Security and Privacy."},{"key":"e_1_3_2_1_8_1","unstructured":"Isaac Caswell Onkur Sen and Allen Nie. 2015. Exploring adversarial learning on neural network models for text classification. (2015)."},{"key":"e_1_3_2_1_10_1","volume-title":"Proceedings of the 54th Annual Meeting of the Association for Computational Linguistics (ACL).","author":"Chen Danqi","unstructured":"Danqi Chen, Jason Bolton, and Christopher D. Manning. 2016. A Thorough Examination of the CNN\/Daily Mail Reading Comprehension Task. In Proceedings of the 54th Annual Meeting of the Association for Computational Linguistics (ACL)."},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.3115\/v1\/D14-1082"},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1145\/3143561"},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"crossref","unstructured":"N. Chomsky. 1957. Syntactic Structures. Mouton The Hague.","DOI":"10.1515\/9783112316009"},{"key":"e_1_3_2_1_15_1","volume-title":"Proceedings of the 55th Annual Meeting of the Association for Computational Linguistics (ACL).","author":"Chu Chenhui","year":"2017","unstructured":"Chenhui Chu, Raj Dabre, and Sadao Kurohashi. 2017. An empirical comparison of simple domain adaptation methods for neural machine translation. In Proceedings of the 55th Annual Meeting of the Association for Computational Linguistics (ACL)."},{"key":"e_1_3_2_1_16_1","unstructured":"CWMT. [n.d.]. CWMT Datasets. http:\/\/nlp.nju.edu.cn\/cwmt-wmt\/"},{"key":"e_1_3_2_1_17_1","unstructured":"Gareth Davies. 2017. Palestinian man is arrested by police after posting 'Good morning' in Arabic on Facebook which was wrongly translated as 'attack them'. https:\/\/www.dailymail.co.uk\/news\/article-5005489\/Good-morning-Facebook-post-leads-arrest-Palestinian.html"},{"key":"e_1_3_2_1_18_1","volume-title":"Proceedings of the Ninth International Conference on Language Resources and Evaluation (LREC).","author":"de Marneffe Marie-Catherine","unstructured":"Marie-Catherine de Marneffe, Timothy Dozat, Natalia Silveira, Katri Haverinen, Filip Ginter, Joakim Nivre, and Christopher D. Manning. [n.d.]. Universal Stanford dependencies: A cross-linguistic typology. In Proceedings of the Ninth International Conference on Language Resources and Evaluation (LREC)."},{"key":"e_1_3_2_1_19_1","volume-title":"Bert: Pre-training of Deep Bidirectional Transformers for Language Understanding. arXiv preprint arXiv:1810.04805","author":"Devlin Jacob","year":"2018","unstructured":"Jacob Devlin, Ming-Wei Chang, Kenton Lee, and Kristina Toutanova. 2018. Bert: Pre-training of Deep Bidirectional Transformers for Language Understanding. arXiv preprint arXiv:1810.04805 (2018)."},{"key":"e_1_3_2_1_20_1","volume-title":"SirenAttack: Generating Adversarial Audio for End-to-End Acoustic Systems. arXiv preprint arXiv:1901.07846","author":"Du Tianyu","year":"2019","unstructured":"Tianyu Du, Shouling Ji, Jinfeng Li, Qinchen Gu, Ting Wang, and Raheem Beyah. 2019. SirenAttack: Generating Adversarial Audio for End-to-End Acoustic Systems. arXiv preprint arXiv:1901.07846 (2019)."},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.1145\/3338906.3338954"},{"key":"e_1_3_2_1_22_1","volume-title":"Proceedings of the 27th International Conference on Computational Linguistics (COLING).","author":"Ebrahimi Javid","year":"2018","unstructured":"Javid Ebrahimi, Daniel Lowd, and Dejing Dou. 2018. On Adversarial Examples for Character-Level Neural Machine Translation. In Proceedings of the 27th International Conference on Computational Linguistics (COLING)."},{"key":"e_1_3_2_1_23_1","volume-title":"Proceedings of the 2018 IEEE Conference on Computer Vision and Pattern Recognition (CVPR).","author":"Evtimov Ivan","year":"2018","unstructured":"Ivan Evtimov, Kevin Eykholt, Earlence Fernandes, Tadayoshi Kohno, Bo Li, Atul Prakash, Amir Rahmati, and Dawn Song. 2018. Robust Physical-World Attacks on Deep Learning Models. In Proceedings of the 2018 IEEE Conference on Computer Vision and Pattern Recognition (CVPR)."},{"key":"e_1_3_2_1_24_1","unstructured":"Hugging Face. [n.d.]. Transformers: State-of-the-art Natural Language Processing for TensorFlow 2.0 and PyTorch. https:\/\/github.com\/huggingface\/transformers"},{"key":"e_1_3_2_1_25_1","unstructured":"Facebook. 2019. How do I translate a post or comment written in another language? https:\/\/www.facebook.com\/help\/509936952489634?helpref=faq_content"},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.1145\/3293882.3330566"},{"key":"e_1_3_2_1_27_1","volume-title":"Proceedings of the 55th Annual Meeting of the Association for Computational Linguistics (ACL).","author":"Gehring Jonas","unstructured":"Jonas Gehring, Michael Auli, David Grangier, and Yann N. Dauphin. 2017. A Convolutional Encoder Model for Neural Machine Translation. In Proceedings of the 55th Annual Meeting of the Association for Computational Linguistics (ACL)."},{"key":"e_1_3_2_1_28_1","volume-title":"Proceedings of the 34th International Conference on Machine Learning (ICML).","author":"Gehring Jonas","unstructured":"Jonas Gehring, Michael Auli, David Grangier, and Yann N. Dauphin. 2017. Convolutional Sequence to Sequence Learning. In Proceedings of the 34th International Conference on Machine Learning (ICML)."},{"key":"e_1_3_2_1_29_1","volume-title":"Proceedings of the 3rd International Conference on Learning Representations (ICLR).","author":"Goodfellow Ian J","year":"2015","unstructured":"Ian J Goodfellow, Jonathon Shlens, and Christian Szegedy. 2015. Explaining and Harnessing Adversarial Examples. Proceedings of the 3rd International Conference on Learning Representations (ICLR)."},{"key":"e_1_3_2_1_30_1","unstructured":"Google. [n.d.]. Google Translate. https:\/\/translate.google.com\/"},{"key":"e_1_3_2_1_31_1","unstructured":"Stanford NLP Group. [n.d.]. Stanford CoreNLP - Natural language software. https:\/\/stanfordnlp.github.io\/CoreNLP\/"},{"key":"e_1_3_2_1_32_1","unstructured":"Hany Hassan Anthony Aue Chang Chen Vishal Chowdhary Jonathan Clark Christian Federmann Xuedong Huang Marcin Junczys-Dowmunt William Lewis Mu Li et al. 2018. Achieving Human Parity on Automatic Chinese to English News Translation. arXiv preprint arXiv:1803.05567 (2018)."},{"key":"e_1_3_2_1_33_1","volume-title":"Towards Structured Evaluation of Deep Neural Network Supervisors. In 2019 IEEE International Conference On Artificial Intelligence Testing (AITest).","author":"Henriksson Jens","year":"2019","unstructured":"Jens Henriksson, Christian Berger, Markus Borg, Lars Tornberg, Cristofer Englund, Sankar Raman Sathyamoorthy, and Stig Ursing. 2019. Towards Structured Evaluation of Deep Neural Network Supervisors. In 2019 IEEE International Conference On Artificial Intelligence Testing (AITest)."},{"key":"e_1_3_2_1_34_1","volume-title":"Error Classification of Machine Translation A Corpus-based Study on Chinese-English Patent Translation. Translation Studies Quarterly","author":"Hsu Jiuan-An","year":"2014","unstructured":"Jiuan-An Hsu. 2014. Error Classification of Machine Translation A Corpus-based Study on Chinese-English Patent Translation. Translation Studies Quarterly (2014)."},{"key":"e_1_3_2_1_35_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/P16-1195"},{"key":"e_1_3_2_1_36_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/N18-1170"},{"key":"e_1_3_2_1_37_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D17-1215"},{"key":"e_1_3_2_1_38_1","volume-title":"Adversarial Logit Pairing. arXiv preprint arXiv:1803.06373","author":"Kannan Harini","year":"2018","unstructured":"Harini Kannan, Alexey Kurakin, and Ian Goodfellow. 2018. Adversarial Logit Pairing. arXiv preprint arXiv:1803.06373 (2018)."},{"key":"e_1_3_2_1_39_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE.2019.00108"},{"key":"e_1_3_2_1_40_1","volume-title":"Kingma and Jimmy Ba","author":"Diederik","year":"2014","unstructured":"Diederik P. Kingma and Jimmy Ba. 2014. Adam: A Method for Stochastic Optimization. http:\/\/arxiv.org\/abs\/1412.6980"},{"key":"e_1_3_2_1_41_1","volume-title":"Proceedings of the 32nd International Conference on International Conference on Machine Learning -","volume":"37","author":"Kusner Matt J.","unstructured":"Matt J. Kusner, Yu Sun, Nicholas I. Kolkin, and Kilian Q. Weinberger. 2015. From Word Embeddings to Document Distances. In Proceedings of the 32nd International Conference on International Conference on Machine Learning - Volume 37 (ICML'15). 957--966."},{"key":"e_1_3_2_1_42_1","unstructured":"Fred. Lambert. 2016. Understanding the fatal Tesla accident on Autopilot and the NHTSA probe. https:\/\/electrek.co\/2016\/07\/01\/understanding-fatal-tesla-accident-autopilot-nhtsa-probe\/"},{"key":"e_1_3_2_1_43_1","doi-asserted-by":"publisher","DOI":"10.1145\/2594291.2594334"},{"key":"e_1_3_2_1_44_1","first-page":"707","article-title":"Binary codes capable of correcting deletions, insertions, and reversals","volume":"10","author":"Levenshtein Vladimir I.","year":"1966","unstructured":"Vladimir I. Levenshtein. 1966. Binary codes capable of correcting deletions, insertions, and reversals. Soviet Physics Doklady 10 (1966), 707--710. Issue 8.","journal-title":"Soviet Physics Doklady"},{"key":"e_1_3_2_1_45_1","unstructured":"Sam Levin. 2018. Tesla fatal crash: 'autopilot' mode sped up car before driver killed report finds. https:\/\/www.theguardian.com\/technology\/2018\/jun\/07\/tesla-fatal-crash-silicon-valley-autopilot-mode-report"},{"key":"e_1_3_2_1_46_1","doi-asserted-by":"publisher","DOI":"10.14722\/ndss.2019.23138"},{"key":"e_1_3_2_1_47_1","volume-title":"Many-Core Compiler Fuzzing. In ACM SIGPLAN Conference on Programming Language Design and Implementation (PLDI).","author":"Lidbury Christopher","unstructured":"Christopher Lidbury, Andrei Lascu, Nathan Chong, and Alastair F. Donaldson. 2015. Many-Core Compiler Fuzzing. In ACM SIGPLAN Conference on Programming Language Design and Implementation (PLDI)."},{"key":"e_1_3_2_1_48_1","volume-title":"Rouge: A Package for Automatic Evaluation of Summaries. Text Summarization Branches Out","author":"Lin Chin-Yew","year":"2004","unstructured":"Chin-Yew Lin. 2004. Rouge: A Package for Automatic Evaluation of Summaries. Text Summarization Branches Out (2004)."},{"key":"e_1_3_2_1_49_1","volume-title":"Proceedings of the 7th International Conference on Learning Representations (ICLR).","author":"Lin Ji","year":"2019","unstructured":"Ji Lin, Chuang Gan, and Song Han. 2019. Defensive Quantization: When Efficiency Meets Robustness. In Proceedings of the 7th International Conference on Learning Representations (ICLR)."},{"key":"e_1_3_2_1_50_1","volume-title":"Proceedings of the 37th International Conference on Software Engineering (ICSE).","author":"Lindvall Mikael","unstructured":"Mikael Lindvall, Dharmalingam Ganesan, Ragnar Ardal, and Robert E. Wiegand. 2015. Metamorphic Model-based Testing Applied on NASA DAT-an experience report. In Proceedings of the 37th International Conference on Software Engineering (ICSE)."},{"key":"e_1_3_2_1_51_1","volume-title":"Manning","author":"Luong Minh-Thang","year":"2015","unstructured":"Minh-Thang Luong, Hieu Pham, and Christopher D. Manning. 2015. Effective Approaches to Attention-based Neural Machine Translation. CoRR abs\/1508.04025 (2015). arXiv:1508.04025 http:\/\/arxiv.org\/abs\/1508.04025"},{"key":"e_1_3_2_1_52_1","volume-title":"Proceedings of the 2015 Conference on Empirical Methods in Natural Language Processing (EMNLP).","author":"Luong Thang","unstructured":"Thang Luong, Hieu Pham, and Christopher D. Manning. 2015. Effective Approaches to Attention-based Neural Machine Translation. In Proceedings of the 2015 Conference on Empirical Methods in Natural Language Processing (EMNLP)."},{"key":"e_1_3_2_1_53_1","doi-asserted-by":"publisher","DOI":"10.1145\/3238147.3238202"},{"key":"e_1_3_2_1_54_1","doi-asserted-by":"publisher","DOI":"10.1109\/ISSRE.2018.00021"},{"key":"e_1_3_2_1_55_1","doi-asserted-by":"publisher","DOI":"10.1145\/3236024.3236082"},{"key":"e_1_3_2_1_56_1","doi-asserted-by":"publisher","DOI":"10.14722\/ndss.2019.23415"},{"key":"e_1_3_2_1_57_1","unstructured":"Fiona Macdonald. 2015. The Greatest Mistranslations Ever. http:\/\/www.bbc.com\/culture\/story\/20150202-the-greatest-mistranslations-ever"},{"key":"e_1_3_2_1_58_1","volume-title":"Proceedings of the 6th International Conference on Learning Representations (ICLR).","author":"Madry Aleksander","year":"2018","unstructured":"Aleksander Madry, Aleksandar Makelov, Ludwig Schmidt, Dimitris Tsipras, and Adrian Vladu. 2018. Towards Deep Learning Models Resistant to Adversarial Attacks. In Proceedings of the 6th International Conference on Learning Representations (ICLR)."},{"key":"e_1_3_2_1_59_1","volume-title":"Efficient Estimation of Word Representations in Vector Space. In ICLR Workshops.","author":"Mikolov Tomas","year":"2013","unstructured":"Tomas Mikolov, Kai Chen, Greg Corrado, and Jeffrey Dean. 2013. Efficient Estimation of Word Representations in Vector Space. In ICLR Workshops."},{"key":"e_1_3_2_1_60_1","volume-title":"Proceedings of the 29th Conference on Neural Information Processing Systems (NeurIPS).","author":"Mikolov Tomas","year":"2013","unstructured":"Tomas Mikolov, Ilya Sutskever, Kai Chen, Greg S Corrado, and Jeff Dean. 2013. Distributed Representations of Words and Phrases and their Compositionality. In Proceedings of the 29th Conference on Neural Information Processing Systems (NeurIPS)."},{"key":"e_1_3_2_1_61_1","volume-title":"Proceedings of the 5th International Conference on Learning Representations (ICLR).","author":"Miyato Takeru","year":"2017","unstructured":"Takeru Miyato, Andrew M Dai, and Ian Goodfellow. 2017. Adversarial Training Methods for Semi-Supervised Text Classification. In Proceedings of the 5th International Conference on Learning Representations (ICLR)."},{"key":"e_1_3_2_1_62_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/P18-1176"},{"key":"e_1_3_2_1_63_1","volume-title":"Proceedings of the 20th International Conference on Software Engineering and Knowledge Engineering (SEKE).","author":"Murphy Christian","year":"2008","unstructured":"Christian Murphy, Gail E. Kaiser, Lifeng Hu, and Leon Wu. 2008. Properties of Machine Learning Applications for Use in Metamorphic Testing. In Proceedings of the 20th International Conference on Software Engineering and Knowledge Engineering (SEKE)."},{"key":"e_1_3_2_1_64_1","unstructured":"Arika Okrent. 2016. 9 Little Translation Mistakes That Caused Big Problems. http:\/\/mentalfloss.com\/article\/48795\/9-little-translation-mistakes-caused-big-problems"},{"key":"e_1_3_2_1_65_1","unstructured":"Thuy Ong. 2017. Facebook apologizes after wrong translation sees Palestinian man arrested for posting 'good morning'. https:\/\/www.theverge.com\/us-world\/2017\/10\/24\/16533496\/facebook-apology-wrong-translation-palestinian-arrested-post-good-morning"},{"key":"e_1_3_2_1_66_1","doi-asserted-by":"publisher","DOI":"10.1109\/SP.2016.41"},{"key":"e_1_3_2_1_67_1","volume-title":"Proceedings of the 40th Annual Meeting on Association for Computational Linguistics (ACL).","author":"Papineni Kishore","year":"2002","unstructured":"Kishore Papineni, Salim Roukos, Todd Ward, and Wei-Jing Zhu. 2002. BLEU: A Method for Automatic Evaluation of Machine Translation. In Proceedings of the 40th Annual Meeting on Association for Computational Linguistics (ACL)."},{"key":"e_1_3_2_1_68_1","doi-asserted-by":"publisher","DOI":"10.1145\/3132747.3132785"},{"key":"e_1_3_2_1_69_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE.2019.00107"},{"key":"e_1_3_2_1_70_1","doi-asserted-by":"publisher","DOI":"10.1145\/3276517"},{"key":"e_1_3_2_1_71_1","volume-title":"Proceedings of the 35th International Conference on Machine Learning (ICML).","author":"Qin Yao","year":"2018","unstructured":"Yao Qin, Nicholas Carlini, Ian Goodfellow, Garrison Cottrell, and Colin Raffel. 2018. Imperceptible, Robust, and Targeted Adversarial Examples for Automatic Speech Recognition. In Proceedings of the 35th International Conference on Machine Learning (ICML)."},{"key":"e_1_3_2_1_72_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/P18-1079"},{"key":"e_1_3_2_1_73_1","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2016.2532875"},{"key":"e_1_3_2_1_74_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/P16-1009"},{"key":"e_1_3_2_1_75_1","volume-title":"Proceedings of the 7th International Conference on Learning Representations (ICLR).","author":"Shen Yikang","year":"2019","unstructured":"Yikang Shen, Shawn Tab, Alessandro Sordoni, and Aaron Courville. 2019. Ordered Neurons: Integrating Tree Structures into Recurrent Neural Networks. In Proceedings of the 7th International Conference on Learning Representations (ICLR)."},{"key":"e_1_3_2_1_76_1","volume-title":"Proceedings of the 30th Conference on Neural Information Processing Systems (NeurIPS).","author":"Sutskever Ilya","year":"2014","unstructured":"Ilya Sutskever, Oriol Vinyals, and Quoc V Le. 2014. Sequence to Sequence Learning with Neural Networks. In Proceedings of the 30th Conference on Neural Information Processing Systems (NeurIPS)."},{"key":"e_1_3_2_1_77_1","volume-title":"Proceedings of the 34th Conference on Neural Information Processing Systems (NeurIPS).","author":"Tao Guanhong","year":"2018","unstructured":"Guanhong Tao, Shiqing Ma, Yingqi Liu, and Xiangyu Zhang. 2018. Attacks Meet Interpretability: Attribute-steered Detection of Adversarial Samples. In Proceedings of the 34th Conference on Neural Information Processing Systems (NeurIPS)."},{"key":"e_1_3_2_1_78_1","doi-asserted-by":"publisher","DOI":"10.1177\/107769905303000401"},{"key":"e_1_3_2_1_79_1","doi-asserted-by":"publisher","DOI":"10.1145\/3180155.3180220"},{"key":"e_1_3_2_1_80_1","unstructured":"Tree. 2013. Adventures in Mistranslation: HSBC's Call to \"Do Nothing\". https:\/\/contentequalsmoney.com\/mistranslation-hsbcs-call-to-do-nothing\/"},{"key":"e_1_3_2_1_81_1","unstructured":"Barak Turovsky. 2016. Ten years of Google Translate. https:\/\/blog.google\/products\/translate\/ten-years-of-google-translate\/"},{"key":"e_1_3_2_1_82_1","unstructured":"Twitter. 2019. About Tweet translation. https:\/\/help.twitter.com\/en\/using-twitter\/translate-tweets"},{"key":"e_1_3_2_1_83_1","volume-title":"Proceedings of the 33rd Conference on Neural Information Processing Systems (NeurIPS).","author":"Vaswani Ashish","year":"2017","unstructured":"Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, Llion Jones, Aidan N. Gomez, and Illia Kaiser, Lukasz abd Polosukhin. 2017. Attention is All you Need. In Proceedings of the 33rd Conference on Neural Information Processing Systems (NeurIPS)."},{"key":"e_1_3_2_1_84_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE.2019.00126"},{"key":"e_1_3_2_1_85_1","unstructured":"WMT. [n.d.]. WMT Datasets. http:\/\/statmt.org\/wmt17\/translation-task.html"},{"key":"e_1_3_2_1_86_1","unstructured":"Yonghui Wu Mike Schuster Zhifeng Chen Quoc V Le Mohammad Norouzi Wolfgang Macherey Maxim Krikun Yuan Cao Qin Gao Klaus Macherey et al. 2016. Google's Neural Machine Translation System: Bridging the Gap Between Human and Machine Translation. arXiv preprint arXiv:1609.08144 (2016)."},{"key":"e_1_3_2_1_87_1","volume-title":"Christian Murphy, Gail Kaiser, Baowen Xu, and Tsong Yueh Chen.","author":"Xie Xiaoyuan","year":"2011","unstructured":"Xiaoyuan Xie, Joshua WK Ho, Christian Murphy, Gail Kaiser, Baowen Xu, and Tsong Yueh Chen. 2011. Testing and Validating Machine Learning Classifiers by Metamorphic Testing. Journal of Systems and Software (JSS) 84 (2011). Issue 4."},{"key":"e_1_3_2_1_88_1","doi-asserted-by":"publisher","DOI":"10.1145\/3293882.3330579"},{"key":"e_1_3_2_1_89_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2019.00935"},{"key":"e_1_3_2_1_90_1","doi-asserted-by":"publisher","DOI":"10.14722\/ndss.2018.23198"},{"key":"e_1_3_2_1_91_1","volume-title":"Proceedings of the 2019 IEEE Conference on Computer Vision and Pattern Recognition (CVPR).","author":"Yang Dawei","year":"2019","unstructured":"Dawei Yang, Chaowei Xiao, Bo Li, Jia Deng, and Mingyan Liu. 2019. Realistic Adversarial Examples in 3D Meshes. In Proceedings of the 2019 IEEE Conference on Computer Vision and Pattern Recognition (CVPR)."},{"key":"e_1_3_2_1_92_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/P18-1166"},{"key":"e_1_3_2_1_93_1","doi-asserted-by":"publisher","DOI":"10.1145\/2642937.2642994"},{"key":"e_1_3_2_1_94_1","volume-title":"Machine Learning Testing: Survey, Landscapes and Horizons. arXiv preprint arXiv:1906.10742","author":"Zhang Jie M.","year":"2019","unstructured":"Jie M. Zhang, Mark Harman, Lei Ma, and Yang Liu. 2019. Machine Learning Testing: Survey, Landscapes and Horizons. arXiv preprint arXiv:1906.10742 (2019)."},{"key":"e_1_3_2_1_95_1","doi-asserted-by":"publisher","DOI":"10.1145\/3238147.3238187"},{"key":"e_1_3_2_1_96_1","volume-title":"Testing Untestable Neural Machine Translation: An Industrial Case. arXiv preprint arXiv:1807.02340","author":"Zheng Wujie","year":"2018","unstructured":"Wujie Zheng, Wenyu Wang, Dian Liu, Changrong Zhang, Qinsong Zeng, Yuetang Deng, Wei Yang, Pinjia He, and Tao Xie. 2018. Testing Untestable Neural Machine Translation: An Industrial Case. arXiv preprint arXiv:1807.02340 (2018)."},{"key":"e_1_3_2_1_97_1","volume-title":"Proceedings of the 25th Australasian Software Engineering Conference (ASWEC).","author":"Zhou Zhi Quan","year":"2018","unstructured":"Zhi Quan Zhou and Liqun Sun. 2018. Metamorphic Testing for Machine Translations: MT4MT. In Proceedings of the 25th Australasian Software Engineering Conference (ASWEC)."},{"key":"e_1_3_2_1_98_1","volume-title":"Metamorphic Testing for Software Quality Assessment: A Study of Search Engines","author":"Zhou Zhi Quan","year":"2016","unstructured":"Zhi Quan Zhou, Shaowen Xiang, and Tsong Yueh Chen. 2016. Metamorphic Testing for Software Quality Assessment: A Study of Search Engines. IEEE Transactions on Software Engineering (TSE) 42 (2016). Issue 3."},{"key":"e_1_3_2_1_99_1","volume-title":"Proceedings of the 51st Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers). Association for Computational Linguistics, 434--443","author":"Zhu Muhua","year":"2013","unstructured":"Muhua Zhu, Yue Zhang, Wenliang Chen, Min Zhang, and Jingbo Zhu. 2013. Fast and Accurate Shift-Reduce Constituent Parsing. In Proceedings of the 51st Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers). Association for Computational Linguistics, 434--443."},{"key":"e_1_3_2_1_100_1","unstructured":"Chris. Ziegler. 2016. A Google self-driving car caused a crash for the first time. https:\/\/www.theverge.com\/2016\/2\/29\/11134344\/google-self-driving-car-crash-report"}],"event":{"name":"ICSE '20: 42nd International Conference on Software Engineering","location":"Seoul South Korea","acronym":"ICSE '20","sponsor":["SIGSOFT ACM Special Interest Group on Software Engineering","KIISE Korean Institute of Information Scientists and Engineers","IEEE CS"]},"container-title":["Proceedings of the ACM\/IEEE 42nd International Conference on Software Engineering"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3377811.3380339","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3377811.3380339","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T23:23:57Z","timestamp":1750202637000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3377811.3380339"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2020,6,27]]},"references-count":98,"alternative-id":["10.1145\/3377811.3380339","10.1145\/3377811"],"URL":"https:\/\/doi.org\/10.1145\/3377811.3380339","relation":{},"subject":[],"published":{"date-parts":[[2020,6,27]]},"assertion":[{"value":"2020-10-01","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}