{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,3,2]],"date-time":"2026-03-02T22:57:33Z","timestamp":1772492253596,"version":"3.50.1"},"publisher-location":"New York, NY, USA","reference-count":58,"publisher":"ACM","license":[{"start":{"date-parts":[[2020,6,27]],"date-time":"2020-06-27T00:00:00Z","timestamp":1593216000000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"name":"the Luxembourg National Research Funds (FNR)","award":["C17\/IS\/11686509\/CODEMATES"],"award-info":[{"award-number":["C17\/IS\/11686509\/CODEMATES"]}]},{"name":"the ERC","award":["741278"],"award-info":[{"award-number":["741278"]}]},{"name":"the National Key Research and Development Program of China","award":["2017YFB1001803"],"award-info":[{"award-number":["2017YFB1001803"]}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2020,6,27]]},"DOI":"10.1145\/3377811.3380420","type":"proceedings-article","created":{"date-parts":[[2020,10,1]],"date-time":"2020-10-01T18:25:38Z","timestamp":1601576738000},"page":"974-985","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":94,"title":["Automatic testing and improvement of machine translation"],"prefix":"10.1145","author":[{"given":"Zeyu","family":"Sun","sequence":"first","affiliation":[{"name":"Peking University, MoE"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Jie M.","family":"Zhang","sequence":"additional","affiliation":[{"name":"University College London"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Mark","family":"Harman","sequence":"additional","affiliation":[{"name":"University College London"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Mike","family":"Papadakis","sequence":"additional","affiliation":[{"name":"University of Luxembourg"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Lu","family":"Zhang","sequence":"additional","affiliation":[{"name":"Peking University, MoE"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2020,10]]},"reference":[{"key":"e_1_3_2_1_1_1","volume-title":"https:\/\/catalog.ldc.upenn.edu\/LDC2013T19","author":"Weischedel Ralph","unstructured":"Ralph Weischedel, Martha Palmer, Mitchell Marcus, Eduard Hovy, Sameer Pradhan, Lance Ramshaw, Nianwen Xue, Ann Taylor, Jeff Kaufman, Michelle Franchini, Mohammed El-Bachouti, Robert Belvin, Ann Houston. 2013. OntoNotes. https:\/\/catalog.ldc.upenn.edu\/LDC2013T19."},{"key":"e_1_3_2_1_2_1","volume-title":"Proc. ICLR.","author":"Belinkov Yonatan","year":"2018","unstructured":"Yonatan Belinkov and Yonatan Bisk. 2018. Synthetic and natural noise both break neural machine translation. In Proc. ICLR."},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/P19-1425"},{"key":"e_1_3_2_1_5_1","volume-title":"Towards robust neural machine translation. arXiv preprint arXiv:1805.06130","author":"Cheng Yong","year":"2018","unstructured":"Yong Cheng, Zhaopeng Tu, Fandong Meng, Junjie Zhai, and Yang Liu. 2018. Towards robust neural machine translation. arXiv preprint arXiv:1805.06130 (2018)."},{"key":"e_1_3_2_1_6_1","unstructured":"CWMT. 2018. The CWMT Dataset. http:\/\/nlp.nju.edu.cn\/cwmt-wmt\/."},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"publisher","DOI":"10.3115\/1289189.1289273"},{"key":"e_1_3_2_1_8_1","volume-title":"Ethnologue: Languages of the world.","author":"Eberhard David M","year":"2019","unstructured":"David M Eberhard, Gary F Simons, and Charles D Fennig. 2019. Ethnologue: Languages of the world. (2019)."},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/P18-2006"},{"key":"e_1_3_2_1_10_1","unstructured":"Free Software Foundation. 2019. GNU Wdiff. https:\/\/www.gnu.org\/software\/wdiff\/"},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.1017\/S0021853700005636"},{"key":"e_1_3_2_1_12_1","volume-title":"word2vec Explained: deriving Mikolov et al.'s negative-sampling word-embedding method. arXiv preprint arXiv:1402.3722","author":"Goldberg Yoav","year":"2014","unstructured":"Yoav Goldberg and Omer Levy. 2014. word2vec Explained: deriving Mikolov et al.'s negative-sampling word-embedding method. arXiv preprint arXiv:1402.3722 (2014)."},{"key":"e_1_3_2_1_13_1","volume-title":"Advances in Neural Information Processing Systems 27","author":"Goodfellow Ian","unstructured":"Ian Goodfellow, Jean Pouget-Abadie, Mehdi Mirza, Bing Xu, David Warde-Farley, Sherjil Ozair, Aaron Courville, and Yoshua Bengio. 2014. Generative Adversarial Nets. In Advances in Neural Information Processing Systems 27, Z. Ghahramani, M. Welling, C. Cortes, N. D. Lawrence, and K. Q. Weinberger (Eds.). Curran Associates, Inc., 2672--2680. http:\/\/papers.nips.cc\/paper\/5423-generative-adversarial-nets.pdf"},{"key":"e_1_3_2_1_14_1","unstructured":"Google. 2019. Google Translate. http:\/\/translate.google.com."},{"key":"e_1_3_2_1_15_1","volume-title":"Proceedings of the Australasian Language Technology Association Workshop","author":"Graham Yvette","year":"2012","unstructured":"Yvette Graham, Timothy Baldwin, Aaron Harwood, Alistair Moffat, and Justin Zobel. 2012. Measurement of progress in machine translation. In Proceedings of the Australasian Language Technology Association Workshop 2012. 70--78."},{"key":"e_1_3_2_1_16_1","volume-title":"Thirty-Second AAAI Conference on Artificial Intelligence.","author":"Gu Jiatao","year":"2018","unstructured":"Jiatao Gu, Yong Wang, Kyunghyun Cho, and Victor OK Li. 2018. Search engine guided neural machine translation. In Thirty-Second AAAI Conference on Artificial Intelligence."},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/N19-1122"},{"key":"e_1_3_2_1_18_1","volume-title":"Achieving Human Parity on Automatic Chinese to English News Translation. CoRR abs\/1803.05567","author":"Hassan Hany","year":"2018","unstructured":"Hany Hassan, Anthony Aue, Chang Chen, Vishal Chowdhary, Jonathan Clark, Christian Federmann, Xuedong Huang, Marcin Junczys-Dowmunt, William Lewis, Mu Li, Shujie Liu, Tie-Yan Liu, Renqian Luo, Arul Menezes, Tao Qin, Frank Seide, Xu Tan, Fei Tian, Lijun Wu, Shuangzhi Wu, Yingce Xia, Dongdong Zhang, Zhirui Zhang, and Ming Zhou. 2018. Achieving Human Parity on Automatic Chinese to English News Translation. CoRR abs\/1803.05567 (2018). arXiv:1803.05567 http:\/\/arxiv.org\/abs\/1803.05567"},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","DOI":"10.1109\/HPCA.2018.00059"},{"key":"e_1_3_2_1_20_1","first-page":"18","volume-title":"Proceedings of the 13th Conference of the Association for Machine Translation in the Americas, AMTA 2018","volume":"80","author":"Heigold Georg","year":"2018","unstructured":"Georg Heigold, Stalin Varanasi, G\u00fcnter Neumann, and Josef van Genabith. 2018. How Robust Are Character-Based Word Embeddings in Tagging and MT Against Wrod Scramlbing or Randdm Nouse?. In Proceedings of the 13th Conference of the Association for Machine Translation in the Americas, AMTA 2018, Boston, MA, USA, March 17-21, 2018 - Volume 1: Research Papers. 68--80. https:\/\/aclanthology.info\/papers\/W18-1807\/w18-1807"},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.1145\/359581.359603"},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2010.62"},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.1007\/s11432-018-1465-6"},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"publisher","DOI":"10.1145\/3213846.3213871"},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.1145\/3213846.3213871"},{"key":"e_1_3_2_1_26_1","volume-title":"Training on Synthetic Noise Improves Robustness to Natural Noise in Machine Translation. arXiv preprint arXiv:1902.01509","author":"Karpukhin Vladimir","year":"2019","unstructured":"Vladimir Karpukhin, Omer Levy, Jacob Eisenstein, and Marjan Ghazvininejad. 2019. Training on Synthetic Noise Improves Robustness to Natural Noise in Machine Translation. arXiv preprint arXiv:1902.01509 (2019)."},{"key":"e_1_3_2_1_27_1","volume-title":"On the impact of various types of noise on neural machine translation. arXiv preprint arXiv:1805.12282","author":"Khayrallah Huda","year":"2018","unstructured":"Huda Khayrallah and Philipp Koehn. 2018. On the impact of various types of noise on neural machine translation. arXiv preprint arXiv:1805.12282 (2018)."},{"key":"e_1_3_2_1_28_1","volume-title":"Genprog: A generic method for automatic software repair. Ieee transactions on software engineering 38, 1","author":"Goues Claire Le","year":"2011","unstructured":"Claire Le Goues, ThanhVu Nguyen, Stephanie Forrest, and Westley Weimer. 2011. Genprog: A generic method for automatic software repair. Ieee transactions on software engineering 38, 1 (2011), 54--72."},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.5555\/2886521.2886640"},{"key":"e_1_3_2_1_30_1","volume-title":"The Stanford CoreNLP Natural Language Processing Toolkit","author":"Manning Christopher D.","unstructured":"Christopher D. Manning, Mihai Surdeanu, John Bauer, Jenny Finkel, Steven J. Bethard, and David McClosky. 2014. The Stanford CoreNLP Natural Language Processing Toolkit. In Association for Computational Linguistics (ACL) System Demonstrations. 55--60. http:\/\/www.aclweb.org\/anthology\/P\/P14\/P14-5010"},{"key":"e_1_3_2_1_31_1","volume-title":"Strategic Insights: Lost in Translation. https:\/\/ssi.armywarcollege.edu\/index.cfm\/articles\/Lost-In-Translation\/2017\/08\/17","author":"Mason M. Chris","year":"2017","unstructured":"M. Chris Mason. 2017. Strategic Insights: Lost in Translation. https:\/\/ssi.armywarcollege.edu\/index.cfm\/articles\/Lost-In-Translation\/2017\/08\/17"},{"key":"e_1_3_2_1_32_1","volume-title":"Yves Le Traon, and Mark Harman","author":"Papadakis Mike","year":"2019","unstructured":"Mike Papadakis, Marinos Kintis, Jie Zhang, Yue Jia, Yves Le Traon, and Mark Harman. 2019. Mutation testing advances: an analysis and survey. In Advances in Computers. Vol. 112. Elsevier, 275--378."},{"key":"e_1_3_2_1_33_1","volume-title":"Proceedings of the 40th annual meeting on association for computational linguistics. Association for Computational Linguistics, 311--318","author":"Papineni Kishore","year":"2002","unstructured":"Kishore Papineni, Salim Roukos, Todd Ward, and Wei-Jing Zhu. 2002. BLEU: a method for automatic evaluation of machine translation. In Proceedings of the 40th annual meeting on association for computational linguistics. Association for Computational Linguistics, 311--318."},{"key":"e_1_3_2_1_34_1","unstructured":"Parmy Olson. 2018. The Algorithm That Helped Google Translate Become Sexist. https:\/\/www.forbes.com\/sites\/parmyolson\/2018\/02\/15\/the-algorithm-that-helped-google-translate-become-sexist\/#224101cb7daa."},{"key":"e_1_3_2_1_35_1","doi-asserted-by":"publisher","DOI":"10.3115\/v1\/D14-1162"},{"key":"e_1_3_2_1_36_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/P18-1079"},{"key":"e_1_3_2_1_37_1","doi-asserted-by":"publisher","DOI":"10.1109\/34.682181"},{"key":"e_1_3_2_1_38_1","volume-title":"English Gigaword","author":"Parker Robert","year":"2011","unstructured":"Robert Parker, David Graf, Junbo Kong, Ke Chen, Kazuaki Maeda. 2011. English Gigaword Fifth Edition. https:\/\/catalog.ldc.upenn.edu\/LDC2011T07."},{"key":"e_1_3_2_1_39_1","volume-title":"Proceedings of the 32Nd IEEE\/ACM International Conference on Automated Software Engineering (ASE","author":"Saha Ripon K.","year":"2017","unstructured":"Ripon K. Saha, Yingjun Lyu, Hiroaki Yoshida, and Mukul R. Prasad. 2017. ELIXIR: Effective Object Oriented Program Repair. In Proceedings of the 32Nd IEEE\/ACM International Conference on Automated Software Engineering (ASE 2017). IEEE Press, Piscataway, NJ, USA, 648--659. http:\/\/dl.acm.org\/citation.cfm?id=3155562.3155643"},{"key":"e_1_3_2_1_40_1","unstructured":"SpaCy. 2019. SpaCy. https:\/\/spacy.io\/."},{"key":"e_1_3_2_1_41_1","volume-title":"International Workshop on Spoken Language Translation (IWSLT).","author":"Sperber Matthias","year":"2017","unstructured":"Matthias Sperber, Jan Niehues, and Alex Waibel. 2017. Toward robust neural machine translation for noisy input sequences. In International Workshop on Spoken Language Translation (IWSLT)."},{"key":"e_1_3_2_1_42_1","doi-asserted-by":"publisher","DOI":"10.1109\/ASWEC.2018.00021"},{"key":"e_1_3_2_1_43_1","unstructured":"Zeyu Sun. 2019. TransRepair Homepage. https:\/\/github.com\/zysszy\/TransRepair."},{"key":"e_1_3_2_1_44_1","volume-title":"Treebanks","author":"Taylor Ann","unstructured":"Ann Taylor, Mitchell Marcus, and Beatrice Santorini. 2003. The Penn treebank: an overview. In Treebanks. Springer, 5--22."},{"key":"e_1_3_2_1_45_1","volume-title":"Tensor2Tensor for Neural Machine Translation. CoRR abs\/1803.07416","author":"Vaswani Ashish","year":"2018","unstructured":"Ashish Vaswani, Samy Bengio, Eugene Brevdo, Francois Chollet, Aidan N. Gomez, Stephan Gouws, Llion Jones, \u0141ukasz Kaiser, Nal Kalchbrenner, Niki Parmar, Ryan Sepassi, Noam Shazeer, and Jakob Uszkoreit. 2018. Tensor2Tensor for Neural Machine Translation. CoRR abs\/1803.07416 (2018). http:\/\/arxiv.org\/abs\/1803.07416"},{"key":"e_1_3_2_1_46_1","volume-title":"Proceedings of the 31st International Conference on Neural Information Processing Systems. Curran Associates Inc., 6000--6010","author":"Vaswani Ashish","year":"2017","unstructured":"Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, Llion Jones, Aidan N Gomez, \u0141ukasz Kaiser, and Illia Polosukhin. 2017. Attention is all you need. In Proceedings of the 31st International Conference on Neural Information Processing Systems. Curran Associates Inc., 6000--6010."},{"key":"e_1_3_2_1_47_1","unstructured":"VoiceBoxer. 2016. WHAT ABOUT ENGLISH IN CHINA? http:\/\/voiceboxer.com\/english-in-china\/."},{"key":"e_1_3_2_1_48_1","doi-asserted-by":"publisher","DOI":"10.1017\/S0266078412000235"},{"key":"e_1_3_2_1_49_1","unstructured":"Wikipedia. 2014. Wikipedia. https:\/\/dumps.wikimedia.org\/."},{"key":"e_1_3_2_1_50_1","unstructured":"WMT. 2018. News-Commentary. http:\/\/data.statmt.org\/wmt18\/translation-task\/."},{"key":"e_1_3_2_1_51_1","volume-title":"Proceedings of the 32Nd IEEE\/ACM International Conference on Automated Software Engineering (ASE","author":"Xin Qi","year":"2017","unstructured":"Qi Xin and Steven P. Reiss. 2017. Leveraging Syntax-related Code for Automated Program Repair. In Proceedings of the 32Nd IEEE\/ACM International Conference on Automated Software Engineering (ASE 2017). IEEE Press, Piscataway, NJ, USA, 660--670. http:\/\/dl.acm.org\/citation.cfm?id=3155562.3155644"},{"key":"e_1_3_2_1_52_1","doi-asserted-by":"publisher","DOI":"10.1145\/2642937.2642994"},{"key":"e_1_3_2_1_53_1","volume-title":"Guiding neural machine translation with retrieved translation pieces. arXiv preprint arXiv:1804.02559","author":"Zhang Jingyi","year":"2018","unstructured":"Jingyi Zhang, Masao Utiyama, Eiichro Sumita, Graham Neubig, and Satoshi Nakamura. 2018. Guiding neural machine translation with retrieved translation pieces. arXiv preprint arXiv:1804.02559 (2018)."},{"key":"e_1_3_2_1_54_1","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2018.2809496"},{"key":"e_1_3_2_1_55_1","doi-asserted-by":"publisher","DOI":"10.1109\/ISSRE.2014.27"},{"key":"e_1_3_2_1_56_1","volume-title":"Machine Learning Testing: Survey, Landscapes and Horizons. arXiv preprint arXiv:1906.10742","author":"Zhang Jie M","year":"2019","unstructured":"Jie M Zhang, Mark Harman, Lei Ma, and Yang Liu. 2019. Machine Learning Testing: Survey, Landscapes and Horizons. arXiv preprint arXiv:1906.10742 (2019)."},{"key":"e_1_3_2_1_57_1","doi-asserted-by":"publisher","DOI":"10.1007\/s13042-010-0001-0"},{"key":"e_1_3_2_1_58_1","volume-title":"Generating Natural Adversarial Examples. CoRR abs\/1710.11342","author":"Zhao Zhengli","year":"2017","unstructured":"Zhengli Zhao, Dheeru Dua, and Sameer Singh. 2017. Generating Natural Adversarial Examples. CoRR abs\/1710.11342 (2017). arXiv:1710.11342 http:\/\/arxiv.org\/abs\/1710.11342"},{"key":"e_1_3_2_1_59_1","volume-title":"Proceedings of the Tenth International Conference on Language Resources and Evaluation (LREC","author":"Ziemski Micha\u0142","year":"2016","unstructured":"Micha\u0142 Ziemski, Marcin Junczys-Dowmunt, and Bruno Pouliquen. 2016. The united nations parallel corpus v1. 0. In Proceedings of the Tenth International Conference on Language Resources and Evaluation (LREC 2016). 3530--3534."}],"event":{"name":"ICSE '20: 42nd International Conference on Software Engineering","location":"Seoul South Korea","acronym":"ICSE '20","sponsor":["SIGSOFT ACM Special Interest Group on Software Engineering","KIISE Korean Institute of Information Scientists and Engineers","IEEE CS"]},"container-title":["Proceedings of the ACM\/IEEE 42nd International Conference on Software Engineering"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3377811.3380420","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3377811.3380420","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T22:41:40Z","timestamp":1750200100000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3377811.3380420"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2020,6,27]]},"references-count":58,"alternative-id":["10.1145\/3377811.3380420","10.1145\/3377811"],"URL":"https:\/\/doi.org\/10.1145\/3377811.3380420","relation":{},"subject":[],"published":{"date-parts":[[2020,6,27]]},"assertion":[{"value":"2020-10-01","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}