{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,1,17]],"date-time":"2026-01-17T19:10:06Z","timestamp":1768677006540,"version":"3.49.0"},"publisher-location":"New York, NY, USA","reference-count":48,"publisher":"ACM","license":[{"start":{"date-parts":[[2024,10,27]],"date-time":"2024-10-27T00:00:00Z","timestamp":1729987200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["62172202"],"award-info":[{"award-number":["62172202"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]},{"name":"Major Program of the Natural Science Foundation of Jiangsu Higher Education Institutions of China","award":["22KJA520008"],"award-info":[{"award-number":["22KJA520008"]}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2024,10,27]]},"DOI":"10.1145\/3691620.3695069","type":"proceedings-article","created":{"date-parts":[[2024,10,18]],"date-time":"2024-10-18T15:39:19Z","timestamp":1729265959000},"page":"758-769","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":2,"title":["Evaluating Terminology Translation in Machine Translation Systems via Metamorphic Testing"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0009-0005-3665-6663","authenticated-orcid":false,"given":"Yihui","family":"Xu","sequence":"first","affiliation":[{"name":"School of Computer Science and Technology, Soochow University, Suzhou, China"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-2282-7175","authenticated-orcid":false,"given":"Yanhui","family":"Li","sequence":"additional","affiliation":[{"name":"National Key Laboratory for Novel Software Technology, Nanjing University, Nanjing, China"}]},{"ORCID":"https:\/\/orcid.org\/0009-0001-5491-0443","authenticated-orcid":false,"given":"Jun","family":"Wang","sequence":"additional","affiliation":[{"name":"National Key Laboratory for Novel Software Technology, Nanjing University, Nanjing, China"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-8667-0456","authenticated-orcid":false,"given":"Xiaofang","family":"Zhang","sequence":"additional","affiliation":[{"name":"School of Computer Science and Technology, Soochow University, Suzhou, China"}]}],"member":"320","published-online":{"date-parts":[[2024,10,27]]},"reference":[{"key":"e_1_3_2_1_1_1","unstructured":"5j9. 2024. wikitextparser. https:\/\/github.com\/5j9\/wikitextparser"},{"key":"e_1_3_2_1_2_1","first-page":"1","article-title":"The efficacy of round-trip translation for MT evaluation","volume":"14","author":"Aiken Milam","year":"2010","unstructured":"Milam Aiken and Mina Park. 2010. The efficacy of round-trip translation for MT evaluation. Translation Journal 14, 1 (2010), 1--10.","journal-title":"Translation Journal"},{"key":"e_1_3_2_1_3_1","volume-title":"Contextual String Embeddings for Sequence Labeling. In COLING 2018, 27th International Conference on Computational Linguistics. 1638--1649","author":"Akbik Alan","year":"2018","unstructured":"Alan Akbik, Duncan Blythe, and Roland Vollgraf. 2018. Contextual String Embeddings for Sequence Labeling. In COLING 2018, 27th International Conference on Computational Linguistics. 1638--1649."},{"key":"e_1_3_2_1_4_1","unstructured":"Antonios Anastasopoulos Laurent Besacier James Cross Matthias Gall\u00e9 Philipp Koehn Vassilina Nikoulina et al. 2021. On the evaluation of machine translation for terminology consistency. arXiv preprint arXiv:2106.11891 (2021)."},{"key":"e_1_3_2_1_5_1","first-page":"5087","article-title":"Biasfinder: Metamorphic test generation to uncover bias for sentiment analysis systems","volume":"48","author":"Asyrofi Muhammad Hilmi","year":"2021","unstructured":"Muhammad Hilmi Asyrofi, Zhou Yang, Imam Nur Bani Yusuf, Hong Jin Kang, Ferdian Thung, and David Lo. 2021. Biasfinder: Metamorphic test generation to uncover bias for sentiment analysis systems. IEEE Transactions on Software Engineering 48, 12 (2021), 5087--5101.","journal-title":"IEEE Transactions on Software Engineering"},{"key":"e_1_3_2_1_6_1","volume-title":"Proceedings of the acl workshop on intrinsic and extrinsic evaluation measures for machine translation and\/or summarization. 65--72","author":"Banerjee Satanjeev","year":"2005","unstructured":"Satanjeev Banerjee and Alon Lavie. 2005. METEOR: An automatic metric for MT evaluation with improved correlation with human judgments. In Proceedings of the acl workshop on intrinsic and extrinsic evaluation measures for machine translation and\/or summarization. 65--72."},{"key":"e_1_3_2_1_7_1","volume-title":"The oracle problem in software testing: A survey","author":"Barr Earl T","year":"2014","unstructured":"Earl T Barr, Mark Harman, Phil McMinn, Muzammil Shahbaz, and Shin Yoo. 2014. The oracle problem in software testing: A survey. IEEE transactions on software engineering 41, 5 (2014), 507--525."},{"key":"e_1_3_2_1_8_1","volume-title":"The cognitive shift in terminology and specialized translation. MonTI. Monograf\u00edas de Traducci\u00f3n e Interpretaci\u00f3n 1","author":"Ben\u00edtez Pamela Faber","year":"2009","unstructured":"Pamela Faber Ben\u00edtez. 2009. The cognitive shift in terminology and specialized translation. MonTI. Monograf\u00edas de Traducci\u00f3n e Interpretaci\u00f3n 1 (2009), 107--134."},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2021.eacl-main.271"},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.7202\/002101ar"},{"key":"e_1_3_2_1_11_1","volume-title":"Working with specialized language: A practical guide to using corpora","author":"Bowker Lynne","unstructured":"Lynne Bowker and Jennifer Pearson. 2002. Working with specialized language: A practical guide to using corpora. Routledge."},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.1145\/3490488"},{"key":"e_1_3_2_1_13_1","unstructured":"Rory Cellan-Jones. 2018. Google Translate serves up 'scummy Welsh' translations. https:\/\/www.bbc.com\/news\/technology-45012357"},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.1109\/ASE51524.2021.9678670"},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1145\/3143561","article-title":"Metamorphic testing: A review of challenges and opportunities","volume":"51","author":"Chen Tsong Yueh","year":"2018","unstructured":"Tsong Yueh Chen, Fei-Ching Kuo, Huai Liu, Pak-Lok Poon, Dave Towey, TH Tse, and Zhi Quan Zhou. 2018. Metamorphic testing: A review of challenges and opportunities. ACM Computing Surveys (CSUR) 51, 1 (2018), 1--27.","journal-title":"ACM Computing Surveys (CSUR)"},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/P19-1294"},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2021.eacl-main.181"},{"key":"e_1_3_2_1_18_1","unstructured":"Google. 2024. Google Translate. https:\/\/translate.google.com"},{"key":"e_1_3_2_1_19_1","unstructured":"Xinxing Gu. 2023. New features make Translate more accessible for its 1 billion users. https:\/\/blog.google\/products\/translate\/new-features-make-translate-more-accessible-for-its-1-billion-users"},{"key":"e_1_3_2_1_20_1","volume-title":"Keith Stevens, Noah Constant, Yun-Hsuan Sung, Brian Strope, et al.","author":"Guo Mandy","year":"2018","unstructured":"Mandy Guo, Qinlan Shen, Yinfei Yang, Heming Ge, Daniel Cer, Gustavo Hernandez Abrego, Keith Stevens, Noah Constant, Yun-Hsuan Sung, Brian Strope, et al. 2018. Effective parallel corpus mining using bilingual sentence embeddings. arXiv preprint arXiv: 1807.11906 (2018)."},{"key":"e_1_3_2_1_21_1","volume-title":"Evaluating Terminology Translation in MT. In International Conference on Computational Linguistics and Intelligent Text Processing. Springer, 495--520","author":"Haque Rejwanul","year":"2019","unstructured":"Rejwanul Haque, Mohammed Hasanuzzaman, and Andy Way. 2019. Evaluating Terminology Translation in MT. In International Conference on Computational Linguistics and Intelligent Text Processing. Springer, 495--520."},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.1145\/3377811.3380339"},{"key":"e_1_3_2_1_23_1","unstructured":"IATE. 2024. The terminology database of the European Union. https:\/\/iate.europa.eu"},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"publisher","DOI":"10.1109\/ASE51524.2021.9678715"},{"key":"e_1_3_2_1_25_1","volume-title":"Proceedings of the Workshop on Terminology resources and computation, LREC 2000 Conference.","author":"Johnson Ian","year":"2000","unstructured":"Ian Johnson and Alastair Macphail. 2000. Iate-inter-agency terminology exchange: development of a single central terminology database for the institutions and agencies of the european union. In Proceedings of the Workshop on Terminology resources and computation, LREC 2000 Conference."},{"key":"e_1_3_2_1_26_1","volume-title":"Understanding the Effectiveness of Large Language Models in Detecting Security Vulnerabilities. arXiv preprint arXiv:2311.16169","author":"Khare Avishree","year":"2023","unstructured":"Avishree Khare, Saikat Dutta, Ziyang Li, Alaia Solko-Breslin, Rajeev Alur, and Mayur Naik. 2023. Understanding the Effectiveness of Large Language Models in Detecting Security Vulnerabilities. arXiv preprint arXiv:2311.16169 (2023)."},{"key":"e_1_3_2_1_27_1","volume-title":"Rouge: A package for automatic evaluation of summaries. In Text summarization branches out. 74--81.","author":"Lin Chin-Yew","year":"2004","unstructured":"Chin-Yew Lin. 2004. Rouge: A package for automatic evaluation of summaries. In Text summarization branches out. 74--81."},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"publisher","DOI":"10.1145\/3441691"},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.coling-main.348"},{"key":"e_1_3_2_1_30_1","unstructured":"Microsoft. 2024. Bing Microsoft Translator. https:\/\/www.bing.com\/Translator\/"},{"key":"e_1_3_2_1_31_1","volume-title":"MTEB: Massive text embedding benchmark. arXiv preprint arXiv:2210.07316","author":"Muennighoff Niklas","year":"2022","unstructured":"Niklas Muennighoff, Nouamane Tazi, Lo\u00efc Magne, and Nils Reimers. 2022. MTEB: Massive text embedding benchmark. arXiv preprint arXiv:2210.07316 (2022)."},{"key":"e_1_3_2_1_32_1","unstructured":"OpenAI. 2023. GPT-3.5 Turbo fine-tuning and API updates. https:\/\/openai.com\/index\/gpt-3-5-turbo-fine-tuning-and-api-updates\/"},{"key":"e_1_3_2_1_33_1","volume-title":"Proceedings of the 40th annual meeting of the Association for Computational Linguistics. 311--318","author":"Papineni Kishore","year":"2002","unstructured":"Kishore Papineni, Salim Roukos, Todd Ward, and Wei-Jing Zhu. 2002. Bleu: a method for automatic evaluation of machine translation. In Proceedings of the 40th annual meeting of the Association for Computational Linguistics. 311--318."},{"key":"e_1_3_2_1_34_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D19-1410"},{"key":"e_1_3_2_1_35_1","doi-asserted-by":"publisher","DOI":"10.1613\/jair.1.12007"},{"key":"e_1_3_2_1_36_1","doi-asserted-by":"publisher","DOI":"10.1145\/3377811.3380420"},{"key":"e_1_3_2_1_37_1","doi-asserted-by":"publisher","DOI":"10.1145\/3510003.3510206"},{"key":"e_1_3_2_1_38_1","unstructured":"Yuqing Tang Chau Tran Xian Li Peng-Jen Chen Naman Goyal Vishrav Chaudhary Jiatao Gu and Angela Fan. 2020. Multilingual Translation with Extensible Multilingual Pretraining and Finetuning. (2020). arXiv:2008.00401 [cs.CL]"},{"key":"e_1_3_2_1_39_1","first-page":"21","article-title":"Translation Studies in the year 2000: the state of the art. Terminology in theory and practice. 2002)","volume":"6","author":"Thelen Marcel","year":"2002","unstructured":"Marcel Thelen. 2002. Translation Studies in the year 2000: the state of the art. Terminology in theory and practice. 2002). Translation and Meaning. Part 6 (2002). 21--39.","journal-title":"Translation and Meaning. Part"},{"key":"e_1_3_2_1_40_1","unstructured":"Liang Tian Derek F Wong Lidia S Chao Paulo Quaresma Francisco Oliveira and Lu Yi. 2014. UM-Corpus: A Large English-Chinese Parallel Corpus for Statistical Machine Translation.. In LREC. 1837--1842."},{"key":"e_1_3_2_1_41_1","volume-title":"UniTE: Unified translation evaluation. arXiv preprint arXiv:2204.13346","author":"Wan Yu","year":"2022","unstructured":"Yu Wan, Dayiheng Liu, Baosong Yang, Haibo Zhang, Boxing Chen, Derek F Wong, and Lidia S Chao. 2022. UniTE: Unified translation evaluation. arXiv preprint arXiv:2204.13346 (2022)."},{"key":"e_1_3_2_1_42_1","doi-asserted-by":"publisher","DOI":"10.1145\/3597926.3598081"},{"key":"e_1_3_2_1_43_1","unstructured":"Wiktionary. 2024. Wiktionary the free dictionary. https:\/\/en.wiktionary.org\/"},{"key":"e_1_3_2_1_44_1","unstructured":"Troy Wolverton. 2018. Google CEO Sundar Pichai revealed a jaw-dropping fact about its translation app that shows how much money is still sitting on the table. https:\/\/www.businessinsider.com\/sundar-pichai-google-translate-143-billion-words-daily-2018-7"},{"key":"e_1_3_2_1_45_1","volume-title":"Introducci\u00f3n a la Teor\u00eda General de la Terminolog\u00eda ya la Lexicograf\u00eda Terminogr\u00e1fica","author":"Cabr\u00e9 Eugene","year":"1998","unstructured":"Eugene W\u00dcSTER and MT Cabr\u00e9. 1998. Introducci\u00f3n a la Teor\u00eda General de la Terminolog\u00eda ya la Lexicograf\u00eda Terminogr\u00e1fica. Barcelona: IULA (1998)."},{"key":"e_1_3_2_1_46_1","volume-title":"C-pack: Packaged resources to advance general chinese embedding. arXiv preprint arXiv:2309.07597","author":"Xiao Shitao","year":"2023","unstructured":"Shitao Xiao, Zheng Liu, Peitian Zhang, and Niklas Muennighof. 2023. C-pack: Packaged resources to advance general chinese embedding. arXiv preprint arXiv:2309.07597 (2023)."},{"key":"e_1_3_2_1_47_1","doi-asserted-by":"publisher","DOI":"10.1145\/3238147.3238187"},{"key":"e_1_3_2_1_48_1","volume-title":"Machine Translation Testing via Syntactic Tree Pruning. ACM Transactions on Software Engineering and Methodology","author":"Zhang Quanjun","year":"2024","unstructured":"Quanjun Zhang, Juan Zhai, Chunrong Fang, Jiawei Liu, Weisong Sun, Haichuan Hu, and Qingyu Wang. 2024. Machine Translation Testing via Syntactic Tree Pruning. ACM Transactions on Software Engineering and Methodology (2024)."}],"event":{"name":"ASE '24: 39th IEEE\/ACM International Conference on Automated Software Engineering","location":"Sacramento CA USA","acronym":"ASE '24","sponsor":["SIGAI ACM Special Interest Group on Artificial Intelligence","SIGSOFT ACM Special Interest Group on Software Engineering","IEEE CS"]},"container-title":["Proceedings of the 39th IEEE\/ACM International Conference on Automated Software Engineering"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3691620.3695069","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3691620.3695069","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,19]],"date-time":"2025-06-19T00:04:07Z","timestamp":1750291447000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3691620.3695069"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,10,27]]},"references-count":48,"alternative-id":["10.1145\/3691620.3695069","10.1145\/3691620"],"URL":"https:\/\/doi.org\/10.1145\/3691620.3695069","relation":{},"subject":[],"published":{"date-parts":[[2024,10,27]]},"assertion":[{"value":"2024-10-27","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}