{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,10,1]],"date-time":"2025-10-01T15:15:29Z","timestamp":1759331729993,"version":"3.37.3"},"reference-count":61,"publisher":"Springer Science and Business Media LLC","issue":"1","license":[{"start":{"date-parts":[[2024,10,30]],"date-time":"2024-10-30T00:00:00Z","timestamp":1730246400000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2024,10,30]],"date-time":"2024-10-30T00:00:00Z","timestamp":1730246400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Empir Software Eng"],"published-print":{"date-parts":[[2025,1]]},"DOI":"10.1007\/s10664-024-10549-2","type":"journal-article","created":{"date-parts":[[2024,10,30]],"date-time":"2024-10-30T15:42:30Z","timestamp":1730302950000},"update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":1,"title":["Towards effectively testing machine translation systems from white-box perspectives"],"prefix":"10.1007","volume":"30","author":[{"given":"Hanying","family":"Shao","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Zishuo","family":"Ding","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Weiyi","family":"Shang","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Jinqiu","family":"Yang","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Nikolaos","family":"Tsantalis","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2024,10,30]]},"reference":[{"key":"10549_CR1","unstructured":"(2019) Spacy. https:\/\/spacy.io\/"},{"key":"10549_CR2","unstructured":"Bahdanau D, Cho K, Bengio Y (2015) Neural machine translation by jointly learning to align and translate. In: Bengio Y, LeCun Y (eds) 3rd International Conference on Learning Representations, ICLR 2015, San Diego, CA, USA, May 7-9, 2015, Conference Track Proceedings, arXiv:1409.0473"},{"key":"10549_CR3","unstructured":"Belinkov Y, Bisk Y (2018) Synthetic and natural noise both break neural machine translation. In: 6th International Conference on Learning Representations, ICLR 2018, Vancouver, BC, Canada, April 30 - May 3, 2018, Conference Track Proceedings, OpenReview.net, https:\/\/openreview.net\/forum?id=BJ8vJebC-"},{"key":"10549_CR4","doi-asserted-by":"publisher","unstructured":"Chen J, Ding Z, Tang Y, Sayagh M, Li H, Adams B, Shang W (2023) Iopv: On inconsistent option performance variations. In: Proceedings of the 31st ACM Joint European Software Engineering Conference and Symposium on the Foundations of Software Engineering, Association for Computing Machinery, New York, NY, USA, ESEC\/FSE 2023, pp 845\u2013857, https:\/\/doi.org\/10.1145\/3611643.3616319","DOI":"10.1145\/3611643.3616319"},{"key":"10549_CR5","unstructured":"deepctrl (2024) deepctrl-sft-data. https:\/\/www.modelscope.cn\/datasets\/deepctrl\/deepctrl-sft-data\/summary"},{"key":"10549_CR6","doi-asserted-by":"publisher","unstructured":"Deng C, Liu M, Qin Y, Zhang J, Duan HX, Sun D (2022) ValCAT: Variable-length contextualized adversarial transformations using encoder-decoder language model. In: Proceedings of the 2022 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, Association for Computational Linguistics, Seattle, United States, pp 1735\u20131746, https:\/\/doi.org\/10.18653\/v1\/2022.naacl-main.125, https:\/\/aclanthology.org\/2022.naacl-main.125","DOI":"10.18653\/v1\/2022.naacl-main.125"},{"key":"10549_CR7","doi-asserted-by":"publisher","unstructured":"Devlin J, Chang M, Lee K, Toutanova K (2019) BERT: pre-training of deep bidirectional transformers for language understanding. In: Burstein J, Doran C, Solorio T (eds) Proceedings of the 2019 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, NAACL-HLT 2019, Minneapolis, MN, USA, June 2-7, 2019, Volume 1 (Long and Short Papers), Association for Computational Linguistics, pp 4171\u20134186, https:\/\/doi.org\/10.18653\/v1\/n19-1423","DOI":"10.18653\/v1\/n19-1423"},{"key":"10549_CR8","doi-asserted-by":"publisher","unstructured":"Ding Z, Tang Y, Li Y, Li H, Shang W (2023) On the temporal relations between logging and code. In: 2023 IEEE\/ACM 45th International conference on software engineering (ICSE), pp 843\u2013854, https:\/\/doi.org\/10.1109\/ICSE48619.2023.00079","DOI":"10.1109\/ICSE48619.2023.00079"},{"key":"10549_CR9","unstructured":"Fursov I, Zaytsev A, Kluchnikov N, Kravchenko A, Burnaev E (2020) Differentiable language model adversarial attacks on categorical sequence classifiers. arXiv:2006.11078"},{"key":"10549_CR10","unstructured":"Ghader H, Monz C (2017) What does attention in neural machine translation pay attention to? In: Kondrak G, Watanabe T (eds) Proceedings of the Eighth International Joint Conference on Natural Language Processing, IJCNLP 2017, Taipei, Taiwan, November 27 - December 1, 2017 - Volume 1: Long Papers, Asian Federation of Natural Language Processing, pp 30\u201339, https:\/\/aclanthology.org\/I17-1004\/"},{"key":"10549_CR11","unstructured":"Goodfellow IJ, Shlens J, Szegedy C (2015) Explaining and harnessing adversarial examples. In: Bengio Y, LeCun Y (eds) 3rd International Conference on Learning Representations, ICLR 2015, San Diego, CA, USA, May 7\u20139, 2015, Conference Track Proceedings, arXiv:1412.6572"},{"key":"10549_CR12","doi-asserted-by":"crossref","unstructured":"Grosse K, Papernot N, Manoharan P, Backes M, McDaniel PD (2016) Adversarial perturbations against deep neural networks for malware classification. arXiv:1606.04435","DOI":"10.1109\/SP.2016.41"},{"key":"10549_CR13","doi-asserted-by":"crossref","unstructured":"Gu J, Wang Y, Cho K, Li VOK (2018) Search engine guided neural machine translation. In: McIlraith SA, Weinberger KQ (eds) Proceedings of the Thirty-Second AAAI Conference on Artificial Intelligence, (AAAI-18), the 30th innovative Applications of Artificial Intelligence (IAAI-18), and the 8th AAAI Symposium on Educational Advances in Artificial Intelligence (EAAI-18), New Orleans, Louisiana, USA, February 2-7, 2018, AAAI Press, pp 5133\u20135140, https:\/\/www.aaai.org\/ocs\/index.php\/AAAI\/AAAI18\/paper\/view\/17282","DOI":"10.1609\/aaai.v32i1.12013"},{"key":"10549_CR14","unstructured":"Guo J, Zhang Z, Zhang L, Xu L, Chen B, Chen E, Luo W (2021) Towards variable-length textual adversarial attacks. arXiv:2104.08139"},{"key":"10549_CR15","doi-asserted-by":"publisher","unstructured":"Gupta S, He P, Meister C, Su Z (2020) Machine translation testing via pathological invariance. In: Devanbu P, Cohen MB, Zimmermann T (eds) ESEC\/FSE \u201920: 28th ACM Joint European Software Engineering Conference and Symposium on the Foundations of Software Engineering, Virtual Event, USA, November 8-13, 2020, ACM, pp 863\u2013875, https:\/\/doi.org\/10.1145\/3368089.3409756","DOI":"10.1145\/3368089.3409756"},{"key":"10549_CR16","doi-asserted-by":"publisher","unstructured":"He P, Meister C, Su Z (2020) Structure-invariant testing for machine translation. In: Rothermel G, Bae D (eds) ICSE \u201920: 42nd International Conference on Software Engineering, Seoul, South Korea, 27 June - 19 July, 2020, ACM, pp 961\u2013973, https:\/\/doi.org\/10.1145\/3377811.3380339","DOI":"10.1145\/3377811.3380339"},{"key":"10549_CR17","doi-asserted-by":"publisher","unstructured":"He P, Meister C, Su Z (2021) Testing machine translation via referential transparency. In: 43rd IEEE\/ACM International conference on software engineering, ICSE 2021, Madrid, Spain, 22-30 May 2021, IEEE, pp 410\u2013422, https:\/\/doi.org\/10.1109\/ICSE43902.2021.00047","DOI":"10.1109\/ICSE43902.2021.00047"},{"key":"10549_CR18","unstructured":"Helsinki-NLP (2020a) opus-2020-07-14. https:\/\/github.com\/Helsinki-NLP\/Tatoeba-Challenge\/blob\/master\/models\/eng-zho\/README.md"},{"key":"10549_CR19","unstructured":"Helsinki-NLP (2020b) opus-mt-en-zh. https:\/\/huggingface.co\/Helsinki-NLP\/opus-mt-en-zh"},{"key":"10549_CR20","doi-asserted-by":"crossref","unstructured":"Hintikka J (1984) A hundred years later: The rise and fall of frege\u2019s influence in language theory. Synthese 59:27\u201349, https:\/\/api.semanticscholar.org\/CorpusID:46975316","DOI":"10.1007\/BF00873280"},{"issue":"5","key":"10549_CR21","doi-asserted-by":"publisher","first-page":"350","DOI":"10.1145\/359581.359603","volume":"20","author":"JW Hunt","year":"1977","unstructured":"Hunt JW, Szymanski TG (1977) A fast algorithm for computing longest subsequences. Commun ACM 20(5):350\u2013353","journal-title":"Commun ACM"},{"key":"10549_CR22","doi-asserted-by":"crossref","unstructured":"Kalyan KS, Rajasekharan A, Sangeetha S (2021) Ammus : A survey of transformer-based pretrained models in natural language processing. arXiv:2108.05542","DOI":"10.1016\/j.jbi.2021.103982"},{"key":"10549_CR23","doi-asserted-by":"publisher","unstructured":"Karpukhin V, Levy O, Eisenstein J, Ghazvininejad M (2019) Training on synthetic noise improves robustness to natural noise in machine translation. In: Xu W, Ritter A, Baldwin T, Rahimi A (eds) Proceedings of the 5th Workshop on Noisy User-generated Text, W-NUT@EMNLP 2019, Hong Kong, China, November 4, 2019, Association for Computational Linguistics, pp 42\u201347, https:\/\/doi.org\/10.18653\/v1\/D19-5506","DOI":"10.18653\/v1\/D19-5506"},{"key":"10549_CR24","doi-asserted-by":"publisher","unstructured":"Khan S, Naseer M, Hayat M, Zamir SW, Khan FS, Shah M (2022) Transformers in vision: A survey. ACM Comput Surv 54(10s), https:\/\/doi.org\/10.1145\/3505244","DOI":"10.1145\/3505244"},{"key":"10549_CR25","doi-asserted-by":"publisher","unstructured":"Khayrallah H, Koehn P (2018) On the impact of various types of noise on neural machine translation. In: Birch A, Finch AM, Luong M, Neubig G, Oda Y (eds) Proceedings of the 2nd Workshop on Neural Machine Translation and Generation, NMT@ACL 2018, Melbourne, Australia, July 20, 2018, Association for Computational Linguistics, pp 74\u201383, https:\/\/doi.org\/10.18653\/v1\/w18-2709","DOI":"10.18653\/v1\/w18-2709"},{"key":"10549_CR26","doi-asserted-by":"crossref","unstructured":"Li J, Ji S, Du T, Li B, Wang T (2019) Textbugger: Generating adversarial text against real-world applications. In: 26th Annual Network and Distributed System Security Symposium, NDSS 2019, San Diego, California, USA, February 24-27, 2019, The Internet Society, https:\/\/www.ndss-symposium.org\/ndss-paper\/textbugger-generating-adversarial-text-against-real-world-applications\/","DOI":"10.14722\/ndss.2019.23138"},{"key":"10549_CR27","doi-asserted-by":"publisher","unstructured":"Li J, Wu Y, Gaur Y, Wang C, Zhao R, Liu S (2020) On the comparison of popular end-to-end models for large scale speech recognition. In: Meng H, Xu B, Zheng TF (eds) Interspeech 2020, 21st Annual Conference of the International Speech Communication Association, Virtual Event, Shanghai, China, 25-29 October 2020, ISCA, pp 1\u20135, https:\/\/doi.org\/10.21437\/Interspeech.2020-2846","DOI":"10.21437\/Interspeech.2020-2846"},{"key":"10549_CR28","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1155\/2022\/9121636","volume":"2022","author":"Z Lihua","year":"2022","unstructured":"Lihua Z (2022) The relationship between machine translation and human translation under the influence of artificial intelligence machine translation. Mobile Inf Syst 2022:1\u20138. https:\/\/doi.org\/10.1155\/2022\/9121636","journal-title":"Mobile Inf Syst"},{"issue":"12","key":"10549_CR29","doi-asserted-by":"publisher","first-page":"5046","DOI":"10.1109\/TSE.2023.3323969","volume":"49","author":"S Liu","year":"2023","unstructured":"Liu S, Dou S, Chen J, Zhang Z, Lu Y (2023) Differential testing of machine translators based on compositional semantics. IEEE Trans Softw Eng 49(12):5046\u20135059. https:\/\/doi.org\/10.1109\/TSE.2023.3323969","journal-title":"IEEE Trans Softw Eng"},{"key":"10549_CR30","doi-asserted-by":"publisher","first-page":"243","DOI":"10.1515\/text.1.1988.8.3.243","volume":"8","author":"W Mann","year":"1988","unstructured":"Mann W, Thompson S (1988) Rethorical structure theory: Toward a functional theory of text organization. Text 8:243\u2013281. https:\/\/doi.org\/10.1515\/text.1.1988.8.3.243","journal-title":"Text"},{"key":"10549_CR31","doi-asserted-by":"publisher","unstructured":"Manning C, Surdeanu M, Bauer J, Finkel J, Bethard S, McClosky D (2014) The Stanford CoreNLP natural language processing toolkit. In: Bontcheva K, Zhu J (eds) Proceedings of 52nd Annual Meeting of the Association for Computational Linguistics: System Demonstrations, Association for Computational Linguistics, Baltimore, Maryland, pp 55\u201360, https:\/\/doi.org\/10.3115\/v1\/P14-5010, https:\/\/aclanthology.org\/P14-5010","DOI":"10.3115\/v1\/P14-5010"},{"key":"10549_CR32","unstructured":"de\u00a0Marneffe MC, Dozat T, Silveira N, Haverinen K, Ginter F, Nivre J, Manning CD (2014) Universal Stanford dependencies: A cross-linguistic typology. In: Proceedings of the Ninth International Conference on Language Resources and Evaluation (LREC\u201914), European Language Resources Association (ELRA), Reykjavik, Iceland, pp 4585\u20134592, http:\/\/www.lrec-conf.org\/proceedings\/lrec2014\/pdf\/1062_Paper.pdf"},{"key":"10549_CR33","unstructured":"Mood A, Graybill F, Boes D (1973) Introduction to the Theory of Statistics. International Student edition, McGraw-Hill, https:\/\/books.google.ca\/books?id=Viu2AAAAIAAJ"},{"key":"10549_CR34","doi-asserted-by":"publisher","unstructured":"Nguyen T, Nguyen L, Tran P, Nguyen H (2021) Improving transformer-based neural machine translation with prior alignments. Complex 2021:5515407:1\u20135515407:10, https:\/\/doi.org\/10.1155\/2021\/5515407","DOI":"10.1155\/2021\/5515407"},{"key":"10549_CR35","doi-asserted-by":"publisher","unstructured":"Papineni K, Roukos S, Ward T, Zhu WJ (2002) Bleu: A method for automatic evaluation of machine translation. In: Proceedings of the 40th annual meeting on association for computational linguistics, association for computational linguistics, USA, ACL \u201902, pp 311\u2013318. https:\/\/doi.org\/10.3115\/1073083.1073135","DOI":"10.3115\/1073083.1073135"},{"issue":"11","key":"10549_CR36","doi-asserted-by":"publisher","first-page":"137","DOI":"10.1145\/3361566","volume":"62","author":"K Pei","year":"2019","unstructured":"Pei K, Cao Y, Yang J, Jana S (2019) Deepxplore: automated whitebox testing of deep learning systems. Commun ACM 62(11):137\u2013145. https:\/\/doi.org\/10.1145\/3361566","journal-title":"Commun ACM"},{"key":"10549_CR37","doi-asserted-by":"publisher","unstructured":"Pennington J, Socher R, Manning CD (2014) Glove: Global vectors for word representation. In: Moschitti A, Pang B, Daelemans W (eds) Proceedings of the 2014 Conference on Empirical Methods in Natural Language Processing, EMNLP 2014, October 25-29, 2014, Doha, Qatar, A meeting of SIGDAT, a Special Interest Group of the ACL, ACL, pp 1532\u20131543, https:\/\/doi.org\/10.3115\/V1\/D14-1162","DOI":"10.3115\/V1\/D14-1162"},{"key":"10549_CR38","doi-asserted-by":"publisher","unstructured":"Pesu D, Zhou ZQ, Zhen J, Towey D (2018) A monte carlo method for metamorphic testing of machine translation services. In: Xie X, Pullum LL, Poon P (eds) 3rd IEEE\/ACM International Workshop on Metamorphic Testing, MET 2018, Gothenburg, Sweden, May 27, 2018, ACM, pp 38\u201345, https:\/\/doi.org\/10.1145\/3193977.3193980","DOI":"10.1145\/3193977.3193980"},{"issue":"5","key":"10549_CR39","doi-asserted-by":"publisher","first-page":"522","DOI":"10.1109\/34.682181","volume":"20","author":"E Ristad","year":"1998","unstructured":"Ristad E, Yianilos P (1998) Learning string-edit distance. IEEE Trans Pattern Anal Machine Intell 20(5):522\u2013532. https:\/\/doi.org\/10.1109\/34.682181","journal-title":"IEEE Trans Pattern Anal Machine Intell"},{"issue":"5","key":"10549_CR40","doi-asserted-by":"publisher","first-page":"503","DOI":"10.1108\/00220410410560582","volume":"60","author":"S Robertson","year":"2004","unstructured":"Robertson S (2004) Understanding inverse document frequency: on theoretical arguments for IDF. J Documentation 60(5):503\u2013520. https:\/\/doi.org\/10.1108\/00220410410560582","journal-title":"J Documentation"},{"key":"10549_CR41","doi-asserted-by":"publisher","first-page":"1801","DOI":"10.1109\/TASLP.2020.2998278","volume":"28","author":"K Song","year":"2020","unstructured":"Song K, Zhou X, Yu H, Huang Z, Zhang Y, Luo W, Duan X, Zhang M (2020) Towards better word alignment in transformer. IEEE ACM Trans Audio Speech Lang Process 28:1801\u20131812. https:\/\/doi.org\/10.1109\/TASLP.2020.2998278","journal-title":"IEEE ACM Trans Audio Speech Lang Process"},{"key":"10549_CR42","doi-asserted-by":"publisher","unstructured":"Sun C, Fu A, Poon P, Xie X, Liu H, Chen TY (2021) Metric$$\\$^{\\wedge }\\{+\\}\\$+:$$ A metamorphic relation identification technique based on input plus output domains. IEEE Trans Software Eng 47(9):1764\u20131785. https:\/\/doi.org\/10.1109\/TSE.2019.2934848","DOI":"10.1109\/TSE.2019.2934848"},{"key":"10549_CR43","doi-asserted-by":"publisher","unstructured":"Sun Z, Zhang JM, Harman M, Papadakis M, Zhang L (2020) Automatic testing and improvement of machine translation. In: Rothermel G, Bae D (eds) ICSE \u201920: 42nd International Conference on Software Engineering, Seoul, South Korea, 27 June - 19 July, 2020, ACM, pp 974\u2013985, https:\/\/doi.org\/10.1145\/3377811.3380420","DOI":"10.1145\/3377811.3380420"},{"key":"10549_CR44","doi-asserted-by":"publisher","unstructured":"Sun Z, Zhang JM, Xiong Y, Harman M, Papadakis M, Zhang L (2022) Improving machine translation systems via isotopic replacement. In: 44th IEEE\/ACM 44th International conference on software engineering, ICSE 2022, Pittsburgh, PA, USA, May 25-27, 2022, ACM, pp 1181\u20131192, https:\/\/doi.org\/10.1145\/3510003.3510206","DOI":"10.1145\/3510003.3510206"},{"key":"10549_CR45","doi-asserted-by":"publisher","unstructured":"Touvron H, Lavril T, Izacard G, Martinet X, Lachaux M, Lacroix T, Rozi\u00e8re B, Goyal N, Hambro E, Azhar F, Rodriguez A, Joulin A, Grave E, Lample G (2023) Llama: Open and efficient foundation language models. CoRR abs\/2302.13971, https:\/\/doi.org\/10.48550\/ARXIV.2302.13971, arXiv:2302.13971","DOI":"10.48550\/ARXIV.2302.13971"},{"key":"10549_CR46","unstructured":"Vaswani A, Shazeer N, Parmar N, Uszkoreit J, Jones L, Gomez AN, Kaiser L, Polosukhin I (2017) Attention is all you need. In: Guyon I, von Luxburg U, Bengio S, Wallach HM, Fergus R, Vishwanathan SVN, Garnett R (eds) Advances in Neural Information Processing Systems 30: Annual Conference on Neural Information Processing Systems 2017, December 4-9, 2017, Long Beach, CA, USA, pp 5998\u20136008, https:\/\/proceedings.neurips.cc\/paper\/2017\/hash\/3f5ee243547dee91fbd053c1c4a845aa-Abstract.html"},{"key":"10549_CR47","doi-asserted-by":"publisher","unstructured":"Wang L, Zheng X (2020) Improving grammatical error correction models with purpose-built adversarial examples. In: Webber B, Cohn T, He Y, Liu Y (eds) Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing, EMNLP 2020, Online, November 16-20, 2020, Association for Computational Linguistics, pp 2858\u20132869, https:\/\/doi.org\/10.18653\/v1\/2020.emnlp-main.228","DOI":"10.18653\/v1\/2020.emnlp-main.228"},{"key":"10549_CR48","doi-asserted-by":"publisher","unstructured":"Wang W, Zheng W, Liu D, Zhang C, Zeng Q, Deng Y, Yang W, He P, Xie T (2019) Detecting failures of neural machine translation in the absence of reference translations. In: 49th Annual IEEE\/IFIP International Conference on Dependable Systems and Networks, DSN (Industry Track) 2019, Portland, OR, USA, June 24-27, 2019, IEEE, pp 1\u20134, https:\/\/doi.org\/10.1109\/DSN-Industry.2019.00007","DOI":"10.1109\/DSN-Industry.2019.00007"},{"key":"10549_CR49","unstructured":"WMT18 (2018) Wmt.2018.news-commentary. http:\/\/data.statmt.org\/wmt18\/translation-task\/"},{"key":"10549_CR50","unstructured":"Xia F (2000) The part-of-speech tagging guidelines for the penn chinese treebank (3.0)"},{"issue":"4","key":"10549_CR51","doi-asserted-by":"publisher","first-page":"1293","DOI":"10.1109\/TR.2020.2972266","volume":"69","author":"X Xie","year":"2020","unstructured":"Xie X, Zhang Z, Chen TY, Liu Y, Poon P, Xu B (2020) METTLE: A metamorphic testing approach to assessing and validating unsupervised machine learning systems. IEEE Trans Reliab 69(4):1293\u20131322. https:\/\/doi.org\/10.1109\/TR.2020.2972266","journal-title":"IEEE Trans Reliab"},{"key":"10549_CR52","doi-asserted-by":"publisher","unstructured":"Xie X, Yin P, Chen S (2022) Boosting the revealing of detected violations in deep learning testing: A diversity-guided method. In: 37th IEEE\/ACM International Conference on Automated Software Engineering, ASE 2022, Rochester, MI, USA, October 10-14, 2022, ACM, pp 17:1\u201317:13, https:\/\/doi.org\/10.1145\/3551349.3556919","DOI":"10.1145\/3551349.3556919"},{"key":"10549_CR53","doi-asserted-by":"crossref","unstructured":"Yan Y, Li R, Wang S, Zhang F, Wu W, Xu W (2021) Consert: A contrastive framework for self-supervised sentence representation transfer. arXiv:2105.11741","DOI":"10.18653\/v1\/2021.acl-long.393"},{"key":"10549_CR54","doi-asserted-by":"publisher","first-page":"127","DOI":"10.1016\/j.jbusres.2016.10.007","volume":"72","author":"C Zhang","year":"2017","unstructured":"Zhang C, Laroche M, Richard MO (2017) The differential roles of verbs, nouns, and adjectives in english and chinese messages among bilingual consumers. J Business Res 72:127\u2013135","journal-title":"J Business Res"},{"key":"10549_CR55","doi-asserted-by":"publisher","unstructured":"Zhang H, Zhou H, Miao N, Li L (2019) Generating fluent adversarial examples for natural languages. In: Korhonen A, Traum DR, M\u00e0rquez L (eds) Proceedings of the 57th Conference of the Association for Computational Linguistics, ACL 2019, Florence, Italy, July 28- August 2, 2019, Volume 1: Long Papers, Association for Computational Linguistics, pp 5564\u20135569, https:\/\/doi.org\/10.18653\/v1\/p19-1559","DOI":"10.18653\/v1\/p19-1559"},{"key":"10549_CR56","doi-asserted-by":"publisher","unstructured":"Zhang J, van Genabith J (2021) A bidirectional transformer based alignment model for unsupervised word alignment. In: Proceedings of the 59th Annual Meeting of the Association for Computational Linguistics and the 11th International Joint Conference on Natural Language Processing (Volume 1: Long Papers), Association for Computational Linguistics, Online, pp 283\u2013292, https:\/\/doi.org\/10.18653\/v1\/2021.acl-long.24, https:\/\/aclanthology.org\/2021.acl-long.24","DOI":"10.18653\/v1\/2021.acl-long.24"},{"key":"10549_CR57","doi-asserted-by":"publisher","unstructured":"Zhang J, Utiyama M, Sumita E, Neubig G, Nakamura S (2018) Guiding neural machine translation with retrieved translation pieces. In: Proceedings of the 2018 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, Volume 1 (Long Papers), Association for Computational Linguistics, New Orleans, Louisiana, pp 1325\u20131335, https:\/\/doi.org\/10.18653\/v1\/N18-1120, https:\/\/aclanthology.org\/N18-1120","DOI":"10.18653\/v1\/N18-1120"},{"issue":"2","key":"10549_CR58","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1109\/TSE.2019.2962027","volume":"48","author":"JM Zhang","year":"2022","unstructured":"Zhang JM, Harman M, Ma L, Liu Y (2022) Machine learning testing: Survey, landscapes and horizons. IEEE Trans Software Eng 48(2):1\u201336. https:\/\/doi.org\/10.1109\/TSE.2019.2962027","journal-title":"IEEE Trans Software Eng"},{"key":"10549_CR59","doi-asserted-by":"publisher","unstructured":"Zhang Q, Zhai J, Fang C, Liu J, Sun W, Hu H, Wang Q (2024) Machine translation testing via syntactic tree pruning. CoRR abs\/2401.00751, https:\/\/doi.org\/10.48550\/ARXIV.2401.00751, arXiv:2401.00751","DOI":"10.48550\/ARXIV.2401.00751"},{"key":"10549_CR60","unstructured":"Zhichen\u00a0Zhang LC Xin\u00a0LU (2024) Llama3-chinese. https:\/\/github.com\/seanzhang-zhichen\/llama3-chinese"},{"key":"10549_CR61","doi-asserted-by":"publisher","unstructured":"Zhou ZQ, Sun L (2018) Metamorphic testing for machine translations: MT4MT. In: 25th Australasian Software Engineering Conference, ASWEC 2018, Adelaide, Australia, November 26-30, 2018, IEEE Computer Society, pp 96\u2013100, https:\/\/doi.org\/10.1109\/ASWEC.2018.00021","DOI":"10.1109\/ASWEC.2018.00021"}],"container-title":["Empirical Software Engineering"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10664-024-10549-2.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s10664-024-10549-2\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10664-024-10549-2.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,2,23]],"date-time":"2025-02-23T01:26:33Z","timestamp":1740273993000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s10664-024-10549-2"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,10,30]]},"references-count":61,"journal-issue":{"issue":"1","published-print":{"date-parts":[[2025,1]]}},"alternative-id":["10549"],"URL":"https:\/\/doi.org\/10.1007\/s10664-024-10549-2","relation":{},"ISSN":["1382-3256","1573-7616"],"issn-type":[{"type":"print","value":"1382-3256"},{"type":"electronic","value":"1573-7616"}],"subject":[],"published":{"date-parts":[[2024,10,30]]},"assertion":[{"value":"19 September 2024","order":1,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"30 October 2024","order":2,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Declarations"}},{"value":"The authors declare that they have no conflict of interest.","order":2,"name":"Ethics","group":{"name":"EthicsHeading","label":"Conflict of interest"}}],"article-number":"13"}}