{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,3,11]],"date-time":"2026-03-11T23:42:31Z","timestamp":1773272551308,"version":"3.50.1"},"reference-count":42,"publisher":"Springer Science and Business Media LLC","issue":"3","license":[{"start":{"date-parts":[[2018,5,8]],"date-time":"2018-05-08T00:00:00Z","timestamp":1525737600000},"content-version":"tdm","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Machine Translation"],"published-print":{"date-parts":[[2018,9]]},"DOI":"10.1007\/s10590-018-9220-z","type":"journal-article","created":{"date-parts":[[2018,5,8]],"date-time":"2018-05-08T08:46:33Z","timestamp":1525769193000},"page":"217-235","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":27,"title":["Human versus automatic quality evaluation of NMT and PBSMT"],"prefix":"10.1007","volume":"32","author":[{"ORCID":"https:\/\/orcid.org\/0000-0001-6300-797X","authenticated-orcid":false,"given":"Dimitar","family":"Shterionov","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Riccardo","family":"Superbo","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Pat","family":"Nagle","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Laura","family":"Casanellas","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Tony","family":"O\u2019Dowd","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Andy","family":"Way","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2018,5,8]]},"reference":[{"key":"9220_CR1","doi-asserted-by":"crossref","unstructured":"Agarwal A, Lavie A (2008) METEOR, M-BLEU and M-TER: evaluation metrics for high-correlation with human rankings of machine translation output. In: Proceedings of the third workshop on statistical machine translation, Columbus, Ohio, pp 115\u2013118","DOI":"10.3115\/1626394.1626406"},{"key":"9220_CR2","unstructured":"Bahdanau D, Cho K, Bengio Y (2015) Neural machine translation by jointly learning to align and translate. In: Proceedings of the 6th international conference on learning representations (ICLR 2015), San Diego, CA, USA"},{"key":"9220_CR3","doi-asserted-by":"crossref","unstructured":"Bentivogli L, Bisazza A, Cettolo M, Federico M (2016) Neural versus phrase-based machine translation quality: a case study. In: Proceedings of the 2016 conference on empirical methods in natural language processing, Austin, Texas, pp 257\u2013267","DOI":"10.18653\/v1\/D16-1025"},{"key":"9220_CR4","unstructured":"Callison-Burch C, Osborne M, Koehn P (2006) Re-evaluating the role of BLEU in machine translation research. In: Proceedings of the eleventh conference of the European chapter of the association for computational linguistics, Trento, Italy, pp 249\u2013256"},{"issue":"1","key":"9220_CR5","doi-asserted-by":"publisher","first-page":"109","DOI":"10.1515\/pralin-2017-0013","volume":"108","author":"S Castilho","year":"2017","unstructured":"Castilho S, Moorkens J, Gaspari F, Calixto I, Tinsley J, Way A (2017) Is neural machine translation the new state of the art? Prague Bull Math Linguist 108(1):109\u2013120","journal-title":"Prague Bull Math Linguist"},{"key":"9220_CR6","unstructured":"Cer D, Manning CD, Jurafsky D (2010) The best lexical metric for phrase-based statistical MT system optimization. In: Human language technologies: the 2010 annual conference of the North American chapter of the association for computational linguistics, Los Angeles, California, pp 555\u2013563"},{"key":"9220_CR7","unstructured":"Cettolo M, Niehues J, St\u00fcker S, Bentivogli L, Cattoni R, Federico M (2015) The IWSLT 2015 evaluation campaign. In: Proceedings of the 12th international workshop on spoken language translation, Da Nang, Vietnam, pp 2\u201314"},{"key":"9220_CR8","doi-asserted-by":"crossref","unstructured":"Chen B, Cherry C (2014) A systematic comparison of smoothing techniques for sentence-level BLEU. In: Proceedings of the ninth workshop on statistical machine translation (WMT@ACL 2014), Baltimore, Maryland, USA, pp 362\u2013367","DOI":"10.3115\/v1\/W14-3346"},{"key":"9220_CR9","doi-asserted-by":"crossref","unstructured":"Chiang D (2005) A hierarchical phrase-based model for statistical machine translation. In: Proceedings of the 43rd annual meeting of the association for computational linguistics (ACL\u201905), Ann Arbor, Michigan, pp 263\u2013270","DOI":"10.3115\/1219840.1219873"},{"key":"9220_CR10","doi-asserted-by":"crossref","unstructured":"Chiang D, DeNeefe S, Chan YS, Ng HT (2008) Decomposability of translation metrics for improved evaluation and efficient algorithms. In: Proceedings of the conference on empirical methods in natural language processing, Honolulu, Hawaii, USA, pp 610\u2013619","DOI":"10.3115\/1613715.1613791"},{"key":"9220_CR11","doi-asserted-by":"crossref","unstructured":"Cho K, van Merri\u00ebnboer B, G\u00fcl\u00e7ehre \u00c7, Bahdanau D, Bougares F, Schwenk H, Bengio Y (2014) Learning phrase representations using RNN encoder\u2013decoder for statistical machine translation. In: Proceedings of the 2014 conference on empirical methods in natural language processing, Doha, Qatar, pp 1724\u20131734","DOI":"10.3115\/v1\/D14-1179"},{"key":"9220_CR12","doi-asserted-by":"crossref","unstructured":"Chung J, Cho K, Bengio Y (2016) A character-level decoder without explicit segmentation for neural machine translation. In: Proceedings of the 54th annual meeting of the association for computational linguistics, ACL 2016, vol 1, long papers, Berlin, Germany, pp 1693\u20131703","DOI":"10.18653\/v1\/P16-1160"},{"issue":"2","key":"9220_CR13","first-page":"245","volume":"31","author":"MR Costa-Juss\u00e0","year":"2012","unstructured":"Costa-Juss\u00e0 MR, Farr\u00fas M, Mari\u00f1o JB, Fonollosa JAR (2012) Study and comparison of rule-based and statistical Catalan-Spanish machine translation systems. Comput Inform 31(2):245\u2013270","journal-title":"Comput Inform"},{"key":"9220_CR14","unstructured":"Crego JM, Kim J, Klein G, Rebollo A, Yang K, Senellart J, Akhanov E, Brunelle P, Coquard A, Deng Y, Enoue S, Geiss C, Johanson J, Khalsa A, Khiari R, Ko B, Kobus C, Lorieux J, Martins L, Nguyen D, Priori A, Riccardi T, Segal N, Servan C, Tiquet C, Wang B, Yang J, Zhang D, Zhou J, Zoldan P (2016) Systran\u2019s pure neural machine translation systems. CoRR \n                    arXiv:1610.05540"},{"key":"9220_CR15","doi-asserted-by":"publisher","first-page":"1282","DOI":"10.3389\/fpsyg.2017.01282","volume":"8","author":"J Daems","year":"2017","unstructured":"Daems J, Vandepitte S, Hartsuiker RJ, Macken L (2017) Identifying the machine translation error types with the greatest impact on post-editing effort. Front Psychol 8:1282","journal-title":"Front Psychol"},{"key":"9220_CR16","unstructured":"Dyer C, Chahuneau V, Smith NA (2013) A simple, fast, and effective reparameterization of IBM model 2. In: Proceedings of the 2013 conference of the North American chapter of the association for computational linguistics: human language technologies (NAACL), Atlanta, USA, pp 644\u2013649"},{"issue":"1","key":"9220_CR17","doi-asserted-by":"publisher","first-page":"174","DOI":"10.1002\/asi.21674","volume":"63","author":"M Farr\u00fas","year":"2012","unstructured":"Farr\u00fas M, Costa-juss\u00e0 MR, Popovi\u0107 M (2012) Study and correlation analysis of linguistic, perceptual, and automatic machine translation evaluations. J Assoc Inf Sci Technol 63(1):174\u2013184","journal-title":"J Assoc Inf Sci Technol"},{"issue":"5","key":"9220_CR18","doi-asserted-by":"publisher","first-page":"378","DOI":"10.1037\/h0031619","volume":"76","author":"JL Fleiss","year":"1971","unstructured":"Fleiss JL (1971) Measuring nominal scale agreement among many raters. Psychol Bull 76(5):378\u2013382","journal-title":"Psychol Bull"},{"key":"9220_CR19","unstructured":"Ha TL, Niehues J, Eunah C, Mediani M, Waibel A (2015) The KIT translation systems for IWSLT 2015. In: Proceedings of the 12th international workshop on spoken language translation, Da Nang, Vietnam, pp 62\u201369"},{"key":"9220_CR20","unstructured":"Junczys-Dowmunt M, Dwojak T, Hoang H (2016) Is neural machine translation ready for deployment? A case study on 30 translation directions. In: Proceedings of the 9th international workshop on spoken language translation, Seattle, WA"},{"key":"9220_CR21","unstructured":"Kingma DP, Ba J (2014) Adam: a method for stochastic optimization. CoRR \n                    arXiv:1412.6980"},{"key":"9220_CR22","doi-asserted-by":"crossref","unstructured":"Klein G, Kim Y, Deng Y, Senellart J, Rush AM (2017) Opennmt: open-source toolkit for neural machine translation. In: Proceedings of the 55th annual meeting of the association for computational linguistics, ACL 2017, System Demonstrations, Vancouver, Canada, pp 67\u201372","DOI":"10.18653\/v1\/P17-4012"},{"key":"9220_CR23","doi-asserted-by":"crossref","unstructured":"Klubi\u010dka F, Toral A, S\u00e1nchez-Cartagena VM (2017) Fine-grained human evaluation of neural versus phrase-based machine translation. The Prague Bulletin of Mathematical Linguistics, pp 121\u2013132","DOI":"10.1515\/pralin-2017-0014"},{"key":"9220_CR24","volume-title":"Statistical machine translation","author":"P Koehn","year":"2010","unstructured":"Koehn P (2010) Statistical machine translation, 1st edn. Cambridge University Press, New York, NY","edition":"1"},{"key":"9220_CR25","doi-asserted-by":"crossref","unstructured":"Koehn P, Hoang H, Birch A, Callison-Burch C, Federico M, Bertoldi N, Cowan B, Shen W, Moran C, Zens R, Dyer C, Bojar O, Constantin A, Herbst E (2007) Moses: open source toolkit for statistical machine translation. In: Proceedings of the 45th annual meeting of the association for computational linguistics companion volume proceedings of the demo and poster sessions, Prague, Czech Republic, pp 177\u2013180","DOI":"10.3115\/1557769.1557821"},{"issue":"1","key":"9220_CR26","doi-asserted-by":"publisher","first-page":"159","DOI":"10.2307\/2529310","volume":"33","author":"JR Landis","year":"1977","unstructured":"Landis JR, Koch GG (1977) The measurement of observer agreement for categorical data. Biometrics 33(1):159\u2013174","journal-title":"Biometrics"},{"key":"9220_CR27","unstructured":"Luong MT, Manning CD (2015) Stanford neural machine translation systems for spoken language domains. In: Proceedings of the 12th international workshop on spoken language translation (IWSLT), Da Nang, Vietnam, pp 76\u201379"},{"key":"9220_CR28","doi-asserted-by":"crossref","unstructured":"Melamed ID, Green R, Turian JP (2003) Precision and recall of machine translation. In: Human Language Technology Conference of the North American Chapter of the Association for Computational Linguistics, HLT-NAACL 2003, Edmonton, Canada, pp 61\u201363","DOI":"10.3115\/1073483.1073504"},{"issue":"1","key":"9220_CR29","doi-asserted-by":"publisher","first-page":"19","DOI":"10.1162\/089120103321337421","volume":"29","author":"F Och","year":"2003","unstructured":"Och F, Ney H (2003) A systematic comparison of various statistical alignment models. Comput Linguist 29(1):19\u201351","journal-title":"Comput Linguist"},{"key":"9220_CR30","unstructured":"Papineni K, Roukos S, Ward T, Zhu WJ (2002) BLEU: a method for automatic evaluation of machine translation. In: Proceedings of the 40th annual meeting on association for computational linguistics, Philadelphia, Pennsylvania, USA, pp 311\u2013318"},{"key":"9220_CR31","doi-asserted-by":"crossref","unstructured":"Popovi\u0107 M (2015) chrF: character n-gram f-score for automatic MT evaluation. In: Proceedings of the tenth workshop on statistical machine translation (WMT@EMNLP 2015), Lisbon, Portugal, pp 392\u2013395","DOI":"10.18653\/v1\/W15-3049"},{"key":"9220_CR32","doi-asserted-by":"crossref","unstructured":"Sennrich R, Haddow B, Birch A (2016) Neural machine translation of rare words with subword units. In: Proceedings of the 54th annual meeting of the association for computational linguistics, ACL 2016, vol 1, Long Papers, Berlin, Germany, pp 1715\u20131725","DOI":"10.18653\/v1\/P16-1162"},{"key":"9220_CR33","unstructured":"Shterionov D, Du J, Palminteri MA, Casanellas L, O\u2019Dowd T, Way A (2016) Improving KantanMT training efficiency with FastAlign. In: Proceedings of AMTA 2016, the twelfth conference of the Association for Machine Translation in the Americas, vol 2, MT Users\u2019 Track, Austin, TX, USA, pp 222\u2013231"},{"key":"9220_CR34","unstructured":"Shterionov D, Nagle P, Casanellas L, Superbo R, ODowd T (2017) Empirical evaluation of NMT and PBSMT quality for large-scale translation production. In: Proceedings of the user track of the 20th annual conference of the European Association for Machine Translation (EAMT), Prague, Czech Republic, pp 74\u201379"},{"key":"9220_CR35","unstructured":"Smith A, Hardmeier C, Tiedemann J (2016) Climbing Mont BLEU: the strange world of reachable high-BLEU translations. In: Proceedings of the 19th annual conference of the European Association for Machine Translation, EAMT 2017, Riga, Latvia, pp 269\u2013281"},{"key":"9220_CR36","unstructured":"Snover M, Dorr B, Schwartz R, Micciulla L, Makhoul J (2006) A study of translation edit rate with targeted human annotation. In: AMTA 2006. Proceedings of the 7th conference of the association for machine translation of the Americas. Visions for the future of machine translation, Cambridge, Massachusetts, USA, pp 223\u2013231"},{"key":"9220_CR37","unstructured":"Sutskever I, Vinyals O, Le QV (2014) Sequence to sequence learning with neural networks. In: Proceedings of advances in neural information processing systems 27: annual conference on neural information processing systems, Montreal, Quebec, Canada, pp 3104\u20133112"},{"key":"9220_CR38","unstructured":"Vanmassenhove E, Du J, Way A (2016) Improving subject-verb agreement in SMT. In: Proceedings of the fifth workshop on hybrid approaches to translation, Riga, Latvia"},{"key":"9220_CR39","volume-title":"The Bloomsbury Companion to language industry studies","author":"A Way","year":"2018","unstructured":"Way A (2018a) Machine translation: where are we at today? In: Angelone E, Massey G, Ehrensberger-Dow M (eds) The Bloomsbury Companion to language industry studies. Bloomsbury, London"},{"key":"9220_CR40","volume-title":"Translation quality assessment: from principles to practice","author":"A Way","year":"2018","unstructured":"Way A (2018b) Quality expectations of machine translation. In: Moorkens J, Castilho S, Gaspari F, Doherty S (eds) Translation quality assessment: from principles to practice. Springer, Berlin"},{"key":"9220_CR41","unstructured":"Wu Y, Schuster M, Chen Z, Le QV, Norouzi M, Macherey W, Krikun M, Cao Y, Gao Q, Macherey K, Klingner J, Shah A, Johnson M, Liu X, Kaiser L, Gouws S, Kato Y, Kudo T, Kazawa H, Stevens K, Kurian G, Patil N, Wang W, Young C, Smith J, Riesa J, Rudnick A, Vinyals O, Corrado G, Hughes M, Dean J (2016) Google\u2019s neural machine translation system: bridging the gap between human and machine translation. CoRR \n                    arXiv:1609.08144"},{"key":"9220_CR42","unstructured":"Ziemski M, Junczys-Dowmunt M, Pouliquen B (2016) The United Nations Parallel Corpus v1.0. In: Proceedings of the tenth international conference on language resources and evaluation, Portoro\u017e, Slovenia, pp 3530\u20133534"}],"container-title":["Machine Translation"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/link.springer.com\/article\/10.1007\/s10590-018-9220-z\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s10590-018-9220-z.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s10590-018-9220-z.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2019,5,7]],"date-time":"2019-05-07T19:36:58Z","timestamp":1557257818000},"score":1,"resource":{"primary":{"URL":"http:\/\/link.springer.com\/10.1007\/s10590-018-9220-z"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2018,5,8]]},"references-count":42,"journal-issue":{"issue":"3","published-print":{"date-parts":[[2018,9]]}},"alternative-id":["9220"],"URL":"https:\/\/doi.org\/10.1007\/s10590-018-9220-z","relation":{},"ISSN":["0922-6567","1573-0573"],"issn-type":[{"value":"0922-6567","type":"print"},{"value":"1573-0573","type":"electronic"}],"subject":[],"published":{"date-parts":[[2018,5,8]]},"assertion":[{"value":"4 September 2017","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"9 April 2018","order":2,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"8 May 2018","order":3,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}}]}}