{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,5,20]],"date-time":"2026-05-20T07:24:57Z","timestamp":1779261897692,"version":"3.51.4"},"reference-count":30,"publisher":"Springer Science and Business Media LLC","issue":"3","license":[{"start":{"date-parts":[[2018,2,10]],"date-time":"2018-02-10T00:00:00Z","timestamp":1518220800000},"content-version":"tdm","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"}],"funder":[{"DOI":"10.13039\/501100001602","name":"Science Foundation Ireland","doi-asserted-by":"crossref","award":["Grant 13\/RC\/2106"],"award-info":[{"award-number":["Grant 13\/RC\/2106"]}],"id":[{"id":"10.13039\/501100001602","id-type":"DOI","asserted-by":"crossref"}]},{"name":"FP7 Seventh Framework Programme","award":["PIAP-GA-2012-324414"],"award-info":[{"award-number":["PIAP-GA-2012-324414"]}]},{"DOI":"10.13039\/501100001711","name":"Schweizerischer Nationalfonds zur F\u00f6rderung der Wissenschaftlichen Forschung","doi-asserted-by":"crossref","award":["74Z0_160501"],"award-info":[{"award-number":["74Z0_160501"]}],"id":[{"id":"10.13039\/501100001711","id-type":"DOI","asserted-by":"crossref"}]}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Machine Translation"],"published-print":{"date-parts":[[2018,9]]},"DOI":"10.1007\/s10590-018-9214-x","type":"journal-article","created":{"date-parts":[[2018,2,10]],"date-time":"2018-02-10T03:30:45Z","timestamp":1518233445000},"page":"195-215","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":23,"title":["Quantitative fine-grained human evaluation of machine translation systems: a case study on English to Croatian"],"prefix":"10.1007","volume":"32","author":[{"ORCID":"https:\/\/orcid.org\/0000-0001-9712-6141","authenticated-orcid":false,"given":"Filip","family":"Klubi\u010dka","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Antonio","family":"Toral","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"V\u00edctor M.","family":"S\u00e1nchez-Cartagena","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2018,2,10]]},"reference":[{"key":"9214_CR1","unstructured":"Bahdanau D, Cho K, Bengio Y (2015) Neural machine translation by jointly learning to align and translate. In: Proceedings of international conference on learning representations, San Diego, CA, USA, 2015"},{"key":"9214_CR2","doi-asserted-by":"crossref","unstructured":"Bentivogli L, Bisazza A, Cettolo M, Federico M (2016) Neural versus phrase-based machine translation quality: a case study. In: Proceedings of the 2016 conference on empirical methods in natural language processing, Austin, Texas, USA, pp 257\u2013267","DOI":"10.18653\/v1\/D16-1025"},{"key":"9214_CR3","doi-asserted-by":"publisher","first-page":"159","DOI":"10.1515\/pralin-2017-0017","volume":"108","author":"A Burchardt","year":"2017","unstructured":"Burchardt A, Macketanz V, Dehdari J, Heigold G, Peter JT, Williams P (2017) A linguistic evaluation of rule-based, phrase-based, and neural MT engines. Prague Bull Math Linguist 108:159\u2013170","journal-title":"Prague Bull Math Linguist"},{"key":"9214_CR4","doi-asserted-by":"crossref","unstructured":"Callison-Burch C, Fordyce C, Koehn P, Monz C, Schroeder J (2007) (Meta-) evaluation of machine translation. In: Proceedings of the second workshop on statistical machine translation, Prague, Czech Republic, pp 136\u2013158","DOI":"10.3115\/1626355.1626373"},{"key":"9214_CR5","doi-asserted-by":"publisher","first-page":"109","DOI":"10.1515\/pralin-2017-0013","volume":"108","author":"S Castilho","year":"2017","unstructured":"Castilho S, Moorkens J, Gaspari F, Calixto I, Tinsley J, Way A (2017) Is neural machine translation the new state of the art? Prague Bull Math Linguist 108:109\u2013120","journal-title":"Prague Bull Math Linguist"},{"issue":"1","key":"9214_CR6","doi-asserted-by":"publisher","first-page":"37","DOI":"10.1177\/001316446002000104","volume":"20","author":"J Cohen","year":"1960","unstructured":"Cohen J (1960) A coefficient of agreement for nominal scales. Educ Psychol Meas 20(1):37\u201346","journal-title":"Educ Psychol Meas"},{"key":"9214_CR7","doi-asserted-by":"crossref","unstructured":"Devlin J, Zbib R, Huang Z, Lamar T, Schwartz R, Makhoul J (2014) Fast and robust neural network joint models for statistical machine translation. In: Proceedings of association for computational linguistics conference, Baltimore, Maryland, USA, pp 1370\u20131380","DOI":"10.3115\/v1\/P14-1129"},{"key":"9214_CR8","unstructured":"Durrani N, Schmid H, Fraser A (2011) A joint sequence translation model with integrated reordering. In: Proceedings of the 49th annual meeting of the association for computational linguistics: human language technologies, vol 1. Portland, Oregon, USA, pp 1045\u20131054"},{"key":"9214_CR9","doi-asserted-by":"crossref","unstructured":"Galley M, Manning CD (2008) A simple and effective hierarchical phrase reordering model. In: Proceedings of the conference on empirical methods in natural language processing, Waikiki, Honolulu, Hawaii, pp 848\u2013856","DOI":"10.3115\/1613715.1613824"},{"key":"9214_CR10","doi-asserted-by":"publisher","first-page":"121","DOI":"10.1515\/pralin-2017-0014","volume":"108","author":"F Klubi\u010dka","year":"2017","unstructured":"Klubi\u010dka F, Toral A, S\u00e1nchez-Cartagena VM (2017) Fine-grained human evaluation of neural versus phrase-based machine translation. Prague Bull Math Linguist 108:121\u2013132","journal-title":"Prague Bull Math Linguist"},{"key":"9214_CR11","unstructured":"Koehn P (2004) Statistical significance tests for machine translation evaluation. In: Proceedings of the conference on empirical methods in natural language processing, Barcelona, Spain, pp 388\u2013395"},{"key":"9214_CR12","unstructured":"Koehn P, Hoang H (2007) Factored translation models. In: Proceedings of conference on empirical methods on natural language processing and computational natural language learning, Jeju Island, Korea, pp 868\u2013876"},{"key":"9214_CR13","doi-asserted-by":"crossref","unstructured":"Koehn P, Och FJ, Marcu D (2003) Statistical phrase-based translation. In: Proceedings of the 2003 conference of the north american chapter of the association for computational linguistics on human language technology, Edmonton, Canada, pp 48\u201354","DOI":"10.21236\/ADA461156"},{"key":"9214_CR14","doi-asserted-by":"crossref","unstructured":"Koehn P, Hoang H, Birch A, Callison-Burch C, Federico M, Bertoldi N, Cowan B, Shen W, Moran C, Zens R, et\u00a0al (2007) Moses: Open source toolkit for statistical machine translation. In: Proceedings of the 45th annual meeting of the ACL on interactive poster and demonstration sessions, Prague, Czech Republic, pp 177\u2013180","DOI":"10.3115\/1557769.1557821"},{"key":"9214_CR15","doi-asserted-by":"crossref","unstructured":"Ljube\u0161i\u0107 N, Klubi\u010dka F (2014) bs,hr,sr WaC-web corpora of Bosnian, Croatian and Serbian. In: Proceedings of the 9th web as corpus workshop (WaC-9), Gothenburg, Sweden, pp 29\u201335","DOI":"10.3115\/v1\/W14-0405"},{"key":"9214_CR16","doi-asserted-by":"crossref","unstructured":"Lommel AR, Burchardt A, Uszkoreit H (2014a) Multidimensional quality metrics (MQM): a framework for declaring and describing translation quality metrics. Tradum\u00e0tica: tecnologies de la traducci\u00f3 pp 455\u2013463","DOI":"10.5565\/rev\/tradumatica.77"},{"key":"9214_CR17","unstructured":"Lommel AR, Popovic M, Burchardt A (2014b) Assessing inter-annotator agreement for translation error annotation. In: MTE: workshop on automatic and manual metrics for operational translation evaluation, Reykjavik, Iceland"},{"key":"9214_CR18","unstructured":"Moore RC, Lewis W (2010) Intelligent selection of language model training data. In: Proceedings of the association for computational linguistics 2010 conference short papers, Stroudsburg, PA, USA, pp 220\u2013224"},{"key":"9214_CR19","unstructured":"Papineni K, Roukos S, Ward T, Zhu WJ (2002) BLEU: a method for automatic evaluation of machine translation. In: Proceedings of the 40th annual meeting on association for computational linguistics, pp 311\u2013318"},{"key":"9214_CR20","unstructured":"Pascanu R, Mikolov T, Bengio Y (2013) On the difficulty of training recurrent neural networks. In: Dasgupta S, Mcallester D (eds) Proceedings of the 30th international conference on machine learning (ICML-13), JMLR workshop and conference proceedings, vol 28. Atlanta, USA, pp 1310\u20131318"},{"key":"9214_CR21","doi-asserted-by":"crossref","unstructured":"Plackett RL (1983) Karl pearson and the chi-squared test. International statistical review\/revue internationale de statistique pp 59\u201372","DOI":"10.2307\/1402731"},{"key":"9214_CR22","doi-asserted-by":"publisher","first-page":"209","DOI":"10.1515\/pralin-2017-0021","volume":"108","author":"M Popovi\u0107","year":"2017","unstructured":"Popovi\u0107 M (2017) Comparing language related issues for NMT and PBMT between German and English. Prague Bull Math Linguist 108:209\u2013220","journal-title":"Prague Bull Math Linguist"},{"issue":"2","key":"9214_CR23","first-page":"354","volume":"4","author":"VM S\u00e1nchez-Cartagena","year":"2016","unstructured":"S\u00e1nchez-Cartagena VM, Ljube\u0161i\u0107 N, Klubi\u010dka F (2016) Dealing with data sparseness in SMT with factored models and morphological expansion: a case study on Croatian. Baltic J Mod Comput 4(2):354\u2013360","journal-title":"Baltic J Mod Comput"},{"key":"9214_CR24","doi-asserted-by":"crossref","unstructured":"Sennrich R, Haddow B, Birch A (2016) Neural machine translation of rare words with subword units. In: Proceedings of the 54th annual meeting of the association for computational linguistics, Berlin, Germany, pp 1715\u20131725","DOI":"10.18653\/v1\/P16-1162"},{"key":"9214_CR25","doi-asserted-by":"crossref","unstructured":"Sennrich R, Firat O, Cho K, Birch A, Haddow B, Hitschler J, Junczys-Dowmunt M, Lubli S, Barone AVM, Mokry J, Nadejde M (2017) Nematus: a toolkit for neural machine translation. In: Proceedings of the European association for computational linguistics 2017 software demonstrations, Valencia, Spain, pp 65\u201368","DOI":"10.18653\/v1\/E17-3017"},{"key":"9214_CR26","unstructured":"Snover M, Dorr B, Schwartz R, Micciulla L, Makhoul J (2006) A study of translation edit rate with targeted human annotation. In: Proceedings of association for machine translation in the Americas, Cambridge, Massachusetts, USA, pp 223\u2013231"},{"key":"9214_CR27","first-page":"237","volume-title":"Current Issues in Linguistic Theory","author":"J\u00f6rg Tiedemann","year":"2009","unstructured":"Tiedemann J (2009) News from OPUS\u2014a collection of multilingual parallel corpora with tools and interfaces. In: Recent advances in natural language processing. Borovets, Bulgaria, pp 237\u2013248"},{"key":"9214_CR28","unstructured":"Tiedemann J (2012) Parallel data, tools and interfaces in OPUS. In: Proceedings of the eighth international conference on language resources and evaluation, Istanbul, Turkey, pp 2214\u20132218"},{"key":"9214_CR29","doi-asserted-by":"crossref","unstructured":"Toral A, S\u00e1nchez-Cartagena VM (2017) A multifaceted evaluation of neural versus phrase-based machine translation for 9 language directions. In: Proceedings of the 15th Conference of the European chapter of the association for computational linguistics, Valencia, Spain, pp 1063\u20131073","DOI":"10.18653\/v1\/E17-1100"},{"key":"9214_CR30","unstructured":"Zeiler MD (2012) ADADELTA: an adaptive learning rate method. arXiv preprint arXiv:1212.5701"}],"container-title":["Machine Translation"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/link.springer.com\/article\/10.1007\/s10590-018-9214-x\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s10590-018-9214-x.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s10590-018-9214-x.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2019,10,10]],"date-time":"2019-10-10T15:36:46Z","timestamp":1570721806000},"score":1,"resource":{"primary":{"URL":"http:\/\/link.springer.com\/10.1007\/s10590-018-9214-x"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2018,2,10]]},"references-count":30,"journal-issue":{"issue":"3","published-print":{"date-parts":[[2018,9]]}},"alternative-id":["9214"],"URL":"https:\/\/doi.org\/10.1007\/s10590-018-9214-x","relation":{},"ISSN":["0922-6567","1573-0573"],"issn-type":[{"value":"0922-6567","type":"print"},{"value":"1573-0573","type":"electronic"}],"subject":[],"published":{"date-parts":[[2018,2,10]]},"assertion":[{"value":"24 August 2017","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"27 January 2018","order":2,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"10 February 2018","order":3,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}}]}}