{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,5]],"date-time":"2026-10-05T15:35:12Z","timestamp":1791214512565,"version":"4.1.0"},"reference-count":47,"publisher":"Springer Science and Business Media LLC","issue":"3-4","license":[{"start":{"date-parts":[[2013,8,30]],"date-time":"2013-08-30T00:00:00Z","timestamp":1377820800000},"content-version":"tdm","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["Machine Translation"],"published-print":{"date-parts":[[2013,12]]},"DOI":"10.1007\/s10590-013-9141-9","type":"journal-article","created":{"date-parts":[[2013,8,29]],"date-time":"2013-08-29T12:44:04Z","timestamp":1377780244000},"page":"213-238","source":"Crossref","is-referenced-by-count":7,"title":["Quality estimation for machine translation: some lessons learned"],"prefix":"10.1007","volume":"27","author":[{"given":"Guillaume","family":"Wisniewski","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Anil Kumar","family":"Singh","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Fran\u00e7ois","family":"Yvon","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2013,8,30]]},"reference":[{"key":"9141_CR1","unstructured":"Albrecht J, Hwa R (2007) A re-examination of machine learning approaches for sentence-level mt evaluation. In: Proceedings of the 45th annual meeting of the association of computational linguistics, Association for Computational Linguistics, Prague, Czech Republic, pp 880\u2013887"},{"issue":"4","key":"9141_CR2","doi-asserted-by":"crossref","first-page":"555","DOI":"10.1162\/coli.07-034-R2","volume":"34","author":"R Artstein","year":"2008","unstructured":"Artstein R, Poesio M (2008) Inter-coder agreement for computational linguistics. Comput Linguist 34(4):555\u2013596. doi: 10.1162\/coli.07-034-R2","journal-title":"Comput Linguist"},{"key":"9141_CR3","unstructured":"Bach N, Huang F, Al-Onaizan Y (2011) Goodness: a method for measuring machine translation confidence. In: Proceedings of the 49th annual meeting of the association for computational linguistics: human language technologies, Association for Computational Linguistics, Portland, Oregon, USA, pp 211\u2013219"},{"key":"9141_CR4","doi-asserted-by":"crossref","first-page":"3","DOI":"10.2466\/pr0.1966.19.1.3","volume":"19","author":"J Bartko","year":"1966","unstructured":"Bartko J (1966) The intraclass correlation coefficient as a measure of reliability. Psychol Rep 19:3\u201311","journal-title":"Psychol Rep"},{"key":"9141_CR5","doi-asserted-by":"crossref","unstructured":"Blatz J, Fitzgerald E, Foster G, Gandrabur S, Goutte C, Kulesza A, Sanchis A, Ueffing N (2004) Confidence estimation for machine translation. In: Proceedings of coling, (2004) COLING. Geneva, Switzerland, pp 315\u2013321","DOI":"10.3115\/1220355.1220401"},{"issue":"1","key":"9141_CR6","doi-asserted-by":"crossref","first-page":"5","DOI":"10.1023\/A:1010933404324","volume":"45","author":"L Breiman","year":"2001","unstructured":"Breiman L (2001) Random forests. Mach Learn 45(1):5\u201332","journal-title":"Mach Learn"},{"issue":"2","key":"9141_CR7","first-page":"79","volume":"16","author":"PF Brown","year":"1990","unstructured":"Brown PF, Cocke J, Pietra SD, Pietra VJD, Jelinek F, Lafferty JD, Mercer RL, Roossin PS (1990) A statistical approach to machine translation. Comput Linguist 16(2):79\u201385","journal-title":"Comput Linguist"},{"issue":"4","key":"9141_CR8","doi-asserted-by":"crossref","first-page":"877","DOI":"10.1093\/biomet\/82.4.877","volume":"82","author":"P Burman","year":"1995","unstructured":"Burman P, Nolan D (1995) A general akaike-type criterion for model selection in robust regression. Biometrika 82(4):877\u2013886","journal-title":"Biometrika"},{"key":"9141_CR9","doi-asserted-by":"crossref","unstructured":"Burstein J, Kukich K, Wolff S, Lu C, Chodorow M, Braden-Harder L, Harris MD (1998) Automated scoring using a hybrid feature identification technique. In: Proceedings of the 36th annual meeting of the Association for Computational Linguistics and 17th international conference on computational linguistics, vol 1. Association for Computational Linguistics, Montreal, Quebec, Canada, pp 206\u2013210. doi: 10.3115\/980845.980879","DOI":"10.3115\/980845.980879"},{"key":"9141_CR10","unstructured":"Callison-Burch C, Koehn P, Monz C, Post M, Soricut R, Specia L (2012) Findings of the 2012 workshop on statistical machine translation. In: Proceedings of the seventh workshop on statistical machine translation, Association for Computational Linguistics, Montr\u00e9al, Canada, pp 10\u201351"},{"key":"9141_CR11","unstructured":"Drucker H, Burges CJC, Kaufman L, Smola AJ, Vapnik V (1996) Support vector regression machines. In: Mozer M, Jordan MI, Petsche T (eds) NIPS. MIT Press, Cambridge, MA, pp 155\u2013161"},{"key":"9141_CR12","unstructured":"Efron B, Tibshirani R (1993) An introduction to the Bootstrap. Chapman and Hall\/CRC monographs on statistics and applied probability series, Chapman & Hall, New York"},{"key":"9141_CR13","unstructured":"Felice M, Specia L (2012) Linguistic features for quality estimation. In: Proceedings of the seventh workshop on statistical machine translation, Association for Computational Linguistics, Montr\u00e9al, Canada, pp 96\u2013103"},{"issue":"5","key":"9141_CR14","doi-asserted-by":"crossref","first-page":"378","DOI":"10.1037\/h0031619","volume":"76","author":"JL Fleiss","year":"1971","unstructured":"Fleiss JL (1971) Measuring nominal scale agreement among many raters. Psychol Bull 76(5):378\u2013382","journal-title":"Psychol Bull"},{"key":"9141_CR15","doi-asserted-by":"crossref","unstructured":"de Gispert A, Blackwood G, Iglesias G, Byrne W (2012) N-gram posterior probability confidence measures for statistical machine translation: an empirical study. Mach Transl 27(2): 85\u2013114","DOI":"10.1007\/s10590-012-9132-2"},{"key":"9141_CR16","volume-title":"Covariate shift and local learning by distribution matching","author":"A Gretton","year":"2009","unstructured":"Gretton A, Smola A, Huang J, Schmittfull M, Borgwardt K, Sch\u00f6lkopf B (2009) Covariate shift and local learning by distribution matching. MIT Press, Cambridge, MA"},{"key":"9141_CR17","first-page":"1157","volume":"3","author":"I Guyon","year":"2003","unstructured":"Guyon I, Elisseeff A (2003) An introduction to variable and feature selection. J Mach Learn Res 3:1157\u20131182","journal-title":"J Mach Learn Res"},{"key":"9141_CR18","doi-asserted-by":"crossref","first-page":"10","DOI":"10.1145\/1656274.1656278","volume":"11","author":"M Hall","year":"2009","unstructured":"Hall M, Frank E, Holmes G, Pfahringer B, Reutemann P, Witten IH (2009) The WEKA data mining software: an update. SIGKDD Explor 11:10\u201318","journal-title":"SIGKDD Explor"},{"key":"9141_CR19","volume-title":"The elements of statistical learning","author":"T Hastie","year":"2003","unstructured":"Hastie T, Tibshirani R, Friedman JH (2003) The elements of statistical learning. Springer, New York"},{"key":"9141_CR20","doi-asserted-by":"crossref","unstructured":"Kanungo T, Orr D (2009) Predicting the readability of short web summaries. In: Proceedings of the second ACM international conference on web search and data mining, ACM, New York, NY, USA, pp 202\u2013211","DOI":"10.1145\/1498759.1498827"},{"key":"9141_CR21","volume-title":"Statistical machine translation","author":"P Koehn","year":"2010","unstructured":"Koehn P (2010) Statistical machine translation. Cambridge University Press, Cambridge"},{"key":"9141_CR22","doi-asserted-by":"crossref","unstructured":"Koehn P, Monz C (2006) Manual and automatic evaluation of machine translation between european languages. In: Proceedings on the workshop on statistical machine translation, Association for Computational Linguistics, New York, pp 102\u2013121","DOI":"10.3115\/1654650.1654666"},{"key":"9141_CR23","doi-asserted-by":"crossref","unstructured":"Koehn P, Hoang H, Birch A, Callison-Burch C, Federico M, Bertoldi N, Cowan B, Shen W, Moran C, Zens R, Dyer C, Bojar O, Constantin A, Herbst E (2007) Moses: open source toolkit for statistical machine translation. In: Proceedings of the 45th annual meeting of the Association for Computational Linguistics Companion Volume Proceedings of the Demo and Poster Sessions, Association for Computational Linguistics, Prague, Czech Republic, pp 177\u2013180","DOI":"10.3115\/1557769.1557821"},{"key":"9141_CR24","unstructured":"Koponen M (2012) Comparing human perceptions of post-editing effort with post-editing operations. In: Proceedings of the seventh workshop on statistical machine translation, Association for Computational Linguistics, Montr\u00e9al, Canada, pp 181\u2013190"},{"key":"9141_CR25","unstructured":"Le HS, Oparin I, Allauzen A, Gauvain JL, Yvon F (2011) Structured output layer neural network language model. In: Proceedings of IEEE international conference on acoustic, speech and signal processing, Prague, Czech Republic, pp 5524\u20135527"},{"key":"9141_CR26","unstructured":"Le HS, Lavergne T, Allauzen A, Apidianaki M, Gong L, Max A, Sokolov A, Wisniewski G, Yvon F (2012) Limsi @ wmt12. In: Proceedings of the seventh workshop on statistical machine translation, Association for Computational Linguistics, Montr\u00e9al, Canada, pp 330\u2013337"},{"issue":"9","key":"9141_CR27","doi-asserted-by":"crossref","first-page":"917","DOI":"10.1109\/TC.1977.1674939","volume":"26","author":"PM Narendra","year":"1977","unstructured":"Narendra PM, Fukunaga K (1977) A branch and bound algorithm for feature subset selection. IEEE Trans Comput 26(9):917\u2013922","journal-title":"IEEE Trans Comput"},{"key":"9141_CR28","first-page":"2825","volume":"12","author":"F Pedregosa","year":"2011","unstructured":"Pedregosa F, Varoquaux G, Gramfort A, Michel V, Thirion B, Grisel O, Blondel M, Prettenhofer P, Weiss R, Dubourg V, Vanderplas J, Passos A, Cournapeau D, Brucher M, Perrot M, Duchesnay E (2011) Scikit-learn: machine learning in Python. J Mach Lear Res 12:2825\u20132830","journal-title":"J Mach Lear Res"},{"key":"9141_CR29","unstructured":"Popovi\u0107 M (2011) Morphemes and pos tags for n-gram based evaluation metrics. In: Proceedings of the sixth workshop on statistical machine translation, Association for Computational Linguistics, Edinburgh, Scotland, pp 104\u2013107"},{"key":"9141_CR30","unstructured":"Popovic M (2012) Morpheme- and pos-based ibm1 and language model scores for translation quality estimation. In: Proceedings of the seventh workshop on statistical machine translation, Association for Computational Linguistics, Montr\u00e9al, Canada, pp 133\u2013137"},{"key":"9141_CR31","unstructured":"Popovi\u0107 M, Vilar D, Avramidis E, Burchardt A (2011) Evaluation without references: Ibm1 scores as evaluation metrics. In: Proceedings of the sixth workshop on statistical machine translation, Association for Computational Linguistics, Edinburgh, Scotland, pp 99\u2013103"},{"key":"9141_CR32","unstructured":"Quinlan RJ (1992) Learning with continuous classes. In: 5th Australian joint conference on artificial intelligence, pp 343\u2013348"},{"key":"9141_CR33","volume-title":"Dataset shift in machine learning","author":"J Quionero-Candela","year":"2009","unstructured":"Quionero-Candela J, Sugiyama M, Schwaighofer A, Lawrence ND (2009) Dataset shift in machine learning. MIT Press, Cambridge"},{"key":"9141_CR34","unstructured":"Quirk C (2004) Training a sentence-level machine translation confidence metric. In: Proceedings of the 4th international conference on language resources and evaluation, pp 825\u2013828"},{"key":"9141_CR35","unstructured":"Rifkin RM, Lippert RA (2007) Notes on regularized least squares. Tech Rep MIT-CSAIL-TR-2007-025, MIT-CSAIL"},{"key":"9141_CR36","unstructured":"Rubino R, Foster J, Wagner J, Roturier J, Samad Zadeh Kaljahi R, Hollowood F (2012) Dcu-symantec submission for the wmt 2012 quality estimation task. In: Proceedings of the seventh workshop on statistical machine translation, Association for Computational Linguistics, Montr\u00e9al, Canada, pp 138\u2013144"},{"key":"9141_CR37","unstructured":"Schmid H (1995) Improvements in part-of-speech tagging with an application to german. In: Proceedings of the ACL SIGDAT-Workshop, ACL, Dublin, Ireland"},{"key":"9141_CR38","doi-asserted-by":"crossref","first-page":"420","DOI":"10.1037\/0033-2909.86.2.420","volume":"86","author":"P Shrout","year":"1979","unstructured":"Shrout P, Fleiss J (1979) Intraclass correlation: uses in assessing rater reliability. Psychol Bull 86:420\u2013428","journal-title":"Psychol Bull"},{"key":"9141_CR39","unstructured":"Snover M, Dorr B, Schwartz R, Micciulla L, Makhoul J (2006) A study of translation edit rate with targeted human annotation. In: Proceedings of AMTA, pp 223\u2013231"},{"key":"9141_CR40","doi-asserted-by":"crossref","DOI":"10.1075\/btl.35","volume-title":"Computers and translation: a translator\u2019s guide","author":"H Somers","year":"2003","unstructured":"Somers H (2003) Computers and translation: a translator\u2019s guide. John Benjamins Publishing Company, Amsterdam"},{"key":"9141_CR41","unstructured":"Soricut R, Echihabi A (2010) Trustrank: inducing trust in automatic translations via ranking. In: Proceedings of the 48th annual meeting of the Association for Computational Linguistics, Uppsala, Sweden, pp 612\u2013621"},{"key":"9141_CR42","unstructured":"Soricut R, Bach N, Wang Z (2012) The sdl language weaver systems in the wmt12 quality estimation shared task. In: Proceedings of the seventh workshop on statistical machine translation, Association for Computational Linguistics, Montr\u00e9al, Canada, pp 145\u2013151"},{"key":"9141_CR43","unstructured":"Specia L (2011) Exploiting objective annotations for measuring translation post-editing effort. In: Proceedings of the 15th conference of EAMT, Leuven, Belgium, pp 73\u201380"},{"issue":"1","key":"9141_CR44","doi-asserted-by":"crossref","first-page":"39","DOI":"10.1007\/s10590-010-9077-2","volume":"24","author":"L Specia","year":"2010","unstructured":"Specia L, Raj D, Turchi M (2010) Machine translation evaluation versus quality estimation. Mach Transl 24(1):39\u201350","journal-title":"Mach Transl"},{"key":"9141_CR45","unstructured":"Wang Y, Witten IH (1997) Induction of model trees for predicting continuous classes. In: 9th European conference on machine learning, Springer, pp 128\u2013137"},{"key":"9141_CR46","unstructured":"Xiong D, Zhang M, Li H (2010) Error detection for statistical machine translation using linguistic features. In: Proceedings of the 48th annual meeting of the Association for Computational Linguistics, Uppsala, Sweden, pp 604\u2013611"},{"key":"9141_CR47","unstructured":"Zhuang Y, Wisniewski G, Yvon F (2012) Non-linear models for confidence estimation. In: Proceedings of the seventh workshop on statistical machine translation, Association for Computational Linguistics, Montr\u00e9al, Canada, pp 157\u2013162"}],"container-title":["Machine Translation"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s10590-013-9141-9.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/article\/10.1007\/s10590-013-9141-9\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s10590-013-9141-9","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2019,7,22]],"date-time":"2019-07-22T15:46:41Z","timestamp":1563810401000},"score":1,"resource":{"primary":{"URL":"http:\/\/link.springer.com\/10.1007\/s10590-013-9141-9"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2013,8,30]]},"references-count":47,"journal-issue":{"issue":"3-4","published-print":{"date-parts":[[2013,12]]}},"alternative-id":["9141"],"URL":"https:\/\/doi.org\/10.1007\/s10590-013-9141-9","relation":{},"ISSN":["0922-6567","1573-0573"],"issn-type":[{"value":"0922-6567","type":"print"},{"value":"1573-0573","type":"electronic"}],"subject":[],"published":{"date-parts":[[2013,8,30]]}}}