{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,24]],"date-time":"2026-04-24T04:05:20Z","timestamp":1777003520523,"version":"3.51.4"},"reference-count":31,"publisher":"Springer Science and Business Media LLC","issue":"2","license":[{"start":{"date-parts":[[2013,10,4]],"date-time":"2013-10-04T00:00:00Z","timestamp":1380844800000},"content-version":"tdm","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["Mach Learn"],"published-print":{"date-parts":[[2014,5]]},"DOI":"10.1007\/s10994-013-5411-2","type":"journal-article","created":{"date-parts":[[2013,10,3]],"date-time":"2013-10-03T15:42:03Z","timestamp":1380814923000},"page":"165-181","source":"Crossref","is-referenced-by-count":51,"title":["Sequence labeling with multiple annotators"],"prefix":"10.1007","volume":"95","author":[{"given":"Filipe","family":"Rodrigues","sequence":"first","affiliation":[]},{"given":"Francisco","family":"Pereira","sequence":"additional","affiliation":[]},{"given":"Bernardete","family":"Ribeiro","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2013,10,4]]},"reference":[{"issue":"18","key":"5411_CR1","doi-asserted-by":"crossref","first-page":"3596","DOI":"10.1093\/bioinformatics\/bti609","volume":"21","author":"J. Allen","year":"2005","unstructured":"Allen, J., & Salzberg, S. (2005). JIGSAW: integration of multiple sources of evidence for gene prediction. Bioinformatics, 21(18), 3596\u20133603.","journal-title":"Bioinformatics"},{"issue":"1","key":"5411_CR2","doi-asserted-by":"crossref","first-page":"142","DOI":"10.1101\/gr.1562804","volume":"14","author":"J. Allen","year":"2004","unstructured":"Allen, J., Pertea, M., & Salzberg, S. (2004). Computational gene prediction using multiple sources of evidence. Genome Research, 14(1), 142\u2013148.","journal-title":"Genome Research"},{"key":"5411_CR3","volume-title":"Sixth international workshop on information integration on the web","author":"K. Bellare","year":"2007","unstructured":"Bellare, K., & Mccallum, A. (2007). Learning extractors from unlabeled text using relevant databases. In Sixth international workshop on information integration on the web."},{"key":"5411_CR4","first-page":"1","volume-title":"Proceedings of the NAACL HLT 2010 workshop on creating speech and language data with amazon\u2019s mechanical turk","author":"C. Callison-Burch","year":"2010","unstructured":"Callison-Burch, C., & Dredze, M. (2010). Creating speech and language data with amazon\u2019s mechanical turk. In Proceedings of the NAACL HLT 2010 workshop on creating speech and language data with amazon\u2019s mechanical turk (pp. 1\u201312)."},{"issue":"1","key":"5411_CR5","first-page":"20","volume":"28","author":"A. P. Dawid","year":"1979","unstructured":"Dawid, A. P., & Skene, A. M. (1979). Maximum likelihood estimation of observer error-rates using the EM algorithm. Journal of the Royal Statistical Society. Series C. Applied Statistics, 28(1), 20\u201328.","journal-title":"Journal of the Royal Statistical Society. Series C. Applied Statistics"},{"issue":"1","key":"5411_CR6","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1111\/j.2517-6161.1977.tb01600.x","volume":"39","author":"A. P. Dempster","year":"1977","unstructured":"Dempster, A. P., Laird, N. M., & Rubin, D. B. (1977). Maximum likelihood from incomplete data via the em algorithm. Journal of the Royal Statistical Society, Series B, 39(1), 1\u201338.","journal-title":"Journal of the Royal Statistical Society, Series B"},{"key":"5411_CR7","doi-asserted-by":"crossref","first-page":"619","DOI":"10.1145\/1458082.1458165","volume-title":"Proceedings of the 17th ACM conference on information and knowledge management","author":"P. Donmez","year":"2008","unstructured":"Donmez, P., & Carbonell, J. (2008). Proactive learning: cost-sensitive active learning with multiple imperfect oracles. In Proceedings of the 17th ACM conference on information and knowledge management (pp. 619\u2013628)."},{"key":"5411_CR8","first-page":"826","volume-title":"Proceedings of the SIAM international conference on data mining","author":"P. Donmez","year":"2010","unstructured":"Donmez, P., Schneider, J., & Carbonell, J. (2010). A probabilistic framework to learn from multiple annotators with time-varying accuracy. In Proceedings of the SIAM international conference on data mining (pp. 826\u2013837)."},{"key":"5411_CR9","volume-title":"ECML-PKDD 2009 workshop on learning from multi-label data","author":"M. Dredze","year":"2009","unstructured":"Dredze, M., Talukdar, P., & Crammer, K. (2009). Sequence learning from data with multiple labels. In ECML-PKDD 2009 workshop on learning from multi-label data."},{"key":"5411_CR10","first-page":"407","volume-title":"Proceedings of the 2011 European conference on machine learning and knowledge discovery in databases","author":"E. Fernandes","year":"2011","unstructured":"Fernandes, E., & Brefeld, U. (2011). Learning from partially annotated sequences. In Proceedings of the 2011 European conference on machine learning and knowledge discovery in databases (pp. 407\u2013422)."},{"key":"5411_CR11","first-page":"159","volume-title":"Proceedings of the 21st international conference on artificial neural networks","author":"P. Groot","year":"2011","unstructured":"Groot, P., Birlutiu, A., & Heskes, T. (2011). Learning from multiple annotators with Gaussian processes. In Proceedings of the 21st international conference on artificial neural networks (Vol.\u00a06792, pp. 159\u2013164)."},{"key":"5411_CR12","volume-title":"Crowdsourcing: why the power of the crowd is driving the future of business","author":"J. Howe","year":"2008","unstructured":"Howe, J. (2008). Crowdsourcing: why the power of the crowd is driving the future of business (1st edn.). New York: Crown Publishing Group.","edition":"1"},{"key":"5411_CR13","first-page":"282","volume-title":"Proceedings of the 18th international conference on machine learning","author":"J. Lafferty","year":"2001","unstructured":"Lafferty, J., McCallum, A., & Pereira, F. (2001). Conditional random fields: probabilistic models for segmenting and labeling sequence data. In Proceedings of the 18th international conference on machine learning (pp. 282\u2013289)."},{"key":"5411_CR14","first-page":"1546","volume-title":"Proceedings of the conference on empirical methods in natural language processing","author":"F. Laws","year":"2011","unstructured":"Laws, F., Scheible, C., & Sch\u00fctze, M. (2011). Active learning with amazon mechanical turk. In Proceedings of the conference on empirical methods in natural language processing. Stroudsburg: Association for Computational Linguistics (pp. 1546\u20131556)."},{"key":"5411_CR15","first-page":"71","volume-title":"Proceedings of the NAACL HLT 2010 workshop on creating speech and language data with amazon\u2019s mechanical turk","author":"N. Lawson","year":"2010","unstructured":"Lawson, N., Eustice, K., Perkowitz, M., & Yetisgen-Yildiz, M. (2010). Annotating large email datasets for named entity recognition with mechanical turk. In Proceedings of the NAACL HLT 2010 workshop on creating speech and language data with amazon\u2019s mechanical turk. Stroudsburg: Association for Computational Linguistics (pp. 71\u201379)."},{"key":"5411_CR16","doi-asserted-by":"crossref","first-page":"503","DOI":"10.1007\/BF01589116","volume":"45","author":"D. Liu","year":"1989","unstructured":"Liu, D., & Nocedal, J. (1989). On the limited memory BFGS method for large scale optimization. Mathematical Programming, 45, 503\u2013528.","journal-title":"Mathematical Programming"},{"key":"5411_CR17","first-page":"207","volume-title":"Human language technologies, HLT\u00a0\u201910","author":"S. Novotney","year":"2010","unstructured":"Novotney, S., & Callison-Burch, C. (2010). Cheap, fast and good enough: automatic speech recognition with non-expert transcription. In Human language technologies, HLT\u00a0\u201910. Stroudsburg: Association for Computational Linguistics (pp. 207\u2013215)."},{"key":"5411_CR18","first-page":"257","volume-title":"Proceedings of the IEEE","author":"L. R. Rabiner","year":"1989","unstructured":"Rabiner, L. R. (1989). A tutorial on hidden Markov models and selected applications in speech recognition. In Proceedings of the IEEE (pp. 257\u2013286)."},{"key":"5411_CR19","first-page":"82","volume-title":"Proceedings of the third workshop on very large corpora","author":"L. Ramshaw","year":"1995","unstructured":"Ramshaw, L., & Marcus, M. (1995). Text chunking using transformation-based learning. In Proceedings of the third workshop on very large corpora. Stroudsburg: Association for Computational Linguistics (pp. 82\u201394)."},{"key":"5411_CR20","first-page":"889","volume-title":"Proceedings of the 26th international conference on machine learning","author":"V. Raykar","year":"2009","unstructured":"Raykar, V., Yu, S., Zhao, L., Jerebko, A., Florin, C., Valadez, G., Bogoni, L., & Moy, L. (2009). Supervised learning from multiple experts: whom to trust when everyone lies a bit. In Proceedings of the 26th international conference on machine learning (pp. 889\u2013896)."},{"key":"5411_CR21","unstructured":"Raykar, V., Yu, S., Zhao, L., Valadez, G., Florin, C., Bogoni, L., & Moy, L. (2010). Learning from crowds. Journal of Machine Learning Research, 1297\u20131322."},{"key":"5411_CR22","doi-asserted-by":"crossref","first-page":"142","DOI":"10.3115\/1119176.1119195","volume-title":"Proceedings of the 7th conference on natural language learning at HLT-NAACL","author":"E. Sang","year":"2003","unstructured":"Sang, E., & Meulder, F. D. (2003). Introduction to the conll-2003 shared task: language-independent named entity recognition. In Proceedings of the 7th conference on natural language learning at HLT-NAACL (Vol.\u00a04, pp. 142\u2013147)."},{"key":"5411_CR23","doi-asserted-by":"crossref","first-page":"614","DOI":"10.1145\/1401890.1401965","volume-title":"Proceedings of the 14th ACM SIGKDD international conference on knowledge discovery and data mining","author":"V. Sheng","year":"2008","unstructured":"Sheng, V., Provost, F., & Ipeirotis, P. (2008). Get another label? Improving data quality and data mining using multiple, noisy labelers. In Proceedings of the 14th ACM SIGKDD international conference on knowledge discovery and data mining (pp. 614\u2013622)."},{"key":"5411_CR24","unstructured":"Smyth, P., Fayyad, U., Burl, M., Perona, P., & Baldi, P. (1995). Inferring ground truth from subjective labelling of venus images. Advances in Neural Information Processing Systems, 1085\u20131092."},{"key":"5411_CR25","doi-asserted-by":"crossref","first-page":"254","DOI":"10.3115\/1613715.1613751","volume-title":"Proceedings of the conference on empirical methods in natural language processing","author":"R. Snow","year":"2008","unstructured":"Snow, R., O\u2019Connor, B., Jurafsky, D., & Ng, A. (2008). Cheap and fast\u2014but is it good?: evaluating non-expert annotations for natural language tasks. In Proceedings of the conference on empirical methods in natural language processing (pp. 254\u2013263)."},{"key":"5411_CR26","volume-title":"The wisdom of crowds: why the many are smarter than the few and how collective wisdom shapes business, economies, societies, and nations","author":"J. Surowiecki","year":"2004","unstructured":"Surowiecki, J. (2004). The wisdom of crowds: why the many are smarter than the few and how collective wisdom shapes business, economies, societies, and nations. New York: Doubleday."},{"key":"5411_CR27","volume-title":"Introduction to conditional random fields for relational learning","author":"C. Sutton","year":"2006","unstructured":"Sutton, C., & McCallum, A. (2006). Introduction to conditional random fields for relational learning. Cambridge: MIT Press."},{"key":"5411_CR28","first-page":"243","volume-title":"Proceedings of the fourth linguistic annotation workshop","author":"R. Voyer","year":"2010","unstructured":"Voyer, R., Nygaard, V., Fitzgerald, W., & Copperman, H. (2010). A hybrid model for annotating named entity training corpora. In Proceedings of the fourth linguistic annotation workshop. Stroudsburg: Association for Computational Linguistics (pp. 243\u2013246)."},{"key":"5411_CR29","first-page":"1571","volume-title":"Proceedings of the 22nd international joint conference on artificial intelligence","author":"O. Wu","year":"2011","unstructured":"Wu, O., Hu, W., & Gao, J. (2011). Learning to rank under multiple annotators. In Proceedings of the 22nd international joint conference on artificial intelligence (pp. 1571\u20131576)."},{"key":"5411_CR30","first-page":"932","volume":"9","author":"Y. Yan","year":"2010","unstructured":"Yan, Y., Rosales, R., Fung, G., Schmidt, M., Valadez, G., Bogoni, L., Moy, L., & Dy, J. (2010). Modeling annotator expertise: learning when everybody knows a bit of something. Journal of Machine Learning Research, 9, 932\u2013939.","journal-title":"Journal of Machine Learning Research"},{"key":"5411_CR31","first-page":"1161","volume-title":"Proceedings of the 28th international conference on machine learning","author":"Y. Yan","year":"2011","unstructured":"Yan, Y., Rosales, R., Fung, G., & Dy, J. (2011). Active learning from crowds. In Proceedings of the 28th international conference on machine learning (pp. 1161\u20131168)."}],"container-title":["Machine Learning"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s10994-013-5411-2.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/article\/10.1007\/s10994-013-5411-2\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s10994-013-5411-2","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,5,18]],"date-time":"2024-05-18T11:39:18Z","timestamp":1716032358000},"score":1,"resource":{"primary":{"URL":"http:\/\/link.springer.com\/10.1007\/s10994-013-5411-2"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2013,10,4]]},"references-count":31,"journal-issue":{"issue":"2","published-print":{"date-parts":[[2014,5]]}},"alternative-id":["5411"],"URL":"https:\/\/doi.org\/10.1007\/s10994-013-5411-2","relation":{},"ISSN":["0885-6125","1573-0565"],"issn-type":[{"value":"0885-6125","type":"print"},{"value":"1573-0565","type":"electronic"}],"subject":[],"published":{"date-parts":[[2013,10,4]]}}}