{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,3,12]],"date-time":"2026-03-12T23:48:09Z","timestamp":1773359289185,"version":"3.50.1"},"publisher-location":"Berlin, Heidelberg","reference-count":29,"publisher":"Springer Berlin Heidelberg","isbn-type":[{"value":"9783642164378","type":"print"},{"value":"9783642164385","type":"electronic"}],"license":[{"start":{"date-parts":[[2010,1,1]],"date-time":"2010-01-01T00:00:00Z","timestamp":1262304000000},"content-version":"unspecified","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2010]]},"DOI":"10.1007\/978-3-642-16438-5_21","type":"book-chapter","created":{"date-parts":[[2010,11,13]],"date-time":"2010-11-13T09:16:46Z","timestamp":1289639806000},"page":"301-315","source":"Crossref","is-referenced-by-count":7,"title":["A Methodology towards Effective and Efficient Manual Document Annotation: Addressing Annotator Discrepancy and Annotation Quality"],"prefix":"10.1007","author":[{"given":"Ziqi","family":"Zhang","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Sam","family":"Chapman","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Fabio","family":"Ciravegna","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","reference":[{"key":"21_CR1","doi-asserted-by":"crossref","unstructured":"Bermingham, A., Smeaton, A.: A Study of Inter-Annotator Agreement for Opinion Retrieval. In: Proceedings of SIGIR 2009 (2009)","DOI":"10.1145\/1571941.1572127"},{"key":"21_CR2","unstructured":"Brants, T.: Inter-annotator agreement for a German newspaper corpus. In: Proceedings of the Second International Conference on Language Resources and Evaluation, LREC (2000)"},{"key":"21_CR3","doi-asserted-by":"crossref","unstructured":"Byrne, K.: Nested Named Entity Recognition in Historical Archive Text. In: Proceedings of International Conference on Semantic Computing (2007)","DOI":"10.1109\/ICSC.2007.107"},{"issue":"2","key":"21_CR4","first-page":"249","volume":"22","author":"J. Carletta","year":"1996","unstructured":"Carletta, J.: Assessing agreement on classification tasks: the kappa statistic. Computational Linguistics\u00a022(2), 249\u2013254 (1996)","journal-title":"Computational Linguistics"},{"key":"21_CR5","doi-asserted-by":"crossref","unstructured":"Colosimo, M., Morgan, A., Yeh, A., Colombe, J., Hirschman, L.: Data preparation and internannotator agreement: BioCreAtIvE Task 1B. BMC Bioinformatics (2005)","DOI":"10.1186\/1471-2105-6-S1-S12"},{"key":"21_CR6","unstructured":"Ciravegna, F., Lavelli, A., Satta, G.: Bringing information extraction out of the labs: the Pinocchio Environment. In: Proceedings of the 14th European Conference on Artificial Intelligence (2000)"},{"key":"21_CR7","unstructured":"Cucchiarini, C., Strik, H.: Automatic transcription agreement: An overview, pp. 347\u2013350 (2003)"},{"key":"21_CR8","unstructured":"Ehrmann, M.: Les entites nommees, de la linguistique au TAL: statut theorique et methods de desambiguisation. Ph.D. thesis, Univ. Paris (2008)"},{"key":"21_CR9","unstructured":"Ferro, L., Mani, I., Sundheim, B., Wilson, G.: TIDES Temporal Annotation Guidelines. Draft Version 1.0. MITRE Technical Report MTR 00W0000094 (October 2000)"},{"key":"21_CR10","doi-asserted-by":"crossref","unstructured":"Fort, K., Ehrmann, M., Nazarenko, A.: Towards a methodology for named entities anntoation. In: Proceedings of the Third Linguistic Annotation Workshop, ACL-IJNLP, pp. 142\u2013145 (2009)","DOI":"10.3115\/1698381.1698406"},{"key":"21_CR11","doi-asserted-by":"crossref","unstructured":"Grishman, R., Sundheim, B.: Message understanding conference - 6: A brief history. In: Proceedings of International Conference on Computational Linguistics (1996)","DOI":"10.3115\/992628.992709"},{"key":"21_CR12","unstructured":"Gut, U., Bayerl, P.S.: Measuring the Reliability of Manual Annotations of Speech Corpora. In: Proceedings of Speech Prosody (2004), Nara, pp. 565\u2013568 (2004)"},{"key":"21_CR13","doi-asserted-by":"crossref","unstructured":"Hripcsak, G., Rothschild, A.: Agreement, the F-measure and Reliability in Information Retrieval. Journal of the American Medical Informatics Association, 296\u2013298 (2005)","DOI":"10.1197\/jamia.M1733"},{"key":"21_CR14","doi-asserted-by":"crossref","unstructured":"Hripcsak, G., Wilcox, A.: Reference standards, judges, and comparison subjects: roles for experts in evaluating system performance. J. Am. Med. Inform. Assoc., 1\u201315 (2002)","DOI":"10.1136\/jamia.2002.0090001"},{"key":"21_CR15","doi-asserted-by":"crossref","unstructured":"Iria, J.: Automating Knowledge Capture in the Aerospace Domain. In: Proceedings of the Fifth International Conference on Knowledge Capture, pp. 97\u2013104 (2009)","DOI":"10.1145\/1597735.1597753"},{"key":"21_CR16","doi-asserted-by":"crossref","unstructured":"Jeffrey, S., Richards, J., Ciravegna, F., Chapman, S., Zhang, Z.: The Archaeotools project: Faceted Classification and Natural Language Processing in an Archaeological Context. In: Special Theme Issues of the Philosophical Transactions of the Royal Society A, Crossing Boundaries: Computational Science, E-Science and Global E-Infrastructures (2009)","DOI":"10.1098\/rsta.2009.0038"},{"key":"21_CR17","doi-asserted-by":"crossref","unstructured":"Kim, J., Ohta, T., Tsujii, J.: Corpus annotations for mining biomedical events from literature. In: BMC Bioinformatics (2008)","DOI":"10.1186\/1471-2105-9-10"},{"key":"21_CR18","unstructured":"Linguistic Data Consortium, Automatic Content Extraction (ACE) (2008), http:\/\/projects.ldc.upenn.edu\/ace\/"},{"key":"21_CR19","doi-asserted-by":"crossref","unstructured":"Minkov, E., Wang, R., Cohen, W.: Extracting Personal Names from Email: Applying Named Entity Recognition to Informal Text. In: Proceedings of HLT\/EMNLP 2005 (2005)","DOI":"10.3115\/1220575.1220631"},{"key":"21_CR20","doi-asserted-by":"crossref","unstructured":"Morante, R., Asch, V., Daelemans, W.: A memory\u2013based learning approach to event extraction in biomedical texts. In: Proceedings of the Workshop on BioNLP: Shared Task, pp. 59\u201367 (2009)","DOI":"10.3115\/1572340.1572349"},{"key":"21_CR21","unstructured":"Murphy, T., McIntosh, T., Curran, J.: Named entity recognition for astronomy literature. In: Australian Language Technology Workshop (2006)"},{"key":"21_CR22","unstructured":"Nadeau, D.: PhD Thesis: Semi-Supervised Named Entity Recognition: Learning to Recognize 100 Entity Types with Little Supervision (2007)"},{"key":"21_CR23","unstructured":"Ng, H., Lim, C., Foo, S.: A Case Study on Inter-Annotator Agreement for Word Sense Disambiguation. In: Proceedings of the ACL SIGLEX Workshop on Standardizing Lexical Resources SIGLEX 1999, pp. 9\u201313 (1999)"},{"key":"21_CR24","doi-asserted-by":"crossref","unstructured":"Ohta, T., Tateisi, Y., Kim, J.: The GENIA corpus: an annotated research abstract corpus in molecular biology domain. In: Proceedings of the Second International Conference on Human Language Technology Research, pp. 82\u201386 (2002)","DOI":"10.3115\/1289189.1289260"},{"key":"21_CR25","unstructured":"Olsson, F.: PhD thesis: Bootstrapping Named Entity Annotation by Means of Active Machine Learning: A Method for Creating Corpora (2008)"},{"key":"21_CR26","doi-asserted-by":"crossref","unstructured":"Pyysalo, S., Ginter, F., Heimonen, J., Bj\u00f6rne, J., Boberg, J., J\u00e4rvinen, J., Salakoski, T.: BioInfer: a corpus for information extraction in the biomedical domain. BMC Bioinformatics (2007)","DOI":"10.1186\/1471-2105-8-50"},{"key":"21_CR27","unstructured":"Saracevic, T.: Individual differences in organizing, searching, and retrieving information. In: Proceedings of the 54th Annual ASIS Meeting, pp. 82\u201386 (1991)"},{"key":"21_CR28","doi-asserted-by":"crossref","unstructured":"Tanabe, L., Xie, N., Thom, L., Matten, W., Wilbur, W.: GENETAG: a tagged corpus for gene\/protein named entity recognition. BMC Bioinformatics (2005)","DOI":"10.1186\/1471-2105-6-S1-S3"},{"key":"21_CR29","doi-asserted-by":"crossref","unstructured":"Wilbur, W., Rzhetsky, A., Shatkay, H.: New directions in biomedical text annotation: definitions, guidelines and corpus construction. Bioinformatics (2006)","DOI":"10.1186\/1471-2105-7-356"}],"container-title":["Lecture Notes in Computer Science","Knowledge Engineering and Management by the Masses"],"original-title":[],"link":[{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-642-16438-5_21","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2019,6,6]],"date-time":"2019-06-06T04:19:42Z","timestamp":1559794782000},"score":1,"resource":{"primary":{"URL":"http:\/\/link.springer.com\/10.1007\/978-3-642-16438-5_21"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2010]]},"ISBN":["9783642164378","9783642164385"],"references-count":29,"URL":"https:\/\/doi.org\/10.1007\/978-3-642-16438-5_21","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"value":"0302-9743","type":"print"},{"value":"1611-3349","type":"electronic"}],"subject":[],"published":{"date-parts":[[2010]]}}}