{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2022,4,5]],"date-time":"2022-04-05T15:49:17Z","timestamp":1649173757770},"reference-count":56,"publisher":"Springer Science and Business Media LLC","issue":"4","license":[{"start":{"date-parts":[[2013,11,15]],"date-time":"2013-11-15T00:00:00Z","timestamp":1384473600000},"content-version":"tdm","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["Lang Resources & Evaluation"],"published-print":{"date-parts":[[2014,12]]},"DOI":"10.1007\/s10579-013-9258-8","type":"journal-article","created":{"date-parts":[[2013,11,14]],"date-time":"2013-11-14T02:42:52Z","timestamp":1384396972000},"page":"561-599","source":"Crossref","is-referenced-by-count":0,"title":["Evaluating machine-assisted annotation in under-resourced settings"],"prefix":"10.1007","volume":"48","author":[{"given":"Paul","family":"Felt","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Eric K.","family":"Ringger","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Kevin","family":"Seppi","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Kristian S.","family":"Heal","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Robbie A.","family":"Haertel","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Deryle","family":"Lonsdale","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2013,11,15]]},"reference":[{"key":"9258_CR1","unstructured":"Alex, B., Grover, C., Haddow, B., Kabadjov, M., Klein, E., Matthews, M., et al. (2008). Assisted curation: Does text mining really help? In Pacific symposium on biocomputing, Citeseer, Vol. 5, pp. 56\u201367."},{"key":"9258_CR2","unstructured":"Baldridge, J., & Osborne, M. (2004). Active learning and the total cost of annotation. In Proceedings of the 42nd annual meeting of the association for computational linguistics, pp. 9\u201316."},{"issue":"2","key":"9258_CR3","first-page":"237","volume":"25","author":"S. Bangalore","year":"1999","unstructured":"Bangalore, S., & Joshi, A. (1999). Supertagging: An approach to almost parsing. Computational Linguistics, 25(2), 237\u2013265.","journal-title":"Computational Linguistics"},{"key":"9258_CR4","unstructured":"Barque, L., Nasr, A., & Polgu\u00e9re, A. (2010). From the definitions of the \u2018Tr\u00e9sor de la Langue Fran\u00e7aise\u2019 to a semantic database of the French language. In Proceedings of the XIV Euralex international congress, pp. 245\u2013252."},{"issue":"4","key":"9258_CR5","doi-asserted-by":"crossref","first-page":"778","DOI":"10.5964\/ejop.v7i4.163","volume":"7","author":"J. Barry","year":"2011","unstructured":"Barry, J. (2011). Doing Bayesian data analysis: A tutorial with R and BUGS. Europe\u2019s Journal of Psychology 7(4), 778\u2013779.","journal-title":"Europe\u2019s Journal of Psychology"},{"issue":"2","key":"9258_CR6","first-page":"159","volume":"76","author":"J. Berger","year":"1988","unstructured":"Berger, J., & Berry, D. (1988). Statistical analysis and the illusion of objectivity. American Scientist, 76(2), 159\u2013165.","journal-title":"American Scientist"},{"key":"9258_CR7","unstructured":"Brants, T., & Plaehn, O. (2000). Interactive corpus annotation. In Proceedings of the second international conference on language resources and evaluation."},{"key":"9258_CR8","unstructured":"Carmen, M., Felt, P., Haertel, R., Lonsdale, D., McClanahan, P., Merkling, O., et al. (2010). Tag dictionaries accelerate manual annotation. In Proceedings of the seventh international conference on language resources and evaluation."},{"key":"9258_CR9","unstructured":"Carter, D. (1997). The TreeBanker. A tool for supervised training of parsed corpora. In Proceedings of the workshop on computational environments for grammar development and linguistic engineering, pp. 9\u201315."},{"key":"9258_CR10","unstructured":"Chang, M., Ratinov, L., & Roth, D. (2007). Guiding semi-supervision with constraint-driven learning. In Proceedings of the 45th annual meeting of the association for computational linguistics, pp. 280\u2013287."},{"key":"9258_CR11","doi-asserted-by":"crossref","unstructured":"Chiou, F., Chiang, D., & Palmer, M. (2001). Facilitating treebank annotation using a statistical parser. In Proceedings of the first international conference on human language technology research, pp. 1\u20134.","DOI":"10.3115\/1072133.1072178"},{"issue":"1","key":"9258_CR12","doi-asserted-by":"crossref","first-page":"80","DOI":"10.1145\/234173.234209","volume":"39","author":"J. Cowie","year":"1996","unstructured":"Cowie, J., & Lehnert, W. (1996). Information extraction. Communications of the ACM 39(1), 80\u201391.","journal-title":"Communications of the ACM"},{"key":"9258_CR13","doi-asserted-by":"crossref","unstructured":"Dandapat, S., Biswas, P., Choudhury, M., & Bali, K. (2009) Complex linguistic annotation\u2014No easy way out!: A case from Bangla and Hindi POS labeling tasks. In Proceedings of the third linguistic annotation workshop, pp. 10\u201318.","DOI":"10.3115\/1698381.1698383"},{"issue":"2","key":"9258_CR14","doi-asserted-by":"crossref","first-page":"159","DOI":"10.1075\/ijcl.14.2.02dav","volume":"14","author":"M. Davies","year":"2009","unstructured":"Davies, M. (2009). The 385+ million word corpus of contemporary American english (1990-2008+): Design, architecture, and linguistic insights. International Journal of Corpus Linguistics, 14(2), 159\u2013190.","journal-title":"International Journal of Corpus Linguistics"},{"key":"9258_CR15","unstructured":"Davies, M. (2010). The corpus of historical American english: 400 million words, 1810\u20132009. Available online at http:\/\/corpus.byu.edu\/coha\/ ."},{"key":"9258_CR16","doi-asserted-by":"crossref","unstructured":"Druck, G., Mann, G., & McCallum, A. (2008). Learning from labeled features using generalized expectation criteria. In Proceedings of the 31st annual international conference on research and development in information retrieval, pp. 595\u2013602.","DOI":"10.1145\/1390334.1390436"},{"key":"9258_CR17","unstructured":"Felt, P., Merkling, O., Carmen, M., Ringger, E., Lemmon, W., Seppi, K., et al. (2010). CCASH: A web application framework for efficient distributed language resource development. In Proceedings of the seventh international conference on language resources and evaluation."},{"key":"9258_CR18","unstructured":"Felt, P., Ringger, E., Seppi, K., Heal, K., Haertel, R., & Lonsdale, D. (2012). First results in a study evaluating pre-labeling and correction propagation for machine-assisted Syriac morphological analysis. In Proceedings of the Eighth international conference on language resources and evaluation."},{"issue":"3","key":"9258_CR19","doi-asserted-by":"crossref","first-page":"59","DOI":"10.1609\/aimag.v31i3.2303","volume":"31","author":"D. Ferrucci","year":"2010","unstructured":"Ferrucci, D., Brown, E., Chu-Carroll, J., Fan, J., Gondek, D., Kalyanpur, A., et\u00a0al (2010). Building watson: An overview of the DeepQA project. AI Magazine, 31(3), 59\u201379.","journal-title":"AI Magazine"},{"key":"9258_CR20","unstructured":"Fort, K., & Sagot, B. (2010). Influence of pre-annotation on POS-tagged corpus development. In Proceedings of the fourth linguistic annotation workshop, association for computational linguistics, pp. 56\u201363."},{"key":"9258_CR21","doi-asserted-by":"crossref","unstructured":"Ganchev, K., Pereira, F., Mandel, M., Carroll, S.,& White, P. (2007). Semi-automated named entity annotation. In Proceedings of the 45th annual meeting of the association for computational linguistics: Linguistic annotation workshop, pp. 53\u201356.","DOI":"10.3115\/1642059.1642068"},{"key":"9258_CR22","unstructured":"Ganchev, K., Gra\u00e7a, J., Blitzer, J., & Taskar, B. (2008). Multi-view learning over structured and non-identical outputs. In Proceedings of the conference on Uncertainty in Artificial Intelligence, pp. 204\u2013211."},{"key":"9258_CR23","volume-title":"Bayesian data analysis","author":"A. Gelman","year":"2004","unstructured":"Gelman, A. (2004). Bayesian data aAnalysis. CRC Press: Boca Raton, FL."},{"key":"9258_CR24","doi-asserted-by":"crossref","unstructured":"Haertel, R., Ringger, E., Seppi, K., Carroll, J., & McClanahan, P. (2008a). Assessing the costs of sampling methods in active learning for annotation. In Proceedings of the 46th annual meeting of the association for computational linguistics, pp. 65\u201368.","DOI":"10.3115\/1557690.1557708"},{"key":"9258_CR25","unstructured":"Haertel, R., Seppi, K., Ringger, E., & Carroll, J. (2008b). Return on investment for active learning. In Proceedings of the 22nd annual conference on neural information processing systems: Workshop on cost-sensitive learning."},{"key":"9258_CR27","first-page":"461","volume":"247","author":"G. Kiraz","year":"1994","unstructured":"Kiraz, G. (1994). Automatic concordance generation of Syriac texts. Orientalia Christiana Analecta, 247, 461\u2013475.","journal-title":"Orientalia Christiana Analecta"},{"key":"9258_CR28","doi-asserted-by":"crossref","unstructured":"Kittur, A., Chi, E., & Suh, B. (2008). Crowdsourcing user studies with mechanical turk. In Proceedings of the 26th annual conference on human factors in computing systems, pp. 453\u2013456.","DOI":"10.1145\/1357054.1357127"},{"issue":"4","key":"9258_CR29","doi-asserted-by":"crossref","first-page":"495","DOI":"10.1162\/coli.2009.35.4.35402","volume":"35","author":"B. Klebanov","year":"2009","unstructured":"Klebanov, B., & Beigman, E. (2009). From annotator agreement to noise models. Computational Linguistics 35(4), 495\u2013503.","journal-title":"Computational Linguistics"},{"key":"9258_CR30","unstructured":"Kristjansson, T., Culotta, A., Viola, P., & McCallum, A. (2004). Interactive information extraction with constrained conditional random fields. In Proceedings of the 19th conference on artificial intelligence, pp.412\u2013418."},{"key":"9258_CR31","doi-asserted-by":"crossref","unstructured":"Leech, G., Garside, R., & Bryant, M. (1994). CLAWS4: The tagging of the British National Corpus. In Proceedings of the 15th conference on computational linguistics, association for computational linguistics, Vol. 1, pp. 622\u2013628.","DOI":"10.3115\/991886.991996"},{"key":"9258_CR32","doi-asserted-by":"crossref","unstructured":"Liang, P., Jordan, M., & Klein, D. (2009). Learning from measurements in exponential families. In Proceedings of the 26th annual international conference on machine learning, pp. 641\u2013648.","DOI":"10.1145\/1553374.1553457"},{"key":"9258_CR33","doi-asserted-by":"crossref","unstructured":"Liang, P., Jordan, M. I., & Klein, D. (2013). Learning dependency-based compositional semantics. Computational Linguistics, 39(2), 389\u2013446.","DOI":"10.1162\/COLI_a_00127"},{"key":"9258_CR34","volume-title":"Foundations of statistical natural language processing","author":"C. Manning","year":"1999","unstructured":"Manning, C., & Sch\u00fctze, H. (1999). Foundations of statistical natural language processing. MIT Press: Cambridge, MA."},{"issue":"2","key":"9258_CR35","first-page":"313","volume":"19","author":"M. Marcus","year":"1993","unstructured":"Marcus, M., Marcinkiewicz, M., & Santorini, B. (1993). Building a large annotated corpus of English: The Penn Treebank. Computational Linguistics, 19(2), 313\u2013330.","journal-title":"Computational Linguistics"},{"key":"9258_CR36","unstructured":"McClanahan, P., Busby, G., Haertel, R., Heal, K., Lonsdale, D., Seppi, K., et al. (2010) A probabilistic morphological analyzer for Syriac. In Proceedings of the conference on empirical methods in natural language processing, pp. 810\u2013820."},{"key":"9258_CR37","doi-asserted-by":"crossref","unstructured":"Menke, J., & Martinez, T. (2004). Using permutations instead of student\u2019s t distribution for P-values in paired-difference algorithm comparisons. In Proceedings of the international joint conference on neural networks, pp. 1331\u20131335.","DOI":"10.1109\/IJCNN.2004.1380138"},{"issue":"6014","key":"9258_CR38","first-page":"176","volume":"331","author":"J. Michel","year":"2011","unstructured":"Michel, J., Shen, Y., Aiden, A., Veres, A., Gray, M., Pickett, J., et\u00a0al. (2011). Quantitative analysis of culture using millions of digitized books. Science 331(6014), 176.","journal-title":"HortScience"},{"key":"9258_CR39","unstructured":"Naseem, T., Chen, H., Barzilay, R., & Johnson, M. (2010). Using universal linguistic knowledge to guide grammar induction. In Proceedings of the conference on empirical methods in natural language processing, pp. 1234\u20131244."},{"key":"9258_CR40","doi-asserted-by":"crossref","unstructured":"Ngai, G., & Yarowsky, D. (2000). Rule writing or annotation: cost-efficient resource usage for base noun phrase chunking. In Proceedings of the 38th annual meeting of the association for computational linguistics, pp. 117\u2013125.","DOI":"10.3115\/1075218.1075234"},{"key":"9258_CR41","doi-asserted-by":"crossref","unstructured":"Oepen, S., Toutanova, K., Shieber, S., Manning, C., Flickinger, D., & Brants, T. (2002). The LinGO redwoods treebank motivation and preliminary applications. In: Proceedings of the 19th international conference on computational linguistics, association for computational linguistics, Vol. 2, pp. 1\u20135.","DOI":"10.3115\/1071884.1071909"},{"issue":"1-2","key":"9258_CR42","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1561\/1500000011","volume":"2","author":"B. Pang","year":"2008","unstructured":"Pang, B., & Lee, L. (2008). Opinion mining and sentiment analysis. Foundations and Trends in Information Retrieval, 2(1\u20132), 1\u2013135.","journal-title":"Foundations and Trends in Information Retrieval"},{"key":"9258_CR43","doi-asserted-by":"crossref","unstructured":"Pang, B., Lee, L., & Vaithyanathan, S. (2002). Thumbs up?: Sentiment classification using machine learning techniques. In Proceedings of the conference on empirical methods in natural language processing, pp. 79\u201386.","DOI":"10.3115\/1118693.1118704"},{"key":"9258_CR44","doi-asserted-by":"crossref","unstructured":"Rehbein, I., Ruppenhofer, J., & Sporleder, C. (2009). Assessing the benefits of partial automatic pre-labeling for frame-semantic annotation. In Proceedings of the third linguistic annotation workshop, association for computational linguistics, pp. 19\u201326.","DOI":"10.3115\/1698381.1698384"},{"key":"9258_CR45","unstructured":"Ringger, E., Carmen, M., Haertel, R., Seppi, K., Lonsdale, D., McClanahan, P., et al. (2008). Assessing the costs of machine-assisted corpus annotation through a user study. In: Proceedings of the sixth international conference on language resources and evaluation."},{"key":"9258_CR46","unstructured":"Settles, B. (2010). Active learning literature survey. Computer sciences technical report 1648, Madison: University of Wisconsin."},{"key":"9258_CR47","doi-asserted-by":"crossref","unstructured":"Shneiderman, B., & Plaisant, C. (2006). Strategies for evaluating information visualization tools: Multi-dimensional in-depth long-term case studies. In: Proceedings of the 2006 AVI workshop on beyond time and errors: Novel evaluation methods for information visualization, pp. 1\u20137.","DOI":"10.1145\/1168149.1168158"},{"issue":"2","key":"9258_CR49","doi-asserted-by":"crossref","first-page":"1","DOI":"10.2200\/S00361ED1V01Y201105HLT013","volume":"4","author":"N.A. Smith","year":"2011","unstructured":"Smith, N. A. (2011) Linguistic structure prediction. Synthesis Lectures on Human Language Technologies, 4(2), 1\u2013274.","journal-title":"Synthesis Lectures on Human Language Technologies"},{"issue":"1","key":"9258_CR48","doi-asserted-by":"crossref","first-page":"15","DOI":"10.1093\/llc\/15.1.15","volume":"15","author":"D. Smith","year":"2000","unstructured":"Smith, D., Rydberg-Cox, J., & Crane, G. (2000). The perseus project: A digital library for the humanities. Literary and Linguistic Computing, 15(1), 15\u201325","journal-title":"Literary and Linguistic Computing"},{"issue":"4","key":"9258_CR50","doi-asserted-by":"crossref","first-page":"521","DOI":"10.1162\/089120101753342653","volume":"27","author":"W. Soon","year":"2001","unstructured":"Soon, W., Ng, H., & Lim, D. (2001). A machine learning approach to coreference resolution of noun phrases. Computational Linguistics 27(4), 521\u2013544.","journal-title":"Computational Linguistics"},{"key":"9258_CR51","unstructured":"Stan Development Team. (2013). Stan: A c++ library for probability and sampling, version 1.1. URL http:\/\/mc-stan.org\/ ."},{"key":"9258_CR52","doi-asserted-by":"crossref","unstructured":"Tanaka, T., Bond, F., Oepen, S., & Fujita, S. (2005) High precision treebanking: Blazing useful trees using POS information. In Proceedings of the 43rd annual meeting on association for computational linguistics, association for computational linguistics, pp. 330\u2013337.","DOI":"10.3115\/1219840.1219881"},{"key":"9258_CR53","doi-asserted-by":"crossref","unstructured":"Tomanek, K., Wermter, J.,& Hahn, U. (2007) Efficient annotation with the jena annotation environment (JANE). In Proceedings of the 45th annual meeting of the association for computational linguistics: Linguistic annotation workshop, pp. 9\u201316.","DOI":"10.3115\/1642059.1642061"},{"key":"9258_CR54","unstructured":"Tov, E., & Reynolds. N., (2006) The dead sea scrolls electronic library. Brigham Young University and the Neal A. Maxwell Institute for Religious Scholarship, Provo, UT."},{"issue":"8","key":"9258_CR55","doi-asserted-by":"crossref","first-page":"58","DOI":"10.1145\/1378704.1378719","volume":"51","author":"L. Von Ahn","year":"2008","unstructured":"Von Ahn, L., & Dabbish, L. (2008). Designing games with a purpose. Communications of the ACM, 51(8), 58\u201367.","journal-title":"Communications of the ACM"},{"key":"9258_CR56","unstructured":"Wright, W. (1871). Apocryphal acts of the apostles. London: Williams and Norgate."},{"key":"9258_CR57","unstructured":"Zettlemoyer, L., & Collins, M. (2005). Learning to map sentences to logical form: Structured classification with probabilistic categorial grammars. In Proceedings of the twenty-first conference annual conference on uncertainty in artificial intelligence (pp. 658\u2013666). Arlington, VA: AUAI Press."}],"container-title":["Language Resources and Evaluation"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s10579-013-9258-8.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/article\/10.1007\/s10579-013-9258-8\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s10579-013-9258-8","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2019,8,1]],"date-time":"2019-08-01T19:50:16Z","timestamp":1564689016000},"score":1,"resource":{"primary":{"URL":"http:\/\/link.springer.com\/10.1007\/s10579-013-9258-8"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2013,11,15]]},"references-count":56,"journal-issue":{"issue":"4","published-print":{"date-parts":[[2014,12]]}},"alternative-id":["9258"],"URL":"https:\/\/doi.org\/10.1007\/s10579-013-9258-8","relation":{},"ISSN":["1574-020X","1574-0218"],"issn-type":[{"value":"1574-020X","type":"print"},{"value":"1574-0218","type":"electronic"}],"subject":[],"published":{"date-parts":[[2013,11,15]]}}}