{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,12,30]],"date-time":"2025-12-30T23:54:39Z","timestamp":1767138879449,"version":"build-2238731810"},"publisher-location":"Cham","reference-count":41,"publisher":"Springer International Publishing","isbn-type":[{"value":"9783319089577","type":"print"},{"value":"9783319089584","type":"electronic"}],"license":[{"start":{"date-parts":[[2014,1,1]],"date-time":"2014-01-01T00:00:00Z","timestamp":1388534400000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2014,1,1]],"date-time":"2014-01-01T00:00:00Z","timestamp":1388534400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2014]]},"DOI":"10.1007\/978-3-319-08958-4_25","type":"book-chapter","created":{"date-parts":[[2014,7,25]],"date-time":"2014-07-25T06:57:11Z","timestamp":1406271431000},"page":"303-314","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":6,"title":["Crowdsourcing for Language Resource Development: Criticisms About Amazon Mechanical Turk Overpowering Use"],"prefix":"10.1007","author":[{"given":"Kar\u00ebn","family":"Fort","sequence":"first","affiliation":[]},{"given":"Gilles","family":"Adda","sequence":"additional","affiliation":[]},{"given":"Beno\u00eet","family":"Sagot","sequence":"additional","affiliation":[]},{"given":"Joseph","family":"Mariani","sequence":"additional","affiliation":[]},{"given":"Alain","family":"Couillault","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2014,7,26]]},"reference":[{"issue":"2","key":"25_CR1","doi-asserted-by":"publisher","first-page":"413","DOI":"10.1162\/COLI_a_00057","volume":"37","author":"K Fort","year":"2011","unstructured":"Fort, K., Adda, G., Cohen, K.B.: Amazon mechanical turk: Gold mine or coal mine? Comput. Linguist. (Editorial) 37(2), 413\u2013420 (2011)","journal-title":"Comput. Linguist. (Editorial)"},{"key":"25_CR2","doi-asserted-by":"crossref","unstructured":"Ross, J., Irani, L., Silberman, M.S., Zaldivar, A., Tomlinson, B.: Who are the crowdworkers?: shifting demographics in mechanical turk. In: Proceedings of the 28th of the International Conference Extended Abstracts on Human Factors in Computing Systems, CHI EA \u201910. ACM, New York (2010)","DOI":"10.1145\/1753846.1753873"},{"key":"25_CR3","unstructured":"Ipeirotis, P.: Demographics of mechanical turk. CeDER Working Papers, March 2010. http:\/\/hdl.handle.net\/2451\/29585 (2010). CeDER-10-01"},{"key":"25_CR4","unstructured":"Biewald, L.: Better crowdsourcing through automated methods for quality control. In: SIGIR 2010 Workshop on Crowdsourcing for Search Evaluation, January 2010 (2010)"},{"key":"25_CR5","doi-asserted-by":"crossref","unstructured":"Silberman, M.S., Ross, J., Irani, L., Tomlinson, B.: Sellers\u2019 problems in human computation markets. In: Proceedings of the ACM SIGKDD Workshop on Human Computation, HCOMP \u201910, pp. 18\u201321 (2010)","DOI":"10.1145\/1837885.1837891"},{"key":"25_CR6","unstructured":"Ross, J., Zaldivar, A., Irani, L., Tomlinson, B.: Who are the turkers? worker demographics in amazon mechanical turk. Social Code Report 2009-01. http:\/\/www.ics.uci.edu\/jwross\/pubs\/SocialCode-2009-01.pdf (2009)"},{"key":"25_CR7","doi-asserted-by":"crossref","unstructured":"Chilton, L.B., Horton, J.J., Miller, R.C., Azenkot, S.: Task search in a human computation market. In: Proceedings of the ACM SIGKDD Workshop on Human Computation, HCOMP \u201910, pp. 1\u20139 (2010)","DOI":"10.1145\/1837885.1837889"},{"key":"25_CR8","unstructured":"Adda, G., Mariani, J.: Language resources and amazon mechanical turk: legal, ethical and other issues. In: LISLR 2010, \u201cLegal Issues for Sharing Language Resources workshop\u201d, LREC 2010, Valletta, Malta, May 2010 (2010)"},{"key":"25_CR9","unstructured":"Novotney, S., Callison-Burch, C.: Cheap, fast and good enough: automatic speech recognition with non-expert transcription. In: Human Language Technologies: The 2010 Annual Conference of the North American Chapter of the Association for Computational Linguistics, HLT \u201910, Los Angeles, California, USA, pp. 207\u2013215 (2010)"},{"key":"25_CR10","unstructured":"Callison-Burch, C., Dredze, M.: Creating speech and language data with amazon\u2019s mechanical turk. In: CSLDAMT \u201910: Proceedings of the NAACL HLT 2010 Workshop on Creating Speech and Language Data with Amazon\u2019s Mechanical Turk, Los Angeles, California, USA (2010)"},{"key":"25_CR11","unstructured":"Kaisser, M., Lowe, J.B.: Creating a research collection of question answer sentence pairs with amazon\u2019s mechanical turk. In: Proceedings of the International Language Resources and Evaluation Conference (LREC), Marrakech, Morocco (2008)"},{"key":"25_CR12","unstructured":"Xu, F., Klakow, D.: Paragraph acquisition and selection for list question using amazon\u2019s mechanical turk. In: Proceedings of the International Language Resources and Evaluation Conference (LREC), Valletta, Malta, May 2010, pp. 2340\u20132345 (2010)"},{"key":"25_CR13","doi-asserted-by":"crossref","unstructured":"Marge, M., Banerjee, S., Rudnicky, A.I.: Using the amazon mechanical turk for transcription of spoken language. In: IEEE International Conference on Acoustics Speech and Signal Processing (ICASSP), Dallas, USA, 14\u201319 March 2010, pp. 5270\u20135273 (2010)","DOI":"10.1109\/ICASSP.2010.5494979"},{"key":"25_CR14","unstructured":"Cook, P., Stevenson, S.: Automatically identifying changes in the semantic orientation of words. In: Proceedings of the International Language Resources and Evaluation Conference (LREC), Valletta, Malta, May 2010 (2010)"},{"key":"25_CR15","unstructured":"Bhardwaj, V., Passonneau, R., Salleb-Aouissi, A., Ide, N.: Anveshan: a tool for analysis of multiple annotators\u2019 labeling behavior. In: Proceedings of the Fourth Linguistic Annotation Workshop (LAW IV), Uppsala, Sweden (2010)"},{"key":"25_CR16","doi-asserted-by":"crossref","unstructured":"Snow, R., O\u2019Connor, B., Jurafsky, D., Ng., A.Y.: Cheap and fast - but is it good? evaluating non-expert annotations for natural language tasks. In: Proceedings of EMNLP 2008. pp. 254\u2013263 (2008)","DOI":"10.3115\/1613715.1613751"},{"key":"25_CR17","unstructured":"Gillick, D., Liu, Y.: Non-expert evaluation of summarization systems is risky. In: Proceedings of the NAACL HLT 2010 Workshop on Creating Speech and Language Data with Amazon\u2019s Mechanical Turk, CSLDAMT \u201910, Los Angeles, California, USA (2010)"},{"key":"25_CR18","unstructured":"Tratz, S., Hovy, E.: A taxonomy, dataset, and classifier for automatic noun compound interpretation. In: Proceedings of the 48th Annual Meeting of the Association for Computational Linguistics, Uppsala, Sweden, July 2010, pp. 678\u2013687 (2010)"},{"key":"25_CR19","unstructured":"Wais, P., Lingamneni, S., Cook, D., Fennell, J., Goldenberg, B., Lubarov, D., Marin, D., Simons, H.: Towards building a high-quality workforce with mechanical turk. In: Proceedings of Computational Social Science and the Wisdom of Crowds (NIPS), December 2010 (2010)"},{"key":"25_CR20","doi-asserted-by":"crossref","unstructured":"Kochhar, S., Mazzocchi, S., Paritosh, P.: The anatomy of a large-scale human computation engine. In: Proceedings of Human Computation Workshop at the 16th ACM SIKDD Conference on Knowledge Discovery and Data Mining, KDD 2010, Washington D.C. (2010)","DOI":"10.1145\/1837885.1837890"},{"key":"25_CR21","unstructured":"Goldwater, S., Griffiths, T.: A fully bayesian approach to unsupervised part-of-speech tagging. In: Proceedings of ACL, Prague, Czech Republic (2007)"},{"key":"25_CR22","unstructured":"H\u00e4nig, C.: Improvements in unsupervised co-occurrence based parsing. In: Proceedings of the Fourteenth Conference on Computational Natural Language Learning, CoNLL \u201910, Uppsala, Sweden, pp. 1\u20138 (2010)"},{"key":"25_CR23","series-title":"Chapman & Hall\/CRC Computer Science & Data Analysis","doi-asserted-by":"crossref","DOI":"10.1201\/9781420010800","volume-title":"Semisupervised Learning for Computational Linguistics","author":"Steven Abney","year":"2007","unstructured":"Abney, S.: Semisupervised Learning for Computational Linguistics. 1\u00e8re edn. Chapman & Hall\/CRC, New York (2007)"},{"key":"25_CR24","doi-asserted-by":"crossref","unstructured":"Yarowsky, D.: Unsupervised word sense disambiguation rivaling supervised methods. In: Proceedings of the 33rd Annual Meeting of the Association for Computational Linguistics, Cambridge, MA, USA, pp. 189\u2013196 (1995)","DOI":"10.3115\/981658.981684"},{"key":"25_CR25","doi-asserted-by":"crossref","unstructured":"Blum, A., Mitchell, T.: Combining labeled and unlabeled data with co-training. In: COLT: Proceedings of the Workshop on Computational Learning Theory. Morgan Kaufmann Publishers (1998)","DOI":"10.1145\/279943.279962"},{"key":"25_CR26","first-page":"705","volume-title":"Advances in Neural Information Processing Systems","author":"DA Cohn","year":"1995","unstructured":"Cohn, D.A., Ghahramani, Z., Jordan, M.I.: Active learning with statistical models. In: Tesauro, G., Touretzky, D., Leen, T. (eds.) Advances in Neural Information Processing Systems, vol. 7, pp. 705\u2013712. The MIT Press, Cambridge (1995)"},{"key":"25_CR27","doi-asserted-by":"crossref","unstructured":"Smith, N., Eisner, J.: Contrastive estimation: training log-linear models on unlabeled data. In: Proceedings of the 43th Annual Meeting of the Association for Computational Linguistics (ACL\u201905), Ann Arbor, Michigan, USA, pp. 354\u2013362 (2005)","DOI":"10.3115\/1219840.1219884"},{"key":"25_CR28","series-title":"Lecture Notes in Computer Science (Lecture Notes in Artificial Intelligence)","doi-asserted-by":"publisher","first-page":"156","DOI":"10.1007\/11551874_20","volume-title":"Text, Speech and Dialogue","author":"B Sagot","year":"2005","unstructured":"Sagot, B.: Automatic acquisition of a Slovak lexicon from a raw corpus. In: Matou\u0161ek, V., Mautner, P., Pavelka, T. (eds.) TSD 2005. LNCS (LNAI), vol. 3658, pp. 156\u2013163. Springer, Heidelberg (2005)"},{"key":"25_CR29","doi-asserted-by":"crossref","unstructured":"Watson, R., Briscoe, T., Carroll, J.: Semi-supervised training of a statistical parser from unlabeled partially-bracketed data. In: Proceedings of the 10th International Conference on Parsing Technologies, IWPT \u201907, Prague, Czech Republic (2007)","DOI":"10.3115\/1621410.1621413"},{"key":"25_CR30","unstructured":"Fort, K., Sagot, B.: Influence of Pre-annotation on POS-tagged corpus development. In: Proceedings of the Fourth ACL Linguistic Annotation Workshop, Uppsala, Sweden (2010)"},{"key":"25_CR31","unstructured":"Erk, K., Kowalski, A., Pado, S.: The SALSA annotation tool. In: Duchier, D., Kruijff, G.J.M. (eds.) Proceedings of the Workshop on Prospects and Advances in the Syntax\/Semantics Interface, Nancy, France (2003)"},{"key":"25_CR32","unstructured":"Yetisgen-Yildiz, M., Solti, I., Xia, F., Halgrim, S.R.: Preliminary experience with amazon\u2019s mechanical turk for annotating medical named entities. In: Proceedings of the NAACL HLT 2010 Workshop on Creating Speech and Language Data with Amazon\u2019s Mechanical Turk, CSLDAMT \u201910, Los Angeles, California, USA, pp. 180\u2013183 (2010)"},{"key":"25_CR33","unstructured":"Finin, T., Murnane, W., Karandikar, A., Keller, N., Martineau, J., Dredze, M.: Annotating named entities in twitter data with crowdsourcing. In: Proceedings of the NAACL HLT 2010 Workshop on Creating Speech and Language Data with Amazon\u2019s Mechanical Turk, CSLDAMT \u201910, Los Angeles, California, USA (2010)"},{"key":"25_CR34","unstructured":"Lawson, N., Eustice, K., Perkowitz, M., Yetisgen-Yildiz, M.: Annotating large email datasets for named entity recognition with mechanical turk. In: Proceedings of the NAACL HLT 2010 Workshop on Creating Speech and Language Data with Amazon\u2019s Mechanical Turk, CSLDAMT \u201910, Los Angeles, California, USA, pp. 71\u201379 (2010)"},{"key":"25_CR35","unstructured":"Nothman, J., Curran, J.R., Murphy, T.: Transforming Wikipedia into named entity training data. In: Proceedings of the Australian Language Technology Workshop (2008)"},{"key":"25_CR36","doi-asserted-by":"crossref","unstructured":"Balasuriya, D., Ringland, N., Nothman, J., Murphy, T., Curran, J.R.: Named entity recognition in Wikipedia. In: People\u2019s Web \u201909: Proceedings of the 2009 Workshop on The People\u2019s Web Meets NLP, Suntec, Singapore, pp. 10\u201318 (2009)","DOI":"10.3115\/1699765.1699767"},{"key":"25_CR37","doi-asserted-by":"crossref","unstructured":"St\u00fcrenberg, M., Goecke, D., Die-wald, N., Cramer, I., Mehler, A.: Web-based annotation of anaphoric relations and lexical chains. In: ACL Workshop on Linguistic Annotation Workshop (LAW), Prague, Czech Republic (2007)","DOI":"10.3115\/1642059.1642082"},{"key":"25_CR38","doi-asserted-by":"publisher","first-page":"92","DOI":"10.1109\/MC.2006.196","volume":"39","author":"L von Ahn","year":"2006","unstructured":"von Ahn, L.: Games with a purpose. IEEE Comput. Mag. 39, 92\u201394 (2006)","journal-title":"IEEE Comput. Mag."},{"key":"25_CR39","unstructured":"Chamberlain, J., Poesio, M., Kruschwitz, U.: Phrase detectives: a web-based collaborative annotation game. In: Proceedings of the International Conference on Semantic Systems (I-Semantics\u201908), Graz, Austria (2008)"},{"key":"25_CR40","doi-asserted-by":"crossref","unstructured":"Hughes, T., Nakajima, K., Ha, L., Vasu, A., Moreno, P., LeBeau, M.: Building transcribed speech corpora quickly and cheaply for many languages. In: Proceedings of Interspeech, Makuhari, Chiba, Japan, September 2010, pp. 1914\u20131917 (2010)","DOI":"10.21437\/Interspeech.2010-551"},{"key":"25_CR41","unstructured":"Couillault, A., Fort, K.: Charte \u00c9thique et Big Data : parce que mon corpus le vaut bien ! In: Linguistique, Langues et Parole : Statuts, Usages et M\u00e9susages, Strasburg, France, July 2013, 4 p (2013)"}],"container-title":["Lecture Notes in Computer Science","Human Language Technology Challenges for Computer Science and Linguistics"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-319-08958-4_25","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,7,12]],"date-time":"2024-07-12T04:49:42Z","timestamp":1720759782000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-3-319-08958-4_25"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2014]]},"ISBN":["9783319089577","9783319089584"],"references-count":41,"aliases":["10.1007\/978-3-319-14120-6_25"],"URL":"https:\/\/doi.org\/10.1007\/978-3-319-08958-4_25","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"value":"0302-9743","type":"print"},{"value":"1611-3349","type":"electronic"}],"subject":[],"published":{"date-parts":[[2014]]},"assertion":[{"value":"26 July 2014","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}}]}}