{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,10,22]],"date-time":"2025-10-22T03:01:02Z","timestamp":1761102062197,"version":"3.40.3"},"publisher-location":"Berlin, Heidelberg","reference-count":41,"publisher":"Springer Berlin Heidelberg","isbn-type":[{"type":"print","value":"9783642364143"},{"type":"electronic","value":"9783642364150"}],"license":[{"start":{"date-parts":[[2013,1,1]],"date-time":"2013-01-01T00:00:00Z","timestamp":1356998400000},"content-version":"unspecified","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2013]]},"DOI":"10.1007\/978-3-642-36415-0_9","type":"book-chapter","created":{"date-parts":[[2013,1,23]],"date-time":"2013-01-23T17:29:25Z","timestamp":1358962165000},"page":"132-154","source":"Crossref","is-referenced-by-count":9,"title":["An Introduction to Crowdsourcing for Language and Multimedia Technology Research"],"prefix":"10.1007","author":[{"given":"Gareth J. F.","family":"Jones","sequence":"first","affiliation":[]}],"member":"297","reference":[{"issue":"2","key":"9_CR1","doi-asserted-by":"publisher","first-page":"9","DOI":"10.1145\/1480506.1480508","volume":"42","author":"O. Alonso","year":"2008","unstructured":"Alonso, O., Rose, D.E., Stewart, B.: Crowdsourcing for relevance evaluation. SIGIR Forum\u00a042(2), 9\u201315 (2008)","journal-title":"SIGIR Forum"},{"key":"9_CR2","unstructured":"Ambati, V., Vogel, S., Carbonell, J.: Active learning and crowd-sourcing for machine translation. In: Proceedings of the Seventh International Conference on Language Resources and Evaluation (LREC 2012), pp. 2169\u20132174 (2010)"},{"key":"9_CR3","doi-asserted-by":"crossref","unstructured":"Callison-Burch, C.: Fast, cheap, and creative: Evaluating translation quality using Amazon\u2019s Mechanical Turk. In: Proceedings of the 2009 Conference on Empirical Methods in Natural Language Processing (EMNLP 2009), pp. 286\u2013295 (2009)","DOI":"10.3115\/1699510.1699548"},{"key":"9_CR4","first-page":"1","volume-title":"Proceedings of the NAACL HLT 2010 Workshop on Creating Speech and Language Data with Amazon\u2019s Mechanical Turk (CSLDAMT 2010)","author":"C. Callison-Burch","year":"2010","unstructured":"Callison-Burch, C., Dredze, M.: Creating speech and language data with Amazon\u2019s Mechanical Turk. In: Proceedings of the NAACL HLT 2010 Workshop on Creating Speech and Language Data with Amazon\u2019s Mechanical Turk (CSLDAMT 2010), pp. 1\u201312. Association for Computational Linguistics, Stroudsburg (2010)"},{"issue":"4","key":"9_CR5","doi-asserted-by":"publisher","first-page":"86","DOI":"10.1145\/1924421.1924442","volume":"54","author":"A. Doan","year":"2011","unstructured":"Doan, A., Ramakrishnan, R., Halevy, A.Y.: Crowdsourcing systems on the World-Wide Web. Communications of the ACM\u00a054(4), 86\u201396 (2011)","journal-title":"Communications of the ACM"},{"key":"9_CR6","unstructured":"Eskevich, M., Jones, G.J.F., Larson, M., Ordelman, R.: Creating a Data Collection for Evaluating Rich Speech Retrieval. In: Proceedings of the Eighth International Conference on Language Resources and Evaluation (LREC 2012), Istanbul, Turkey (2012)"},{"key":"9_CR7","unstructured":"Evanini, K., Higgins, D., Zechner, K.: Using Amazon Mechanical Turk for transcription of non-native speech. In: Proceedings of the NAACL HLT 2010 Workshop on Creating Speech and Language Data with Amazon\u2019s Mechanical Turk (CSLDAMT 2010), pp. 53\u201356. Association for Computational Linguistics (2010)"},{"key":"9_CR8","doi-asserted-by":"crossref","unstructured":"Goto, M., Ogata, J.: Podcastle: Recent advances of a spoken document retrieval service improved by anonymous user contributions. In: Proceedings of Interspeech 2011 (2011)","DOI":"10.21437\/Interspeech.2011-769"},{"key":"9_CR9","unstructured":"Grady, C., Lease, M.: Crowdsourcing document relevance assessment with Mechanical Turk. In: Proceedings of the NAACL HLT 2010 Workshop on Creating Speech and Language Data with Amazon\u2019s Mechanical Turk (CSLDAMT 2010), pp. 172\u2013179. Association for Computational Linguistics (2010)"},{"key":"9_CR10","doi-asserted-by":"crossref","unstructured":"Horton, J.J.: Employer expectations, peer effects and productivity: Evidence from a series of field experiments. CoRR, abs\/1008.2437 (2010)","DOI":"10.2139\/ssrn.1652993"},{"key":"9_CR11","unstructured":"Ipeirotis, P.G., Provost, F., Sheng, V., Wang, J.: Repeated Labeling Using Multiple Noisy Labelers. SSRN eLibrary (2010)"},{"key":"9_CR12","doi-asserted-by":"crossref","unstructured":"Kittur, A., Chi, E.H., Suh, B.: Crowdsourcing user studies with Mechanical Turk. In: Proceedings of the 2008 Conference on Human Factors in Computing Systems (CHI 2008), pp. 453\u2013456. ACM (2008)","DOI":"10.1145\/1357054.1357127"},{"key":"9_CR13","unstructured":"Kunchukuttan, A., Roy, S., Patel, P., Ladha, K., Gupta, S., Khapra, M.M., Bhattacharyya, P.: Experiences in resource generation for machine translation through crowdsourcing. In: Proceedings of the Eighth International Conference on Language Resources and Evaluation (LREC 2012), Istanbul, Turkey (2012)"},{"key":"9_CR14","unstructured":"Lane, I., Waibel, A., Eck, M., Rottmann, K.: Tools for collecting speech corpora via Mechanical-Turk. In: Proceedings of the NAACL HLT 2010 Workshop on Creating Speech and Language Data with Amazon\u2019s Mechanical Turk (CSLDAMT 2010), pp. 184\u2013187. Association for Computational Linguistics (2010)"},{"key":"9_CR15","unstructured":"Larson, M., Eskevich, M., Ordelman, R., Kofler, C., Schmiedeke, S., Jones, G.J.F.: Overview of Mediaeval 2011 Rich Speech Retrieval Task and Genre Tagging Task. In: MediaEval 2011 Workshop Notes Proceedings, vol.\u00a0807. CEUR-WS.org (2011)"},{"issue":"3","key":"9_CR16","doi-asserted-by":"publisher","first-page":"15","DOI":"10.1109\/MMUL.2012.27","volume":"19","author":"M. Larson","year":"2012","unstructured":"Larson, M., Soleymani, M., Eskevich, M., Serdyukov, P., Ordelman, R., Jones, G.J.F.: The Community and the Crowd: Multimedia Benchmark Dataset Development. IEEE Multimedia\u00a019(3), 15\u201323 (2012)","journal-title":"IEEE Multimedia"},{"key":"9_CR17","doi-asserted-by":"publisher","first-page":"23","DOI":"10.1145\/2380116.2380122","volume-title":"Proceedings of 25th ACM Symposium on User Interface Software and Technology (UIST 2012)","author":"W.S. Lasecki","year":"2012","unstructured":"Lasecki, W.S., Miller, C.D., Sadilek, A., Abumoussa, A., Borrello, D., Kushalnagar, R., Bigham, J.P.: Real-time captioning by groups of non-experts. In: Proceedings of 25th ACM Symposium on User Interface Software and Technology (UIST 2012), pp. 23\u201334. ACM, Cambridge (2012)"},{"key":"9_CR18","unstructured":"Liao, S., Wu, C., Huerta, J.M.: Evaluating human correction quality for machine translation from crowdsourcing. In: Recent Advances in Natural Language Processing (RANLP 2011), pp. 598\u2013603 (2011)"},{"key":"9_CR19","doi-asserted-by":"crossref","unstructured":"Lloret, E., Plaza, L., Aker, A.: Analyzing the capabilities of crowdsourcing services for text summarization. In: Language Resources and Evaluation (LRE) (2012)","DOI":"10.1007\/s10579-012-9198-8"},{"key":"9_CR20","doi-asserted-by":"crossref","unstructured":"Marge, M., Banerjee, S., Rudnicky, A.I.: Using the Amazon Mechanical Turk for transcription of spoken language. In: Proceedings of the IEEE International Conference on Acoustics, Speech, and Signal Processing (ICASSP 2010), pp. 5270\u20135273. IEEE (2010)","DOI":"10.1109\/ICASSP.2010.5494979"},{"key":"9_CR21","doi-asserted-by":"publisher","first-page":"77","DOI":"10.1145\/1600150.1600175","volume-title":"Proceedings of the ACM SIGKDD Workshop on Human Computation (HCOMP 2009)","author":"W. Mason","year":"2009","unstructured":"Mason, W., Watts, D.J.: Financial incentives and the \u201cperformance of crowds\u201d. In: Proceedings of the ACM SIGKDD Workshop on Human Computation (HCOMP 2009), pp. 77\u201385. ACM, New York (2009)"},{"key":"9_CR22","doi-asserted-by":"crossref","unstructured":"McGraw, I., Cyphers, S., Pasupat, P., Liu, J., Glass, J.: Automating crowd-supervised learning for spoken language systems. In: Proceedings of Interspeech 2012 (2012)","DOI":"10.21437\/Interspeech.2012-568"},{"key":"9_CR23","first-page":"207","volume-title":"The 2010 Annual Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies (HLT 2010)","author":"S. Novotney","year":"2010","unstructured":"Novotney, S., Callison-Burch, C.: Cheap, fast and good enough: automatic speech recognition with non-expert transcription. In: The 2010 Annual Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies (HLT 2010), pp. 207\u2013215. Association for Computational Linguistics, Stroudsburg (2010)"},{"key":"9_CR24","doi-asserted-by":"crossref","unstructured":"Ogata, J., Goto, M.: PodCastle: Collaborative training of language models on the basis of the wisdom of crowds. In: Proceedings of Interspeech 2012 (2012)","DOI":"10.21437\/Interspeech.2012-621"},{"issue":"5","key":"9_CR25","doi-asserted-by":"crossref","first-page":"411","DOI":"10.1017\/S1930297500002205","volume":"5","author":"G. Paolacci","year":"2010","unstructured":"Paolacci, G., Chandler, J., Ipeirotis, P.G.: Running Experiments on Amazon Mechanical Turk. Judgment and Decision Making\u00a05(5), 411\u2013419 (2010)","journal-title":"Judgment and Decision Making"},{"key":"9_CR26","doi-asserted-by":"crossref","unstructured":"Parent, G., Eskenazi, M.: Speaking to the Crowd: looking at past achievements in using crowdsourcing for speech and predicting future challenges. In: Proceedings of Interspeech 2011, pp. 3037\u20133040 (2011)","DOI":"10.21437\/Interspeech.2011-760"},{"issue":"6055","key":"9_CR27","doi-asserted-by":"publisher","first-page":"509","DOI":"10.1126\/science.1205869","volume":"334","author":"G. Pickard","year":"2011","unstructured":"Pickard, G., Pan, W., Rahwan, I., Cebrian, M., Crane, R., Madan, A., Pentland, A.: Time-critical social mobilization. Science\u00a0334(6055), 509\u2013512 (2011)","journal-title":"Science"},{"key":"9_CR28","first-page":"139","volume-title":"Proceedings of the NAACL HLT 2010 Workshop on Creating Speech and Language Data with Amazon\u2019s Mechanical Turk (CSLDAMT 2010)","author":"C. Rashtchian","year":"2010","unstructured":"Rashtchian, C., Young, P., Hodosh, M., Hockenmaier, J.: Collecting image annotations using Amazon\u2019s Mechanical Turk. In: Proceedings of the NAACL HLT 2010 Workshop on Creating Speech and Language Data with Amazon\u2019s Mechanical Turk (CSLDAMT 2010), pp. 139\u2013147. Association for Computational Linguistics, Stroudsburg (2010)"},{"key":"9_CR29","first-page":"1297","volume":"11","author":"V.C. Rayker","year":"2010","unstructured":"Rayker, V.C., Yu, S., Zhao, L.H., Hermosillo Valadez, G., Floring, C., Bogoni, L., May, L.: Learning from crowds. Journal of Machine Learning Research\u00a011, 1297\u20131322 (2010)","journal-title":"Journal of Machine Learning Research"},{"key":"9_CR30","doi-asserted-by":"publisher","first-page":"2863","DOI":"10.1145\/1753846.1753873","volume-title":"Proceedings of the 28th of the International Conference Extended Abstracts on Human Factors in Computing Systems (CHI EA 2010)","author":"J. Ross","year":"2010","unstructured":"Ross, J., Irani, L., Silberman, M.S., Zaldivar, A., Tomlinson, B.: Who are the crowdworkers?: shifting demographics in Mechanical Turk. In: Proceedings of the 28th of the International Conference Extended Abstracts on Human Factors in Computing Systems (CHI EA 2010), pp. 2863\u20132872. ACM, New York (2010)"},{"key":"9_CR31","doi-asserted-by":"publisher","first-page":"614","DOI":"10.1145\/1401890.1401965","volume-title":"Proceedings of the 14th ACM SIGKDD International Conference on Knowledge Discovery and Data Mining (KDD 2008)","author":"V.S. Sheng","year":"2008","unstructured":"Sheng, V.S., Provost, F., Ipeirotis, P.G.: Get another label? improving data quality and data mining using multiple, noisy labelers. In: Proceedings of the 14th ACM SIGKDD International Conference on Knowledge Discovery and Data Mining (KDD 2008), pp. 614\u2013622. ACM, New York (2008)"},{"key":"9_CR32","doi-asserted-by":"crossref","unstructured":"Snoek, C.G., Freiburg, B., Oomen, J., Ordelman, R.: Crowdsourcing rock n\u2019 roll multimedia retrieval. In: Proceedings of the 18th ACM International Conference on Multimedia (ACM MM 2010), pp. 1535\u20131538. ACM (2010)","DOI":"10.1145\/1873951.1874278"},{"key":"9_CR33","doi-asserted-by":"crossref","first-page":"254","DOI":"10.3115\/1613715.1613751","volume-title":"Proceedings of the Conference on Empirical Methods in Natural Language Processing (EMNLP 2008)","author":"R. Snow","year":"2008","unstructured":"Snow, R., O\u2019Connor, B., Jurafsky, D., Ng, A.Y.: Cheap and fast \u2014 but is it good?: Evaluating non-expert annotations for natural language tasks. In: Proceedings of the Conference on Empirical Methods in Natural Language Processing (EMNLP 2008), pp. 254\u2013263. Association for Computational Linguistics, Stroudsburg (2008)"},{"key":"9_CR34","unstructured":"Soleymani, M., Larson, M.: Crowdsourcing for Affective Annotation of Video: Development of a Viewer-reported Boredom Corpus. In: Carvalho, V., Lease, M., Yilmaz (eds.) Proceedings of the SIGIR 2010 Workshop on Crowdsourcing for Search Evaluation (CSE 2010). ACM (2010)"},{"key":"9_CR35","doi-asserted-by":"crossref","unstructured":"Sorokin, A., Forsyth, D.: Utility data annotation with Amazon Mechanical Turk. In: Proceedings of the First IEEE Workshop on Internet Vision at CVPR 2008, pp. 1\u20138. IEEE (2008)","DOI":"10.1109\/CVPRW.2008.4562953"},{"issue":"6","key":"9_CR36","doi-asserted-by":"publisher","first-page":"92","DOI":"10.1109\/MC.2006.196","volume":"39","author":"L. Ahn von","year":"2006","unstructured":"von Ahn, L.: Games with a Purpose. Computer\u00a039(6), 92\u201394 (2006)","journal-title":"Computer"},{"key":"9_CR37","doi-asserted-by":"crossref","first-page":"319","DOI":"10.1145\/985692.985733","volume-title":"Proceedings of the SIGCHI Conference on Human Factors in Computing Systems (CHI 2004)","author":"L. Ahn von","year":"2004","unstructured":"von Ahn, L., Dabbish, L.: Labeling images with a computer game. In: Proceedings of the SIGCHI Conference on Human Factors in Computing Systems (CHI 2004), pp. 319\u2013326. ACM, New York (2004)"},{"key":"9_CR38","unstructured":"Wang, J., Ipeirotis, P.G., Provost, F.: Managing crowdsourcing workers. In: Proceedings of the Winter Conference on Business Intelligence (2011)"},{"key":"9_CR39","doi-asserted-by":"publisher","first-page":"227","DOI":"10.1145\/2207676.2207709","volume-title":"Proceedings of the 2012 ACM Annual Conference on Human Factors in Computing Systems (CHI 2012)","author":"W. Willett","year":"2012","unstructured":"Willett, W., Heer, J., Agrawala, M.: Strategies for crowdsourcing social data analysis. In: Proceedings of the 2012 ACM Annual Conference on Human Factors in Computing Systems (CHI 2012), pp. 227\u2013236. ACM, New York (2012)"},{"key":"9_CR40","doi-asserted-by":"crossref","unstructured":"Wu, S.-Y., Thawonmas, R., Chen, K.-T.: Video summarization via crowdsourcing. In: Extended Abstracts on Human Factors in Computing Systems (CHI 2011), pp. 1531\u20131536. ACM (2011)","DOI":"10.1145\/1979742.1979803"},{"key":"9_CR41","doi-asserted-by":"crossref","first-page":"77","DOI":"10.1145\/1814433.1814443","volume-title":"Proceedings of the 8th International Conference on Mobile Systems, Applications, and Services (MobiSys 2010)","author":"T. Yan","year":"2010","unstructured":"Yan, T., Kumar, V., Ganesan, D.: Crowdsearch: exploiting crowds for accurate real-time image search on mobile phones. In: Proceedings of the 8th International Conference on Mobile Systems, Applications, and Services (MobiSys 2010), pp. 77\u201390. ACM, New York (2010)"}],"container-title":["Lecture Notes in Computer Science","Information Retrieval Meets Information Visualization"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-642-36415-0_9","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2023,6,28]],"date-time":"2023-06-28T22:30:52Z","timestamp":1687991452000},"score":1,"resource":{"primary":{"URL":"http:\/\/link.springer.com\/10.1007\/978-3-642-36415-0_9"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2013]]},"ISBN":["9783642364143","9783642364150"],"references-count":41,"URL":"https:\/\/doi.org\/10.1007\/978-3-642-36415-0_9","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"type":"print","value":"0302-9743"},{"type":"electronic","value":"1611-3349"}],"subject":[],"published":{"date-parts":[[2013]]}}}