{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,10,1]],"date-time":"2025-10-01T15:30:43Z","timestamp":1759332643658},"publisher-location":"Berlin, Heidelberg","reference-count":15,"publisher":"Springer Berlin Heidelberg","isbn-type":[{"type":"print","value":"9783642201608"},{"type":"electronic","value":"9783642201615"}],"license":[{"start":{"date-parts":[[2011,1,1]],"date-time":"2011-01-01T00:00:00Z","timestamp":1293840000000},"content-version":"unspecified","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2011]]},"DOI":"10.1007\/978-3-642-20161-5_16","type":"book-chapter","created":{"date-parts":[[2011,4,12]],"date-time":"2011-04-12T07:41:23Z","timestamp":1302594083000},"page":"153-164","source":"Crossref","is-referenced-by-count":42,"title":["Design and Implementation of Relevance Assessments Using Crowdsourcing"],"prefix":"10.1007","author":[{"given":"Omar","family":"Alonso","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Ricardo","family":"Baeza-Yates","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","reference":[{"issue":"6","key":"16_CR1","doi-asserted-by":"publisher","first-page":"92","DOI":"10.1109\/MC.2006.196","volume":"39","author":"L. Ahn von","year":"2006","unstructured":"von Ahn, L.: Games with a purpose. IEEE Computer\u00a039(6), 92\u201394 (2006)","journal-title":"IEEE Computer"},{"key":"16_CR2","unstructured":"Alonso, O., Mizzaro, S.: Can we get rid of TREC Assessors? Using Mechanical Turk for Relevance Assessment. In: SIGIR Workshop Future of IR Evaluation (2009)"},{"key":"16_CR3","doi-asserted-by":"crossref","unstructured":"Alonso, O., Schenkel, R., Theobald, M.: Crowdsourcing Assessments for XML Ranked Retrieval. In: 32 ECIR, Milton Keynes, UK (2010)","DOI":"10.1007\/978-3-642-12275-0_57"},{"key":"16_CR4","unstructured":"Alonso, O., Baeza-Yates, R.: An Analysis of Crowdsourcing Relevance Assessments in Spanish. In: CERI 2010, Madrid, Spain (2010)"},{"key":"16_CR5","unstructured":"Bradburn, N., Sudman, S., Wansink, B.: Asking Questions: The Definitive Guide to Questionnaire Design. Josey-Bass (2004)"},{"key":"16_CR6","doi-asserted-by":"crossref","unstructured":"Callison-Burch, C.: Fast, Cheap, and Creative: Evaluating Translation Quality Using Amazon\u2019s Mechanical Turk. In: Proceedings of EMNLP (2009)","DOI":"10.3115\/1699510.1699548"},{"key":"16_CR7","unstructured":"Grady, C., Lease, M.: Crowdsourcing Document Relevance Assessment with Mechanical Turk. In: NAACL HLT Workshop on Creating Speech and Language Data with Amazons Mechanical Turk (2010)"},{"key":"16_CR8","doi-asserted-by":"crossref","unstructured":"Kazai, G., Milic-Frayling, N., Costello, J.: Towards Methods for the Collective Gathering and Quality Control of Relevance Assessments. In: 32 SIGIR (2009)","DOI":"10.1145\/1571941.1572019"},{"key":"16_CR9","doi-asserted-by":"crossref","unstructured":"Kinney, K., Huffman, S., Zhai, J.: How Evaluator Domain Expertise Affects Search Result Relevance Judgments. In: 17 CIKM (2008)","DOI":"10.1145\/1458082.1458160"},{"key":"16_CR10","volume-title":"Harnessing Crowds: Mapping the Genome of Collective Intelligence","author":"T.W. Malone","year":"2009","unstructured":"Malone, T.W., Laubacher, R., Dellarocas, C.: Harnessing Crowds: Mapping the Genome of Collective Intelligence. MIT Press, Cambridge (2009)"},{"key":"16_CR11","doi-asserted-by":"crossref","unstructured":"Mason, W., Watts, D.: Financial Incentives and the \u2018Performance of Crowds\u2019. In: HCOMP Workshop at KDD, Paris, France (2009)","DOI":"10.1145\/1600150.1600175"},{"key":"16_CR12","doi-asserted-by":"crossref","unstructured":"Nov, O., Naaman, M., Ye, C.: What Drives Content Tagging: The Case of Photos on Flickr. In: CHI, Florence, Italy (2008)","DOI":"10.1145\/1357054.1357225"},{"key":"16_CR13","doi-asserted-by":"crossref","unstructured":"Snow, R., O\u2019 Connor, B., Jurafsky, D., Ng, A.Y.: Cheap and Fast - But is it Good? Evaluating Non-Expert Annotations for Natural Language Tasks. In: EMNLP (2008)","DOI":"10.3115\/1613715.1613751"},{"key":"16_CR14","doi-asserted-by":"crossref","unstructured":"Tang, J., Sanderson, M.: Evaluation and User Preference Study on Spatial Diversity. In: 32 ECIR, Milton Keynes, UK (2010)","DOI":"10.1007\/978-3-642-12275-0_18"},{"key":"16_CR15","unstructured":"Voorhees, E.: Personal communication (2009)"}],"container-title":["Lecture Notes in Computer Science","Advances in Information Retrieval"],"original-title":[],"link":[{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-642-20161-5_16","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2019,5,23]],"date-time":"2019-05-23T00:58:22Z","timestamp":1558573102000},"score":1,"resource":{"primary":{"URL":"http:\/\/link.springer.com\/10.1007\/978-3-642-20161-5_16"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2011]]},"ISBN":["9783642201608","9783642201615"],"references-count":15,"URL":"https:\/\/doi.org\/10.1007\/978-3-642-20161-5_16","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"type":"print","value":"0302-9743"},{"type":"electronic","value":"1611-3349"}],"subject":[],"published":{"date-parts":[[2011]]}}}