{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,9,27]],"date-time":"2026-09-27T01:43:23Z","timestamp":1790473403650,"version":"4.1.0"},"publisher-location":"Berlin, Heidelberg","reference-count":13,"publisher":"Springer Berlin Heidelberg","isbn-type":[{"value":"9783642201608","type":"print"},{"value":"9783642201615","type":"electronic"}],"license":[{"start":{"date-parts":[[2011,1,1]],"date-time":"2011-01-01T00:00:00Z","timestamp":1293840000000},"content-version":"unspecified","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2011]]},"DOI":"10.1007\/978-3-642-20161-5_17","type":"book-chapter","created":{"date-parts":[[2011,4,12]],"date-time":"2011-04-12T03:41:23Z","timestamp":1302579683000},"page":"165-176","source":"Crossref","is-referenced-by-count":85,"title":["In Search of Quality in Crowdsourcing for Search Engine Evaluation"],"prefix":"10.1007","author":[{"given":"Gabriella","family":"Kazai","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","reference":[{"key":"17_CR1","unstructured":"Alonso, O., Mizzaro, S.: Can we get rid of TREC assessors? using Mechanical Turk for relevance assessment. In: Proceedings of the SIGIR 2009 Workshop on the Future of IR Evaluation, pp. 557\u2013566 (2009)"},{"issue":"2","key":"17_CR2","doi-asserted-by":"publisher","first-page":"9","DOI":"10.1145\/1480506.1480508","volume":"42","author":"O. Alonso","year":"2008","unstructured":"Alonso, O., Rose, D.E., Stewart, B.: Crowdsourcing for relevance evaluation. SIGIR Forum\u00a042(2), 9\u201315 (2008)","journal-title":"SIGIR Forum"},{"key":"17_CR3","doi-asserted-by":"crossref","unstructured":"Bailey, P., Craswell, N., Soboroff, I., Thomas, P., de Vries, A.P., Yilmaz, E.: Relevance assessment: are judges exchangeable and does it matter. In: SIGIR 2008: Proceedings of the 31st Annual International ACM SIGIR Conference, pp. 667\u2013674 (2008)","DOI":"10.1145\/1390334.1390447"},{"key":"17_CR4","unstructured":"Grady, C., Lease, M.: Crowdsourcing document relevance assessment with mechanical turk. In: Proceedings of the NAACL HLT 2010 Workshop on Creating Speech and Language Data with Amazon\u2019s Mechanical Turk, pp. 172\u2013179 (2010)"},{"key":"17_CR5","volume-title":"Crowdsourcing: Why the Power of the Crowd Is Driving the Future of Business","author":"J. Howe","year":"2008","unstructured":"Howe, J.: Crowdsourcing: Why the Power of the Crowd Is Driving the Future of Business. Crown Publishing Group, New York (2008)"},{"key":"17_CR6","unstructured":"Kapelner, A., Chandler, D.: Preventing satisficing in online surveys: A \u2018kapcha\u2019 to ensure higher quality data. In: The World\u2019s First Conference on the Future of Distributed Work (CrowdConf 2010) (2010)"},{"key":"17_CR7","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"145","DOI":"10.1007\/978-3-642-14556-8_16","volume-title":"Focused Retrieval and Evaluation","author":"G. Kazai","year":"2010","unstructured":"Kazai, G., Koolen, M., Doucet, A., Landoni, M.: Overview of the INEX 2009 book track. In: Geva, S., Kamps, J., Trotman, A. (eds.) INEX 2009. LNCS, vol.\u00a06203, pp. 145\u2013159. Springer, Heidelberg (2010)"},{"key":"17_CR8","unstructured":"Le, J., Edmonds, A., Hester, V., Biewald, L.: Ensuring quality in crowdsourced search relevance evaluation. In: SIGIR Workshop on Crowdsourcing for Search Evaluation, pp. 17\u201320 (2010)"},{"key":"17_CR9","first-page":"24","volume":"18","author":"P. Marsden","year":"2009","unstructured":"Marsden, P.: Crowdsourcing. Contagious Magazine\u00a018, 24\u201328 (2009)","journal-title":"Contagious Magazine"},{"key":"17_CR10","doi-asserted-by":"crossref","unstructured":"Mason, W., Watts, D.J.: Financial incentives and the \u201dperformance of crowds\u201d. In: HCOMP 2009: Proceedings of the ACM SIGKDD Workshop on Human Computation, pp. 77\u201385 (2009)","DOI":"10.1145\/1600150.1600175"},{"key":"17_CR11","unstructured":"Quinn, A.J., Bederson, B.B.: A taxonomy of distributed human computation. Technical Report HCIL-2009-23, University of Maryland (2009)"},{"key":"17_CR12","doi-asserted-by":"crossref","unstructured":"von Ahn, L., Dabbish, L.: Labeling images with a computer game. In: Proceedings of the SIGCHI Conference on Human Factors in Computing Systems, CHI 2004, pp. 319\u2013326 (2004)","DOI":"10.1145\/985692.985733"},{"key":"17_CR13","unstructured":"Zhu, D., Carterette, B.: An analysis of assessor behavior in crowdsourced preference judgments. In: SIGIR 2010 Workshop on Crowdsourcing for Search Evaluation (2010)"}],"container-title":["Lecture Notes in Computer Science","Advances in Information Retrieval"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-642-20161-5_17","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2019,5,22]],"date-time":"2019-05-22T20:34:26Z","timestamp":1558557266000},"score":1,"resource":{"primary":{"URL":"http:\/\/link.springer.com\/10.1007\/978-3-642-20161-5_17"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2011]]},"ISBN":["9783642201608","9783642201615"],"references-count":13,"URL":"https:\/\/doi.org\/10.1007\/978-3-642-20161-5_17","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"value":"0302-9743","type":"print"},{"value":"1611-3349","type":"electronic"}],"subject":[],"published":{"date-parts":[[2011]]}}}