{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,11,18]],"date-time":"2025-11-18T12:14:29Z","timestamp":1763468069711,"version":"3.40.3"},"publisher-location":"Berlin, Heidelberg","reference-count":20,"publisher":"Springer Berlin Heidelberg","isbn-type":[{"type":"print","value":"9783642289965"},{"type":"electronic","value":"9783642289972"}],"license":[{"start":{"date-parts":[[2012,1,1]],"date-time":"2012-01-01T00:00:00Z","timestamp":1325376000000},"content-version":"tdm","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2012]]},"DOI":"10.1007\/978-3-642-28997-2_16","type":"book-chapter","created":{"date-parts":[[2012,4,5]],"date-time":"2012-04-05T08:36:53Z","timestamp":1333615013000},"page":"182-194","source":"Crossref","is-referenced-by-count":35,"title":["On Aggregating Labels from Multiple Crowd Workers to Infer Relevance of Documents"],"prefix":"10.1007","author":[{"given":"Mehdi","family":"Hosseini","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Ingemar J.","family":"Cox","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Nata\u0161a","family":"Mili\u0107-Frayling","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Gabriella","family":"Kazai","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Vishwa","family":"Vinay","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","reference":[{"key":"16_CR1","doi-asserted-by":"crossref","unstructured":"Kazai, G., Kamps, J., Koolen, M., Milic-Frayling, N.: Crowdsourcing for Book Search Evaluation: Impact of HIT Design on Comparative System Ranking. In: Proceedings of the 34th International ACM SIGIR Conference on Research and Development in Information, pp. 205\u2013214 (2011)","DOI":"10.1145\/2009916.2009947"},{"key":"16_CR2","unstructured":"Alonso, O., Mizzaro, S.: Can we get rid of TREC assessors? Using Mechanical Turk for relevance assessment. In: SIGIR 2009: Workshop on the Future of IR Evaluation, Boston (2009)"},{"key":"16_CR3","doi-asserted-by":"crossref","unstructured":"Smucker, M.D., Jethani, C.P.: Measuring assessor accuracy: a comparison of nist assessors and user study participants. In: Proceeding of the 34th International ACM SIGIR Conference on Research and Development in Information Retrieval, SIGIR 2011, Beijing, China, July 25-29, pp. 1231\u20131232 (2011)","DOI":"10.1145\/2009916.2010134"},{"issue":"4","key":"16_CR4","doi-asserted-by":"publisher","first-page":"291","DOI":"10.1108\/eb026436","volume":"23","author":"C.A. Cuadra","year":"1967","unstructured":"Cuadra, C.A., Katter, R.V.: Opening the Black Box of \u2018Relevance\u2019. Journal of Documentation\u00a023(4), 291\u2013303 (1967)","journal-title":"Journal of Documentation"},{"issue":"5","key":"16_CR5","doi-asserted-by":"publisher","first-page":"697","DOI":"10.1016\/S0306-4573(00)00010-8","volume":"36","author":"E. Voorhees","year":"2000","unstructured":"Voorhees, E.: Variations in relevance judgments and the measurement of retrieval effectiveness. Inf. Process. Manage.\u00a036(5), 697\u2013716 (2000)","journal-title":"Inf. Process. Manage."},{"key":"16_CR6","doi-asserted-by":"crossref","unstructured":"Buckley, C., Voorhees, E.: Evaluating evaluation measure stability. In: Proceedings of SIGIR, pp. 33\u201340 (2000)","DOI":"10.1145\/345508.345543"},{"key":"16_CR7","doi-asserted-by":"crossref","unstructured":"Carterette, B., Pavlu, V., Kanoulas, E., Aslam, J.A., Allen, J.: Evaluation Over Thousands of Queries. In: Proceedings of SIGIR, pp. 651\u2013658 (2008)","DOI":"10.1145\/1390334.1390445"},{"key":"16_CR8","doi-asserted-by":"crossref","unstructured":"Aslam, J.A., Pavlu, V., Yilmaz, E.: A Statistical Method for System Evaluation Using Incomplete Judgments. In: Proceedings of SIGIR, pp. 541\u2013548 (2006)","DOI":"10.1145\/1148170.1148263"},{"key":"16_CR9","doi-asserted-by":"crossref","unstructured":"Carterette, B., Soboroff, I.: The effect of assessor error on IR system evaluation. In: Proceeding of the 33rd International ACM SIGIR Conference on Research and Development in Information Retrieval, pp. 539\u2013546 (2010)","DOI":"10.1145\/1835449.1835540"},{"key":"16_CR10","doi-asserted-by":"crossref","unstructured":"Bailey, P., et al.: Relevance assessment: are judges exchangeable and does it matter. In: Proceedings of the 31st Annual International ACM SIGIR Conference on Research and Development in Information Retrieval, pp. 667\u2013674 (2008)","DOI":"10.1145\/1390334.1390447"},{"key":"16_CR11","doi-asserted-by":"crossref","unstructured":"Scholer, F., Turpin, A., Sanderson, M.: Quantifying Test Collection Quality Based on the Consistency of Relevance Judgements. In: Proceeding of the 34th International ACM SIGIR Conference on Research and Development in Information Retrieval, pp. 1063\u20131072 (2011)","DOI":"10.1145\/2009916.2010057"},{"key":"16_CR12","doi-asserted-by":"crossref","unstructured":"Winter, M., Duncan, W.: Financial incentives and the \u201dperformance of crowd\u201d. In: Proceedings of the ACM SIGKDD Workshop on Human Computation, pp. 77\u201385 (2009)","DOI":"10.1145\/1600150.1600175"},{"key":"16_CR13","doi-asserted-by":"crossref","unstructured":"Snow, R., O\u2019Connor, B., Urafsky, D., Ng, A.Y.: Cheap and fast\u2014but is it good?: evaluating non-expert annotations for natural language tasks. In: Proceedings of the Conference on Empirical Methods in Natural Language Processing, Stroudsburg, PA, USA, pp. 254\u2013263 (2008)","DOI":"10.3115\/1613715.1613751"},{"key":"16_CR14","unstructured":"Smucker, M.D., Jethani, C.P.: The Crowd vs. the Lab: A Comparison of Crowd-Sourced and University Laboratory Participant Behavior. In: Proceedings of the SIGIR 2011 Workshop on Crowdsourcing for Information Retrieval, Beijing (2011)"},{"key":"16_CR15","unstructured":"Kumar, A., Lease, M.: Modeling Annotator Accuracies for Supervised Learning. In: WSDM 2011 Workshop on Crowdsourcing for Search and Data Mining, Hong Kong (2011)"},{"key":"16_CR16","doi-asserted-by":"crossref","unstructured":"Kasneci, G., Gael, J.V., Stern, D.H., Graepel, T.: CoBayes: bayesian knowledge corroboration with assessors of unknown areas of expertise. In: Proceedings of the Forth International Conference on Web Search and Web Data Mining, pp. 465\u2013474 (2011)","DOI":"10.1145\/1935826.1935896"},{"key":"16_CR17","doi-asserted-by":"crossref","unstructured":"Welinder, P., Perona, P.: Online crowdsourcing: rating annotators and obtaining cost-effective labels. In: CVPR 2010: IEEE Conference on Computer Vision and Pattern, pp. 1526\u20131534 (2010)","DOI":"10.1109\/CVPRW.2010.5543189"},{"key":"16_CR18","doi-asserted-by":"crossref","unstructured":"Ipeirotis, P.G., Provost, F., Wang, J.: Quality Management on Amazon Mechanical Turk. In: Proceedings of the ACM SIGKDD Workshop on Human Computation, pp. 64\u201367 (2010)","DOI":"10.1145\/1837885.1837906"},{"issue":"1","key":"16_CR19","doi-asserted-by":"publisher","first-page":"20","DOI":"10.2307\/2346806","volume":"28","author":"P. Dawid","year":"1979","unstructured":"Dawid, P., Skene, A.M.: Maximum Likelihood Estimation of Observer Error-Rates Using the EM Algorithm. Applied Statistics\u00a028(1), 20\u201328 (1979)","journal-title":"Applied Statistics"},{"key":"16_CR20","doi-asserted-by":"crossref","unstructured":"Bernstein, Y., Zobel, J.: Redundant documents and search effectiveness. In: Proceedings of the 14th ACM International Conference on Information and Knowledge Management, pp. 736\u2013743 (2005)","DOI":"10.1145\/1099554.1099733"}],"container-title":["Lecture Notes in Computer Science","Advances in Information Retrieval"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-642-28997-2_16","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,3,25]],"date-time":"2025-03-25T14:15:40Z","timestamp":1742912140000},"score":1,"resource":{"primary":{"URL":"http:\/\/link.springer.com\/10.1007\/978-3-642-28997-2_16"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2012]]},"ISBN":["9783642289965","9783642289972"],"references-count":20,"URL":"https:\/\/doi.org\/10.1007\/978-3-642-28997-2_16","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"type":"print","value":"0302-9743"},{"type":"electronic","value":"1611-3349"}],"subject":[],"published":{"date-parts":[[2012]]}}}