{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,5,3]],"date-time":"2025-05-03T04:02:27Z","timestamp":1746244947486,"version":"3.40.4"},"publisher-location":"Cham","reference-count":19,"publisher":"Springer International Publishing","isbn-type":[{"type":"print","value":"9783319060279"},{"type":"electronic","value":"9783319060286"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2014]]},"DOI":"10.1007\/978-3-319-06028-6_13","type":"book-chapter","created":{"date-parts":[[2014,3,24]],"date-time":"2014-03-24T09:24:27Z","timestamp":1395653067000},"page":"148-159","source":"Crossref","is-referenced-by-count":4,"title":["Tolerance of Effectiveness Measures to Relevance Judging Errors"],"prefix":"10.1007","author":[{"given":"Le","family":"Li","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Mark D.","family":"Smucker","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","reference":[{"key":"13_CR1","unstructured":"Alonso, O., Mizzaro, S.: Can we get rid of TREC assessors? using Mechanical Turk for relevance assessment. In: Proceedings of the SIGIR 2009 Workshop on the Future of IR Evaluation, pp. 15\u201316 (July 2009)"},{"key":"13_CR2","unstructured":"McCreadie, R., Macdonald, C., Ounis, I.: Crowdsourcing blog track top news judgments at TREC. In: WSDM 2011 Workshop on Crowdsourcing for Search and Data Mining (2011)"},{"key":"13_CR3","doi-asserted-by":"crossref","unstructured":"Smucker, M.D., Kazai, G., Lease, M.: Overview of the TREC 2012 crowdsourcing track (2012)","DOI":"10.6028\/NIST.SP.500-298.crowd-overview"},{"key":"13_CR4","first-page":"697","volume":"36","author":"E.M. Voorhees","year":"2000","unstructured":"Voorhees, E.M.: Variations in relevance judgments and the measurement of retrieval effectiveness. IPM\u00a036, 697\u2013716 (2000)","journal-title":"IPM"},{"key":"13_CR5","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"182","DOI":"10.1007\/978-3-642-28997-2_16","volume-title":"Advances in Information Retrieval","author":"M. Hosseini","year":"2012","unstructured":"Hosseini, M., Cox, I., Mili\u0107-Frayling, N., Kazai, G., Vinay, V.: On aggregating labels from multiple crowd workers to infer relevance of documents. In: Baeza-Yates, R., de Vries, A.P., Zaragoza, H., Cambazoglu, B.B., Murdock, V., Lempel, R., Silvestri, F. (eds.) ECIR 2012. LNCS, vol.\u00a07224, pp. 182\u2013194. Springer, Heidelberg (2012)"},{"key":"13_CR6","first-page":"1297","volume":"99","author":"V.C. Raykar","year":"2010","unstructured":"Raykar, V.C., Yu, S., Zhao, L.H., Valadez, G.H., Florin, C., Bogoni, L., Moy, L.: Learning from crowds. The Journal of Machine Learning Research\u00a099, 1297\u20131322 (2010)","journal-title":"The Journal of Machine Learning Research"},{"key":"13_CR7","doi-asserted-by":"crossref","unstructured":"Yilmaz, E., Aslam, J.A., Robertson, S.: A new rank correlation coefficient for information retrieval. In: SIGIR, pp. 587\u2013594 (2008)","DOI":"10.1145\/1390334.1390435"},{"key":"13_CR8","doi-asserted-by":"crossref","unstructured":"Macmillan, N.A., Creelman, C.D.: Detection theory: A user\u2019s guide. Psychology Press (2004)","DOI":"10.4324\/9781410611147"},{"key":"13_CR9","doi-asserted-by":"crossref","unstructured":"Voorhees, E.M., Harman, D.: Overview of the Eighth Text REtrieval Conference (TREC-8). In: Proceedings of TREC, vol.\u00a08, pp. 1\u201324 (1999)","DOI":"10.6028\/NIST.SP.500-246"},{"key":"13_CR10","doi-asserted-by":"crossref","unstructured":"Voorhees, E.M.: Overview of TREC 2005. In: Proceedings of TREC (2005)","DOI":"10.6028\/NIST.SP.500-266.overview-overview"},{"key":"13_CR11","doi-asserted-by":"crossref","unstructured":"Smucker, M.D., Jethani, C.P.: Measuring assessor accuracy: a comparison of NIST assessors and user study participants. In: SIGIR, pp. 1231\u20131232 (2011)","DOI":"10.1145\/2009916.2010134"},{"key":"13_CR12","unstructured":"Smucker, M., Jethani, C.: The crowd vs. the lab: A comparison of crowd-sourced and university laboratory participant behavior. In: Proceedings of the SIGIR 2011 Workshop on Crowdsourcing for Information Retrieval (2011)"},{"issue":"1\/2","key":"13_CR13","doi-asserted-by":"publisher","first-page":"81","DOI":"10.2307\/2332226","volume":"30","author":"M.G. Kendall","year":"1938","unstructured":"Kendall, M.G.: A new measure of rank correlation. Biometrika\u00a030(1\/2), 81\u201393 (1938)","journal-title":"Biometrika"},{"key":"13_CR14","doi-asserted-by":"crossref","unstructured":"Carterette, B., Soboroff, I.: The effect of assessor error on IR system evaluation. In: SIGIR, pp. 539\u2013546 (2010)","DOI":"10.1145\/1835449.1835540"},{"key":"13_CR15","doi-asserted-by":"crossref","unstructured":"Webber, W., Chandar, P., Carterette, B.: Alternative assessor disagreement and retrieval depth. In: CIKM, pp. 125\u2013134 (2012)","DOI":"10.1145\/2396761.2396781"},{"key":"13_CR16","unstructured":"Vuurens, J., de Vries, A.P., Eickhoff, C.: How much spam can you take? In: SIGIR 2011 Workshop on Crowdsourcing for Information Retrieval, CIR (2011)"},{"issue":"5","key":"13_CR17","first-page":"697","volume":"36","author":"E. Voorhees","year":"2000","unstructured":"Voorhees, E.: Variations in relevance judgments and the measurement of retrieval effectiveness. IPM\u00a036(5), 697\u2013716 (2000)","journal-title":"IPM"},{"key":"13_CR18","doi-asserted-by":"crossref","unstructured":"Bailey, P., Craswell, N., Soboroff, I., Thomas, P., de Vries, A., Yilmaz, E.: Relevance assessment: are judges exchangeable and does it matter. In: SIGIR, pp. 667\u2013674 (2008)","DOI":"10.1145\/1390334.1390447"},{"key":"13_CR19","doi-asserted-by":"crossref","unstructured":"Kinney, K., Huffman, S., Zhai, J.: How evaluator domain expertise affects search result relevance judgments. In: CIKM, pp. 591\u2013598 (2008)","DOI":"10.1145\/1458082.1458160"}],"container-title":["Lecture Notes in Computer Science","Advances in Information Retrieval"],"original-title":[],"link":[{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-319-06028-6_13","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,5,2]],"date-time":"2025-05-02T04:09:41Z","timestamp":1746158981000},"score":1,"resource":{"primary":{"URL":"http:\/\/link.springer.com\/10.1007\/978-3-319-06028-6_13"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2014]]},"ISBN":["9783319060279","9783319060286"],"references-count":19,"URL":"https:\/\/doi.org\/10.1007\/978-3-319-06028-6_13","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"type":"print","value":"0302-9743"},{"type":"electronic","value":"1611-3349"}],"subject":[],"published":{"date-parts":[[2014]]}}}