{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,9,25]],"date-time":"2026-09-25T12:02:27Z","timestamp":1790337747920,"version":"4.1.0"},"reference-count":58,"publisher":"Springer Science and Business Media LLC","issue":"2","license":[{"start":{"date-parts":[[2012,7,20]],"date-time":"2012-07-20T00:00:00Z","timestamp":1342742400000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2012,7,20]],"date-time":"2012-07-20T00:00:00Z","timestamp":1342742400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Inf Retrieval"],"published-print":{"date-parts":[[2013,4]]},"DOI":"10.1007\/s10791-012-9205-0","type":"journal-article","created":{"date-parts":[[2012,7,19]],"date-time":"2012-07-19T11:05:36Z","timestamp":1342695936000},"page":"138-178","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":98,"title":["An analysis of human factors and label accuracy in crowdsourcing relevance judgments"],"prefix":"10.1007","volume":"16","author":[{"given":"Gabriella","family":"Kazai","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Jaap","family":"Kamps","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Natasa","family":"Milic-Frayling","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2012,7,20]]},"reference":[{"key":"9205_CR2","doi-asserted-by":"crossref","unstructured":"Alonso, O., & Baeza-Yates, R. A. (2011). Design and implementation of relevance assessments using crowdsourcing. In Advances in information retrieval \u201433rd European conference on IR research (ECIR 2011), LNCS, Vol. 6611 (pp. 153\u2013164). Springer.","DOI":"10.1007\/978-3-642-20161-5_16"},{"key":"9205_CR3","unstructured":"Alonso, O., & Mizzaro, S. (2009). Can we get rid of TREC assessors? using mechanical turk for relevance assessment. In Proceedings of the SIGIR 2009 workshop on the future of IR evaluation (pp. 557\u2013566)."},{"issue":"2","key":"9205_CR4","doi-asserted-by":"publisher","first-page":"9","DOI":"10.1145\/1480506.1480508","volume":"42","author":"O. Alonso","year":"2008","unstructured":"Alonso, O., Rose, D. E., & Stewart, B. (2008). Crowdsourcing for relevance evaluation. SIGIR Forum, 42(2), 9\u201315.","journal-title":"SIGIR Forum"},{"key":"9205_CR5","doi-asserted-by":"crossref","unstructured":"Alonso, O., Schenkel, R., & Theobald, M. (2010). Crowdsourcing assessments for xml ranked retrieval. In Advances in information retrieval, 32nd European conference on IR research (ECIR 2010), LNCS, Vol. 5993 (pp. 602\u2013606). Springer.","DOI":"10.1007\/978-3-642-12275-0_57"},{"key":"9205_CR6","doi-asserted-by":"crossref","unstructured":"Bailey, P., Craswell, N., Soboroff, I., Thomas, P., de Vries, A. P., & Yilmaz, E. (2008). Relevance assessment: Are judges exchangeable and does it matter. In SIGIR \u201908: Proceedings of the 31st annual international ACM SIGIR conference (pp. 667\u2013674). New York, NY: ACM.","DOI":"10.1145\/1390334.1390447"},{"key":"9205_CR7","doi-asserted-by":"crossref","unstructured":"Behrend, T. S., Sharek, D. J., Meade, A. W., & Wiebe, E. N. (2011). The viability of crowdsourcing for survey research. Behavior Research Methods.","DOI":"10.1037\/e518362013-534"},{"key":"9205_CR8","doi-asserted-by":"crossref","unstructured":"Carterette, B., & Soboroff, I. (2010). The effect of assessor error on ir system evaluation. In Proceeding of the 33rd international ACM SIGIR conference on research and development in information retrieval, SIGIR \u201910 (pp. 539\u2013546). New York, NY: ACM.","DOI":"10.1145\/1835449.1835540"},{"key":"9205_CR9","volume-title":"Principles of Economics","author":"K. E. Case","year":"2011","unstructured":"Case, K. E., Fair, R. C., & Oster, S. C. (2011). Principles of Economics (10th  ed.). Englewood Cliffs NJ: Prentice-Hall.","edition":"10"},{"issue":"4","key":"9205_CR10","doi-asserted-by":"publisher","first-page":"661","DOI":"10.1137\/070710111","volume":"51","author":"A. Clauset","year":"2009","unstructured":"Clauset, A., Shalizi, C. R., & Newman, M. E. J. (2009). Power-law distributions in empirical data. SIAM Review, 51(4), 661\u2013703.","journal-title":"SIAM Review"},{"key":"9205_CR11","doi-asserted-by":"publisher","first-page":"173","DOI":"10.1108\/eb050097","volume":"19","author":"C. W. Cleverdon","year":"1967","unstructured":"Cleverdon, C. W. (1967). The Cranfield tests on index language devices. Aslib, 19, 173\u2013192.","journal-title":"Aslib"},{"key":"9205_CR12","doi-asserted-by":"crossref","unstructured":"Cormack, G. V., Palmer, C. R., & Clarke, C. L. A. (1998). Efficient construction of large test collections. In Proceedings of the 21st annual international ACM SIGIR conference on research and development in information retrieval, SIGIR \u201998 (pp. 282\u2013289). New York, NY: ACM.","DOI":"10.1145\/290941.291009"},{"key":"9205_CR13","doi-asserted-by":"publisher","first-page":"86","DOI":"10.1145\/1924421.1924442","volume":"54","author":"A. Doan","year":"2011","unstructured":"Doan, A., Ramakrishnan, R., & Halevy, A. Y. (2011). Crowdsourcing systems on the world-wide web. Commun. ACM, 54, 86\u201396.","journal-title":"Communications of the ACM"},{"key":"9205_CR14","doi-asserted-by":"crossref","unstructured":"Downs, J. S., Holbrook, M. B., Sheng, S., & Cranor, L. F. (2010). Are your participants gaming the system? Screening mechanical turk workers. In Proceedings of the 28th international conference on human factors in computing systems (CHI \u201910)  (pp. 2399\u20132402). ACM.","DOI":"10.1145\/1753326.1753688"},{"key":"9205_CR15","unstructured":"Eickhoff C., & de Vries, A. P. (2011). How crowdsourcable is your task? In Proceedings of the workshop on crowdsourcing for search and data mining (CSDM 2011) (pp. 11\u201314). ACM."},{"key":"9205_CR16","unstructured":"Feild, H., Jones, R., Miller, R. C., Nayak, R., Churchill, E. F., & Velipasaoglu, E. (2010). Logging the search self-efficacy of Amazon mechanical turkers. In M. Lease, V. Carvalho, & E. Yilmaz (Eds.), Proceedings of the ACM SIGIR 2010 workshop on crowdsourcing for search evaluation (CSE 2010) (pp. 27\u201330). Geneva, Switzerland."},{"issue":"2","key":"9205_CR17","doi-asserted-by":"crossref","first-page":"203","DOI":"10.1037\/h0041593","volume":"58","author":"L. Festinger","year":"1959","unstructured":"Festinger, L., & Carlsmith, J. M. (1959). Cognitive consequences of forced compliance. Journal of Abnormal and Social Psychology, 58(2), 203\u2013210. http:\/\/psychclassics.yorku.ca\/Festinger\/.","journal-title":"Journal of Abnormal and Social Psychology"},{"key":"9205_CR18","doi-asserted-by":"publisher","first-page":"378","DOI":"10.1037\/h0031619","volume":"76","author":"J. L. Fleiss","year":"1971","unstructured":"Fleiss, J. L. (1971). Measuring nominal scale agreement among many raters. Psychological Bulletin, 76, 378\u2013382.","journal-title":"Psychological Bulletin"},{"key":"9205_CR19","unstructured":"Grady, C., & Lease, M. (2010). Crodsourcing document relevance assessment with mechanical turk. In Proceedings of the NAACL HLT 2010 workshop on creating speech and language data with Amazon\u2019s mechanical turk (pp. 172\u2013179)."},{"key":"9205_CR20","unstructured":"Grimes, C., Tang, D., & Russell, D. M. (2007). Query logs alone are not enough. In E. Amitay, C. G. Murray, & J. Teevan (Eds.), Query log analysis: Social and technological challenges. A workshop at the 16th International World Wide Web Conference (WWW 2007)."},{"key":"9205_CR21","doi-asserted-by":"crossref","unstructured":"Hirth, M., Ho\u00dffeld, T., & Tran-Gia, P. (2011) Anatomy of a crowdsourcing platform\u2014using the example of microworkers.com. In Workshop on future internet and next generation networks (FINGNet). Seoul, Korea.","DOI":"10.1109\/IMIS.2011.89"},{"key":"9205_CR22","volume-title":"Crowdsourcing: Why the Power of the Crowd Is Driving the Future of Business","author":"J. Howe","year":"2008","unstructured":"Howe, J. (2008). Crowdsourcing: Why the power of the crowd Is driving the future of business. New York, NY: Crown Publishing Group."},{"key":"9205_CR23","unstructured":"Ipeirotis, P. (2008). Mechanical turk: The demographics. Blog post. http:\/\/behind-the-enemy-lines.blogspot.com\/2008\/03\/mechanical-turk-demographics.html."},{"key":"9205_CR24","unstructured":"Ipeirotis, P. (2010a). The new demographics of mechanical turk. Blog post. http:\/\/behind-the-enemy-lines.blogspot.com\/2010\/03\/new-demographics-of-mechanical-turk.html."},{"key":"9205_CR25","doi-asserted-by":"publisher","first-page":"16","DOI":"10.1145\/1869086.1869094","volume":"17","author":"P. G. Ipeirotis","year":"2010","unstructured":"Ipeirotis, P. G. (2010b). Analyzing the amazon mechanical turk marketplace. XRDS, 17, 16\u201321.","journal-title":"XRDS"},{"key":"9205_CR26","doi-asserted-by":"crossref","unstructured":"Ipeirotis P. G., Provost, F., & Wang, J. (2010). Quality management on amazon mechanical turk. In Proceedings of the ACM SIGKDD workshop on human computation, HCOMP \u201910 (pp. 64\u201367). New York, NY: ACM.","DOI":"10.1145\/1837885.1837906"},{"key":"9205_CR27","doi-asserted-by":"crossref","unstructured":"Jain, S., & Parkes, D. C. (2009). The role of game theory in human computation systems. In Proceedings of the ACM SIGKDD workshop on human computation, HCOMP \u201909 (pp. 58\u201361). New York, NY: ACM.","DOI":"10.1145\/1600150.1600171"},{"key":"9205_CR28","doi-asserted-by":"crossref","unstructured":"Kamps, J., Koolen, M., & Trotman, A. (2009). Comparative analysis of clicks and judgments for IR evaluation. In Proceedings of the workshop on web search click data (WSCD 2009) (pp. 80\u201387). New York NY: ACM Press.","DOI":"10.1145\/1507509.1507522"},{"key":"9205_CR29","unstructured":"Kapelner, A., & Chandler, D. (2010) Preventing satisficing in online surveys: A \u2018kapcha\u2019 to ensure higher quality data. In The world\u2019s first conference on the future of distributed work (CrowdConf2010)."},{"key":"9205_CR30","doi-asserted-by":"crossref","unstructured":"Kasneci, G., Van Gael, J., Herbrich, R., & Graepel, T. (2010). Bayesian knowledge corroboration with logical rules and user feedback. In Proceedings of the 2010 European conference on machine learning and knowledge discovery in databases: Part II, ECML PKDD\u201910 (pp. 1\u201318). Berlin : Springer.","DOI":"10.1007\/978-3-642-15883-4_1"},{"key":"9205_CR31","doi-asserted-by":"crossref","unstructured":"Kazai, G. (2011). In search of quality in crowdsourcing for search engine evaluation. In Advances in Information retrieval \u201433rd European conference on IR research (ECIR 2011), LNCS , Vol. 6611 (pp. 165\u2013176). Springer.","DOI":"10.1007\/978-3-642-20161-5_17"},{"key":"9205_CR32","doi-asserted-by":"crossref","unstructured":"Kazai, G., Doucet, A., & Landoni, M. (2008). Overview of the inex 2008 book track. In INEX (pp. 106\u2013123).","DOI":"10.1007\/978-3-642-03761-0_12"},{"key":"9205_CR33","doi-asserted-by":"crossref","unstructured":"Kazai, G., Kamps, J., Koolen, M., & Milic-Frayling, N. (2011a). Crowdsourcing for book search evaluation: Impact of quality on comparative system ranking. In Proceedings of the 34th annual international ACM SIGIR conference on research and development in information retrieval. ACM.","DOI":"10.1145\/2009916.2009947"},{"key":"9205_CR34","doi-asserted-by":"crossref","unstructured":"Kazai, G., Kamps, J., & Milic-Frayling, N. (2011b). Worker types and personality traits in crowdsourcing relevance labels. In Proceedings of the 20th ACM international conference on Information and knowledge management (pp. 1941\u20131944). ACM.","DOI":"10.1145\/2063576.2063860"},{"key":"9205_CR35","doi-asserted-by":"crossref","unstructured":"Kazai, G., Milic-Frayling, N., & Costello, J. (2009). Towards methods for the collective gathering and quality control of relevance assessments. In Proceedings of the 32nd international ACM SIGIR conference on research and development in information retrieval, SIGIR \u201909 (pp. 452\u2013459). New York, NY: ACM.","DOI":"10.1145\/1571941.1572019"},{"key":"9205_CR36","doi-asserted-by":"crossref","unstructured":"Kittur, A., Chi, E. H., & Suh, B. (2008). Crowdsourcing user studies with mechanical turk. In Proceeding of the twenty-sixth annual SIGCHI conference on human factors in computing systems (CHI \u201908) (pp. 453\u2013456). ACM.","DOI":"10.1145\/1357054.1357127"},{"issue":"1","key":"9205_CR37","doi-asserted-by":"publisher","first-page":"159","DOI":"10.2307\/2529310","volume":"33","author":"J. R. Landis","year":"1977","unstructured":"Landis, J. R., & Koch, G. G. (1977). The measurement of observer agreement for categorical data. Biometrics, 33(1), 159\u2013174.","journal-title":"Biometrics"},{"key":"9205_CR38","unstructured":"Le, J., Edmonds, A., Hester, V., & Biewald, L. (2010) Ensuring quality in crowdsourced search relevance evaluation. In V. Carvalho, M. Lease, & E. Yilmaz (Eds.), SIGIR Workshop on crowdsourcing for search evaluation (pp. 17\u201320). New York, NY: ACM."},{"key":"9205_CR39","unstructured":"Lease, M. (2011). On quality control and machine learning in crowdsourcing. In Proceedings of the 3rd human computation workshop (HCOMP) at AAAI (pp. 97\u2013102)."},{"key":"9205_CR40","unstructured":"Lease, M., & Kazai, G. (2011). Overview of the trec 2011 crowdsourcing track. In Proceedings of the text retrieval conference (TREC)."},{"key":"9205_CR41","first-page":"24","volume":"18","author":"P. Marsden","year":"2009","unstructured":"Marsden, P. (2009). Crowdsourcing. Contagious Magazine, 18, 24\u201328.","journal-title":"Contagious Magazine"},{"key":"9205_CR42","doi-asserted-by":"crossref","unstructured":"Mason, W., & Suri, S. (2011). Conducting behavioral research on amazons mechanical turk. Behavior Research Methods.","DOI":"10.3758\/s13428-011-0124-6"},{"key":"9205_CR43","doi-asserted-by":"crossref","unstructured":"Mason, W., & Watts, D. J. (2009). Financial incentives and the \u201cperformance of crowds\u201d. In HCOMP \u201909: Proceedings of the ACM SIGKDD workshop on human computation (pp. 77\u201385). New York, NY: ACM.","DOI":"10.1145\/1600150.1600175"},{"key":"9205_CR44","doi-asserted-by":"crossref","unstructured":"Nowak, S., & R\u00fcger, S. (2010). How reliable are annotations via crowdsourcing: A study about inter-annotator agreement for multi-label image annotation. In MIR \u201910: Proceedings of the international conference on Multimedia information retrieval (pp. 557\u2013566). New York, NY: ACM.","DOI":"10.1145\/1743384.1743478"},{"key":"9205_CR45","volume-title":"Questionnaire Design and Attitude Measurement","author":"A. N. Oppenheim","year":"1966","unstructured":"Oppenheim, A. N. (1966). Questionnaire design and attitude measurement. London: Heinemann."},{"key":"9205_CR46","unstructured":"Quinn, A. J., & Bederson, B. B. (2009). A taxonomy of distributed human computation. Technical Report HCIL-2009-23. University of Maryland."},{"key":"9205_CR47","doi-asserted-by":"crossref","unstructured":"Quinn, A. J., & Bederson, B. B. (2011). Human computation: A survey and taxonomy of a growing field. In Proceedings of CHI 2011.","DOI":"10.1145\/1978942.1979148"},{"key":"9205_CR48","doi-asserted-by":"crossref","unstructured":"Radlinski, F., Kurup, M., & Joachims, T. (2008). How does clickthrough data reflect retrieval quality?. In J. G. Shanahan, S. Amer-Yahia, I. Manolescu, Y. Zhang, D. A. Evans, A. Kolcz, K. S. Choi, & A. Chowdhury (Eds). CIKM (pp. 43\u201352). ACM.","DOI":"10.1145\/1458082.1458092"},{"key":"9205_CR49","doi-asserted-by":"crossref","unstructured":"Ross, J., Irani, L., Silberman, M. S., Zaldivar, A., & Tomlinson, B. (2010). Who are the crowdworkers?: Shifting demographics in mechanical turk. In Proceedings of the 28th international conference on human factors in computing systems, CHI 2010, extended abstracts volume (pp. 2863\u20132872). ACM.","DOI":"10.1145\/1753846.1753873"},{"key":"9205_CR50","doi-asserted-by":"publisher","unstructured":"Rzeszotarski, J. M., & Kittur, A. (2011). Instrumenting the crowd: using implicit behavioral measures to predict task performance. In Proceedings of the 24th annual ACM symposium on User interface software and technology, UIST \u201911 (pp. 13\u201322). New York, NY: ACM. doi:10.1145\/2047196.2047199. url:http:\/\/doi.acm.org\/10.1145\/2047196.2047199.","DOI":"10.1145\/2047196.2047199"},{"key":"9205_CR51","doi-asserted-by":"crossref","unstructured":"Shaw, A., Horton, J., & Chen, D. (2011). Designing incentives for inexpert human raters. In Proceedings of the ACM Conference on computer supported cooperative work, CSCW \u201911.","DOI":"10.1145\/1958824.1958865"},{"key":"9205_CR52","doi-asserted-by":"crossref","unstructured":"Silberman, M. S., Ross, J., Irani, L., & Tomlinson, B. (2010). Sellers\u2019 problems in human computation markets. In Proceedings of the ACM SIGKDD workshop on human computation (HCOMP \u201910) (pp. 18\u201321). ACM.","DOI":"10.1145\/1837885.1837891"},{"key":"9205_CR53","doi-asserted-by":"crossref","unstructured":"Snow, R., O\u2019Connor, B., Jurafsky, D., & Ng, A. Y. (2008). Cheap and fast\u2014but is it good?: Evaluating non-expert annotations for natural language tasks. In Proceedings of the conference on empirical methods in natural language processing (EMNLP \u201908) (pp. 254\u2013263). ACL.","DOI":"10.3115\/1613715.1613751"},{"key":"9205_CR1","doi-asserted-by":"crossref","unstructured":"von Ahn, L., & Dabbish, L. (2004). Labeling images with a computer game. In Proceedings of the SIGCHI conference on Human factors in computing systems, CHI \u201904 (pp. 319\u2013326). New York, NY: ACM.","DOI":"10.1145\/985692.985733"},{"issue":"5","key":"9205_CR54","doi-asserted-by":"publisher","first-page":"697","DOI":"10.1016\/S0306-4573(00)00010-8","volume":"36","author":"E. M. Voorhees","year":"2000","unstructured":"Voorhees, E. M. (2000). Variations in relevance judgments and the measurement of retrieval effectiveness. Information Processing & Management, 36(5), 697\u2013716.","journal-title":"Information Processing Management"},{"key":"9205_CR55","volume-title":"TREC: Experimentation and Evaluation in Information Retrieval","year":"2005","unstructured":"Voorhees, E. M., & Harman, D. K. (Eds.). (2005). TREC: Experimentation and evaluation in information retrieval. Cambridge, MA: MIT Press."},{"key":"9205_CR56","unstructured":"Vuurens, J., Vries, A. P. D., & Eickhoff, C. (2011). How much Spam can you take? An analysis of crowdsourcing results to increase accuracy. In M. Lease, V. Hester, A. Sorokin, & E. Yilmaz (Eds.), Proceedings of the ACM SIGIR 2011 workshop on crowdsourcing for information retrieval (CIR 2011) (pp. 48\u201355). Beijing, China."},{"key":"9205_CR57","unstructured":"Welinder, P., Branson, S., Belongie, S., & Perona, P. (2010). The multidimensional wisdom of crowds. In J. Lafferty, C. K. I. Williams, J. Shawe-Taylor, R. Zemel, & A. Culotta (Eds.), Advances in neural information processing systems (NIPS \u201910) (pp. 2424\u20132432)."},{"key":"9205_CR58","unstructured":"Zhu, D., & Carterette, B. (2010). An analysis of assessor behavior in crowdsourced preference judgments. In SIGIR 2010 workshop on crowdsourcing for search evaluation."}],"container-title":["Information Retrieval"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10791-012-9205-0.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s10791-012-9205-0\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s10791-012-9205-0","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10791-012-9205-0.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,1,2]],"date-time":"2024-01-02T10:12:07Z","timestamp":1704190327000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s10791-012-9205-0"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2012,7,20]]},"references-count":58,"journal-issue":{"issue":"2","published-print":{"date-parts":[[2013,4]]}},"alternative-id":["9205"],"URL":"https:\/\/doi.org\/10.1007\/s10791-012-9205-0","relation":{},"ISSN":["1386-4564","1573-7659"],"issn-type":[{"value":"1386-4564","type":"print"},{"value":"1573-7659","type":"electronic"}],"subject":[],"published":{"date-parts":[[2012,7,20]]},"assertion":[{"value":"13 May 2011","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"26 June 2012","order":2,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"20 July 2012","order":3,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"This content has been made available to all.","name":"free","label":"Free to read"}]}}