{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,4]],"date-time":"2026-04-04T06:19:07Z","timestamp":1775283547697,"version":"3.50.1"},"reference-count":48,"publisher":"Springer Science and Business Media LLC","issue":"2","license":[{"start":{"date-parts":[[2012,7,20]],"date-time":"2012-07-20T00:00:00Z","timestamp":1342742400000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2012,7,20]],"date-time":"2012-07-20T00:00:00Z","timestamp":1342742400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Inf Retrieval"],"published-print":{"date-parts":[[2013,4]]},"DOI":"10.1007\/s10791-012-9204-1","type":"journal-article","created":{"date-parts":[[2012,7,19]],"date-time":"2012-07-19T12:07:28Z","timestamp":1342699648000},"page":"101-120","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":43,"title":["Implementing crowdsourcing-based relevance experimentation: an industrial perspective"],"prefix":"10.1007","volume":"16","author":[{"given":"Omar","family":"Alonso","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2012,7,20]]},"reference":[{"key":"9204_CR2","unstructured":"Alonso, O., & Baeza-Yates, R. (2011). An analysis of crowdsourcing relevance assessments in Spanish. In CERI 2011."},{"key":"9204_CR3","doi-asserted-by":"crossref","unstructured":"Alonso, O., & Baeza-Yates, R. (2011). Design and implementation of relevance assessments using crowdsourcing. In ECIR 2011.","DOI":"10.1007\/978-3-642-20161-5_16"},{"key":"9204_CR1","doi-asserted-by":"publisher","unstructured":"Alonso, O., & Mizzaro, S. (2012). Using crowdsourcing for TREC relevance assessment. Information Processing and Management. doi:10.1016\/j.ipm.2012.01.004.","DOI":"10.1016\/j.ipm.2012.01.004"},{"key":"9204_CR4","doi-asserted-by":"crossref","unstructured":"Alonso, O., & Lease M. (2011). Crowdsourcing for information retrieval: Principles, methods, and applications. In  SIGIR, Tutorial.","DOI":"10.1145\/2009916.2010170"},{"issue":"4","key":"9204_CR5","doi-asserted-by":"crossref","first-page":"555","DOI":"10.1162\/coli.07-034-R2","volume":"34","author":"R. Artstein","year":"2008","unstructured":"Artstein, R., & Poesio, M. (2008). Inter-coder agreement for computational linguistics. Journal of Computational Linguistics, 34(4), 555\u2013596.","journal-title":"Journal of Computational Linguistics"},{"key":"9204_CR7","doi-asserted-by":"crossref","unstructured":"Bernstein, M., Little, G., Miller, R., Hartmann, B., Ackerman, M. Karger, D. et\u00a0al. (2010). Soylent: A word processor with a crowd inside. In UIST 2010.","DOI":"10.1145\/1866029.1866078"},{"key":"9204_CR6","doi-asserted-by":"crossref","unstructured":"Blanco, R., Halpin, H., Herzig, D., Mika, P., Pound, J., Thompson, H. et\u00a0al. (2011). Repeatable and reliable search system evaluation using crowdsourcing. In SIGIR 2011.","DOI":"10.1145\/2009916.2010039"},{"key":"9204_CR8","unstructured":"Bloodgood, M., & Callison-Burch, C. (2010). Bucking the trend: Large-scale cost-focused active learning for statistical machine translation. In ACL 2010."},{"key":"9204_CR9","doi-asserted-by":"crossref","unstructured":"Callison-Burch, C. (2009). Fast, cheap, and creative: Evaluating translation quality using Amazon\u2019s Mechanical Turk. In Proceedings of EMNLP.","DOI":"10.3115\/1699510.1699548"},{"key":"9204_CR10","doi-asserted-by":"crossref","unstructured":"Carterette, B., & Soboroff, I. (2011). The effect of assessor error on IR system evaluation. In SIGIR 2011.","DOI":"10.1145\/1835449.1835540"},{"key":"9204_CR12","unstructured":"Chandrasekar, R. et\u00a0al. (Eds) (2010). Proceedings of the human computation workshop (HCOMP)."},{"key":"9204_CR11","doi-asserted-by":"crossref","unstructured":"Chilton, L., Horton, J., Miller, R., & Azenkot, S. (2010). Task search in a human computation market. In HCOMP workshop 2010.","DOI":"10.1145\/1837885.1837889"},{"key":"9204_CR13","unstructured":"Cleverdon, C. (1970). The effects of variations in relevance assessments in comparative experimental tests of index languages. Cranfield Library Report, 3."},{"key":"9204_CR14","unstructured":"Eickhoff, C., & de Vries, A. (2011). How crowdsourcable is your task? In WSDM CSDM Workshop 2011."},{"key":"9204_CR15","volume-title":"Improving survey questions","author":"F Fowler","year":"1995","unstructured":"Fowler, F. (1995). Improving survey questions. Thousands Oaks, CA: Sage Publications."},{"key":"9204_CR16","unstructured":"Grady, C., & Lease, M. (2010). Crowdsourcing document relevance assessment with Mechanical Turk. In NAACL HLT workshop on creating speech and language data with Amazon Mechanical Turk."},{"key":"9204_CR17","unstructured":"Grier, D. (2006). When computers were human. Princenton University Press."},{"key":"9204_CR18","doi-asserted-by":"crossref","unstructured":"Harman, D. (2011). Information retrieval evaluation. Morgan & Claypool Publishers.","DOI":"10.1007\/978-3-031-02276-0"},{"key":"9204_CR19","doi-asserted-by":"crossref","unstructured":"Ipeirotis, P., & Paritosh, P. (2011). Managing crowdsourced human computation: A tutorial. In WWW 2011.","DOI":"10.1145\/1963192.1963314"},{"key":"9204_CR20","doi-asserted-by":"crossref","unstructured":"Ipeirotis, P., Provost, F., & Wang, J. (2010). Quality management on Amazon Mechanical Turk. In KDD-HCOMP 2010.","DOI":"10.1145\/1837885.1837906"},{"key":"9204_CR22","unstructured":"Kapelner, A., & Chandler, D. (2010). Preventing Satisficing in online surveys. In CrowdConf 2010."},{"key":"9204_CR23","unstructured":"Kazai, G. (2010). An exploration of the effects of task parameters on the performance of crowds. In WWW 2010."},{"key":"9204_CR24","doi-asserted-by":"crossref","unstructured":"Kazai, G. (2011). In search of quality in crowdsourcing for search engine evaluation. In ECIR 2011.","DOI":"10.1007\/978-3-642-20161-5_17"},{"key":"9204_CR25","doi-asserted-by":"crossref","unstructured":"Kazai, G., Milic-Frayling, N., & Costello, J. (2009). Towards methods for the collective gathering and quality control of relevance assessments. In SIGIR 2009.","DOI":"10.1145\/1571941.1572019"},{"key":"9204_CR26","doi-asserted-by":"crossref","unstructured":"Kern, R., Thies, H., & Satzger, G. (2010). Statistical quality control for human-based electronic services. In ICSOC 2010.","DOI":"10.1007\/978-3-642-17358-5_17"},{"key":"9204_CR27","doi-asserted-by":"crossref","unstructured":"Kinney, K., Huffman, S., & Zhai, J. (2008). How evaluator domain expertise affects search result relevance judgments. In CIKM 2008.","DOI":"10.1145\/1458082.1458160"},{"key":"9204_CR28","doi-asserted-by":"crossref","unstructured":"Kittur, A., Chi, E., & Suh, B. (2008). Crowdsourcing user studies with Mechanical Turk. In SIGCHI 2008.","DOI":"10.1145\/1357054.1357127"},{"key":"9204_CR29","volume-title":"Content Analysis (Second Edition).","author":"K Krippendorff","year":"2004","unstructured":"Krippendorff, K. (2004). Content analysis, 2nd edn. Thousands Oaks, CA: Sage Publications."},{"key":"9204_CR30","unstructured":"Lease, M., Carvalho, V., & Yilmaz, E. (Eds) (2010). Proceedings of the ACM SIGIR 2010 workshop on crowdsourcing for search evaluation (CSE 2010). SIGIR 2010."},{"key":"9204_CR31","doi-asserted-by":"crossref","unstructured":"Lease, M., Carvalho, V., & Yilmaz, E. (Eds) (2011). Proceedings of the workshop on crowdsourcing for search and data mining (CSDM). WSDM 2011.","DOI":"10.1145\/1935826.1935828"},{"key":"9204_CR33","unstructured":"Lease, M., & Kazai, G. (2011). Overview of the TREC 2011 crowdsourcing task. In TREC 2011."},{"key":"9204_CR32","unstructured":"Lease, M., & Yilmaz, E. (Eds) (2011). Proceedings of the ACM SIGIR workshop on crowdsourcing for information retrieval (CIR 2011). SIGIR 2011."},{"key":"9204_CR34","doi-asserted-by":"crossref","unstructured":"Law, E., & von Ahn, L. (2011). Human computation. Morgan & Claypool Publishers.","DOI":"10.1007\/978-3-031-01555-7"},{"key":"9204_CR35","doi-asserted-by":"crossref","unstructured":"Law, E., Bennett, P., & Horvitz, E. (2011). The effects of choice in routing relevance judgments. In SIGIR 2011.","DOI":"10.1145\/2009916.2010082"},{"key":"9204_CR36","unstructured":"Laws, F., Scheible, C., & Sch\u00fctze, H. (2011). Active learning with Amazon Mechanical Turk. In EMNLP 2011."},{"key":"9204_CR37","volume-title":"Sampling: Design and analysis","author":"S Lohr","year":"1999","unstructured":"Lohr, S. (1999). Sampling: Design and analysis. Boston, CA: Brooks\/Cole Publishing."},{"key":"9204_CR38","doi-asserted-by":"crossref","unstructured":"Marshall, C., & Shipman, F. (2011). The ownership and reuse of visual media. In JCDL 2011.","DOI":"10.1145\/1998076.1998108"},{"issue":"1","key":"9204_CR40","doi-asserted-by":"crossref","first-page":"1","DOI":"10.3758\/s13428-011-0124-6","volume":"44","author":"W. Mason","year":"2011","unstructured":"Mason, W., & Suri, S. (2011). Conducting behavioral research on amazon\u2019s Mechanical Turk. Behavior Research Methods. 44(1), 1\u201323.","journal-title":"Behavior Research Methods"},{"key":"9204_CR39","doi-asserted-by":"crossref","unstructured":"Mason, W., & Watts, D. (2009). Financial incentives and the \u2018Performance of Crowds\u2019. In HCOMP workshop at KDD.","DOI":"10.1145\/1600150.1600175"},{"key":"9204_CR41","volume-title":"Usability engineering","author":"J Nielsen","year":"1994","unstructured":"Nielsen, J. (1994). Usability engineering. San Francisco, CA: Morgan Kaufmann."},{"issue":"5","key":"9204_CR42","doi-asserted-by":"crossref","first-page":"411","DOI":"10.1017\/S1930297500002205","volume":"5","author":"G. Paolacci","year":"2010","unstructured":"Paolacci, G., Chandler, J., & Ipeirotis, P. (2010). Running experiments on Amazon Mechanical Turk. Judgment and Decision Making, 5(5), 411\u2013419.","journal-title":"Judgment and Decision Making"},{"key":"9204_CR43","doi-asserted-by":"crossref","unstructured":"Sheng, V., Provost, F., & Ipeirotis, P. (2008). Get another label? Improving data quality and data mining using multiple, noisy labelers. In KDD 2008.","DOI":"10.1145\/1401890.1401965"},{"key":"9204_CR44","doi-asserted-by":"crossref","unstructured":"Snow, R., O\u2019 Connor, B., Jurafsky, D., & Ng, A.Y. (2008). Cheap and fast\u2014but is it good? Evaluating non-expert annotations for natural language tasks. In EMNLP 2008.","DOI":"10.3115\/1613715.1613751"},{"key":"9204_CR45","doi-asserted-by":"crossref","unstructured":"Smucker, M., & Jethani, C. (2011). Measuring assessor accuracy: A comparison of NIST assessors and user study participants. In SIGIR 2011.","DOI":"10.1145\/2009916.2010134"},{"key":"9204_CR46","unstructured":"Smucker, M. (2011). Crowdsourcing with a crowd of one and other. In TREC 2011."},{"key":"9204_CR47","doi-asserted-by":"crossref","unstructured":"Tang, J., & Sanderson, M. (2010). Evaluation and user preference study on spatial diversity. In ECIR 2010.","DOI":"10.1007\/978-3-642-12275-0_18"},{"key":"9204_CR48","unstructured":"Voorhees, E., & Harman, D. (2005). TREC experiment and evaluation in information retrieval. In MIT Press."},{"key":"9204_CR49","unstructured":"Zaidand,O., & Callison-Burch, C. (2011). Crowdsourcing translation: Professional quality from non-professionals. In ACL 2011."}],"container-title":["Information Retrieval"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10791-012-9204-1.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s10791-012-9204-1\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s10791-012-9204-1","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10791-012-9204-1.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,1,2]],"date-time":"2024-01-02T15:11:40Z","timestamp":1704208300000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s10791-012-9204-1"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2012,7,20]]},"references-count":48,"journal-issue":{"issue":"2","published-print":{"date-parts":[[2013,4]]}},"alternative-id":["9204"],"URL":"https:\/\/doi.org\/10.1007\/s10791-012-9204-1","relation":{},"ISSN":["1386-4564","1573-7659"],"issn-type":[{"value":"1386-4564","type":"print"},{"value":"1573-7659","type":"electronic"}],"subject":[],"published":{"date-parts":[[2012,7,20]]},"assertion":[{"value":"16 May 2011","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"21 June 2012","order":2,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"20 July 2012","order":3,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"This content has been made available to all.","name":"free","label":"Free to read"}]}}