{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,1,8]],"date-time":"2026-01-08T01:26:26Z","timestamp":1767835586483,"version":"3.49.0"},"publisher-location":"New York, NY, USA","reference-count":30,"publisher":"ACM","license":[{"start":{"date-parts":[[2013,2,23]],"date-time":"2013-02-23T00:00:00Z","timestamp":1361577600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2013,2,23]]},"DOI":"10.1145\/2441776.2441847","type":"proceedings-article","created":{"date-parts":[[2013,2,22]],"date-time":"2013-02-22T19:25:33Z","timestamp":1361561133000},"page":"639-648","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":22,"title":["Enhancing reliability using peer consistency evaluation in human computation"],"prefix":"10.1145","author":[{"given":"Shih-Wen","family":"Huang","sequence":"first","affiliation":[{"name":"University of Illinois at Urbana-Champaign, Urbana, Illinois, USA"}]},{"given":"Wai-Tat","family":"Fu","sequence":"additional","affiliation":[{"name":"University of Illinois at Urbana-Champaign, Urbana, Illinois, USA"}]}],"member":"320","published-online":{"date-parts":[[2013,2,23]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"publisher","DOI":"10.5555\/1766171.1766196"},{"key":"e_1_3_2_1_2_1","first-page":"3","article-title":"Social preferences and the response to incentives: Evidence from personnel data","volume":"120","author":"Bandiera O.","year":"2005","unstructured":"Bandiera , O. , Barankay , I. , and Rasul , I . Social preferences and the response to incentives: Evidence from personnel data . The Quarterly Journal of Economics 120 , 3 ( 2005 ), 917--962. Bandiera, O., Barankay, I., and Rasul, I. Social preferences and the response to incentives: Evidence from personnel data. The Quarterly Journal of Economics 120, 3 (2005), 917--962.","journal-title":"The Quarterly Journal of Economics"},{"key":"e_1_3_2_1_3_1","series-title":"Series C (Applied Statistics) 28, 1","first-page":"20","volume-title":"Maximum likelihood estimation of observer error-rates using the em algorithm. Journal of the Royal Statistical Society","author":"Dawid A. P.","year":"1979","unstructured":"Dawid , A. P. , and Skene , A. M . Maximum likelihood estimation of observer error-rates using the em algorithm. Journal of the Royal Statistical Society . Series C (Applied Statistics) 28, 1 ( 1979 ), pp. 20 -- 28 . Dawid, A. P., and Skene, A. M. Maximum likelihood estimation of observer error-rates using the em algorithm. Journal of the Royal Statistical Society. Series C (Applied Statistics) 28, 1 (1979), pp. 20--28."},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.1145\/2145204.2145355"},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1162\/003355300554917"},{"key":"e_1_3_2_1_6_1","volume-title":"Proceedings of WSDM 2011 Workshop on Crowdsourcing for Search and Data Mining","author":"Harris C. G.","year":"2011","unstructured":"Harris , C. G. You're hired! an examination of crowdsourcing incentive models in human resourse tasks . In Proceedings of WSDM 2011 Workshop on Crowdsourcing for Search and Data Mining ( 2011 ). Harris, C. G. You're hired! an examination of crowdsourcing incentive models in human resourse tasks. In Proceedings of WSDM 2011 Workshop on Crowdsourcing for Search and Data Mining (2011)."},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"publisher","DOI":"10.1109\/IMIS.2011.91"},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.1145\/1837885.1837908"},{"key":"e_1_3_2_1_9_1","volume-title":"Proceedings of HCOMP12: The 4th Workshop on Human Computation","author":"Huang S.-W.","year":"2012","unstructured":"Huang , S.-W. , and Fu , W . -T. Systematic analysis of output agreement games: Effects of gaming environment, social interaction, and feedback . In Proceedings of HCOMP12: The 4th Workshop on Human Computation ( 2012 ). Huang, S.-W., and Fu, W.-T. Systematic analysis of output agreement games: Effects of gaming environment, social interaction, and feedback. In Proceedings of HCOMP12: The 4th Workshop on Human Computation (2012)."},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.1145\/1869086.1869094"},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.1145\/1837885.1837906"},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.1145\/2145204.2145357"},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1145\/2047196.2047202"},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.1145\/2145204.2145354"},{"key":"e_1_3_2_1_15_1","volume-title":"Human Computation. Synthesis Lectures on Artificial Intelligence and Machine Learning","author":"Law E.","year":"2011","unstructured":"Law , E. , and Von Ahn , L. Human Computation. Synthesis Lectures on Artificial Intelligence and Machine Learning . Morgan & Claypool Publishers , 2011 . Law, E., and Von Ahn, L. Human Computation. Synthesis Lectures on Artificial Intelligence and Machine Learning. Morgan & Claypool Publishers, 2011."},{"key":"e_1_3_2_1_16_1","volume-title":"Proceedings of the SIGIR 2010 Workshop on Crowdsourcing for Search Evaluation","author":"Le J.","year":"2010","unstructured":"Le , J. , Edmonds , A. , Hester , V. , and Biewald , L . Ensuring quality in crowdsourced search relevance evaluation: The effects of training qustion distribution . In Proceedings of the SIGIR 2010 Workshop on Crowdsourcing for Search Evaluation ( 2010 ). Le, J., Edmonds, A., Hester, V., and Biewald, L. Ensuring quality in crowdsourced search relevance evaluation: The effects of training qustion distribution. In Proceedings of the SIGIR 2010 Workshop on Crowdsourcing for Search Evaluation (2010)."},{"key":"e_1_3_2_1_17_1","volume-title":"Proceedings of the AAAI Workshop on Human Computation (HCOMP)","author":"Liem B.","year":"2011","unstructured":"Liem , B. , Zhang , H. , and Chen , Y . An Iterative Dual Pathway Structure for Speech-to-Text Transcription . In Proceedings of the AAAI Workshop on Human Computation (HCOMP) ( 2011 ). Liem, B., Zhang, H., and Chen, Y. An Iterative Dual Pathway Structure for Speech-to-Text Transcription. In Proceedings of the AAAI Workshop on Human Computation (HCOMP) (2011)."},{"key":"e_1_3_2_1_18_1","volume-title":"Proceedings of HCOMP12: The 4th Workshop on Human Computation","author":"Lin C.","year":"2012","unstructured":"Lin , C. , Mausam , M. , and Weld , D . Crowdsourcing control: Moving beyond multiple choice . In Proceedings of HCOMP12: The 4th Workshop on Human Computation ( 2012 ). Lin, C., Mausam, M., and Weld, D. Crowdsourcing control: Moving beyond multiple choice. In Proceedings of HCOMP12: The 4th Workshop on Human Computation (2012)."},{"key":"e_1_3_2_1_19_1","volume-title":"AAAI","author":"Lin C. H.","year":"2012","unstructured":"Lin , C. H. , Mausam , and Weld, D. S . Dynamically switching between synergistic workflows for crowdsourcing . In AAAI ( 2012 ). Lin, C. H., Mausam, and Weld, D. S. Dynamically switching between synergistic workflows for crowdsourcing. In AAAI (2012)."},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.1145\/1600150.1600159"},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.1145\/1809400.1809422"},{"key":"e_1_3_2_1_22_1","volume-title":"Proceedings of HCOMP11: The 3rd Workshop on Human Computation","author":"Oleson D.","year":"2011","unstructured":"Oleson , D. , Sorokin , A. , Laughlin , G. , Hester , V. , Le , J. , and Biewald , L . Programmatic gold: Targeted and scalable quality assurance in crowdsourcing . In Proceedings of HCOMP11: The 3rd Workshop on Human Computation ( 2011 ). Oleson, D., Sorokin, A., Laughlin, G., Hester, V., Le, J., and Biewald, L. Programmatic gold: Targeted and scalable quality assurance in crowdsourcing. In Proceedings of HCOMP11: The 3rd Workshop on Human Computation (2011)."},{"key":"e_1_3_2_1_23_1","volume-title":"Proceedings of CrowdSearch: Crowdsourcing Web search 2012","author":"Paritosh P.","year":"2012","unstructured":"Paritosh , P. Human computation must be reproducible . In Proceedings of CrowdSearch: Crowdsourcing Web search 2012 ( 2012 ). Paritosh, P. Human computation must be reproducible. In Proceedings of CrowdSearch: Crowdsourcing Web search 2012 (2012)."},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"publisher","DOI":"10.1145\/1978942.1979148"},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.1145\/1520340.1520597"},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.1145\/1837885.1837905"},{"key":"e_1_3_2_1_27_1","doi-asserted-by":"publisher","DOI":"10.5555\/1613715.1613751"},{"key":"e_1_3_2_1_28_1","volume-title":"Proceedings of HCOMP11: The 3rd Workshop on Human Computation","author":"Sun Y.-A.","year":"2011","unstructured":"Sun , Y.-A. , Roy , S. , and Little , G. D . Beyond independent agreement: A tournament selection approach for quality assurance of human computation tasks . In Proceedings of HCOMP11: The 3rd Workshop on Human Computation ( 2011 ). Sun, Y.-A., Roy, S., and Little, G. D. Beyond independent agreement: A tournament selection approach for quality assurance of human computation tasks. In Proceedings of HCOMP11: The 3rd Workshop on Human Computation (2011)."},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.1145\/985692.985733"},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"publisher","DOI":"10.1145\/1378704.1378719"}],"event":{"name":"CSCW '13: Computer Supported Cooperative Work","location":"San Antonio Texas USA","acronym":"CSCW '13","sponsor":["SIGCHI ACM Special Interest Group on Computer-Human Interaction"]},"container-title":["Proceedings of the 2013 conference on Computer supported cooperative work"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/2441776.2441847","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/2441776.2441847","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,18]],"date-time":"2025-06-18T08:19:14Z","timestamp":1750234754000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/2441776.2441847"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2013,2,23]]},"references-count":30,"alternative-id":["10.1145\/2441776.2441847","10.1145\/2441776"],"URL":"https:\/\/doi.org\/10.1145\/2441776.2441847","relation":{},"subject":[],"published":{"date-parts":[[2013,2,23]]},"assertion":[{"value":"2013-02-23","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}