{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,1,14]],"date-time":"2026-01-14T23:32:49Z","timestamp":1768433569773,"version":"3.49.0"},"publisher-location":"New York, NY, USA","reference-count":35,"publisher":"ACM","license":[{"start":{"date-parts":[[2013,8,11]],"date-time":"2013-08-11T00:00:00Z","timestamp":1376179200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2013,8,11]]},"DOI":"10.1145\/2487575.2487595","type":"proceedings-article","created":{"date-parts":[[2013,8,13]],"date-time":"2013-08-13T12:31:21Z","timestamp":1376397081000},"page":"686-694","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":52,"title":["Evaluating the crowd with confidence"],"prefix":"10.1145","author":[{"given":"Manas","family":"Joglekar","sequence":"first","affiliation":[{"name":"Stanford University, Stanford, CA, USA"}]},{"given":"Hector","family":"Garcia-Molina","sequence":"additional","affiliation":[{"name":"Stanford University, Stanford, CA, USA"}]},{"given":"Aditya","family":"Parameswaran","sequence":"additional","affiliation":[{"name":"Stanford University, Stanford, CA, USA"}]}],"member":"320","published-online":{"date-parts":[[2013,8,11]]},"reference":[{"key":"e_1_3_2_1_1_1","unstructured":"Mechanical Turk. http:\/\/mturk.com.  Mechanical Turk. http:\/\/mturk.com."},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.1145\/1480506.1480508"},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1145\/2339530.2339707"},{"key":"e_1_3_2_1_4_1","volume-title":"Statistical Inference","author":"Casella George","year":"2001","unstructured":"George Casella and Roger Berger . Statistical Inference . Duxbury Resource Center , 2001 . George Casella and Roger Berger. Statistical Inference. Duxbury Resource Center, 2001."},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1145\/2408776.2408787"},{"key":"e_1_3_2_1_6_1","unstructured":"Image Comparison Dataset. http:\/\/www.stanford.edu\/$\\sim$manasrj\/ic\\_data.tar.gz.  Image Comparison Dataset. http:\/\/www.stanford.edu\/$\\sim$manasrj\/ic\\_data.tar.gz."},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"publisher","DOI":"10.2307\/2346806"},{"issue":"1","key":"e_1_3_2_1_8_1","first-page":"1","article-title":"Maximum likelihood from incomplete data via the em algorithm. JOURNAL OF THE ROYAL STATISTICAL SOCIETY","volume":"39","author":"Dempster A. P.","year":"1977","unstructured":"A. P. Dempster , N. M. Laird , and D. B. Rubin . Maximum likelihood from incomplete data via the em algorithm. JOURNAL OF THE ROYAL STATISTICAL SOCIETY , SERIES B , 39 ( 1 ): 1 -- 38 , 1977 . A. P. Dempster, N. M. Laird, and D. B. Rubin. Maximum likelihood from incomplete data via the em algorithm. JOURNAL OF THE ROYAL STATISTICAL SOCIETY, SERIES B, 39(1):1--38, 1977.","journal-title":"SERIES B"},{"key":"e_1_3_2_1_9_1","volume-title":"NIPS.","author":"J. Whitehill","year":"2009","unstructured":"J. Whitehill et al. Whose vote should count more: Optimal integration of labels from labelers of unknown expertise . In NIPS. 2009 . J. Whitehill et al. Whose vote should count more: Optimal integration of labels from labelers of unknown expertise. In NIPS. 2009."},{"key":"e_1_3_2_1_10_1","volume-title":"USENIX Security Symposium '10","author":"Recaptchas M. Motoyama","unstructured":"M. Motoyama et al. Recaptchas : Understanding captcha-solving services in an economic context . In USENIX Security Symposium '10 . M. Motoyama et al. Recaptchas : Understanding captcha-solving services in an economic context. In USENIX Security Symposium '10."},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.1145\/1557019.1557053"},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.5555\/1613715.1613751"},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1145\/1553374.1553488"},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.1145\/2213836.2213880"},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.1561\/2000000034"},{"key":"e_1_3_2_1_16_1","volume-title":"Infolab technical report","author":"Joglekar Manas","year":"2012","unstructured":"Manas Joglekar , Hector Garcia-Molina , and Aditya Parameswaran . Infolab technical report , 2012 . http:\/\/ilpubs.stanford.edu:8090\/1051\/. Manas Joglekar, Hector Garcia-Molina, and Aditya Parameswaran. Infolab technical report, 2012. http:\/\/ilpubs.stanford.edu:8090\/1051\/."},{"key":"e_1_3_2_1_17_1","volume-title":"Budget-optimal task allocation for reliable crowdsourcing systems. CoRR, abs\/1110.3564","author":"Karger David R.","year":"2011","unstructured":"David R. Karger , Sewoong Oh , and Devavrat Shah . Budget-optimal task allocation for reliable crowdsourcing systems. CoRR, abs\/1110.3564 , 2011 . David R. Karger, Sewoong Oh, and Devavrat Shah. Budget-optimal task allocation for reliable crowdsourcing systems. CoRR, abs\/1110.3564, 2011."},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.14778\/2336664.2336676"},{"key":"e_1_3_2_1_19_1","volume-title":"VLDB","author":"Marcus A.","year":"2012","unstructured":"A. Marcus , E. Wu , D. Karger , S. Madden , and R. Miller . Human-powered sorts and joins . In VLDB , 2012 . A. Marcus, E. Wu, D. Karger, S. Madden, and R. Miller. Human-powered sorts and joins. In VLDB, 2012."},{"key":"e_1_3_2_1_20_1","volume-title":"McLachlan and Thriyambakam Krishnan","author":"Geoffrey","year":"2008","unstructured":"Geoffrey J. McLachlan and Thriyambakam Krishnan . The EM Algorithm and Extensions (Wiley Series in Probability and Statistics). Wiley-Interscience , 2 edition, March 2008 . Geoffrey J. McLachlan and Thriyambakam Krishnan. The EM Algorithm and Extensions (Wiley Series in Probability and Statistics). Wiley-Interscience, 2 edition, March 2008."},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.1145\/2213836.2213878"},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.14778\/1952376.1952377"},{"key":"e_1_3_2_1_23_1","volume-title":"NIPS","author":"R. Gomes","year":"2011","unstructured":"R. Gomes et al. Crowdclustering . In NIPS , 2011 . R. Gomes et al. Crowdclustering. In NIPS, 2011."},{"key":"e_1_3_2_1_24_1","volume-title":"Identifying reliable workers swiftly. Infolab technical report","author":"Ramesh A.","year":"2012","unstructured":"A. Ramesh , A. Parameswaran , H. Garcia-Molina , and N. Polyzotis . Identifying reliable workers swiftly. Infolab technical report , Stanford University , 2012 . A. Ramesh, A. Parameswaran, H. Garcia-Molina, and N. Polyzotis. Identifying reliable workers swiftly. Infolab technical report, Stanford University, 2012."},{"key":"e_1_3_2_1_25_1","first-page":"491","article-title":"Eliminating spammers and ranking annotators for crowdsourced labeling tasks","volume":"13","author":"Raykar Vikas C.","year":"2012","unstructured":"Vikas C. Raykar and Shipeng Yu . Eliminating spammers and ranking annotators for crowdsourced labeling tasks . Journal of Machine Learning Research , 13 : 491 -- 518 , 2012 . Vikas C. Raykar and Shipeng Yu. Eliminating spammers and ranking annotators for crowdsourced labeling tasks. Journal of Machine Learning Research, 13:491--518, 2012.","journal-title":"Journal of Machine Learning Research"},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICIP.2011.6116320"},{"key":"e_1_3_2_1_27_1","doi-asserted-by":"publisher","DOI":"10.1002\/9780470316696"},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"publisher","DOI":"10.1145\/1401890.1401965"},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.1145\/2339530.2339571"},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"publisher","DOI":"10.1016\/S0169-7439(01)00132-0"},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"publisher","DOI":"10.14778\/2350229.2350263"},{"key":"e_1_3_2_1_32_1","volume-title":"All of Statistics","author":"Wasserman Larry","year":"2003","unstructured":"Larry Wasserman . All of Statistics . Springer , 2003 . Larry Wasserman. All of Statistics. Springer, 2003."},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPRW.2010.5543189"},{"key":"e_1_3_2_1_34_1","doi-asserted-by":"publisher","DOI":"10.1080\/01621459.1927.10502953"},{"key":"e_1_3_2_1_35_1","doi-asserted-by":"publisher","DOI":"10.5555\/1699510.1699518"}],"event":{"name":"KDD' 13: The 19th ACM SIGKDD International Conference on Knowledge Discovery and Data Mining","location":"Chicago Illinois USA","acronym":"KDD' 13","sponsor":["SIGMOD ACM Special Interest Group on Management of Data","SIGKDD ACM Special Interest Group on Knowledge Discovery in Data"]},"container-title":["Proceedings of the 19th ACM SIGKDD international conference on Knowledge discovery and data mining"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/2487575.2487595","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/2487575.2487595","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,18]],"date-time":"2025-06-18T08:35:55Z","timestamp":1750235755000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/2487575.2487595"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2013,8,11]]},"references-count":35,"alternative-id":["10.1145\/2487575.2487595","10.1145\/2487575"],"URL":"https:\/\/doi.org\/10.1145\/2487575.2487595","relation":{},"subject":[],"published":{"date-parts":[[2013,8,11]]},"assertion":[{"value":"2013-08-11","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}