{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,11,18]],"date-time":"2025-11-18T09:54:42Z","timestamp":1763459682126,"version":"3.45.0"},"publisher-location":"New York, NY, USA","reference-count":73,"publisher":"ACM","license":[{"start":{"date-parts":[[2016,5,7]],"date-time":"2016-05-07T00:00:00Z","timestamp":1462579200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"name":"NSF","award":["1351131"],"award-info":[{"award-number":["1351131"]}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2016,5,7]]},"DOI":"10.1145\/2858036.2858115","type":"proceedings-article","created":{"date-parts":[[2016,5,5]],"date-time":"2016-05-05T10:08:22Z","timestamp":1462442902000},"page":"3167-3179","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":44,"title":["Embracing Error to Enable Rapid Crowdsourcing"],"prefix":"10.1145","author":[{"given":"Ranjay A.","family":"Krishna","sequence":"first","affiliation":[{"name":"Stanford University, Stanford, CA, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Kenji","family":"Hata","sequence":"additional","affiliation":[{"name":"Stanford University, Stanford, CA, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Stephanie","family":"Chen","sequence":"additional","affiliation":[{"name":"Stanford University, Stanford, CA, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Joshua","family":"Kravitz","sequence":"additional","affiliation":[{"name":"Stanford University, Stanford, CA, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"David A.","family":"Shamma","sequence":"additional","affiliation":[{"name":"Yahoo! Research, San Francisco, CA, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Li","family":"Fei-Fei","sequence":"additional","affiliation":[{"name":"Stanford University, Palo Alto, CA, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Michael S.","family":"Bernstein","sequence":"additional","affiliation":[{"name":"Stanford University, Palo Alto, CA, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2016,5,7]]},"reference":[{"key":"e_1_3_2_2_1_1","doi-asserted-by":"publisher","DOI":"10.1145\/2047196.2047201"},{"key":"e_1_3_2_2_2_1","doi-asserted-by":"publisher","DOI":"10.1145\/1866029.1866078"},{"key":"e_1_3_2_2_3_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2013.89"},{"key":"e_1_3_2_2_4_1","doi-asserted-by":"publisher","DOI":"10.1609\/hcomp.v1i1.13091"},{"key":"e_1_3_2_2_5_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2014.473"},{"key":"e_1_3_2_2_6_1","doi-asserted-by":"publisher","DOI":"10.5555\/1888089.1888123"},{"key":"e_1_3_2_2_7_1","volume-title":"From detection to identification: Response to multiple targets in rapid serial visual presentation. Perception & psychophysics 42, 2","author":"Broadbent Donald E","year":"1987","unstructured":"Donald E Broadbent and Margaret HP Broadbent. 1987. From detection to identification: Response to multiple targets in rapid serial visual presentation. Perception & psychophysics 42, 2 (1987), 105--113."},{"key":"e_1_3_2_2_8_1","doi-asserted-by":"publisher","DOI":"10.1145\/2441776.2441936"},{"key":"e_1_3_2_2_9_1","doi-asserted-by":"publisher","unstructured":"Stuart K Card Allen Newell and Thomas P Moran. 1983. The psychology of human-computer interaction. (1983).","DOI":"10.5555\/578027"},{"key":"e_1_3_2_2_10_1","doi-asserted-by":"publisher","DOI":"10.1145\/2702123.2702145"},{"key":"e_1_3_2_2_11_1","doi-asserted-by":"publisher","DOI":"10.1145\/2470654.2466265"},{"key":"e_1_3_2_2_12_1","volume-title":"Chelsea T Finn, and Robert A Sinkin.","author":"Colligan Lacey","year":"2015","unstructured":"Lacey Colligan, Henry WW Potts, Chelsea T Finn, and Robert A Sinkin. 2015. Cognitive workload changes for nurses transitioning from a legacy system with paper documentation to a commercial electronic health record. International journal of medical informatics 84, 7 (2015), 469--476."},{"key":"e_1_3_2_2_13_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2009.5206848"},{"key":"e_1_3_2_2_14_1","doi-asserted-by":"publisher","DOI":"10.1145\/2556288.2557011"},{"key":"e_1_3_2_2_15_1","doi-asserted-by":"publisher","DOI":"10.1145\/2556288.2556998"},{"key":"e_1_3_2_2_16_1","doi-asserted-by":"publisher","DOI":"10.1167\/7.1.10"},{"key":"e_1_3_2_2_17_1","doi-asserted-by":"publisher","DOI":"10.1145\/1518701.1518736"},{"key":"e_1_3_2_2_18_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2014.81"},{"key":"e_1_3_2_2_19_1","doi-asserted-by":"publisher","DOI":"10.1162\/jocn_a_00790"},{"key":"e_1_3_2_2_20_1","doi-asserted-by":"publisher","DOI":"10.1145\/1869086.1869094"},{"key":"e_1_3_2_2_21_1","doi-asserted-by":"publisher","DOI":"10.1145\/1837885.1837906"},{"key":"e_1_3_2_2_22_1","doi-asserted-by":"publisher","DOI":"10.1145\/2470654.2470742"},{"key":"e_1_3_2_2_23_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2013.166"},{"key":"e_1_3_2_2_24_1","unstructured":"Tatiana Josephy Matt Lease and Praveen Paritosh. 2013. CrowdScale 2013: Crowdsourcing at Scale workshop report. (2013)."},{"key":"e_1_3_2_2_25_1","doi-asserted-by":"publisher","DOI":"10.5555\/2343576.2343643"},{"key":"e_1_3_2_2_26_1","volume-title":"2011 49th Annual Allerton Conference on. IEEE, 284--291","author":"Karger David R","year":"2011","unstructured":"David R Karger, Sewoong Oh, and Devavrat Shah. 2011. Budget-optimal crowdsourcing using low-rank matrix approximations. In Communication, Control, and Computing (Allerton), 2011 49th Annual Allerton Conference on. IEEE, 284--291."},{"key":"e_1_3_2_2_27_1","doi-asserted-by":"publisher","DOI":"10.1287\/opre.2013.1235"},{"key":"e_1_3_2_2_28_1","doi-asserted-by":"publisher","DOI":"10.1145\/1357054.1357127"},{"key":"e_1_3_2_2_29_1","unstructured":"Alex Krizhevsky Ilya Sutskever and Geoffrey E Hinton. 2012. Imagenet classification with deep convolutional neural networks. In Advances in neural information processing systems. 1097--1105."},{"key":"e_1_3_2_2_30_1","doi-asserted-by":"publisher","DOI":"10.1145\/2470654.2466420"},{"key":"e_1_3_2_2_31_1","doi-asserted-by":"publisher","DOI":"10.1145\/2702123.2702416"},{"key":"e_1_3_2_2_32_1","doi-asserted-by":"publisher","DOI":"10.1145\/2380116.2380122"},{"key":"e_1_3_2_2_33_1","doi-asserted-by":"publisher","DOI":"10.1145\/2047196.2047200"},{"key":"e_1_3_2_2_34_1","first-page":"3","article-title":"Guest Editorial","volume":"12","author":"Lewis David D.","year":"1994","unstructured":"David D. Lewis and Philip J. Hayes. 1994. Guest Editorial. ACM Transactions on Information Systems 12, 3 (July 1994), 231.","journal-title":"ACM Transactions on Information Systems"},{"key":"e_1_3_2_2_35_1","doi-asserted-by":"publisher","DOI":"10.1073\/pnas.092277599"},{"key":"e_1_3_2_2_36_1","first-page":"239","article-title":"Detecting emotion in music","volume":"3","author":"Li Tao","year":"2003","unstructured":"Tao Li and Mitsunori Ogihara. 2003. Detecting emotion in music. In ISMIR, Vol. 3. 239--240.","journal-title":"ISMIR"},{"key":"e_1_3_2_2_37_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2014.34"},{"key":"e_1_3_2_2_38_1","volume-title":"Piotr Dollar, and C Lawrence Zitnick","author":"Lin Tsung-Yi","year":"2014","unstructured":"Tsung-Yi Lin, Michael Maire, Serge Belongie, James Hays, Pietro Perona, Deva Ramanan, Piotr Dollar, and C Lawrence Zitnick. 2014. Microsoft COCO: Common objects in context. In Computer Vision-ECCV 2014. Springer, 740--755."},{"key":"e_1_3_2_2_39_1","doi-asserted-by":"publisher","unstructured":"Adam Marcus and Aditya Parameswaran. 2015. Crowdsourced data management: industry and academic perspectives. Foundations and Trends in Databases. 10.1561\/1900000044","DOI":"10.1561\/1900000044"},{"key":"e_1_3_2_2_40_1","doi-asserted-by":"publisher","DOI":"10.1145\/2531602.2531663"},{"key":"e_1_3_2_2_41_1","volume-title":"Conducting behavioral research on Amazons Mechanical Turk. Behavior research methods 44, 1","author":"Mason Winter","year":"2012","unstructured":"Winter Mason and Siddharth Suri. 2012. Conducting behavioral research on Amazons Mechanical Turk. Behavior research methods 44, 1 (2012), 1--23."},{"key":"e_1_3_2_2_42_1","volume-title":"Contextual correlates of semantic similarity. Language and cognitive processes 6, 1","author":"Miller George A","year":"1991","unstructured":"George A Miller and Walter G Charles. 1991. Contextual correlates of semantic similarity. Language and cognitive processes 6, 1 (1991), 1--28."},{"key":"e_1_3_2_2_43_1","doi-asserted-by":"publisher","DOI":"10.1561\/1500000011"},{"key":"e_1_3_2_2_44_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-33712-3_26"},{"key":"e_1_3_2_2_45_1","doi-asserted-by":"publisher","DOI":"10.5555\/2898607.2898793"},{"key":"e_1_3_2_2_46_1","doi-asserted-by":"publisher","DOI":"10.1037\/0278-7393.2.5.509"},{"key":"e_1_3_2_2_47_1","doi-asserted-by":"publisher","DOI":"10.1037\/h0027470"},{"key":"e_1_3_2_2_48_1","volume-title":"Attention gating in short-term visual memory. Psychological review 93, 2","author":"Reeves Adam","year":"1986","unstructured":"Adam Reeves and George Sperling. 1986. Attention gating in short-term visual memory. Psychological review 93, 2 (1986), 180."},{"key":"e_1_3_2_2_49_1","doi-asserted-by":"publisher","DOI":"10.1007\/s11263-015-0816-y"},{"key":"e_1_3_2_2_50_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2015.7298824"},{"key":"e_1_3_2_2_51_1","doi-asserted-by":"publisher","DOI":"10.1145\/2702123.2702508"},{"key":"e_1_3_2_2_52_1","doi-asserted-by":"publisher","DOI":"10.1023\/A:1007649029923"},{"key":"e_1_3_2_2_53_1","doi-asserted-by":"publisher","DOI":"10.1145\/2647868.2654908"},{"key":"e_1_3_2_2_54_1","doi-asserted-by":"publisher","DOI":"10.1145\/1401890.1401965"},{"key":"e_1_3_2_2_55_1","doi-asserted-by":"publisher","DOI":"10.1609\/hcomp.v1i1.13088"},{"key":"e_1_3_2_2_56_1","unstructured":"K. Simonyan and A. Zisserman. 2014. Very Deep Convolutional Networks for Large-Scale Image Recognition. CoRR abs\/1409.1556 (2014)."},{"key":"e_1_3_2_2_57_1","volume-title":"KDD Workshop. 109--120","author":"Smyth Padhraic","year":"1994","unstructured":"Padhraic Smyth, Michael C Burl, Usama M Fayyad, and Pietro Perona. 1994. Knowledge Discovery in Large Image Databases: Dealing with Uncertainties in Ground Truth.. In KDD Workshop. 109--120."},{"key":"e_1_3_2_2_58_1","unstructured":"Padhraic Smyth Usama Fayyad Michael Burl Pietro Perona and Pierre Baldi. 1995. Inferring ground truth from subjective labelling of venus images. (1995)."},{"key":"e_1_3_2_2_59_1","doi-asserted-by":"publisher","DOI":"10.5555\/1613715.1613751"},{"key":"e_1_3_2_2_60_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2011.5995330"},{"key":"e_1_3_2_2_61_1","volume-title":"Workshops at the Twenty-Sixth AAAI Conference on Artificial Intelligence.","author":"Su Hao","year":"2012","unstructured":"Hao Su, Jia Deng, and Li Fei-Fei. 2012. Crowdsourcing annotations for visual object detection. In Workshops at the Twenty-Sixth AAAI Conference on Artificial Intelligence."},{"key":"e_1_3_2_2_62_1","volume-title":"Adaptively learning the crowd kernel. arXiv preprint arXiv:1105.1033","author":"Tamuz Omer","year":"2011","unstructured":"Omer Tamuz, Ce Liu, Serge Belongie, Ohad Shamir, and Adam Tauman Kalai. 2011. Adaptively learning the crowd kernel. arXiv preprint arXiv:1105.1033 (2011)."},{"key":"e_1_3_2_2_63_1","doi-asserted-by":"publisher","DOI":"10.1145\/2812802"},{"key":"e_1_3_2_2_64_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2010.5540055"},{"key":"e_1_3_2_2_65_1","volume-title":"Show and tell: A neural image caption generator. arXiv preprint arXiv:1411.4555","author":"Vinyals Oriol","year":"2014","unstructured":"Oriol Vinyals, Alexander Toshev, Samy Bengio, and Dumitru Erhan. 2014. Show and tell: A neural image caption generator. arXiv preprint arXiv:1411.4555 (2014)."},{"key":"e_1_3_2_2_66_1","doi-asserted-by":"publisher","DOI":"10.1007\/s11263-012-0564-1"},{"key":"e_1_3_2_2_67_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2011.6126539"},{"key":"e_1_3_2_2_68_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2014.115"},{"key":"e_1_3_2_2_69_1","volume-title":"Dynamics of automatic and controlled visual attention. Science 238, 4828","author":"Weichselgartner Erich","year":"1987","unstructured":"Erich Weichselgartner and George Sperling. 1987. Dynamics of automatic and controlled visual attention. Science 238, 4828 (1987), 778--780."},{"key":"e_1_3_2_2_70_1","unstructured":"Peter Welinder Steve Branson Pietro Perona and Serge J Belongie. 2010. The multidimensional wisdom of crowds. In Advances in neural information processing systems. 2424--2432."},{"key":"e_1_3_2_2_71_1","unstructured":"Jacob Whitehill Ting-fan Wu Jacob Bergsma Javier R Movellan and Paul L Ruvolo. 2009. Whose vote should count more: Optimal integration of labels from labelers of unknown expertise. In Advances in neural information processing systems. 2035--2043."},{"key":"e_1_3_2_2_72_1","doi-asserted-by":"publisher","DOI":"10.1145\/571985.572014"},{"key":"e_1_3_2_2_73_1","unstructured":"Dengyong Zhou Sumit Basu Yi Mao and John C Platt. 2012. Learning from the wisdom of crowds by minimax entropy. In Advances in Neural Information Processing Systems. 2195--2203."}],"event":{"name":"CHI'16: CHI Conference on Human Factors in Computing Systems","sponsor":["SIGCHI ACM Special Interest Group on Computer-Human Interaction"],"location":"San Jose California USA","acronym":"CHI'16"},"container-title":["Proceedings of the 2016 CHI Conference on Human Factors in Computing Systems"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/2858036.2858115","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/2858036.2858115","content-type":"application\/pdf","content-version":"vor","intended-application":"syndication"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/2858036.2858115","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,11,18]],"date-time":"2025-11-18T09:50:11Z","timestamp":1763459411000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/2858036.2858115"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2016,5,7]]},"references-count":73,"alternative-id":["10.1145\/2858036.2858115","10.1145\/2858036"],"URL":"https:\/\/doi.org\/10.1145\/2858036.2858115","relation":{},"subject":[],"published":{"date-parts":[[2016,5,7]]},"assertion":[{"value":"2016-05-07","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}