{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,6,19]],"date-time":"2025-06-19T04:07:32Z","timestamp":1750306052269,"version":"3.41.0"},"publisher-location":"New York, NY, USA","reference-count":46,"publisher":"ACM","license":[{"start":{"date-parts":[[2017,5,2]],"date-time":"2017-05-02T00:00:00Z","timestamp":1493683200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"DOI":"10.13039\/100000001","name":"National Science Foundation","doi-asserted-by":"publisher","award":["CCF-1144520"],"award-info":[{"award-number":["CCF-1144520"]}],"id":[{"id":"10.13039\/100000001","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2017,5,2]]},"DOI":"10.1145\/3025453.3026025","type":"proceedings-article","created":{"date-parts":[[2017,5,2]],"date-time":"2017-05-02T18:37:22Z","timestamp":1493750242000},"page":"2347-2358","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":6,"title":["VoxPL"],"prefix":"10.1145","author":[{"given":"Daniel W.","family":"Barowy","sequence":"first","affiliation":[{"name":"University of Massachusetts, Amherst, MA, USA"}]},{"given":"Emery D.","family":"Berger","sequence":"additional","affiliation":[{"name":"University of Massachusetts Amherst, Amherst, USA"}]},{"given":"Daniel G.","family":"Goldstein","sequence":"additional","affiliation":[{"name":"Microsoft Research, New York, NY, USA"}]},{"given":"Siddharth","family":"Suri","sequence":"additional","affiliation":[{"name":"Microsoft Research, New York, NY, USA"}]}],"member":"320","published-online":{"date-parts":[[2017,5,2]]},"reference":[{"key":"e_1_3_2_2_3_1","doi-asserted-by":"publisher","DOI":"10.1145\/2487575.2487600"},{"key":"e_1_3_2_2_4_1","doi-asserted-by":"publisher","DOI":"10.1145\/2384616.2384663"},{"key":"e_1_3_2_2_5_1","doi-asserted-by":"publisher","DOI":"10.1145\/2927928"},{"key":"e_1_3_2_2_6_1","doi-asserted-by":"publisher","DOI":"10.1002\/9780470377963.ch1"},{"key":"e_1_3_2_2_7_1","doi-asserted-by":"publisher","DOI":"10.1214\/aos\/1176345637"},{"key":"e_1_3_2_2_8_1","volume-title":"Proceedings of the First AAAI Conference on Human Computation and Crowdsourcing, HCOMP 2013","author":"Bragg Jonathan","year":"2013","unstructured":"Jonathan Bragg , Mausam, and Daniel S. Weld . 2013. Crowdsourcing Multi-Label Classification for Taxonomy Creation . In Proceedings of the First AAAI Conference on Human Computation and Crowdsourcing, HCOMP 2013 , November 7-9, 2013 , Palm Springs, CA, USA. http:\/\/www. aaai.org\/ocs\/index.php\/HCOMP\/HCOMP13\/paper\/view\/7560 Jonathan Bragg, Mausam, and Daniel S. Weld. 2013. Crowdsourcing Multi-Label Classification for Taxonomy Creation. In Proceedings of the First AAAI Conference on Human Computation and Crowdsourcing, HCOMP 2013, November 7-9, 2013, Palm Springs, CA, USA. http:\/\/www. aaai.org\/ocs\/index.php\/HCOMP\/HCOMP13\/paper\/view\/7560"},{"key":"e_1_3_2_2_9_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.artint.2013.06.002"},{"key":"e_1_3_2_2_10_1","doi-asserted-by":"publisher","DOI":"10.1145\/2556288.2557011"},{"key":"e_1_3_2_2_11_1","doi-asserted-by":"publisher","DOI":"10.1080\/01621459.1961.10482090"},{"key":"e_1_3_2_2_12_1","doi-asserted-by":"publisher","DOI":"10.1214\/aos\/1176344552"},{"key":"e_1_3_2_2_13_1","doi-asserted-by":"publisher","DOI":"10.2307\/3314608"},{"key":"e_1_3_2_2_14_1","doi-asserted-by":"publisher","DOI":"10.1080\/01621459.1987.10478410"},{"key":"e_1_3_2_2_15_1","doi-asserted-by":"publisher","DOI":"10.1145\/1989323.1989331"},{"key":"e_1_3_2_2_16_1","doi-asserted-by":"publisher","DOI":"10.1038\/075450a0"},{"key":"e_1_3_2_2_17_1","volume-title":"Proceedings of the First AAAI Conference on Human Computation and Crowdsourcing, HCOMP 2013","author":"Hartman Bj\u00f6rn","year":"2013","unstructured":"Bj\u00f6rn Hartman and Eric Horvitz ( Eds .). 2013 . Proceedings of the First AAAI Conference on Human Computation and Crowdsourcing, HCOMP 2013 , November 7 --9 , 2013, Palm Springs, CA, USA. AAAI. http:\/\/www.aaai.org\/Library\/HCOMP\/hcomp13contents.php Bj\u00f6rn Hartman and Eric Horvitz (Eds.). 2013. Proceedings of the First AAAI Conference on Human Computation and Crowdsourcing, HCOMP 2013, November 7--9, 2013, Palm Springs, CA, USA. AAAI. http:\/\/www.aaai.org\/Library\/HCOMP\/hcomp13contents.php"},{"key":"e_1_3_2_2_18_1","first-page":"65","article-title":"A Simple Sequentially Rejective Multiple Test Procedure","volume":"6","author":"Holm Sture","year":"1979","unstructured":"Sture Holm . 1979 . A Simple Sequentially Rejective Multiple Test Procedure . Scandinavian Journal of Statistics 6 , 2 (1979), 65 -- 70 . DOI: http:\/\/dx.doi.org\/10.2307\/4615733 10.2307\/4615733 Sture Holm. 1979. A Simple Sequentially Rejective Multiple Test Procedure. Scandinavian Journal of Statistics 6, 2 (1979), 65--70. DOI: http:\/\/dx.doi.org\/10.2307\/4615733","journal-title":"Scandinavian Journal of Statistics"},{"key":"e_1_3_2_2_19_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPRW.2008.4562973"},{"key":"e_1_3_2_2_20_1","doi-asserted-by":"publisher","DOI":"10.1371\/journal.pmed.0020124"},{"key":"e_1_3_2_2_21_1","doi-asserted-by":"publisher","DOI":"10.1145\/1837885.1837906"},{"key":"e_1_3_2_2_22_1","doi-asserted-by":"publisher","DOI":"10.1145\/2470654.2470742"},{"key":"e_1_3_2_2_24_1","volume-title":"Weighty Issue: Cow Guessing Game Helps To Explain The Stock Market. (15","author":"Kestenbaum David","year":"2015","unstructured":"David Kestenbaum . 2015 . Weighty Issue: Cow Guessing Game Helps To Explain The Stock Market. (15 Aug. 2015). http:\/\/www.npr.org\/2015\/08\/20\/432978431 David Kestenbaum. 2015. Weighty Issue: Cow Guessing Game Helps To Explain The Stock Market. (15 Aug. 2015). http:\/\/www.npr.org\/2015\/08\/20\/432978431"},{"key":"e_1_3_2_2_25_1","doi-asserted-by":"publisher","DOI":"10.1145\/2441776.2441923"},{"key":"e_1_3_2_2_26_1","doi-asserted-by":"publisher","DOI":"10.1145\/2047196.2047202"},{"key":"e_1_3_2_2_27_1","doi-asserted-by":"publisher","DOI":"10.1145\/2858036.2858115"},{"key":"e_1_3_2_2_28_1","doi-asserted-by":"publisher","DOI":"10.1145\/331960.331977"},{"key":"e_1_3_2_2_29_1","volume-title":"Crowdsourcing Control: Moving Beyond Multiple Choice. In Proceedings of the Twenty-Eighth Conference on Uncertainty in Artificial Intelligence","author":"Lin Christopher H.","year":"2012","unstructured":"Christopher H. Lin , Mausam, and Daniel S. Weld . 2012 . Crowdsourcing Control: Moving Beyond Multiple Choice. In Proceedings of the Twenty-Eighth Conference on Uncertainty in Artificial Intelligence , Catalina Island, CA, USA , August 14-18, 2012 , Nando de Freitas and Kevin P. Murphy (Eds.). AUAI Press, 491--500. https:\/\/dslpitt.org\/papers\/12\/p491-lin.pdf Christopher H. Lin, Mausam, and Daniel S. Weld. 2012. Crowdsourcing Control: Moving Beyond Multiple Choice. In Proceedings of the Twenty-Eighth Conference on Uncertainty in Artificial Intelligence, Catalina Island, CA, USA, August 14-18, 2012, Nando de Freitas and Kevin P. Murphy (Eds.). AUAI Press, 491--500. https:\/\/dslpitt.org\/papers\/12\/p491-lin.pdf"},{"key":"e_1_3_2_2_30_1","doi-asserted-by":"publisher","DOI":"10.1145\/1866029.1866040"},{"key":"e_1_3_2_2_31_1","doi-asserted-by":"publisher","DOI":"10.1609\/hcomp.v2i1.13168"},{"key":"e_1_3_2_2_32_1","doi-asserted-by":"publisher","DOI":"10.14778\/2535568.2448944"},{"key":"e_1_3_2_2_33_1","volume-title":"Miller","author":"Marcus Adam","year":"2011","unstructured":"Adam Marcus , Eugene Wu , Samuel Madden , and Robert C . Miller . 2011 . Crowdsourced Databases : Query Processing with People. In CIDR 2011, Fifth Biennial Conference on Innovative Data Systems Research, Asilomar, CA, USA, January 9-12, 2011, Online Proceedings . www.cidrdb.org, 211--214. http: \/\/www.cidrdb.org\/cidr2011\/Papers\/CIDR11_Paper29.pdf Adam Marcus, Eugene Wu, Samuel Madden, and Robert C. Miller. 2011. Crowdsourced Databases: Query Processing with People. In CIDR 2011, Fifth Biennial Conference on Innovative Data Systems Research, Asilomar, CA, USA, January 9-12, 2011, Online Proceedings. www.cidrdb.org, 211--214. http: \/\/www.cidrdb.org\/cidr2011\/Papers\/CIDR11_Paper29.pdf"},{"key":"e_1_3_2_2_34_1","doi-asserted-by":"publisher","DOI":"10.3758\/s13428-011-0124-6"},{"key":"e_1_3_2_2_35_1","doi-asserted-by":"publisher","DOI":"10.1145\/2505515.2505755"},{"key":"e_1_3_2_2_36_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2015.146"},{"key":"e_1_3_2_2_37_1","unstructured":"National Institute of Standards and Technology \/ SEMATECH. 2013. NIST\/SEMATECH e-Handbook of Statistical Methods. http:\/\/www.itl.nist.gov\/div898\/handbook  National Institute of Standards and Technology \/ SEMATECH. 2013. NIST\/SEMATECH e-Handbook of Statistical Methods. http:\/\/www.itl.nist.gov\/div898\/handbook"},{"key":"e_1_3_2_2_38_1","doi-asserted-by":"publisher","DOI":"10.1145\/2047196.2047198"},{"key":"e_1_3_2_2_39_1","doi-asserted-by":"publisher","DOI":"10.1109\/WACV.2009.5403087"},{"volume-title":"Human-Machine Hybrid Computation. In CHI 2011 Workshop On Crowdsourcing And Human Computation. http: \/\/crowdresearch.org\/chi2011-workshop\/papers\/quinn.pdf","author":"Alexander","key":"e_1_3_2_2_40_1","unstructured":"Alexander J. Quinn and Benjamin B. Bederson. 2011 . Human-Machine Hybrid Computation. In CHI 2011 Workshop On Crowdsourcing And Human Computation. http: \/\/crowdresearch.org\/chi2011-workshop\/papers\/quinn.pdf Alexander J. Quinn and Benjamin B. Bederson. 2011. Human-Machine Hybrid Computation. In CHI 2011 Workshop On Crowdsourcing And Human Computation. http: \/\/crowdresearch.org\/chi2011-workshop\/papers\/quinn.pdf"},{"key":"e_1_3_2_2_41_1","doi-asserted-by":"publisher","DOI":"10.1145\/503272.503288"},{"key":"e_1_3_2_2_42_1","doi-asserted-by":"publisher","DOI":"10.1109\/JPROC.2006.884093"},{"key":"e_1_3_2_2_43_1","volume-title":"Ng","author":"Snow Rion","year":"2008","unstructured":"Rion Snow , Brendan O'Connor , Daniel Jurafsky , and Andrew Y . Ng . 2008 . Cheap and Fast - But is it Good? Evaluating Non-Expert Annotations for Natural Language Tasks. In 2008 Conference on Empirical Methods in Natural Language Processing, EMNLP 2008, Proceedings of the Conference, 25-27 October 2008, Honolulu, Hawaii, USA, A meeting of SIGDAT, a Special Interest Group of the ACL. ACL, 254--263. http:\/\/www.aclweb.org\/anthology\/D08-1027 Rion Snow, Brendan O'Connor, Daniel Jurafsky, and Andrew Y. Ng. 2008. Cheap and Fast - But is it Good? Evaluating Non-Expert Annotations for Natural Language Tasks. In 2008 Conference on Empirical Methods in Natural Language Processing, EMNLP 2008, Proceedings of the Conference, 25-27 October 2008, Honolulu, Hawaii, USA, A meeting of SIGDAT, a Special Interest Group of the ACL. ACL, 254--263. http:\/\/www.aclweb.org\/anthology\/D08-1027"},{"key":"e_1_3_2_2_44_1","unstructured":"James Surowiecki. 2005. The Wisdom of Crowds. Anchor.  James Surowiecki. 2005. The Wisdom of Crowds. Anchor."},{"key":"e_1_3_2_2_45_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2014.220"},{"key":"e_1_3_2_2_46_1","doi-asserted-by":"publisher","DOI":"10.1214\/ss\/1032280214"},{"key":"e_1_3_2_2_47_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.1991.139758"},{"key":"e_1_3_2_2_48_1","doi-asserted-by":"publisher","DOI":"10.1145\/985692.985733"},{"volume-title":"All of Nonparametric Statistics","author":"Wasserman L.","key":"e_1_3_2_2_49_1","unstructured":"L. Wasserman . 2006. All of Nonparametric Statistics . Springer New York . L. Wasserman. 2006. All of Nonparametric Statistics. Springer New York."}],"event":{"name":"CHI '17: CHI Conference on Human Factors in Computing Systems","sponsor":["SIGCHI ACM Special Interest Group on Computer-Human Interaction"],"location":"Denver Colorado USA","acronym":"CHI '17"},"container-title":["Proceedings of the 2017 CHI Conference on Human Factors in Computing Systems"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3025453.3026025","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3025453.3026025","content-type":"application\/pdf","content-version":"vor","intended-application":"syndication"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3025453.3026025","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,18]],"date-time":"2025-06-18T03:03:28Z","timestamp":1750215808000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3025453.3026025"}},"subtitle":["Programming with the Wisdom of the Crowd"],"short-title":[],"issued":{"date-parts":[[2017,5,2]]},"references-count":46,"alternative-id":["10.1145\/3025453.3026025","10.1145\/3025453"],"URL":"https:\/\/doi.org\/10.1145\/3025453.3026025","relation":{},"subject":[],"published":{"date-parts":[[2017,5,2]]},"assertion":[{"value":"2017-05-02","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}