{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,4]],"date-time":"2026-07-04T02:11:02Z","timestamp":1783131062965,"version":"3.54.6"},"publisher-location":"New York, NY, USA","reference-count":158,"publisher":"ACM","license":[{"start":{"date-parts":[[2013,2,23]],"date-time":"2013-02-23T00:00:00Z","timestamp":1361577600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2013,2,23]]},"DOI":"10.1145\/2441776.2441923","type":"proceedings-article","created":{"date-parts":[[2013,2,22]],"date-time":"2013-02-22T19:25:33Z","timestamp":1361561133000},"page":"1301-1318","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":724,"title":["The future of crowd work"],"prefix":"10.1145","author":[{"given":"Aniket","family":"Kittur","sequence":"first","affiliation":[{"name":"Carnegie Mellon University, Pittsburgh, Pennsylvania, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Jeffrey V.","family":"Nickerson","sequence":"additional","affiliation":[{"name":"Stevens Institute of Technology, Hoboken, New Jersey, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Michael","family":"Bernstein","sequence":"additional","affiliation":[{"name":"Computer Science Department, Stanford University, Stanford, CA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Elizabeth","family":"Gerber","sequence":"additional","affiliation":[{"name":"Northwestern University, Evanston, Illinois, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Aaron","family":"Shaw","sequence":"additional","affiliation":[{"name":"Northwestern University, Evanston, Illinois, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"John","family":"Zimmerman","sequence":"additional","affiliation":[{"name":"Carnegie Mellon University, Pittsburgh, Pennsylvania, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Matt","family":"Lease","sequence":"additional","affiliation":[{"name":"University of Texas at Austin, Austin, Texas, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"John","family":"Horton","sequence":"additional","affiliation":[{"name":"oDesk Research & Harvard University, Cambridge, Massachusetts, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2013,2,23]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"publisher","DOI":"10.1145\/2047196.2047203"},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.1145\/985692.985733"},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.5555\/1766171.1766196"},{"key":"e_1_3_2_1_4_1","volume":"2011","author":"Albrecht","unstructured":"Albrecht , J. The 2011 Nobel Memorial Prize in Search Theory. Department of Economics, Georgetown University. http:\/\/www9.georgetown.edu\/faculty\/albrecht\/SJESurvey.pdf, (2011). Albrecht, J. The 2011 Nobel Memorial Prize in Search Theory. Department of Economics, Georgetown University. http:\/\/www9.georgetown.edu\/faculty\/albrecht\/SJESurvey.pdf, (2011).","journal-title":"J. The"},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1145\/2187836.2187950"},{"key":"e_1_3_2_1_6_1","first-page":"4","article-title":"Crowdsourcing Higher Education","volume":"7","author":"Anderson M","year":"2011","unstructured":"Anderson , M . Crowdsourcing Higher Education : A Design Proposal for Distributed Learning. MERLOT Journal of Online Learning and Teaching 7 , 4 ( 2011 ), 576--590. Anderson, M. Crowdsourcing Higher Education: A Design Proposal for Distributed Learning. MERLOT Journal of Online Learning and Teaching 7, 4 (2011), 576--590.","journal-title":"A Design Proposal for Distributed Learning. MERLOT Journal of Online Learning and Teaching"},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"publisher","DOI":"10.1145\/2207676.2208699"},{"key":"e_1_3_2_1_8_1","volume-title":"Optimal Design of Crowdsourcing Contests. ICIS 2009 Proceedings","author":"Archak N.","year":"2009","unstructured":"Archak , N. and Sundararajan , A . Optimal Design of Crowdsourcing Contests. ICIS 2009 Proceedings , ( 2009 ), 200. Archak, N. and Sundararajan, A. Optimal Design of Crowdsourcing Contests. ICIS 2009 Proceedings, (2009), 200."},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1145\/72551.72552"},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.2307\/2118383"},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.1145\/1979742.1979606"},{"key":"e_1_3_2_1_12_1","volume-title":"Yale Univ Pr","author":"Benkler Y.","year":"2006","unstructured":"Benkler , Y. The wealth of networks: How social production transforms markets and freedom . Yale Univ Pr , 2006 . Benkler, Y. The wealth of networks: How social production transforms markets and freedom. Yale Univ Pr, 2006."},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1145\/2047196.2047201"},{"key":"e_1_3_2_1_14_1","volume-title":"Proc. Collective Intelligence 2012","author":"Bernstein M. S.","year":"2012","unstructured":"Bernstein , M. S. , Karger , D. R. , Miller , R. C. , and Brandt , J . Analytic Methods for Optimizing Realtime Crowdsourcing . Proc. Collective Intelligence 2012 , ( 2012 ). Bernstein, M. S., Karger, D. R., Miller, R. C., and Brandt, J. Analytic Methods for Optimizing Realtime Crowdsourcing. Proc. Collective Intelligence 2012, (2012)."},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.1145\/1866029.1866078"},{"key":"e_1_3_2_1_16_1","volume-title":"Fifth International AAAI Conference on Weblogs and Social Media, AAAI Publications","author":"Bernstein M. S.","year":"2011","unstructured":"Bernstein , M. S. , Monroy-Hern\u00e1ndez , A. , Harry , D. , Andr\u00e9 , P. , Panovich , K. , and Vargas , G . 4chan and \/b\/: An Analysis of Anonymity and Ephemerality in a Large Online Community . Fifth International AAAI Conference on Weblogs and Social Media, AAAI Publications ( 2011 ). Bernstein, M. S., Monroy-Hern\u00e1ndez, A., Harry, D., Andr\u00e9, P., Panovich, K., and Vargas, G. 4chan and \/b\/: An Analysis of Anonymity and Ephemerality in a Large Online Community. Fifth International AAAI Conference on Weblogs and Social Media, AAAI Publications (2011)."},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.1145\/1866029.1866080"},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1080\/0260293990240405"},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","DOI":"10.1287\/mnsc.1110.1322"},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.5210\/fm.v13i6.2159"},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.1016\/S0169-7552(98)00110-X"},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.1145\/1099203.1099205"},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.5555\/1866696.1866697"},{"key":"e_1_3_2_1_24_1","volume-title":"Proceedings of the 2009 Conference on Empirical Methods in Natural Language Processing","volume":"1","author":"Callison-Burch C.","year":"2009","unstructured":"Callison-Burch , C. Fast , cheap, and creative : evaluating translation quality using Amazon's Mechanical Turk . Proceedings of the 2009 Conference on Empirical Methods in Natural Language Processing : Volume 1-Volume 1 , ( 2009 ), 286--295. Callison-Burch, C. Fast, cheap, and creative: evaluating translation quality using Amazon's Mechanical Turk. Proceedings of the 2009 Conference on Empirical Methods in Natural Language Processing: Volume 1-Volume 1, (2009), 286--295."},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.1016\/S0167-739X(00)00104-7"},{"key":"e_1_3_2_1_26_1","volume-title":"Breaking monotony with meaning: Motivation in crowdsourcing markets","author":"Chandler D.","year":"2010","unstructured":"Chandler , D. and Kapelner , A . Breaking monotony with meaning: Motivation in crowdsourcing markets . University of Chicago mimeo , ( 2010 ). Chandler, D. and Kapelner, A. Breaking monotony with meaning: Motivation in crowdsourcing markets. University of Chicago mimeo, (2010)."},{"key":"e_1_3_2_1_27_1","first-page":"3","volume":"5","author":"Chen J. J.","year":"2011","unstructured":"Chen , J. J. , Menezes , N. J. , and Bradley , A. D . Opportunities for Crowdsourcing Research on Amazon Mechanical Turk. Interfaces 5 , ( 2011 ), 3 . Chen, J. J., Menezes, N. J., and Bradley, A. D. Opportunities for Crowdsourcing Research on Amazon Mechanical Turk. Interfaces 5, (2011), 3.","journal-title":"Opportunities for Crowdsourcing Research on Amazon Mechanical Turk. Interfaces"},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"publisher","DOI":"10.1162\/DAED_a_00114"},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.1145\/1837885.1837889"},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"publisher","DOI":"10.1111\/j.1468-0335.1937.tb00002.x"},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"publisher","DOI":"10.1038\/nature09304"},{"key":"e_1_3_2_1_32_1","volume-title":"Proc. AAAI '10","author":"Dai P.","year":"2010","unstructured":"Dai , P. , Mausam , and Weld, D . Decision-theoretic control of crowd-sourced workflows . Proc. AAAI '10 , ( 2010 ). Dai, P., Mausam, and Weld, D. Decision-theoretic control of crowd-sourced workflows. Proc. AAAI '10, (2010)."},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"publisher","DOI":"10.5555\/2900423.2900606"},{"key":"e_1_3_2_1_34_1","unstructured":"Dean J. and Ghemawat S. MapReduce: Simplified Data Processing on Large Clusters. To appear in OSDI (2004) 1.   Dean J. and Ghemawat S. MapReduce: Simplified Data Processing on Large Clusters. To appear in OSDI (2004) 1."},{"key":"e_1_3_2_1_35_1","doi-asserted-by":"publisher","DOI":"10.1145\/2282338.2282361"},{"key":"e_1_3_2_1_36_1","volume-title":"Proc. 22nd Annual Conference on Learning Theory","author":"Dekel O.","year":"2009","unstructured":"Dekel , O. and Shamir , O . Vox populi: Collecting high-quality labels from a crowd . Proc. 22nd Annual Conference on Learning Theory , ( 2009 ). Dekel, O. and Shamir, O. Vox populi: Collecting high-quality labels from a crowd. Proc. 22nd Annual Conference on Learning Theory, (2009)."},{"key":"e_1_3_2_1_37_1","volume-title":"Collective Intelligence","author":"Della Penn N.","year":"2012","unstructured":"Della Penn , N. and Reid , M. D . Crowd & Prejudice: An Impossibility Theorem for Crowd Labelling without a Gold Standard . Collective Intelligence , ( 2012 ). Della Penn, N. and Reid, M. D. Crowd & Prejudice: An Impossibility Theorem for Crowd Labelling without a Gold Standard. Collective Intelligence, (2012)."},{"key":"e_1_3_2_1_38_1","doi-asserted-by":"publisher","DOI":"10.1145\/501158.501177"},{"key":"e_1_3_2_1_39_1","doi-asserted-by":"publisher","DOI":"10.1023\/A:1022883727209"},{"key":"e_1_3_2_1_40_1","volume-title":"Voyager","author":"Doctorow C.","year":"2010","unstructured":"Doctorow , C. For the Win . Voyager , 2010 . Doctorow, C. For the Win. Voyager, 2010."},{"key":"e_1_3_2_1_41_1","doi-asserted-by":"publisher","DOI":"10.5555\/646334.687813"},{"key":"e_1_3_2_1_42_1","doi-asserted-by":"publisher","DOI":"10.1145\/2145204.2145355"},{"key":"e_1_3_2_1_43_1","doi-asserted-by":"publisher","DOI":"10.1145\/1753326.1753688"},{"key":"e_1_3_2_1_44_1","volume-title":"Proceedings of CrowdConf","author":"Felstiner A.","year":"2010","unstructured":"Felstiner , A. Sweatshop or Paper Route?: Child Labor Laws and In-Game Work . Proceedings of CrowdConf , ( 2010 ). Felstiner, A. Sweatshop or Paper Route?: Child Labor Laws and In-Game Work. Proceedings of CrowdConf, (2010)."},{"key":"e_1_3_2_1_45_1","first-page":"143","article-title":"Working the Crowd","volume":"32","author":"Felstiner A","year":"2011","unstructured":"Felstiner , A . Working the Crowd : Employment and Labor Law in the Crowdsourcing Industry. Berkeley J. Emp. & Lab. L. 32 , ( 2011 ), 143 -- 143 . Felstiner, A. Working the Crowd: Employment and Labor Law in the Crowdsourcing Industry. Berkeley J. Emp. & Lab. L. 32, (2011), 143--143.","journal-title":"Employment and Labor Law in the Crowdsourcing Industry. Berkeley J. Emp. & Lab. L."},{"key":"e_1_3_2_1_46_1","doi-asserted-by":"publisher","DOI":"10.1145\/1989323.1989331"},{"key":"e_1_3_2_1_47_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-540-69132-7_21"},{"key":"e_1_3_2_1_48_1","unstructured":"Gerber E. and Dontcheva M. Career Aspirations for Crowdworkers. In preparation.  Gerber E. and Dontcheva M. Career Aspirations for Crowdworkers. In preparation."},{"key":"e_1_3_2_1_49_1","volume-title":"Crowdfunding: Why creators and supporters participate","author":"Gerber E.","year":"2012","unstructured":"Gerber , E. , Hui , J. , and Kuo , P . Crowdfunding: Why creators and supporters participate . Segal Design Institute , Evanston, IL , 2012 . Gerber, E., Hui, J., and Kuo, P. Crowdfunding: Why creators and supporters participate. Segal Design Institute, Evanston, IL, 2012."},{"key":"e_1_3_2_1_50_1","volume-title":"European Commission","author":"Ghosh R.","year":"2002","unstructured":"Ghosh , R. and Glott , R . Free\/Libre and Open Source Software: Survey and Study . European Commission , 2002 . Ghosh, R. and Glott, R. Free\/Libre and Open Source Software: Survey and Study. European Commission, 2002."},{"key":"e_1_3_2_1_51_1","doi-asserted-by":"publisher","DOI":"10.1145\/2231816.2231817"},{"key":"e_1_3_2_1_52_1","volume-title":"UNU-MERIT","author":"Glott R.","year":"2010","unstructured":"Glott , R. , Ghosh , R. , and Schmidt , P . Wikipedia Survey . UNU-MERIT , Maastricht, Netherlands , 2010 . Glott, R., Ghosh, R., and Schmidt, P. Wikipedia Survey. UNU-MERIT, Maastricht, Netherlands, 2010."},{"key":"e_1_3_2_1_53_1","volume-title":"Proc. Graphics Interface'91","author":"Greenberg S.","year":"1991","unstructured":"Greenberg , S. and Bohnet , R . GroupSketch: A multi-user sketchpad for geographically-distributed small groups . Proc. Graphics Interface'91 , ( 1991 ). Greenberg, S. and Bohnet, R. GroupSketch: A multi-user sketchpad for geographically-distributed small groups. Proc. Graphics Interface'91, (1991)."},{"key":"e_1_3_2_1_54_1","volume-title":"When Computers Were Human","author":"Grier D. A.","year":"2005","unstructured":"Grier , D. A. When Computers Were Human . Princeton University Press , 2005 . Grier, D. A. When Computers Were Human. Princeton University Press, 2005."},{"key":"e_1_3_2_1_55_1","volume-title":"Proc. HCOMP '11","author":"Grier D. A.","year":"2011","unstructured":"Grier , D. A. Error Identification and Correction in Human Computation: Lessons from the WPA . Proc. HCOMP '11 , ( 2011 ). Grier, D. A. Error Identification and Correction in Human Computation: Lessons from the WPA. Proc. HCOMP '11, (2011)."},{"key":"e_1_3_2_1_56_1","doi-asserted-by":"publisher","DOI":"10.1145\/2207676.2208320"},{"key":"e_1_3_2_1_57_1","volume-title":"Motivation through the design of work: Test of a theory. Organizational behavior and human performance 16, 2","author":"Hackman J. R.","year":"1976","unstructured":"Hackman , J. R. and Oldham , G. R . Motivation through the design of work: Test of a theory. Organizational behavior and human performance 16, 2 ( 1976 ), 250--279. Hackman, J. R. and Oldham, G. R. Motivation through the design of work: Test of a theory. Organizational behavior and human performance 16, 2 (1976), 250--279."},{"key":"e_1_3_2_1_58_1","doi-asserted-by":"publisher","DOI":"10.1145\/2207676.2208619"},{"key":"e_1_3_2_1_59_1","doi-asserted-by":"publisher","DOI":"10.1145\/1965724.1965744"},{"key":"e_1_3_2_1_60_1","doi-asserted-by":"crossref","DOI":"10.7551\/mitpress\/2464.001.0001","volume-title":"Distributed work","author":"Hinds P.","year":"2002","unstructured":"Hinds , P. Distributed work . The MIT Press , 2002 . Hinds, P. Distributed work. The MIT Press, 2002."},{"key":"e_1_3_2_1_61_1","volume-title":"Task partitioning: An innovation process variable. Research policy 19, 5","author":"Von Hippel E.","year":"1990","unstructured":"Von Hippel , E. Task partitioning: An innovation process variable. Research policy 19, 5 ( 1990 ), 407--418. Von Hippel, E. Task partitioning: An innovation process variable. Research policy 19, 5 (1990), 407--418."},{"key":"e_1_3_2_1_62_1","doi-asserted-by":"publisher","DOI":"10.1093\/jleo\/7.special_issue.24"},{"key":"e_1_3_2_1_63_1","doi-asserted-by":"publisher","DOI":"10.1145\/1807342.1807376"},{"key":"e_1_3_2_1_64_1","doi-asserted-by":"publisher","DOI":"10.1145\/1837885.1837906"},{"key":"e_1_3_2_1_65_1","volume-title":"http:\/\/www.behind-the-enemy-lines.com\/2010\/07\/mechanical-turk-low-wages-and-market.html","author":"Ipeirotis P. G.","year":"2010","unstructured":"Ipeirotis , P. G. Mechanical Turk , Low Wages, and the Market for Lemons. http:\/\/www.behind-the-enemy-lines.com\/2010\/07\/mechanical-turk-low-wages-and-market.html , 2010 . Ipeirotis, P. G. Mechanical Turk, Low Wages, and the Market for Lemons. http:\/\/www.behind-the-enemy-lines.com\/2010\/07\/mechanical-turk-low-wages-and-market.html, 2010."},{"key":"e_1_3_2_1_66_1","doi-asserted-by":"publisher","DOI":"10.1145\/142750.142977"},{"key":"e_1_3_2_1_67_1","volume-title":"Proc. AAMAS '12","author":"Kamar E.","year":"2012","unstructured":"Kamar , E. , Hacker , S. , and Horvitz , E . Combining Human and Machine Intelligence in Large-scale Crowdsourcing . Proc. AAMAS '12 , ( 2012 ). Kamar, E., Hacker, S., and Horvitz, E. Combining Human and Machine Intelligence in Large-scale Crowdsourcing. Proc. AAMAS '12, (2012)."},{"key":"e_1_3_2_1_68_1","doi-asserted-by":"publisher","DOI":"10.1126\/science.1127207"},{"key":"e_1_3_2_1_69_1","doi-asserted-by":"publisher","DOI":"10.1023\/A:1008689307411"},{"key":"e_1_3_2_1_70_1","first-page":"769","article-title":"On the folly of rewarding A, while hoping for B","author":"Kerr S","year":"1975","unstructured":"Kerr , S . On the folly of rewarding A, while hoping for B . Academy of Management Journal, ( 1975 ), 769 -- 783 . Kerr, S. On the folly of rewarding A, while hoping for B. Academy of Management Journal, (1975), 769--783.","journal-title":"Academy of Management Journal, ("},{"key":"e_1_3_2_1_71_1","doi-asserted-by":"publisher","DOI":"10.1145\/1926180.1926195"},{"key":"e_1_3_2_1_72_1","doi-asserted-by":"publisher","DOI":"10.1145\/1357054.1357127"},{"key":"e_1_3_2_1_73_1","doi-asserted-by":"publisher","DOI":"10.1145\/2145204.2145357"},{"key":"e_1_3_2_1_74_1","doi-asserted-by":"publisher","DOI":"10.1145\/1518701.1518928"},{"key":"e_1_3_2_1_75_1","doi-asserted-by":"publisher","DOI":"10.1145\/2047196.2047202"},{"key":"e_1_3_2_1_76_1","doi-asserted-by":"publisher","DOI":"10.1145\/1460563.1460639"},{"key":"e_1_3_2_1_77_1","doi-asserted-by":"publisher","DOI":"10.1145\/1869086.1869096"},{"key":"e_1_3_2_1_78_1","doi-asserted-by":"publisher","DOI":"10.1145\/1837885.1837890"},{"key":"e_1_3_2_1_79_1","doi-asserted-by":"publisher","DOI":"10.1109\/MIC.2012.72"},{"key":"e_1_3_2_1_80_1","doi-asserted-by":"publisher","DOI":"10.1145\/1979742.1979865"},{"key":"e_1_3_2_1_81_1","doi-asserted-by":"publisher","DOI":"10.1145\/1215942.1215943"},{"key":"e_1_3_2_1_82_1","volume-title":"Perspectives on Free and Open Source Software","author":"Lakhani K.","year":"2005","unstructured":"Lakhani , K. and Wolf , B . Why hackers do what they do: Understanding motivation and effort in free\/open source software projects . In J. Feller, B. Fitzgerald, S. A. Hissam and K. R. Lakhani, eds., Perspectives on Free and Open Source Software . MIT Press , 2005 , 3--22. Lakhani, K. and Wolf, B. Why hackers do what they do: Understanding motivation and effort in free\/open source software projects. In J. Feller, B. Fitzgerald, S. A. Hissam and K. R. Lakhani, eds., Perspectives on Free and Open Source Software. MIT Press, 2005, 3--22."},{"key":"e_1_3_2_1_83_1","doi-asserted-by":"publisher","DOI":"10.1145\/2047196.2047200"},{"key":"e_1_3_2_1_84_1","volume-title":"Proc. Collective Intelligence 2012","author":"Lasecki W. S.","year":"2012","unstructured":"Lasecki , W. S. , White , S. C. , Murray , K. I. , and Bigham , J. P . Crowd memory: Learning in the collective . Proc. Collective Intelligence 2012 , ( 2012 ). Lasecki, W. S., White, S. C., Murray, K. I., and Bigham, J. P. Crowd memory: Learning in the collective. Proc. Collective Intelligence 2012, (2012)."},{"key":"e_1_3_2_1_85_1","volume-title":"Proc. SIGIR 2010 Workshop on Crowdsourcing for Search Evaluation","author":"Le J.","year":"2010","unstructured":"Le , J. , Edmonds , A. , Hester , V. , and Biewald , L . Ensuring quality in crowdsourced search relevance evaluation: The effects of training question distribution . Proc. SIGIR 2010 Workshop on Crowdsourcing for Search Evaluation , ( 2010 ), 21--26. Le, J., Edmonds, A., Hester, V., and Biewald, L. Ensuring quality in crowdsourced search relevance evaluation: The effects of training question distribution. Proc. SIGIR 2010 Workshop on Crowdsourcing for Search Evaluation, (2010), 21--26."},{"key":"e_1_3_2_1_86_1","doi-asserted-by":"publisher","DOI":"10.1145\/1978942.1979048"},{"key":"e_1_3_2_1_87_1","doi-asserted-by":"publisher","DOI":"10.1145\/1866029.1866040"},{"key":"e_1_3_2_1_88_1","doi-asserted-by":"crossref","unstructured":"Littler C. R. Understanding Taylorism. British Journal of Sociology (1978) 185--202.  Littler C. R. Understanding Taylorism. British Journal of Sociology (1978) 185--202.","DOI":"10.2307\/589888"},{"key":"e_1_3_2_1_89_1","doi-asserted-by":"publisher","DOI":"10.1145\/174666.174668"},{"key":"e_1_3_2_1_90_1","doi-asserted-by":"publisher","DOI":"10.1145\/214762.214766"},{"key":"e_1_3_2_1_91_1","volume-title":"Proc. AAAI '11","author":"Mao A.","year":"2011","unstructured":"Mao , A. , Parkes , D. C. , Procaccia , A. D. , and Zhang , H . Human Computation and Multiagent Systems: An Algorithmic Perspective . Proc. AAAI '11 , ( 2011 ). Mao, A., Parkes, D. C., Procaccia, A. D., and Zhang, H. Human Computation and Multiagent Systems: An Algorithmic Perspective. Proc. AAAI '11, (2011)."},{"key":"e_1_3_2_1_92_1","unstructured":"March J. G. and Simon H. A. Organizations. (1958).  March J. G. and Simon H. A. Organizations. (1958)."},{"key":"e_1_3_2_1_93_1","doi-asserted-by":"publisher","DOI":"10.14778\/2535568.2448944"},{"key":"e_1_3_2_1_94_1","doi-asserted-by":"publisher","DOI":"10.1145\/1600150.1600175"},{"key":"e_1_3_2_1_95_1","volume-title":"Proc. LREC","author":"McGraw I.","year":"2010","unstructured":"McGraw , I. , Lee , C. , Hetherington , L. , Seneff , S. , and Glass , J. R . Collecting voices from the cloud . Proc. LREC , ( 2010 ). McGraw, I., Lee, C., Hetherington, L., Seneff, S., and Glass, J. R. Collecting voices from the cloud. Proc. LREC, (2010)."},{"key":"e_1_3_2_1_96_1","volume-title":"Communicating and mobile systems: the {symbol for pi}-calculus","author":"Milner R.","year":"1999","unstructured":"Milner , R. Communicating and mobile systems: the {symbol for pi}-calculus . Cambridge Univ Pr , 1999 . Milner, R. Communicating and mobile systems: the {symbol for pi}-calculus. Cambridge Univ Pr, 1999."},{"key":"e_1_3_2_1_97_1","volume-title":"An emerging strategy of \"direct\" research. Administrative science quarterly 24, 4","author":"Mintzberg H.","year":"1979","unstructured":"Mintzberg , H. An emerging strategy of \"direct\" research. Administrative science quarterly 24, 4 ( 1979 ), 582--589. Mintzberg, H. An emerging strategy of \"direct\" research. Administrative science quarterly 24, 4 (1979), 582--589."},{"key":"e_1_3_2_1_98_1","doi-asserted-by":"publisher","DOI":"10.1145\/99332.99369"},{"key":"e_1_3_2_1_99_1","doi-asserted-by":"publisher","DOI":"10.3233\/ICG-1999-22402"},{"key":"e_1_3_2_1_100_1","volume-title":"Proc. HCOMP '11","author":"Narula P.","year":"2011","unstructured":"Narula , P. , Gutheim 1, P., Rolnitzky , D. , Kulkarni , A. , and Hartmann , B . MobileWorks: A Mobile Crowdsourcing Platform for Workers at the Bottom of the Pyramid . Proc. HCOMP '11 , ( 2011 ). Narula, P., Gutheim1, P., Rolnitzky, D., Kulkarni, A., and Hartmann, B. MobileWorks: A Mobile Crowdsourcing Platform for Workers at the Bottom of the Pyramid. Proc. HCOMP '11, (2011)."},{"key":"e_1_3_2_1_101_1","doi-asserted-by":"publisher","DOI":"10.1109\/HICSS.2011.75"},{"key":"e_1_3_2_1_102_1","doi-asserted-by":"publisher","DOI":"10.5555\/576915"},{"key":"e_1_3_2_1_103_1","doi-asserted-by":"publisher","DOI":"10.1145\/2047196.2047198"},{"key":"e_1_3_2_1_104_1","doi-asserted-by":"publisher","DOI":"10.17705\/1thci.00005"},{"key":"e_1_3_2_1_105_1","doi-asserted-by":"publisher","DOI":"10.1145\/1978942.1979148"},{"key":"e_1_3_2_1_106_1","volume-title":"Bulletin of the American Astronomical Society","author":"Raddick J.","year":"2008","unstructured":"Raddick , J. , Lintott , C. , Bamford , S. , Galaxy Zoo : Motivations of Citizen Scientists . Bulletin of the American Astronomical Society , ( 2008 ), 240. Raddick, J., Lintott, C., Bamford, S., et al. Galaxy Zoo: Motivations of Citizen Scientists. Bulletin of the American Astronomical Society, (2008), 240."},{"key":"e_1_3_2_1_107_1","doi-asserted-by":"publisher","DOI":"10.1017\/CBO9780511813740.012"},{"key":"e_1_3_2_1_108_1","doi-asserted-by":"publisher","DOI":"10.1007\/s12130-999-1026-0"},{"key":"e_1_3_2_1_109_1","doi-asserted-by":"publisher","DOI":"10.5555\/998675.999472"},{"key":"e_1_3_2_1_110_1","doi-asserted-by":"publisher","DOI":"10.5555\/1793114.1793127"},{"key":"e_1_3_2_1_111_1","doi-asserted-by":"publisher","DOI":"10.1016\/S0278-0984(02)11030-3"},{"key":"e_1_3_2_1_112_1","volume-title":"Dilemmas in a general theory of planning. Policy sciences 4, 2","author":"Rittel H. W. J.","year":"1973","unstructured":"Rittel , H. W. J. and Webber , M. M . Dilemmas in a general theory of planning. Policy sciences 4, 2 ( 1973 ), 155--169. Rittel, H. W. J. and Webber, M. M. Dilemmas in a general theory of planning. Policy sciences 4, 2 (1973), 155--169."},{"key":"e_1_3_2_1_113_1","volume-title":"Cognitive Tutor: Applied research in mathematics education. Psychonomic bulletin & review 14, 2","author":"Ritter S.","year":"2007","unstructured":"Ritter , S. , Anderson , J. R. , Koedinger , K. R. , and Corbett , A . Cognitive Tutor: Applied research in mathematics education. Psychonomic bulletin & review 14, 2 ( 2007 ), 249--255. Ritter, S., Anderson, J. R., Koedinger, K. R., and Corbett, A. Cognitive Tutor: Applied research in mathematics education. Psychonomic bulletin & review 14, 2 (2007), 249--255."},{"key":"e_1_3_2_1_114_1","doi-asserted-by":"publisher","DOI":"10.1145\/192844.192875"},{"key":"e_1_3_2_1_115_1","volume-title":"Proceedings of the Fifth International AAAI Conference on Weblogs and Social Media: Barcelona","author":"Rogstadius J.","year":"2011","unstructured":"Rogstadius , J. , Kostakos , V. , Kittur , A. , Smus , B. , Laredo , J. , and Vukovic , M . An Assessment of Intrinsic and Extrinsic Motivation on Task Performance in Crowdsourcing Markets . Proceedings of the Fifth International AAAI Conference on Weblogs and Social Media: Barcelona , Spain , ( 2011 ). Rogstadius, J., Kostakos, V., Kittur, A., Smus, B., Laredo, J., and Vukovic, M. An Assessment of Intrinsic and Extrinsic Motivation on Task Performance in Crowdsourcing Markets. Proceedings of the Fifth International AAAI Conference on Weblogs and Social Media: Barcelona, Spain, (2011)."},{"key":"e_1_3_2_1_116_1","doi-asserted-by":"publisher","DOI":"10.1145\/1753846.1753873"},{"key":"e_1_3_2_1_117_1","doi-asserted-by":"publisher","DOI":"10.1111\/1468-0262.00335"},{"key":"e_1_3_2_1_118_1","doi-asserted-by":"publisher","DOI":"10.2307\/2087434"},{"key":"e_1_3_2_1_119_1","doi-asserted-by":"publisher","DOI":"10.1145\/2380116.2380125"},{"key":"e_1_3_2_1_120_1","doi-asserted-by":"publisher","DOI":"10.1145\/2047196.2047199"},{"key":"e_1_3_2_1_121_1","volume-title":"Paris","author":"Sadlon E.","year":"2008","unstructured":"Sadlon , E. , Barrett , S. , Sakamoto , Y. , and Nickerson , J. V . The Karma of Digg: Reciprocity in Online Social Networks . Paris , December 2008 . Proceedings of the 18th Annual Workshop on Information Technologies and Systems (WITS) , (2008). Sadlon, E., Barrett, S., Sakamoto, Y., and Nickerson, J. V. The Karma of Digg: Reciprocity in Online Social Networks. Paris, December 2008. Proceedings of the 18th Annual Workshop on Information Technologies and Systems (WITS), (2008)."},{"key":"e_1_3_2_1_122_1","doi-asserted-by":"publisher","DOI":"10.1145\/2093548.2093553"},{"key":"e_1_3_2_1_123_1","volume-title":"Taking CSCW seriously. Computer Supported Cooperative Work (CSCW) 1, 1","author":"Schmidt K.","year":"1992","unstructured":"Schmidt , K. and Bannon , L . Taking CSCW seriously. Computer Supported Cooperative Work (CSCW) 1, 1 ( 1992 ), 7--40. Schmidt, K. and Bannon, L. Taking CSCW seriously. Computer Supported Cooperative Work (CSCW) 1, 1 (1992), 7--40."},{"key":"e_1_3_2_1_124_1","first-page":"1","article-title":"Voluntary Engagement in an Open Web-Based Encyclopedia","volume":"12","author":"Schroer J.","year":"2009","unstructured":"Schroer , J. and Hertel , G . Voluntary Engagement in an Open Web-Based Encyclopedia : Wikipedians and Why They Do It. Media Psychology 12 , 1 ( 2009 ), 96. Schroer, J. and Hertel, G. Voluntary Engagement in an Open Web-Based Encyclopedia: Wikipedians and Why They Do It. Media Psychology 12, 1 (2009), 96.","journal-title":"Wikipedians and Why They Do It. Media Psychology"},{"key":"e_1_3_2_1_125_1","doi-asserted-by":"crossref","unstructured":"Shahaf D. and Horvitz E. Generalized task markets for human and machine computation. (2010).  Shahaf D. and Horvitz E. Generalized task markets for human and machine computation. (2010).","DOI":"10.1609\/aaai.v24i1.7652"},{"key":"e_1_3_2_1_126_1","first-page":"3","volume":"10","author":"Shamir B.","year":"1985","unstructured":"Shamir , B. and Salomon , I . Work-at-Home and the Quality of Working Life. The Academy of Management Review 10 , 3 ( 1985 ), 455--464. Shamir, B. and Salomon, I. Work-at-Home and the Quality of Working Life. The Academy of Management Review 10, 3 (1985), 455--464.","journal-title":"Work-at-Home and the Quality of Working Life. The Academy of Management Review"},{"key":"e_1_3_2_1_127_1","doi-asserted-by":"publisher","DOI":"10.1145\/1958824.1958865"},{"key":"e_1_3_2_1_128_1","volume-title":"System Sciences, 2003. Proceedings of the 36th Annual Hawaii International Conference on Social Sciences","author":"Shen M.","year":"2003","unstructured":"Shen , M. , Tzeng , G. H. , and Liu , D. R . Multi-criteria task assignment in workflow management systems . System Sciences, 2003. Proceedings of the 36th Annual Hawaii International Conference on Social Sciences , ( 2003 ). Shen, M., Tzeng, G. H., and Liu, D. R. Multi-criteria task assignment in workflow management systems. System Sciences, 2003. Proceedings of the 36th Annual Hawaii International Conference on Social Sciences, (2003)."},{"key":"e_1_3_2_1_129_1","doi-asserted-by":"publisher","DOI":"10.1145\/1401890.1401965"},{"key":"e_1_3_2_1_130_1","doi-asserted-by":"publisher","DOI":"10.1145\/1869086.1869100"},{"key":"e_1_3_2_1_131_1","doi-asserted-by":"publisher","DOI":"10.1145\/1837885.1837891"},{"key":"e_1_3_2_1_132_1","doi-asserted-by":"publisher","DOI":"10.1145\/280277.280278"},{"key":"e_1_3_2_1_133_1","volume-title":"The Wealth of Nations (1776)","author":"Smith A.","year":"1937","unstructured":"Smith , A. The Wealth of Nations (1776) . New York : Modern Library , ( 1937 ), 740. Smith, A. The Wealth of Nations (1776). New York: Modern Library, (1937), 740."},{"key":"e_1_3_2_1_134_1","doi-asserted-by":"publisher","DOI":"10.5555\/1613715.1613751"},{"key":"e_1_3_2_1_135_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPRW.2008.4562953"},{"key":"e_1_3_2_1_136_1","volume-title":"Spectra","author":"Sterling B.","year":"1999","unstructured":"Sterling , B. A good old-fashioned future: stories . Spectra , 1999 . Sterling, B. A good old-fashioned future: stories. Spectra, 1999."},{"key":"e_1_3_2_1_137_1","volume-title":"Shaping things","author":"Sterling B.","year":"2005","unstructured":"Sterling , B. Shaping things . 2005 . Sterling, B. Shaping things. 2005."},{"key":"e_1_3_2_1_138_1","doi-asserted-by":"publisher","DOI":"10.1023\/A:1011457324641"},{"key":"e_1_3_2_1_139_1","volume-title":"The Wisdom of Crowds","author":"Surowiecki J.","year":"2005","unstructured":"Surowiecki , J. The Wisdom of Crowds . Random House , New York , 2005 . Surowiecki, J. The Wisdom of Crowds. Random House, New York, 2005."},{"key":"e_1_3_2_1_140_1","doi-asserted-by":"publisher","DOI":"10.1145\/1924421.1924441"},{"key":"e_1_3_2_1_141_1","volume-title":"The principles of scientific management","author":"Taylor F. W.","year":"1911","unstructured":"Taylor , F. W. The principles of scientific management . Harper & Brothers , New York , 1911 . Taylor, F. W. The principles of scientific management. Harper & Brothers, New York, 1911."},{"key":"e_1_3_2_1_142_1","volume-title":"The evolution of socio-technical systems: A conceptual framework and an action research program","author":"Trist E. L.","unstructured":"Trist , E. L. The evolution of socio-technical systems: A conceptual framework and an action research program . Ontario Quality of Working Life Center . Trist, E. L. The evolution of socio-technical systems: A conceptual framework and an action research program. Ontario Quality of Working Life Center."},{"key":"e_1_3_2_1_143_1","doi-asserted-by":"crossref","unstructured":"Van de Ven A. H. Delbecq A. L. and Koenig  Jr R. Determinants of coordination modes within organizations. American sociological review (1976) 322--338.  Van de Ven A. H. Delbecq A. L. and Koenig Jr R. Determinants of coordination modes within organizations. American sociological review (1976) 322--338.","DOI":"10.2307\/2094477"},{"key":"e_1_3_2_1_144_1","doi-asserted-by":"publisher","DOI":"10.1109\/COMSNETS.2012.6151333"},{"key":"e_1_3_2_1_145_1","doi-asserted-by":"publisher","DOI":"10.1145\/2187836.2187928"},{"key":"e_1_3_2_1_146_1","volume-title":"Personalized Online Education - A Crowdsourcing Challenge. Workshops at the Twenty-Sixth AAAI Conference on Artificial Intelligence","author":"Weld D. S.","year":"2012","unstructured":"Weld , D. S. , Adar , E. , Chilton , L. , Hoffmann , R. , and Horvitz , E . Personalized Online Education - A Crowdsourcing Challenge. Workshops at the Twenty-Sixth AAAI Conference on Artificial Intelligence , ( 2012 ). Weld, D. S., Adar, E., Chilton, L., Hoffmann, R., and Horvitz, E. Personalized Online Education - A Crowdsourcing Challenge. Workshops at the Twenty-Sixth AAAI Conference on Artificial Intelligence, (2012)."},{"key":"e_1_3_2_1_147_1","first-page":"7","article-title":"The multidimensional wisdom of crowds","volume":"6","author":"Welinder P.","year":"2010","unstructured":"Welinder , P. , Branson , S. , Belongie , S. , and Perona , P . The multidimensional wisdom of crowds . Neural Information Processing Systems 6 , 7 ( 2010 ), 1--9. Welinder, P., Branson, S., Belongie, S., and Perona, P. The multidimensional wisdom of crowds. Neural Information Processing Systems 6, 7 (2010), 1--9.","journal-title":"Neural Information Processing Systems"},{"key":"e_1_3_2_1_148_1","volume-title":"The economics of organization: The transaction cost approach. American journal of sociology 87, 3","author":"Williamson O. E.","year":"1981","unstructured":"Williamson , O. E. The economics of organization: The transaction cost approach. American journal of sociology 87, 3 ( 1981 ), 548--577. Williamson, O. E. The economics of organization: The transaction cost approach. American journal of sociology 87, 3 (1981), 548--577."},{"key":"e_1_3_2_1_149_1","volume-title":"Evidence for a collective intelligence factor in the performance of human groups. science 330, 6004","author":"Woolley A. W.","year":"2010","unstructured":"Woolley , A. W. , Chabris , C. F. , Pentland , A. , Hashmi , N. , and Malone , T. W . Evidence for a collective intelligence factor in the performance of human groups. science 330, 6004 ( 2010 ), 686--688. Woolley, A. W., Chabris, C. F., Pentland, A., Hashmi, N., and Malone, T. W. Evidence for a collective intelligence factor in the performance of human groups. science 330, 6004 (2010), 686--688."},{"key":"e_1_3_2_1_150_1","doi-asserted-by":"publisher","DOI":"10.5555\/647067.715342"},{"key":"e_1_3_2_1_151_1","doi-asserted-by":"publisher","DOI":"10.1145\/1084805.1084814"},{"key":"e_1_3_2_1_152_1","doi-asserted-by":"publisher","DOI":"10.1145\/1978942.1979147"},{"key":"e_1_3_2_1_153_1","doi-asserted-by":"publisher","DOI":"10.1145\/2448116.2448118"},{"key":"e_1_3_2_1_154_1","doi-asserted-by":"publisher","DOI":"10.1145\/2207676.2207708"},{"key":"e_1_3_2_1_155_1","volume-title":"Robust Active Learning Using Crowdsourced Annotations for Activity Recognition. Workshops at the Twenty-Fifth AAAI Conference on Artificial Intelligence","author":"Zhao L.","year":"2011","unstructured":"Zhao , L. , Sukthankar , G. , and Sukthankar , R . Robust Active Learning Using Crowdsourced Annotations for Activity Recognition. Workshops at the Twenty-Fifth AAAI Conference on Artificial Intelligence , ( 2011 ). Zhao, L., Sukthankar, G., and Sukthankar, R. Robust Active Learning Using Crowdsourced Annotations for Activity Recognition. Workshops at the Twenty-Fifth AAAI Conference on Artificial Intelligence, (2011)."},{"key":"e_1_3_2_1_156_1","doi-asserted-by":"publisher","DOI":"10.1145\/1978942.1979453"},{"key":"e_1_3_2_1_157_1","doi-asserted-by":"publisher","DOI":"10.1098\/rsta.2008.0116"},{"key":"e_1_3_2_1_158_1","volume-title":"Oppenheimer Question. Proceedings of Collective Intelligence","author":"Zittrain J.","year":"2012","unstructured":"Zittrain , J. Human Computing's Oppenheimer Question. Proceedings of Collective Intelligence , ( 2012 ). Zittrain, J. Human Computing's Oppenheimer Question. Proceedings of Collective Intelligence, (2012)."}],"event":{"name":"CSCW '13: Computer Supported Cooperative Work","location":"San Antonio Texas USA","acronym":"CSCW '13","sponsor":["SIGCHI ACM Special Interest Group on Computer-Human Interaction"]},"container-title":["Proceedings of the 2013 conference on Computer supported cooperative work"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/2441776.2441923","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/2441776.2441923","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,18]],"date-time":"2025-06-18T08:19:14Z","timestamp":1750234754000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/2441776.2441923"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2013,2,23]]},"references-count":158,"alternative-id":["10.1145\/2441776.2441923","10.1145\/2441776"],"URL":"https:\/\/doi.org\/10.1145\/2441776.2441923","relation":{},"subject":[],"published":{"date-parts":[[2013,2,23]]},"assertion":[{"value":"2013-02-23","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}