{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,5,6]],"date-time":"2026-05-06T05:33:30Z","timestamp":1778045610017,"version":"3.51.4"},"publisher-location":"New York, NY, USA","reference-count":90,"publisher":"ACM","license":[{"start":{"date-parts":[[2015,8,19]],"date-time":"2015-08-19T00:00:00Z","timestamp":1439942400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2015,8,19]]},"DOI":"10.1145\/2788993.2789841","type":"proceedings-article","created":{"date-parts":[[2015,9,22]],"date-time":"2015-09-22T19:31:27Z","timestamp":1442950287000},"page":"1-10","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":17,"title":["Measuring the crowd"],"prefix":"10.1145","author":[{"given":"Eoin","family":"Cullina","sequence":"first","affiliation":[{"name":"NUI Galway, Newcastle, Galway, Ireland"}]},{"given":"Kieran","family":"Conboy","sequence":"additional","affiliation":[{"name":"NUI Galway, Newcastle, Galway, Ireland"}]},{"given":"Lorraine","family":"Morgan","sequence":"additional","affiliation":[{"name":"NUI Galway, Newcastle, Galway, Ireland"}]}],"member":"320","published-online":{"date-parts":[[2015,8,19]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"publisher","DOI":"10.1145\/1978542.1978560"},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.5465\/amr.2010.0146"},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1109\/MIC.2013.20"},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.bushor.2013.12.009"},{"key":"e_1_3_2_1_5_1","first-page":"1","article-title":". Analysis Of Social Interactions In A Social News Application","volume":"1","author":"Alpar P.","year":"2011","journal-title":"ANALYSIS"},{"key":"e_1_3_2_1_6_1","volume-title":"Proceedings of the 7th World Conference on Mass Customization, Personalization, and Co-Creation (MCPC 2014","author":"Anbardan Y. Z.","year":"2014"},{"key":"e_1_3_2_1_7_1","volume-title":"ICIS 2009 Proceedings, 200","author":"Archak N."},{"key":"e_1_3_2_1_8_1","volume-title":"ESCOM (European Software Control and Metrics conference), 337--346","author":"Azuma M.","year":"2001"},{"key":"e_1_3_2_1_9_1","volume-title":"Computer Systems and Industrial Informatics (ICCSII), 2012 International Conference on IEEE, 1--6.","author":"Badri M."},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"crossref","volume-title":"Typologies and taxonomies: an introduction to classification techniques","author":"Bailey K. D.","DOI":"10.4135\/9781412986397"},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"crossref","unstructured":"Baldwin C. and Hippel E. V. 2010. Modeling a Paradigm Shift: From Producer Innovation to User and Open Collaborative Innovation Modeling a Paradigm Shift: From Producer Innovation to User and Open Collaborative Innovation 1. Introduction and Overview 1--37.  Baldwin C. and Hippel E. V. 2010. Modeling a Paradigm Shift: From Producer Innovation to User and Open Collaborative Innovation Modeling a Paradigm Shift: From Producer Innovation to User and Open Collaborative Innovation 1. Introduction and Overview 1--37.","DOI":"10.2139\/ssrn.1502864"},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.1287\/mnsc.1120.1599"},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.2307\/248684"},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.1016\/S0048-7333(99)00067-0"},{"key":"e_1_3_2_1_15_1","first-page":"199","article-title":"Crowdsourcing: How to Benefit from (Too) Many Great Ideas","volume":"12","author":"Blohm I.","year":"2013","journal-title":"Management Information Systems Quarterly Executive"},{"key":"e_1_3_2_1_16_1","first-page":"14","article-title":"From Crowds to Collaborators: Initiating Effort & Catalyzing Interactions Among Online Creative Workers. Harvard Business School Technology & Operations Mgt. Unit","author":"Boudreau K.","year":"2014","journal-title":"Working Paper"},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.1287\/mnsc.1110.1322"},{"key":"e_1_3_2_1_18_1","volume-title":"Crowdsourcing: From Theory to Practice and Long-Term Perspectives.","author":"Bozzon A.","year":"2014"},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","DOI":"10.1177\/1354856507084420"},{"key":"e_1_3_2_1_20_1","volume-title":"Using Crowdsourcing In Government","author":"Brabham D. C."},{"key":"e_1_3_2_1_21_1","unstructured":"Carlson K. D. and Kavanagh M. J. 2011. HR Metrics and Workforce analytics. Human Resource Information Systems: Basics Applications and Future Directions 150.  Carlson K. D. and Kavanagh M. J. 2011. HR Metrics and Workforce analytics. Human Resource Information Systems: Basics Applications and Future Directions 150."},{"key":"e_1_3_2_1_22_1","unstructured":"Challenge.Gov 2015. Government Challenges Your Solutions.  Challenge.Gov 2015. Government Challenges Your Solutions."},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.1109\/MNET.2010.5430141"},{"key":"e_1_3_2_1_24_1","unstructured":"Cook S. 2008. Why contributors contribute. Harvard Business Review.  Cook S. 2008. Why contributors contribute. Harvard Business Review."},{"key":"e_1_3_2_1_25_1","unstructured":"Costa-Font J. Jofre-Bonet M. and Yen S. T. 2012. Nonmonetary rewards could be used to incentivise blood donation. British Politics and Policy at LSE.  Costa-Font J. Jofre-Bonet M. and Yen S. T. 2012. Nonmonetary rewards could be used to incentivise blood donation. British Politics and Policy at LSE."},{"key":"e_1_3_2_1_26_1","volume-title":"AMCIS 2012 Proceedings","author":"Erickson L."},{"key":"e_1_3_2_1_27_1","doi-asserted-by":"publisher","DOI":"10.1177\/0165551512437638"},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"crossref","unstructured":"Fenton N. and Bieman J. 2014. Software metrics: a rigorous and practical approach. CRC Press   Fenton N. and Bieman J. 2014. Software metrics: a rigorous and practical approach. CRC Press","DOI":"10.1201\/b17461"},{"key":"e_1_3_2_1_29_1","unstructured":"Fishwick C. 2014. Tomnod -- the online search party looking for Malaysian Airlines flight MH370. In The Guardian.  Fishwick C. 2014. Tomnod -- the online search party looking for Malaysian Airlines flight MH370. In The Guardian."},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.respol.2013.07.005"},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"publisher","DOI":"10.1109\/MIS.2011.52"},{"key":"e_1_3_2_1_32_1","volume-title":"Proceedings of the 22nd Australasian Conference on Information Systems (ACIS","author":"Geiger D.","year":"2011"},{"key":"e_1_3_2_1_33_1","volume-title":"Thirty Third International Conference on Information Systems, Orlando.","author":"Geiger D."},{"key":"e_1_3_2_1_34_1","volume-title":"Americas Conference on Information Systems (AMCIS)","author":"Geiger D."},{"key":"e_1_3_2_1_35_1","doi-asserted-by":"publisher","DOI":"10.1109\/52.391837"},{"key":"e_1_3_2_1_36_1","doi-asserted-by":"publisher","DOI":"10.1002\/bltj.20473"},{"key":"e_1_3_2_1_37_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.ijpe.2003.08.003"},{"key":"e_1_3_2_1_38_1","doi-asserted-by":"publisher","DOI":"10.1145\/2207676.2208320"},{"key":"e_1_3_2_1_39_1","unstructured":"Harris C. G. 2011. You're Hired! An Examination of Crowdsourcing Incentive Models in Human Resource Tasks 15--18.  Harris C. G. 2011. You're Hired! An Examination of Crowdsourcing Incentive Models in Human Resource Tasks 15--18."},{"key":"e_1_3_2_1_40_1","unstructured":"Hetmank L. 2013. Components and Functions of Crowdsourcing Systems-A Systematic Literature Review. In Wirtschaftsinformatik 4.  Hetmank L. 2013. Components and Functions of Crowdsourcing Systems-A Systematic Literature Review. In Wirtschaftsinformatik 4."},{"key":"e_1_3_2_1_41_1","doi-asserted-by":"crossref","unstructured":"Hodson H. 2013. Crowdsourcing grows up as workers unite. New Scientist 217 2903 22--23.  Hodson H. 2013. Crowdsourcing grows up as workers unite. New Scientist 217 2903 22--23.","DOI":"10.1016\/S0262-4079(13)60346-2"},{"key":"e_1_3_2_1_42_1","doi-asserted-by":"publisher","DOI":"10.1145\/1979742.1979669"},{"key":"e_1_3_2_1_43_1","volume-title":"Proceedings of the 23rd International Teletraffic Congress International Teletraffic Congress, 142--149","author":"Ho\u00dffeld T."},{"key":"e_1_3_2_1_44_1","unstructured":"Howe B. J. 2006. The Rise of Crowdsourcing. In Wired 1--5.  Howe B. J. 2006. The Rise of Crowdsourcing. In Wired 1--5."},{"key":"e_1_3_2_1_45_1","volume-title":"Demographics of mechanical turk","author":"Ipeirotis P. G."},{"key":"e_1_3_2_1_46_1","doi-asserted-by":"publisher","DOI":"10.1145\/1837885.1837906"},{"key":"e_1_3_2_1_47_1","doi-asserted-by":"publisher","DOI":"10.1177\/000312240607100404"},{"key":"e_1_3_2_1_48_1","volume-title":"Project management metrics, KPIs, and dashboards: a guide to measuring and monitoring project performance","author":"Kerzner H. R."},{"key":"e_1_3_2_1_49_1","unstructured":"Kickstarter 2014.  Kickstarter 2014."},{"key":"e_1_3_2_1_50_1","doi-asserted-by":"crossref","unstructured":"Kittur A. Smus B. Khamkar S. and Kraut R. E. 2011. CrowdForge ACM Press New York New York USA 43--43. DOI= http:\/\/dx.doi.org\/10.1145\/2047196.2047202.  Kittur A. Smus B. Khamkar S. and Kraut R. E. 2011. CrowdForge ACM Press New York New York USA 43--43. DOI= http:\/\/dx.doi.org\/10.1145\/2047196.2047202.","DOI":"10.1145\/2047196.2047202"},{"key":"e_1_3_2_1_51_1","doi-asserted-by":"publisher","DOI":"10.1145\/2380718.2380739"},{"key":"e_1_3_2_1_52_1","unstructured":"Lakhani K. Garvin D. and Lonstein E. 2013. TopCoder (A): Developing Software through Crowdsourcing Harvard University. General Management Unit Case 610-032 (2010).  Lakhani K. Garvin D. and Lonstein E. 2013. TopCoder (A): Developing Software through Crowdsourcing Harvard University. General Management Unit Case 610-032 (2010)."},{"key":"e_1_3_2_1_53_1","doi-asserted-by":"publisher","DOI":"10.2753\/MIS0742-1222260108"},{"key":"e_1_3_2_1_54_1","volume-title":"Informing Science: International Journal of an Emerging Transdiscipline 9, 1, 181--212.","author":"Levy Y.","year":"2006"},{"key":"e_1_3_2_1_55_1","doi-asserted-by":"publisher","DOI":"10.1016\/S0048-7333(99)00063-3"},{"key":"e_1_3_2_1_56_1","volume-title":"Proceedings of the American Society for Information Science and Technology 48","author":"Liu T. X.","year":"2011"},{"key":"e_1_3_2_1_57_1","doi-asserted-by":"publisher","DOI":"10.1109\/SCC.2010.74"},{"key":"e_1_3_2_1_58_1","doi-asserted-by":"publisher","DOI":"10.1177\/0021943603259363"},{"key":"e_1_3_2_1_59_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.jsis.2013.07.004"},{"key":"e_1_3_2_1_60_1","doi-asserted-by":"publisher","DOI":"10.1093\/scipol\/scs009"},{"key":"e_1_3_2_1_61_1","doi-asserted-by":"publisher","DOI":"10.1161\/CIRCOUTCOMES.113.000140"},{"key":"e_1_3_2_1_62_1","doi-asserted-by":"crossref","unstructured":"Mergel I. 2012. Challenges as Game Changers for Collaborative Knowledge Incubation in the Public Sector. Available at SSRN 1992964.  Mergel I. 2012. Challenges as Game Changers for Collaborative Knowledge Incubation in the Public Sector. Available at SSRN 1992964.","DOI":"10.2139\/ssrn.1992964"},{"key":"e_1_3_2_1_63_1","volume-title":"1st International Conference on Mass Customisation and Open Innovation Conference","author":"Morgan L."},{"key":"e_1_3_2_1_64_1","volume-title":"European Conference in Information Systems.","author":"Nickerson R."},{"key":"e_1_3_2_1_65_1","doi-asserted-by":"publisher","DOI":"10.1057\/ejis.2012.26"},{"key":"e_1_3_2_1_66_1","unstructured":"Oxford Dictionary 2015. Definition of Noun \"Metric\" Oxford University Press Oxford Dictionaries  Oxford Dictionary 2015. Definition of Noun \"Metric\" Oxford University Press Oxford Dictionaries"},{"key":"e_1_3_2_1_67_1","doi-asserted-by":"publisher","DOI":"10.1287\/isre.13.2.151.88"},{"key":"e_1_3_2_1_68_1","doi-asserted-by":"publisher","DOI":"10.1145\/2441776.2441920"},{"key":"e_1_3_2_1_69_1","doi-asserted-by":"publisher","DOI":"10.1109\/HICSS.2014.431"},{"key":"e_1_3_2_1_70_1","doi-asserted-by":"publisher","DOI":"10.1145\/1978942.1979148"},{"key":"e_1_3_2_1_71_1","first-page":"202","volume":"2012","author":"Ren J.","year":"2011","journal-title":"Exploring the Process of Web-based Crowdsourcing Innovation. In AMCIS"},{"key":"e_1_3_2_1_72_1","volume-title":"An Assessment of Intrinsic and Extrinsic Motivation on Task Performance in Crowdsourcing Markets. In ICWSM","author":"Rogstadius J.","year":"2011"},{"key":"e_1_3_2_1_73_1","unstructured":"Roth Y. and C\u00e9r\u00e9 J. 2014. Big Brands Increased Investment In Crowdsourcing By Nearly 50% In 2014. In The State of Crowdsourcing in 2015\" Trend Report eYeka pressroom.  Roth Y. and C\u00e9r\u00e9 J. 2014. Big Brands Increased Investment In Crowdsourcing By Nearly 50% In 2014. In The State of Crowdsourcing in 2015\" Trend Report eYeka pressroom."},{"key":"e_1_3_2_1_74_1","volume-title":"ACIS 2010 Proceedings, 76","author":"Rouse A. C.","year":"2010"},{"key":"e_1_3_2_1_75_1","doi-asserted-by":"publisher","DOI":"10.1080\/10580530.2013.739883"},{"key":"e_1_3_2_1_76_1","doi-asserted-by":"publisher","DOI":"10.1145\/2441776.2441791"},{"key":"e_1_3_2_1_77_1","doi-asserted-by":"publisher","DOI":"10.5437\/08956308X5504074"},{"key":"e_1_3_2_1_78_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.infsof.2011.06.007"},{"key":"e_1_3_2_1_79_1","doi-asserted-by":"publisher","DOI":"10.1145\/2568225.2568249"},{"key":"e_1_3_2_1_80_1","unstructured":"Sullivan M. 2006. Historical Crowdfunding Examples. In Fundavlog.  Sullivan M. 2006. Historical Crowdfunding Examples. In Fundavlog."},{"key":"e_1_3_2_1_81_1","doi-asserted-by":"crossref","unstructured":"Torfing J. 2013. Collaborative innovation in the public sector. In Handbook of Innovation in Public Services 301--316.  Torfing J. 2013. Collaborative innovation in the public sector. In Handbook of Innovation in Public Services 301--316.","DOI":"10.4337\/9781849809757.00032"},{"key":"e_1_3_2_1_82_1","unstructured":"Trompette P. Chanal V. and Pelissier C. 2008. Crowdsourcing as a way to access external knowledge for innovation: Control incentive and coordination in hybrid forms of innovation 1--29.  Trompette P. Chanal V. and Pelissier C. 2008. Crowdsourcing as a way to access external knowledge for innovation: Control incentive and coordination in hybrid forms of innovation 1--29."},{"key":"e_1_3_2_1_83_1","unstructured":"Unilever 2015. Unilever launches crowdsourcing initiative to drive sustainable growth ambition. DOI=http:\/\/foundry.unilever.com\/unilever-launches-crowdsourcing-initiative-to-drive-sustainable-growth-ambition\/  Unilever 2015. Unilever launches crowdsourcing initiative to drive sustainable growth ambition. DOI=http:\/\/foundry.unilever.com\/unilever-launches-crowdsourcing-initiative-to-drive-sustainable-growth-ambition\/"},{"key":"e_1_3_2_1_84_1","doi-asserted-by":"publisher","DOI":"10.1109\/SERVICES-I.2009.56"},{"key":"e_1_3_2_1_85_1","first-page":"2","article-title":"Analyzing the past to prepare for the future: Writing a literature review","volume":"26","author":"Webster J.","year":"2002","journal-title":"Management Information Systems Quarterly"},{"key":"e_1_3_2_1_86_1","doi-asserted-by":"publisher","DOI":"10.1080\/13662710801970142"},{"key":"e_1_3_2_1_87_1","doi-asserted-by":"publisher","DOI":"10.1108\/01443331111104779"},{"key":"e_1_3_2_1_88_1","doi-asserted-by":"publisher","DOI":"10.7903\/cmr.1145"},{"key":"e_1_3_2_1_89_1","doi-asserted-by":"publisher","DOI":"10.1145\/1386790.1386829"},{"key":"e_1_3_2_1_90_1","volume-title":"Applications of case study research (applied social research Methods). Series","author":"Yin R. K.","edition":"4"}],"event":{"name":"OpenSym '15: The 11th International Symposium on Open Collaboration","location":"San Francisco California","acronym":"OpenSym '15","sponsor":["SIGWEB ACM Special Interest Group on Hypertext, Hypermedia, and Web","SIGSOFT ACM Special Interest Group on Software Engineering"]},"container-title":["Proceedings of the 11th International Symposium on Open Collaboration"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/2788993.2789841","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/2788993.2789841","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,18]],"date-time":"2025-06-18T06:12:30Z","timestamp":1750227150000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/2788993.2789841"}},"subtitle":["a preliminary taxonomy of crowdsourcing metrics"],"short-title":[],"issued":{"date-parts":[[2015,8,19]]},"references-count":90,"alternative-id":["10.1145\/2788993.2789841","10.1145\/2788993"],"URL":"https:\/\/doi.org\/10.1145\/2788993.2789841","relation":{},"subject":[],"published":{"date-parts":[[2015,8,19]]},"assertion":[{"value":"2015-08-19","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}