{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,5,6]],"date-time":"2026-05-06T05:33:29Z","timestamp":1778045609608,"version":"3.51.4"},"reference-count":101,"publisher":"IEEE","content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2014,5]]},"DOI":"10.1109\/rcis.2014.6861072","type":"proceedings-article","created":{"date-parts":[[2014,7,29]],"date-time":"2014-07-29T17:06:55Z","timestamp":1406653615000},"page":"1-12","source":"Crossref","is-referenced-by-count":104,"title":["The four pillars of crowdsourcing: A reference model"],"prefix":"10.1109","author":[{"given":"Mahmood","family":"Hosseini","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Keith","family":"Phalp","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Jacqui","family":"Taylor","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Raian","family":"Ali","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"263","reference":[{"key":"35","doi-asserted-by":"publisher","DOI":"10.1177\/0165551512437638"},{"key":"36","doi-asserted-by":"publisher","DOI":"10.1007\/s10796-012-9350-4"},{"key":"33","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE.2013.6606645"},{"key":"34","doi-asserted-by":"publisher","DOI":"10.1109\/RE.2013.6636712"},{"key":"39","doi-asserted-by":"publisher","DOI":"10.1108\/03055721011024955"},{"key":"37","first-page":"114","article-title":"Opinion mining of Spanish customer comments with non-expert annotations on mechanical turk","author":"mellebeek","year":"2010","journal-title":"Proceedings of the NAACL HLT 2010 Workshop on Creating Speech and Language Data with Amazon's Mechanical Turk"},{"key":"38","first-page":"534","article-title":"Adaptive task assignment for crowdsourced classification","author":"ho","year":"2013","journal-title":"Proceedings of the 30th International Conference on Machine Learning"},{"key":"43","doi-asserted-by":"publisher","DOI":"10.1109\/SERVICES-I.2009.56"},{"key":"42","doi-asserted-by":"publisher","DOI":"10.14236\/ewic\/sohuman2013.2"},{"key":"41","article-title":"Crowdsourcing: What can be outsourced to the crowd, and why?","author":"schenk","year":"2009","journal-title":"Workshop on Open Source Innovation"},{"key":"40","article-title":"Activating the crowd: Exploiting user-item reciprocity for recommendation","author":"larson","year":"2013","journal-title":"The First Workshop on Crowdsourcing and Human Computation for Recommender Systems ACM Conference Series on Recommender Systems ACM RECSYS"},{"key":"22","doi-asserted-by":"publisher","DOI":"10.1080\/12297119.2010.9707437"},{"key":"23","first-page":"1","article-title":"How to invent a new business model based on crowdsourcing: The Crowdspirit case","author":"chanal","year":"2008","journal-title":"Actes de la Confe?rence Internationale de Management Strate?gique Nice"},{"key":"24","doi-asserted-by":"crossref","DOI":"10.7903\/cmr.1145","article-title":"Crowdsourcing and its application in marketing activities","volume":"5","author":"whitla","year":"2009","journal-title":"Contemporary Management Research"},{"key":"25","doi-asserted-by":"publisher","DOI":"10.1145\/2390803.2390808"},{"key":"26","doi-asserted-by":"publisher","DOI":"10.2196\/jmir.2513"},{"key":"27","doi-asserted-by":"publisher","DOI":"10.1007\/978-94-007-1384-0_4"},{"key":"28","doi-asserted-by":"publisher","DOI":"10.1108\/01443331111104779"},{"key":"29","doi-asserted-by":"publisher","DOI":"10.1109\/MIS.2011.52"},{"key":"3","doi-asserted-by":"publisher","DOI":"10.1145\/2487575.2487600"},{"key":"2","article-title":"Towards task recommendation in micro-task markets","author":"ambati","year":"2011","journal-title":"Human Computation"},{"key":"1","author":"howe","year":"2006","journal-title":"Crowdsourcing A Definition"},{"key":"30","doi-asserted-by":"publisher","DOI":"10.1016\/j.envsoft.2012.03.002"},{"key":"7","doi-asserted-by":"publisher","DOI":"10.1186\/2047-217X-2-2"},{"key":"6","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE.2013.6606679"},{"key":"32","doi-asserted-by":"publisher","DOI":"10.1145\/2025113.2025196"},{"key":"5","article-title":"Crowdsourcing to mobile users: A study of the role of platforms and tasks","volume":"14","author":"della mea","year":"2013","journal-title":"DBCrowd 2013"},{"key":"31","article-title":"Social adaptation: When software gives users a voice","author":"ali","year":"2012","journal-title":"ENASE'11"},{"key":"4","doi-asserted-by":"publisher","DOI":"10.1145\/2030100.2030102"},{"key":"9","doi-asserted-by":"crossref","first-page":"1157","DOI":"10.1145\/2488388.2488489","article-title":"Pricing mechanisms for crowdsourcing markets","author":"singer","year":"2013","journal-title":"Proceedings of the 12th international conference on World Wide Web"},{"key":"8","doi-asserted-by":"publisher","DOI":"10.1145\/2506364.2506368"},{"key":"19","doi-asserted-by":"publisher","DOI":"10.1177\/1745691610393980"},{"key":"17","doi-asserted-by":"publisher","DOI":"10.1109\/IMIS.2011.89"},{"key":"18","doi-asserted-by":"publisher","DOI":"10.2196\/jmir.2620"},{"key":"15","article-title":"What's the right price? Pricing tasks for finishing on time","author":"faradani","year":"2011","journal-title":"Human Computation"},{"key":"16","doi-asserted-by":"publisher","DOI":"10.1145\/1979742.1979606"},{"key":"13","article-title":"Leveraging human computation markets for interactive evolution","author":"lehman","year":"2013","journal-title":"International Conference on Machine Learning 2013 Workshop Machine Learning Meets Crowdsourcing"},{"key":"14","author":"howe","year":"0","journal-title":"The Rise of Crowdsourcing"},{"key":"11","doi-asserted-by":"publisher","DOI":"10.1145\/1753846.1753873"},{"key":"12","first-page":"26","article-title":"Mechanical cheat: Spamming schemes and adversarial techniques on crowdsourcing platforms","author":"difallah","year":"2012","journal-title":"Crowds"},{"key":"21","doi-asserted-by":"publisher","DOI":"10.1080\/13691181003624090"},{"key":"20","doi-asserted-by":"publisher","DOI":"10.1145\/1357054.1357127"},{"key":"99","doi-asserted-by":"publisher","DOI":"10.1109\/RE.2012.6345835"},{"key":"101","first-page":"345","article-title":"Crowdsourcing the evaluation of a domain-adapted named entity recognition system","author":"sayeed","year":"2010","journal-title":"Human Language Technologies The 2010 Annual Conference of the North American Chapter of the Association for Computational Linguistics"},{"key":"100","article-title":"Towards crowdsourcing for requirements engineering","author":"hosseini","year":"2013","journal-title":"20th International Working Conference on Requirements Engineering Foundation for Software Quality - Empirical Track"},{"key":"98","doi-asserted-by":"publisher","DOI":"10.1145\/2024645.2024657"},{"key":"97","doi-asserted-by":"publisher","DOI":"10.1145\/1806799.1806844"},{"key":"96","doi-asserted-by":"publisher","DOI":"10.1109\/JPROC.2006.887293"},{"key":"95","doi-asserted-by":"publisher","DOI":"10.1007\/b107408"},{"key":"94","doi-asserted-by":"publisher","DOI":"10.1016\/j.websem.2007.11.011"},{"key":"93","author":"surowiecki","year":"2005","journal-title":"The Wisdom of Crowds"},{"key":"92","doi-asserted-by":"publisher","DOI":"10.1145\/1920261.1920263"},{"key":"91","doi-asserted-by":"publisher","DOI":"10.1109\/MC.2008.123"},{"key":"90","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1007\/978-3-642-02161-9_1","article-title":"Software engineering for self-adaptive systems: A research roadmap","author":"cheng","year":"2009","journal-title":"Software Engineering for Self-Adaptive Systems"},{"key":"10","article-title":"Crowdsourcing the verification of relationships in biomedical ontologies","author":"mortensen","year":"2013","journal-title":"AMIA Annual Symposium"},{"key":"88","article-title":"An assessment of intrinsic and extrinsic motivation on task performance in crowdsourcing markets","author":"rogstadius","year":"2011","journal-title":"ICWSM"},{"key":"89","doi-asserted-by":"publisher","DOI":"10.1145\/1979742.1979902"},{"key":"79","article-title":"Winner-take-all crowdsourcing contests with stochastic production","author":"cavallo","year":"2012","journal-title":"Workshop on Social Computing and User Generated Content EC-12 Valencia Spain"},{"key":"78","article-title":"Wrapper generation supervised by a noisy crowd","volume":"8","author":"crescenzi","year":"2013","journal-title":"DBCrowd 2013"},{"key":"77","doi-asserted-by":"publisher","DOI":"10.1108\/10878570810888713"},{"key":"82","article-title":"Programmatic gold: Targeted and scalable quality assurance in crowdsourcing","volume":"11","author":"oleson","year":"2011","journal-title":"Human Computation"},{"key":"83","article-title":"Crowdsourcing gold-HIT creation at scale: Challenges and adaptive exploration approaches","author":"abraham","year":"2013","journal-title":"HCOMP 2013 Workshop"},{"key":"80","article-title":"Perspectives on infrastructure for crowdsourcing","volume":"7","author":"alonso","year":"2011","journal-title":"Crowdsourcing for Search and Data Mining CSDM 2011"},{"key":"81","article-title":"Crowdsourcing workshop: The emergence of affective crowdsourcing","author":"morris","year":"2011","journal-title":"Proceedings of the 2011 Annual Conference Extended Abstracts on Human Factors in Computing Systems"},{"key":"86","doi-asserted-by":"publisher","DOI":"10.1145\/1989323.1989331"},{"key":"87","doi-asserted-by":"crossref","DOI":"10.21236\/ADA235785","author":"kang","year":"1990","journal-title":"Feature-Oriented Domain Analysis (FODA) Feasibility Study"},{"key":"84","article-title":"99designs: An analysis of creative competition in crowdsourced design","author":"araujo","year":"2013","journal-title":"First AAAI Conference on Human Computation and Crowdsourcing"},{"key":"85","doi-asserted-by":"publisher","DOI":"10.1177\/1354856507084420"},{"key":"67","doi-asserted-by":"crossref","first-page":"165","DOI":"10.1007\/978-3-642-20161-5_17","article-title":"In search of quality in crowdsourcing for search engine evaluation","author":"kazai","year":"2011","journal-title":"Advances in Information Retrieval"},{"key":"66","doi-asserted-by":"crossref","first-page":"1167","DOI":"10.1145\/2488388.2488490","article-title":"Truthful incentives in crowdsourcing tasks using regret minimization mechanisms","author":"singla","year":"2013","journal-title":"Proceedings of the 12th international conference on World Wide Web"},{"key":"69","doi-asserted-by":"publisher","DOI":"10.1145\/1809400.1809422"},{"key":"68","article-title":"Reducing label cost by combining feature labels and crowdsourcing","author":"pujara","year":"2011","journal-title":"Proc ICML Workshop Combining Learn Strategies Reduce Label Cost"},{"key":"70","author":"brito","year":"2007","journal-title":"Hack Mash & Peer Crowdsourcing Government Transparency"},{"key":"71","article-title":"Understanding and designing for cultural differences on crowdsourcing marketplaces","author":"hsieh","year":"2011","journal-title":"CHI 2011 Workshop on Crowdsourcing and Human Computation"},{"key":"72","doi-asserted-by":"publisher","DOI":"10.1145\/1985394.1985395"},{"key":"73","doi-asserted-by":"publisher","DOI":"10.1016\/j.indmarman.2013.05.006"},{"key":"74","doi-asserted-by":"publisher","DOI":"10.1145\/2390034.2390038"},{"key":"75","first-page":"1","article-title":"Un(der) paid innovators: The commercial utilization of consumer work through crowdsourcing","volume":"4","author":"kleemann","year":"2008","journal-title":"Science Technology & Innovation Studies"},{"key":"76","doi-asserted-by":"publisher","DOI":"10.1007\/s13222-012-0092-8"},{"key":"59","doi-asserted-by":"publisher","DOI":"10.1504\/IJCRC.2013.056925"},{"key":"58","doi-asserted-by":"publisher","DOI":"10.1145\/1480506.1480508"},{"key":"57","doi-asserted-by":"publisher","DOI":"10.1145\/2470654.2470683"},{"key":"56","article-title":"Learning sociocultural knowledge via crowdsourced examples","author":"li","year":"2012","journal-title":"Proceedings of the 4th AAAI Workshop on Human Computation"},{"key":"55","first-page":"142","article-title":"Modeling of crowdsourcing platforms and granularity of work organization in future internet","author":"hossfeld","year":"2011","journal-title":"Proceedings of the 23rd International Teletraffic Congress"},{"key":"64","doi-asserted-by":"publisher","DOI":"10.5210\/fm.v13i6.2159"},{"key":"65","doi-asserted-by":"publisher","DOI":"10.1145\/2493432.2493481"},{"key":"62","doi-asserted-by":"publisher","DOI":"10.1111\/j.1540-5885.2011.00893.x"},{"key":"63","article-title":"Crowdsourcing for human subjects research","author":"schmidt","year":"2010","journal-title":"Proceedings of CrowdConf"},{"key":"60","article-title":"CrowdREquire: A requirements engineering crowdsourcing platform","author":"adepetu","year":"2012","journal-title":"AAAI Spring Symposium Series"},{"key":"61","doi-asserted-by":"crossref","first-page":"153","DOI":"10.1145\/2488388.2488403","article-title":"Reactive crowdsourcing","author":"bozzon","year":"2013","journal-title":"Proceedings of the 12th international conference on World Wide Web"},{"key":"49","doi-asserted-by":"publisher","DOI":"10.1109\/MC.2011.155"},{"key":"48","doi-asserted-by":"publisher","DOI":"10.1145\/792550.792552"},{"key":"45","article-title":"Error rate analysis of labeling by crowdsourcing","author":"li","year":"2013","journal-title":"International Conference on Machine Learning 2013 Workshop Machine Learning Meets Crowdsourcing"},{"key":"44","first-page":"1","article-title":"Some thoughts on a framework for crowdsourcing","author":"erickson","year":"2011","journal-title":"CHI 2011 Workshop on Crowdsourcing and Human Computation"},{"key":"47","doi-asserted-by":"crossref","first-page":"21","DOI":"10.1007\/BFb0013570","article-title":"Is it an agent, or just a program?: A taxonomy for autonomous agents","author":"franklin","year":"1997","journal-title":"Intelligent Agents III Agent Theories Architectures and Languages"},{"key":"46","doi-asserted-by":"publisher","DOI":"10.1109\/52.43044"},{"key":"51","doi-asserted-by":"publisher","DOI":"10.1145\/1963405.1963473"},{"key":"52","doi-asserted-by":"publisher","DOI":"10.1142\/S0218843012400011"},{"key":"53","doi-asserted-by":"publisher","DOI":"10.1145\/1852786.1852832"},{"key":"54","doi-asserted-by":"publisher","DOI":"10.1145\/2441776.2441848"},{"key":"50","doi-asserted-by":"publisher","DOI":"10.1016\/j.isprsjprs.2010.06.005"}],"event":{"name":"2014 IEEE Eighth International Conference on Research Challenges in Information Science (RCIS)","location":"Marrakech, Morocco","start":{"date-parts":[[2014,5,28]]},"end":{"date-parts":[[2014,5,30]]}},"container-title":["2014 IEEE Eighth International Conference on Research Challenges in Information Science (RCIS)"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx7\/6848887\/6860531\/06861072.pdf?arnumber=6861072","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2019,8,13]],"date-time":"2019-08-13T05:36:08Z","timestamp":1565674568000},"score":1,"resource":{"primary":{"URL":"http:\/\/ieeexplore.ieee.org\/document\/6861072\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2014,5]]},"references-count":101,"URL":"https:\/\/doi.org\/10.1109\/rcis.2014.6861072","relation":{},"subject":[],"published":{"date-parts":[[2014,5]]}}}