{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,1,24]],"date-time":"2026-01-24T01:06:20Z","timestamp":1769216780401,"version":"3.49.0"},"reference-count":49,"publisher":"Springer Science and Business Media LLC","issue":"1","license":[{"start":{"date-parts":[[2018,7,13]],"date-time":"2018-07-13T00:00:00Z","timestamp":1531440000000},"content-version":"tdm","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"}],"funder":[{"name":"NPRP","award":["06-1220-1-233"],"award-info":[{"award-number":["06-1220-1-233"]}]}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Knowl Inf Syst"],"published-print":{"date-parts":[[2019,7]]},"DOI":"10.1007\/s10115-018-1235-5","type":"journal-article","created":{"date-parts":[[2018,7,13]],"date-time":"2018-07-13T10:01:30Z","timestamp":1531476090000},"page":"1-24","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":26,"title":["Overview of the crowdsourcing process"],"prefix":"10.1007","volume":"60","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-9590-8403","authenticated-orcid":false,"given":"Lobna","family":"Nassar","sequence":"first","affiliation":[]},{"given":"Fakhri","family":"Karray","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2018,7,13]]},"reference":[{"key":"1235_CR1","unstructured":"Aaron S, John H, Daniel C (2011) Designing incentives for inexpert human raters. In: Proceedings of the ACM conference on computer supported cooperative work, CSCW\u201911"},{"key":"1235_CR2","doi-asserted-by":"publisher","unstructured":"Aditya P, Scott C (2011) Identifying topical authorities in microblogs. In Proceedings of ACM conference on web search and data mining (WSDM), pp 45\u201354. \n                    https:\/\/doi.org\/10.1145\/1935826.1935843","DOI":"10.1145\/1935826.1935843"},{"key":"1235_CR3","unstructured":"Alexander S, David F (2008) Utility data annotation with amazon mechanical turk. In: First IEEE workshop on internet vision at CVPR\u201908"},{"key":"1235_CR4","doi-asserted-by":"publisher","first-page":"758","DOI":"10.1177\/0165551509346786","volume":"35","author":"AH Bernardo","year":"2009","unstructured":"Bernardo AH, Daniel MR, Fang W (2009) Crowdsourcing, attention and productivity. J Inf Sci 35:758\u2013765. \n                    https:\/\/doi.org\/10.1177\/0165551509346786","journal-title":"J Inf Sci"},{"key":"1235_CR5","unstructured":"Bin Y, Yan W, Ling L (2015) CrowdTrust: a context-aware trust model for workers selection in crowdsourcing environments. In: 22nd IEEE international conference on web services (IEEE ICWS, research track, acceptance rate 17.4%), June 27\u2013July 2, 2015, New York, USA"},{"key":"1235_CR6","unstructured":"Catherine G, Matthew L (2010) Crowdsourcing document relevance assessment with mechanical turk. In: Proceedings of the NAACL HLT 2010 workshop on creating speech and language data with Amazon\u2019s mechanical turk, pp 172\u2013179"},{"issue":"2","key":"1235_CR7","doi-asserted-by":"publisher","first-page":"291","DOI":"10.1016\/S0306-4573(99)00017-5","volume":"36","author":"LAC Charles","year":"2000","unstructured":"Charles LAC, Gordon VC, Elizabeth AT (2000) Relevance ranking for one to three term queries. Inf Process Manag 36(2):291\u2013311","journal-title":"Inf Process Manag"},{"key":"1235_CR8","doi-asserted-by":"publisher","unstructured":"Damon H, Sepandar DK (2010) The anatomy of a large-scale social search engine. In: Proceedings of 19th ACM international conference on world wide web. ACM, New York, pp 431\u2013440. \n                    https:\/\/doi.org\/10.1145\/1772690.1772735","DOI":"10.1145\/1772690.1772735"},{"issue":"2","key":"1235_CR9","doi-asserted-by":"publisher","first-page":"138","DOI":"10.1007\/s10791-012-9205-0","volume":"16","author":"K Gabriella","year":"2013","unstructured":"Gabriella K, Jaap K, Natasa M (2013) An analysis of human factors and label accuracy in crowdsourcing relevance judgments. Inf Retr 16(2):138\u2013178. \n                    https:\/\/doi.org\/10.1007\/s10791-012-9205-0","journal-title":"Inf Retr"},{"key":"1235_CR10","doi-asserted-by":"publisher","first-page":"509","DOI":"10.1126\/science.1205869","volume":"334","author":"P Galen","year":"2011","unstructured":"Galen P, Iyad R, Wei P et al (2011) Time-critical social mobilization. Science 334:509\u2013512. \n                    https:\/\/doi.org\/10.1126\/science.1205869","journal-title":"Science"},{"key":"1235_CR11","unstructured":"Gianluca S, Gang W, Manuel E et al (2013) Follow the green: growth and dynamics in twitter follower markets. In: Proceedings of IMC"},{"key":"1235_CR12","unstructured":"Haoqi Z, Eric H, Yiling C et al (2012) Task routing for prediction tasks. In: Proceeding of 11th international conference autonomous agents and multiagent systems, vol 2. International foundation for autonomous agents and multi-agent Systems, Richland, pp 889\u2013896"},{"key":"1235_CR13","first-page":"10","volume":"26","author":"G Huiji","year":"2011","unstructured":"Huiji G, Geoffrey B, Goolsby Rebecca (2011) Harnessing the crowdsourcing power of social media for disaster relief. Intell Syst IEEE 26:10\u201314","journal-title":"Intell Syst IEEE"},{"key":"1235_CR14","doi-asserted-by":"publisher","first-page":"68","DOI":"10.1109\/mc.2012.295","volume":"46","author":"R Iyad","year":"2013","unstructured":"Iyad R, Sohan D, Alex R et al (2013) Global manhunt pushes the limits of social mobilization. Computer 46:68\u201375. \n                    https:\/\/doi.org\/10.1109\/mc.2012.295","journal-title":"Computer"},{"key":"1235_CR15","unstructured":"Jacob W, Paul R, Ting-fan W et al (2009) Whose vote should count more: optimal integration of labels from labelers of unknown expertise. In: Neural information processing systems (NIPS)\u201909, pp 2035\u20132043"},{"key":"1235_CR16","unstructured":"Jia D, Wei D, Richard S et al (2009) ImageNet: a large-scale hierarchical image database. In: CVPR"},{"key":"1235_CR17","unstructured":"Jianshu W, Ee Peng L, Jing J et al (2010) Twitterrank: finding topic-sensitive influential twitterers. In: Proceedings of ACM conference on web search and data mining (WSDM)"},{"key":"1235_CR18","unstructured":"Jing W, Panagiotis GI, Foster P (2016) Cost-effective quality assurance in crowd labeling. Inf Syst Res (Forthcoming), Dec 2016, NYU Working Paper No. 2451\/31833"},{"issue":"1","key":"1235_CR19","first-page":"55","volume":"65","author":"R Ju","year":"2015","unstructured":"Ju R, Yaoxue Z, Kuan Z et al (2015) SACRM: social aware crowdsourcing with reputation management in mobile sensing. Comput Commun (Elsevier) 65(1):55\u201365","journal-title":"Comput Commun (Elsevier)"},{"key":"1235_CR20","unstructured":"Julien B, Shourya R, Gueyoung J et al (2013) Crowdsourcing translation by leveraging tournament selection and lattice-based string alignment. In: AAAI conference on human computation and crowdsourcing (HCOMP) (Works in Progress\/Demos)"},{"key":"1235_CR21","unstructured":"Ke M, Licia C, Mark H et al (2015) Survey of the use of crowdsourcing in software engineering. RN 15 (2015), 01"},{"key":"1235_CR22","unstructured":"Kyumin L, Prithivi T, James C (2013). Crowdturfers, campaigns, and social media: tracking and revealing crowdsourced manipulation of social media. In: Proceedings of ICWSM"},{"issue":"1","key":"1235_CR23","first-page":"1","volume":"5","author":"L Kyumin","year":"2014","unstructured":"Kyumin L, Steve W, Hancheng G (2014) Characterizing and automatically detecting crowdturfing in Fiverr and Twitter. Soc Netw Anal Min 5(1):1\u201316","journal-title":"Soc Netw Anal Min"},{"key":"1235_CR24","first-page":"1","volume":"4","author":"A Leavitt","year":"2009","unstructured":"Leavitt A, Evan B, David F et al (2009) The influentials: new approaches for analyzing influence on twitter. Web Ecol Proj 4:1\u201318","journal-title":"Web Ecol Proj"},{"key":"1235_CR25","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1007\/978-1-4614-7163-9_110201-1","volume-title":"Detecting crowdturfing in social media. Encyclopedia of social network analysis and mining","author":"W Liang","year":"2017","unstructured":"Liang W, Huan L (2017) Detecting crowdturfing in social media. Encyclopedia of social network analysis and mining. Springer, New York, pp 1\u20139. \n                    https:\/\/doi.org\/10.1007\/978-1-4614-7163-9_110196-1"},{"key":"1235_CR26","doi-asserted-by":"publisher","first-page":"92","DOI":"10.1109\/mc.2006.196","volume":"39","author":"VA Luis","year":"2006","unstructured":"Luis VA (2006) Games with a purpose. Computer 39:92\u201394. \n                    https:\/\/doi.org\/10.1109\/mc.2006.196","journal-title":"Computer"},{"key":"1235_CR27","unstructured":"Luis VA, Laura D (2004) Labeling images with a computer game. In: SIGCHI conference on Human factors in computing systems, pp 319\u2013326"},{"key":"1235_CR28","unstructured":"Manuel C, Lorenzo C, Andrea VA et al (2012) Finding red balloons with \u201csplit\u201d contracts: robustness to individuals\u2019 selfishness. In: ACM symposium on theory of computing (STOC)"},{"key":"1235_CR29","unstructured":"Manuel C, Iyad R, Victoriano I et al (2016) Searching for someone. Illustrated by Beatriz Travieso. Published in MIT media lab. Sponsored by the Data61 Unit at CSIRO. \n                    https:\/\/medium.com\/mit-media-lab\/searching-for-someone-688f6c12ff42#.tlleaq622"},{"key":"1235_CR30","first-page":"260","volume-title":"ISWC 2013, Part II. LNCS","author":"A Maribel","year":"2013","unstructured":"Maribel A, Amrapali Z, Elena S et al (2013) Crowdsourcing linked data quality assessment. In: Harith A et al (eds) ISWC 2013, Part II. LNCS, vol 8219. Springer, Heidelberg, pp 260\u2013276"},{"key":"1235_CR31","unstructured":"Marti L, Stefan V (2011) Dirty jobs: the role of freelance labor in web service abuse. In: Proceedings of the 20th USENIX security symposium, USESEC\u201911, San Francisco, CA"},{"key":"1235_CR32","unstructured":"Meeyoung C, Hamed H, Fabri\u0131cio B et al (2010) Measuring user influence in twitter: the million follower fallacy. In: Proceedings of AAAI conference on weblogs and social media (ICWSM)"},{"issue":"2","key":"1235_CR33","doi-asserted-by":"publisher","first-page":"76","DOI":"10.1109\/MIC.2013.20","volume":"17","author":"A Mohammad","year":"2013","unstructured":"Mohammad A, Boualem B, Aleksandar I et al (2013) Quality control in crowdsourcing systems: issues and directions. IEEE Internet Comput 17(2):76\u201381. \n                    https:\/\/doi.org\/10.1109\/MIC.2013.20","journal-title":"IEEE Internet Comput"},{"key":"1235_CR34","first-page":"1","volume":"2","author":"Q Nguyen","year":"2013","unstructured":"Nguyen Q, Nguyen T, Lam T et al (2013) An\u00a0evaluation\u00a0of aggregation techniques\u00a0in\u00a0crowdsourcing. WISE 2:1\u201315","journal-title":"WISE"},{"key":"1235_CR35","unstructured":"Panagiotis I, Foster P, Jing W (2010) Quality management on amazon mechanical turk. In Proceedings of the ACM SIGKDD workshop on human computation (HCOMP\u201910), pp 64\u201367"},{"key":"1235_CR36","unstructured":"Pei-Yun H, Prem M, Vikas S (2009) Data quality from crowdsourcing: a study of annotation selection criteria. In: Proceedings of the NAACL HLT workshop on active learning for natural language processing. Association for Computational Linguistics, pp 27\u201335"},{"key":"1235_CR37","unstructured":"Peter W, Pietro P (2010) Online crowdsourcing: rating annotators and obtaining cost effective labels. In: IEEE conference on computer vision and pattern recognition workshops (ACVHL)"},{"key":"1235_CR38","unstructured":"Peter W, Steve B, Serge B et al (2010) The multidimensional wisdom of crowds. In: Neural information processing systems conference (NIPS), vol 6"},{"key":"1235_CR39","doi-asserted-by":"crossref","unstructured":"Petros V,\u00a0Hector G-M, Kerui H et al (2012) Max algorithms in crowdsourcing environments. In: Proceedings of the 2012 international conference on the world wide web, 2012, pp 989\u2013998. \n                    http:\/\/dx.doi.org\/10.1145\/2187836.2187969","DOI":"10.1145\/2187836.2187969"},{"issue":"1","key":"1235_CR40","first-page":"20","volume":"28","author":"D Philip","year":"1979","unstructured":"Philip D, Skene AM (1979) Maximum likelihood estimation of observer error-rates using the EM algorithm. J R Stat Soc Ser C (Appl Stat) 28(1):20\u201328","journal-title":"J R Stat Soc Ser C (Appl Stat)"},{"key":"1235_CR41","unstructured":"Rion S, Brendan O, Daniel J et al (2008) Cheap and fast\u2014but is it good? Evaluating non-expert annotations for natural language tasks. In: EMNLP"},{"key":"1235_CR42","unstructured":"Sam M, Mike JJ, Didier GL (2014) A flexible framework for assessing the quality of crowdsourced data. In: 17th annual international AGILE conference, Castell\u00f3n, Spain"},{"key":"1235_CR43","unstructured":"Saptarshi G, Naveen S, Fabricio B et al (2012) Cognos: crowdsourcing search for topic experts in microblogs. In: Proceedings of the 35th international ACM SIGIR conference on Research and development in information retrieval, pp 575\u2013590"},{"key":"1235_CR44","doi-asserted-by":"publisher","unstructured":"Shih-Wen H, Wai-Tat F (2013) Enhancing reliability using peer consistency evaluation in human computation. In: Computer supported cooperative work (CSCW), San Antonio, TX, USA, pp 639\u2013648. \n                    https:\/\/doi.org\/10.1145\/2441776.2441847","DOI":"10.1145\/2441776.2441847"},{"key":"1235_CR45","unstructured":"Stefanie N, Stefan R (2010) How reliable are annotations via crowdsourcing? A study about inter-annotator agreement for multi-label image annotation. In: The 11th ACM international conference on multimedia information retrieval (MIR), Philadelphia, USA, pp 29\u201331"},{"key":"1235_CR46","unstructured":"Twitter: Who to Follow. \n                    http:\/\/twitter.com\/#!\/who_to_follow"},{"issue":"10","key":"1235_CR47","first-page":"e45924","volume":"7","author":"N Victor","year":"2012","unstructured":"Victor N, Iyad R, Manuel C et al (2012) Verification in Referral-Based Crowdsourcing. PLOS One 7(10):e45924","journal-title":"Verification in Referral-Based Crowdsourcing. PLOS One"},{"issue":"8","key":"1235_CR48","first-page":"612","volume":"9","author":"Z Xiaohang","year":"2016","unstructured":"Xiaohang Z, Guoliang L, Jianhua F (2016) Crowdsourced top-k algorithms: an experimental evaluation. PVLDB 9(8):612\u2013623","journal-title":"PVLDB"},{"key":"1235_CR49","unstructured":"Yu-An S, Shourya R, Greg DL (2011) Beyond independent agreement: a tournament selection approach for quality assurance of human computation tasks. In: Proceedings of HCOMP11: the 3rd workshop on human computation"}],"container-title":["Knowledge and Information Systems"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s10115-018-1235-5.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/article\/10.1007\/s10115-018-1235-5\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s10115-018-1235-5.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2019,7,17]],"date-time":"2019-07-17T10:25:09Z","timestamp":1563359109000},"score":1,"resource":{"primary":{"URL":"http:\/\/link.springer.com\/10.1007\/s10115-018-1235-5"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2018,7,13]]},"references-count":49,"journal-issue":{"issue":"1","published-print":{"date-parts":[[2019,7]]}},"alternative-id":["1235"],"URL":"https:\/\/doi.org\/10.1007\/s10115-018-1235-5","relation":{},"ISSN":["0219-1377","0219-3116"],"issn-type":[{"value":"0219-1377","type":"print"},{"value":"0219-3116","type":"electronic"}],"subject":[],"published":{"date-parts":[[2018,7,13]]},"assertion":[{"value":"9 October 2016","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"17 March 2018","order":2,"name":"revised","label":"Revised","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"14 April 2018","order":3,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"13 July 2018","order":4,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}}]}}