{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,3,28]],"date-time":"2026-03-28T03:46:04Z","timestamp":1774669564174,"version":"3.50.1"},"reference-count":143,"publisher":"Institute of Electrical and Electronics Engineers (IEEE)","issue":"9","license":[{"start":{"date-parts":[[2016,9,1]],"date-time":"2016-09-01T00:00:00Z","timestamp":1472688000000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/ieeexplore.ieee.org\/Xplorehelp\/downloads\/license-information\/IEEE.html"}],"funder":[{"DOI":"10.13039\/501100002920","name":"Hong Kong RGC Project","doi-asserted-by":"publisher","award":["N_HKUST637\/13"],"award-info":[{"award-number":["N_HKUST637\/13"]}],"id":[{"id":"10.13039\/501100002920","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100002920","name":"Hong Kong RGC Project","doi-asserted-by":"publisher","award":["ANR-13-CORD-0020"],"award-info":[{"award-number":["ANR-13-CORD-0020"]}],"id":[{"id":"10.13039\/501100002920","id-type":"DOI","asserted-by":"publisher"}]},{"name":"National Grand Fundamental Research 973 Program of China","award":["2014CB340303"],"award-info":[{"award-number":["2014CB340303"]}]},{"DOI":"10.13039\/501100001809","name":"National Science Foundation of China (NSFC)","doi-asserted-by":"publisher","award":["61502021"],"award-info":[{"award-number":["61502021"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100001809","name":"National Science Foundation of China (NSFC)","doi-asserted-by":"publisher","award":["61328202"],"award-info":[{"award-number":["61328202"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100001809","name":"NSFC","doi-asserted-by":"publisher","award":["U1301253"],"award-info":[{"award-number":["U1301253"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["IEEE Trans. Knowl. Data Eng."],"published-print":{"date-parts":[[2016,9,1]]},"DOI":"10.1109\/tkde.2016.2555805","type":"journal-article","created":{"date-parts":[[2016,4,21]],"date-time":"2016-04-21T18:18:28Z","timestamp":1461262708000},"page":"2246-2266","source":"Crossref","is-referenced-by-count":112,"title":["A Survey of General-Purpose Crowdsourcing Techniques"],"prefix":"10.1109","volume":"28","author":[{"given":"Anand Inasu","family":"Chittilappilly","sequence":"first","affiliation":[]},{"given":"Lei","family":"Chen","sequence":"additional","affiliation":[]},{"given":"Sihem","family":"Amer-Yahia","sequence":"additional","affiliation":[]}],"member":"263","reference":[{"key":"ref39","article-title":"VizWiz: Nearly real-time answers to visual questions","author":"bigham","year":"0","journal-title":"Proc ACM Symp User Interface Softw Technol"},{"key":"ref38","doi-asserted-by":"publisher","DOI":"10.1145\/2207676.2207710"},{"key":"ref33","doi-asserted-by":"publisher","DOI":"10.1111\/j.1467-937X.2009.00534.x"},{"key":"ref32","doi-asserted-by":"publisher","DOI":"10.14778\/2536274.2536288"},{"key":"ref31","doi-asserted-by":"publisher","DOI":"10.1145\/2506364.2506371"},{"key":"ref30","year":"0"},{"key":"ref37","doi-asserted-by":"publisher","DOI":"10.1145\/1866029.1866078"},{"key":"ref36","doi-asserted-by":"publisher","DOI":"10.1145\/2339530.2339707"},{"key":"ref35","first-page":"467","article-title":"Task assignment optimization in knowledge-intensive crowdsourcing","volume":"24","author":"roy","year":"2015","journal-title":"Proc VLDB Endowment"},{"key":"ref34","first-page":"12","article-title":"Online assignment of heterogeneous tasks in crowdsourcing markets","author":"assadi","year":"2015","journal-title":"AAAI Conf Human Comp & Crowdsourcing"},{"key":"ref28","year":"0"},{"key":"ref27","year":"0"},{"key":"ref29","year":"0"},{"key":"ref20","year":"0"},{"key":"ref22","year":"0"},{"key":"ref21","year":"0"},{"key":"ref24","year":"0"},{"key":"ref23","year":"0"},{"key":"ref101","first-page":"670","article-title":"Divide and conquer : Crowdsourcing the creation of cross-lingual textual entailment corpora","author":"negri","year":"2011","journal-title":"Proc Conf Empirical Methods Natural Language Process"},{"key":"ref26","year":"0"},{"key":"ref100","doi-asserted-by":"publisher","DOI":"10.1145\/1518701.1518821"},{"key":"ref25","year":"0"},{"key":"ref50","doi-asserted-by":"publisher","DOI":"10.1145\/2723372.2749431"},{"key":"ref51","doi-asserted-by":"publisher","DOI":"10.1038\/nature09304"},{"key":"ref59","first-page":"7","article-title":"MetropolItalia: A crowdsourcing platform for linguistic field research","author":"kneissl","year":"0","journal-title":"Proc IADIS Int Conf WWW\/Internet"},{"key":"ref58","first-page":"11","article-title":"How crowdsourcable is your task","author":"eickhoff","year":"0","journal-title":"Proceedings of Workshop on Crowdsourcing for Search and Data Mining"},{"key":"ref57","doi-asserted-by":"publisher","DOI":"10.1145\/1924421.1924442"},{"key":"ref56","doi-asserted-by":"publisher","DOI":"10.1145\/1566374.1566384"},{"key":"ref55","first-page":"137","article-title":"CrowdQ: Crowdsourced query understanding","author":"demartini","year":"0","journal-title":"Proc Biennial Conf Innovative Data Syst Res"},{"key":"ref54","doi-asserted-by":"publisher","DOI":"10.2307\/2346806"},{"key":"ref53","doi-asserted-by":"publisher","DOI":"10.1145\/2448496.2448524"},{"key":"ref52","doi-asserted-by":"publisher","DOI":"10.1016\/j.artint.2013.06.002"},{"key":"ref40","doi-asserted-by":"publisher","DOI":"10.1109\/ICDE.2012.122"},{"key":"ref4","year":"0"},{"key":"ref3","year":"0"},{"key":"ref6","year":"0"},{"key":"ref5","year":"0"},{"key":"ref8","year":"0"},{"key":"ref49","doi-asserted-by":"publisher","DOI":"10.14778\/2794367.2794372"},{"key":"ref7","year":"0"},{"key":"ref9","year":"0"},{"key":"ref46","doi-asserted-by":"publisher","DOI":"10.1109\/MNET.2010.5430141"},{"key":"ref45","author":"chen","year":"0"},{"key":"ref48","doi-asserted-by":"publisher","DOI":"10.14778\/2733004.2733047"},{"key":"ref47","author":"chen","year":"0"},{"key":"ref42","doi-asserted-by":"publisher","DOI":"10.1177\/1354856507084420"},{"key":"ref41","first-page":"215","author":"bollob\u00e1s","year":"1998","journal-title":"Random Graphs"},{"key":"ref44","doi-asserted-by":"publisher","DOI":"10.1109\/CSE.2009.166"},{"key":"ref43","doi-asserted-by":"publisher","DOI":"10.5210\/fm.v13i6.2159"},{"key":"ref127","article-title":"The Problems with Crowdsourcing and how to Fix them","author":"wagorn","year":"0"},{"key":"ref126","doi-asserted-by":"crossref","first-page":"1465","DOI":"10.1126\/science.1160379","article-title":"reCAPTCHA: Human-based character recognition via web security measures","volume":"321","author":"von ahn","year":"2008","journal-title":"Science"},{"key":"ref125","doi-asserted-by":"publisher","DOI":"10.1145\/985692.985733"},{"key":"ref124","doi-asserted-by":"publisher","DOI":"10.1145\/2449396.2449398"},{"key":"ref73","doi-asserted-by":"publisher","DOI":"10.1145\/2207676.2208700"},{"key":"ref72","year":"0"},{"key":"ref129","doi-asserted-by":"publisher","DOI":"10.1145\/2463676.2465280"},{"key":"ref71","year":"0"},{"key":"ref128","doi-asserted-by":"publisher","DOI":"10.14778\/2350229.2350263"},{"key":"ref70","doi-asserted-by":"publisher","DOI":"10.1145\/1807342.1807376"},{"key":"ref76","doi-asserted-by":"publisher","DOI":"10.1527\/tjsai.27.133"},{"key":"ref130","doi-asserted-by":"publisher","DOI":"10.14778\/2536336.2536337"},{"key":"ref77","year":"0"},{"key":"ref74","year":"0"},{"key":"ref75","doi-asserted-by":"publisher","DOI":"10.1145\/1837885.1837906"},{"key":"ref133","doi-asserted-by":"publisher","DOI":"10.1109\/ALLERTON.2014.7028448"},{"key":"ref134","doi-asserted-by":"publisher","DOI":"10.1016\/j.eswa.2014.06.044"},{"key":"ref131","first-page":"2035","article-title":"Whose vote should count more: Optimal integration of labels from labelers of unknown expertise","author":"whitehill","year":"2009","journal-title":"Proc Neural Inf Process Syst"},{"key":"ref78","first-page":"1953","article-title":"Iterative learning for reliable crowdsourcing systems","author":"karger","year":"2011","journal-title":"Proc Neural Inform Process Syst"},{"key":"ref132","doi-asserted-by":"publisher","DOI":"10.1109\/CSE.2009.430"},{"key":"ref79","article-title":"An exploration of the influence that task parameters have on the performance of crowds","author":"kazai","year":"0","journal-title":"Proceedings of CrowdConf"},{"key":"ref136","doi-asserted-by":"publisher","DOI":"10.1145\/1978942.1979147"},{"key":"ref135","doi-asserted-by":"publisher","DOI":"10.1007\/978-94-007-7618-0_22"},{"key":"ref138","doi-asserted-by":"publisher","DOI":"10.1007\/s11063-014-9343-z"},{"key":"ref137","doi-asserted-by":"publisher","DOI":"10.1109\/PASSAT\/SocialCom.2011.203"},{"key":"ref60","first-page":"3102","article-title":"A model for aggregating contributions of synergistic crowdsourcing workflows","author":"fang","year":"2014","journal-title":"Proc AAAI Nat Conf Artificial Intell"},{"key":"ref139","doi-asserted-by":"publisher","DOI":"10.14778\/2536360.2536374"},{"key":"ref62","first-page":"183","article-title":"Crowdsourcing platform for large-scale speech data collection","author":"freitas","year":"2010","journal-title":"VI Jornadas en Tecnologa del Habla and II Iberian SLTech Workshop"},{"key":"ref61","article-title":"What's the right price? Pricing tasks for finishing on time","author":"faridani","year":"2011","journal-title":"Proc AAAI Nat Conf Artificial Intell"},{"key":"ref63","doi-asserted-by":"publisher","DOI":"10.1162\/003355300554917"},{"key":"ref64","doi-asserted-by":"publisher","DOI":"10.14778\/2824032.2824062"},{"key":"ref140","doi-asserted-by":"publisher","DOI":"10.1109\/INFCOM.2012.6195597"},{"key":"ref65","first-page":"15","article-title":"You're hired! an examination of crowdsourcing incentive models in human resource tasks","author":"harris","year":"2011","journal-title":"Proceedings of Workshop on Crowdsourcing for Search and Data Mining"},{"key":"ref141","first-page":"1031","article-title":"QASCA: A quality-aware task assignment system for crowdsourcing applications","author":"zheng","year":"2015","journal-title":"Proc ACM SIGMOD Int Conf Manag Data"},{"key":"ref66","doi-asserted-by":"publisher","DOI":"10.1145\/2207676.2208619"},{"key":"ref142","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-540-92185-1_63"},{"key":"ref67","volume":"57","author":"hirth","year":"2013","journal-title":"Proc of Mathematical and Computer Modelling"},{"key":"ref143","doi-asserted-by":"publisher","DOI":"10.1145\/2212776.2223826"},{"key":"ref68","first-page":"534","article-title":"Adaptive task assignment for crowdsourced classification","author":"ho","year":"2013","journal-title":"Proc Annu Int Conf Mach Learn"},{"key":"ref2","year":"0"},{"key":"ref69","first-page":"45","article-title":"Online task assignment in crowdsourcing markets","author":"ho","year":"2012","journal-title":"Proc AAAI Nat Conf Artificial Intell"},{"key":"ref1","year":"0"},{"key":"ref109","doi-asserted-by":"publisher","DOI":"10.1145\/1753846.1753873"},{"key":"ref95","doi-asserted-by":"publisher","DOI":"10.1561\/1900000044"},{"key":"ref108","first-page":"1","article-title":"Supervised learning from multiple experts: Whom to trust when everyone lies a bit","author":"raykar","year":"2009","journal-title":"Proc Annu Int Conf Mach Learn"},{"key":"ref94","author":"lynch","year":"0"},{"key":"ref107","doi-asserted-by":"publisher","DOI":"10.14778\/2809974.2809977"},{"key":"ref93","doi-asserted-by":"publisher","DOI":"10.1109\/BigData.2013.6691593"},{"key":"ref106","first-page":"1","article-title":"An evaluation of aggregation techniques in crowdsourcing","author":"hung","year":"0","journal-title":"Proc Web Inf Syst Eng"},{"key":"ref92","doi-asserted-by":"publisher","DOI":"10.14778\/2336664.2336676"},{"key":"ref105","doi-asserted-by":"publisher","DOI":"10.14778\/2536206.2536207"},{"key":"ref91","doi-asserted-by":"publisher","DOI":"10.1145\/1772690.1772843"},{"key":"ref104","doi-asserted-by":"publisher","DOI":"10.1145\/2430456.2430462"},{"key":"ref90","first-page":"21","article-title":"Ensuring quality in crowdsourced search relevance evaluation: The effects of training question distribution","author":"le","year":"2010","journal-title":"Proc SIGIR Workshop Crowdsourcing for Search Evaluation"},{"key":"ref103","first-page":"160","article-title":"Answering queries using humans, algorithms and databases","author":"parameswaran","year":"2011","journal-title":"Proc Conf Innovative Data Syst Res"},{"key":"ref102","doi-asserted-by":"publisher","DOI":"10.1145\/2047196.2047198"},{"key":"ref111","first-page":"1257","article-title":"Probabilistic matrix factorization","author":"salakhutdinov","year":"0","journal-title":"Proc Neural Inf Process Syst"},{"key":"ref112","doi-asserted-by":"publisher","DOI":"10.1056\/NEJMp1506643"},{"key":"ref110","article-title":"11 of the 12 Best Global Brands use creative crowdsourcing","author":"roth","year":"0"},{"key":"ref98","first-page":"361","article-title":"Reward system for completing FAQs","volume":"202","author":"moreno","year":"2009","journal-title":"New Front Appl Artif Intell"},{"key":"ref99","doi-asserted-by":"publisher","DOI":"10.14778\/2367502.2367537"},{"key":"ref96","doi-asserted-by":"publisher","DOI":"10.14778\/2047485.2047487"},{"key":"ref97","doi-asserted-by":"publisher","DOI":"10.1145\/1600150.1600175"},{"key":"ref10","year":"0"},{"key":"ref11","year":"0"},{"key":"ref12","year":"0"},{"key":"ref13","year":"0"},{"key":"ref14","year":"0"},{"key":"ref15","year":"0"},{"key":"ref118","first-page":"1","article-title":"Digital volunteerism during disaster: Crowdsourcing information processing","author":"starbird","year":"2011","journal-title":"Proc SIGCHI Conf Human Factors Comput Syst"},{"key":"ref16","year":"0"},{"key":"ref82","article-title":"When crowdsourcing doesnt work","author":"keith","year":"0"},{"key":"ref117","doi-asserted-by":"publisher","DOI":"10.1109\/CVPRW.2008.4562953"},{"key":"ref17","year":"0"},{"key":"ref81","doi-asserted-by":"publisher","DOI":"10.1145\/2424321.2424346"},{"key":"ref18","year":"0"},{"key":"ref84","article-title":"Testing the wisdom of crowds","author":"killian","year":"0"},{"key":"ref119","doi-asserted-by":"publisher","DOI":"10.14778\/2733004.2733024"},{"key":"ref19","year":"0"},{"key":"ref83","first-page":"1","article-title":"Quality control of crowd labeling through expert evaluation","author":"khattak","year":"2011","journal-title":"Proceedings of the NIPS Workshop on Computational Social Science and the Wisdom of Crowds"},{"key":"ref114","doi-asserted-by":"publisher","DOI":"10.1145\/1869086.1869100"},{"key":"ref113","doi-asserted-by":"publisher","DOI":"10.1145\/1401890.1401965"},{"key":"ref116","first-page":"254","article-title":"Cheap and fast but is it good?: Evaluating non-expert annotations for natural language tasks","author":"snow","year":"2008","journal-title":"Proc Conf Empirical Methods Natural Language Process"},{"key":"ref80","doi-asserted-by":"publisher","DOI":"10.1145\/2063576.2063860"},{"key":"ref115","year":"0"},{"key":"ref120","doi-asserted-by":"publisher","DOI":"10.1109\/ICDE.2014.6816736"},{"key":"ref89","doi-asserted-by":"publisher","DOI":"10.1145\/2380116.2380122"},{"key":"ref121","doi-asserted-by":"publisher","DOI":"10.1145\/1978942.1979146"},{"key":"ref122","year":"0"},{"key":"ref123","author":"ahn","year":"0"},{"key":"ref85","doi-asserted-by":"publisher","DOI":"10.1145\/1357054.1357127"},{"key":"ref86","doi-asserted-by":"publisher","DOI":"10.1109\/MC.2009.263"},{"key":"ref87","doi-asserted-by":"publisher","DOI":"10.1145\/2145204.2145354"},{"key":"ref88","doi-asserted-by":"publisher","DOI":"10.1007\/s10044-002-0173-7"}],"container-title":["IEEE Transactions on Knowledge and Data Engineering"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx7\/69\/7533522\/07456302.pdf?arnumber=7456302","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2022,1,12]],"date-time":"2022-01-12T16:02:50Z","timestamp":1642003370000},"score":1,"resource":{"primary":{"URL":"http:\/\/ieeexplore.ieee.org\/document\/7456302\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2016,9,1]]},"references-count":143,"journal-issue":{"issue":"9"},"URL":"https:\/\/doi.org\/10.1109\/tkde.2016.2555805","relation":{},"ISSN":["1041-4347"],"issn-type":[{"value":"1041-4347","type":"print"}],"subject":[],"published":{"date-parts":[[2016,9,1]]}}}