{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,1,28]],"date-time":"2026-01-28T03:20:43Z","timestamp":1769570443214,"version":"3.49.0"},"reference-count":96,"publisher":"Institute of Electrical and Electronics Engineers (IEEE)","license":[{"start":{"date-parts":[[2022,1,1]],"date-time":"2022-01-01T00:00:00Z","timestamp":1640995200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/legalcode"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["IEEE Access"],"published-print":{"date-parts":[[2022]]},"DOI":"10.1109\/access.2022.3207292","type":"journal-article","created":{"date-parts":[[2022,9,16]],"date-time":"2022-09-16T19:35:52Z","timestamp":1663356952000},"page":"99709-99723","source":"Crossref","is-referenced-by-count":4,"title":["What Quality Control Mechanisms do We Need for High-Quality Crowd Work?"],"prefix":"10.1109","volume":"10","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-1049-3040","authenticated-orcid":false,"given":"Margeret","family":"Hall","sequence":"first","affiliation":[{"name":"Department of Strategy and Innovation, Wirtschaftsuniversität Wien, Vienna, Austria"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-7457-9148","authenticated-orcid":false,"given":"Mohammad Farhad","family":"Afzali","sequence":"additional","affiliation":[{"name":"College of Information Science and Technology, University of Nebraska Omaha, Omaha, NE, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Markus","family":"Krause","sequence":"additional","affiliation":[{"name":"Brainworks.ai, Berkeley, CA, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-9379-3879","authenticated-orcid":false,"given":"Simon","family":"Caton","sequence":"additional","affiliation":[{"name":"School of Computer Science, University College Dublin, Dublin, Ireland"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"263","reference":[{"key":"ref1","doi-asserted-by":"publisher","DOI":"10.1109\/MIC.2013.20"},{"key":"ref2","doi-asserted-by":"publisher","DOI":"10.24251\/hicss.2019.636"},{"key":"ref3","doi-asserted-by":"publisher","DOI":"10.1016\/j.jvb.2020.103435"},{"key":"ref4","doi-asserted-by":"publisher","DOI":"10.1038\/s43018-020-0065-z"},{"key":"ref5","doi-asserted-by":"publisher","DOI":"10.17705\/1jais.00441"},{"key":"ref6","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-15168-7_55"},{"key":"ref7","doi-asserted-by":"publisher","DOI":"10.1109\/CGC.2013.87"},{"key":"ref8","doi-asserted-by":"publisher","DOI":"10.1109\/CGC.2013.90"},{"key":"ref9","first-page":"1","volume-title":"Quality-based pricing for crowdsourced workers","author":"Wang","year":"2013"},{"key":"ref10","doi-asserted-by":"publisher","DOI":"10.1145\/1837885.1837906"},{"key":"ref11","doi-asserted-by":"publisher","DOI":"10.1145\/2468356.2468386"},{"key":"ref12","first-page":"43","article-title":"Programmatic gold: Targeted and scalable quality assurance in crowdsourcing","volume-title":"Proc. AAAI Workshop Hum. Comput.","author":"Oleson"},{"key":"ref13","first-page":"97","article-title":"On quality control and machine learning in crowdsourcing","volume-title":"Proc. AAAI Workshop","volume":"11","author":"Lease"},{"key":"ref14","doi-asserted-by":"publisher","DOI":"10.1145\/3290605.3300761"},{"key":"ref15","doi-asserted-by":"publisher","DOI":"10.1145\/2998181.2998248"},{"key":"ref16","doi-asserted-by":"publisher","DOI":"10.25300\/misq\/2016\/40.2.01"},{"key":"ref17","doi-asserted-by":"publisher","DOI":"10.1145\/2675133.2675158"},{"key":"ref18","doi-asserted-by":"publisher","DOI":"10.1002\/poi3.111"},{"key":"ref19","first-page":"15","article-title":"Dealing with sub-optimal crowd work: Implications of current quality control practices","volume-title":"Proc. CHI","author":"Gadiraju"},{"key":"ref20","doi-asserted-by":"publisher","DOI":"10.1145\/1538788.1538808"},{"key":"ref21","doi-asserted-by":"publisher","DOI":"10.1145\/2047196.2047199"},{"key":"ref22","doi-asserted-by":"publisher","DOI":"10.1145\/2499149.2499168"},{"key":"ref23","first-page":"20","article-title":"Mechanical cheat: Spamming schemes and adversarial techniques on crowdsourcing platforms","volume-title":"Proc. CEUR Workshop","volume":"842","author":"Difallah"},{"key":"ref24","doi-asserted-by":"publisher","DOI":"10.1145\/2858036.2858490"},{"key":"ref25","doi-asserted-by":"publisher","DOI":"10.1145\/2858036.2858237"},{"key":"ref26","doi-asserted-by":"publisher","DOI":"10.1016\/j.jebo.2013.03.003"},{"key":"ref27","doi-asserted-by":"publisher","DOI":"10.1109\/HICSS.2016.542"},{"key":"ref28","first-page":"121","article-title":"Brave new digital work? New forms of performance control in crowdwork","volume-title":"Research in the Sociology of Work","volume":"33","author":"Gerber","year":"2019"},{"key":"ref29","doi-asserted-by":"publisher","DOI":"10.1145\/3313831.3376677"},{"key":"ref30","doi-asserted-by":"publisher","DOI":"10.1002\/poi3.93"},{"key":"ref31","doi-asserted-by":"publisher","DOI":"10.1002\/poi3.102"},{"key":"ref32","doi-asserted-by":"publisher","DOI":"10.1007\/s10726-018-9578-6"},{"key":"ref33","doi-asserted-by":"publisher","DOI":"10.1002\/poi3.95"},{"key":"ref34","doi-asserted-by":"publisher","DOI":"10.1145\/2998181.2998197"},{"key":"ref35","doi-asserted-by":"publisher","DOI":"10.1145\/2396761.2398697"},{"key":"ref36","doi-asserted-by":"publisher","DOI":"10.1145\/2063576.2063860"},{"key":"ref37","doi-asserted-by":"publisher","DOI":"10.1145\/3078714.3078715"},{"key":"ref38","doi-asserted-by":"publisher","DOI":"10.1609\/hcomp.v4i1.13283"},{"key":"ref39","doi-asserted-by":"publisher","DOI":"10.1145\/2810239"},{"key":"ref40","doi-asserted-by":"publisher","DOI":"10.1145\/1357054.1357127"},{"key":"ref41","doi-asserted-by":"publisher","DOI":"10.1145\/2818048.2819942"},{"key":"ref42","doi-asserted-by":"publisher","DOI":"10.1145\/2441776.2441923"},{"key":"ref43","first-page":"1","article-title":"Managing the crowd: Towards a taxonomy of crowdsourcing processes","volume-title":"Proc. 17th Americas Conf. Inf. Syst.","author":"Geiger"},{"key":"ref44","doi-asserted-by":"publisher","DOI":"10.1007\/s10606-018-9336-y"},{"key":"ref45","doi-asserted-by":"publisher","DOI":"10.1145\/3119930"},{"key":"ref46","doi-asserted-by":"publisher","DOI":"10.1145\/2872427.2883036"},{"key":"ref47","doi-asserted-by":"publisher","DOI":"10.1609\/icwsm.v13i01.3216"},{"issue":"3","key":"ref48","first-page":"1","article-title":"Opportunities for crowdsourcing research on Amazon mechanical Turk","volume":"5","author":"Chen","year":"2011","journal-title":"Interface"},{"key":"ref49","doi-asserted-by":"publisher","DOI":"10.1093\/pan\/mpr057"},{"key":"ref50","doi-asserted-by":"publisher","DOI":"10.1145\/1401890.1401965"},{"key":"ref51","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-17358-5_17"},{"key":"ref52","doi-asserted-by":"publisher","DOI":"10.1016\/j.riob.2010.09.001"},{"key":"ref53","doi-asserted-by":"publisher","DOI":"10.1016\/j.jebo.2008.01.004"},{"key":"ref54","doi-asserted-by":"publisher","DOI":"10.1145\/1978942.1979148"},{"key":"ref55","doi-asserted-by":"publisher","DOI":"10.1145\/1979742.1979593"},{"key":"ref56","doi-asserted-by":"publisher","DOI":"10.2753\/mis0742-1222290308"},{"key":"ref57","doi-asserted-by":"publisher","DOI":"10.1037\/0003-066x.55.1.68"},{"key":"ref58","doi-asserted-by":"publisher","DOI":"10.1080\/07421222.2016.1172459"},{"key":"ref59","doi-asserted-by":"publisher","DOI":"10.1145\/2858036.2858539"},{"key":"ref60","first-page":"114","article-title":"Predicting crowd-based translation quality with language-independent feature vectors","volume-title":"Proc. Workshops 26th AAAI Conf. Artif. Intell.","author":"Runge"},{"issue":"1","key":"ref61","doi-asserted-by":"crossref","first-page":"5","DOI":"10.1023\/A:1010933404324","article-title":"Random forests","volume":"45","author":"Breiman","year":"2001","journal-title":"Mach. Learn."},{"key":"ref62","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-540-87479-9_34"},{"key":"ref63","doi-asserted-by":"publisher","DOI":"10.1007\/bf00117831"},{"issue":"10","key":"ref64","first-page":"2825","article-title":"Scikit-learn: Machine learning in Python","volume":"12","author":"Pedregosa","year":"2012","journal-title":"J. Mach. Learn. Res."},{"key":"ref65","doi-asserted-by":"publisher","DOI":"10.1111\/j.2517-6161.1977.tb01600.x"},{"key":"ref66","doi-asserted-by":"publisher","DOI":"10.1080\/19312458.2011.568376"},{"key":"ref67","doi-asserted-by":"publisher","DOI":"10.4135\/9781071878781"},{"key":"ref68","doi-asserted-by":"publisher","DOI":"10.1177\/001316447003000105"},{"key":"ref69","first-page":"832","article-title":"Sentence similarity based on relevance","volume-title":"Proc. 12th Int. Conf. Inf. Process. Manage. Uncertainty Knowl.-Based Syst.","volume":"2008","author":"Feng"},{"key":"ref70","doi-asserted-by":"publisher","DOI":"10.1145\/1459352.1459355"},{"key":"ref71","first-page":"1419","article-title":"WikiRelate! Computing semantic relatedness using Wikipedia","volume-title":"Proc. AAAI","author":"Strube"},{"key":"ref72","first-page":"315","article-title":"Measuring semantic similarity in the taxonomy of WordNet","volume-title":"Proc. 28th Australas. Comput. Sci. Conf.","author":"Yang"},{"key":"ref73","first-page":"1","article-title":"Using information content to evaluate semantic similarity in a taxonomy","volume-title":"Proc. 14th Int. Joint Conf. Artif. Intell. (IJCAI)","author":"Resnik"},{"key":"ref74","doi-asserted-by":"publisher","DOI":"10.1145\/1963405.1963455"},{"key":"ref75","doi-asserted-by":"publisher","DOI":"10.1145\/371920.372094"},{"key":"ref76","doi-asserted-by":"publisher","DOI":"10.1007\/978-1-4614-8806-4_31"},{"key":"ref77","doi-asserted-by":"publisher","DOI":"10.1609\/aimag.v31i3.2303"},{"key":"ref78","doi-asserted-by":"publisher","DOI":"10.1145\/1837885.1837900"},{"key":"ref79","volume-title":"Homo Ludens in the Loop: Playful Human Computation Systems","author":"Krause","year":"2014"},{"key":"ref80","first-page":"1","article-title":"Using monolingual human computation to improve language translation via targeted paraphrase","volume-title":"Proc. ACM SIGKDD Workshop Hum. Comput.","author":"Resnik"},{"key":"ref81","doi-asserted-by":"publisher","DOI":"10.1145\/1978942.1979111"},{"key":"ref82","first-page":"1220","article-title":"Crowdsourcing translation: Professional quality from non-professionals","volume-title":"Proc. ACL","author":"Zaidan"},{"key":"ref83","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2015.123"},{"key":"ref84","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2009.5206848"},{"key":"ref85","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2008.4587637"},{"key":"ref86","first-page":"487","article-title":"Learning deep features for scene recognition using places database","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","volume":"27","author":"Zhou"},{"key":"ref87","article-title":"LSUN: Construction of a large-scale image dataset using deep learning with humans in the loop","volume-title":"arXiv:1506.03365","author":"Yu","year":"2015"},{"key":"ref88","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2013.81"},{"key":"ref89","doi-asserted-by":"publisher","DOI":"10.1098\/rspa.1937.0109"},{"key":"ref90","article-title":"On-the-job learning for micro-task workers","volume-title":"Proc. Hum. Comput.","author":"Streuer"},{"key":"ref91","doi-asserted-by":"publisher","DOI":"10.1145\/2851581.2856488"},{"key":"ref92","doi-asserted-by":"publisher","DOI":"10.1145\/2858036.2858121"},{"key":"ref93","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-92052-8_38"},{"key":"ref94","article-title":"Work-learn: Necessary and sufficient conditions for upskilling homeless adults with entry-level programming and tech sector skills","volume-title":"Proc. ICIS TREOs","volume":"53","author":"Hall"},{"key":"ref95","doi-asserted-by":"publisher","DOI":"10.1145\/3170427.3174351"},{"key":"ref96","doi-asserted-by":"publisher","DOI":"10.1145\/3272973.3274545"}],"container-title":["IEEE Access"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx7\/6287639\/9668973\/09893786.pdf?arnumber=9893786","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,1,22]],"date-time":"2024-01-22T22:14:47Z","timestamp":1705961687000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/9893786\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2022]]},"references-count":96,"URL":"https:\/\/doi.org\/10.1109\/access.2022.3207292","relation":{},"ISSN":["2169-3536"],"issn-type":[{"value":"2169-3536","type":"electronic"}],"subject":[],"published":{"date-parts":[[2022]]}}}