{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,16]],"date-time":"2026-04-16T19:09:01Z","timestamp":1776366541282,"version":"3.51.2"},"reference-count":51,"publisher":"Institute of Electrical and Electronics Engineers (IEEE)","license":[{"start":{"date-parts":[[2021,1,1]],"date-time":"2021-01-01T00:00:00Z","timestamp":1609459200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/legalcode"},{"start":{"date-parts":[[2021,1,1]],"date-time":"2021-01-01T00:00:00Z","timestamp":1609459200000},"content-version":"am","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/legalcode"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["IEEE Access"],"published-print":{"date-parts":[[2021]]},"DOI":"10.1109\/access.2021.3127994","type":"journal-article","created":{"date-parts":[[2021,11,13]],"date-time":"2021-11-13T00:23:28Z","timestamp":1636763008000},"page":"153058-153071","source":"Crossref","is-referenced-by-count":6,"title":["Aggregating Reliable Submissions in Crowdsourcing Systems"],"prefix":"10.1109","volume":"9","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-3390-9360","authenticated-orcid":false,"given":"Ayswarya R.","family":"Kurup","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-9422-6689","authenticated-orcid":false,"given":"G. P.","family":"Sajeev","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-5646-3213","authenticated-orcid":false,"given":"J.","family":"Swaminathan","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"263","reference":[{"key":"ref39","first-page":"977","article-title":"Pairwise hits: Quality estimation from pairwise comparisons in creator-evaluator crowdsourcing process","author":"sunahase","year":"2017","journal-title":"Proc 21st AAAI Conf Artif Intell"},{"key":"ref38","first-page":"254","article-title":"Cheap and fast–but is it good? Evaluating non-expert annotations for natural language tasks","author":"snow","year":"2008","journal-title":"Proc Conf Empirical Methods Natural Lang Process"},{"key":"ref33","doi-asserted-by":"publisher","DOI":"10.1109\/79.543975"},{"key":"ref32","doi-asserted-by":"publisher","DOI":"10.1145\/1416950.1416952"},{"key":"ref31","first-page":"49","article-title":"Framework and literature analysis for crowdsourcing’s answer aggregation","volume":"60","author":"moayedikia","year":"2017","journal-title":"J Comput Inf Syst"},{"key":"ref30","doi-asserted-by":"publisher","DOI":"10.1016\/j.dss.2019.05.005"},{"key":"ref37","first-page":"1297","article-title":"Learning from crowds","volume":"11","author":"raykar","year":"2010","journal-title":"J Mach Learn Res"},{"key":"ref36","doi-asserted-by":"publisher","DOI":"10.1155\/2017\/7831897"},{"key":"ref35","doi-asserted-by":"publisher","DOI":"10.1109\/TKDE.2009.191"},{"key":"ref34","doi-asserted-by":"publisher","DOI":"10.1177\/0165551514550140"},{"key":"ref28","first-page":"692","article-title":"Variational inference for crowdsourcing","author":"liu","year":"2012","journal-title":"Proc Adv Neural Inf Process Syst"},{"key":"ref27","doi-asserted-by":"publisher","DOI":"10.1016\/j.ins.2021.05.060"},{"key":"ref29","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-01012-6_16"},{"key":"ref2","doi-asserted-by":"publisher","DOI":"10.1007\/978-1-4842-3564-5_8"},{"key":"ref1","year":"2021","journal-title":"Figure Eight"},{"key":"ref20","first-page":"2177","article-title":"Neural word embedding as implicit matrix factorization","volume":"27","author":"levy","year":"2014","journal-title":"Proc 27th Int Conf Neural Inf Process Syst"},{"key":"ref22","first-page":"165","article-title":"The wisdom of minority: Discovering and targeting the right group of workers for crowdsourcing","author":"li","year":"2014","journal-title":"Proc Int World Wide Web Conf"},{"key":"ref21","doi-asserted-by":"publisher","DOI":"10.1109\/TKDE.2016.2535242"},{"key":"ref24","doi-asserted-by":"publisher","DOI":"10.1007\/s11432-020-3118-7"},{"key":"ref23","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-01421-6_57"},{"key":"ref26","doi-asserted-by":"publisher","DOI":"10.1109\/TKDE.2013.19"},{"key":"ref25","doi-asserted-by":"publisher","DOI":"10.1109\/TBDATA.2016.2612236"},{"key":"ref50","doi-asserted-by":"publisher","DOI":"10.1007\/s10796-012-9350-4"},{"key":"ref51","doi-asserted-by":"publisher","DOI":"10.1109\/TKDE.2018.2849385"},{"key":"ref10","doi-asserted-by":"publisher","DOI":"10.1016\/j.jsis.2012.03.002"},{"key":"ref11","doi-asserted-by":"publisher","DOI":"10.14778\/2824032.2824136"},{"key":"ref40","doi-asserted-by":"publisher","DOI":"10.1145\/3368986"},{"key":"ref12","first-page":"1468","article-title":"Crowdsourcing: A review and suggestions for future research","volume":"20","author":"ghezzi","year":"2017","journal-title":"Int J Manage Rev"},{"key":"ref13","first-page":"1","article-title":"An evaluation of aggregation techniques in crowdsourcing","author":"hung","year":"2013","journal-title":"Proc Int Conf Web Inf Syst Eng"},{"key":"ref14","doi-asserted-by":"publisher","DOI":"10.1145\/3226028"},{"key":"ref15","doi-asserted-by":"publisher","DOI":"10.1109\/TNNLS.2021.3082496"},{"key":"ref16","author":"kurup","year":"2020","journal-title":"Task-Aggregation"},{"key":"ref17","doi-asserted-by":"publisher","DOI":"10.1109\/ISED48680.2019.9096224"},{"key":"ref18","doi-asserted-by":"publisher","DOI":"10.1504\/IJWET.2019.105588"},{"key":"ref19","doi-asserted-by":"publisher","DOI":"10.1016\/j.elerap.2020.100946"},{"key":"ref4","doi-asserted-by":"publisher","DOI":"10.1145\/3148148"},{"key":"ref3","doi-asserted-by":"publisher","DOI":"10.1145\/2487575.2487600"},{"key":"ref6","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1111\/j.2517-6161.1977.tb01600.x","article-title":"Maximum likelihood from incomplete data via the EM algorithm","volume":"39","author":"dempster","year":"1977","journal-title":"J Roy Statist Soc B Statist Methodol"},{"key":"ref5","doi-asserted-by":"publisher","DOI":"10.2307\/2346806"},{"key":"ref8","doi-asserted-by":"publisher","DOI":"10.1145\/2723372.2750550"},{"key":"ref49","doi-asserted-by":"publisher","DOI":"10.1007\/s10462-016-9491-9"},{"key":"ref7","doi-asserted-by":"publisher","DOI":"10.1145\/2872427.2883030"},{"key":"ref9","author":"feldman","year":"2007","journal-title":"The Text Mining Handbook Advanced Approaches in Analyzing Unstructured Data"},{"key":"ref46","doi-asserted-by":"publisher","DOI":"10.1016\/S0306-4573(00)00010-8"},{"key":"ref45","doi-asserted-by":"publisher","DOI":"10.1613\/jair.5175"},{"key":"ref48","doi-asserted-by":"publisher","DOI":"10.1109\/TNNLS.2018.2890148"},{"key":"ref47","doi-asserted-by":"publisher","DOI":"10.1017\/ATSIP.2019.12"},{"key":"ref42","doi-asserted-by":"publisher","DOI":"10.1007\/s10115-020-01475-y"},{"key":"ref41","doi-asserted-by":"publisher","DOI":"10.1016\/j.ipm.2019.01.005"},{"key":"ref44","doi-asserted-by":"publisher","DOI":"10.1145\/2566486.2567989"},{"key":"ref43","doi-asserted-by":"publisher","DOI":"10.1145\/3379336.3381499"}],"container-title":["IEEE Access"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx7\/6287639\/9312710\/09614193.pdf?arnumber=9614193","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,9,12]],"date-time":"2024-09-12T02:07:13Z","timestamp":1726106833000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/9614193\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2021]]},"references-count":51,"URL":"https:\/\/doi.org\/10.1109\/access.2021.3127994","relation":{},"ISSN":["2169-3536"],"issn-type":[{"value":"2169-3536","type":"electronic"}],"subject":[],"published":{"date-parts":[[2021]]}}}