{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,10,27]],"date-time":"2025-10-27T16:16:48Z","timestamp":1761581808561,"version":"3.37.3"},"reference-count":104,"publisher":"Institute of Electrical and Electronics Engineers (IEEE)","issue":"7","license":[{"start":{"date-parts":[[2020,7,1]],"date-time":"2020-07-01T00:00:00Z","timestamp":1593561600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/ieeexplore.ieee.org\/Xplorehelp\/downloads\/license-information\/IEEE.html"},{"start":{"date-parts":[[2020,7,1]],"date-time":"2020-07-01T00:00:00Z","timestamp":1593561600000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2020,7,1]],"date-time":"2020-07-01T00:00:00Z","timestamp":1593561600000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"}],"funder":[{"name":"Visvesvaraya Young Faculty Research Fellowship","award":["2015-2016"],"award-info":[{"award-number":["2015-2016"]}]}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["IEEE Trans. Knowl. Data Eng."],"published-print":{"date-parts":[[2020,7,1]]},"DOI":"10.1109\/tkde.2019.2904064","type":"journal-article","created":{"date-parts":[[2019,3,8]],"date-time":"2019-03-08T20:00:51Z","timestamp":1552075251000},"page":"1234-1248","source":"Crossref","is-referenced-by-count":12,"title":["A Review of Judgment Analysis Algorithms for Crowdsourced Opinions"],"prefix":"10.1109","volume":"32","author":[{"given":"Sujoy","family":"Chatterjee","sequence":"first","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0001-8128-3521","authenticated-orcid":false,"given":"Anirban","family":"Mukhopadhyay","sequence":"additional","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0001-7781-1264","authenticated-orcid":false,"given":"Malay","family":"Bhattacharyya","sequence":"additional","affiliation":[]}],"member":"263","reference":[{"key":"ref39","doi-asserted-by":"crossref","first-page":"75","DOI":"10.1145\/1124772.1124784","article-title":"Verbosity: A game for collecting common-sense knowledge","author":"von ahn","year":"2006","journal-title":"Proc SIGCHI Conf Hum Factors Comput Syst"},{"key":"ref38","doi-asserted-by":"publisher","DOI":"10.1109\/ICDE.2016.7498229"},{"key":"ref33","first-page":"1","article-title":"Computing crowd consensus with partial agreement","volume":"30","author":"nguyen","year":"2017","journal-title":"IEEE Trans Knowl Data Eng"},{"key":"ref32","first-page":"21","article-title":"Towards managing knowledge collection from volunteer contributors","author":"chklovski","year":"2005","journal-title":"Proc AAAI Spring Symp Knowl Collection Volunteer Contrib"},{"key":"ref31","article-title":"Crowd-powered data mining","volume":"abs 1806 4968","author":"chai","year":"2018","journal-title":"CoRR"},{"key":"ref30","first-page":"819","article-title":"How to grade a test without knowing the answers: A bayesian graphical model for adaptive crowdsourcing and aptitude testing","author":"bachrach","year":"2012","journal-title":"Proc 29th Int Conf Mach Learn"},{"key":"ref37","doi-asserted-by":"publisher","DOI":"10.1109\/ICDAR.1999.791712"},{"key":"ref36","first-page":"47","article-title":"The public acquisition of commonsense knowledge","author":"singh","year":"2002","journal-title":"Proc AAAI Spring Symp Acquiring Linguistic Knowl Inf Access"},{"key":"ref35","doi-asserted-by":"publisher","DOI":"10.1016\/j.patrec.2013.05.012"},{"key":"ref34","doi-asserted-by":"publisher","DOI":"10.1145\/1837885.1837906"},{"key":"ref28","first-page":"2","article-title":"Report of crowdscale shared task challenge 2013","author":"liu","year":"2013","journal-title":"Proc Crowdscale Shared Task Challenge"},{"key":"ref27","first-page":"1","article-title":"Opinion ensembling: Learning from dependent judgements of the crowd","author":"bhattacharyya","year":"2013","journal-title":"Proc Crowdscale Shared Task Challenge"},{"key":"ref29","first-page":"3","article-title":"Bayesian combination of crowd-based tweet sentiment analysis judgments","author":"venanzi","year":"2013","journal-title":"Proc Crowdscale Shared Task Challenge"},{"key":"ref20","doi-asserted-by":"publisher","DOI":"10.3115\/1613715.1613751"},{"key":"ref22","doi-asserted-by":"publisher","DOI":"10.14778\/3055540.3055547"},{"key":"ref21","first-page":"1621","article-title":"Max-margin majority voting for learning from crowds","author":"tian","year":"2015","journal-title":"Proc 28th Int Conf Neural Inf Process Syst"},{"key":"ref24","doi-asserted-by":"publisher","DOI":"10.1145\/2556288.2557011"},{"key":"ref23","doi-asserted-by":"publisher","DOI":"10.1145\/2463676.2465318"},{"key":"ref101","doi-asserted-by":"publisher","DOI":"10.1093\/bioinformatics\/btm158"},{"key":"ref26","doi-asserted-by":"publisher","DOI":"10.1145\/985692.985733"},{"key":"ref100","doi-asserted-by":"publisher","DOI":"10.1016\/j.ygeno.2008.05.003"},{"key":"ref25","doi-asserted-by":"publisher","DOI":"10.1145\/1124772.1124782"},{"key":"ref50","doi-asserted-by":"publisher","DOI":"10.1145\/3035918.3064055"},{"key":"ref51","doi-asserted-by":"publisher","DOI":"10.1007\/s41019-017-0037-1"},{"key":"ref59","first-page":"2424","article-title":"The multidimensional wisdom of crowds","author":"welinder","year":"2010","journal-title":"Proc Neural Inf Process Syst"},{"key":"ref58","first-page":"2035","article-title":"Whose vote should count more: Optimal integration of labels from labelers of unknown expertise","author":"whitehill","year":"2009","journal-title":"Proc Neural Inf Process Syst"},{"key":"ref57","first-page":"778","article-title":"The theory and practice of item response theory","volume":"75","author":"ayala","year":"2010","journal-title":"Psychometrika"},{"key":"ref56","first-page":"225","article-title":"True label + confusions: A spectrum of probabilistic models in analyzing multiple ratings","author":"liu","year":"2012","journal-title":"Proc Int Conf Mach Learn"},{"key":"ref55","first-page":"1120","article-title":"Learning whom to trust with MACE","author":"hovy","year":"2013","journal-title":"Proc NAACL-HLT"},{"key":"ref54","first-page":"491","article-title":"Eliminating spammers and ranking annotators for crowdsourced labeling tasks","volume":"13","author":"raykar","year":"2011","journal-title":"J Mach Learn Res"},{"key":"ref53","doi-asserted-by":"publisher","DOI":"10.14778\/3025111.3025118"},{"key":"ref52","doi-asserted-by":"publisher","DOI":"10.1145\/2723372.2749430"},{"key":"ref40","doi-asserted-by":"publisher","DOI":"10.1145\/2063576.2063860"},{"key":"ref4","doi-asserted-by":"publisher","DOI":"10.1016\/j.ins.2016.09.036"},{"key":"ref3","doi-asserted-by":"publisher","DOI":"10.1145\/2732587.2732607"},{"key":"ref6","doi-asserted-by":"publisher","DOI":"10.1177\/1354856507084420"},{"key":"ref5","doi-asserted-by":"publisher","DOI":"10.1109\/ICDE.2017.207"},{"key":"ref8","doi-asserted-by":"publisher","DOI":"10.1109\/CVPRW.2008.4562953"},{"key":"ref49","doi-asserted-by":"publisher","DOI":"10.1145\/2783258.2783314"},{"key":"ref7","first-page":"1085","article-title":"Inferring ground truth from subjective labelling of venus images","author":"smyth","year":"1994","journal-title":"Proc Neural Inf Process Syst"},{"key":"ref9","doi-asserted-by":"publisher","DOI":"10.1145\/985692.985733"},{"key":"ref46","doi-asserted-by":"publisher","DOI":"10.1145\/2882903.2882953"},{"key":"ref45","doi-asserted-by":"publisher","DOI":"10.1109\/TKDE.2016.2535242"},{"key":"ref48","doi-asserted-by":"publisher","DOI":"10.1145\/2723372.2750550"},{"key":"ref47","doi-asserted-by":"publisher","DOI":"10.1109\/TKDE.2016.2518669"},{"key":"ref42","first-page":"99","article-title":"Using the amazon mechanical turk to transcribe and annotate meeting speech for extractive summarization","author":"marge","year":"2010","journal-title":"NAACL HLT Workshop on Creating Speech and Language Data with Amazon's Mechanical Turk"},{"key":"ref41","doi-asserted-by":"publisher","DOI":"10.1145\/1772690.1772843"},{"key":"ref44","doi-asserted-by":"publisher","DOI":"10.1145\/3035918.3054776"},{"key":"ref43","doi-asserted-by":"publisher","DOI":"10.1109\/TKDE.2017.2656086"},{"journal-title":"Probabilistic Graphical Models Principles and Techniques - Adaptive Computation and Machine Learning","year":"2009","author":"koller","key":"ref73"},{"key":"ref72","doi-asserted-by":"publisher","DOI":"10.2307\/2346806"},{"key":"ref71","article-title":"Quality enhancement by weighted rank aggregation of crowd opinion","author":"chatterjee","year":"2017","journal-title":"Proc WiP Track 5th AAAI Conf Hum Comput Crowdsourcing"},{"key":"ref70","doi-asserted-by":"publisher","DOI":"10.1109\/ICASSP.2012.6288327"},{"key":"ref76","first-page":"1","article-title":"Scoring workers in crowdsourcing: How many control questions are enough?","author":"liu","year":"2013","journal-title":"Proc 27th Annu Conf Neural Inf Process Syst"},{"key":"ref77","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-71246-8_7"},{"key":"ref74","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/P17-1028"},{"key":"ref75","first-page":"3537","article-title":"Spectral methods meet em: A provably optimal algorithm for crowdsourcing","volume":"17","author":"zhang","year":"2016","journal-title":"J Mach Learn Res"},{"key":"ref78","first-page":"193","article-title":"On optimality of jury selection in crowdsourcing","author":"zheng","year":"2015","journal-title":"Proc Intl Conf Extending Database Technology"},{"key":"ref79","first-page":"1","article-title":"Quality control of crowd labeling through expert evaluation","author":"khattak","year":"2011","journal-title":"Proc 2nd Workshop Comput Social Sci Wisdom Crowds"},{"key":"ref60","first-page":"101","article-title":"Improving quality of crowdsourced labels via probabilistic matrix factorization","author":"jung","year":"2012","journal-title":"Proceedings of the 4th AAAI Workshop on Human Computation"},{"key":"ref62","first-page":"2445","article-title":"Group decision making with probabilistic graphical model","author":"chatterjee","year":"0","journal-title":"Proc CHI Extended Abstracts"},{"key":"ref61","doi-asserted-by":"publisher","DOI":"10.1109\/TKDE.2015.2504974"},{"key":"ref63","first-page":"92","article-title":"Identifying and accounting for task-dependent bias in crowdsourcing","author":"kamar","year":"2015","journal-title":"Proc 3rd AAAI Conf Hum Comput Crowdsourcing"},{"key":"ref64","first-page":"32","article-title":"MicroTalk: Using argumentation to improve crowdsourcing accuracy","author":"drapeau","year":"0","journal-title":"Proc 4th AAAI Conf Hum Comput Crowdsourcing"},{"key":"ref65","article-title":"Consensus of dependent opinions","author":"chatterjee","year":"0","journal-title":"Proc WiP Track 4th AAAI Conf Hum Comput Crowdsourcing"},{"key":"ref66","doi-asserted-by":"publisher","DOI":"10.1016\/j.ins.2017.01.036"},{"key":"ref67","first-page":"16","article-title":"Smart city planning with constrained crowd judgment analysis","author":"chatterjee","year":"2017","journal-title":"Proc AAAI Spring Symp AI Soc Good"},{"key":"ref68","article-title":"Constrained crowd judgment analysis","author":"chatterjee","year":"2017","journal-title":"ACM SIGWEB Newslett Autumn"},{"key":"ref2","doi-asserted-by":"publisher","DOI":"10.1037\/0033-2909.134.3.404"},{"key":"ref69","doi-asserted-by":"publisher","DOI":"10.1109\/TPDS.2016.2515092"},{"key":"ref1","volume":"1","author":"brunswik","year":"1952","journal-title":"The Conceptual Framework of Psychology (International Encyclopedia of Unified Science)"},{"key":"ref95","doi-asserted-by":"publisher","DOI":"10.14778\/2733085.2733105"},{"key":"ref94","doi-asserted-by":"publisher","DOI":"10.1145\/2588555.2610509"},{"key":"ref93","first-page":"692","article-title":"Variational inference for crowdsourcing","author":"liu","year":"2012","journal-title":"Proc 25th Int Conf Neural Inf Process Syst"},{"key":"ref92","first-page":"2195","article-title":"Learning from the wisdom of crowds by minimax entropy","author":"zhou","year":"2012","journal-title":"Proc 25th Int Conf Neural Inf Process Syst"},{"key":"ref91","doi-asserted-by":"publisher","DOI":"10.14778\/2735496.2735505"},{"key":"ref104","article-title":"A markov chain based ensemble method for crowdsourced clustering","author":"chatterjee","year":"0","journal-title":"Proc WiP Track 4th AAAI Conf Human Comput Crowdsourcing arXiv preprint arXiv 1609 01484"},{"key":"ref90","doi-asserted-by":"publisher","DOI":"10.1007\/s11263-018-1065-7"},{"key":"ref103","article-title":"Image clustering without ground truth","author":"dash","year":"2016","journal-title":"Proc GroupSight Workshop Collocated 4th AAAI Conf Human Comput Crowdsourcing"},{"key":"ref102","doi-asserted-by":"publisher","DOI":"10.1016\/j.csda.2006.11.006"},{"key":"ref98","article-title":"T-crowd: Effective crowdsourcing for tabular data","volume":"abs 1708 2125","author":"shan","year":"2017","journal-title":"CoRR"},{"key":"ref99","doi-asserted-by":"publisher","DOI":"10.1145\/3077136.3080679"},{"key":"ref96","first-page":"1061","article-title":"Active learning from crowds with unsure option","author":"zhong","year":"2015","journal-title":"Proc 24th Int Conf Artif Intell"},{"key":"ref97","first-page":"3439","article-title":"Crowdsourced action-model acquisition for planning","author":"zhuo","year":"2015","journal-title":"Proc 29th AAAI Conf Artif Intell"},{"key":"ref10","doi-asserted-by":"publisher","DOI":"10.1109\/ICDE.2017.56"},{"key":"ref11","first-page":"1757","article-title":"Learning from multiple sources","volume":"9","author":"crammer","year":"2008","journal-title":"J Mach Learn Res"},{"key":"ref12","article-title":"Online decision making in crowdsourcing markets: Theoretical challenges (position paper)","volume":"abs 1308 1746","author":"slivkins","year":"2013","journal-title":"CoRR"},{"key":"ref13","doi-asserted-by":"publisher","DOI":"10.1145\/2488388.2488414"},{"key":"ref14","doi-asserted-by":"publisher","DOI":"10.1145\/2187836.2187900"},{"key":"ref15","doi-asserted-by":"publisher","DOI":"10.1145\/2441776.2441923"},{"key":"ref16","doi-asserted-by":"publisher","DOI":"10.1145\/2484028.2484199"},{"key":"ref82","doi-asserted-by":"publisher","DOI":"10.1109\/TPAMI.2018.2817205"},{"key":"ref17","first-page":"787","article-title":"Learning to aggregate ordinal labels by maximizing separating width","author":"chen","year":"2017","journal-title":"Proc 34th Int Conf Mach Learn"},{"key":"ref81","first-page":"3233","article-title":"Identifying unreliable and adversarial workers in crowdsourced labeling tasks","volume":"18","author":"jagabathula","year":"2017","journal-title":"J Mach Learn Res"},{"key":"ref18","doi-asserted-by":"publisher","DOI":"10.1145\/1753846.1753873"},{"key":"ref84","first-page":"1555","article-title":"Semi-supervised learning from crowds using deep generative models","author":"atarashi","year":"2018","journal-title":"Proc 32nd AAAI Conf Artif Intell 30th Innovative Appl Artif Intell 8th AAAI Symp Educ Adv Artif Intell"},{"key":"ref19","first-page":"2035","article-title":"SQUARE: A benchmark for research on computing crowd consensus","author":"sheshadri","year":"2013","journal-title":"Proc AAAI Conf Hum Comput"},{"key":"ref83","doi-asserted-by":"publisher","DOI":"10.24963\/ijcai.2017\/184"},{"key":"ref80","first-page":"1953","article-title":"Iterative learning for reliable crowdsourcing systems","author":"karger","year":"2011","journal-title":"Proc Adv Neural Inf Process Syst"},{"key":"ref89","article-title":"Combining preference and absolute judgments in a crowd-sourced setting","author":"ye","year":"2013","journal-title":"Proc ICML Workshop"},{"key":"ref85","first-page":"3581","article-title":"Semi-supervised learning with deep generative models","author":"kingma","year":"2014","journal-title":"Proc 27th Int Conf Neural Inf Process Syst"},{"key":"ref86","first-page":"1611","article-title":"Deep learning from crowds","author":"rodrigues","year":"2018","journal-title":"Proc 32nd AAAI Conf Artif Intell 30th Innovative Appl Artif Intell 8th AAAI Symp Edu Adv Artif Intell"},{"key":"ref87","doi-asserted-by":"publisher","DOI":"10.1109\/ICASSP.2011.5946971"},{"key":"ref88","doi-asserted-by":"publisher","DOI":"10.1145\/2433396.2433420"}],"container-title":["IEEE Transactions on Knowledge and Data Engineering"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx7\/69\/9108388\/08663457.pdf?arnumber=8663457","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2022,9,13]],"date-time":"2022-09-13T15:35:35Z","timestamp":1663083335000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/8663457\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2020,7,1]]},"references-count":104,"journal-issue":{"issue":"7"},"URL":"https:\/\/doi.org\/10.1109\/tkde.2019.2904064","relation":{},"ISSN":["1041-4347","1558-2191","2326-3865"],"issn-type":[{"type":"print","value":"1041-4347"},{"type":"electronic","value":"1558-2191"},{"type":"electronic","value":"2326-3865"}],"subject":[],"published":{"date-parts":[[2020,7,1]]}}}