{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,6,22]],"date-time":"2026-06-22T13:15:29Z","timestamp":1782134129391,"version":"3.54.5"},"reference-count":77,"publisher":"Institute of Electrical and Electronics Engineers (IEEE)","issue":"3","license":[{"start":{"date-parts":[[2025,5,1]],"date-time":"2025-05-01T00:00:00Z","timestamp":1746057600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/ieeexplore.ieee.org\/Xplorehelp\/downloads\/license-information\/IEEE.html"},{"start":{"date-parts":[[2025,5,1]],"date-time":"2025-05-01T00:00:00Z","timestamp":1746057600000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2025,5,1]],"date-time":"2025-05-01T00:00:00Z","timestamp":1746057600000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"}],"funder":[{"DOI":"10.13039\/100000001","name":"NSF","doi-asserted-by":"publisher","award":["2312546"],"award-info":[{"award-number":["2312546"]}],"id":[{"id":"10.13039\/100000001","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000001","name":"NSF","doi-asserted-by":"publisher","award":["2126052"],"award-info":[{"award-number":["2126052"]}],"id":[{"id":"10.13039\/100000001","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000001","name":"NSF","doi-asserted-by":"publisher","award":["2220292"],"award-info":[{"award-number":["2220292"]}],"id":[{"id":"10.13039\/100000001","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000001","name":"NSF","doi-asserted-by":"publisher","award":["2312547"],"award-info":[{"award-number":["2312547"]}],"id":[{"id":"10.13039\/100000001","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000001","name":"NSF","doi-asserted-by":"publisher","award":["2212318"],"award-info":[{"award-number":["2212318"]}],"id":[{"id":"10.13039\/100000001","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000001","name":"NSF","doi-asserted-by":"publisher","award":["2332547"],"award-info":[{"award-number":["2332547"]}],"id":[{"id":"10.13039\/100000001","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["IEEE Signal Process. Mag."],"published-print":{"date-parts":[[2025,5]]},"DOI":"10.1109\/msp.2025.3572636","type":"journal-article","created":{"date-parts":[[2025,9,15]],"date-time":"2025-09-15T17:39:40Z","timestamp":1757957980000},"page":"84-106","source":"Crossref","is-referenced-by-count":7,"title":["Learning From Crowdsourced Noisy Labels: A signal processing perspective"],"prefix":"10.1109","volume":"42","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-1951-5234","authenticated-orcid":false,"given":"Shahana","family":"Ibrahim","sequence":"first","affiliation":[{"name":"University of Central Florida, Orlando, FL, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-5420-758X","authenticated-orcid":false,"given":"Panagiotis A.","family":"Traganitis","sequence":"additional","affiliation":[{"name":"Michigan State University, East Lansing, MI, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-4847-9586","authenticated-orcid":false,"given":"Xiao","family":"Fu","sequence":"additional","affiliation":[{"name":"Oregon State University, Corvallis, OR, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-0196-0260","authenticated-orcid":false,"given":"Georgios B.","family":"Giannakis","sequence":"additional","affiliation":[{"name":"University of Minnesota, Minneapolis, MN, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"263","reference":[{"key":"ref1-sidebar2","first-page":"297","article-title":"Decentralized detection","volume-title":"Advanced Statistical Signal Processing","volume":"2","author":"Tsitsiklis","year":"1993"},{"key":"ref2-sidebar2","doi-asserted-by":"publisher","DOI":"10.1109\/5.720247"},{"key":"ref3-sidebar2","doi-asserted-by":"publisher","DOI":"10.1109\/18.490552"},{"key":"ref4-sidebar2","doi-asserted-by":"publisher","DOI":"10.1109\/icassp.2018.8462493"},{"key":"ref1-sidebar4","first-page":"53,728","article-title":"Direct preference optimization: Your language model is secretly a reward model","volume-title":"Proc. 37th Int. Conf. Neural Inf. Process. Syst. (NIPS)","volume":"36","author":"Rafailov","year":"2023"},{"key":"ref2-sidebar4","article-title":"Provably robust DPO: Aligning language models with noisy feedback","author":"Chowdhury","year":"2024"},{"key":"ref3-sidebar4","article-title":"Aligning crowd feedback via distributional preference reward modeling","author":"Li","year":"2024"},{"key":"ref4-sidebar4","article-title":"MaxMin-RLHF: Alignment with diverse human preferences","author":"Chakraborty","year":"2024"},{"key":"ref1","volume-title":"Data collection and labeling market size, share & trends analysis report","year":"2022"},{"key":"ref2","article-title":"Beyond AMT: An analysis of crowd work platforms","author":"Vakharia","year":"2013"},{"key":"ref3","article-title":"A survey of large language models","author":"Zhao","year":"2023"},{"key":"ref4","doi-asserted-by":"publisher","DOI":"10.1111\/j.1365-2966.2008.14252.x"},{"key":"ref5","doi-asserted-by":"publisher","DOI":"10.1016\/j.biocon.2009.05.006"},{"key":"ref6","doi-asserted-by":"publisher","DOI":"10.1016\/j.neuroimage.2012.08.075"},{"key":"ref7","doi-asserted-by":"publisher","DOI":"10.1016\/j.chb.2018.10.028"},{"key":"ref8","doi-asserted-by":"publisher","DOI":"10.1109\/jproc.2015.2460697"},{"issue":"165","key":"ref9","first-page":"1","article-title":"Double or nothing: Multiplicative incentive mechanisms for crowdsourcing","volume":"17","author":"Shah","year":"2016","journal-title":"J. Mach. Learn. Res."},{"key":"ref10","doi-asserted-by":"publisher","DOI":"10.2307\/2346806"},{"key":"ref11","doi-asserted-by":"publisher","DOI":"10.14778\/3137765.3137827"},{"key":"ref12","doi-asserted-by":"publisher","DOI":"10.1109\/tsp.2018.2860562"},{"key":"ref13","first-page":"1260","article-title":"Spectral methods meet EM: A provably optimal algorithm for crowdsourcing","volume-title":"Proc. Adv. Neural Inform. Process. Syst.","author":"Zhang","year":"2014"},{"key":"ref14","first-page":"7847","article-title":"Crowdsourcing via pairwise co-occurrences: Identifiability and algorithms","volume-title":"Proc. 33rd Int. Conf. Neural Inf. Process. Syst.","author":"Ibrahim","year":"2019"},{"key":"ref15","article-title":"Deep learning from crowdsourced labels: Coupled cross-entropy minimization, identifiability, and regularization","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Ibrahim","year":"2023"},{"key":"ref16","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v32i1.11506"},{"key":"ref17","doi-asserted-by":"publisher","DOI":"10.1109\/cvpr.2019.01150"},{"key":"ref18","first-page":"1297","article-title":"Learning from crowds","volume":"11","author":"Raykar","year":"2010","journal-title":"J. Mach. Learn. Res."},{"key":"ref19","doi-asserted-by":"publisher","DOI":"10.1287\/opre.2013.1235"},{"key":"ref20","doi-asserted-by":"publisher","DOI":"10.1006\/inco.1994.1009"},{"key":"ref21","doi-asserted-by":"publisher","DOI":"10.1109\/icassp40776.2020.9053945"},{"key":"ref22","first-page":"1345","article-title":"Multi-armed bandit algorithms for crowdsourcing systems with online estimation of workers ability","volume-title":"Proc. Int. Conf. Auton. Agents MultiAgent Syst.","author":"Rangi","year":"2018"},{"key":"ref23","doi-asserted-by":"publisher","DOI":"10.1145\/2487575.2487593"},{"key":"ref24","doi-asserted-by":"publisher","DOI":"10.1109\/jstsp.2014.2316116"},{"key":"ref25","doi-asserted-by":"publisher","DOI":"10.24963\/ijcai.2020\/402"},{"key":"ref26","doi-asserted-by":"publisher","DOI":"10.1145\/1993574.1993599"},{"key":"ref27","doi-asserted-by":"publisher","DOI":"10.1109\/allerton.2011.6120180"},{"key":"ref28","first-page":"2195","article-title":"Learning from the wisdom of crowds by minimax entropy","volume-title":"Proc. Adv. Neural Inform. Process. Syst.","volume":"25","author":"Zhou","year":"2012"},{"key":"ref29","doi-asserted-by":"publisher","DOI":"10.1145\/2488388.2488414"},{"key":"ref30","first-page":"262","article-title":"Aggregating ordinal labels from crowds by minimax conditional entropy","volume-title":"Proc. 31st Int. Conf. Mach. Learn. (ICML)","author":"Zhou","year":"2014"},{"key":"ref31","first-page":"2035","article-title":"Whose vote should count more: Optimal integration of labels from labelers of unknown expertise","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","volume":"22","author":"Whitehill","year":"2009"},{"key":"ref32","first-page":"619","article-title":"Bayesian classifier combination","volume-title":"Proc. 15th Int. Conf. Artif. Intell. Statist.","author":"Kim","year":"2012"},{"key":"ref33","doi-asserted-by":"publisher","DOI":"10.1109\/tsp.2021.3090960"},{"key":"ref34","first-page":"407","article-title":"Estimating the accuracies of multiple classifiers without labeled data","volume-title":"Proc. 18th Int. Conf. Artif. Intell. Statist. (AISTATS)","author":"Jaffe","year":"2015"},{"key":"ref35","first-page":"4544","article-title":"Crowdsourcing via annotator co-occurrence imputation and provable symmetric nonnegative matrix factorization","volume-title":"Proc. 38th Int. Conf. Mach. Learn.","author":"Ibrahim","year":"2021"},{"key":"ref36","first-page":"692","article-title":"Variational inference for crowdsourcing","volume":"25","author":"Liu","year":"2012","journal-title":"Proc. Adv. Neural Inf. Process. Syst."},{"key":"ref37","doi-asserted-by":"publisher","DOI":"10.1145\/2588555.2610509"},{"key":"ref38","first-page":"1621","article-title":"Max-margin majority voting for learning from crowds","volume":"28","author":"Tian","year":"2015","journal-title":"Proc. Adv. Neural Inform. Process. Syst."},{"key":"ref39","doi-asserted-by":"publisher","DOI":"10.1109\/tnnls.2021.3082496"},{"key":"ref40","doi-asserted-by":"publisher","DOI":"10.1109\/tkde.2020.3046645"},{"key":"ref41","doi-asserted-by":"publisher","DOI":"10.1109\/tit.1967.1054010"},{"key":"ref42","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D19-1101"},{"key":"ref43","first-page":"351","article-title":"Unsupervised ensemble learning with dependent classifiers","volume-title":"Proc. 19th Int. Conf. Artif. Intell. Statist.","author":"Jaffe","year":"2016"},{"key":"ref44","doi-asserted-by":"publisher","DOI":"10.1109\/ieeeconf56349.2022.10052052"},{"key":"ref45","first-page":"603","article-title":"Exact exponent in optimal rates for crowdsourcing","volume":"48","author":"Gao","year":"2016","journal-title":"Proc. 33rd Int. Conf. Int. Conf. Mach. Learn. (ICML)"},{"key":"ref46","doi-asserted-by":"publisher","DOI":"10.1109\/tsp.2018.2862383"},{"key":"ref47","doi-asserted-by":"publisher","DOI":"10.1109\/tsp.2017.2690524"},{"key":"ref48","doi-asserted-by":"publisher","DOI":"10.1109\/msp.2018.2877582"},{"key":"ref49","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-86523-8_33"},{"key":"ref50","doi-asserted-by":"publisher","DOI":"10.1007\/s11263-007-0090-8"},{"key":"ref51","first-page":"433","article-title":"Gaussian process classification and active learning with multiple annotators","volume":"32","author":"Rodrigues","year":"2014","journal-title":"Proc. 31st Int. Conf. Int. Conf. Mach. Learn. (ICML)"},{"key":"ref52","doi-asserted-by":"publisher","DOI":"10.1007\/s10994-013-5411-2"},{"key":"ref53","first-page":"7829","article-title":"Max-MIG: An information theoretic approach for joint learning from crowds","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Cao","year":"2019"},{"key":"ref54","first-page":"15,750","article-title":"Disentangling human error from the ground truth in segmentation of medical images","volume":"33","author":"Zhang","year":"2020","journal-title":"Proc. 34th Int. Conf. Neural Inf. Process. Syst. (NIPS)"},{"key":"ref55","first-page":"347","article-title":"Label correction of crowdsourced noisy annotations with an instance-dependent noise transition model","volume":"36","author":"Guo","year":"2023","journal-title":"Proc. 37th Int. Conf. Neural Inf. Process. Syst. (NIPS)"},{"key":"ref56","first-page":"97,261","article-title":"Noisy label learning with instance-dependent outliers: Identifiability via crowd wisdom","volume":"37","author":"Nguyen","year":"2024","journal-title":"Proc. Adv. Neural Inf. Process. Syst."},{"key":"ref57","article-title":"Learning with noisy labels revisited: A study using real-world human annotations","volume-title":"Proc. Int. Conf. Learn. Representations (ICLR)","author":"Wei","year":"2022"},{"key":"ref58","first-page":"1486","article-title":"Iterative Bayesian learning for crowdsourced regression","volume":"89","author":"Ok","year":"2019","journal-title":"Proc. 22nd Int. Conf. Artif. Intell. Statist. (AISTATS)"},{"key":"ref59","first-page":"5225","article-title":"Crowdsourcing regression: A spectral approach","volume-title":"Proc. Int. Conf. Artif. Intell. Statist.","author":"Tenzer","year":"2022"},{"key":"ref60","doi-asserted-by":"publisher","DOI":"10.1145\/2433396.2433420"},{"key":"ref61","first-page":"25,980","article-title":"Deep clustering with incomplete noisy pairwise annotations: A geometric regularization approach","volume-title":"Proc. 40th Int. Conf. Mach. Learn. (ICML)","author":"Nguyen","year":"2023"},{"key":"ref62","article-title":"Fairness and bias in truth discovery algorithms: An experimental analysis","author":"Lazier","year":"2023"},{"key":"ref63","doi-asserted-by":"publisher","DOI":"10.1145\/3306618.3314282"},{"issue":"16","key":"ref64","first-page":"491","article-title":"Eliminating spammers and ranking annotators for crowdsourced labeling tasks","volume":"13","author":"Raykar","year":"2012","journal-title":"J. Mach. Learn. Res."},{"key":"ref65","first-page":"21,841","article-title":"Adversarial crowdsourcing through robust rank-one matrix completion","volume":"33","author":"Ma","year":"2020","journal-title":"Proc. 34th Conf. Neural Inf. Process. Syst."},{"key":"ref66","doi-asserted-by":"publisher","DOI":"10.1109\/icdm51629.2021.00174"},{"key":"ref67","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v36i5.20467"},{"key":"ref68","article-title":"A survey of reinforcement learning from human feedback","author":"Kaufmann","year":"2023"},{"key":"ref69","article-title":"Crowd-prefRL: Preference-based reward learning from crowds","author":"Chhan","year":"2024"}],"container-title":["IEEE Signal Processing Magazine"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx8\/79\/11164540\/11164541.pdf?arnumber=11164541","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,9,16]],"date-time":"2025-09-16T17:36:07Z","timestamp":1758044167000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/11164541\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,5]]},"references-count":77,"journal-issue":{"issue":"3"},"URL":"https:\/\/doi.org\/10.1109\/msp.2025.3572636","relation":{},"ISSN":["1053-5888","1558-0792"],"issn-type":[{"value":"1053-5888","type":"print"},{"value":"1558-0792","type":"electronic"}],"subject":[],"published":{"date-parts":[[2025,5]]}}}