{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,2,21]],"date-time":"2025-02-21T09:54:30Z","timestamp":1740131670590,"version":"3.37.3"},"reference-count":50,"publisher":"Institute of Electrical and Electronics Engineers (IEEE)","issue":"6","license":[{"start":{"date-parts":[[2021,6,1]],"date-time":"2021-06-01T00:00:00Z","timestamp":1622505600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/ieeexplore.ieee.org\/Xplorehelp\/downloads\/license-information\/IEEE.html"},{"start":{"date-parts":[[2021,6,1]],"date-time":"2021-06-01T00:00:00Z","timestamp":1622505600000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2021,6,1]],"date-time":"2021-06-01T00:00:00Z","timestamp":1622505600000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"}],"funder":[{"DOI":"10.13039\/100000006","name":"Office of Naval Research MURI","doi-asserted-by":"publisher","award":["DOD-002888"],"award-info":[{"award-number":["DOD-002888"]}],"id":[{"id":"10.13039\/100000006","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000181","name":"Air Force Office of Scientific Research","doi-asserted-by":"publisher","award":["AFOSR-FA9550-14-1-001"],"award-info":[{"award-number":["AFOSR-FA9550-14-1-001"]}],"id":[{"id":"10.13039\/100000181","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000006","name":"Office of Naval Research","doi-asserted-by":"publisher","award":["ONR-N00014"],"award-info":[{"award-number":["ONR-N00014"]}],"id":[{"id":"10.13039\/100000006","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000001","name":"National Science Foundation","doi-asserted-by":"publisher","award":["CIF: 31712-23800","CRII: CIF: 1755656","CIF: 1763734","NSF CAREER 1942124"],"award-info":[{"award-number":["CIF: 31712-23800","CRII: CIF: 1755656","CIF: 1763734","NSF CAREER 1942124"]}],"id":[{"id":"10.13039\/100000001","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["IEEE Trans. Inform. Theory"],"published-print":{"date-parts":[[2021,6]]},"DOI":"10.1109\/tit.2020.3045613","type":"journal-article","created":{"date-parts":[[2020,12,17]],"date-time":"2020-12-17T21:14:50Z","timestamp":1608239690000},"page":"4162-4184","source":"Crossref","is-referenced-by-count":9,"title":["A Permutation-Based Model for Crowd Labeling: Optimal Estimation and Robustness"],"prefix":"10.1109","volume":"67","author":[{"ORCID":"https:\/\/orcid.org\/0000-0001-5158-9677","authenticated-orcid":false,"given":"Nihar B.","family":"Shah","sequence":"first","affiliation":[]},{"given":"Sivaraman","family":"Balakrishnan","sequence":"additional","affiliation":[]},{"given":"Martin J.","family":"Wainwright","sequence":"additional","affiliation":[]}],"member":"263","reference":[{"key":"ref39","doi-asserted-by":"publisher","DOI":"10.1214\/18-AOS1772"},{"key":"ref38","first-page":"7246","article-title":"Simple, robust and optimal ranking from pairwise comparisons","volume":"18","author":"shah","year":"2018","journal-title":"J Mach Learn Res"},{"key":"ref33","first-page":"38","article-title":"On a modification of chebyshev’s inequality and of the error formula of Laplace","volume":"1","author":"bernstein","year":"1924","journal-title":"Ann Sci Inst Sav Ukraine Sec Math"},{"journal-title":"Elements of Information Theory","year":"2012","author":"cover","key":"ref32"},{"key":"ref31","first-page":"739","article-title":"Estimate of the number of signals in error correcting codes","volume":"117","author":"varshamov","year":"1957","journal-title":"Dok Akad Nauk SSSR"},{"key":"ref30","doi-asserted-by":"publisher","DOI":"10.1002\/j.1538-7305.1952.tb01393.x"},{"key":"ref37","first-page":"1","article-title":"Low permutation-rank matrices: Structural properties and noisy completion","volume":"20","author":"shah","year":"2019","journal-title":"J Mach Learn Res"},{"key":"ref36","doi-asserted-by":"publisher","DOI":"10.3150\/17-BEJ1000"},{"key":"ref35","doi-asserted-by":"publisher","DOI":"10.1214\/15-AOP1025"},{"key":"ref34","doi-asserted-by":"publisher","DOI":"10.1137\/1032121"},{"key":"ref28","doi-asserted-by":"publisher","DOI":"10.1109\/TPAMI.2005.151"},{"key":"ref27","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2009.5206848"},{"key":"ref29","doi-asserted-by":"publisher","DOI":"10.1214\/009117905000000044"},{"key":"ref2","first-page":"1297","article-title":"Learning from crowds","volume":"11","author":"raykar","year":"2010","journal-title":"J Mach Learn Res"},{"key":"ref1","doi-asserted-by":"publisher","DOI":"10.1145\/1401890.1401965"},{"key":"ref20","first-page":"2035","article-title":"Whose vote should count more: Optimal integration of labels from labelers of unknown expertise","author":"whitehill","year":"2009","journal-title":"Proc Adv Neural Inf Process Syst"},{"key":"ref22","doi-asserted-by":"publisher","DOI":"10.1109\/TIT.2019.2903249"},{"key":"ref21","doi-asserted-by":"publisher","DOI":"10.1109\/TIT.2016.2634418"},{"key":"ref24","first-page":"407","article-title":"Estimating the accuracies of multiple classifiers without labeled data","author":"jaffe","year":"2015","journal-title":"Proc Artif Intell Statist"},{"key":"ref23","doi-asserted-by":"publisher","DOI":"10.1073\/pnas.1219097111"},{"key":"ref26","first-page":"1","article-title":"Novel dataset for fine-grained image categorization","author":"khosla","year":"2011","journal-title":"Proc 1st Workshop Fine-Grained Vis Categorization IEEE Conf Comput Vis Pattern Recognit"},{"key":"ref25","first-page":"1","article-title":"Pap-smear benchmark data for pattern classification","author":"jantzen","year":"2005","journal-title":"Proc Nature Inspired Smart Inf Syst (NiSIS)"},{"journal-title":"The Probabilistic Method","year":"2001","author":"matou\u0161ek","key":"ref50"},{"key":"ref10","first-page":"603","article-title":"Exact exponent in optimal rates for crowdsourcing","author":"gao","year":"2016","journal-title":"Proc Int Conf Mach Learn (ICML)"},{"key":"ref11","doi-asserted-by":"publisher","DOI":"10.2307\/2346806"},{"key":"ref40","first-page":"2037","article-title":"Breaking the 1\/$\\sqrt{n}$ barrier: Faster rates for permutation-based models in polynomial time","author":"mao","year":"2018","journal-title":"Proc Conf Learn Theory"},{"key":"ref12","first-page":"11","article-title":"How crowdsourcable is your task","author":"eickhoff","year":"2011","journal-title":"Proc Crowdsourcing Search Data Mining"},{"key":"ref13","first-page":"5725","article-title":"Double or nothing: Multiplicative incentive mechanisms for crowdsourcing","volume":"17","author":"shah","year":"2016","journal-title":"J Mach Learn Res"},{"key":"ref14","doi-asserted-by":"publisher","DOI":"10.1109\/PASSAT\/SocialCom.2011.203"},{"key":"ref15","doi-asserted-by":"publisher","DOI":"10.1145\/2702123.2702443"},{"key":"ref16","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-24258-3_8"},{"key":"ref17","first-page":"4844","article-title":"Achieving budget-optimality with adaptive schemes in crowdsourcing","volume":"29","author":"khetan","year":"2016","journal-title":"Proc Adv Neural Inf Process Syst"},{"key":"ref18","first-page":"2204","article-title":"Learning from the wisdom of crowds by minimax entropy","author":"zhou","year":"2012","journal-title":"Proc Adv Neural Inf Process Syst"},{"key":"ref19","article-title":"Regularized minimax conditional entropy for crowdsourcing","author":"zhou","year":"2015","journal-title":"arXiv 1503 07240"},{"key":"ref4","doi-asserted-by":"publisher","DOI":"10.1109\/Allerton.2011.6120180"},{"key":"ref3","first-page":"1953","article-title":"Iterative learning for reliable crowdsourcing systems","author":"karger","year":"2011","journal-title":"Proc Adv Neural Inf Process Syst"},{"key":"ref6","first-page":"692","article-title":"Variational inference for crowdsourcing","author":"liu","year":"2012","journal-title":"Proc Adv Neural Inf Process Syst"},{"key":"ref5","doi-asserted-by":"crossref","first-page":"167","DOI":"10.1145\/1993574.1993599","article-title":"Who moderates the moderators? Crowdsourcing abuse detection in user-generated content","author":"ghosh","year":"2011","journal-title":"Proc ACM Conf Electron Commerce"},{"key":"ref8","doi-asserted-by":"publisher","DOI":"10.1145\/2488388.2488414"},{"key":"ref7","article-title":"Minimax optimal convergence rates for estimating ground truth from crowdsourced labels","author":"gao","year":"2013","journal-title":"arXiv 1310 5764"},{"key":"ref49","doi-asserted-by":"publisher","DOI":"10.1090\/S0002-9947-1943-0009262-5"},{"key":"ref9","first-page":"3537","article-title":"Spectral methods meet EM: A provably optimal algorithm for crowdsourcing","volume":"17","author":"zhang","year":"2016","journal-title":"J Mach Learn Res"},{"key":"ref46","article-title":"Loss functions, axioms, and peer review","author":"noothigattu","year":"2020","journal-title":"Proc ICML Workshop Incentives Mach Learn"},{"key":"ref45","first-page":"864","article-title":"Your 2 is my 1, your 3 is my 9: Handling arbitrary miscalibrations in ratings","author":"wang","year":"2019","journal-title":"Proc AAMAS"},{"key":"ref48","article-title":"Debiasing evaluations that are biased by evaluations","author":"wang","year":"2021","journal-title":"Proc AAAI"},{"key":"ref47","first-page":"828","article-title":"PeerReview4All: Fair and accurate reviewer assignment in peer review","author":"stelmakh","year":"2019","journal-title":"Proceedings Algorithmic Learning Theory (ALT)"},{"key":"ref42","first-page":"1331","article-title":"A nonasymptotic law of iterated logarithm for general m-estimators","author":"dalalyan","year":"2020","journal-title":"Proc Int Conf Artif Intell Statist"},{"key":"ref41","first-page":"2780","article-title":"Better algorithms for estimating non-parametric models in crowd-sourcing and rank aggregation","author":"liu","year":"2020","journal-title":"Proc Conf Learn Theory"},{"key":"ref44","first-page":"5286","article-title":"On testing for biases in peer review","author":"stelmakh","year":"2019","journal-title":"Proc NIPS"},{"key":"ref43","article-title":"Global empirical risk minimizers with ‘shape constraints’ are rate optimal in general dimensions","author":"han","year":"2019","journal-title":"arXiv 1905 12823"}],"container-title":["IEEE Transactions on Information Theory"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx7\/18\/9437270\/09296844.pdf?arnumber=9296844","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2023,10,15]],"date-time":"2023-10-15T14:53:40Z","timestamp":1697381620000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/9296844\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2021,6]]},"references-count":50,"journal-issue":{"issue":"6"},"URL":"https:\/\/doi.org\/10.1109\/tit.2020.3045613","relation":{},"ISSN":["0018-9448","1557-9654"],"issn-type":[{"type":"print","value":"0018-9448"},{"type":"electronic","value":"1557-9654"}],"subject":[],"published":{"date-parts":[[2021,6]]}}}