{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,3,13]],"date-time":"2026-03-13T04:56:38Z","timestamp":1773377798161,"version":"3.50.1"},"reference-count":25,"publisher":"IEEE","license":[{"start":{"date-parts":[[2022,6,26]],"date-time":"2022-06-26T00:00:00Z","timestamp":1656201600000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2022,6,26]],"date-time":"2022-06-26T00:00:00Z","timestamp":1656201600000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2022,6,26]]},"DOI":"10.1109\/isit50566.2022.9834340","type":"proceedings-article","created":{"date-parts":[[2022,8,3]],"date-time":"2022-08-03T15:34:22Z","timestamp":1659540862000},"page":"3168-3173","source":"Crossref","is-referenced-by-count":0,"title":["No Rose for MLE: Inadmissibility of MLE for Evaluation Aggregation Under Levels of Expertise"],"prefix":"10.1109","author":[{"given":"Charvi","family":"Rastogi","sequence":"first","affiliation":[{"name":"Carnegie Mellon University,Machine Learning Department"}]},{"given":"Ivan","family":"Stelmakh","sequence":"additional","affiliation":[{"name":"Carnegie Mellon University,Machine Learning Department"}]},{"given":"Nihar","family":"Shah","sequence":"additional","affiliation":[{"name":"Carnegie Mellon University,Machine Learning Department"}]},{"given":"Sivaraman","family":"Balakrishnan","sequence":"additional","affiliation":[{"name":"Carnegie Mellon University,Machine Learning Department"}]}],"member":"263","reference":[{"key":"ref10","first-page":"1953","article-title":"Iterative learning for reliable crowdsourcing systems","author":"karger","year":"2011","journal-title":"Proceedings of the 24th International Conference on Neural Information Processing Systems NIPS’11"},{"key":"ref11","doi-asserted-by":"publisher","DOI":"10.1126\/science.1216549"},{"key":"ref12","doi-asserted-by":"publisher","DOI":"10.2307\/1914288"},{"key":"ref13","first-page":"2554","article-title":"Accurate integration of crowdsourced labels using workers’ self-reported confidence scores","author":"oyama","year":"2013"},{"key":"ref14","first-page":"1297","article-title":"Learning from crowds","volume":"11","author":"raykar","year":"2010","journal-title":"Journal of Machine Learning Research"},{"key":"ref15","article-title":"An overview of challenges, experiments, and computational solutions in peer review","author":"shah","year":"2021","journal-title":"Communications of the ACM (to appear)"},{"key":"ref16","doi-asserted-by":"publisher","DOI":"10.1109\/TIT.2020.3045613"},{"key":"ref17","article-title":"Double or nothing: Multiplicative incentive mechanisms for crowdsourcing","volume":"28","author":"shah","year":"2015","journal-title":"Advances in neural information processing systems"},{"key":"ref18","doi-asserted-by":"publisher","DOI":"10.1145\/1401890.1401965"},{"key":"ref19","first-page":"254","article-title":"Cheap and fast – but is it good? Evaluating non-expert annotations for natural language tasks","author":"snow","year":"2008","journal-title":"Proceedings of the 2008 Conference on Empirical Methods in Natural Language Processing"},{"key":"ref4","first-page":"1631","article-title":"Understanding malicious behavior in crowdsourcing platforms: The case of online surveys","author":"gadiraju","year":"2015","journal-title":"Proceedings of the 33rd Annual ACM Conference on Human Factors in Computing Systems"},{"key":"ref3","article-title":"A. for the Advancement of Artificial Intelligence","year":"0"},{"key":"ref6","doi-asserted-by":"publisher","DOI":"10.1145\/1993574.1993599"},{"key":"ref5","article-title":"Minimax optimal convergence rates for estimating ground truth from crowdsourced labels","author":"gao","year":"2013"},{"key":"ref8","doi-asserted-by":"publisher","DOI":"10.1126\/science.1221403"},{"key":"ref7","doi-asserted-by":"crossref","first-page":"99","DOI":"10.1214\/cbms\/1462297310","article-title":"The new likelihoods and the Neyman-Scott problems","author":"ghosh","year":"1994","journal-title":"Higher Order Asymptotics"},{"key":"ref2","doi-asserted-by":"publisher","DOI":"10.2307\/2346806"},{"key":"ref9","doi-asserted-by":"publisher","DOI":"10.1109\/Allerton.2011.6120180"},{"key":"ref1","doi-asserted-by":"publisher","DOI":"10.1145\/2488388.2488414"},{"key":"ref20","article-title":"Lessons learned from an experiment in crowdsourcing complex citizen engineering tasks with amazon mechanical turk","author":"staffelbach","year":"2014"},{"key":"ref22","doi-asserted-by":"publisher","DOI":"10.1109\/PASSAT\/SocialCom.2011.203"},{"key":"ref21","article-title":"PeerReview4All: Fair and accurate reviewer assignment in peer review","author":"stelmakh","year":"2021","journal-title":"JMLR"},{"key":"ref24","article-title":"Aggregating ordinal labels from crowds by minimax conditional entropy","volume":"2","author":"zhou","year":"2014"},{"key":"ref23","first-page":"1260","article-title":"Spectral methods meet EM: A provably optimal algorithm for crowdsourcing","volume":"27","author":"zhang","year":"2014","journal-title":"Advances in neural information processing systems"},{"key":"ref25","article-title":"Regularized minimax conditional entropy for crowdsourcing","author":"zhou","year":"2015"}],"event":{"name":"2022 IEEE International Symposium on Information Theory (ISIT)","location":"Espoo, Finland","start":{"date-parts":[[2022,6,26]]},"end":{"date-parts":[[2022,7,1]]}},"container-title":["2022 IEEE International Symposium on Information Theory (ISIT)"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx7\/9834325\/9834269\/09834340.pdf?arnumber=9834340","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,3,12]],"date-time":"2026-03-12T20:35:11Z","timestamp":1773347711000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/9834340\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2022,6,26]]},"references-count":25,"URL":"https:\/\/doi.org\/10.1109\/isit50566.2022.9834340","relation":{},"subject":[],"published":{"date-parts":[[2022,6,26]]}}}