{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,5,6]],"date-time":"2026-05-06T15:07:23Z","timestamp":1778080043951,"version":"3.51.4"},"publisher-location":"New York, NY, USA","reference-count":40,"publisher":"ACM","license":[{"start":{"date-parts":[[2023,7,18]],"date-time":"2023-07-18T00:00:00Z","timestamp":1689638400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"name":"the National Key Research and Development Program of China","award":["2021YFF0901005"],"award-info":[{"award-number":["2021YFF0901005"]}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2023,7,19]]},"DOI":"10.1145\/3539618.3592007","type":"proceedings-article","created":{"date-parts":[[2023,7,19]],"date-time":"2023-07-19T00:22:23Z","timestamp":1689726143000},"page":"2103-2107","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":5,"title":["Learning from Crowds with Annotation Reliability"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-2767-4588","authenticated-orcid":false,"given":"Zhi","family":"Cao","sequence":"first","affiliation":[{"name":"Anhui Province Key Laboratory of Big Data Analysis and Application, School of Computer Science and Technology, University of Science and Technology of China & State Key Laboratory of Cognitive Intelligence, Hefei, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-4835-4102","authenticated-orcid":false,"given":"Enhong","family":"Chen","sequence":"additional","affiliation":[{"name":"Anhui Province Key Laboratory of Big Data Analysis and Application, School of Computer Science and Technology, University of Science and Technology of China & State Key Laboratory of Cognitive Intelligence, Hefei, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-7607-9443","authenticated-orcid":false,"given":"Ye","family":"Huang","sequence":"additional","affiliation":[{"name":"Anhui Province Key Laboratory of Big Data Analysis and Application, School of Computer Science and Technology, University of Science and Technology of China & State Key Laboratory of Cognitive Intelligence, Hefei, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-3905-9352","authenticated-orcid":false,"given":"Shuanghong","family":"Shen","sequence":"additional","affiliation":[{"name":"Anhui Province Key Laboratory of Big Data Analysis and Application, School of Computer Science and Technology, University of Science and Technology of China & State Key Laboratory of Cognitive Intelligence, Hefei, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-1661-0420","authenticated-orcid":false,"given":"Zhenya","family":"Huang","sequence":"additional","affiliation":[{"name":"Anhui Province Key Laboratory of Big Data Analysis and Application, School of Computer Science and Technology, University of Science and Technology of China & State Key Laboratory of Cognitive Intelligence, Hefei, China"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2023,7,18]]},"reference":[{"key":"e_1_3_2_2_1_1","doi-asserted-by":"publisher","DOI":"10.1109\/TMI.2016.2528120"},{"key":"e_1_3_2_2_2_1","volume-title":"Proceedings of the 29th International Coference on International Conference on Machine Learning. Omnipress","author":"Bachrach Yoram","year":"2012","unstructured":"Yoram Bachrach, Thore Graepel, Thomas P. Minka, and Jo W Guiver. 2012. How to Grade a Test without Knowing the Answers: A Bayesian Graphical Model for Adaptive Crowdsourcing and Aptitude Testing. In Proceedings of the 29th International Coference on International Conference on Machine Learning. Omnipress, Madison, WI, USA, 819--826."},{"key":"e_1_3_2_2_3_1","volume-title":"International Conference on Learning Representations.","author":"Cao Peng","year":"2019","unstructured":"Peng Cao, Yilun Xu, Yuqing Kong, and Yizhou Wang. 2019. Max-MIG: an Information Theoretic Approach for Joint Learning from Crowds. In International Conference on Learning Representations."},{"key":"e_1_3_2_2_4_1","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v36i5.20467"},{"key":"e_1_3_2_2_5_1","volume-title":"Proceedings of the Twenty-Ninth International Joint Conference on Artificial Intelligence. International Joint Conferences on Artificial Intelligence Organization, 1512--1518","author":"Chen Zhijun","year":"2021","unstructured":"Zhijun Chen, Huimin Wang, Hailong Sun, Pengpeng Chen, Tao Han, Xudong Liu, and Jie Yang. 2021. Structured Probabilistic End-to-End Learning from Crowds. In Proceedings of the Twenty-Ninth International Joint Conference on Artificial Intelligence. International Joint Conferences on Artificial Intelligence Organization, 1512--1518."},{"key":"e_1_3_2_2_6_1","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v35i7.16730"},{"key":"e_1_3_2_2_7_1","doi-asserted-by":"publisher","DOI":"10.1145\/3447548.3467409"},{"key":"e_1_3_2_2_8_1","first-page":"1","article-title":"Maximum Likelihood Estimation of Observer Error-Rates Using the EM Algorithm","volume":"28","author":"Philip Dawid A.","year":"1979","unstructured":"A. Philip Dawid and Allan Skene. 1979. Maximum Likelihood Estimation of Observer Error-Rates Using the EM Algorithm. Journal of the Royal Statistical Society: Series C (Applied Statistics), Vol. 28, 1 (March 1979), 20--28.","journal-title":"Journal of the Royal Statistical Society: Series C (Applied Statistics)"},{"key":"e_1_3_2_2_9_1","doi-asserted-by":"publisher","DOI":"10.1111\/j.1745-3984.2008.00069.x"},{"key":"e_1_3_2_2_10_1","doi-asserted-by":"publisher","DOI":"10.1145\/2187836.2187900"},{"key":"e_1_3_2_2_11_1","volume-title":"Who Said What: Modeling Individual Labelers Improves Classification. Proceedings of the AAAI Conference on Artificial Intelligence","volume":"32","author":"Guan Melody Y.","year":"2018","unstructured":"Melody Y. Guan, Varun Gulshan, Andrew M. Dai, and Geoffrey E. Hinton. 2018. Who Said What: Modeling Individual Labelers Improves Classification. Proceedings of the AAAI Conference on Artificial Intelligence, Vol. 32, 1 (April 2018)."},{"key":"e_1_3_2_2_12_1","doi-asserted-by":"publisher","DOI":"10.1145\/3447548.3467347"},{"key":"e_1_3_2_2_13_1","volume-title":"The Eleventh International Conference on Learning Representations.","author":"Ibrahim Shahana","year":"2023","unstructured":"Shahana Ibrahim, Tri Nguyen, and Xiao Fu. 2023. Deep Learning From Crowdsourced Labels: Coupled Cross-Entropy Minimization, Identifiability, and Regularization. In The Eleventh International Conference on Learning Representations."},{"key":"e_1_3_2_2_14_1","volume-title":"Zachary Chase Lipton, and Anima Anandkumar","author":"Khetan Ashish","year":"2017","unstructured":"Ashish Khetan, Zachary Chase Lipton, and Anima Anandkumar. 2017. Learning From Noisy Singly-labeled Data. ArXiv, Vol. abs\/1712.04577 (2017)."},{"key":"e_1_3_2_2_15_1","volume-title":"Kingma and Jimmy Ba","author":"Diederik","year":"2014","unstructured":"Diederik P. Kingma and Jimmy Ba. 2014. Adam: A Method for Stochastic Optimization. arXiv preprint arXiv:1412.6980 (December 2014)."},{"key":"e_1_3_2_2_16_1","volume-title":"Handbook of Systemic Autoimmune Diseases","volume":"1","author":"Krizhevsky Alex","year":"2009","unstructured":"Alex Krizhevsky. 2009. Learning Multiple Layers of Features from Tiny Images. Handbook of Systemic Autoimmune Diseases, Vol. 1, 4 (2009)."},{"key":"e_1_3_2_2_17_1","doi-asserted-by":"publisher","DOI":"10.1145\/3209978.3210033"},{"key":"e_1_3_2_2_18_1","volume-title":"Error Rate Bounds and Iterative Weighted Majority Voting for Crowdsourcing. arXiv preprint arXiv:1412.4086 (November","author":"Li Hongwei","year":"2014","unstructured":"Hongwei Li and Bin Yu. 2014. Error Rate Bounds and Iterative Weighted Majority Voting for Crowdsourcing. arXiv preprint arXiv:1412.4086 (November 2014)."},{"key":"e_1_3_2_2_19_1","doi-asserted-by":"publisher","DOI":"10.1145\/3397271.3401239"},{"key":"e_1_3_2_2_20_1","doi-asserted-by":"publisher","DOI":"10.14778\/2735496.2735505"},{"key":"e_1_3_2_2_21_1","doi-asserted-by":"publisher","DOI":"10.1145\/2897350.2897352"},{"key":"e_1_3_2_2_22_1","volume-title":"Learning by Applying: A General Framework for Mathematical Reasoning via Enhancing Explicit Knowledge Learning. arXiv preprint arXiv:2302.05717","author":"Liu Jiayu","year":"2023","unstructured":"Jiayu Liu, Zhenya Huang, Chengxiang Zhai, and Qi Liu. 2023. Learning by Applying: A General Framework for Mathematical Reasoning via Enhancing Explicit Knowledge Learning. arXiv preprint arXiv:2302.05717 (2023)."},{"key":"e_1_3_2_2_23_1","volume-title":"Human Uncertainty Makes Classification More Robust. In 2019 IEEE\/CVF International Conference on Computer Vision (ICCV). 9616--9625","author":"Peterson Joshua C.","year":"2019","unstructured":"Joshua C. Peterson, Ruairidh M. Battleday, Thomas L. Griffiths, and Olga Russakovsky. 2019. Human Uncertainty Makes Classification More Robust. In 2019 IEEE\/CVF International Conference on Computer Vision (ICCV). 9616--9625."},{"key":"e_1_3_2_2_24_1","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v32i1.11506"},{"key":"e_1_3_2_2_25_1","volume-title":"Proceedings of the 31st International Conference on Machine Learning. PMLR","author":"Rodrigues Filipe","year":"2014","unstructured":"Filipe Rodrigues, Francisco Camara Pereira, and Bernardete Ribeiro. 2014. Gaussian Process Classification and Active Learning with Multiple Annotators. In Proceedings of the 31st International Conference on Machine Learning. PMLR, Bejing, China, 433--441."},{"key":"e_1_3_2_2_26_1","doi-asserted-by":"publisher","DOI":"10.1145\/3397271.3401112"},{"key":"e_1_3_2_2_27_1","doi-asserted-by":"publisher","DOI":"10.1007\/s11263-007-0090-8"},{"key":"e_1_3_2_2_28_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICDM51629.2021.00065"},{"key":"e_1_3_2_2_29_1","volume-title":"Proceedings of the Conference on Empirical Methods in Natural Language Processing. Association for Computational Linguistics, USA, 254--263","author":"Snow Rion","unstructured":"Rion Snow, Brendan T. O'Connor, Dan Jurafsky, and A. Ng. 2008. Cheap and Fast -- But is it Good? Evaluating Non-Expert Annotations for Natural Language Tasks. In Proceedings of the Conference on Empirical Methods in Natural Language Processing. Association for Computational Linguistics, USA, 254--263."},{"key":"e_1_3_2_2_30_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2019.01150"},{"key":"e_1_3_2_2_31_1","doi-asserted-by":"publisher","DOI":"10.1145\/2566486.2567989"},{"key":"e_1_3_2_2_32_1","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v34i04.6080"},{"key":"e_1_3_2_2_33_1","volume-title":"Deep Learning From Multiple Noisy Annotators as A Union","author":"Wei Hongxin","unstructured":"Hongxin Wei, Renchunzi Xie, Lei Feng, Bo Han, and Bo An. 2022. Deep Learning From Multiple Noisy Annotators as A Union. IEEE transactions on neural networks and learning systems, Vol. PP (2022)."},{"key":"e_1_3_2_2_34_1","volume-title":"Advances in Neural Information Processing Systems. Curran Associates","author":"Welinder Peter","unstructured":"Peter Welinder, Steve Branson, Serge J. Belongie, and Pietro Perona. 2010. The Multidimensional Wisdom of Crowds. In Advances in Neural Information Processing Systems. Curran Associates, Inc."},{"key":"e_1_3_2_2_35_1","volume-title":"Proceedings of the 22nd International Conference on Neural Information Processing Systems. Curran Associates Inc.","author":"Whitehill Jacob","year":"2035","unstructured":"Jacob Whitehill, Paul Ruvolo, Tingfan Wu, Jacob Bergsma, and Javier R. Movellan. 2009. Whose Vote Should Count More: Optimal Integration of Labels from Labelers of Unknown Expertise. In Proceedings of the 22nd International Conference on Neural Information Processing Systems. Curran Associates Inc., Red Hook, NY, USA, 2035--2043."},{"key":"e_1_3_2_2_36_1","volume-title":"Dy","author":"Yan Yan","year":"2014","unstructured":"Yan Yan, R\u00f3mer Rosales, Glenn Fung, Subramanian Ramanathan, and Jennifer G. Dy. 2014. Learning from multiple annotators with varying expertise. Machine Learning, Vol. 95 (October 2014), 291--327."},{"key":"e_1_3_2_2_37_1","doi-asserted-by":"publisher","DOI":"10.1109\/TKDE.2018.2881185"},{"key":"e_1_3_2_2_38_1","volume-title":"Making the Relation Matters: Relation of Relation Learning Network for Sentence Semantic Matching. In AAAI Conference on Artificial Intelligence.","author":"Zhang Kun","year":"2021","unstructured":"Kun Zhang, Le Wu, Guangyi Lv, Meng Wang, Enhong Chen, and Shulan Ruan. 2021. Making the Relation Matters: Relation of Relation Learning Network for Sentence Semantic Matching. In AAAI Conference on Artificial Intelligence."},{"key":"e_1_3_2_2_39_1","volume-title":"An Ensemble Learning Approach with Gradient Resampling for Class-Imbalance Problems. INFORMS Journal on Computing","author":"Zhao Hongke","year":"2023","unstructured":"Hongke Zhao, Chuang Zhao, Xi Zhang, Nanlin Liu, Hengshu Zhu, Qi Liu, and Hui Xiong. 2023. An Ensemble Learning Approach with Gradient Resampling for Class-Imbalance Problems. INFORMS Journal on Computing (2023)."},{"key":"e_1_3_2_2_40_1","doi-asserted-by":"publisher","DOI":"10.14778\/3055540.3055547"}],"event":{"name":"SIGIR '23: The 46th International ACM SIGIR Conference on Research and Development in Information Retrieval","location":"Taipei Taiwan","acronym":"SIGIR '23","sponsor":["SIGIR ACM Special Interest Group on Information Retrieval"]},"container-title":["Proceedings of the 46th International ACM SIGIR Conference on Research and Development in Information Retrieval"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3539618.3592007","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3539618.3592007","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T16:37:48Z","timestamp":1750178268000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3539618.3592007"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023,7,18]]},"references-count":40,"alternative-id":["10.1145\/3539618.3592007","10.1145\/3539618"],"URL":"https:\/\/doi.org\/10.1145\/3539618.3592007","relation":{},"subject":[],"published":{"date-parts":[[2023,7,18]]},"assertion":[{"value":"2023-07-18","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}