{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,6,18]],"date-time":"2025-06-18T04:16:48Z","timestamp":1750220208636,"version":"3.41.0"},"publisher-location":"New York, NY, USA","reference-count":52,"publisher":"ACM","license":[{"start":{"date-parts":[[2022,8,14]],"date-time":"2022-08-14T00:00:00Z","timestamp":1660435200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2022,8,14]]},"DOI":"10.1145\/3534678.3539184","type":"proceedings-article","created":{"date-parts":[[2022,8,12]],"date-time":"2022-08-12T19:06:41Z","timestamp":1660331201000},"page":"3645-3655","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":2,"title":["Crowdsourcing with Contextual Uncertainty"],"prefix":"10.1145","author":[{"given":"Viet-An","family":"Nguyen","sequence":"first","affiliation":[{"name":"Meta, Menlo Park, CA, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Peibei","family":"Shi","sequence":"additional","affiliation":[{"name":"Meta, Menlo Park, CA, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Jagdish","family":"Ramakrishnan","sequence":"additional","affiliation":[{"name":"Meta, Menlo Park, CA, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Narjes","family":"Torabi","sequence":"additional","affiliation":[{"name":"Meta, Menlo Park, CA, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Nimar S.","family":"Arora","sequence":"additional","affiliation":[{"name":"Meta, Menlo Park, CA, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Udi","family":"Weinsberg","sequence":"additional","affiliation":[{"name":"Meta, Menlo Park, CA, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Michael","family":"Tingley","sequence":"additional","affiliation":[{"name":"Meta, Menlo Park, CA, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2022,8,14]]},"reference":[{"key":"e_1_3_2_1_1_1","unstructured":"Michael Betancourt. 2020. Robust Gaussian Process Modeling. https:\/\/betanalpha.github.io\/assets\/case_studies\/gaussian_processes.html"},{"key":"e_1_3_2_1_2_1","volume-title":"ECML PKDD Workshop: Languages for Data Mining and Machine Learning . 108--122","author":"Buitinck Lars","year":"2013","unstructured":"Lars Buitinck, Gilles Louppe, Mathieu Blondel, Fabian Pedregosa, Andreas Mueller, Olivier Grisel, Vlad Niculae, Peter Prettenhofer, Alexandre Gramfort, Jaques Grobler, Robert Layton, Jake VanderPlas, Arnaud Joly, Brian Holt, and Ga\u00eb l Varoquaux. 2013. API design for machine learning software: experiences from the scikit-learn project. In ECML PKDD Workshop: Languages for Data Mining and Machine Learning . 108--122."},{"key":"e_1_3_2_1_3_1","unstructured":"Peng Cao Yilun Xu Yuqing Kong and Yizhou Wang. 2019. Max-MIG: an Information Theoretic Approach for Joint Learning from Crowds. In ICLR ."},{"key":"e_1_3_2_1_4_1","volume-title":"Multilevel Bayesian models of categorical data annotation. Unpublished manuscript","author":"Carpenter Bob","year":"2008","unstructured":"Bob Carpenter. 2008. Multilevel Bayesian models of categorical data annotation. Unpublished manuscript , Vol. 17, 122 (2008), 45--50."},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.18637\/jss.v076.i01"},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v35i7.16730"},{"key":"e_1_3_2_1_7_1","volume-title":"Maximum likelihood estimation of observer error-rates using the EM algorithm. Applied Statistics","author":"Dawid Alexander Philip","year":"1979","unstructured":"Alexander Philip Dawid and Allan M Skene. 1979. Maximum likelihood estimation of observer error-rates using the EM algorithm. Applied Statistics (1979)."},{"key":"e_1_3_2_1_8_1","unstructured":"Facebook. 2016. Introducing FBLearner Flow: Facebook's AI backbone . https:\/\/code.fb.com\/core-data\/introducing-fblearner-flow-facebook-s-ai-backbone\/"},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"crossref","unstructured":"Perry Groot Adriana Birlutiu and Tom Heskes. 2011. Learning from Multiple Annotators with Gaussian Processes. In ICANN. 159--164.","DOI":"10.1007\/978-3-642-21738-8_21"},{"key":"e_1_3_2_1_10_1","volume-title":"Hinton","author":"Guan Melody Y.","year":"2017","unstructured":"Melody Y. Guan, Varun Gulshan, Andrew M. Dai, and Geoffrey E. Hinton. 2017. Who Said What: Modeling Individual Labelers Improves Classification. In AAAI ."},{"key":"e_1_3_2_1_11_1","volume-title":"Weinberger","author":"Guo Chuan","year":"2017","unstructured":"Chuan Guo, Geoff Pleiss, Yu Sun, and Kilian Q. Weinberger. 2017. On Calibration of Modern Neural Networks. In ICML. 1321--1330."},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"crossref","unstructured":"M. Hein M. Andriushchenko and J. Bitterwolf. 2019. Why ReLU Networks Yield High-Confidence Predictions Far Away From the Training Data and How to Mitigate the Problem. In CVPR. 41--50.","DOI":"10.1109\/CVPR.2019.00013"},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10618-013-0306-1"},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.1145\/1837885.1837906"},{"key":"e_1_3_2_1_15_1","volume-title":"Noise: How to Overcome the High, Hidden Cost of Inconsistent Decision Making . https:\/\/hbr.org\/2016\/10\/noise","author":"Kahneman Daniel","year":"2016","unstructured":"Daniel Kahneman, Andrew M. Rosenfield, Linnea Gandhi, and Tom Blaser. 2016. Noise: How to Overcome the High, Hidden Cost of Inconsistent Decision Making . https:\/\/hbr.org\/2016\/10\/noise"},{"key":"e_1_3_2_1_16_1","unstructured":"Hyun-Chul Kim and Zoubin Ghahramani. 2012. Bayesian classifier combination. In AISTATS. 619--627."},{"key":"e_1_3_2_1_17_1","volume-title":"Telmo Silva Filho, and Peter Flach","author":"Kull Meelis","year":"2017","unstructured":"Meelis Kull, Telmo Silva Filho, and Peter Flach. 2017. Beta calibration: a well-founded and easily implemented improvement on logistic calibration for binary classifiers. In AISTATS. 623--631."},{"key":"e_1_3_2_1_18_1","unstructured":"Balaji Lakshminarayanan Alexander Pritzel and Charles Blundell. 2017. Simple and Scalable Predictive Uncertainty Estimation Using Deep Ensembles. In NeurIPS ."},{"key":"e_1_3_2_1_19_1","unstructured":"Yuan Li Benjamin Rubinstein and Trevor Cohn. 2019. Exploiting Worker Correlation for Label Aggregation in Crowdsourcing. In ICML . 3886--3895."},{"key":"e_1_3_2_1_20_1","volume-title":"MLJ","volume":"68","author":"Lin Hsuan-Tien","year":"2007","unstructured":"Hsuan-Tien Lin, Chih-Jen Lin, and Ruby C. Weng. 2007. A Note on Platt's Probabilistic Outputs for Support Vector Machines . MLJ , Vol. 68, 3 (Oct. 2007), 267--276."},{"key":"e_1_3_2_1_21_1","unstructured":"Yu Liu. 2019. Estimating the prevalence of rare events - theory and practice . https:\/\/www.unofficialgoogledatascience.com\/2019\/08\/estimating-prevalence-of-rare-events.html"},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"crossref","unstructured":"C. Long G. Hua and A. Kapoor. 2013. Active Visual Recognition with Expertise Estimation in Crowdsourcing. In ICCV . 3000--3007.","DOI":"10.1109\/ICCV.2013.373"},{"key":"e_1_3_2_1_23_1","unstructured":"P. Morales-Alvarez P. Ruiz S. Coughlin R. Molina Soriano and A. Katsaggelos. 2020. Scalable Variational Gaussian Processes for Crowdsourcing: Glitch Detection in LIGO . TPAMI (2020)."},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.inffus.2018.12.008"},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.1109\/TGRS.2017.2758922"},{"key":"e_1_3_2_1_26_1","volume-title":"Yee Whye Teh, and Fernando Perez-Cruz","author":"Moreno Pablo G.","year":"2015","unstructured":"Pablo G. Moreno, Antonio Art\u00e9s-Rodr'iguez, Yee Whye Teh, and Fernando Perez-Cruz. 2015. Bayesian Nonparametric Crowdsourcing. JMLR (2015), 1607--1627."},{"key":"e_1_3_2_1_27_1","doi-asserted-by":"crossref","unstructured":"Mahdi Pakdaman Naeini Gregory F. Cooper and Milos Hauskrecht. 2015. Obtaining Well Calibrated Probabilities Using Bayesian Binning. In AAAI . 2901--2907.","DOI":"10.1609\/aaai.v29i1.9602"},{"key":"e_1_3_2_1_28_1","unstructured":"An T. Nguyen Byron C. Wallace and Matthew Lease. 2016. A Correlated Worker Model for Grouped Imbalanced and Multitask Data. In UAI . 537--546."},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.1145\/3394486.3403304"},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"publisher","DOI":"10.1162\/tacl_a_00185"},{"key":"e_1_3_2_1_31_1","volume-title":"Comparing Bayesian Models of Annotation. TACL","author":"Paun Silviu","year":"2018","unstructured":"Silviu Paun, Bob Carpenter, Jon Chamberlain, Dirk Hovy, Udo Kruschwitz, and Massimo Poesio. 2018. Comparing Bayesian Models of Annotation. TACL (2018)."},{"key":"e_1_3_2_1_32_1","doi-asserted-by":"crossref","unstructured":"John C. Platt. 1999. Probabilistic Outputs for Support Vector Machines and Comparisons to Regularized Likelihood Methods. In Advances in Large Margin Classifiers . 61--74.","DOI":"10.7551\/mitpress\/1113.003.0008"},{"key":"e_1_3_2_1_33_1","volume-title":"JMLR (2005)","author":"Qui\u00f1onero-Candela Joaquin","year":"2005","unstructured":"Joaquin Qui\u00f1onero-Candela and Carl Edward Rasmussen. 2005. A unifying view of sparse approximate Gaussian process regression. JMLR (2005), 1939--1959."},{"volume-title":"Gaussian Processes in Machine Learning","author":"Rasmussen Carl Edward","key":"e_1_3_2_1_34_1","unstructured":"Carl Edward Rasmussen. 2004. Gaussian Processes in Machine Learning .Springer Berlin Heidelberg, Berlin, Heidelberg, 63--71."},{"key":"e_1_3_2_1_35_1","doi-asserted-by":"publisher","DOI":"10.14778\/3157794.3157797"},{"key":"e_1_3_2_1_36_1","volume-title":"Luca Bogoni, and Linda Moy.","author":"Raykar Vikas C","year":"2009","unstructured":"Vikas C Raykar, Shipeng Yu, Linda H Zhao, Anna Jerebko, Charles Florin, Gerardo Hermosillo Valadez, Luca Bogoni, and Linda Moy. 2009. Supervised learning from multiple experts: whom to trust when everyone lies a bit. In ICML . 889--896."},{"key":"e_1_3_2_1_37_1","volume-title":"JMLR","volume":"11","author":"Raykar Vikas C.","year":"2010","unstructured":"Vikas C. Raykar, Shipeng Yu, Linda H. Zhao, Gerardo Hermosillo Valadez, Charles Florin, Luca Bogoni, and Linda Moy. 2010. Learning From Crowds. JMLR , Vol. 11 (Aug. 2010), 1297--1322."},{"key":"e_1_3_2_1_38_1","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v32i1.11506"},{"key":"e_1_3_2_1_39_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.patrec.2013.05.012"},{"key":"e_1_3_2_1_40_1","unstructured":"Filipe Rodrigues Francisco C. Pereira and Bernardete Ribeiro. 2014. Gaussian Process Classification and Active Learning with Multiple Annotators. In ICML ."},{"key":"e_1_3_2_1_41_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.patcog.2018.11.021"},{"key":"e_1_3_2_1_42_1","doi-asserted-by":"crossref","unstructured":"Matteo Venanzi John Guiver Gabriella Kazai Pushmeet Kohli and Milad Shokouhi. 2014. Community-based Bayesian Aggregation Models for Crowdsourcing. In WWW . 155--164.","DOI":"10.1145\/2566486.2567989"},{"key":"e_1_3_2_1_43_1","unstructured":"Peter Welinder Steve Branson Pietro Perona and Serge J Belongie. 2010. The multidimensional wisdom of crowds. In NeurIPS. 2424--2432."},{"key":"e_1_3_2_1_44_1","unstructured":"Jonathan Wenger Hedvig Kjellstr\u00f6m and Rudolph Triebel. 2020. Non-parametric calibration for classification. In AISTATS. PMLR 178--190."},{"key":"e_1_3_2_1_45_1","volume-title":"Jacob Bergsma, Javier R. Movellan, and Paul L. Ruvolo.","author":"Whitehill Jacob","year":"2009","unstructured":"Jacob Whitehill, Ting fan Wu, Jacob Bergsma, Javier R. Movellan, and Paul L. Ruvolo. 2009. Whose Vote Should Count More: Optimal Integration of Labels from Labelers of Unknown Expertise. In NIPS. 2035--2043."},{"key":"e_1_3_2_1_46_1","unstructured":"Andrew Wilson and Hannes Nickisch. 2015. Kernel interpolation for scalable structured Gaussian processes (KISS-GP). In ICML . 1775--1784."},{"key":"e_1_3_2_1_47_1","doi-asserted-by":"publisher","DOI":"10.1080\/01621459.1927.10502953"},{"key":"e_1_3_2_1_48_1","unstructured":"Yan Yan R\u00f3mer Rosales Glenn Fung Mark Schmidt Gerardo Hermosillo Luca Bogoni Linda Moy and Jennifer Dy. 2010. Modeling annotator expertise: Learning when everybody knows a bit of something. In AISTATS . 932--939."},{"key":"e_1_3_2_1_49_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10994-013-5412-1"},{"key":"e_1_3_2_1_50_1","unstructured":"Bianca Zadrozny and Charles Elkan. 2001. Obtaining Calibrated Probability Estimates from Decision Trees and Naive Bayesian Classifiers. In ICML. 609--616."},{"key":"e_1_3_2_1_51_1","doi-asserted-by":"publisher","DOI":"10.14778\/3055540.3055547"},{"key":"e_1_3_2_1_52_1","volume-title":"Platt","author":"Zhou Dengyong","year":"2012","unstructured":"Dengyong Zhou, Sumit Basu, Yi Mao, and John C. Platt. 2012. Learning from the Wisdom of Crowds by Minimax Entropy. In NeurIPS. 2195--2203."}],"event":{"name":"KDD '22: The 28th ACM SIGKDD Conference on Knowledge Discovery and Data Mining","sponsor":["SIGMOD ACM Special Interest Group on Management of Data","SIGKDD ACM Special Interest Group on Knowledge Discovery in Data"],"location":"Washington DC USA","acronym":"KDD '22"},"container-title":["Proceedings of the 28th ACM SIGKDD Conference on Knowledge Discovery and Data Mining"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3534678.3539184","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3534678.3539184","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T19:02:59Z","timestamp":1750186979000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3534678.3539184"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2022,8,14]]},"references-count":52,"alternative-id":["10.1145\/3534678.3539184","10.1145\/3534678"],"URL":"https:\/\/doi.org\/10.1145\/3534678.3539184","relation":{},"subject":[],"published":{"date-parts":[[2022,8,14]]},"assertion":[{"value":"2022-08-14","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}