{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,6,18]],"date-time":"2025-06-18T04:10:46Z","timestamp":1750219846975,"version":"3.41.0"},"publisher-location":"New York, NY, USA","reference-count":44,"publisher":"ACM","license":[{"start":{"date-parts":[[2023,7,18]],"date-time":"2023-07-18T00:00:00Z","timestamp":1689638400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"name":"Dutch Ministry of Educa- tion Culture and Science"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2023,7,19]]},"DOI":"10.1145\/3539618.3591685","type":"proceedings-article","created":{"date-parts":[[2023,7,19]],"date-time":"2023-07-19T00:22:23Z","timestamp":1689726143000},"page":"729-738","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":0,"title":["Extending Label Aggregation Models with a Gaussian Process to Denoise Crowdsourcing Labels"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0001-6381-1087","authenticated-orcid":false,"given":"Dan","family":"Li","sequence":"first","affiliation":[{"name":"University of Amsterdam & Elsevier, Amsterdam, Netherlands"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-1086-0202","authenticated-orcid":false,"given":"Maarten","family":"de Rijke","sequence":"additional","affiliation":[{"name":"University of Amsterdam, Amsterdam, Netherlands"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2023,7,18]]},"reference":[{"key":"e_1_3_2_1_1_1","volume-title":"Aggnet: deep learning from crowds for mitosis detection in breast cancer histology images","author":"Albarqouni Shadi","year":"2016","unstructured":"Shadi Albarqouni, Christoph Baur, Felix Achilles, Vasileios Belagiannis, Stefanie Demirci, and Nassir Navab. 2016. Aggnet: deep learning from crowds for mitosis detection in breast cancer histology images. IEEE transactions on medical imaging, Vol. 35, 5 (2016), 1313--1321."},{"key":"e_1_3_2_1_2_1","volume-title":"The Perspectivist Data Manifesto. https:\/\/pdai.info\/. [Online","author":"Basile Valerio","year":"2023","unstructured":"Valerio Basile. [n.,d.]. The Perspectivist Data Manifesto. https:\/\/pdai.info\/. [Online; accessed 2-January-2023]."},{"key":"e_1_3_2_1_3_1","volume-title":"International Conference on Learning Representations.","author":"Cao Peng","year":"2019","unstructured":"Peng Cao, Yilun Xu, Yuqing Kong, and Yizhou Wang. 2019. Max-MIG: an Information Theoretic Approach for Joint Learning from Crowds. In International Conference on Learning Representations."},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.1145\/2433396.2433420"},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1145\/3331184.3331303"},{"key":"e_1_3_2_1_6_1","volume-title":"Maximum Likelihood Estimation of Observer Error-rates using the EM Algorithm. Applied statistics","author":"Dawid Alexander Philip","year":"1979","unstructured":"Alexander Philip Dawid and Allan M Skene. 1979. Maximum Likelihood Estimation of Observer Error-rates using the EM Algorithm. Applied statistics (1979), 20--28."},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"publisher","DOI":"10.1145\/2187836.2187900"},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.1145\/3459637.3482032"},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1145\/3336191.3371875"},{"key":"e_1_3_2_1_10_1","unstructured":"Peter A Flach Jos\u00e9 Hern\u00e1ndez-Orallo and C\u00e8sar Ferri Ramirez. 2011. A Coherent Interpretation of AUC as a Measure of Aggregated Classification Performance. In ICML."},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"crossref","unstructured":"Meric Altug Gemalmaz and Ming Yin. 2021. Accounting for Confirmation Bias in Crowdsourced Label Aggregation.. In IJCAI. 1729--1735.","DOI":"10.24963\/ijcai.2021\/238"},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-21738-8_21"},{"key":"e_1_3_2_1_13_1","volume-title":"Advances in Neural Information Processing Systems","volume":"32","author":"Hamelijnck Oliver","year":"2019","unstructured":"Oliver Hamelijnck, Theodoros Damoulas, Kangrui Wang, and Mark Girolami. 2019. Multi-resolution Multi-task Gaussian Processes. Advances in Neural Information Processing Systems, Vol. 32 (2019)."},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.1145\/3336191.3371857"},{"key":"e_1_3_2_1_15_1","volume-title":"Proceedings of the 2013 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies. 1120--1130","author":"Hovy Dirk","year":"2013","unstructured":"Dirk Hovy, Taylor Berg-Kirkpatrick, Ashish Vaswani, and Eduard Hovy. 2013. Learning Whom to Trust with MACE. In Proceedings of the 2013 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies. 1120--1130."},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.1145\/3269206.3271779"},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.14778\/3067421.3067431"},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.artint.2020.103351"},{"key":"e_1_3_2_1_19_1","volume-title":"An Analysis of Human Factors and Label Accuracy in Crowdsourcing Relevance Judgments. Information retrieval","author":"Kazai Gabriella","year":"2013","unstructured":"Gabriella Kazai, Jaap Kamps, and Natasa Milic-Frayling. 2013. An Analysis of Human Factors and Label Accuracy in Crowdsourcing Relevance Judgments. Information retrieval, Vol. 16, 2 (2013), 138--178."},{"key":"e_1_3_2_1_20_1","unstructured":"Hyun-Chul Kim and Zoubin Ghahramani. 2012. Bayesian Classifier Combination. In Artificial Intelligence and Statistics. 619--627."},{"key":"e_1_3_2_1_21_1","volume-title":"Variational Learning on Aggregate Outputs with Gaussian Processes. Advances in neural information processing systems","author":"Law Ho Chung","year":"2018","unstructured":"Ho Chung Law, Dino Sejdinovic, Ewan Cameron, Tim Lucas, Seth Flaxman, Katherine Battle, and Kenji Fukumizu. 2018. Variational Learning on Aggregate Outputs with Gaussian Processes. Advances in neural information processing systems , Vol. 31 (2018)."},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.1145\/3442381.3450047"},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.1007\/s11432-020-3118-7"},{"volume-title":"Probabilistic Models for Aggregating Crowdsourced Annotations. Ph.,D. Dissertation","author":"Yuan Li.","key":"e_1_3_2_1_24_1","unstructured":"Yuan Li. 2019. Probabilistic Models for Aggregating Crowdsourced Annotations. Ph.,D. Dissertation. University of Melbourne, Parkville, Victoria, Australia."},{"key":"e_1_3_2_1_25_1","first-page":"1299","article-title":"GPflow: A Gaussian Process Library Using TensorFlow","volume":"18","author":"Matthews Alexander G.","year":"2017","unstructured":"Alexander G. de G. Matthews, Mark van der Wilk, Tom Nickson, Keisuke Fujii, Alexis Boukouvalas, Pablo Le\u00f3n-Villagr\u00e1, Zoubin Ghahramani, and James Hensman. 2017. GPflow: A Gaussian Process Library Using TensorFlow. The Journal of Machine Learning Research, Vol. 18, 1 (2017), 1299--1304.","journal-title":"The Journal of Machine Learning Research"},{"volume-title":"The EM algorithm and extensions","author":"McLachlan Geoffrey J","key":"e_1_3_2_1_26_1","unstructured":"Geoffrey J McLachlan and Thriyambakam Krishnan. 2007. The EM algorithm and extensions. John Wiley & Sons."},{"key":"e_1_3_2_1_27_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.inffus.2018.12.008"},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"publisher","DOI":"10.1145\/3480965"},{"volume-title":"Learning in graphical models","author":"Neal Radford M","key":"e_1_3_2_1_29_1","unstructured":"Radford M Neal and Geoffrey E Hinton. 1998. A View of the EM Algorithm that Justifies Incremental, Sparse, and Other Variants. In Learning in graphical models. Springer, 355--368."},{"volume-title":"Advanced lectures on machine learning","author":"Rasmussen Carl Edward","key":"e_1_3_2_1_30_1","unstructured":"Carl Edward Rasmussen. 2004. Gaussian Processes in Machine Learning. In Advanced lectures on machine learning. Springer, 63--71."},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"publisher","DOI":"10.5555\/1756006.1859894"},{"key":"e_1_3_2_1_32_1","volume-title":"Gaussian Process Classification and Active Learning with Multiple Annotators. In International Conference on Machine Learning. 433--441","author":"Rodrigues Filipe","year":"2014","unstructured":"Filipe Rodrigues, Francisco Pereira, and Bernardete Ribeiro. 2014. Gaussian Process Classification and Active Learning with Multiple Annotators. In International Conference on Machine Learning. 433--441."},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"publisher","DOI":"10.1145\/3485447.3511960"},{"key":"e_1_3_2_1_34_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.patcog.2018.11.021"},{"key":"e_1_3_2_1_35_1","doi-asserted-by":"publisher","DOI":"10.1145\/3488560.3502182"},{"key":"e_1_3_2_1_36_1","volume-title":"Advances in Neural Information Processing Systems","volume":"32","author":"Tanaka Yusuke","year":"2019","unstructured":"Yusuke Tanaka, Toshiyuki Tanaka, Tomoharu Iwata, Takeshi Kurashima, Maya Okawa, Yasunori Akagi, and Hiroyuki Toda. 2019. Spatially Aggregated Gaussian Processes with Multivariate Areal Outputs. Advances in Neural Information Processing Systems, Vol. 32 (2019)."},{"key":"e_1_3_2_1_37_1","volume-title":"Proc. ACM SIGIR Workshop on Crowdsourcing for Information Retrieval (CIR?11)","author":"Vuurens Jeroen","year":"2011","unstructured":"Jeroen Vuurens, Arjen P de Vries, and Carsten Eickhoff. 2011. How Much Spam Can You Take? An Analysis of Crowdsourcing Results to Increase Accuracy. In Proc. ACM SIGIR Workshop on Crowdsourcing for Information Retrieval (CIR?11). 21--26."},{"key":"e_1_3_2_1_38_1","unstructured":"Jacob Whitehill Ting-fan Wu Jacob Bergsma Javier R Movellan and Paul L Ruvolo. 2009. Whose Vote Should Count More: Optimal Integration of Labels from Labelers of Unknown Expertise. In Advances in neural information processing systems. 2035--2043."},{"key":"e_1_3_2_1_39_1","volume-title":"Exploiting Heterogeneous Graph Neural Networks with Latent Worker\/Task Correlation Information for Label Aggregation in Crowdsourcing. ACM Transactions on Knowledge Discovery from Data (TKDD)","author":"Wu Hanlu","year":"2021","unstructured":"Hanlu Wu, Tengfei Ma, Lingfei Wu, Fangli Xu, and Shouling Ji. 2021. Exploiting Heterogeneous Graph Neural Networks with Latent Worker\/Task Correlation Information for Label Aggregation in Crowdsourcing. ACM Transactions on Knowledge Discovery from Data (TKDD), Vol. 16, 2 (2021), 1--18."},{"key":"e_1_3_2_1_40_1","volume-title":"Label Aggregation with Clustering for Biased Crowdsourced Labeling. In 2022 14th International Conference on Machine Learning and Computing (ICMLC). 165--169","author":"Wu Ming","year":"2022","unstructured":"Ming Wu, Qianmu Li, Jing Zhang, and Jun Hou. 2022. Label Aggregation with Clustering for Biased Crowdsourced Labeling. In 2022 14th International Conference on Machine Learning and Computing (ICMLC). 165--169."},{"key":"e_1_3_2_1_41_1","volume-title":"Advances in Neural Information Processing Systems","volume":"32","author":"Yousefi Fariba","year":"2019","unstructured":"Fariba Yousefi, Michael T Smith, and Mauricio Alvarez. 2019. Multi-task Learning for Aggregated Data Using Gaussian Processes. Advances in Neural Information Processing Systems, Vol. 32 (2019)."},{"key":"e_1_3_2_1_42_1","volume-title":"The Eleventh International Conference on Learning Representations. https:\/\/openreview.net\/forum?id=q0nmYciuuZN","author":"Zhao Jianan","year":"2023","unstructured":"Jianan Zhao, Meng Qu, Chaozhuo Li, Hao Yan, Qian Liu, Rui Li, Xing Xie, and Jian Tang. 2023. Learning on Large-scale Text-attributed Graphs via Variational Inference. In The Eleventh International Conference on Learning Representations. https:\/\/openreview.net\/forum?id=q0nmYciuuZN"},{"key":"e_1_3_2_1_43_1","doi-asserted-by":"publisher","DOI":"10.14778\/3055540.3055547"},{"key":"e_1_3_2_1_44_1","doi-asserted-by":"publisher","DOI":"10.1145\/3292500.3332277"}],"event":{"name":"SIGIR '23: The 46th International ACM SIGIR Conference on Research and Development in Information Retrieval","sponsor":["SIGIR ACM Special Interest Group on Information Retrieval"],"location":"Taipei Taiwan","acronym":"SIGIR '23"},"container-title":["Proceedings of the 46th International ACM SIGIR Conference on Research and Development in Information Retrieval"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3539618.3591685","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3539618.3591685","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T16:46:59Z","timestamp":1750178819000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3539618.3591685"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023,7,18]]},"references-count":44,"alternative-id":["10.1145\/3539618.3591685","10.1145\/3539618"],"URL":"https:\/\/doi.org\/10.1145\/3539618.3591685","relation":{},"subject":[],"published":{"date-parts":[[2023,7,18]]},"assertion":[{"value":"2023-07-18","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}