{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,8,2]],"date-time":"2025-08-02T19:09:28Z","timestamp":1754161768075,"version":"3.41.2"},"publisher-location":"New York, NY, USA","reference-count":33,"publisher":"ACM","content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2025,8,4]]},"DOI":"10.1145\/3715928.3737480","type":"proceedings-article","created":{"date-parts":[[2025,7,29]],"date-time":"2025-07-29T07:33:43Z","timestamp":1753774423000},"page":"188-197","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":0,"title":["Rationale-aware Label Aggregation in Crowdsourcing"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0009-0004-9041-3235","authenticated-orcid":false,"given":"Tomoya","family":"Nishio","sequence":"first","affiliation":[{"name":"University of Tsukuba, Tsukuba Science City, Japan"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-3265-7029","authenticated-orcid":false,"given":"Hiroyoshi","family":"Ito","sequence":"additional","affiliation":[{"name":"University of Tsukuba, Tsukuba Science City, Japan"}]},{"ORCID":"https:\/\/orcid.org\/0009-0009-1118-6078","authenticated-orcid":false,"given":"Takumi","family":"Tamura","sequence":"additional","affiliation":[{"name":"University of Tsukuba, Tsukuba Science City, Japan"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-4606-9065","authenticated-orcid":false,"given":"Atsuyuki","family":"Morishima","sequence":"additional","affiliation":[{"name":"University of Tsukuba, Tsukuba Science City, Japan"}]}],"member":"320","published-online":{"date-parts":[[2025,8,3]]},"reference":[{"key":"e_1_3_3_2_2_2","doi-asserted-by":"publisher","DOI":"10.1145\/2009916.2010039"},{"key":"e_1_3_3_2_3_2","doi-asserted-by":"publisher","DOI":"10.1109\/CSCWD49262.2021.9437769"},{"key":"e_1_3_3_2_4_2","doi-asserted-by":"publisher","unstructured":"Florian Daniel Pavel Kucherbaev Cinzia Cappiello Boualem Benatallah and Mohammad Allahbakhsh. 2018. Quality Control in Crowdsourcing: A Survey of Quality Attributes Assessment Techniques and Assurance Actions. ACM Comput. Surv. 51 1 (Jan. 2018) 7:1\u20137:40. 10.1145\/3148148","DOI":"10.1145\/3148148"},{"key":"e_1_3_3_2_5_2","doi-asserted-by":"publisher","unstructured":"A.\u00a0P. Dawid and A.\u00a0M. Skene. 1979. Maximum Likelihood Estimation of Observer Error-Rates Using the EM Algorithm. Journal of the Royal Statistical Society. Series C (Applied Statistics) 28 1 (1979) 20\u201328. https:\/\/doi.org\/10.2307\/2346806 jstor:2346806","DOI":"10.2307\/2346806"},{"key":"e_1_3_3_2_6_2","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2011.6126394"},{"key":"e_1_3_3_2_7_2","doi-asserted-by":"publisher","unstructured":"G\u00fcl\u015een Eryi\u011fit Ali \u015eenta\u015f and Johanna Monti. 2023. Gamified Crowdsourcing for Idiom Corpora Construction. Natural Language Engineering 29 4 (July 2023) 909\u2013941. https:\/\/doi.org\/10.1017\/S1351324921000401","DOI":"10.1017\/S1351324921000401"},{"key":"e_1_3_3_2_8_2","doi-asserted-by":"publisher","unstructured":"Thomas\u00a0Mosgaard Giselsson Rasmus\u00a0Nyholm J\u00f8rgensen Peter\u00a0Kryger Jensen Mads Dyrmann and Henrik\u00a0Skov Midtiby. 2017. A Public Image Database for Benchmark of Plant Seedling Classification Algorithms. https:\/\/doi.org\/10.48550\/arXiv.1711.05458 arxiv:https:\/\/arXiv.org\/abs\/1711.05458\u00a0[cs]","DOI":"10.48550\/arXiv.1711.05458"},{"key":"e_1_3_3_2_9_2","doi-asserted-by":"publisher","unstructured":"Tanya Goyal Tyler McDonnell Mucahid Kutlu Tamer Elsayed and Matthew Lease. 2018. Your Behavior Signals Your Reliability: Modeling Crowd Behavioral Traces to Ensure Quality Relevance Annotations. Proceedings of the AAAI Conference on Human Computation and Crowdsourcing 6 (June 2018) 41\u201349. https:\/\/doi.org\/10.1609\/hcomp.v6i1.13331","DOI":"10.1609\/hcomp.v6i1.13331"},{"key":"e_1_3_3_2_10_2","doi-asserted-by":"publisher","unstructured":"Hideaki Imamura Issei Sato and Masashi Sugiyama. 2018. Analysis of Minimax Error Rate for Crowdsourcing and Its Application to Worker Clustering Model. https:\/\/doi.org\/10.48550\/arXiv.1802.04551 arxiv:https:\/\/arXiv.org\/abs\/1802.04551\u00a0[stat]","DOI":"10.48550\/arXiv.1802.04551"},{"key":"e_1_3_3_2_11_2","doi-asserted-by":"publisher","DOI":"10.1145\/3269206.3271779"},{"key":"e_1_3_3_2_12_2","doi-asserted-by":"publisher","DOI":"10.1145\/2009916.2009947"},{"key":"e_1_3_3_2_13_2","unstructured":"A. Khosla Nityananda Jayadevaprakash Bangpeng Yao and Li Fei-Fei. 2012. Novel Dataset for Fine-Grained Image Categorization : Stanford Dogs."},{"key":"e_1_3_3_2_14_2","volume-title":"The Tenth AAAI Conference on Human Computation and Crowdsourcing (HCOMP2022)","author":"Kida Kai","year":"2022","unstructured":"Kai Kida, Hiroyoshi Ito, Masaki Matsubara, Nobutaka Suzuki, and Atsuyuki Morishima. 2022. Aggregating Crowd Intelligence over Open Source Information: An Inference Rule Centric Approach. In The Tenth AAAI Conference on Human Computation and Crowdsourcing (HCOMP2022)."},{"key":"e_1_3_3_2_15_2","first-page":"619","volume-title":"Proceedings of the Fifteenth International Conference on Artificial Intelligence and Statistics","author":"Kim Hyun-Chul","year":"2012","unstructured":"Hyun-Chul Kim and Zoubin Ghahramani. 2012. Bayesian Classifier Combination. In Proceedings of the Fifteenth International Conference on Artificial Intelligence and Statistics. PMLR, 619\u2013627."},{"key":"e_1_3_3_2_16_2","doi-asserted-by":"publisher","DOI":"10.1145\/1357054.1357127"},{"key":"e_1_3_3_2_17_2","doi-asserted-by":"publisher","unstructured":"Adriana Kovashka Olga Russakovsky Li Fei-Fei and Kristen Grauman. 2016. Crowdsourcing in Computer Vision. Foundations and Trends\u00ae in Computer Graphics and Vision 10 2 (2016) 103\u2013175. https:\/\/doi.org\/10.1561\/0600000073 arxiv:https:\/\/arXiv.org\/abs\/1611.02145\u00a0[cs]","DOI":"10.1561\/0600000073"},{"key":"e_1_3_3_2_18_2","first-page":"14","volume-title":"Proceedings of the First Workshop on Bridging Human\u2013Computer Interaction and Natural Language Processing","author":"Lees Alyssa","year":"2021","unstructured":"Alyssa Lees, Daniel Borkan, Ian Kivlichan, Jorge Nario, and Tesh Goyal. 2021. Capturing Covertly Toxic Speech via Crowdsourcing. In Proceedings of the First Workshop on Bridging Human\u2013Computer Interaction and Natural Language Processing, Su\u00a0Lin Blodgett, Michael Madaio, Brendan O\u2019Connor, Hanna Wallach, and Qian Yang (Eds.). Association for Computational Linguistics, Online, 14\u201320."},{"key":"e_1_3_3_2_19_2","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-86517-727"},{"key":"e_1_3_3_2_20_2","doi-asserted-by":"publisher","unstructured":"Tyler McDonnell Matthew Lease Mucahid Kutlu and Tamer Elsayed. 2016. Why Is That Relevant? Collecting Annotator Rationales for Relevance Judgments. Proceedings of the AAAI Conference on Human Computation and Crowdsourcing 4 (Sept. 2016) 139\u2013148. https:\/\/doi.org\/10.1609\/hcomp.v4i1.13287","DOI":"10.1609\/hcomp.v4i1.13287"},{"key":"e_1_3_3_2_21_2","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2023.law-1.8"},{"key":"e_1_3_3_2_22_2","doi-asserted-by":"publisher","unstructured":"Allard Oelen Markus Stocker and S\u00f6ren Auer. 2024. Creating and Validating a Scholarly Knowledge Graph Using Natural Language Processing and Microtask Crowdsourcing. International Journal on Digital Libraries 25 2 (June 2024) 273\u2013285. https:\/\/doi.org\/10.1007\/s00799-023-00360-7","DOI":"10.1007\/s00799-023-00360-7"},{"key":"e_1_3_3_2_23_2","doi-asserted-by":"publisher","DOI":"10.5555\/2540128.2540496"},{"key":"e_1_3_3_2_24_2","series-title":"(ICML \u201900)","first-page":"727","volume-title":"Proceedings of the Seventeenth International Conference on Machine Learning","author":"Pelleg Dan","year":"2000","unstructured":"Dan Pelleg and Andrew\u00a0W. Moore. 2000. X-Means: Extending K-means with Efficient Estimation of the Number of Clusters. In Proceedings of the Seventeenth International Conference on Machine Learning(ICML \u201900). Morgan Kaufmann Publishers Inc., San Francisco, CA, USA, 727\u2013734."},{"key":"e_1_3_3_2_25_2","doi-asserted-by":"publisher","unstructured":"Nils Reimers and Iryna Gurevych. 2019. Sentence-BERT: Sentence Embeddings Using Siamese BERT-Networks. https:\/\/doi.org\/10.48550\/arXiv.1908.10084 arxiv:https:\/\/arXiv.org\/abs\/1908.10084\u00a0[cs]","DOI":"10.48550\/arXiv.1908.10084"},{"key":"e_1_3_3_2_26_2","doi-asserted-by":"publisher","unstructured":"Nils Reimers and Iryna Gurevych. 2020. Making Monolingual Sentence Embeddings Multilingual Using Knowledge Distillation. https:\/\/doi.org\/10.48550\/arXiv.2004.09813 arxiv:https:\/\/arXiv.org\/abs\/2004.09813\u00a0[cs]","DOI":"10.48550\/arXiv.2004.09813"},{"key":"e_1_3_3_2_27_2","doi-asserted-by":"publisher","unstructured":"Martin\u00a0W.P Savelsbergh and Marlin\u00a0W. Ulmer. 2022. Challenges and Opportunities in Crowdsourced Delivery Planning and Operations. 4OR 20 1 (March 2022) 1\u201321. https:\/\/doi.org\/10.1007\/s10288-021-00500-2","DOI":"10.1007\/s10288-021-00500-2"},{"key":"e_1_3_3_2_28_2","doi-asserted-by":"publisher","DOI":"10.1109\/MASS58611.2023.00051"},{"key":"e_1_3_3_2_29_2","doi-asserted-by":"publisher","DOI":"10.1145\/2566486.2567989"},{"key":"e_1_3_3_2_30_2","volume-title":"Caltech-UCSD Birds-200-2011","author":"Wah Catherine","year":"2011","unstructured":"Catherine Wah, Branson Steve, Welinder Peter, Perona Pietro, and Belongle Serge. 2011. Caltech-UCSD Birds-200-2011. Technical Reports CNS-TR-2011-001. California Institute of Technology."},{"key":"e_1_3_3_2_31_2","doi-asserted-by":"publisher","unstructured":"Peter Washington Haik Kalantarian Jack Kent Arman Husic Aaron Kline Emilie Leblanc Cathy Hou Cezmi Mutlu Kaitlyn Dunlap Yordan Penev Nate Stockham Brianna Chrisman Kelley Paskov Jae-Yoon Jung Catalin Voss Nick Haber and Dennis\u00a0P. Wall. 2021. Training Affective Computer Vision Models by Crowdsourcing Soft-Target Labels. Cognitive Computation 13 5 (Sept. 2021) 1363\u20131373. https:\/\/doi.org\/10.1007\/s12559-021-09936-4","DOI":"10.1007\/s12559-021-09936-4"},{"key":"e_1_3_3_2_32_2","volume-title":"Advances in Neural Information Processing Systems","author":"Whitehill Jacob","year":"2009","unstructured":"Jacob Whitehill, Ting-fan Wu, Jacob Bergsma, Javier Movellan, and Paul Ruvolo. 2009. Whose Vote Should Count More: Optimal Integration of Labels from Labelers of Unknown Expertise. In Advances in Neural Information Processing Systems, Vol.\u00a022. Curran Associates, Inc."},{"key":"e_1_3_3_2_33_2","doi-asserted-by":"publisher","unstructured":"Yuchen Zhang Xi Chen Dengyong Zhou and Michael\u00a0I. Jordan. 2014. Spectral Methods Meet EM: A Provably Optimal Algorithm for Crowdsourcing. https:\/\/doi.org\/10.48550\/arXiv.1406.3824 arxiv:https:\/\/arXiv.org\/abs\/1406.3824\u00a0[stat]","DOI":"10.48550\/arXiv.1406.3824"},{"key":"e_1_3_3_2_34_2","doi-asserted-by":"publisher","DOI":"10.5555\/2999325.2999380"}],"event":{"name":"CI 2025: ACM Collective Intelligence Conference","location":"San Diego CA USA","acronym":"CI 2025","sponsor":["SIGCHI ACM Special Interest Group on Computer-Human Interaction"]},"container-title":["Proceedings of the ACM Collective Intelligence Conference"],"original-title":[],"deposited":{"date-parts":[[2025,7,29]],"date-time":"2025-07-29T09:46:48Z","timestamp":1753782408000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3715928.3737480"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,8,3]]},"references-count":33,"alternative-id":["10.1145\/3715928.3737480","10.1145\/3715928"],"URL":"https:\/\/doi.org\/10.1145\/3715928.3737480","relation":{},"subject":[],"published":{"date-parts":[[2025,8,3]]},"assertion":[{"value":"2025-08-03","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}