{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,12,18]],"date-time":"2025-12-18T14:22:40Z","timestamp":1766067760682,"version":"3.40.3"},"publisher-location":"Cham","reference-count":44,"publisher":"Springer Nature Switzerland","isbn-type":[{"type":"print","value":"9783031395383"},{"type":"electronic","value":"9783031395390"}],"license":[{"start":{"date-parts":[[2023,1,1]],"date-time":"2023-01-01T00:00:00Z","timestamp":1672531200000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2023,1,1]],"date-time":"2023-01-01T00:00:00Z","timestamp":1672531200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2023]]},"DOI":"10.1007\/978-3-031-39539-0_13","type":"book-chapter","created":{"date-parts":[[2023,7,30]],"date-time":"2023-07-30T00:02:38Z","timestamp":1690675358000},"page":"147-162","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":3,"title":["Learn2Agree: Fitting with\u00a0Multiple Annotators Without Objective Ground Truth"],"prefix":"10.1007","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-9819-088X","authenticated-orcid":false,"given":"Chongyang","family":"Wang","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Yuan","family":"Gao","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Chenyou","family":"Fan","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Junjie","family":"Hu","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Tin Lum","family":"Lam","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Nicholas D.","family":"Lane","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Nadia","family":"Bianchi-Berthouze","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2023,7,30]]},"reference":[{"issue":"4","key":"13_CR1","doi-asserted-by":"publisher","first-page":"435","DOI":"10.1109\/TAFFC.2015.2462830","volume":"7","author":"MS Aung","year":"2015","unstructured":"Aung, M.S., et al.: The automatic detection of chronic pain-related expression: requirements, challenges and the multimodal emopain dataset. IEEE Trans. Affect. Comput. 7(4), 435\u2013451 (2015)","journal-title":"IEEE Trans. Affect. Comput."},{"key":"13_CR2","unstructured":"Charpentier, B., Z\u00fcgner, D., G\u00fcnnemann, S.: Posterior network: uncertainty estimation without ood samples via density-based pseudo-counts. arXiv preprint arXiv:2006.09239 (2020)"},{"issue":"1","key":"13_CR3","doi-asserted-by":"publisher","first-page":"37","DOI":"10.1177\/001316446002000104","volume":"20","author":"J Cohen","year":"1960","unstructured":"Cohen, J.: A coefficient of agreement for nominal scales. Educ. Psychol. Measur. 20(1), 37\u201346 (1960)","journal-title":"Educ. Psychol. Measur."},{"issue":"4","key":"13_CR4","doi-asserted-by":"publisher","first-page":"213","DOI":"10.1037\/h0026256","volume":"70","author":"J Cohen","year":"1968","unstructured":"Cohen, J.: Weighted kappa: nominal scale agreement provision for scaled disagreement or partial credit. Psychol. Bull. 70(4), 213 (1968)","journal-title":"Psychol. Bull."},{"key":"13_CR5","doi-asserted-by":"crossref","unstructured":"Cui, Y., Jia, M., Lin, T.Y., Song, Y., Belongie, S.: Class-balanced loss based on effective number of samples. In: Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition, pp. 9268\u20139277 (2019)","DOI":"10.1109\/CVPR.2019.00949"},{"key":"13_CR6","doi-asserted-by":"crossref","unstructured":"Deng, J., Dong, W., Socher, R., Li, L.J., Li, K., Fei-Fei, L.: ImageNet: a large-scale hierarchical image database. In: 2009 IEEE Conference on Computer Vision and Pattern Recognition, pp. 248\u2013255. IEEE (2009)","DOI":"10.1109\/CVPR.2009.5206848"},{"key":"13_CR7","doi-asserted-by":"crossref","unstructured":"Fan, C., et al.: Multi-horizon time series forecasting with temporal attention learning. In: Proceedings of the 25th ACM SIGKDD International Conference on Knowledge Discovery & Data Mining, pp. 2527\u20132535 (2019)","DOI":"10.1145\/3292500.3330662"},{"key":"13_CR8","doi-asserted-by":"crossref","unstructured":"Felipe, S., Singh, A., Bradley, C., Williams, A.C., Bianchi-Berthouze, N.: Roles for personal informatics in chronic pain. In: 2015 9th International Conference on Pervasive Computing Technologies for Healthcare, pp. 161\u2013168. IEEE (2015)","DOI":"10.4108\/icst.pervasivehealth.2015.259501"},{"issue":"5","key":"13_CR9","doi-asserted-by":"publisher","first-page":"378","DOI":"10.1037\/h0031619","volume":"76","author":"JL Fleiss","year":"1971","unstructured":"Fleiss, J.L.: Measuring nominal scale agreement among many raters. Psychol. Bull. 76(5), 378 (1971)","journal-title":"Psychol. Bull."},{"key":"13_CR10","doi-asserted-by":"crossref","unstructured":"Guan, M., Gulshan, V., Dai, A., Hinton, G.: Who said what: modeling individual labelers improves classification. In: Proceedings of the AAAI Conference on Artificial Intelligence, vol. 32 (2018)","DOI":"10.1609\/aaai.v32i1.11756"},{"key":"13_CR11","doi-asserted-by":"crossref","unstructured":"Hao, L., Naiman, D.Q., Naiman, D.Q.: Quantile Regression. Sage (2007)","DOI":"10.4135\/9781412985550"},{"key":"13_CR12","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"107","DOI":"10.1007\/978-3-642-24600-5_14","volume-title":"Affective Computing and Intelligent Interaction","author":"J Healey","year":"2011","unstructured":"Healey, J.: Recording affect in the field: towards methods and metrics for improving ground truth labels. In: D\u2019Mello, S., Graesser, A., Schuller, B., Martin, J.-C. (eds.) ACII 2011. LNCS, vol. 6974, pp. 107\u2013116. Springer, Heidelberg (2011). https:\/\/doi.org\/10.1007\/978-3-642-24600-5_14"},{"issue":"10","key":"13_CR13","doi-asserted-by":"publisher","first-page":"1973","DOI":"10.1109\/TPAMI.2016.2621761","volume":"39","author":"N Hu","year":"2016","unstructured":"Hu, N., Englebienne, G., Lou, Z., Kr\u00f6se, B.: Learning to recognize human activities using soft labels. IEEE Trans. Pattern Anal. Mach. Intell. 39(10), 1973\u20131984 (2016)","journal-title":"IEEE Trans. Pattern Anal. Mach. Intell."},{"key":"13_CR14","first-page":"21713","volume":"33","author":"P Hu","year":"2020","unstructured":"Hu, P., Sclaroff, S., Saenko, K.: Uncertainty-aware learning for zero-shot semantic segmentation. Adv. Neural Inf. Process. Syst. 33, 21713\u201321724 (2020)","journal-title":"Adv. Neural Inf. Process. Syst."},{"key":"13_CR15","doi-asserted-by":"crossref","unstructured":"Huang, G., Liu, Z., Van Der Maaten, L., Weinberger, K.Q.: Densely connected convolutional networks. In: Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition, pp. 4700\u20134708 (2017)","DOI":"10.1109\/CVPR.2017.243"},{"key":"13_CR16","doi-asserted-by":"crossref","unstructured":"Ji, W., et al.: Learning calibrated medical image segmentation via multi-rater agreement modeling. In: Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition, pp. 12341\u201312351 (2021)","DOI":"10.1109\/CVPR46437.2021.01216"},{"key":"13_CR17","doi-asserted-by":"crossref","unstructured":"Karimi, D., Dou, H., Warfield, S.K., Gholipour, A.: Deep learning with noisy labels: exploring techniques and remedies in medical image analysis. Med. Image Anal. 65, 101759 (2020)","DOI":"10.1016\/j.media.2020.101759"},{"key":"13_CR18","doi-asserted-by":"crossref","unstructured":"Kendall, A., Badrinarayanan, V., Cipolla, R.: Bayesian segnet: model uncertainty in deep convolutional encoder-decoder architectures for scene understanding. In: British Machine Vision Conference (2017)","DOI":"10.5244\/C.31.57"},{"key":"13_CR19","unstructured":"Kingma, D.P., Ba, J.: Adam: a method for stochastic optimization. arXiv preprint arXiv:1412.6980 (2014)"},{"issue":"4","key":"13_CR20","doi-asserted-by":"publisher","first-page":"1027","DOI":"10.1109\/TSMCB.2010.2103557","volume":"41","author":"A Kleinsmith","year":"2011","unstructured":"Kleinsmith, A., Bianchi-Berthouze, N., Steed, A.: Automatic recognition of non-acted affective postures. IEEE Trans. Syst. Man Cybern. Part B 41(4), 1027\u20131038 (2011)","journal-title":"IEEE Trans. Syst. Man Cybern. Part B"},{"issue":"4","key":"13_CR21","doi-asserted-by":"publisher","first-page":"143","DOI":"10.1257\/jep.15.4.143","volume":"15","author":"R Koenker","year":"2001","unstructured":"Koenker, R., Hallock, K.F.: Quantile regression. J. Econ. Perspect. 15(4), 143\u2013156 (2001)","journal-title":"J. Econ. Perspect."},{"key":"13_CR22","doi-asserted-by":"publisher","first-page":"144","DOI":"10.1016\/j.patrec.2017.05.018","volume":"105","author":"J de La Torre","year":"2018","unstructured":"de La Torre, J., Puig, D., Valls, A.: Weighted kappa loss function for multi-class classification of ordinal data in deep learning. Pattern Recogn. Lett. 105, 144\u2013154 (2018)","journal-title":"Pattern Recogn. Lett."},{"key":"13_CR23","unstructured":"Lakshminarayanan, B., Pritzel, A., Blundell, C.: Simple and scalable predictive uncertainty estimation using deep ensembles. arXiv preprint arXiv:1612.01474 (2016)"},{"issue":"6","key":"13_CR24","doi-asserted-by":"publisher","first-page":"2557","DOI":"10.1109\/TIP.2016.2544703","volume":"25","author":"TA Lampert","year":"2016","unstructured":"Lampert, T.A., Stumpf, A., Gan\u00e7arski, P.: An empirical study into annotator agreement, ground truth estimation, and algorithm evaluation. IEEE Trans. Image Process. 25(6), 2557\u20132572 (2016)","journal-title":"IEEE Trans. Image Process."},{"issue":"1","key":"13_CR25","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1038\/s41598-017-17876-z","volume":"7","author":"C Leibig","year":"2017","unstructured":"Leibig, C., Allken, V., Ayhan, M.S., Berens, P., Wahl, S.: Leveraging uncertainty information from deep neural networks for disease detection. Sci. Rep. 7(1), 1\u201314 (2017)","journal-title":"Sci. Rep."},{"key":"13_CR26","doi-asserted-by":"crossref","unstructured":"Li, X., Wang, W., Hu, X., Li, J., Tang, J., Yang, J.: Generalized focal loss V2: learning reliable localization quality estimation for dense object detection. In: Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition, pp. 11632\u201311641 (2021)","DOI":"10.1109\/CVPR46437.2021.01146"},{"key":"13_CR27","doi-asserted-by":"crossref","unstructured":"Lin, T.Y., Goyal, P., Girshick, R., He, K., Doll\u00e1r, P.: Focal loss for dense object detection. In: Proceedings of the IEEE International Conference on Computer Vision, pp. 2980\u20132988 (2017)","DOI":"10.1109\/ICCV.2017.324"},{"key":"13_CR28","doi-asserted-by":"crossref","unstructured":"Long, C., Hua, G.: Multi-class multi-annotator active learning with robust gaussian process for visual recognition. In: Proceedings of the IEEE International Conference on Computer Vision, pp. 2839\u20132847 (2015)","DOI":"10.1109\/ICCV.2015.325"},{"key":"13_CR29","doi-asserted-by":"crossref","unstructured":"Long, C., Hua, G., Kapoor, A.: Active visual recognition with expertise estimation in crowdsourcing. In: Proceedings of the IEEE International Conference on Computer Vision, pp. 3000\u20133007 (2013)","DOI":"10.1109\/ICCV.2013.373"},{"key":"13_CR30","unstructured":"Lovchinsky, I., et al.: Discrepancy ratio: evaluating model performance when even experts disagree on the truth. In: International Conference on Learning Representations (2019)"},{"key":"13_CR31","doi-asserted-by":"crossref","unstructured":"Ma, L., St\u00fcckler, J., Kerl, C., Cremers, D.: Multi-view deep learning for consistent semantic mapping with RGB-D cameras. In: 2017 IEEE\/RSJ International Conference on Intelligent Robots and Systems, pp. 598\u2013605. IEEE (2017)","DOI":"10.1109\/IROS.2017.8202213"},{"key":"13_CR32","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"225","DOI":"10.1007\/978-3-642-24600-5_26","volume-title":"Affective Computing and Intelligent Interaction","author":"H Meng","year":"2011","unstructured":"Meng, H., Kleinsmith, A., Bianchi-Berthouze, N.: Multi-score learning for affect recognition: the case of body postures. In: D\u2019Mello, S., Graesser, A., Schuller, B., Martin, J.-C. (eds.) ACII 2011. LNCS, vol. 6974, pp. 225\u2013234. Springer, Heidelberg (2011). https:\/\/doi.org\/10.1007\/978-3-642-24600-5_26"},{"key":"13_CR33","doi-asserted-by":"crossref","unstructured":"Postels, J., Ferroni, F., Coskun, H., Navab, N., Tombari, F.: Sampling-free epistemic uncertainty estimation using approximated variance propagation. In: Proceedings of the IEEE\/CVF International Conference on Computer Vision, pp. 2931\u20132940 (2019)","DOI":"10.1109\/ICCV.2019.00302"},{"key":"13_CR34","unstructured":"Rajpurkar, P., et al.: MURA: large dataset for abnormality detection in musculoskeletal radiographs. arXiv preprint arXiv:1712.06957 (2017)"},{"key":"13_CR35","doi-asserted-by":"crossref","unstructured":"Shen, Y., Zhang, Z., Sabuncu, M.R., Sun, L.: Real-time uncertainty estimation in computer vision via uncertainty-aware distribution distillation. In: Proceedings of the IEEE\/CVF Winter Conference on Applications of Computer Vision, pp. 707\u2013716 (2021)","DOI":"10.1109\/WACV48630.2021.00075"},{"issue":"3\u20134","key":"13_CR36","doi-asserted-by":"publisher","first-page":"335","DOI":"10.1080\/07370024.2015.1085310","volume":"31","author":"A Singh","year":"2016","unstructured":"Singh, A., et al.: Go-with-the-flow: tracking, analysis and sonification of movement and breathing to build confidence in activity despite chronic pain. Hum.-Comput. Interact. 31(3\u20134), 335\u2013383 (2016)","journal-title":"Hum.-Comput. Interact."},{"key":"13_CR37","unstructured":"Surowiecki, J.: The Wisdom of Crowds. Anchor (2005)"},{"key":"13_CR38","doi-asserted-by":"crossref","unstructured":"Tanno, R., Saeedi, A., Sankaranarayanan, S., Alexander, D.C., Silberman, N.: Learning from noisy labels by regularized estimation of annotator confusion. In: Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition, pp. 11244\u201311253 (2019)","DOI":"10.1109\/CVPR.2019.01150"},{"key":"13_CR39","doi-asserted-by":"crossref","unstructured":"Wang, C., Gao, Y., Mathur, A., Williams, A.C.D.C., Lane, N.D., Bianchi-Berthouze, N.: Leveraging activity recognition to enable protective behavior detection in continuous data. Proc. ACM Interact. Mob. Wearable Ubiquit. Technol. 5(2) (2021)","DOI":"10.1145\/3463508"},{"issue":"3","key":"13_CR40","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1145\/3449068","volume":"2","author":"C Wang","year":"2021","unstructured":"Wang, C., Olugbade, T.A., Mathur, A., Williams, A.C.D.C., Lane, N.D., Bianchi-Berthouze, N.: Chronic pain protective behavior detection with deep learning. ACM Trans. Comput. Healthc. 2(3), 1\u201324 (2021)","journal-title":"ACM Trans. Comput. Healthc."},{"issue":"7","key":"13_CR41","doi-asserted-by":"publisher","first-page":"903","DOI":"10.1109\/TMI.2004.828354","volume":"23","author":"SK Warfield","year":"2004","unstructured":"Warfield, S.K., Zou, K.H., Wells, W.M.: Simultaneous truth and performance level estimation (staple): an algorithm for the validation of image segmentation. IEEE Trans. Med. Imaging 23(7), 903\u2013921 (2004)","journal-title":"IEEE Trans. Med. Imaging"},{"key":"13_CR42","unstructured":"Yan, Y., et al.: Modeling annotator expertise: Learning when everybody knows a bit of something. In: Proceedings of the Thirteenth International Conference on Artificial Intelligence and Statistics, pp. 932\u2013939. JMLR Workshop and Conference Proceedings (2010)"},{"issue":"3","key":"13_CR43","doi-asserted-by":"publisher","first-page":"291","DOI":"10.1007\/s10994-013-5412-1","volume":"95","author":"Y Yan","year":"2014","unstructured":"Yan, Y., Rosales, R., Fung, G., Subramanian, R., Dy, J.: Learning from multiple annotators with varying expertise. Mach. Learn. 95(3), 291\u2013327 (2014)","journal-title":"Mach. Learn."},{"key":"13_CR44","unstructured":"Zhang, L., et al.: Disentangling human error from the ground truth in segmentation of medical images. arXiv preprint arXiv:2007.15963 (2020)"}],"container-title":["Lecture Notes in Computer Science","Trustworthy Machine Learning for Healthcare"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-031-39539-0_13","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2023,7,30]],"date-time":"2023-07-30T00:28:52Z","timestamp":1690676932000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-3-031-39539-0_13"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023]]},"ISBN":["9783031395383","9783031395390"],"references-count":44,"URL":"https:\/\/doi.org\/10.1007\/978-3-031-39539-0_13","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"type":"print","value":"0302-9743"},{"type":"electronic","value":"1611-3349"}],"subject":[],"published":{"date-parts":[[2023]]},"assertion":[{"value":"30 July 2023","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"TML4H","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"International Workshop on Trustworthy Machine Learning for Healthcare","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2023","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"4 May 2023","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"4 May 2023","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"1","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"tml4h2023","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"https:\/\/cse.hkust.edu.hk\/~jhc\/2023tml4h.html","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Double-blind","order":1,"name":"type","label":"Type","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"OpenReview","order":2,"name":"conference_management_system","label":"Conference Management System","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"30","order":3,"name":"number_of_submissions_sent_for_review","label":"Number of Submissions Sent for Review","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"16","order":4,"name":"number_of_full_papers_accepted","label":"Number of Full Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"0","order":5,"name":"number_of_short_papers_accepted","label":"Number of Short Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"53% - The value is computed by the equation \"Number of Full Papers Accepted \/ Number of Submissions Sent for Review * 100\" and then rounded to a whole number.","order":6,"name":"acceptance_rate_of_full_papers","label":"Acceptance Rate of Full Papers","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"2.73","order":7,"name":"average_number_of_reviews_per_paper","label":"Average Number of Reviews per Paper","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"1.44","order":8,"name":"average_number_of_papers_per_reviewer","label":"Average Number of Papers per Reviewer","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"No","order":9,"name":"external_reviewers_involved","label":"External Reviewers Involved","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}}]}}