{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,12,6]],"date-time":"2025-12-06T17:15:48Z","timestamp":1765041348290,"version":"3.40.3"},"publisher-location":"Cham","reference-count":37,"publisher":"Springer Nature Switzerland","isbn-type":[{"type":"print","value":"9783031250811"},{"type":"electronic","value":"9783031250828"}],"license":[{"start":{"date-parts":[[2023,1,1]],"date-time":"2023-01-01T00:00:00Z","timestamp":1672531200000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2023,1,1]],"date-time":"2023-01-01T00:00:00Z","timestamp":1672531200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2023]]},"DOI":"10.1007\/978-3-031-25082-8_37","type":"book-chapter","created":{"date-parts":[[2023,2,11]],"date-time":"2023-02-11T09:12:42Z","timestamp":1676106762000},"page":"552-565","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":4,"title":["Variability Matters: Evaluating Inter-Rater Variability in\u00a0Histopathology for\u00a0Robust Cell Detection"],"prefix":"10.1007","author":[{"ORCID":"https:\/\/orcid.org\/0000-0001-6321-0003","authenticated-orcid":false,"given":"Cholmin","family":"Kang","sequence":"first","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0002-6164-2563","authenticated-orcid":false,"given":"Chunggi","family":"Lee","sequence":"additional","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0003-3435-0271","authenticated-orcid":false,"given":"Heon","family":"Song","sequence":"additional","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0003-0416-8479","authenticated-orcid":false,"given":"Minuk","family":"Ma","sequence":"additional","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0002-4298-0903","authenticated-orcid":false,"given":"S\u00e9rgio","family":"Pereira","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2023,2,12]]},"reference":[{"issue":"1","key":"37_CR1","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1038\/s41467-021-21896-9","volume":"12","author":"JA Diao","year":"2021","unstructured":"Diao, J.A., et al.: Human-interpretable image features derived from densely mapped cancer pathology slides predict diverse molecular phenotypes. Nat. Commun. 12(1), 1\u201315 (2021)","journal-title":"Nat. Commun."},{"issue":"2","key":"37_CR2","doi-asserted-by":"publisher","first-page":"112","DOI":"10.1038\/s42256-019-0018-3","volume":"1","author":"B Lutnick","year":"2019","unstructured":"Lutnick, B., et al.: An integrated iterative annotation technique for easing neural network training in medical image analysis. Nat. Mach. Intell. 1(2), 112\u2013119 (2019)","journal-title":"Nat. Mach. Intell."},{"key":"37_CR3","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"209","DOI":"10.1007\/978-3-319-46976-8_22","volume-title":"Deep Learning and Data Labeling for Medical Applications","author":"V Cheplygina","year":"2016","unstructured":"Cheplygina, V., Perez-Rovira, A., Kuo, W., Tiddens, H.A.W.M., de Bruijne, M.: Early experiences with crowdsourcing airway annotations in chest CT. In: Carneiro, G., et al. (eds.) LABELS\/DLMIA -2016. LNCS, vol. 10008, pp. 209\u2013218. Springer, Cham (2016). https:\/\/doi.org\/10.1007\/978-3-319-46976-8_22"},{"issue":"10","key":"37_CR4","doi-asserted-by":"publisher","first-page":"2020","DOI":"10.1109\/TMC.2014.2385097","volume":"14","author":"B Guo","year":"2014","unstructured":"Guo, B., Chen, H., Yu, Z., Xie, X., Huangfu, S., Zhang, D.: FlierMeet: a mobile crowdsensing system for cross-space public information reposting, tagging, and sharing. Trans. Mob. Comput. 14(10), 2020\u20132033 (2014)","journal-title":"Trans. Mob. Comput."},{"key":"37_CR5","doi-asserted-by":"crossref","unstructured":"Li, J., Wong, Y., Zhao, Q., Kankanhalli, M.S.: Learning to learn from noisy labeled data. In: CVPR (2019)","DOI":"10.1109\/CVPR.2019.00519"},{"key":"37_CR6","doi-asserted-by":"publisher","first-page":"352","DOI":"10.1016\/j.neucom.2020.02.119","volume":"400","author":"Y Wang","year":"2020","unstructured":"Wang, Y., Wang, D., Pang, W., Miao, C., Tan, A.H., Zhou, Y.: A systematic density-based clustering method using anchor points. Neurocomputing 400, 352\u2013370 (2020)","journal-title":"Neurocomputing"},{"key":"37_CR7","doi-asserted-by":"crossref","unstructured":"Miller, D., Sunderhauf, N., Milford, M., Dayoub, F.: Class anchor clustering: a loss for distance-based open set recognition. In: Proceedings of the IEEE\/CVF Winter Conference on Applications of Computer Vision, pp. 3570\u20133578 (2021)","DOI":"10.1109\/WACV48630.2021.00361"},{"key":"37_CR8","unstructured":"Krizhevsky, A., Sutskever, I., Hinton, G.E.: ImageNet classification with deep convolutional neural networks. In: Advances in Neural Information Processing Systems, vol. 25 (2012)"},{"key":"37_CR9","doi-asserted-by":"crossref","unstructured":"Girshick, R., Donahue, J., Darrell, T., Malik, J.: Rich feature hierarchies for accurate object detection and semantic segmentation. In: Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition, pp. 580\u2013587 (2014)","DOI":"10.1109\/CVPR.2014.81"},{"issue":"5","key":"37_CR10","doi-asserted-by":"publisher","first-page":"140","DOI":"10.1109\/MSP.2012.2204190","volume":"29","author":"E Meijering","year":"2012","unstructured":"Meijering, E.: Cell segmentation: 50 years down the road [life sciences]. IEEE Signal Process. Mag. 29(5), 140\u2013145 (2012)","journal-title":"IEEE Signal Process. Mag."},{"key":"37_CR11","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"17","DOI":"10.1007\/978-3-319-10581-9_3","volume-title":"Machine Learning in Medical Imaging","author":"T Chen","year":"2014","unstructured":"Chen, T., Chefd\u2019hotel, C.: Deep learning based automatic immune cell detection for immunohistochemistry images. In: Wu, G., Zhang, D., Zhou, L. (eds.) MLMI 2014. LNCS, vol. 8679, pp. 17\u201324. Springer, Cham (2014). https:\/\/doi.org\/10.1007\/978-3-319-10581-9_3"},{"issue":"2","key":"37_CR12","doi-asserted-by":"publisher","first-page":"416","DOI":"10.1109\/JBHI.2016.2526603","volume":"21","author":"Z Gao","year":"2016","unstructured":"Gao, Z., Wang, L., Zhou, L., Zhang, J.: Hep-2 cell image classification with deep convolutional neural networks. IEEE J. Biomed. Health Inform. 21(2), 416\u2013428 (2016)","journal-title":"IEEE J. Biomed. Health Inform."},{"key":"37_CR13","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"411","DOI":"10.1007\/978-3-642-40763-5_51","volume-title":"Medical Image Computing and Computer-Assisted Intervention \u2013 MICCAI 2013","author":"DC Cire\u015fan","year":"2013","unstructured":"Cire\u015fan, D.C., Giusti, A., Gambardella, L.M., Schmidhuber, J.: Mitosis detection in breast cancer histology images with deep neural networks. In: Mori, K., Sakuma, I., Sato, Y., Barillot, C., Navab, N. (eds.) MICCAI 2013. LNCS, vol. 8150, pp. 411\u2013418. Springer, Heidelberg (2013). https:\/\/doi.org\/10.1007\/978-3-642-40763-5_51"},{"key":"37_CR14","unstructured":"Xue, Y., Ray, N.: Cell detection in microscopy images with deep convolutional neural network and compressed sensing. arXiv preprint arXiv:1708.03307 (2017)"},{"key":"37_CR15","doi-asserted-by":"publisher","first-page":"91","DOI":"10.1016\/j.ejca.2019.06.012","volume":"118","author":"A Hekler","year":"2019","unstructured":"Hekler, A., et al.: Deep learning outperformed 11 pathologists in the classification of histopathological melanoma images. Eur. J. Cancer 118, 91\u201396 (2019)","journal-title":"Eur. J. Cancer"},{"issue":"2","key":"37_CR16","doi-asserted-by":"publisher","first-page":"211","DOI":"10.1177\/03009858211067478","volume":"59","author":"CA Bertram","year":"2022","unstructured":"Bertram, C.A., et al.: Computer-assisted mitotic count using a deep learning-based algorithm improves interobserver reproducibility and accuracy. Vet. Pathol. 59(2), 211\u2013226 (2022)","journal-title":"Vet. Pathol."},{"key":"37_CR17","doi-asserted-by":"crossref","unstructured":"Schaekermann, M., Beaton, G., Habib, M., Lim, A., Larson, K., Law, E.: Understanding expert disagreement in medical data analysis through structured adjudication. Proc. ACM Hum.-Comput. Interact. 3(CSCW), 1\u201323 (2019)","DOI":"10.1145\/3359178"},{"issue":"1","key":"37_CR18","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1038\/s41598-020-69534-6","volume":"10","author":"C Haarburger","year":"2020","unstructured":"Haarburger, C., M\u00fcller-Franzes, G., Weninger, L., Kuhl, C., Truhn, D., Merhof, D.: Radiomics feature reproducibility under inter-rater variability in segmentations of CT images. Sci. Rep. 10(1), 1\u201310 (2020)","journal-title":"Sci. Rep."},{"key":"37_CR19","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"665","DOI":"10.1007\/978-3-030-32251-9_73","volume-title":"Medical Image Computing and Computer Assisted Intervention \u2013 MICCAI 2019","author":"CH Sudre","year":"2019","unstructured":"Sudre, C.H., et al.: Let\u2019s agree to disagree: learning highly debatable multirater labelling. In: Shen, D., et al. (eds.) MICCAI 2019. LNCS, vol. 11767, pp. 665\u2013673. Springer, Cham (2019). https:\/\/doi.org\/10.1007\/978-3-030-32251-9_73"},{"key":"37_CR20","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"438","DOI":"10.1007\/978-3-319-10470-6_55","volume-title":"Medical Image Computing and Computer-Assisted Intervention \u2013 MICCAI 2014","author":"L Maier-Hein","year":"2014","unstructured":"Maier-Hein, L., et al.: Can masses of non-experts train highly accurate image classifiers? In: Golland, P., Hata, N., Barillot, C., Hornegger, J., Howe, R. (eds.) MICCAI 2014. LNCS, vol. 8674, pp. 438\u2013445. Springer, Cham (2014). https:\/\/doi.org\/10.1007\/978-3-319-10470-6_55"},{"key":"37_CR21","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"454","DOI":"10.1007\/978-3-319-10470-6_57","volume-title":"Medical Image Computing and Computer-Assisted Intervention \u2013 MICCAI 2014","author":"R Kwitt","year":"2014","unstructured":"Kwitt, R., Hegenbart, S., Rasiwasia, N., V\u00e9csei, A., Uhl, A.: Do we need annotation experts? A case study in celiac disease classification. In: Golland, P., Hata, N., Barillot, C., Hornegger, J., Howe, R. (eds.) MICCAI 2014. LNCS, vol. 8674, pp. 454\u2013461. Springer, Cham (2014). https:\/\/doi.org\/10.1007\/978-3-319-10470-6_57"},{"key":"37_CR22","doi-asserted-by":"crossref","unstructured":"Amgad, M., et al.: NuCLS: a scalable crowdsourcing, deep learning approach and dataset for nucleus classification, localization and segmentation. arXiv:2102.09099 (2021)","DOI":"10.1093\/gigascience\/giac037"},{"key":"37_CR23","doi-asserted-by":"crossref","unstructured":"Veit, A., Alldrin, N., Chechik, G., Krasin, I., Gupta, A., Belongie, S.: Learning from noisy large-scale datasets with minimal supervision. In: CVPR (2017)","DOI":"10.1109\/CVPR.2017.696"},{"key":"37_CR24","unstructured":"Jiang, L., Zhou, Z., Leung, T., Li, L.J., Fei-Fei, L.: MentorNet: learning data-driven curriculum for very deep neural networks on corrupted labels. In: ICML (2018)"},{"issue":"3","key":"37_CR25","doi-asserted-by":"publisher","first-page":"291","DOI":"10.1007\/s10994-013-5412-1","volume":"95","author":"Y Yan","year":"2014","unstructured":"Yan, Y., Rosales, R., Fung, G., Subramanian, R., Dy, J.: Learning from multiple annotators with varying expertise. Mach. Learn. 95(3), 291\u2013327 (2014)","journal-title":"Mach. Learn."},{"key":"37_CR26","unstructured":"Raykar, V.C., et al.: Learning from crowds. J. Mach. Learn. Res. 11(4) (2010)"},{"key":"37_CR27","unstructured":"Rodrigues, F., Pereira, F., Ribeiro, B.: Gaussian process classification and active learning with multiple annotators. In: International Conference on Machine Learning, pp. 433\u2013441. PMLR (2014)"},{"key":"37_CR28","doi-asserted-by":"crossref","unstructured":"Bartolo, M., Thrush, T., Riedel, S., Stenetorp, P., Jia, R., Kiela, D.: Models in the loop: aiding crowdworkers with generative annotation assistants. arXiv:2112.09062 (2021)","DOI":"10.18653\/v1\/2022.naacl-main.275"},{"issue":"1","key":"37_CR29","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1038\/s41598-021-90444-8","volume":"11","author":"M Khened","year":"2021","unstructured":"Khened, M., Kori, A., Rajkumar, H., Krishnamurthi, G., Srinivasan, B.: A generalized deep learning framework for whole-slide image segmentation and analysis. Sci. Rep. 11(1), 1\u201314 (2021)","journal-title":"Sci. Rep."},{"key":"37_CR30","volume":"58","author":"Z Swiderska-Chadaj","year":"2019","unstructured":"Swiderska-Chadaj, Z., et al.: Learning to detect lymphocytes in immunohistochemistry with deep learning. MIA 58, 101547 (2019)","journal-title":"MIA"},{"key":"37_CR31","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"833","DOI":"10.1007\/978-3-030-01234-2_49","volume-title":"Computer Vision \u2013 ECCV 2018","author":"L-C Chen","year":"2018","unstructured":"Chen, L.-C., Zhu, Y., Papandreou, G., Schroff, F., Adam, H.: Encoder-decoder with atrous separable convolution for semantic image segmentation. In: Ferrari, V., Hebert, M., Sminchisescu, C., Weiss, Y. (eds.) ECCV 2018. LNCS, vol. 11211, pp. 833\u2013851. Springer, Cham (2018). https:\/\/doi.org\/10.1007\/978-3-030-01234-2_49"},{"key":"37_CR32","unstructured":"Kingma, D.P., Ba, J.: Adam: a method for stochastic optimization. In: Bengio, Y., LeCun, Y. (eds.) ICLR (2015)"},{"key":"37_CR33","doi-asserted-by":"crossref","unstructured":"Milletari, F., Navab, N., Ahmadi, S.A.: V-Net: fully convolutional neural networks for volumetric medical image segmentation. In: 3DV (2016)","DOI":"10.1109\/3DV.2016.79"},{"key":"37_CR34","unstructured":"Paszke, A., et al.: Automatic differentiation in PyTorch (2017)"},{"key":"37_CR35","doi-asserted-by":"crossref","unstructured":"Shapiro, S.S., Wilk, M.B.: An analysis of variance test for normality (complete samples)$$\\dagger $$. Biometrika 52(3\u20134), 591\u2013611 (1965)","DOI":"10.1093\/biomet\/52.3-4.591"},{"issue":"2","key":"37_CR36","doi-asserted-by":"publisher","first-page":"463","DOI":"10.1007\/s11135-014-0003-1","volume":"49","author":"R Falotico","year":"2015","unstructured":"Falotico, R., Quatto, P.: Fleiss\u2019 kappa statistic without paradoxes. Qual. Quant. 49(2), 463\u2013470 (2015)","journal-title":"Qual. Quant."},{"key":"37_CR37","doi-asserted-by":"crossref","unstructured":"Woolson, R.F.: Wilcoxon signed-rank test. Wiley Encyclopedia of Clinical Trials, pp. 1\u20133 (2007)","DOI":"10.1002\/9780471462422.eoct979"}],"container-title":["Lecture Notes in Computer Science","Computer Vision \u2013 ECCV 2022 Workshops"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-031-25082-8_37","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,3,7]],"date-time":"2024-03-07T13:09:20Z","timestamp":1709816960000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-3-031-25082-8_37"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023]]},"ISBN":["9783031250811","9783031250828"],"references-count":37,"URL":"https:\/\/doi.org\/10.1007\/978-3-031-25082-8_37","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"type":"print","value":"0302-9743"},{"type":"electronic","value":"1611-3349"}],"subject":[],"published":{"date-parts":[[2023]]},"assertion":[{"value":"12 February 2023","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"ECCV","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"European Conference on Computer Vision","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Tel Aviv","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Israel","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2022","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"23 October 2022","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"27 October 2022","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"17","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"eccv2022","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"https:\/\/eccv2022.ecva.net\/","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Double-blind","order":1,"name":"type","label":"Type","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"CMT","order":2,"name":"conference_management_system","label":"Conference Management System","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"5804","order":3,"name":"number_of_submissions_sent_for_review","label":"Number of Submissions Sent for Review","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"1645","order":4,"name":"number_of_full_papers_accepted","label":"Number of Full Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"0","order":5,"name":"number_of_short_papers_accepted","label":"Number of Short Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"28% - The value is computed by the equation \"Number of Full Papers Accepted \/ Number of Submissions Sent for Review * 100\" and then rounded to a whole number.","order":6,"name":"acceptance_rate_of_full_papers","label":"Acceptance Rate of Full Papers","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"3.21","order":7,"name":"average_number_of_reviews_per_paper","label":"Average Number of Reviews per Paper","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"3.91","order":8,"name":"average_number_of_papers_per_reviewer","label":"Average Number of Papers per Reviewer","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"Yes","order":9,"name":"external_reviewers_involved","label":"External Reviewers Involved","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"From the workshops, 367 reviewed full papers have been selected for publication","order":10,"name":"additional_info_on_review_process","label":"Additional Info on Review Process","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}}]}}