{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,3,25]],"date-time":"2025-03-25T14:51:53Z","timestamp":1742914313110,"version":"3.40.3"},"publisher-location":"Cham","reference-count":16,"publisher":"Springer International Publishing","isbn-type":[{"type":"print","value":"9783030877217"},{"type":"electronic","value":"9783030877224"}],"license":[{"start":{"date-parts":[[2021,1,1]],"date-time":"2021-01-01T00:00:00Z","timestamp":1609459200000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"},{"start":{"date-parts":[[2021,1,1]],"date-time":"2021-01-01T00:00:00Z","timestamp":1609459200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2021]]},"DOI":"10.1007\/978-3-030-87722-4_23","type":"book-chapter","created":{"date-parts":[[2021,9,24]],"date-time":"2021-09-24T01:17:40Z","timestamp":1632446260000},"page":"251-262","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":6,"title":["Can Non-specialists Provide High Quality Gold Standard Labels in Challenging Modalities?"],"prefix":"10.1007","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-9062-0013","authenticated-orcid":false,"given":"Samuel","family":"Budd","sequence":"first","affiliation":[]},{"given":"Thomas","family":"Day","sequence":"additional","affiliation":[]},{"given":"John","family":"Simpson","sequence":"additional","affiliation":[]},{"given":"Karen","family":"Lloyd","sequence":"additional","affiliation":[]},{"given":"Jacqueline","family":"Matthew","sequence":"additional","affiliation":[]},{"given":"Emily","family":"Skelton","sequence":"additional","affiliation":[]},{"given":"Reza","family":"Razavi","sequence":"additional","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0002-7813-5023","authenticated-orcid":false,"given":"Bernhard","family":"Kainz","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2021,9,21]]},"reference":[{"key":"23_CR1","unstructured":"Labelbox (2021). https:\/\/labelbox.com. Accessed 27 Feb 2021"},{"key":"23_CR2","unstructured":"Budd, S., et al.: Detecting Hypo-plastic Left Heart Syndrome in Fetal Ultrasound via Disease-specific Atlas Maps, July 2021. https:\/\/arxiv.org\/abs\/2107.02643v1"},{"key":"23_CR3","doi-asserted-by":"publisher","unstructured":"Chang, J.C., Amershi, S., Kamar, E.: Revolt: collaborative crowdsourcing for labeling machine learning datasets. In: Proceedings of the 2017 CHI Conference on Human Factors in Computing Systems, CHI 2017, pp. 2334\u20132346. Association for Computing Machinery, New York (2017). https:\/\/doi.org\/10.1145\/3025453.3026044","DOI":"10.1145\/3025453.3026044"},{"key":"23_CR4","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"209","DOI":"10.1007\/978-3-319-46976-8_22","volume-title":"Deep Learning and Data Labeling for Medical Applications","author":"V Cheplygina","year":"2016","unstructured":"Cheplygina, V., Perez-Rovira, A., Kuo, W., Tiddens, H.A.W.M., de Bruijne, M.: Early experiences with crowdsourcing airway annotations in chest CT. In: Carneiro, G., et al. (eds.) LABELS\/DLMIA -2016. LNCS, vol. 10008, pp. 209\u2013218. Springer, Cham (2016). https:\/\/doi.org\/10.1007\/978-3-319-46976-8_22"},{"key":"23_CR5","unstructured":"Fang, J., Price, B., Price, L.: Pruning non-informative text through non-expert annotations to improve aspect-level sentiment classification. In: Proceedings of the 2nd Workshop on The People\u2019s Web Meets NLP: Collaboratively Constructed Semantic Resources, pp. 37\u201345. Coling 2010 Organizing Committee, Beijing, August 2010. https:\/\/www.aclweb.org\/anthology\/W10-3505"},{"key":"23_CR6","unstructured":"Gal, Y., Ghahramani, Z.: Dropout as a Bayesian approximation: representing model uncertainty in deep learning. In: ICLR 2016, pp. 1050\u20131059 (2016)"},{"issue":"03","key":"23_CR7","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1117\/1.jmi.5.3.034002","volume":"5","author":"E Heim","year":"2018","unstructured":"Heim, E., et al.: Large-scale medical image annotation with crowd-powered algorithms. J. Med. Imaging 5(03), 1 (2018). https:\/\/doi.org\/10.1117\/1.jmi.5.3.034002","journal-title":"J. Med. Imaging"},{"key":"23_CR8","doi-asserted-by":"crossref","unstructured":"Hsueh, P.Y., Melville, P., Sindhwani, V.: Data quality from crowdsourcing: a study of annotation selection criteria. In: Proceedings of the NAACL HLT 2009 Workshop on Active Learning for Natural Language Processing, HLT 2009, pp. 27\u201335. Association for Computational Linguistics (2009)","DOI":"10.3115\/1564131.1564137"},{"key":"23_CR9","unstructured":"Jamison, E., Gurevych, I.: Needle in a haystack: reducing the costs of annotating rare-class instances in imbalanced datasets. In: Proceedings of the 28th Pacific Asia Conference on Language, Information and Computing, pp. 244\u2013253. Department of Linguistics, Chulalongkorn University, Phuket, December 2014. https:\/\/www.aclweb.org\/anthology\/Y14-1030"},{"key":"23_CR10","unstructured":"Rodrigues, F., Pereira, F.C.: Deep learning from crowds. https:\/\/arxiv.org\/pdf\/1709.01779v2.pdf"},{"key":"23_CR11","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"234","DOI":"10.1007\/978-3-319-24574-4_28","volume-title":"Medical Image Computing and Computer-Assisted Intervention \u2013 MICCAI 2015","author":"O Ronneberger","year":"2015","unstructured":"Ronneberger, O., Fischer, P., Brox, T.: U-Net: convolutional networks for biomedical image segmentation. In: Navab, N., Hornegger, J., Wells, W.M., Frangi, A.F. (eds.) MICCAI 2015. LNCS, vol. 9351, pp. 234\u2013241. Springer, Cham (2015). https:\/\/doi.org\/10.1007\/978-3-319-24574-4_28"},{"key":"23_CR12","doi-asserted-by":"crossref","unstructured":"Snow, R., O\u2019Connor, B., Jurafsky, D., Ng, A.: Cheap and fast - but is it good? Evaluating non-expert annotations for natural language tasks. In: Proceedings of the 2008 Conference on Empirical Methods in Natural Language Processing, pp. 254\u2013263. Association for Computational Linguistics, Honolulu, October 2008. https:\/\/www.aclweb.org\/anthology\/D08-1027","DOI":"10.3115\/1613715.1613751"},{"key":"23_CR13","doi-asserted-by":"publisher","first-page":"101693","DOI":"10.1016\/j.media.2020.101693","volume":"63","author":"N Tajbakhsh","year":"2020","unstructured":"Tajbakhsh, N., et al.: Embracing imperfect datasets: a review of deep learning solutions for medical image segmentation. Med. Image Anal. 63, 101693 (2020). https:\/\/doi.org\/10.1016\/j.media.2020.101693","journal-title":"Med. Image Anal."},{"key":"23_CR14","doi-asserted-by":"publisher","first-page":"527","DOI":"10.1016\/j.chb.2016.12.074","volume":"73","author":"R Tinati","year":"2017","unstructured":"Tinati, R., Luczak-Roesch, M., Simperl, E., Hall, W.: An investigation of player motivations in Eyewire, a gamified citizen science project. Comput. Hum. Behav. 73, 527\u2013540 (2017). https:\/\/doi.org\/10.1016\/j.chb.2016.12.074","journal-title":"Comput. Hum. Behav."},{"key":"23_CR15","unstructured":"Wilm, F., et al.: How many annotators do we need? A study on the influence of inter-observer variability on the reliability of automatic mitotic figure assessment, December 2020. http:\/\/arxiv.org\/abs\/2012.02495"},{"issue":"17","key":"23_CR16","doi-asserted-by":"publisher","first-page":"175007","DOI":"10.1088\/1361-6560\/ab99e5","volume":"65","author":"S Yu","year":"2020","unstructured":"Yu, S., et al.: Robustness study of noisy annotation in deep learning based medical image segmentation. Phys. Med. Biol. 65(17), 175007 (2020). https:\/\/doi.org\/10.1088\/1361-6560\/ab99e5","journal-title":"Phys. Med. Biol."}],"container-title":["Lecture Notes in Computer Science","Domain Adaptation and Representation Transfer, and Affordable Healthcare and AI for Resource Diverse Global Health"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-030-87722-4_23","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2023,4,6]],"date-time":"2023-04-06T08:23:16Z","timestamp":1680769396000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-3-030-87722-4_23"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2021]]},"ISBN":["9783030877217","9783030877224"],"references-count":16,"URL":"https:\/\/doi.org\/10.1007\/978-3-030-87722-4_23","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"type":"print","value":"0302-9743"},{"type":"electronic","value":"1611-3349"}],"subject":[],"published":{"date-parts":[[2021]]},"assertion":[{"value":"21 September 2021","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"FAIR","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Workshop on Affordable Healthcare and AI for Resource Diverse Global Health","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Strasbourg","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"France","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2021","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"27 September 2021","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"27 September 2021","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"1","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"fair22021","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"https:\/\/sites.google.com\/view\/fair-workshop-2021\/home","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Double-blind","order":1,"name":"type","label":"Type","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"CMT","order":2,"name":"conference_management_system","label":"Conference Management System","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"17","order":3,"name":"number_of_submissions_sent_for_review","label":"Number of Submissions Sent for Review","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"10","order":4,"name":"number_of_full_papers_accepted","label":"Number of Full Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"0","order":5,"name":"number_of_short_papers_accepted","label":"Number of Short Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"59% - The value is computed by the equation \"Number of Full Papers Accepted \/ Number of Submissions Sent for Review * 100\" and then rounded to a whole number.","order":6,"name":"acceptance_rate_of_full_papers","label":"Acceptance Rate of Full Papers","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"2.8","order":7,"name":"average_number_of_reviews_per_paper","label":"Average Number of Reviews per Paper","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"2.5","order":8,"name":"average_number_of_papers_per_reviewer","label":"Average Number of Papers per Reviewer","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"No","order":9,"name":"external_reviewers_involved","label":"External Reviewers Involved","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"The conference was held virtually due to the COVID-19 pandemic.","order":10,"name":"additional_info_on_review_process","label":"Additional Info on Review Process","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}}]}}