{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,3,4]],"date-time":"2026-03-04T03:01:25Z","timestamp":1772593285459,"version":"3.50.1"},"publisher-location":"New York, NY, USA","reference-count":68,"publisher":"ACM","license":[{"start":{"date-parts":[[2023,8,8]],"date-time":"2023-08-08T00:00:00Z","timestamp":1691452800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by-nc-nd\/4.0\/"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2023,8,8]]},"DOI":"10.1145\/3600211.3604766","type":"proceedings-article","created":{"date-parts":[[2023,8,29]],"date-time":"2023-08-29T18:41:37Z","timestamp":1693334497000},"page":"351-362","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":19,"title":["Ground Truth Or Dare: Factors Affecting The Creation Of Medical Datasets For Training AI"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-0689-6912","authenticated-orcid":false,"given":"Hubert Dariusz","family":"Zaj\u0105c","sequence":"first","affiliation":[{"name":"Department of Computer Science, University of Copenhagen, Denmark"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-1009-1810","authenticated-orcid":false,"given":"Natalia Rozalia","family":"Avlona","sequence":"additional","affiliation":[{"name":"Department of Computer Science, University of Copenhagen, Denmark"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-1392-5999","authenticated-orcid":false,"given":"Finn","family":"Kensing","sequence":"additional","affiliation":[{"name":"Department of Computer Science, University of Copenhagen, Denmark"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-9342-5520","authenticated-orcid":false,"given":"Tariq Osman","family":"Andersen","sequence":"additional","affiliation":[{"name":"Department of Computer Science, University of Copenhagen, Denmark"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-1874-0958","authenticated-orcid":false,"given":"Irina","family":"Shklovski","sequence":"additional","affiliation":[{"name":"Department of Computer Science, University of Copenhagen, Denmark"}]}],"member":"320","published-online":{"date-parts":[[2023,8,29]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE-SEIP.2019.00042"},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.1145\/3411764.3445736"},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1609\/aimag.v36i1.2564"},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.inffus.2019.12.012"},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1162\/tacl_a_00041"},{"key":"e_1_3_2_1_6_1","volume-title":"Bowker and Susan\u00a0Leigh Star","author":"C.","year":"2000","unstructured":"Geoffrey\u00a0C. Bowker and Susan\u00a0Leigh Star. 2000. Sorting Things Out - Classification and Its Consequences. The MIT Press. https:\/\/mitpress.mit.edu\/9780262522953\/"},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"publisher","DOI":"10.1038\/525315a"},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.media.2020.101797"},{"key":"e_1_3_2_1_9_1","volume-title":"Constructing Grounded Theory","author":"Charmaz K","unstructured":"K Charmaz. 2014. Constructing Grounded Theory (2nd ed.).","edition":"2"},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.1109\/TR.2021.3070863"},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.1038\/s41563-019-0345-0"},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.4135\/9781412985833"},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1093\/jamia"},{"key":"e_1_3_2_1_14_1","volume-title":"Whose Ground Truth? Accounting for Individual and Collective Identities Underlying Dataset Annotation. (12","author":"Denton Emily","year":"2021","unstructured":"Emily Denton, Mark D\u00edaz, Ian Kivlichan, Vinodkumar Prabhakaran, and Rachel Rosen. 2021. Whose Ground Truth? Accounting for Individual and Collective Identities Underlying Dataset Annotation. (12 2021). http:\/\/arxiv.org\/abs\/2112.04554"},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.1145\/3514094.3534187"},{"key":"e_1_3_2_1_16_1","volume-title":"Oxford Handbook of Ethics of AI, Markus\u00a0D","author":"Dignum Virginia","unstructured":"Virginia Dignum. 2020. Responsibility and artificial intelligence. In Oxford Handbook of Ethics of AI, Markus\u00a0D. Dubber, Frank Pasquale, and Sunit Das (Eds.). Oxford University Press, Chapter\u00a011, 215\u2013231."},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.1007\/S10618-022-00854-Z"},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1145\/3025453.3025837"},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","DOI":"10.1002\/9781119306696"},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.1145\/3458723"},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.1162\/QSS_A_00144"},{"key":"e_1_3_2_1_22_1","volume-title":"Is an Oxymoron","author":"Gitelman Lisa","unstructured":"Lisa Gitelman. 2013. \u201cRaw Data\u201d Is an Oxymoron. MIT Press. 9\u201310 pages. https:\/\/nyuscholars.nyu.edu\/en\/publications\/raw-data-is-an-oxymoron"},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.1016\/J.ACRA.2021.09.013"},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"publisher","DOI":"10.1145\/3461702.3462620"},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.1080\/1369118X.2020.1751866"},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.1109\/VLHCC.2016.7739680"},{"key":"e_1_3_2_1_27_1","volume-title":"The Dataset Nutrition Label: A Framework To Drive Higher Data Quality Standards. (5","author":"Holland Sarah","year":"2018","unstructured":"Sarah Holland, Ahmed Hosny, Sarah Newman, Joshua Joseph, and Kasia Chmielinski. 2018. The Dataset Nutrition Label: A Framework To Drive Higher Data Quality Standards. (5 2018). http:\/\/arxiv.org\/abs\/1805.03677"},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"publisher","DOI":"10.1145\/3419249.3420149"},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.1177\/0309132519836158"},{"key":"e_1_3_2_1_30_1","volume-title":"Towards a \u2019Science","author":"Hovy Eduard","year":"2010","unstructured":"Eduard Hovy and Julia Lavid. 2010. Towards a \u2019Science\u2019 of Corpus Annotation: A New Methodological Challenge for Corpus Linguistics. INTERNATIONAL JOURNAL OF TRANSLATION 22, 1 (2010)."},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"publisher","DOI":"10.1145\/3442188.3445918"},{"key":"e_1_3_2_1_32_1","doi-asserted-by":"publisher","DOI":"10.1145\/3442188.3445918"},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v33i01.3301590"},{"key":"e_1_3_2_1_34_1","volume-title":"Effect of Radiology Report Labeler Quality on Deep Learning Models for Chest X-Ray Interpretation. (4","author":"Jain Saahil","year":"2021","unstructured":"Saahil Jain, Akshay Smit, Andrew\u00a0Y. Ng, and Pranav Rajpurkar. 2021. Effect of Radiology Report Labeler Quality on Deep Learning Models for Chest X-Ray Interpretation. (4 2021). http:\/\/arxiv.org\/abs\/2104.00793"},{"key":"e_1_3_2_1_35_1","volume-title":"a large publicly available database of labeled chest radiographs. (1","author":"Johnson Alistair","year":"2019","unstructured":"Alistair E.\u00a0W. Johnson, Tom\u00a0J. Pollard, Nathaniel\u00a0R. Greenbaum, Matthew\u00a0P. Lungren, Chih-ying Deng, Yifan Peng, Zhiyong Lu, Roger\u00a0G. Mark, Seth\u00a0J. Berkowitz, and Steven Horng. 2019. MIMIC-CXR-JPG, a large publicly available database of labeled chest radiographs. (1 2019). http:\/\/arxiv.org\/abs\/1901.07042"},{"key":"e_1_3_2_1_36_1","doi-asserted-by":"publisher","DOI":"10.1145\/3461702.3462598"},{"key":"e_1_3_2_1_37_1","doi-asserted-by":"publisher","DOI":"10.3390\/DIAGNOSTICS12123112"},{"key":"e_1_3_2_1_38_1","doi-asserted-by":"publisher","DOI":"10.1055\/s-0038-1637976"},{"key":"e_1_3_2_1_39_1","doi-asserted-by":"publisher","DOI":"10.11613\/BM.2012.031"},{"key":"e_1_3_2_1_40_1","doi-asserted-by":"publisher","DOI":"10.1109\/EMBC.2013.6610779"},{"key":"e_1_3_2_1_41_1","doi-asserted-by":"publisher","DOI":"10.1145\/3555561"},{"key":"e_1_3_2_1_42_1","doi-asserted-by":"publisher","DOI":"10.1145\/3492853"},{"key":"e_1_3_2_1_43_1","doi-asserted-by":"publisher","DOI":"10.1145\/3415186"},{"key":"e_1_3_2_1_44_1","doi-asserted-by":"publisher","DOI":"10.1145\/3555623"},{"key":"e_1_3_2_1_45_1","doi-asserted-by":"publisher","DOI":"10.1145\/3442188.3445880"},{"key":"e_1_3_2_1_46_1","doi-asserted-by":"publisher","DOI":"10.1177\/1527476419837739"},{"key":"e_1_3_2_1_47_1","doi-asserted-by":"publisher","DOI":"10.1145\/3287560.3287596"},{"key":"e_1_3_2_1_48_1","doi-asserted-by":"publisher","DOI":"10.1145\/3290605.3300356"},{"key":"e_1_3_2_1_49_1","doi-asserted-by":"publisher","DOI":"10.1145\/3411764.3445402"},{"key":"e_1_3_2_1_50_1","doi-asserted-by":"publisher","DOI":"10.1109\/ESEM.2019.8870157"},{"key":"e_1_3_2_1_51_1","volume-title":"Exploring large scale public medical image datasets. (7","author":"Oakden-Rayner Luke","year":"2019","unstructured":"Luke Oakden-Rayner. 2019. Exploring large scale public medical image datasets. (7 2019). http:\/\/arxiv.org\/abs\/1907.12720"},{"key":"e_1_3_2_1_52_1","doi-asserted-by":"publisher","DOI":"10.1145\/3368555.3384468"},{"key":"e_1_3_2_1_53_1","doi-asserted-by":"publisher","DOI":"10.1145\/2702123.2702298"},{"key":"e_1_3_2_1_54_1","doi-asserted-by":"publisher","DOI":"10.1162\/coli.2008.34.3.319"},{"key":"e_1_3_2_1_55_1","doi-asserted-by":"publisher","DOI":"10.1177\/0306312719849709"},{"key":"e_1_3_2_1_56_1","doi-asserted-by":"publisher","DOI":"10.1145\/3173574.3173606"},{"key":"e_1_3_2_1_57_1","doi-asserted-by":"publisher","DOI":"10.1145\/3411764.3445518"},{"key":"e_1_3_2_1_58_1","doi-asserted-by":"publisher","DOI":"10.1145\/3476058"},{"key":"e_1_3_2_1_59_1","doi-asserted-by":"publisher","DOI":"10.1145\/3461702.3462594"},{"key":"e_1_3_2_1_60_1","doi-asserted-by":"publisher","DOI":"10.14236\/ewic\/HCI2018.14"},{"key":"e_1_3_2_1_61_1","doi-asserted-by":"publisher","DOI":"10.1023\/A:1008651105359"},{"key":"e_1_3_2_1_62_1","doi-asserted-by":"publisher","DOI":"10.1145\/3491102.3517537"},{"key":"e_1_3_2_1_63_1","doi-asserted-by":"publisher","DOI":"10.1515\/CLLT.2008.010"},{"key":"e_1_3_2_1_64_1","doi-asserted-by":"publisher","DOI":"10.1080\/01436597.2021.1948832"},{"key":"e_1_3_2_1_65_1","doi-asserted-by":"publisher","DOI":"10.1145\/3359313"},{"key":"e_1_3_2_1_66_1","doi-asserted-by":"publisher","unstructured":"Xiaosong Wang Yifan Peng Le Lu Zhiyong Lu Mohammadhadi Bagheri and Ronald\u00a0M. Summers. 2019. ChestX-ray: Hospital-Scale Chest X-ray Database and Benchmarks on Weakly Supervised Classification and Localization of Common Thorax Diseases. In Advances in Computer Vision and Pattern Recognition. 369\u2013392. https:\/\/doi.org\/10.1007\/978-3-030-13969-8_18","DOI":"10.1007\/978-3-030-13969-8_18"},{"key":"e_1_3_2_1_67_1","doi-asserted-by":"publisher","DOI":"10.48340\/ecscw2022_p04"},{"key":"e_1_3_2_1_68_1","doi-asserted-by":"publisher","DOI":"10.1145\/3392826"}],"event":{"name":"AIES '23: AAAI\/ACM Conference on AI, Ethics, and Society","location":"Montr\u00e9al QC Canada","acronym":"AIES '23","sponsor":["SIGAI ACM Special Interest Group on Artificial Intelligence"]},"container-title":["Proceedings of the 2023 AAAI\/ACM Conference on AI, Ethics, and Society"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3600211.3604766","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3600211.3604766","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,18]],"date-time":"2025-06-18T18:55:31Z","timestamp":1750272931000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3600211.3604766"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023,8,8]]},"references-count":68,"alternative-id":["10.1145\/3600211.3604766","10.1145\/3600211"],"URL":"https:\/\/doi.org\/10.1145\/3600211.3604766","relation":{},"subject":[],"published":{"date-parts":[[2023,8,8]]},"assertion":[{"value":"2023-08-29","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}