{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,13]],"date-time":"2026-04-13T18:59:54Z","timestamp":1776106794878,"version":"3.50.1"},"publisher-location":"New York, NY, USA","reference-count":62,"publisher":"ACM","license":[{"start":{"date-parts":[[2025,3,24]],"date-time":"2025-03-24T00:00:00Z","timestamp":1742774400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"funder":[{"name":"Young Investigator Award by the Office of Naval Research.","award":["ONR N000142212188"],"award-info":[{"award-number":["ONR N000142212188"]}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2025,3,24]]},"DOI":"10.1145\/3708359.3712161","type":"proceedings-article","created":{"date-parts":[[2025,3,19]],"date-time":"2025-03-19T12:50:34Z","timestamp":1742388634000},"page":"624-640","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":5,"title":["Empowering Medical Data Labeling for Non-Experts with DANNY: Enhancing Accuracy and Mitigating Over-Reliance on AI"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-1357-951X","authenticated-orcid":false,"given":"Youngseung","family":"Jeon","sequence":"first","affiliation":[{"name":"Electrical and Computer Engineering, UCLA, Los Angeles, California, USA,"}]},{"ORCID":"https:\/\/orcid.org\/0009-0009-2784-2957","authenticated-orcid":false,"given":"Christopher","family":"Hwang","sequence":"additional","affiliation":[{"name":"Electrical and Computer Engineering, UCLA, Los Angeles, California, USA,"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-8527-1744","authenticated-orcid":false,"given":"Xiang 'Anthony'","family":"Chen","sequence":"additional","affiliation":[{"name":"HCI Research, UCLA, Los Angeles, California, USA,"}]}],"member":"320","published-online":{"date-parts":[[2025,3,24]]},"reference":[{"key":"e_1_3_3_3_2_2","doi-asserted-by":"crossref","unstructured":"Moloud Abdar Farhad Pourpanah Sadiq Hussain Dana Rezazadegan Li Liu Mohammad Ghavamzadeh Paul Fieguth Xiaochun Cao Abbas Khosravi U\u00a0Rajendra Acharya et\u00a0al. 2021. A review of uncertainty quantification in deep learning: Techniques applications and challenges. Information fusion 76 (2021) 243\u2013297.","DOI":"10.1016\/j.inffus.2021.05.008"},{"key":"e_1_3_3_3_3_2","doi-asserted-by":"crossref","unstructured":"Sozan\u00a0Mohammed Ahmed and Ramadhan\u00a0J Mstafa. 2022. Identifying Severity Grading of Knee Osteoarthritis from X-ray Images Using an Efficient Mixture of Deep Learning and Machine Learning Models. Diagnostics 12 12 (2022) 2939.","DOI":"10.3390\/diagnostics12122939"},{"key":"e_1_3_3_3_4_2","volume-title":"Medical Imaging with Deep Learning","author":"Ayhan Murat\u00a0Seckin","year":"2022","unstructured":"Murat\u00a0Seckin Ayhan and Philipp Berens. 2022. Test-time data augmentation for estimation of heteroscedastic aleatoric uncertainty in deep neural networks. In Medical Imaging with Deep Learning."},{"key":"e_1_3_3_3_5_2","doi-asserted-by":"crossref","unstructured":"Amir Banifatemi Nicolas Miailhe R Buse\u00a0\u00c7etin Alexandre Cadain Yolanda Lannquist and Cyrus Hodes. 2021. Democratizing AI for humanity: A common goal. Reflections on Artificial Intelligence for Humanity (2021) 228\u2013236.","DOI":"10.1007\/978-3-030-69128-8_14"},{"key":"e_1_3_3_3_6_2","doi-asserted-by":"crossref","unstructured":"Katarzyna Borys Yasmin\u00a0Alyssa Schmitt Meike Nauta Christin Seifert Nicole Kr\u00e4mer Christoph\u00a0M Friedrich and Felix Nensa. 2023. Explainable AI in Medical Imaging: An overview for clinical practitioners\u2013Saliency-based XAI approaches. European journal of radiology (2023) 110787.","DOI":"10.1016\/j.ejrad.2023.110787"},{"key":"e_1_3_3_3_7_2","doi-asserted-by":"crossref","unstructured":"Zana Bu\u00e7inca Maja\u00a0Barbara Malaya and Krzysztof\u00a0Z Gajos. 2021. To trust or to think: cognitive forcing functions can reduce overreliance on AI in AI-assisted decision-making. Proceedings of the ACM on Human-Computer Interaction 5 CSCW1 (2021) 1\u201321.","DOI":"10.1145\/3449287"},{"key":"e_1_3_3_3_8_2","doi-asserted-by":"publisher","DOI":"10.1145\/3290605.3300234"},{"key":"e_1_3_3_3_9_2","volume-title":"Graphics Interface 2022","author":"Chang Chia-Ming","year":"2022","unstructured":"Chia-Ming Chang, Yi He, Xi Yang, Haoran Xie, and Takeo Igarashi. 2022. DualLabel: secondary labels for challenging image annotation. In Graphics Interface 2022."},{"key":"e_1_3_3_3_10_2","doi-asserted-by":"publisher","DOI":"10.1145\/3411764.3445165"},{"key":"e_1_3_3_3_11_2","doi-asserted-by":"publisher","DOI":"10.1145\/3025453.3026044"},{"key":"e_1_3_3_3_12_2","unstructured":"Pingjun Chen. 2018. Knee osteoarthritis severity grading dataset. Mendeley Data 1 (2018) 21\u201323."},{"key":"e_1_3_3_3_13_2","doi-asserted-by":"publisher","DOI":"10.1109\/ICSMC.2000.884424"},{"key":"e_1_3_3_3_14_2","unstructured":"Victoria Clarke Virginia Braun and Nikki Hayfield. 2015. Thematic analysis. Qualitative Psychology: A Practical Guide to Research Methods (2015) 222\u2013248."},{"key":"e_1_3_3_3_15_2","doi-asserted-by":"crossref","unstructured":"Andy Cockburn Carl Gutwin Joey Scarr and Sylvain Malacria. 2014. Supporting novice to expert transitions in user interfaces. ACM Computing Surveys (CSUR) 47 2 (2014) 1\u201336.","DOI":"10.1145\/2659796"},{"key":"e_1_3_3_3_16_2","unstructured":"Cathrine Damgaard Trine\u00a0Naja Eriksen Dovile Juodelyte Veronika Cheplygina and Amelia Jim\u00e9nez-S\u00e1nchez. 2023. Augmenting chest x-ray datasets with non-expert annotations. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2309.02244 (2023)."},{"key":"e_1_3_3_3_17_2","doi-asserted-by":"crossref","unstructured":"Roc\u00edo Del\u00a0Amor Julio Silva-Rodr\u00edguez and Valery Naranjo. 2023. Labeling confidence for uncertainty-aware histology image classification. Computerized Medical Imaging and Graphics 107 (2023) 102231.","DOI":"10.1016\/j.compmedimag.2023.102231"},{"key":"e_1_3_3_3_18_2","doi-asserted-by":"publisher","DOI":"10.1145\/3397481.3450698"},{"key":"e_1_3_3_3_19_2","doi-asserted-by":"crossref","unstructured":"Murat Dikmen and Catherine Burns. 2022. The effects of domain knowledge on trust in explainable AI and task performance: A case of peer-to-peer lending. International Journal of Human-Computer Studies 162 (2022) 102792.","DOI":"10.1016\/j.ijhcs.2022.102792"},{"key":"e_1_3_3_3_20_2","doi-asserted-by":"crossref","unstructured":"Yi-Li Fang Hai-Long Sun Peng-Peng Chen and Ting Deng. 2017. Improving the quality of crowdsourced image labeling via label similarity. Journal of Computer Science and Technology 32 (2017) 877\u2013889.","DOI":"10.1007\/s11390-017-1770-7"},{"key":"e_1_3_3_3_21_2","unstructured":"Matthew Grissinger. 2019. Understanding human over-reliance on technology. Pharmacy and Therapeutics 44 6 (2019) 320."},{"key":"e_1_3_3_3_22_2","doi-asserted-by":"crossref","unstructured":"Thomas Grote. 2021. Trustworthy medical AI systems need to know when they don\u2019t know. Journal of medical ethics 47 5 (2021) 337\u2013338.","DOI":"10.1136\/medethics-2021-107463"},{"key":"e_1_3_3_3_23_2","doi-asserted-by":"crossref","unstructured":"Andrzej Grzybowski Kai Jin and Hongkang Wu. 2024. Challenges of artificial intelligence in medicine and dermatology. Clinics in dermatology (2024).","DOI":"10.1016\/j.clindermatol.2023.12.013"},{"key":"e_1_3_3_3_24_2","doi-asserted-by":"crossref","unstructured":"Hongyan Gu Yuan Liang Yifan Xu Christopher\u00a0Kazu Williams Shino Magaki Negar Khanlou Harry Vinters Zesheng Chen Shuo Ni Chunxu Yang et\u00a0al. 2023. Improving workflow integration with XPath: Design and evaluation of a human-AI diagnosis system in pathology. ACM Transactions on Computer-Human Interaction 30 2 (2023) 1\u201337.","DOI":"10.1145\/3577011"},{"key":"e_1_3_3_3_25_2","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2016.90"},{"key":"e_1_3_3_3_26_2","doi-asserted-by":"crossref","unstructured":"Eyke H\u00fcllermeier and Willem Waegeman. 2021. Aleatoric and epistemic uncertainty in machine learning: An introduction to concepts and methods. Machine Learning 110 (2021) 457\u2013506.","DOI":"10.1007\/s10994-021-05946-3"},{"key":"e_1_3_3_3_27_2","doi-asserted-by":"crossref","unstructured":"Tim Hulsen. 2023. Explainable artificial intelligence (XAI): concepts and challenges in healthcare. AI 4 3 (2023) 652\u2013666.","DOI":"10.3390\/ai4030034"},{"key":"e_1_3_3_3_28_2","doi-asserted-by":"crossref","unstructured":"Vincent Humphrey Matthew Rodell and Annette Eicker. 2023. Using satellite-based terrestrial water storage data: A review. Surveys in Geophysics 44 5 (2023) 1489\u20131517.","DOI":"10.1007\/s10712-022-09754-9"},{"key":"e_1_3_3_3_29_2","unstructured":"Youngseung Jeon Matthew\u00a0K Hong Yan-Ying Chen Kalani Murakami Jonathan Li Xiang\u00a0Anthony Chen and Matthew Klenk. [n. d.]. Weaving ML with Human Aesthetic Assessments to Augment Design Space Exploration. ([n. d.])."},{"key":"e_1_3_3_3_30_2","doi-asserted-by":"crossref","unstructured":"Youngseung Jeon Seung\u00a0Gon Jeon and Kyungsik Han. 2020. Better targeting of consumers: Modeling multifactorial gender and biological sex from Instagram posts. User Modeling and User-Adapted Interaction 30 5 (2020) 833\u2013866.","DOI":"10.1007\/s11257-020-09260-w"},{"key":"e_1_3_3_3_31_2","doi-asserted-by":"publisher","DOI":"10.1145\/3411764.3445093"},{"key":"e_1_3_3_3_32_2","doi-asserted-by":"crossref","unstructured":"Joanne\u00a0M Jordan G\u00a0Fletcher Linder Jordan\u00a0B Renner and John\u00a0G Fryer. 1995. The impact of arthritis in rural populations. Arthritis & Rheumatism: Official Journal of the American College of Rheumatology 8 4 (1995) 242\u2013250.","DOI":"10.1002\/art.1790080407"},{"key":"e_1_3_3_3_33_2","volume-title":"Thinking, fast and slow","author":"Kahneman Daniel","year":"2011","unstructured":"Daniel Kahneman. 2011. Thinking, fast and slow. macmillan."},{"key":"e_1_3_3_3_34_2","doi-asserted-by":"crossref","unstructured":"Daniel Kahneman Shane Frederick et\u00a0al. 2002. Representativeness revisited: Attribute substitution in intuitive judgment. Heuristics and biases: The psychology of intuitive judgment 49 49-81 (2002) 74.","DOI":"10.1017\/CBO9780511808098.004"},{"key":"e_1_3_3_3_35_2","doi-asserted-by":"crossref","unstructured":"Mark\u00a0D Kohn Adam\u00a0A Sassoon and Navin\u00a0D Fernando. 2016. Classifications in brief: Kellgren-Lawrence classification of osteoarthritis. Clinical Orthopaedics and Related Research\u00ae 474 (2016) 1886\u20131893.","DOI":"10.1007\/s11999-016-4732-4"},{"key":"e_1_3_3_3_36_2","doi-asserted-by":"crossref","unstructured":"Ashnil Kumar Shane Dyer Jinman Kim Changyang Li Philip\u00a0HW Leong Michael Fulham and Dagan Feng. 2016. Adapting content-based image retrieval techniques for the semantic annotation of medical images. Computerized Medical Imaging and Graphics 49 (2016) 37\u201345.","DOI":"10.1016\/j.compmedimag.2016.01.001"},{"key":"e_1_3_3_3_37_2","unstructured":"Vivian Lai Chacha Chen Q\u00a0Vera Liao Alison Smith-Renner and Chenhao Tan. 2021. Towards a science of human-ai decision making: a survey of empirical studies. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2112.11471 (2021)."},{"key":"e_1_3_3_3_38_2","doi-asserted-by":"crossref","unstructured":"Jing Lin and Kee\u00a0Yuan Ngiam. 2023. How data science and AI-based technologies impact genomics. Singapore medical journal 64 1 (2023) 59\u201366.","DOI":"10.4103\/singaporemedj.SMJ-2021-438"},{"key":"e_1_3_3_3_39_2","unstructured":"Matthew Lipman. 1987. Critical thinking: What can it be? Analytic Teaching 8 1 (1987)."},{"key":"e_1_3_3_3_40_2","doi-asserted-by":"publisher","DOI":"10.1145\/3563657.3596098"},{"key":"e_1_3_3_3_41_2","unstructured":"Mark Mazumder Colby Banbury Xiaozhe Yao Bojan Karla\u0161 William\u00a0Gaviria Rojas Sudnya Diamos Greg Diamos Lynn He Alicia Parrish Hannah\u00a0Rose Kirk et\u00a0al. 2022. Dataperf: Benchmarks for data-centric ai development. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2207.10062 (2022)."},{"key":"e_1_3_3_3_42_2","doi-asserted-by":"crossref","unstructured":"Eduardo Mosqueira-Rey Elena Hern\u00e1ndez-Pereira David Alonso-R\u00edos Jos\u00e9 Bobes-Bascar\u00e1n and \u00c1ngel Fern\u00e1ndez-Leal. 2023. Human-in-the-loop machine learning: a state of the art. Artificial Intelligence Review 56 4 (2023) 3005\u20133054.","DOI":"10.1007\/s10462-022-10246-w"},{"key":"e_1_3_3_3_43_2","doi-asserted-by":"publisher","DOI":"10.1145\/347642.347697"},{"key":"e_1_3_3_3_44_2","doi-asserted-by":"publisher","DOI":"10.1109\/ICDM.2015.83"},{"key":"e_1_3_3_3_45_2","doi-asserted-by":"publisher","DOI":"10.4018\/978-1-7998-7638-0.ch024"},{"key":"e_1_3_3_3_46_2","unstructured":"Samir Passi and Mihaela Vorvoreanu. 2022. Overreliance on ai literature review. Microsoft Research (2022)."},{"key":"e_1_3_3_3_47_2","doi-asserted-by":"crossref","unstructured":"Luise Pufahl and Mathias Weske. 2019. Batch activity: enhancing business process modeling and enactment with batch processing. Computing 101 12 (2019) 1909\u20131933.","DOI":"10.1007\/s00607-019-00717-4"},{"key":"e_1_3_3_3_48_2","doi-asserted-by":"crossref","unstructured":"Pranav Rajpurkar Emma Chen Oishi Banerjee and Eric\u00a0J Topol. 2022. AI in health and medicine. Nature medicine 28 1 (2022) 31\u201338.","DOI":"10.1038\/s41591-021-01614-0"},{"key":"e_1_3_3_3_49_2","doi-asserted-by":"publisher","DOI":"10.1145\/288392.288596"},{"key":"e_1_3_3_3_50_2","doi-asserted-by":"crossref","unstructured":"Balaji Saibaba Mandeep\u00a0S Dhillon Devendra\u00a0K Chouhan Rajendra\u00a0K Kanojia Mahesh Prakash and Vikas Bachhal. 2015. Significant incidence of extra-articular tibia vara affects radiological outcome of total knee arthroplasty. Knee Surgery & Related Research 27 3 (2015) 173.","DOI":"10.5792\/ksrr.2015.27.3.173"},{"key":"e_1_3_3_3_51_2","unstructured":"Ally Salim\u00a0Jr Megan Allen Kelvin Mariki Kevin\u00a0James Masoy and Jafary Liana. 2023. Understanding how the use of AI decision support tools affect critical thinking and over-reliance on technology by drug dispensers in Tanzania. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2302.09487 (2023)."},{"key":"e_1_3_3_3_52_2","unstructured":"Max Schemmer Patrick Hemmer Niklas K\u00fchl Carina Benz and Gerhard Satzger. 2022. Should I follow AI-based advice? Measuring appropriate reliance in human-AI decision-making. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2204.06916 (2022)."},{"key":"e_1_3_3_3_53_2","doi-asserted-by":"publisher","DOI":"10.1145\/3581641.3584066"},{"key":"e_1_3_3_3_54_2","doi-asserted-by":"publisher","DOI":"10.1117\/12.844207"},{"key":"e_1_3_3_3_55_2","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2017.74"},{"key":"e_1_3_3_3_56_2","unstructured":"Burr Settles. 2009. Active learning literature survey. (2009)."},{"key":"e_1_3_3_3_57_2","unstructured":"Monideepa Tarafdar Cynthia\u00a0M Beath and Jeanne\u00a0W Ross. 2019. Using AI to enhance business operations. MIT Sloan Management Review 60 4 (2019) 37\u201344."},{"key":"e_1_3_3_3_58_2","doi-asserted-by":"crossref","unstructured":"John\u00a0W Tukey. 1949. Comparing individual means in the analysis of variance. Biometrics (1949) 99\u2013114.","DOI":"10.2307\/3001913"},{"key":"e_1_3_3_3_59_2","doi-asserted-by":"crossref","unstructured":"Bas\u00a0HM Van\u00a0der Velden Hugo\u00a0J Kuijf Kenneth\u00a0GA Gilhuijs and Max\u00a0A Viergever. 2022. Explainable artificial intelligence (XAI) in deep learning-based medical image analysis. Medical Image Analysis 79 (2022) 102470.","DOI":"10.1016\/j.media.2022.102470"},{"key":"e_1_3_3_3_60_2","doi-asserted-by":"publisher","DOI":"10.1145\/3397481.3450650"},{"key":"e_1_3_3_3_61_2","doi-asserted-by":"crossref","unstructured":"Maximiliane Wilkesmann and Uwe Wilkesmann. 2011. Knowledge transfer as interaction between experts and novices supported by technology. Vine 41 2 (2011) 96\u2013112.","DOI":"10.1108\/03055721111134763"},{"key":"e_1_3_3_3_62_2","doi-asserted-by":"crossref","unstructured":"Michael Yeomans Anuj Shah Sendhil Mullainathan and Jon Kleinberg. 2019. Making sense of recommendations. Journal of Behavioral Decision Making 32 4 (2019) 403\u2013414.","DOI":"10.1002\/bdm.2118"},{"key":"e_1_3_3_3_63_2","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-59719-1_25"}],"event":{"name":"IUI '25: 30th International Conference on Intelligent User Interfaces","location":"Cagliari Italy","acronym":"IUI '25","sponsor":["SIGAI ACM Special Interest Group on Artificial Intelligence","SIGCHI ACM Special Interest Group on Computer-Human Interaction"]},"container-title":["Proceedings of the 30th International Conference on Intelligent User Interfaces"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3708359.3712161","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3708359.3712161","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,19]],"date-time":"2025-06-19T01:57:07Z","timestamp":1750298227000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3708359.3712161"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,3,24]]},"references-count":62,"alternative-id":["10.1145\/3708359.3712161","10.1145\/3708359"],"URL":"https:\/\/doi.org\/10.1145\/3708359.3712161","relation":{},"subject":[],"published":{"date-parts":[[2025,3,24]]},"assertion":[{"value":"2025-03-24","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}