{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,2]],"date-time":"2026-07-02T23:01:46Z","timestamp":1783033306082,"version":"3.54.6"},"reference-count":42,"publisher":"Springer Science and Business Media LLC","issue":"1","license":[{"start":{"date-parts":[[2021,2,19]],"date-time":"2021-02-19T00:00:00Z","timestamp":1613692800000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0"},{"start":{"date-parts":[[2021,2,19]],"date-time":"2021-02-19T00:00:00Z","timestamp":1613692800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0"}],"funder":[{"DOI":"10.13039\/501100000038","name":"Gouvernement du Canada | Natural Sciences and Engineering Research Council of Canada","doi-asserted-by":"publisher","id":[{"id":"10.13039\/501100000038","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100006112","name":"Microsoft Research","doi-asserted-by":"publisher","id":[{"id":"10.13039\/100006112","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["npj Digit. Med."],"abstract":"<jats:title>Abstract<\/jats:title><jats:p>Artificial intelligence (AI) models for decision support have been developed for clinical settings such as radiology, but little work evaluates the potential impact of such systems. In this study, physicians received chest X-rays and diagnostic advice, some of which was inaccurate, and were asked to evaluate advice quality and make diagnoses. All advice was generated by human experts, but some was labeled as coming from an AI system. As a group, radiologists rated advice as lower quality when it appeared to come from an AI system; physicians with less task-expertise did not. Diagnostic accuracy was significantly worse when participants received inaccurate advice, regardless of the purported source. This work raises important considerations for how advice, AI and non-AI, should be deployed in clinical environments.<\/jats:p>","DOI":"10.1038\/s41746-021-00385-9","type":"journal-article","created":{"date-parts":[[2021,2,20]],"date-time":"2021-02-20T16:02:53Z","timestamp":1613836973000},"update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":399,"title":["Do as AI say: susceptibility in deployment of clinical decision-aids"],"prefix":"10.1038","volume":"4","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-1633-4772","authenticated-orcid":false,"given":"Susanne","family":"Gaube","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-9769-4947","authenticated-orcid":false,"given":"Harini","family":"Suresh","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-7443-1829","authenticated-orcid":false,"given":"Martina","family":"Raue","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Alexander","family":"Merritt","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Seth J.","family":"Berkowitz","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-6600-9580","authenticated-orcid":false,"given":"Eva","family":"Lermer","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Joseph F.","family":"Coughlin","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"John V.","family":"Guttag","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-3771-7975","authenticated-orcid":false,"given":"Errol","family":"Colak","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-6349-7251","authenticated-orcid":false,"given":"Marzyeh","family":"Ghassemi","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2021,2,19]]},"reference":[{"key":"385_CR1","doi-asserted-by":"publisher","first-page":"1347","DOI":"10.1056\/NEJMra1814259","volume":"380","author":"A Rajkomar","year":"2019","unstructured":"Rajkomar, A., Dean, J. & Kohane, I. Machine learning in medicine. N. Engl. J. Med. 380, 1347\u20131358 (2019).","journal-title":"N. Engl. J. Med."},{"key":"385_CR2","doi-asserted-by":"publisher","first-page":"44","DOI":"10.1038\/s41591-018-0300-7","volume":"25","author":"EJ Topol","year":"2019","unstructured":"Topol, E. J. High-performance medicine: the convergence of human and artificial intelligence. Nat. Med. 25, 44\u201356 (2019).","journal-title":"Nat. Med."},{"key":"385_CR3","doi-asserted-by":"publisher","first-page":"719","DOI":"10.1038\/s41551-018-0305-z","volume":"2","author":"K-H Yu","year":"2018","unstructured":"Yu, K.-H., Beam, A. L. & Kohane, I. S. Artificial intelligence in healthcare. Nat. Biomed. Eng. 2, 719\u2013731 (2018).","journal-title":"Nat. Biomed. Eng."},{"key":"385_CR4","doi-asserted-by":"publisher","first-page":"2402","DOI":"10.1001\/jama.2016.17216","volume":"316","author":"V Gulshan","year":"2016","unstructured":"Gulshan, V. et al. Development and validation of a deep learning algorithm for detection of diabetic retinopathy in retinal fundus photographs. JAMA 316, 2402\u20132410 (2016).","journal-title":"JAMA"},{"key":"385_CR5","doi-asserted-by":"publisher","first-page":"115","DOI":"10.1038\/nature21056","volume":"542","author":"A Esteva","year":"2017","unstructured":"Esteva, A. et al. Dermatologist-level classification of skin cancer with deep neural networks. Nature 542, 115\u2013118 (2017).","journal-title":"Nature"},{"key":"385_CR6","doi-asserted-by":"publisher","first-page":"1719","DOI":"10.1016\/S0140-6736(17)30568-8","volume":"389","author":"P Brocklehurst","year":"2017","unstructured":"Brocklehurst, P. et al. Computerised interpretation of fetal heart rate during labour (INFANT): a randomised controlled trial. Lancet 389, 1719\u20131729 (2017).","journal-title":"Lancet"},{"key":"385_CR7","doi-asserted-by":"publisher","first-page":"83","DOI":"10.1097\/AOG.0000000000001799","volume":"129","author":"I Nunes","year":"2017","unstructured":"Nunes, I. et al. Central fetal monitoring with and without computer analysis: a randomized controlled trial. Obstet. Gynecol. 129, 83\u201390 (2017).","journal-title":"Obstet. Gynecol."},{"key":"385_CR8","doi-asserted-by":"publisher","first-page":"425","DOI":"10.1109\/THMS.2017.2648849","volume":"47","author":"P Robinette","year":"2017","unstructured":"Robinette, P., Howard, A. M. & Wagner, A. R. Effect of robot performance on human\u2013robot trust in time-critical situations. IEEE Trans. Hum.-Mach. Syst. 47, 425\u2013436 (2017).","journal-title":"IEEE Trans. Hum.-Mach. Syst."},{"key":"385_CR9","doi-asserted-by":"crossref","unstructured":"Bussone, A., Stumpf, S. & O\u2019Sullivan, D. The role of explanations on trust and reliance in clinical decision support systems. In Proc. 2015 International Conference on Healthcare Informatics 160\u2013169 (IEEE, 2015).","DOI":"10.1109\/ICHI.2015.26"},{"key":"385_CR10","doi-asserted-by":"publisher","first-page":"23","DOI":"10.1038\/s41746-020-0232-8","volume":"3","author":"A Kiani","year":"2020","unstructured":"Kiani, A. et al. Impact of a deep learning assistant on the histopathologic classification of liver cancer.npj Digit. Med. 3, 23 (2020).","journal-title":"npj Digit. Med."},{"key":"385_CR11","doi-asserted-by":"publisher","first-page":"909","DOI":"10.1016\/j.acra.2004.05.012","volume":"11","author":"E Alberdi","year":"2004","unstructured":"Alberdi, E., Povykalo, A., Strigini, L. & Ayton, P. Effects of incorrect computer-aided detection (CAD) output on human decision-making in mammography. Acad. Radiol. 11, 909\u2013918 (2004).","journal-title":"Acad. Radiol."},{"key":"385_CR12","doi-asserted-by":"publisher","first-page":"114","DOI":"10.1037\/xge0000033","volume":"144","author":"BJ Dietvorst","year":"2015","unstructured":"Dietvorst, B. J., Simmons, J. P. & Massey, C. Algorithm aversion: people erroneously avoid algorithms after seeing them err. J. Exp. Psychol. Gen. 144, 114\u2013126 (2015).","journal-title":"J. Exp. Psychol. Gen."},{"key":"385_CR13","doi-asserted-by":"publisher","first-page":"209","DOI":"10.1111\/j.1468-2389.2011.00548.x","volume":"19","author":"DL Diab","year":"2011","unstructured":"Diab, D. L., Pui, S.-Y., Yankelevich, M. & Highhouse, S. Lay perceptions of selection decision aids in US and non-US samples. Int. J. Sel. Assess. 19, 209\u2013216 (2011).","journal-title":"Int. J. Sel. Assess."},{"key":"385_CR14","doi-asserted-by":"publisher","first-page":"455","DOI":"10.1002\/bdm.542","volume":"19","author":"M Promberger","year":"2006","unstructured":"Promberger, M. & Baron, J. Do patients trust computers? J. Behav. Decis. Making 19, 455\u2013468 (2006).","journal-title":"J. Behav. Decis. Making"},{"key":"385_CR15","doi-asserted-by":"publisher","first-page":"90","DOI":"10.1016\/j.obhdp.2018.12.005","volume":"151","author":"JM Logg","year":"2019","unstructured":"Logg, J. M., Minson, J. A. & Moore, D. A. Algorithm appreciation: people prefer algorithmic to human judgment. Organ. Behav. Hum. Decis. Process. 151, 90\u2013103 (2019).","journal-title":"Organ. Behav. Hum. Decis. Process."},{"key":"385_CR16","doi-asserted-by":"publisher","first-page":"155","DOI":"10.1080\/014492998119526","volume":"17","author":"JJ Dijkstra","year":"1998","unstructured":"Dijkstra, J. J., Liebrand, W. B. G. & Timminga, E. Persuasiveness of expert systems. Behav. Inform. Technol. 17, 155\u2013163 (1998).","journal-title":"Behav. Inform. Technol."},{"key":"385_CR17","doi-asserted-by":"publisher","first-page":"98","DOI":"10.1177\/0272989X12465490","volume":"33","author":"AA Povyakalo","year":"2013","unstructured":"Povyakalo, A. A., Alberdi, E., Strigini, L. & Ayton, P. How to discriminate between computer-aided and computer-hindered decisions: a case study in mammography. Med. Decis. Making 33, 98\u2013107 (2013).","journal-title":"Med. Decis. Making"},{"key":"385_CR18","doi-asserted-by":"publisher","first-page":"41","DOI":"10.1038\/s41746-020-0253-3","volume":"3","author":"MP Sendak","year":"2020","unstructured":"Sendak, M. P., Gao, M., Brajer, N. & Balu, S. Presenting machine learning model information to clinical end users with model facts labels.npj Digit. Med. 3, 41 (2020).","journal-title":"npj Digit. Med."},{"key":"385_CR19","doi-asserted-by":"publisher","first-page":"47","DOI":"10.1038\/s41746-020-0254-2","volume":"3","author":"CM Cutillo","year":"2020","unstructured":"Cutillo, C. M. et al. Machine intelligence in healthcare\u2014perspectives on trustworthiness, explainability, usability, and transparency.npj Digit. Med. 3, 47 (2020).","journal-title":"npj Digit. Med."},{"key":"385_CR20","doi-asserted-by":"crossref","unstructured":"Beede, E. et al. A human-centered evaluation of a deep learning system deployed in clinics for the detection of diabetic retinopathy. In Proc. CHI Conference on Human Factors in Computing Systems (CHI \u201820) 1\u201312 (ACM, 2020).","DOI":"10.1145\/3313831.3376718"},{"key":"385_CR21","doi-asserted-by":"publisher","first-page":"2","DOI":"10.1016\/j.cgh.2015.09.026","volume":"14","author":"M Lin","year":"2016","unstructured":"Lin, M., Pappas, S. C., Sellin, J. & El-Serag, H. B. Curbside consultations: the good, the bad, and the ugly. Clin. Gastroenterol. Hepatol. 14, 2\u20134 (2016).","journal-title":"Clin. Gastroenterol. Hepatol."},{"key":"385_CR22","doi-asserted-by":"publisher","first-page":"156","DOI":"10.1016\/j.jacr.2015.09.029","volume":"13","author":"MT Lu","year":"2016","unstructured":"Lu, M. T. et al. Radiologist point-of-care clinical decision support and adherence to guidelines for incidental lung nodules. J. Am. Coll. Radiol. 13, 156\u2013162 (2016).","journal-title":"J. Am. Coll. Radiol."},{"key":"385_CR23","doi-asserted-by":"publisher","first-page":"60","DOI":"10.1148\/radiol.2019182716","volume":"292","author":"A Yala","year":"2019","unstructured":"Yala, A., Lehman, C., Schuster, T., Portnoi, T. & Barzilay, R. A deep learning mammography-based model for improved breast cancer risk prediction. Radiology 292, 60\u201366 (2019).","journal-title":"Radiology"},{"key":"385_CR24","doi-asserted-by":"publisher","first-page":"574","DOI":"10.1148\/radiol.2017162326","volume":"284","author":"P Lakhani","year":"2017","unstructured":"Lakhani, P. & Sundaram, B. Deep learning at chest radiography: automated classification of pulmonary tuberculosis by using convolutional neural networks. Radiology 284, 574\u2013582 (2017).","journal-title":"Radiology"},{"key":"385_CR25","doi-asserted-by":"publisher","first-page":"97","DOI":"10.1097\/01.cpm.0000156704.33941.e2","volume":"12","author":"JP Kanne","year":"2005","unstructured":"Kanne, J. P., Thoongsuwan, N. & Stern, E. J. Common errors and pitfalls in interpretation of the adult chest radiograph. Clin. Pulm. Med. 12, 97\u2013114 (2005).","journal-title":"Clin. Pulm. Med."},{"key":"385_CR26","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1097\/01.CDR.0000406727.37058.bc","volume":"34","author":"KL Humphrey","year":"2011","unstructured":"Humphrey, K. L. et al. Where are they all hiding? Common blind spots on chest radiography. Contemp. Diagn. Radiol. 34, 1\u20135 (2011).","journal-title":"Contemp. Diagn. Radiol."},{"key":"385_CR27","doi-asserted-by":"publisher","first-page":"197","DOI":"10.1053\/j.ro.2015.01.008","volume":"50","author":"PM de Groot","year":"2015","unstructured":"de Groot, P. M., de Groot, P. M., Carter, B. W., Abbott, G. F. & Wu, C. C. Pitfalls in chest radiographic interpretation: blind spots. Semin. Roentgenol. 50, 197\u2013209 (2015).","journal-title":"Semin. Roentgenol."},{"key":"385_CR28","doi-asserted-by":"publisher","first-page":"810","DOI":"10.1126\/science.aaw0029","volume":"363","author":"RB Parikh","year":"2019","unstructured":"Parikh, R. B., Obermeyer, Z. & Navathe, A. S. Regulation of predictive analytics in medicine. Science 363, 810\u2013812 (2019).","journal-title":"Science"},{"key":"385_CR29","doi-asserted-by":"publisher","first-page":"629","DOI":"10.1093\/jcr\/ucz013","volume":"46","author":"C Longoni","year":"2019","unstructured":"Longoni, C., Bonezzi, A. & Morewedge, C. K. Resistance to medical artificial intelligence. J. Consum. Res. 46, 629\u2013650 (2019).","journal-title":"J. Consum. Res."},{"key":"385_CR30","doi-asserted-by":"publisher","first-page":"35","DOI":"10.1016\/j.socec.2010.10.008","volume":"40","author":"A Furnham","year":"2011","unstructured":"Furnham, A. & Boo, H. C. A literature review of the anchoring effect. J. Socio-Econ. 40, 35\u201342 (2011).","journal-title":"J. Socio-Econ."},{"key":"385_CR31","doi-asserted-by":"publisher","first-page":"611","DOI":"10.2214\/AJR.12.10375","volume":"201","author":"CS Lee","year":"2013","unstructured":"Lee, C. S., Nagy, P. G., Weaver, S. J. & Newman-Toker, D. E. Cognitive and system factors contributing to diagnostic errors in radiology. AJR Am. J. Roentgenol. 201, 611\u2013617 (2013).","journal-title":"AJR Am. J. Roentgenol."},{"key":"385_CR32","doi-asserted-by":"publisher","first-page":"236","DOI":"10.1148\/rg.2018170107","volume":"38","author":"LP Busby","year":"2018","unstructured":"Busby, L. P., Courtier, J. L. & Glastonbury, C. M. Bias in radiology: the how and why of misses and misinterpretations. Radiographics 38, 236\u2013247 (2018).","journal-title":"Radiographics"},{"key":"385_CR33","doi-asserted-by":"crossref","unstructured":"Howard, A. Are we trusting AI too much? Examining human-robot interactions in the real world. In Proc. 2020 ACM\/IEEE International Conference on Human-Robot Interaction 1 (ACM, 2020).","DOI":"10.1145\/3319502.3374842"},{"key":"385_CR34","doi-asserted-by":"publisher","first-page":"46","DOI":"10.1109\/MRA.2017.2778743","volume":"25","author":"J Borenstein","year":"2018","unstructured":"Borenstein, J., Wagner, A. R. & Howard, A. Overtrust of pediatric health-care robots: a preliminary survey of parent perspectives. IEEE Robot. Autom. Mag. 25, 46\u201354 (2018).","journal-title":"IEEE Robot. Autom. Mag."},{"key":"385_CR35","doi-asserted-by":"publisher","first-page":"260","DOI":"10.1006\/obhd.2000.2909","volume":"83","author":"Yaniv, I.","year":"2000","unstructured":"Yaniv, I. & Kleinberger, E. Advice taking in decision making: egocentric discounting and reputation formation. Organ. Behav. Hum. Decis. Process. 83, 260\u2013281 (2000).","journal-title":"Organ. Behav. Hum. Decis. Process."},{"key":"385_CR36","doi-asserted-by":"crossref","unstructured":"Ye, S. et al. Human trust after robot mistakes: study of the effects of different forms of robot communication. In Proc. 2019 28th IEEE International Conference on Robot and Human Interactive Communication (RO-MAN) 1\u20137 (IEEE, 2019).","DOI":"10.1109\/RO-MAN46459.2019.8956424"},{"key":"385_CR37","doi-asserted-by":"publisher","first-page":"504","DOI":"10.1177\/0956797617739369","volume":"29","author":"C Gaertig","year":"2018","unstructured":"Gaertig, C. & Simmons, J. P. Do people inherently dislike uncertain advice? Psychol. Sci. 29, 504\u2013520 (2018).","journal-title":"Psychol. Sci."},{"key":"385_CR38","doi-asserted-by":"crossref","unstructured":"Paulhus, D. L. Socially desirable responding on self-reports. In Encyclopedia of Personality and Individual Differences (eds. Zeigler-Hill, V. & Shackelford, T.) 1\u20135 (Springer, Cham, 2017).","DOI":"10.1007\/978-3-319-28099-8_1349-1"},{"key":"385_CR39","doi-asserted-by":"publisher","DOI":"10.1371\/journal.pone.0191943","volume":"13","author":"DA Cook","year":"2018","unstructured":"Cook, D. A. et al. Practice variation and practice guidelines: attitudes of generalist and specialist physicians, nurse practitioners, and physician assistants. PLoS One. 13, e0191943 (2018).","journal-title":"PLoS One."},{"key":"385_CR40","doi-asserted-by":"crossref","unstructured":"Cai, C. J., Winter, S., Steiner, D., Wilcox, L. & Terry, M. \u201cHello AI\u201d: Uncovering the onboarding needs of medical practitioners for human-AI collaborative decision-making. In Proc. ACM on Human-Computer Interaction 104 (ACM, 2019).","DOI":"10.1145\/3359206"},{"key":"385_CR41","doi-asserted-by":"publisher","first-page":"1229","DOI":"10.1038\/s41591-020-0942-0","volume":"26","author":"P Tschandl","year":"2020","unstructured":"Tschandl, P. et al. Human\u2013computer collaboration for skin cancer recognition. Nat. Med. 26, 1229\u20131234 (2020).","journal-title":"Nat. Med."},{"key":"385_CR42","unstructured":"Association of American Medical Colleges. Center for Workforce Studies. 2018 Physician Specialty Data Book. https:\/\/www.aamc.org\/data-reports\/workforce\/report\/physician-specialty-data-report (2018)."}],"container-title":["npj Digital Medicine"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/www.nature.com\/articles\/s41746-021-00385-9.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/www.nature.com\/articles\/s41746-021-00385-9","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/www.nature.com\/articles\/s41746-021-00385-9.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2022,12,3]],"date-time":"2022-12-03T18:44:36Z","timestamp":1670093076000},"score":1,"resource":{"primary":{"URL":"https:\/\/www.nature.com\/articles\/s41746-021-00385-9"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2021,2,19]]},"references-count":42,"journal-issue":{"issue":"1","published-online":{"date-parts":[[2021,12]]}},"alternative-id":["385"],"URL":"https:\/\/doi.org\/10.1038\/s41746-021-00385-9","relation":{},"ISSN":["2398-6352"],"issn-type":[{"value":"2398-6352","type":"electronic"}],"subject":[],"published":{"date-parts":[[2021,2,19]]},"assertion":[{"value":"21 July 2020","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"7 January 2021","order":2,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"19 February 2021","order":3,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"The authors declare no competing interests.","order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Competing interests"}}],"article-number":"31"}}