{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,6,24]],"date-time":"2026-06-24T23:43:56Z","timestamp":1782344636713,"version":"3.54.5"},"reference-count":53,"publisher":"Springer Science and Business Media LLC","issue":"7","license":[{"start":{"date-parts":[[2021,5,31]],"date-time":"2021-05-31T00:00:00Z","timestamp":1622419200000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2021,5,31]],"date-time":"2021-05-31T00:00:00Z","timestamp":1622419200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"funder":[{"DOI":"10.13039\/100000153","name":"NSF | BIO | Division of Biological Infrastructure","doi-asserted-by":"publisher","award":["DBI-1552309"],"award-info":[{"award-number":["DBI-1552309"]}],"id":[{"id":"10.13039\/100000153","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000002","name":"U.S. Department of Health & Human Services | National Institutes of Health","doi-asserted-by":"publisher","award":["R35 GM 128638"],"award-info":[{"award-number":["R35 GM 128638"]}],"id":[{"id":"10.13039\/100000002","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000002","name":"U.S. Department of Health & Human Services | National Institutes of Health","doi-asserted-by":"publisher","award":["R01 AG 061132"],"award-info":[{"award-number":["R01 AG 061132"]}],"id":[{"id":"10.13039\/100000002","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Nat Mach Intell"],"DOI":"10.1038\/s42256-021-00338-7","type":"journal-article","created":{"date-parts":[[2021,5,31]],"date-time":"2021-05-31T17:11:59Z","timestamp":1622481119000},"page":"610-619","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":450,"title":["AI for radiographic COVID-19 detection selects shortcuts over signal"],"prefix":"10.1038","volume":"3","author":[{"ORCID":"https:\/\/orcid.org\/0000-0001-9933-6273","authenticated-orcid":false,"given":"Alex J.","family":"DeGrave","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-1804-7133","authenticated-orcid":false,"given":"Joseph D.","family":"Janizek","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-5833-5215","authenticated-orcid":false,"given":"Su-In","family":"Lee","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2021,5,31]]},"reference":[{"key":"338_CR1","doi-asserted-by":"publisher","unstructured":"Mossa-Basha, M. et al. Policies and guidelines for COVID-19 preparedness: experiences from the University of Washington. Radiology https:\/\/doi.org\/10.1148\/radiol.2020201326 (2020).","DOI":"10.1148\/radiol.2020201326"},{"key":"338_CR2","doi-asserted-by":"publisher","first-page":"3","DOI":"10.1148\/ryai.2020200053","volume":"2","author":"S Kundu","year":"2020","unstructured":"Kundu, S., Elhalawani, H., Gichoya, J. W. & Kahn, C. E.Jr How might AI and chest imaging help unravel COVID-19\u2019s mysteries? Radiol. Artificial Intell 2, 3 (2020).","journal-title":"Radiol. Artificial Intell"},{"key":"338_CR3","unstructured":"Ghoshal, B. & Tucker, A. Estimating uncertainty and interpretability in deep learning for coronavirus (COVID-19) detection. Preprint at https:\/\/arxiv.org\/pdf\/2003.10769.pdf (2020)."},{"key":"338_CR4","doi-asserted-by":"publisher","DOI":"10.1038\/s41598-020-76550-z","volume":"10","author":"L Wang","year":"2020","unstructured":"Wang, L., Lin, Z. Q. & Wong, A. COVID-Net: a tailored deep convolutional neural network design for detection of COVID-19 cases from chest X-ray images. Sci. Rep. 10, 19549 (2020).","journal-title":"Sci. Rep."},{"key":"338_CR5","unstructured":"Hemdan, E. E.-D., Shouman, M. A. & Karar, M. E. COVIDX-Net: a framework of deep learning classifiers to diagnose COVID-19 in X-ray images. Preprint at https:\/\/arxiv.org\/pdf\/2003.11055.pdf (2020)."},{"key":"338_CR6","doi-asserted-by":"publisher","first-page":"103792","DOI":"10.1016\/j.compbiomed.2020.103792","volume":"121","author":"T Ozturk","year":"2020","unstructured":"Ozturk, T. et al. Automated detection of COVID-19 cases using deep neural networks with X-ray images. Comput. Biol. Med. 121, 103792 (2020).","journal-title":"Comput. Biol. Med."},{"key":"338_CR7","doi-asserted-by":"publisher","first-page":"105608","DOI":"10.1016\/j.cmpb.2020.105608","volume":"196","author":"L Brunese","year":"2020","unstructured":"Brunese, L., Mercaldo, F., Reginelli, A. & Santone, A. Explainable deep learning for pulmonary disease and coronavirus COVID-19 detection from X-rays. Comput. Methods Programs Biomed. 196, 105608 (2020).","journal-title":"Comput. Methods Programs Biomed."},{"key":"338_CR8","doi-asserted-by":"crossref","unstructured":"Karim, M. et al. DeepCOVIDExplainer: explainable COVID-19 predictions based on chest X-ray images. Preprint at https:\/\/arxiv.org\/pdf\/2004.04582.pdf (2020).","DOI":"10.1109\/BIBM49941.2020.9313304"},{"key":"338_CR9","doi-asserted-by":"publisher","first-page":"e225","DOI":"10.1016\/S2589-7500(20)30079-0","volume":"2","author":"A Laghi","year":"2020","unstructured":"Laghi, A. Cautions about radiologic diagnosis of COVID-19 infection driven by artificial intelligence. Lancet Digit. Health 2, e225 (2020).","journal-title":"Lancet Digit. Health"},{"key":"338_CR10","doi-asserted-by":"publisher","first-page":"665","DOI":"10.1038\/s42256-020-00257-z","volume":"2","author":"R Geirhos","year":"2020","unstructured":"Geirhos, R. et al. Shortcut learning in deep neural networks. Nat. Mach. Intell. 2, 665\u2013673 (2020).","journal-title":"Nat. Mach. Intell."},{"key":"338_CR11","doi-asserted-by":"publisher","DOI":"10.1038\/s41467-020-17971-2","volume":"11","author":"SA Harmon","year":"2020","unstructured":"Harmon, S. A. et al. Artificial intelligence for the detection of COVID-19 pneumonia on chest CT using multinational datasets. Nat. Commun. 11, 4080 (2020).","journal-title":"Nat. Commun."},{"key":"338_CR12","doi-asserted-by":"publisher","first-page":"574","DOI":"10.1148\/radiol.2017162326","volume":"284","author":"P Lakhani","year":"2017","unstructured":"Lakhani, P. & Sundaram, B. Deep learning at chest radiography: automated classification of pulmonary tuberculosis by using convolutional neural networks. Radiology 284, 574\u2013582 (2017).","journal-title":"Radiology"},{"key":"338_CR13","doi-asserted-by":"publisher","first-page":"105351","DOI":"10.1016\/j.cmpb.2020.105351","volume":"190","author":"MA Al-Masni","year":"2020","unstructured":"Al-Masni, M. A., Kim, D.-H. & Kim, T.-S. Multiple skin lesions diagnostics via integrated deep convolutional networks for segmentation and classification. Comput. Methods Programs Biomed. 190, 105351 (2020).","journal-title":"Comput. Methods Programs Biomed."},{"key":"338_CR14","doi-asserted-by":"crossref","unstructured":"Huang, G., Liu, Z., Van Der Maaten, L. & Weinberger, K. Q. Densely connected convolutional networks. In Proc. IEEE Conference on Computer Vision and Pattern Recognition 4700\u20134708 (IEEE, 2017).","DOI":"10.1109\/CVPR.2017.243"},{"key":"338_CR15","unstructured":"Krizhevsky, A., Sutskever, I. & Hinton, G. E. ImageNet classification with deep convolutional neural networks. In Proc. 25th International Conference on Neural Information Processing Systems 1097\u20131105 (ACM, 2012)."},{"key":"338_CR16","unstructured":"Sundararajan, M., Taly, A. & Yan, Q. Axiomatic attribution for deep networks. In Proc. 34th International Conference on Machine Learning Vol. 70, 3319\u20133328 (PMLR, 2017)."},{"key":"338_CR17","doi-asserted-by":"crossref","unstructured":"Zhu, J.-Y., Park, T., Isola, P. & Efros, A. A. Unpaired image-to-image translation using cycle-consistent adversarial networks. In Proc. 2017 IEEE International Conference on Computer Vision 2242\u20132251 (IEEE, 2017).","DOI":"10.1109\/ICCV.2017.244"},{"key":"338_CR18","unstructured":"Singla, S., Pollack, B., Chen, J. & Batmanghelich, K. Explanation by progressive exaggeration. In International Conference on Learning Representations (2019)."},{"key":"338_CR19","doi-asserted-by":"publisher","unstructured":"Erion, G., Janizek, J. D., Sturmfels, P., Lundberg, S. M. & Lee, S.-I. Improving performance of deep learning models with axiomatic attribution priors and expected gradients. Nat. Mach. Intell. https:\/\/doi.org\/10.1038\/s42256-021-00343-w (2021).","DOI":"10.1038\/s42256-021-00343-w"},{"key":"338_CR20","doi-asserted-by":"publisher","unstructured":"Zhang, R. et al. Diagnosis of COVID-19 pneumonia using chest radiography: value of artificial intelligence. Radiology https:\/\/doi.org\/10.1148\/radiol.2020202944 (2020).","DOI":"10.1148\/radiol.2020202944"},{"key":"338_CR21","doi-asserted-by":"crossref","unstructured":"Sandler, M., Howard, A., Zhu, M., Zhmoginov, A. & Chen, L.-C. MobileNetV2: inverted residuals and linear bottlenecks. In Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition 4510\u20134520 (IEEE, 2018).","DOI":"10.1109\/CVPR.2018.00474"},{"key":"338_CR22","doi-asserted-by":"crossref","unstructured":"Xie, S., Girshick, R., Doll\u00e1r, P., Tu, Z. & He, K. Aggregated residual transformations for deep neural networks. In Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition 5987\u20135995 (IEEE, 2017).","DOI":"10.1109\/CVPR.2017.634"},{"key":"338_CR23","unstructured":"Cohen, J. P., Morrison, P. & Dao, L. COVID-19 image data collection. GitHub https:\/\/github.com\/ieee8023\/covid-chestxray-dataset"},{"key":"338_CR24","doi-asserted-by":"crossref","unstructured":"Wang, X. et al. ChestX-ray8: hospital-scale chest X-ray database and benchmarks on weakly-supervised classification and localization of common thorax diseases. In Proc. IEEE Conference on Computer Vision and Pattern Recognition 2097\u20132106 (IEEE, 2017).","DOI":"10.1109\/CVPR.2017.369"},{"key":"338_CR25","unstructured":"Radiological Society of North America. RSNA pneumonia detection challenge. kaggle https:\/\/www.kaggle.com\/c\/rsna-pneumonia-detection-challenge\/data"},{"key":"338_CR26","doi-asserted-by":"publisher","unstructured":"Wehbe, R. M. et al. DeepCOVID-XR: an artificial intelligence algorithm to detect COVID-19 on chest radiographs trained and tested on a large US clinical dataset. Radiology https:\/\/doi.org\/10.1148\/radiol.2020203511 (2020).","DOI":"10.1148\/radiol.2020203511"},{"key":"338_CR27","doi-asserted-by":"publisher","first-page":"e200079","DOI":"10.1148\/ryai.2020200079","volume":"2","author":"MD Li","year":"2020","unstructured":"Li, M. D. et al. Automated assessment and tracking of COVID-19 pulmonary disease severity on chest radiographs using convolutional Siamese neural networks. Radiol. Artif. Intell. 2, e200079 (2020).","journal-title":"Radiol. Artif. Intell."},{"key":"338_CR28","doi-asserted-by":"publisher","first-page":"E166","DOI":"10.1148\/radiol.2020201874","volume":"296","author":"K Murphy","year":"2020","unstructured":"Murphy, K. et al. COVID-19 on chest radiographs: a multireader evaluation of an artificial intelligence system. Radiology 296, E166\u2013E172 (2020).","journal-title":"Radiology"},{"key":"338_CR29","doi-asserted-by":"publisher","first-page":"101797","DOI":"10.1016\/j.media.2020.101797","volume":"66","author":"A Bustos","year":"2020","unstructured":"Bustos, A., Pertusa, A., Salinas, J.-M. & de la Iglesia-Vay\u00e1, M. PadChest: a large chest X-ray image dataset with multi-label annotated reports. Med. Image Anal. 66, 101797 (2020).","journal-title":"Med. Image Anal."},{"key":"338_CR30","unstructured":"Vay\u00e1, M. d. l. I. et al. BIMCV COVID-19+: a large annotated dataset of RX and CT images from COVID-19 patients. Preprint at https:\/\/arxiv.org\/pdf\/2006.01174.pdf (2020)."},{"key":"338_CR31","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1016\/j.inffus.2021.04.008","volume":"76","author":"G Maguolo","year":"2021","unstructured":"Maguolo, G. & Nanni, L. A critic evaluation of methods for COVID-19 automatic detection from X-ray images. Inform. Fusion 76, 1\u20137 (2021).","journal-title":"Inform. Fusion"},{"key":"338_CR32","doi-asserted-by":"publisher","first-page":"3673","DOI":"10.1038\/s41467-020-17478-w","volume":"11","author":"DC Castro","year":"2020","unstructured":"Castro, D. C., Walker, I. & Glocker, B. Causality matters in medical imaging. Nat. Commun. 11, 3673 (2020).","journal-title":"Nat. Commun."},{"key":"338_CR33","doi-asserted-by":"publisher","first-page":"3923","DOI":"10.1038\/s41467-020-17419-7","volume":"11","author":"JG Richens","year":"2020","unstructured":"Richens, J. G., Lee, C. M. & Johri, S. Improving the accuracy of medical diagnosis with causal machine learning. Nat. Commun. 11, 3923 (2020).","journal-title":"Nat. Commun."},{"key":"338_CR34","doi-asserted-by":"crossref","unstructured":"Janizek, J. D., Erion, G., DeGrave, A. J. & Lee, S.-I. An adversarial approach for the robust classification of pneumonia from chest radiographs. In Proc. ACM Conference on Health, Inference and Learning 69\u201379 (ACM, 2020).","DOI":"10.1145\/3368555.3384458"},{"key":"338_CR35","first-page":"2096","volume":"17","author":"Y Ganin","year":"2016","unstructured":"Ganin, Y. et al. Domain-adversarial training of neural networks. J. Mach. Learn. Res. 17, 2096\u20132030 (2016).","journal-title":"J. Mach. Learn. Res."},{"key":"338_CR36","unstructured":"Sagawa, S., Raghunathan, A., Koh, P. W. & Liang, P. An investigation of why overparameterization exacerbates spurious correlations. In Proc. 37th International Conference on Machine Learning (ICML) Vol. 119, 8346\u20138356 (PMLR, 2020)."},{"key":"338_CR37","doi-asserted-by":"publisher","first-page":"13590","DOI":"10.1038\/s41598-020-70479-z","volume":"10","author":"KK Bressem","year":"2020","unstructured":"Bressem, K. K. et al. Comparing different deep learning architectures for classification of chest radiographs. Sci. Rep. 10, 13590 (2020).","journal-title":"Sci. Rep."},{"key":"338_CR38","doi-asserted-by":"crossref","unstructured":"Quionero-Candela, J., Sugiyama, M., Schwaighofer, A. & Lawrence, N. D. Dataset Shift in Machine Learning (MIT Press, 2009).","DOI":"10.7551\/mitpress\/9780262170055.001.0001"},{"key":"338_CR39","unstructured":"Rajpurkar, P. et al. CheXNet: radiologist-level pneumonia detection on chest X-rays with deep learning. Preprint at https:\/\/arxiv.org\/pdf\/1711.05225.pdf (2017)."},{"key":"338_CR40","doi-asserted-by":"publisher","first-page":"18","DOI":"10.1038\/s41551-019-0487-z","volume":"4","author":"A Mitani","year":"2020","unstructured":"Mitani, A. et al. Detection of anaemia from retinal fundus images via deep learning. Nat. Biomed. Eng. 4, 18\u201327 (2020).","journal-title":"Nat. Biomed. Eng."},{"key":"338_CR41","doi-asserted-by":"publisher","first-page":"e1002683","DOI":"10.1371\/journal.pmed.1002683","volume":"15","author":"JR Zech","year":"2018","unstructured":"Zech, J. R. et al. Variable generalization performance of a deep learning model to detect pneumonia in chest radiographs: a cross-sectional study. PLoS Med. 15, e1002683 (2018).","journal-title":"PLoS Med."},{"key":"338_CR42","doi-asserted-by":"publisher","first-page":"e200034","DOI":"10.1148\/ryct.2020200034","volume":"2","author":"M-Y Ng","year":"2020","unstructured":"Ng, M.-Y. et al. Imaging profile of the COVID-19 infection: radiologic findings and literature review. Radiol. Cardiothorac. Imaging 2, e200034 (2020).","journal-title":"Radiol. Cardiothorac. Imaging"},{"key":"338_CR43","doi-asserted-by":"publisher","first-page":"336","DOI":"10.1007\/s11263-019-01228-7","volume":"128","author":"RR Selvaraju","year":"2020","unstructured":"Selvaraju, R. R. et al. Grad-CAM: visual explanations from deep networks via gradient-based localization. Int. J. Comput. Vision 128, 336\u2013359 (2020).","journal-title":"Int. J. Comput. Vision"},{"key":"338_CR44","doi-asserted-by":"publisher","unstructured":"Wong, H. Y. F. et al. Frequency and distribution of chest radiographic findings in COVID-19 positive patients. Radiology https:\/\/doi.org\/10.1148\/radiol.2020201160 (2020).","DOI":"10.1148\/radiol.2020201160"},{"key":"338_CR45","unstructured":"Gale, W., Oakden-Rayner, L., Carneiro, G., Bradley, A. P. & Palmer, L. J. Detecting hip fractures with radiologist-level performance using deep neural networks. Preprint at https:\/\/arxiv.org\/pdf\/1711.06504.pdf (2017)."},{"key":"338_CR46","doi-asserted-by":"crossref","unstructured":"Deng, J. et al. ImageNet: a large-scale hierarchical image database. In Proc. 2009 IEEE Conference on Computer Vision and Pattern Recognition 248\u2013255 (IEEE, 2009).","DOI":"10.1109\/CVPR.2009.5206848"},{"key":"338_CR47","unstructured":"Paszke, A. et al. PyTorch: an imperative style, high-performance deep learning library. In Advances in Neural Information Processing Systems 32 (eds Wallach, H. et al.) 8024\u20138035 (Curran Associates, 2019); http:\/\/papers.neurips.cc\/paper\/9015-pytorch-an-imperative-style-high-performance-deep-learning-library.pdf"},{"key":"338_CR48","doi-asserted-by":"crossref","unstructured":"Cohen, J. P. et al. 2020. COVID-19 image data collection: prospective predictions are the future. GitHub https:\/\/github.com\/ieee8023\/covid-chestxray-dataset","DOI":"10.59275\/j.melba.2020-48g7"},{"key":"338_CR49","doi-asserted-by":"publisher","first-page":"e22","DOI":"10.23915\/distill.00022","volume":"5","author":"P Sturmfels","year":"2020","unstructured":"Sturmfels, P., Lundberg, S. & Lee, S.-I. Visualizing the impact of feature attribution baselines. Distill 5, e22 (2020).","journal-title":"Distill"},{"key":"338_CR50","doi-asserted-by":"crossref","unstructured":"Ribeiro, M. T., Wu, T., Guestrin, C. & Singh, S. Beyond accuracy: behavioral testing of NLP models with checklist. In Proc. 58th Annual Meeting of the Association for Computational Linguistics 4902\u20134912 (Association for Computational Linguistics, 2020); https:\/\/www.aclweb.org\/anthology\/2020.acl-main.442","DOI":"10.18653\/v1\/2020.acl-main.442"},{"key":"338_CR51","doi-asserted-by":"publisher","first-page":"439","DOI":"10.1086\/341527","volume":"71","author":"BV North","year":"2002","unstructured":"North, B. V., Curtis, D. & Sham, P. C. A note on the calculation of empirical P values from Monte Carlo procedures. Am. J. Human Genet. 71, 439\u2013441 (2002).","journal-title":"Am. J. Human Genet."},{"key":"338_CR52","doi-asserted-by":"publisher","unstructured":"Winther, H. et al. COVID-19 image repository. figshare https:\/\/doi.org\/10.6084\/m9.figshare.12275009","DOI":"10.6084\/m9.figshare.12275009"},{"key":"338_CR53","volume":"7","author":"Y-H Jin","year":"2020","unstructured":"Jin, Y.-H. et al. A rapid advice guideline for the diagnosis and treatment of 2019 novel coronavirus (2019-nCoV) infected pneumonia (standard version). Mil. Med. Res. 7, 4 (2020).","journal-title":"Mil. Med. Res."}],"container-title":["Nature Machine Intelligence"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/www.nature.com\/articles\/s42256-021-00338-7.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/www.nature.com\/articles\/s42256-021-00338-7","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/www.nature.com\/articles\/s42256-021-00338-7.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2023,11,4]],"date-time":"2023-11-04T06:24:49Z","timestamp":1699079089000},"score":1,"resource":{"primary":{"URL":"https:\/\/www.nature.com\/articles\/s42256-021-00338-7"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2021,5,31]]},"references-count":53,"journal-issue":{"issue":"7","published-online":{"date-parts":[[2021,7]]}},"alternative-id":["338"],"URL":"https:\/\/doi.org\/10.1038\/s42256-021-00338-7","relation":{"has-preprint":[{"id-type":"doi","id":"10.1101\/2020.09.13.20193565","asserted-by":"object"}]},"ISSN":["2522-5839"],"issn-type":[{"value":"2522-5839","type":"electronic"}],"subject":[],"published":{"date-parts":[[2021,5,31]]},"assertion":[{"value":"25 September 2020","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"29 March 2021","order":2,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"31 May 2021","order":3,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"The authors declare no competing interests.","order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Competing interests"}},{"value":"This content has been made available to all.","name":"free","label":"Free to read"}]}}