{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,5,7]],"date-time":"2026-05-07T11:39:36Z","timestamp":1778153976251,"version":"3.51.4"},"reference-count":38,"publisher":"Springer Science and Business Media LLC","issue":"5","license":[{"start":{"date-parts":[[2024,5,6]],"date-time":"2024-05-06T00:00:00Z","timestamp":1714953600000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2024,5,6]],"date-time":"2024-05-06T00:00:00Z","timestamp":1714953600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"funder":[{"DOI":"10.13039\/100000143","name":"Division of Computing and Communication Foundations","doi-asserted-by":"publisher","award":["2239787"],"award-info":[{"award-number":["2239787"]}],"id":[{"id":"10.13039\/100000143","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100014582","name":"Whiting School of Engineering, Johns Hopkins University","doi-asserted-by":"publisher","id":[{"id":"10.13039\/100014582","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["J Digit Imaging. Inform. med."],"DOI":"10.1007\/s10278-024-01136-4","type":"journal-article","created":{"date-parts":[[2024,5,6]],"date-time":"2024-05-06T18:02:56Z","timestamp":1715018576000},"page":"2490-2499","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":13,"title":["Gradient-Based Saliency Maps Are Not Trustworthy Visual Explanations of Automated AI Musculoskeletal Diagnoses"],"prefix":"10.1007","volume":"37","author":[{"ORCID":"https:\/\/orcid.org\/0000-0001-8153-0115","authenticated-orcid":false,"given":"Kesavan","family":"Venkatesh","sequence":"first","affiliation":[]},{"given":"Simukayi","family":"Mutasa","sequence":"additional","affiliation":[]},{"given":"Fletcher","family":"Moore","sequence":"additional","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0003-0946-1957","authenticated-orcid":false,"given":"Jeremias","family":"Sulam","sequence":"additional","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0001-9433-8093","authenticated-orcid":false,"given":"Paul H.","family":"Yi","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2024,5,6]]},"reference":[{"issue":"21","key":"1136_CR1","doi-asserted-by":"publisher","first-page":"1981","DOI":"10.1056\/NEJMra2301725","volume":"388","author":"P Rajpurkar","year":"2023","unstructured":"P. Rajpurkar and M. P. Lungren, \u201cThe Current and Future State of AI Interpretation of Medical Images,\u201d N. Engl. J. Med., vol. 388, no. 21, pp. 1981\u20131990, May 2023.","journal-title":"N. Engl. J. Med."},{"issue":"8","key":"1136_CR2","doi-asserted-by":"publisher","first-page":"5455","DOI":"10.1007\/s10462-020-09825-6","volume":"53","author":"A Khan","year":"2020","unstructured":"A. Khan, A. Sohail, U. Zahoora, and A. S. Qureshi, \u201cA survey of the recent architectures of deep convolutional neural networks,\u201d Artif. Intell. Rev., vol. 53, no. 8, pp. 5455\u20135516, Dec. 2020.","journal-title":"Artif. Intell. Rev."},{"key":"1136_CR3","unstructured":"P. Rajpurkar et al., \u201cCheXNet: Radiologist-Level Pneumonia Detection on Chest X-Rays with Deep Learning,\u201d arXiv [cs.CV], 14-Nov-2017."},{"issue":"11","key":"1136_CR4","doi-asserted-by":"publisher","DOI":"10.1371\/journal.pmed.1002686","volume":"15","author":"P Rajpurkar","year":"2018","unstructured":"P. Rajpurkar et al., \u201cDeep learning for chest radiograph diagnosis: A retrospective comparison of the CheXNeXt algorithm to practicing radiologists,\u201d PLoS Med., vol. 15, no. 11, p. e1002686, Nov. 2018.","journal-title":"PLoS Med."},{"key":"1136_CR5","doi-asserted-by":"crossref","unstructured":"R. Ranjbarzadeh, A. Bagherian Kasgari, S. Jafarzadeh Ghoushchi, S. Anari, M. Naseri, and M. Bendechache, \u201cBrain tumor segmentation based on deep learning and an attention mechanism using MRI multi-modalities brain images,\u201d Sci. Rep., vol. 11, no. 1, p. 10930, May 2021.","DOI":"10.1038\/s41598-021-90428-8"},{"key":"1136_CR6","doi-asserted-by":"crossref","unstructured":"L. H. Gilpin, D. Bau, B. Z. Yuan, A. Bajwa, M. Specter, and L. Kagal, \u201cExplaining Explanations: An Overview of Interpretability of Machine Learning,\u201d arXiv [cs.AI], 31-May-2018.","DOI":"10.1109\/DSAA.2018.00018"},{"issue":"11","key":"1136_CR7","doi-asserted-by":"publisher","DOI":"10.1371\/journal.pmed.1002683","volume":"15","author":"JR Zech","year":"2018","unstructured":"J. R. Zech, M. A. Badgeley, M. Liu, A. B. Costa, J. J. Titano, and E. K. Oermann, \u201cVariable generalization performance of a deep learning model to detect pneumonia in chest radiographs: A cross-sectional study,\u201d PLoS Med., vol. 15, no. 11, p. e1002683, Nov. 2018.","journal-title":"PLoS Med."},{"key":"1136_CR8","doi-asserted-by":"crossref","unstructured":"J. Teneggi, P. H. Yi, and J. Sulam, \u201cExamination-level Supervision for Deep Learning\u2013based Intracranial Hemorrhage Detection at Head CT,\u201d Radiology: Artificial Intelligence, p. e230159, Dec. 2023.","DOI":"10.1148\/ryai.230159"},{"issue":"11","key":"1136_CR9","doi-asserted-by":"publisher","DOI":"10.1371\/journal.pmed.1002699","volume":"15","author":"N Bien","year":"2018","unstructured":"N. Bien et al., \u201cDeep-learning-assisted diagnosis for knee magnetic resonance imaging: Development and retrospective validation of MRNet,\u201d PLoS Med., vol. 15, no. 11, p. e1002699, Nov. 2018.","journal-title":"PLoS Med."},{"issue":"1","key":"1136_CR10","doi-asserted-by":"publisher","first-page":"18","DOI":"10.1038\/s41551-019-0487-z","volume":"4","author":"A Mitani","year":"2020","unstructured":"A. Mitani et al., \u201cDetection of anaemia from retinal fundus images via deep learning,\u201d Nat Biomed Eng, vol. 4, no. 1, pp. 18\u201327, Jan. 2020.","journal-title":"Nat Biomed Eng"},{"key":"1136_CR11","doi-asserted-by":"crossref","unstructured":"Z. Kang, E. Xiao, Z. Li, and L. Wang, \u201cDeep Learning Based on ResNet-18 for Classification of Prostate Imaging-Reporting and Data System Category 3 Lesions,\u201d Acad. Radiol., Jan. 2024.","DOI":"10.1016\/j.acra.2023.12.042"},{"issue":"3","key":"1136_CR12","doi-asserted-by":"publisher","DOI":"10.1371\/journal.pone.0299545","volume":"19","author":"L Alzubaidi","year":"2024","unstructured":"L. Alzubaidi et al., \u201cTrustworthy deep learning framework for the detection of abnormalities in X-ray shoulder images,\u201d PLoS One, vol. 19, no. 3, p. e0299545, Mar. 2024.","journal-title":"PLoS One"},{"key":"1136_CR13","unstructured":"J. Adebayo, J. Gilmer, M. Muelly, I. Goodfellow, M. Hardt, and B. Kim, \u201cSanity Checks for Saliency Maps,\u201d arXiv [cs.CV], 08-Oct-2018."},{"issue":"1","key":"1136_CR14","doi-asserted-by":"publisher","DOI":"10.1148\/ryai.220221","volume":"6","author":"J Zhang","year":"2024","unstructured":"J. Zhang, H. Chao, G. Dasegowda, G. Wang, M. K. Kalra, and P. Yan, \u201cRevisiting the Trustworthiness of Saliency Methods in Radiology AI,\u201d Radiol Artif Intell, vol. 6, no. 1, p. e220221, Jan. 2024.","journal-title":"Radiol Artif Intell"},{"issue":"6","key":"1136_CR15","doi-asserted-by":"publisher","DOI":"10.1148\/ryai.2021200267","volume":"3","author":"N Arun","year":"2021","unstructured":"N. Arun et al., \u201cAssessing the Trustworthiness of Saliency Maps for Localizing Abnormalities in Medical Imaging,\u201d Radiol Artif Intell, vol. 3, no. 6, p. e200267, Nov. 2021.","journal-title":"Radiol Artif Intell"},{"issue":"10","key":"1136_CR16","doi-asserted-by":"publisher","first-page":"867","DOI":"10.1038\/s42256-022-00536-x","volume":"4","author":"A Saporta","year":"2022","unstructured":"A. Saporta et al., \u201cBenchmarking saliency methods for chest X-ray interpretation,\u201d Nature Machine Intelligence, vol. 4, no. 10, pp. 867\u2013878, Oct. 2022.","journal-title":"Nature Machine Intelligence"},{"key":"1136_CR17","unstructured":"W. Jin, X. Li, and G. Hamarneh, \u201cOne Map Does Not Fit All: Evaluating Saliency Map Explanation on Multi-Modal Medical Images,\u201d arXiv [cs.CV], 11-Jul-2021."},{"key":"1136_CR18","unstructured":"P. Rajpurkar et al., \u201cMURA: Large Dataset for Abnormality Detection in Musculoskeletal Radiographs,\u201d arXiv [physics.med-ph], 11-Dec-2017."},{"key":"1136_CR19","doi-asserted-by":"crossref","unstructured":"C. Szegedy, V. Vanhoucke, S. Ioffe, J. Shlens, and Z. Wojna, \u201cRethinking the Inception Architecture for Computer Vision,\u201d arXiv [cs.CV], 02-Dec-2015.","DOI":"10.1109\/CVPR.2016.308"},{"key":"1136_CR20","doi-asserted-by":"crossref","unstructured":"G. Huang, Z. Liu, L. van der Maaten, and K. Q. Weinberger, \u201cDensely Connected Convolutional Networks,\u201d arXiv [cs.CV], 25-Aug-2016.","DOI":"10.1109\/CVPR.2017.243"},{"issue":"2","key":"1136_CR21","doi-asserted-by":"publisher","first-page":"498","DOI":"10.1148\/radiol.2018180736","volume":"290","author":"SS Halabi","year":"2019","unstructured":"S. S. Halabi et al., \u201cThe RSNA Pediatric Bone Age Machine Learning Challenge,\u201d Radiology, vol. 290, no. 2, pp. 498\u2013503, Feb. 2019.","journal-title":"Radiology"},{"key":"1136_CR22","doi-asserted-by":"crossref","unstructured":"R. R. Selvaraju, M. Cogswell, A. Das, R. Vedantam, D. Parikh, and D. Batra, \u201cGrad-CAM: Visual Explanations from Deep Networks via Gradient-based Localization,\u201d arXiv [cs.CV], 07-Oct-2016.","DOI":"10.1109\/ICCV.2017.74"},{"key":"1136_CR23","unstructured":"K. Simonyan, A. Vedaldi, and A. Zisserman, \u201cDeep Inside Convolutional Networks: Visualising Image Classification Models and Saliency Maps,\u201d arXiv [cs.CV], 20-Dec-2013."},{"key":"1136_CR24","unstructured":"M. Sundararajan, A. Taly, and Q. Yan, \u201cAxiomatic attribution for deep networks,\u201d in Proceedings of the 34th International Conference on Machine Learning - Volume 70, Sydney, NSW, Australia, 2017, pp. 3319\u20133328."},{"key":"1136_CR25","unstructured":"D. Smilkov, N. Thorat, B. Kim, F. Vi\u00e9gas, and M. Wattenberg, \u201cSmoothGrad: removing noise by adding noise,\u201d arXiv [cs.LG], 12-Jun-2017."},{"key":"1136_CR26","doi-asserted-by":"crossref","unstructured":"A. Kapishnikov, T. Bolukbasi, F. Vi\u00e9gas, and M. Terry, \u201cXRAI: Better Attributions Through Regions,\u201d arXiv [cs.CV], 06-Jun-2019.","DOI":"10.1109\/ICCV.2019.00505"},{"key":"1136_CR27","unstructured":"J. T. Springenberg, A. Dosovitskiy, T. Brox, and M. Riedmiller, \u201cStriving for Simplicity: The All Convolutional Net,\u201d arXiv [cs.LG], 21-Dec-2014."},{"key":"1136_CR28","unstructured":"R. R. Selvaraju, A. Das, R. Vedantam, M. Cogswell, D. Parikh, and D. Batra, \u201cGrad-CAM: Why did you say that?,\u201d arXiv [stat.ML], 22-Nov-2016."},{"key":"1136_CR29","unstructured":"S. Hooker, D. Erhan, P.-J. Kindermans, and B. Kim, \u201cA benchmark for interpretability methods in deep neural networks,\u201d in Proceedings of the 33rd International Conference on Neural Information Processing Systems, Red Hook, NY, USA: Curran Associates Inc., 2019, pp. 9737\u20139748."},{"issue":"6","key":"1136_CR30","doi-asserted-by":"publisher","first-page":"679","DOI":"10.1109\/TPAMI.1986.4767851","volume":"8","author":"J Canny","year":"1986","unstructured":"J. Canny, \u201cA computational approach to edge detection,\u201d IEEE Trans. Pattern Anal. Mach. Intell., vol. 8, no. 6, pp. 679\u2013698, Jun. 1986.","journal-title":"IEEE Trans. Pattern Anal. Mach. Intell."},{"issue":"1","key":"1136_CR31","doi-asserted-by":"publisher","first-page":"9097","DOI":"10.1038\/s41598-021-88578-w","volume":"11","author":"M He","year":"2021","unstructured":"M. He, X. Wang, and Y. Zhao, \u201cA calibrated deep learning ensemble for abnormality detection in musculoskeletal radiographs,\u201d Sci. Rep., vol. 11, no. 1, p. 9097, Apr. 2021.","journal-title":"Sci. Rep."},{"key":"1136_CR32","unstructured":"J. Irvin et al., \u201cCheXpert: A Large Chest Radiograph Dataset with Uncertainty Labels and Expert Comparison,\u201d arXiv [cs.CV], 21-Jan-2019."},{"key":"1136_CR33","doi-asserted-by":"crossref","unstructured":"L. Oakden-Rayner, J. Dunnmon, G. Carneiro, and C. Re, \u201cHidden stratification causes clinically meaningful failures in machine learning for medical imaging,\u201d in Proceedings of the ACM Conference on Health, Inference, and Learning, Toronto, Ontario, Canada, 2020, pp. 151\u2013159.","DOI":"10.1145\/3368555.3384468"},{"key":"1136_CR34","unstructured":"G. Yona and D. Greenfeld, \u201cRevisiting Sanity Checks for Saliency Maps,\u201d arXiv [cs.LG], 27-Oct-2021."},{"key":"1136_CR35","unstructured":"S. Lundberg and S.-I. Lee, \u201cA Unified Approach to Interpreting Model Predictions,\u201d arXiv [cs.AI], 22-May-2017."},{"key":"1136_CR36","unstructured":"J. Teneggi, A. Luster, and J. Sulam, \u201cFast Hierarchical Games for Image Explanations,\u201d arXiv [cs.CV], 13-Apr-2021."},{"key":"1136_CR37","unstructured":"J. Teneggi, B. Bharti, Y. Romano, and J. Sulam, \u201cSHAP-XRT: The Shapley Value Meets Conditional Independence Testing,\u201d Transactions on Machine Learning Research, 11-Jul-2023."},{"key":"1136_CR38","first-page":"71","volume":"12729","author":"Z Liu","year":"2021","unstructured":"Z. Liu, E. Adeli, K. M. Pohl, and Q. Zhao, \u201cGoing Beyond Saliency Maps: Training Deep Models to Interpret Deep Models,\u201d Inf. Process. Med. Imaging, vol. 12729, pp. 71\u201382, Jun. 2021.","journal-title":"Inf. Process. Med. Imaging"}],"container-title":["Journal of Imaging Informatics in Medicine"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10278-024-01136-4.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s10278-024-01136-4\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10278-024-01136-4.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,10,29]],"date-time":"2024-10-29T08:36:20Z","timestamp":1730190980000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s10278-024-01136-4"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,5,6]]},"references-count":38,"journal-issue":{"issue":"5","published-online":{"date-parts":[[2024,10]]}},"alternative-id":["1136"],"URL":"https:\/\/doi.org\/10.1007\/s10278-024-01136-4","relation":{},"ISSN":["2948-2933"],"issn-type":[{"value":"2948-2933","type":"electronic"}],"subject":[],"published":{"date-parts":[[2024,5,6]]},"assertion":[{"value":"8 February 2024","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"30 April 2024","order":2,"name":"revised","label":"Revised","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"1 May 2024","order":3,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"6 May 2024","order":4,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Declarations"}},{"value":"This study used publicly available data and was considered not human subjects research. Therefore, institutional review board review was waived.","order":2,"name":"Ethics","group":{"name":"EthicsHeading","label":"Ethics Approval"}},{"value":"The authors declare no competing interests.","order":3,"name":"Ethics","group":{"name":"EthicsHeading","label":"Competing Interests"}}]}}