{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,5,7]],"date-time":"2026-05-07T15:37:42Z","timestamp":1778168262716,"version":"3.51.4"},"reference-count":34,"publisher":"Springer Science and Business Media LLC","issue":"1","license":[{"start":{"date-parts":[[2021,1,1]],"date-time":"2021-01-01T00:00:00Z","timestamp":1609459200000},"content-version":"tdm","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"},{"start":{"date-parts":[[2021,1,1]],"date-time":"2021-01-01T00:00:00Z","timestamp":1609459200000},"content-version":"vor","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"}],"funder":[{"DOI":"10.13039\/100000049","name":"National Institute on Aging","doi-asserted-by":"publisher","award":["R56 AG052536-01A1"],"award-info":[{"award-number":["R56 AG052536-01A1"]}],"id":[{"id":"10.13039\/100000049","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000002","name":"National Institutes of Health","doi-asserted-by":"publisher","award":["UL1TR001876"],"award-info":[{"award-number":["UL1TR001876"]}],"id":[{"id":"10.13039\/100000002","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000738","name":"U.S. Department of Veterans Affairs","doi-asserted-by":"publisher","award":["CRE 12\u2013315"],"award-info":[{"award-number":["CRE 12\u2013315"]}],"id":[{"id":"10.13039\/100000738","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000738","name":"U.S. Department of Veterans Affairs","doi-asserted-by":"publisher","award":["CRE 12-321"],"award-info":[{"award-number":["CRE 12-321"]}],"id":[{"id":"10.13039\/100000738","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000050","name":"National Heart, Lung, and Blood Institute","doi-asserted-by":"publisher","award":["K08HL136850"],"award-info":[{"award-number":["K08HL136850"]}],"id":[{"id":"10.13039\/100000050","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["J Med Syst"],"published-print":{"date-parts":[[2021,1]]},"DOI":"10.1007\/s10916-020-01701-8","type":"journal-article","created":{"date-parts":[[2021,1,6]],"date-time":"2021-01-06T08:03:41Z","timestamp":1609920221000},"update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":41,"title":["Shedding Light on the Black Box: Explaining Deep Neural Network Prediction of Clinical Outcomes"],"prefix":"10.1007","volume":"45","author":[{"ORCID":"https:\/\/orcid.org\/0000-0001-6419-7963","authenticated-orcid":false,"given":"Yijun","family":"Shao","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Yan","family":"Cheng","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Rashmee U.","family":"Shah","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Charlene R.","family":"Weir","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Bruce E.","family":"Bray","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Qing","family":"Zeng-Treitler","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2021,1,4]]},"reference":[{"key":"1701_CR1","unstructured":"A. Krizhevsky, I. Sutskever, and G. E. Hinton, \"ImageNet Classification with Deep Convolutional Neural Networks,\" in NIPS'12 Proceedings of the 25th International Conference on Neural Information Processing Systems, Lake Tahoe, 2012, vol. 1."},{"key":"1701_CR2","doi-asserted-by":"crossref","unstructured":"W. Xiong et al., \"Toward Human Parity in Conversational Speech Recognition,\" IEEE\/ACM Trans. Audio Speech Lang. Process., vol. 25, no. 11, 2017.","DOI":"10.1109\/TASLP.2017.2756440"},{"key":"1701_CR3","doi-asserted-by":"publisher","unstructured":"D. Silver et al., \"Mastering the game of Go with deep neural networks and tree search,\" Nature, vol. 529, no. 7587, pp. 484-9, doi: https:\/\/doi.org\/10.1038\/nature16961.","DOI":"10.1038\/nature16961"},{"key":"1701_CR4","doi-asserted-by":"publisher","unstructured":"Y. LeCun, Y. Bengio, and G. Hinton, Deep learning, Nature, Research Support, Non-U.S. Gov't Research Support, U.S. Gov't, Non-P.H.S. Rev. vol. 521, no. 7553, pp. 436-44, May 28 2015, doi: https:\/\/doi.org\/10.1038\/nature14539.","DOI":"10.1038\/nature14539"},{"key":"1701_CR5","doi-asserted-by":"publisher","first-page":"514","DOI":"10.1109\/Access.2014.2325029","volume":"2","author":"XW Chen","year":"2014","unstructured":"X. W. Chen and X. T. Lin, Big Data Deep Learning: Challenges and Perspectives, (in English), Leee Access, vol. 2, pp. 514-525, 2014, doi: https:\/\/doi.org\/10.1109\/Access.2014.2325029.","journal-title":"Leee Access"},{"key":"1701_CR6","doi-asserted-by":"publisher","unstructured":"R. Miotto, F. Wang, S. Wang, X. Jiang, and J. T. Dudley, Deep learning for healthcare: review, opportunities and challenges, Brief Bioinform, May 6 2017, doi: https:\/\/doi.org\/10.1093\/bib\/bbx044.","DOI":"10.1093\/bib\/bbx044"},{"key":"1701_CR7","doi-asserted-by":"publisher","first-page":"229","DOI":"10.1016\/j.jbi.2015.05.016","volume":"56","author":"J Futoma","year":"2015","unstructured":"J. Futoma, J. Morris, and J. Lucas, \"A comparison of models for predicting early hospital readmissions,\" J Biomed Inform, vol. 56, pp. 229-38, 2015, doi: https:\/\/doi.org\/10.1016\/j.jbi.2015.05.016.","journal-title":"J Biomed Inform"},{"key":"1701_CR8","volume-title":"Proceedings of the 2016 SIAM International Conference on Data Mining","author":"Y Cheng","year":"2016","unstructured":"Y. Cheng, F. Wang, P. Zheng, and J. Hu (2016), \"Risk prediction with electronic health records: a deep learning approach,\" in Proceedings of the 2016 SIAM International Conference on Data Mining, Miami: Society for Industrial and Applied Mathematics."},{"key":"1701_CR9","unstructured":"E. Choi, M. T. Bahadori, A. Schuetz, W. F. Stewart, and J. Sun, \"Doctor AI: Predicting Clinical Events via Recurrent Neural Networks,\" JMLR Workshop Conf. Proc., vol. 56, pp. 301-318, 2016. [Online]. Available: http:\/\/www.ncbi.nlm.nih.gov\/pubmed\/28286600."},{"issue":"4","key":"1701_CR10","doi-asserted-by":"publisher","first-page":"570","DOI":"10.3348\/kjr.2017.18.4.570","volume":"18","author":"JG Lee","year":"2017","unstructured":"J. G. Lee et al., \"Deep Learning in Medical Imaging: General Overview,\" Korean J Radiol, vol. 18, no. 4, pp. 570-584, 2017, doi: https:\/\/doi.org\/10.3348\/kjr.2017.18.4.570.","journal-title":"Korean J Radiol"},{"issue":"2","key":"1701_CR11","doi-asserted-by":"publisher","first-page":"505","DOI":"10.1148\/rg.2017160130","volume":"37","author":"BJ Erickson","year":"2017","unstructured":"B. J. Erickson, P. Korfiatis, Z. Akkus, and T. L. Kline, \"Machine Learning for Medical Imaging,\" Radiographics, vol. 37, no. 2, pp. 505-515, 2017, doi: https:\/\/doi.org\/10.1148\/rg.2017160130.","journal-title":"Radiographics"},{"key":"1701_CR12","doi-asserted-by":"publisher","unstructured":"L. A. Pastur-Romay, F. Cedron, A. Pazos, and A. B. Porto-Pazos, \"Deep Artificial Neural Networks and Neuromorphic Chips for Big Data Analysis: Pharmaceutical and Bioinformatics Applications,\" Int J Mol Sci, vol. 17, no. 8, 1313 Aug 11 2016, doi: https:\/\/doi.org\/10.3390\/ijms17081313.","DOI":"10.3390\/ijms17081313"},{"key":"1701_CR13","unstructured":"NIST. \"Guidelines for the 2012 TREC Medical Records Track.\" http:\/\/www-nlpir.nist.gov\/projects\/trecmed\/2012. Accessed March 2018."},{"key":"1701_CR14","doi-asserted-by":"publisher","first-page":"818","DOI":"10.1007\/978-3-319-10590-1_53","volume":"8689","author":"MD Zeiler","year":"2014","unstructured":"M. D. Zeiler and R. Fergus, \"Visualizing and Understanding Convolutional Networks, (in English), Lect. Notes Comput. Sc., vol. 8689, pp. 818-833, 2014. [Online]. Available: <Go to ISI>:\/\/WOS:000345524200047.","journal-title":"Lect. Notes Comput. Sc."},{"issue":"7","key":"1701_CR15","doi-asserted-by":"publisher","first-page":"e0130140","DOI":"10.1371\/journal.pone.0130140","volume":"10","author":"S Bach","year":"2015","unstructured":"S. Bach, A. Binder, G. Montavon, F. Klauschen, K. R. Muller, and W. Samek, \"On Pixel-Wise Explanations for Non-Linear Classifier Decisions by Layer-Wise Relevance Propagation,\" PLoS One, vol. 10, no. 7, p. e0130140, 2015, doi: https:\/\/doi.org\/10.1371\/journal.pone.0130140.","journal-title":"PLoS One"},{"key":"1701_CR16","doi-asserted-by":"crossref","unstructured":"Lipton, Z. C., The Mythos of Model Interpretability. Queue, vol 16, no. 3, 2018. https:\/\/arxiv.org\/abs\/1606.03490. Accessed March 2018.","DOI":"10.1145\/3236386.3241340"},{"key":"1701_CR17","doi-asserted-by":"crossref","unstructured":"Y. Dong, H. Su, J. Zhu, and B. Zhang, Improving Interpretability of Deep Neural Networks with Semantic Information, 2017 IEEE Conference on Computer Vision and Pattern Recognition (CVPR), 2017. arXiv preprint arXiv:1703.04096, 2017","DOI":"10.1109\/CVPR.2017.110"},{"key":"1701_CR18","doi-asserted-by":"publisher","unstructured":"Oquab, M., Bottou, L., Laptev, I., and J. Sivic, Learning and Transferring Mid-Level Image Representations using Convolutional Neural Networks, (in English). 2014 IEEE Conference on Computer Vision and Pattern Recognition,\u00a0pp. 1717-1724, 2014,\u00a0https:\/\/doi.org\/10.1109\/Cvpr.2014.222.","DOI":"10.1109\/Cvpr.2014.222"},{"key":"1701_CR19","doi-asserted-by":"crossref","unstructured":"R. Shah, Y. Shao, K. M. Doing-Harris, W. Charlene, Y. Cheng, B. Bray, and Q. Zeng-Treitler, \"Frailty and Cardiovasular Surgery: Deep Neural Network versus Support Vector Machine to Predict Death,\" in ACC.18, Orlando, FL, 2018.","DOI":"10.1016\/S0735-1097(18)31898-9"},{"key":"1701_CR20","first-page":"001","volume":"S3","author":"QT Zeng","year":"2011","unstructured":"Q. T. Zeng, D. Redd, G. Divita, S. Jarad, C. Brandt, and J. R. Nebeker, \"Characterizing Clinical Text and Sublanguage: A Case Study of the VA Clinical Notes,\" J Health Med Informat. S3:001, 2011.","journal-title":"J Health Med Informat."},{"issue":"24","key":"1701_CR21","doi-asserted-by":"publisher","first-page":"3327","DOI":"10.1161\/CIRCULATIONAHA.104.481267","volume":"111","author":"A Kadish","year":"2005","unstructured":"A. Kadish and M. Mehra, \"Heart failure devices: implantable cardioverter-defibrillators and biventricular pacing therapy,\" Circulation, vol. 111, no. 24, pp. 3327-35, 2005, doi: https:\/\/doi.org\/10.1161\/CIRCULATIONAHA.104.481267.","journal-title":"Circulation"},{"key":"1701_CR22","doi-asserted-by":"publisher","unstructured":"M. S. Slaughter et al., \"Advanced heart failure treated with continuous-flow left ventricular assist device,\" N Engl J Med, vol. 361, no. 23, pp. 2241-51, Dec 3 2009, doi: https:\/\/doi.org\/10.1056\/NEJMoa0909938.","DOI":"10.1056\/NEJMoa0909938"},{"issue":"11","key":"1701_CR23","doi-asserted-by":"publisher","first-page":"1324","DOI":"10.1001\/archpsyc.65.11.1324","volume":"65","author":"KH Ladwig","year":"2008","unstructured":"K. H. Ladwig, J. Baumert, B. Marten-Mittag, C. Kolb, B. Zrenner, and C. Schmitt, \"Posttraumatic stress symptoms and predicted mortality in patients with implantable cardioverter-defibrillators: results from the prospective living with an implanted cardioverter-defibrillator study,\" Arch Gen Psychiatry, vol. 65, no. 11, pp. 1324-30, Nov 2008, doi: https:\/\/doi.org\/10.1001\/archpsyc.65.11.1324.","journal-title":"Arch Gen Psychiatry"},{"issue":"3-4","key":"1701_CR24","doi-asserted-by":"publisher","first-page":"450","DOI":"10.1111\/jocn.12200","volume":"23","author":"IM Morken","year":"2014","unstructured":"I. M. Morken, E. Bru, T. M. Norekval, A. I. Larsen, T. Idsoe, and B. Karlsen, \"Perceived support from healthcare professionals, shock anxiety and post-traumatic stress in implantable cardioverter defibrillator recipients,\" J Clin Nurs, vol. 23, no. 3-4, pp. 450-60, 2014, doi: https:\/\/doi.org\/10.1111\/jocn.12200.","journal-title":"J Clin Nurs"},{"issue":"7","key":"1701_CR25","doi-asserted-by":"publisher","first-page":"988","DOI":"10.1016\/j.healun.2015.03.006","volume":"34","author":"KH Magid","year":"2015","unstructured":"K. H. Magid, D. D. Matlock, J. S. Thompson, C. K. McIlvennan, and L. A. Allen, \"The influence of expected risks on decision making for destination therapy left ventricular assist device: An MTurk survey,\" J Heart Lung Transplant, vol. 34, no. 7, pp. 988-90, 2015, doi: https:\/\/doi.org\/10.1016\/j.healun.2015.03.006.","journal-title":"J Heart Lung Transplant"},{"issue":"1","key":"1701_CR26","doi-asserted-by":"publisher","first-page":"e000033","DOI":"10.1136\/openhrt-2013-000033","volume":"1","author":"R Rowe","year":"2014","unstructured":"R. Rowe et al., \"Role of frailty assessment in patients undergoing cardiac interventions,\" Open Heart, vol. 1, no. 1, p. e000033, 2014, doi: https:\/\/doi.org\/10.1136\/openhrt-2013-000033.","journal-title":"Open Heart"},{"issue":"2","key":"1701_CR27","doi-asserted-by":"publisher","first-page":"109","DOI":"10.1053\/j.semtcvs.2010.09.001","volume":"22","author":"J Chikwe","year":"2010","unstructured":"J. Chikwe and D. H. Adams, \"Frailty: the missing element in predicting operative mortality,\" Semin Thorac Cardiovasc Surg, vol. 22, no. 2, pp. 109-10, 2010, doi: https:\/\/doi.org\/10.1053\/j.semtcvs.2010.09.001.","journal-title":"Semin Thorac Cardiovasc Surg"},{"issue":"6","key":"1701_CR28","doi-asserted-by":"publisher","first-page":"1056","DOI":"10.1016\/j.jbi.2011.08.005","volume":"44","author":"J Zhang","year":"2011","unstructured":"J. Zhang and M. F. Walji, \"TURF: toward a unified framework of EHR usability,\" J Biomed Inform, vol. 44, no. 6, pp. 1056-67, 2011, doi: https:\/\/doi.org\/10.1016\/j.jbi.2011.08.005.","journal-title":"J Biomed Inform"},{"key":"1701_CR29","unstructured":"CZ. Che, Y. Cheng, Z. Sun, and Y. Liu, Exploiting Convolutional Neural Network for Risk Prediction with Medical Feature Embedding, arXiv preprint arXiv:1701.07474, 2017."},{"key":"1701_CR30","doi-asserted-by":"crossref","unstructured":"Y. Cheng, F. Wang, P. Zheng, and J. Hu, Risk Prediction with Electronic Health Records: A Deep Learning Approach, presented at the Proceedings of the 2016 SIAM International Conference on Data Mining, 2016.","DOI":"10.1137\/1.9781611974348.49"},{"key":"1701_CR31","volume-title":"1-D Convolutional Neural Networks for Signal Processing Applications, presented at the ICASSP 2019 - 2019 IEEE International Conference on Acoustics","author":"S Kiranyaz","year":"2019","unstructured":"S. Kiranyaz, T. Ince, O. Abdeljaber, O. Avci, and M. Gabbouj, \"1-D Convolutional Neural Networks for Signal Processing Applications, presented at the ICASSP 2019 - 2019 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP), Brighton, 2019."},{"key":"1701_CR32","doi-asserted-by":"crossref","unstructured":"J. B. Bergstra, O. Bastien, F. Lamblin, P. Pascanu, R. Desjardins, G. Turian, J. Warde-Farley, D. Bengio, Y. Theano, A CPU and GPU Math Expression Compiler, in Proceedings of the Python for Scientific Computing Conference (SciPy) 2010.","DOI":"10.25080\/Majora-92bf1922-003"},{"key":"1701_CR33","doi-asserted-by":"publisher","unstructured":"S. S. J. Dieleman, C. Raffel, E. Olso, S. K. S\u00f8nderby, D. Nouri, et. al, Lasagne: First release. https:\/\/doi.org\/10.5281\/zenodo.27878.","DOI":"10.5281\/zenodo.27878"},{"key":"1701_CR34","doi-asserted-by":"crossref","unstructured":"R. P. Anderson, R. Jin, and G. L. Grunkemeier, \"Understanding logistic regression analysis in clinical reports: an introduction,\" Ann Thorac Surg, vol. 75, no. 3, pp. 753-7, 2003. [Online]. Available: http:\/\/www.ncbi.nlm.nih.gov\/pubmed\/12645688. Accessed March 2018.","DOI":"10.1016\/S0003-4975(02)04683-0"}],"container-title":["Journal of Medical Systems"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s10916-020-01701-8.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/article\/10.1007\/s10916-020-01701-8\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s10916-020-01701-8.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2021,1,14]],"date-time":"2021-01-14T08:48:11Z","timestamp":1610614091000},"score":1,"resource":{"primary":{"URL":"http:\/\/link.springer.com\/10.1007\/s10916-020-01701-8"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2021,1]]},"references-count":34,"journal-issue":{"issue":"1","published-print":{"date-parts":[[2021,1]]}},"alternative-id":["1701"],"URL":"https:\/\/doi.org\/10.1007\/s10916-020-01701-8","relation":{},"ISSN":["0148-5598","1573-689X"],"issn-type":[{"value":"0148-5598","type":"print"},{"value":"1573-689X","type":"electronic"}],"subject":[],"published":{"date-parts":[[2021,1]]},"assertion":[{"value":"8 December 2019","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"16 December 2020","order":2,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"4 January 2021","order":3,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Compliance with ethical standards"}},{"value":"All authors declare that they have no conflicts of interest.","order":2,"name":"Ethics","group":{"name":"EthicsHeading","label":"Conflict of interests"}},{"value":"All procedures performed in studies involving human participants were in accordance with the ethical standards of the institutional and\/or national research committee and with the 1964 Helsinki declaration and its later amendments or comparable ethical standards. This article does not contain any studies with animals performed by any of the authors.","order":3,"name":"Ethics","group":{"name":"EthicsHeading","label":"Ethical approval"}},{"value":"Informed consent was obtained from all individual participants included in the study.","order":4,"name":"Ethics","group":{"name":"EthicsHeading","label":"Informed consent"}}],"article-number":"5"}}