{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,6,29]],"date-time":"2026-06-29T21:44:37Z","timestamp":1782769477404,"version":"3.54.5"},"reference-count":41,"publisher":"Springer Science and Business Media LLC","issue":"1","license":[{"start":{"date-parts":[[2020,1,17]],"date-time":"2020-01-17T00:00:00Z","timestamp":1579219200000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2020,1,17]],"date-time":"2020-01-17T00:00:00Z","timestamp":1579219200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"funder":[{"DOI":"10.13039\/100000001","name":"National Science Foundation","doi-asserted-by":"publisher","award":["DGE-1256082"],"award-info":[{"award-number":["DGE-1256082"]}],"id":[{"id":"10.13039\/100000001","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000001","name":"National Science Foundation","doi-asserted-by":"publisher","award":["DBI-1759487"],"award-info":[{"award-number":["DBI-1759487"]}],"id":[{"id":"10.13039\/100000001","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000001","name":"National Science Foundation","doi-asserted-by":"publisher","award":["DBI-1552309"],"award-info":[{"award-number":["DBI-1552309"]}],"id":[{"id":"10.13039\/100000001","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000001","name":"National Science Foundation","doi-asserted-by":"publisher","award":["DBI-1355899"],"award-info":[{"award-number":["DBI-1355899"]}],"id":[{"id":"10.13039\/100000001","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000048","name":"American Cancer Society","doi-asserted-by":"publisher","award":["127332-RSG-15-097-01-TBG"],"award-info":[{"award-number":["127332-RSG-15-097-01-TBG"]}],"id":[{"id":"10.13039\/100000048","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000002","name":"U.S. Department of Health & Human Services | National Institutes of Health","doi-asserted-by":"publisher","award":["R35 GM 128638"],"award-info":[{"award-number":["R35 GM 128638"]}],"id":[{"id":"10.13039\/100000002","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000002","name":"U.S. Department of Health & Human Services | National Institutes of Health","doi-asserted-by":"publisher","award":["R01 NIA AG 061132"],"award-info":[{"award-number":["R01 NIA AG 061132"]}],"id":[{"id":"10.13039\/100000002","id-type":"DOI","asserted-by":"publisher"}]},{"name":"U.S. Department of Health & Human Services | National Institutes of Health"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Nat Mach Intell"],"DOI":"10.1038\/s42256-019-0138-9","type":"journal-article","created":{"date-parts":[[2020,1,17]],"date-time":"2020-01-17T12:02:53Z","timestamp":1579262573000},"page":"56-67","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":8089,"title":["From local explanations to global understanding with explainable AI for trees"],"prefix":"10.1038","volume":"2","author":[{"given":"Scott M.","family":"Lundberg","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-8741-131X","authenticated-orcid":false,"given":"Gabriel","family":"Erion","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Hugh","family":"Chen","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Alex","family":"DeGrave","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Jordan M.","family":"Prutkin","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Bala","family":"Nair","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Ronit","family":"Katz","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Jonathan","family":"Himmelfarb","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Nisha","family":"Bansal","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-5833-5215","authenticated-orcid":false,"given":"Su-In","family":"Lee","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2020,1,17]]},"reference":[{"key":"138_CR1","unstructured":"The state of data science & maching learning. Kaggle https:\/\/www.kaggle.com\/surveys\/2017 (2017)."},{"key":"138_CR2","doi-asserted-by":"crossref","unstructured":"Friedman, J., Hastie, T. & Tibshirani, R. The Elements of Statistical Learning Vol. 1 (Springer Series in Statistics, Springer, 2001).","DOI":"10.1007\/978-0-387-21606-5_1"},{"key":"138_CR3","unstructured":"Lundberg, S. M. & Lee, S.-I. A unified approach to interpreting model predictions. Adv. Neural Inf. Process. Syst. 30, 4768\u20134777 (2017)."},{"key":"138_CR4","unstructured":"Saabas, A. treeinterpreter python package. GitHub https:\/\/github.com\/andosa\/treeinterpreter (2019)."},{"key":"138_CR5","doi-asserted-by":"crossref","unstructured":"Ribeiro, M. T., Singh, S. & Guestrin, C. Why should i trust you?: Explaining the predictions of any classifier. In Proc. 22nd ACM SIGKDD International Conference on Knowledge Discovery and Data Mining 1135\u20131144 (ACM, 2016).","DOI":"10.1145\/2939672.2939778"},{"key":"138_CR6","doi-asserted-by":"crossref","unstructured":"Datta, A., Sen, S. & Zick, Y. Algorithmic transparency via quantitative input influence: theory and experiments with learning systems. In Proc. 2016 IEEE Symposium on Security and Privacy (SP), 598\u2013617 (IEEE, 2016).","DOI":"10.1109\/SP.2016.42"},{"key":"138_CR7","doi-asserted-by":"publisher","first-page":"647","DOI":"10.1007\/s10115-013-0679-x","volume":"41","author":"E \u0160trumbelj","year":"2014","unstructured":"\u0160trumbelj, E. & Kononenko, I. Explaining prediction models and individual predictions with feature contributions. Knowl. Inf. Syst. 41, 647\u2013665 (2014).","journal-title":"Knowl. Inf. Syst."},{"key":"138_CR8","first-page":"1803","volume":"11","author":"D Baehrens","year":"2010","unstructured":"Baehrens, D. et al. How to explain individual classification decisions. J. Mach. Learn. Res. 11, 1803\u20131831 (2010).","journal-title":"J. Mach. Learn. Res."},{"key":"138_CR9","first-page":"307","volume":"2","author":"LS Shapley","year":"1953","unstructured":"Shapley, L. S. A value for n-person games. Contrib. Theor. Games 2, 307\u2013317 (1953).","journal-title":"Contrib. Theor. Games"},{"key":"138_CR10","unstructured":"Sundararajan, M. & Najmi, A. The many Shapley values for model explanation. Preprint at https:\/\/arxiv.org\/abs\/1908.08474 (2019)."},{"key":"138_CR11","unstructured":"Janzing, D., Minorics, L. & Bl\u00f6baum, P. Feature relevance quantification in explainable AI: a causality problem. Preprint at https:\/\/arxiv.org\/abs\/1910.13413 (2019)."},{"key":"138_CR12","doi-asserted-by":"publisher","first-page":"305","DOI":"10.1016\/S0304-3975(00)00251-6","volume":"263","author":"Y Matsui","year":"2001","unstructured":"Matsui, Y. & Matsui, T. NP-completeness for calculating power indices of weighted majority games. Theor. Comput. Sci. 263, 305\u2013310 (2001).","journal-title":"Theor. Comput. Sci."},{"key":"138_CR13","doi-asserted-by":"publisher","first-page":"72","DOI":"10.1016\/j.geb.2005.03.002","volume":"55","author":"K Fujimoto","year":"2006","unstructured":"Fujimoto, K., Kojadinovic, I. & Marichal, J.-L. Axiomatic characterizations of probabilistic and cardinal-probabilistic interaction indices. Games Econ. Behav. 55, 72\u201399 (2006).","journal-title":"Games Econ. Behav."},{"key":"138_CR14","doi-asserted-by":"crossref","unstructured":"Ribeiro, M. T., Singh, S. & Guestrin, C. Anchors: high-precision model-agnostic explanations. In Proc. AAAI Conference on Artificial Intelligence (2018).","DOI":"10.1609\/aaai.v32i1.11491"},{"key":"138_CR15","doi-asserted-by":"publisher","first-page":"2199","DOI":"10.1001\/jama.2018.17163","volume":"320","author":"EH Shortliffe","year":"2018","unstructured":"Shortliffe, E. H. & Sep\u00falveda, M. J. Clinical decision support in the era of artificial intelligence. JAMA 320, 2199\u20132200 (2018).","journal-title":"JAMA"},{"key":"138_CR16","doi-asserted-by":"publisher","first-page":"749","DOI":"10.1038\/s41551-018-0304-0","volume":"2","author":"SM Lundberg","year":"2018","unstructured":"Lundberg, S. M. et al. Explainable machine learning predictions to help anesthesiologists prevent hypoxemia during surgery. Nat. Biomed. Eng. 2, 749\u2013760 (2018).","journal-title":"Nat. Biomed. Eng."},{"key":"138_CR17","first-page":"1","volume":"35","author":"CS Cox","year":"1997","unstructured":"Cox, C. S. et al. Plan and operation of the NHANES I Epidemiologic Followup Study, 1992. Vital Health Stat. 35, 1\u2013231 (1997).","journal-title":"Vital Health Stat."},{"key":"138_CR18","doi-asserted-by":"crossref","unstructured":"Chen, T. & Guestrin, C. Xgboost: a scalable tree boosting system. In Proc. 22nd ACM SIGKDD International Conference on Knowledge Discovery and Data Mining 785\u2013794 (ACM, 2016).","DOI":"10.1145\/2939672.2939785"},{"key":"138_CR19","doi-asserted-by":"publisher","first-page":"96","DOI":"10.1016\/j.neuroimage.2013.10.067","volume":"87","author":"S Haufe","year":"2014","unstructured":"Haufe, S. et al. On the interpretation of weight vectors of linear models in multivariate neuroimaging. Neuroimage 87, 96\u2013110 (2014).","journal-title":"Neuroimage"},{"key":"138_CR20","unstructured":"Kim, B. et al. Interpretability beyond feature attribution: quantitative testing with concept activation vectors (TCAV). In International Conference on Machine Learning (ICLR, 2018)."},{"key":"138_CR21","unstructured":"Yosinski, J., Clune, J., Nguyen, A., Fuchs, T. & Lipson, H. Understanding neural networks through deep visualization. In ICML Deep Learning Workshop (ICML, 2015)."},{"key":"138_CR22","doi-asserted-by":"crossref","unstructured":"Bau, D., Zhou, B., Khosla, A., Oliva, A. & Torralba, A. Network dissection: quantifying interpretability of deep visual representations. In Proc. IEEE Conference on Computer Vision and Pattern Recognition 6541\u20136549 (IEEE, 2017).","DOI":"10.1109\/CVPR.2017.354"},{"key":"138_CR23","doi-asserted-by":"crossref","unstructured":"Leino, K., Sen, S., Datta, A., Fredrikson, M. & Li, L. Influence-directed explanations for deep convolutional networks. In Proc. 2018 IEEE International Test Conference (ITC) 1\u20138 (IEEE, 2018).","DOI":"10.1109\/TEST.2018.8624792"},{"key":"138_CR24","doi-asserted-by":"publisher","first-page":"2103","DOI":"10.1056\/NEJMoa1511939","volume":"373","author":"SR Group","year":"2015","unstructured":"Group, S. R. A randomized trial of intensive versus standard blood-pressure control. N. Engl. J. Med. 373, 2103\u20132116 (2015).","journal-title":"N. Engl. J. Med."},{"key":"138_CR25","first-page":"e38","volume":"133","author":"D Mozaffarian","year":"2016","unstructured":"Mozaffarian, D. et al. Heart disease and stroke statistics-2016 update a report from the American Heart Association. Circulation 133, e38\u2013e48 (2016).","journal-title":"Circulation"},{"key":"138_CR26","doi-asserted-by":"publisher","first-page":"603","DOI":"10.2215\/CJN.09710916","volume":"12","author":"B Bowe","year":"2017","unstructured":"Bowe, B., Xie, Y., Xian, H., Li, T. & Al-Aly, Z. Association between monocyte count and risk of incident CKD and progression to ESRD. Clin. J. Am. Soc. Nephrol. 12, 603\u2013613 (2017).","journal-title":"Clin. J. Am. Soc. Nephrol."},{"key":"138_CR27","doi-asserted-by":"publisher","DOI":"10.1186\/s12882-017-0608-4","volume":"18","author":"F Fan","year":"2017","unstructured":"Fan, F., Jia, J., Li, J., Huo, Y. & Zhang, Y. White blood cell count predicts the odds of kidney function decline in a Chinese community-based population. BMC Nephrol. 18, 190 (2017).","journal-title":"BMC Nephrol."},{"key":"138_CR28","unstructured":"Zinkevich, M. Rules of machine learning: best practices for ML engineering (2017)."},{"key":"138_CR29","doi-asserted-by":"publisher","first-page":"969","DOI":"10.1002\/mds.23116","volume":"25","author":"SM van Rooden","year":"2010","unstructured":"van Rooden, S. M. et al. The identification of Parkinson\u2019s disease subtypes using cluster analysis: a systematic review. Mov. Disord. 25, 969\u2013978 (2010).","journal-title":"Mov. Disord."},{"key":"138_CR30","doi-asserted-by":"publisher","first-page":"8418","DOI":"10.1073\/pnas.0932692100","volume":"100","author":"T S\u00f8rlie","year":"2003","unstructured":"S\u00f8rlie, T. et al. Repeated observation of breast tumor subtypes in independent gene expression data sets. Proc. Natl Acad. Sci. USA 100, 8418\u20138423 (2003).","journal-title":"Proc. Natl Acad. Sci. USA"},{"key":"138_CR31","doi-asserted-by":"publisher","DOI":"10.1038\/s41467-019-08987-4","volume":"10","author":"S Lapuschkin","year":"2019","unstructured":"Lapuschkin, S. et al. Unmasking clever hans predictors and assessing what machines really learn. Nat. Commun. 10, 1096 (2019).","journal-title":"Nat. Commun."},{"key":"138_CR32","doi-asserted-by":"crossref","unstructured":"Pfungst, O. Clever Hans: (the Horse of Mr. Von Osten.) A Contribution to Experimental Animal and Human Psychology (Holt, Rinehart and Winston, 1911).","DOI":"10.5962\/bhl.title.56164"},{"key":"138_CR33","unstructured":"Machine Learning Recommendations for Policymakers (IIF, 2019); https:\/\/www.iif.com\/Publications\/ID\/3574\/Machine-Learning-Recommendations-for-Policymakers"},{"key":"138_CR34","unstructured":"Deeks, A. The judicial demand for explainable artificial intelligence. (2019)."},{"key":"138_CR35","first-page":"2520","volume":"31","author":"G Plumb","year":"2018","unstructured":"Plumb, G., Molitor, D. & Talwalkar, A. S. Model agnostic supervised local explanations. Adv. Neural Inf. Process. Syst. 31, 2520\u20132529 (2018).","journal-title":"Adv. Neural Inf. Process. Syst."},{"key":"138_CR36","doi-asserted-by":"publisher","first-page":"65","DOI":"10.1007\/BF01769885","volume":"14","author":"HP Young","year":"1985","unstructured":"Young, H. P. Monotonic solutions of cooperative games. Int. J. Game Theor. 14, 65\u201372 (1985).","journal-title":"Int. J. Game Theor."},{"key":"138_CR37","unstructured":"Ancona, M., Ceolini, E., Oztireli, C. & Gross, M. Towards better understanding of gradient-based attribution methods for deep neural networks. In Proc. 6th International Conference on Learning Representations (ICLR 2018) (2018)."},{"key":"138_CR38","unstructured":"Hooker, S., Erhan, D., Kindermans, P.-J. & Kim, B. A benchmark for interpretability methods in deep neural networks. In Conference on Neural Information Processing Systems (NIPS, 2019)."},{"key":"138_CR39","unstructured":"Shrikumar, A., Greenside, P., Shcherbina, A. & Kundaje, A. Not just a black box: learning important features through propagating activation differences. Preprint at https:\/\/arxiv.org\/abs\/1605.01713 (2016)."},{"key":"138_CR40","doi-asserted-by":"publisher","DOI":"10.1186\/1471-2156-5-32","volume":"5","author":"KL Lunetta","year":"2004","unstructured":"Lunetta, K. L., Hayward, L. B., Segal, J. & Van Eerdewegh, P. Screening large-scale association study data: exploiting interactions using random forests. BMC Genet. 5, 32 (2004).","journal-title":"BMC Genet."},{"key":"138_CR41","doi-asserted-by":"publisher","DOI":"10.1186\/1471-2105-10-S1-S65","volume":"10","author":"R Jiang","year":"2009","unstructured":"Jiang, R., Tang, W., Wu, X. & Fu, W. A random forest approach to the detection of epistatic interactions in case-control studies. BMC Bioinformatics 10, S65 (2009).","journal-title":"BMC Bioinformatics"}],"container-title":["Nature Machine Intelligence"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/www.nature.com\/articles\/s42256-019-0138-9.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/www.nature.com\/articles\/s42256-019-0138-9","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/www.nature.com\/articles\/s42256-019-0138-9.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2022,12,6]],"date-time":"2022-12-06T22:29:30Z","timestamp":1670365770000},"score":1,"resource":{"primary":{"URL":"https:\/\/www.nature.com\/articles\/s42256-019-0138-9"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2020,1,17]]},"references-count":41,"journal-issue":{"issue":"1","published-online":{"date-parts":[[2020,1]]}},"alternative-id":["138"],"URL":"https:\/\/doi.org\/10.1038\/s42256-019-0138-9","relation":{"has-part":[{"id-type":"doi","id":"10.5194\/amt-15-3663-2022","asserted-by":"object"}]},"ISSN":["2522-5839"],"issn-type":[{"value":"2522-5839","type":"electronic"}],"subject":[],"published":{"date-parts":[[2020,1,17]]},"assertion":[{"value":"27 June 2019","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"6 December 2019","order":2,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"17 January 2020","order":3,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"The authors declare no competing interests.","order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Competing interests"}}]}}