{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,5,13]],"date-time":"2026-05-13T17:20:56Z","timestamp":1778692856213,"version":"3.51.4"},"reference-count":157,"publisher":"Institute of Electrical and Electronics Engineers (IEEE)","license":[{"start":{"date-parts":[[2020,1,1]],"date-time":"2020-01-01T00:00:00Z","timestamp":1577836800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/ieeexplore.ieee.org\/Xplorehelp\/downloads\/license-information\/IEEE.html"},{"start":{"date-parts":[[2020,1,1]],"date-time":"2020-01-01T00:00:00Z","timestamp":1577836800000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2020,1,1]],"date-time":"2020-01-01T00:00:00Z","timestamp":1577836800000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"}],"funder":[{"name":"Hong Kong RGC CRF","award":["No. C6030-18F"],"award-info":[{"award-number":["No. C6030-18F"]}]}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["IEEE Trans. Knowl. Data Eng."],"published-print":{"date-parts":[[2020]]},"DOI":"10.1109\/tkde.2020.2983930","type":"journal-article","created":{"date-parts":[[2020,3,30]],"date-time":"2020-03-30T22:35:26Z","timestamp":1585607726000},"page":"1-1","source":"Crossref","is-referenced-by-count":126,"title":["A Survey of Data-driven and Knowledge-aware eXplainable AI"],"prefix":"10.1109","author":[{"given":"Xiao-Hui","family":"Li","sequence":"first","affiliation":[]},{"given":"Caleb Chen","family":"Cao","sequence":"additional","affiliation":[]},{"given":"Yuhan","family":"Shi","sequence":"additional","affiliation":[]},{"given":"Wei","family":"Bai","sequence":"additional","affiliation":[]},{"given":"Han","family":"Gao","sequence":"additional","affiliation":[]},{"given":"Luyu","family":"Qiu","sequence":"additional","affiliation":[]},{"given":"Cong","family":"Wang","sequence":"additional","affiliation":[]},{"given":"Yuanyuan","family":"Gao","sequence":"additional","affiliation":[]},{"given":"Shenjia","family":"Zhang","sequence":"additional","affiliation":[]},{"given":"Xun","family":"Xue","sequence":"additional","affiliation":[]},{"given":"Lei","family":"Chen","sequence":"additional","affiliation":[]}],"member":"263","reference":[{"key":"ref1","doi-asserted-by":"publisher","DOI":"10.1007\/s11263-015-0816-y"},{"key":"ref2","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-10602-1_48"},{"key":"ref3","doi-asserted-by":"publisher","DOI":"10.1145\/3065386"},{"key":"ref4","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2015.279"},{"key":"ref5","article-title":"Neural machine translation by jointly learning to align and translate","author":"Bahdanau","year":"2014","journal-title":"arXiv:1409.0473"},{"key":"ref6","doi-asserted-by":"publisher","DOI":"10.1038\/nature16961"},{"key":"ref7","doi-asserted-by":"publisher","DOI":"10.1126\/science.aay2400"},{"key":"ref9","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2017.195"},{"key":"ref10","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2016.90"},{"key":"ref11","first-page":"91","article-title":"Faster R-CNN: Towards real-time object detection with region proposal networks","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Ren"},{"key":"ref12","doi-asserted-by":"publisher","DOI":"10.1109\/TNNLS.2018.2886017"},{"key":"ref13","article-title":"Evaluating explanation without ground truth in interpretable machine learning","author":"Yang","year":"2019"},{"key":"ref14","doi-asserted-by":"publisher","DOI":"10.1145\/2939672.2939778"},{"key":"ref15","article-title":"Distilling the knowledge in a neural network","author":"Hinton","year":"2015"},{"key":"ref16","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2017.74"},{"key":"ref17","first-page":"4765","article-title":"A unified approach to interpreting model predictions","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Lundberg"},{"key":"ref18","article-title":"Metrics for explainable AI: Challenges and prospects","author":"Hoffman","year":"2018"},{"key":"ref19","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-01216-8_35"},{"key":"ref20","doi-asserted-by":"publisher","DOI":"10.1145\/3359786"},{"key":"ref21","doi-asserted-by":"publisher","DOI":"10.1109\/ACCESS.2018.2870052"},{"key":"ref22","doi-asserted-by":"publisher","DOI":"10.1145\/3236009"},{"key":"ref23","doi-asserted-by":"publisher","DOI":"10.23919\/MIPRO.2018.8400040"},{"key":"ref24","article-title":"Explanation and justification in machine learning: A survey","volume-title":"Proc. IJCAI Workshop Explainable AI","volume":"8","author":"Biran"},{"key":"ref25","volume-title":"Interpretable Machine Learning","author":"Molnar","year":"2019"},{"key":"ref26","article-title":"Towards a rigorous science of interpretable machine learning","author":"Doshi-Velez","year":"2017"},{"key":"ref27","doi-asserted-by":"publisher","DOI":"10.1109\/DSAA.2018.00018"},{"key":"ref29","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v30i1.10314"},{"key":"ref30","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v32i1.11573"},{"key":"ref31","article-title":"Embedding entities and relations for learning and inference in knowledge bases","author":"Yang","year":"2014"},{"key":"ref32","doi-asserted-by":"publisher","DOI":"10.14711\/thesis-991012786067703412"},{"key":"ref33","doi-asserted-by":"publisher","DOI":"10.1109\/TNNLS.2020.3027314"},{"key":"ref34","first-page":"24","article-title":"Extracting tree-structured representations of trained networks","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Craven"},{"key":"ref35","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-46307-0_29"},{"key":"ref36","article-title":"Distilling a neural network into a soft decision tree","author":"Frosst","year":"2017"},{"key":"ref37","article-title":"All models are wrong but many are useful: Variable importance for black-box, proprietary, or misspecified prediction models, using model class reliance","author":"Fisher","year":"2018"},{"key":"ref38","first-page":"3387","article-title":"Synthesizing the preferred inputs for neurons in neural networks via deep generator networks","volume-title":"Proc. 29th Int. Conf. Neural Inf. Process. Syst.","author":"Nguyen"},{"key":"ref39","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2018.00920"},{"key":"ref40","article-title":"Towards interpretable R-CNN by unfolding latent structures","author":"Wu","year":"2017"},{"key":"ref41","article-title":"Local rule-based explanations of black box decision systems","author":"Guidotti","year":"2018"},{"key":"ref42","first-page":"1803","article-title":"How to explain individual classification decisions","volume":"11","author":"Baehrens","year":"2010","journal-title":"J. Mach. Learn. Res."},{"key":"ref43","article-title":"Deep inside convolutional networks: Visualising image classification models and saliency maps","volume-title":"Proc. Workshop Int. Conf. Learn. Representations","author":"Simonyan"},{"key":"ref44","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-10590-1_53"},{"key":"ref45","article-title":"Axiomatic attribution for deep networks","author":"Sundararajan","year":"2017"},{"key":"ref46","doi-asserted-by":"publisher","DOI":"10.1214\/11-AOAS495"},{"key":"ref47","first-page":"2280","article-title":"Examples are not enough, learn to criticize! criticism for interpretability","volume-title":"Proc. 29th Int. Conf. Neural Inf. Process. Syst.","author":"Kim"},{"key":"ref48","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v32i1.11771"},{"key":"ref49","article-title":"Counterfactual explanations without opening the black box: Automated decisions and the gpdr","volume":"31","author":"Wachter","year":"2017","journal-title":"Harvard J. Law Technol."},{"key":"ref50","doi-asserted-by":"publisher","DOI":"10.1109\/GlobalSIP45357.2019.8969491"},{"key":"ref51","first-page":"2668","article-title":"Interpretability beyond feature attribution: Quantitative testing with concept activation vectors (TCAV)","volume-title":"Proc. 35th Int. Conf. Mach. Learn.","volume":"80","author":"Kim"},{"key":"ref52","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-01237-3_8"},{"key":"ref53","doi-asserted-by":"publisher","DOI":"10.1145\/3278721.3278736"},{"key":"ref54","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v33i01.33015329"},{"key":"ref55","doi-asserted-by":"publisher","DOI":"10.1145\/3308558.3313607"},{"key":"ref56","first-page":"55","article-title":"Enabling trust in clinical decision support recommendations through semantics","volume":"2477","author":"Seneviratne","year":"2019","journal-title":"Proc. Workshop Semantic Web Solutions Large-Scale Biomed. Data Analytics co-located 18th Int. Semantic Web Conf. (ISWC) 2019"},{"key":"ref57","article-title":"Propositional rule extraction from neural networks under background knowledge","volume-title":"Proc. Int. Workshop Neural-Symbolic Learn. Reasoning","author":"Labaf"},{"key":"ref58","article-title":"Explaining trained neural networks with Semantic Web technologies: First steps","author":"Sarker","year":"2017"},{"key":"ref59","volume-title":"Data Mining: Concepts and Techniques","author":"Han","year":"2011"},{"key":"ref60","doi-asserted-by":"publisher","DOI":"10.1016\/S0031-3203(98)00181-2"},{"key":"ref61","doi-asserted-by":"publisher","DOI":"10.1145\/775047.775113"},{"key":"ref62","doi-asserted-by":"publisher","DOI":"10.1109\/CIDM.2009.4938655"},{"key":"ref63","article-title":"Concept tree: High-level representation of variables for more interpretable surrogate decision trees","author":"Renard","year":"2019"},{"key":"ref64","doi-asserted-by":"publisher","DOI":"10.1016\/B978-1-55860-335-6.50013-1"},{"key":"ref65","doi-asserted-by":"publisher","DOI":"10.1016\/0950-7051(96)81920-4"},{"key":"ref66","first-page":"13","article-title":"Rule extraction from trained neural networks using genetic programming","volume-title":"Proc. 13th Int. Conf. Artif. Neural Netw.","author":"Johansson"},{"key":"ref67","first-page":"658","article-title":"The truth is in there-rule extraction from opaque models using genetic programming","volume-title":"Proc. 17th Int. Florida Artif. Intell. Res. Soc. Conf.","author":"Johansson"},{"issue":"1","key":"ref68","first-page":"3","article-title":"Extracting symbolic rules from trained neural network ensembles","volume":"16","author":"Zhou","year":"2003","journal-title":"AI Commun."},{"key":"ref69","doi-asserted-by":"publisher","DOI":"10.1007\/s11063-011-9207-8"},{"key":"ref70","doi-asserted-by":"publisher","DOI":"10.1109\/IJCNN.2001.938448"},{"key":"ref71","doi-asserted-by":"publisher","DOI":"10.1109\/ICDMW.2018.00132"},{"key":"ref72","article-title":"Detecting bias in black-box models using transparent model distillation","volume-title":"Proc. AAAI\/ACM Conf. Artif. Intell. Ethics Soc.","author":"Tan"},{"key":"ref73","article-title":"Learning global additive explanations for neural nets using model distillation","author":"Tan","year":"2018"},{"key":"ref74","doi-asserted-by":"publisher","DOI":"10.1023\/A:I0I0933404324"},{"key":"ref75","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-10925-7_40"},{"issue":"3","key":"ref76","first-page":"1","article-title":"Visualizing higher-layer features of a deep network","volume":"1341","author":"Erhan","year":"2009","journal-title":"Univ. Montreal"},{"key":"ref77","doi-asserted-by":"publisher","DOI":"10.1007\/s11263-016-0911-8"},{"key":"ref78","article-title":"Understanding intra-class knowledge inside CNN","author":"Wei","year":"2015","journal-title":"arXiv:1507.02379"},{"key":"ref79","article-title":"Understanding neural networks through deep visualization","volume-title":"Proc. 31st Int. Conf. Mach. Learn. Workshop Deep Learn.","author":"Yosinski"},{"key":"ref80","article-title":"Multifaceted feature visualization: Uncovering the different types of features learned by each neuron in deep neural networks","author":"Nguyen","year":"2016","journal-title":"arXiv:1602.03616"},{"key":"ref81","article-title":"Inceptionism: Going deeper into neural networks","author":"Mordvintsev","year":"2015","journal-title":"Google Res. Blog"},{"key":"ref82","first-page":"2672","article-title":"Generative adversarial nets","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Goodfellow"},{"key":"ref83","doi-asserted-by":"publisher","DOI":"10.2307\/2699986"},{"key":"ref84","doi-asserted-by":"publisher","DOI":"10.1111\/rssb.12377"},{"key":"ref85","doi-asserted-by":"publisher","DOI":"10.4310\/amsa.2018.v3.n1.a5"},{"key":"ref86","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v32i1.11491"},{"key":"ref87","article-title":"Verifiable reinforcement learning via policy extraction","author":"Bastani","year":"2019"},{"key":"ref88","article-title":"TreeView: Peeking into deep neural networks via feature-space partitioning","author":"Thiagarajan","year":"2016","journal-title":"arXiv:1611.07429"},{"key":"ref89","article-title":"GNNExplainer: Generating explanations for graph neural networks","author":"Ying","year":"2019"},{"key":"ref90","article-title":"SmoothGrad: Removing noise by adding noise","author":"Smilkov","year":"2017"},{"key":"ref91","article-title":"Striving for simplicity: The all convolutional net","author":"Springenberg","year":"2015","journal-title":"arXiv:1412.6806"},{"key":"ref92","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2016.319"},{"key":"ref93","doi-asserted-by":"publisher","DOI":"10.1371\/journal.pone.0130140"},{"key":"ref94","first-page":"3145","article-title":"Learning important features through propagating activation differences","volume-title":"Proc. 34th Int. Conf. Mach. Learn.","author":"Shrikumar"},{"key":"ref95","first-page":"592","article-title":"Explanations based on the missing: Towards contrastive explanations with Pertinent Negatives","volume-title":"Proc. 31st Int. Conf. Neural Inf. Process. Syst.","author":"Dhurandhar"},{"key":"ref96","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2017.371"},{"key":"ref97","first-page":"6967","article-title":"Real time image saliency for black box classifiers","volume-title":"Proc. 30th Int. Conf. Neural Inf. Process. Syst.","author":"Dabkowski"},{"key":"ref98","doi-asserted-by":"publisher","DOI":"10.1515\/9781400881970-018"},{"key":"ref99","article-title":"Explainable AI for trees: From local explanations to global understanding","author":"Lundberg","year":"2019"},{"key":"ref100","doi-asserted-by":"publisher","DOI":"10.1080\/10618600.2014.907095"},{"key":"ref101","first-page":"1952","article-title":"The Bayesian case model: A generative approach for case-based reasoning and prototype classification","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Kim"},{"key":"ref102","doi-asserted-by":"publisher","DOI":"10.1109\/ICDM.2019.00036"},{"key":"ref103","article-title":"Streaming methods for restricted strongly convex functions with applications to prototype selection","author":"Gurumoorthy","year":"2018"},{"key":"ref104","doi-asserted-by":"publisher","DOI":"10.1109\/TVCG.2019.2934267"},{"key":"ref105","doi-asserted-by":"publisher","DOI":"10.1145\/3292500.3330908"},{"key":"ref106","volume-title":"The Book of Why: The New Science of Cause and Effect","author":"Pearl","year":"2018"},{"key":"ref107","article-title":"Interpretable credit application predictions with counterfactual explanations","author":"Grath","year":"2018"},{"key":"ref108","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2015.425"},{"key":"ref109","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-86520-7_40"},{"key":"ref110","article-title":"Inverse classification for comparison-based interpretability in machine learning","author":"Laugel","year":"2017"},{"key":"ref111","article-title":"Measurable counterfactual local explanations for any classifier","author":"White","year":"2019"},{"key":"ref112","doi-asserted-by":"publisher","DOI":"10.1002\/hbe2.117"},{"key":"ref113","doi-asserted-by":"publisher","DOI":"10.1145\/3290605.3300234"},{"key":"ref114","article-title":"Interactively transferring CNN patterns for part localization","author":"Zhang","year":"2017"},{"key":"ref115","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2017.354"},{"key":"ref116","doi-asserted-by":"publisher","DOI":"10.1145\/3301275.3302316"},{"key":"ref117","first-page":"1956","article-title":"Guiding reinforcement learning exploration using natural language","volume-title":"Proc. 17th Int. Conf. Auton. Agents MultiAgent Syst.","author":"Harrison"},{"key":"ref118","article-title":"Learning to generate natural language rationales for game playing agents","volume-title":"Proc. AAAI Conf. Artif. Intell. Interactive Digit. Entertainment Workshops","volume":"2282","author":"Ehsan"},{"key":"ref119","doi-asserted-by":"publisher","DOI":"10.24963\/ijcai.2017\/593"},{"key":"ref120","doi-asserted-by":"publisher","DOI":"10.3390\/a11090137"},{"key":"ref121","doi-asserted-by":"publisher","DOI":"10.1145\/3209978.3210017"},{"key":"ref122","article-title":"","author":"Pa\u00e7ac\u0131","year":"2019"},{"key":"ref123","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v33i01.33013036"},{"key":"ref124","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2017.544"},{"key":"ref125","article-title":"A survey ofevaluation methods and measures for interpretable machine learning","author":"Mohseni","year":"2018"},{"key":"ref126","article-title":"Explanation in Human-AI systems: A literature meta-review, synopsis of key ideas and publications, and bibliography for explainable AI","author":"Mueller","year":"2019"},{"key":"ref127","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2019.00642"},{"key":"ref128","doi-asserted-by":"publisher","DOI":"10.1145\/3308558.3313545"},{"key":"ref129","article-title":"On the robustness of interpretability methods","author":"Alvarez-Melis","year":"2018"},{"key":"ref130","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-28954-6_14"},{"key":"ref131","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v33i01.33013681"},{"key":"ref132","first-page":"2921","article-title":"Fooling neural network interpretations via adversarial model manipulation","volume-title":"Proc. 33rd Int. Conf. Neural Inf. Process. Syst.","author":"Heo"},{"key":"ref133","article-title":"Interpretable deep learning under fire","author":"Zhang","year":"2019"},{"key":"ref134","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2001.937505"},{"key":"ref135","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2009.5206848"},{"key":"ref136","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-46493-0_33"},{"key":"ref137","doi-asserted-by":"publisher","DOI":"10.1145\/1518701.1519023"},{"key":"ref138","doi-asserted-by":"publisher","DOI":"10.1145\/2207676.2207678"},{"key":"ref139","doi-asserted-by":"publisher","DOI":"10.1109\/VLHCC.2013.6645235"},{"key":"ref140","doi-asserted-by":"publisher","DOI":"10.1145\/2678025.2701399"},{"key":"ref141","doi-asserted-by":"publisher","DOI":"10.1016\/j.ijhcs.2013.12.007"},{"key":"ref142","doi-asserted-by":"publisher","DOI":"10.1145\/1620545.1620576"},{"key":"ref143","article-title":"How do humans understand explanations from machine learning systems? An evaluation of the human-interpretability of explanation","author":"Narayanan","year":"2018"},{"key":"ref144","doi-asserted-by":"publisher","DOI":"10.1007\/s11257-011-9117-5"},{"key":"ref145","doi-asserted-by":"publisher","DOI":"10.1145\/1378773.1378804"},{"key":"ref146","article-title":"The promise and peril of human evaluation for model interpretability","author":"Herman","year":"2017"},{"key":"ref147","article-title":"End to end learning for self-driving cars","author":"Bojarski","year":"2016","journal-title":"arXiv:1604.07316"},{"key":"ref148","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2017.320"},{"key":"ref149","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2019.01084"},{"key":"ref150","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2019.00886"},{"key":"ref151","doi-asserted-by":"publisher","DOI":"10.1007\/978-1-4899-7668-0"},{"key":"ref152","article-title":"Effects of augmented situational awareness on driver trust in semi-autonomous vehicle operation","author":"Petersen","year":"2017"},{"key":"ref153","doi-asserted-by":"publisher","DOI":"10.1145\/3173386.3177057"},{"key":"ref154","doi-asserted-by":"publisher","DOI":"10.1109\/RE.2018.00-21"},{"key":"ref155","doi-asserted-by":"publisher","DOI":"10.1016\/j.techfore.2015.12.014"},{"key":"ref156","doi-asserted-by":"publisher","DOI":"10.18178\/ijimt.2018.9.2.791"},{"key":"ref157","doi-asserted-by":"publisher","DOI":"10.2139\/ssrn.3435104"},{"key":"ref158","article-title":"An interpretable model with globally consistent explanations for credit risk","author":"Chen","year":"2018"},{"key":"ref163","doi-asserted-by":"publisher","DOI":"10.1016\/j.websem.2017.05.003"}],"container-title":["IEEE Transactions on Knowledge and Data Engineering"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx7\/69\/4358933\/09050829.pdf?arnumber=9050829","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,1,9]],"date-time":"2024-01-09T22:23:23Z","timestamp":1704839003000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/9050829\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2020]]},"references-count":157,"URL":"https:\/\/doi.org\/10.1109\/tkde.2020.2983930","relation":{},"ISSN":["1041-4347","1558-2191","2326-3865"],"issn-type":[{"value":"1041-4347","type":"print"},{"value":"1558-2191","type":"electronic"},{"value":"2326-3865","type":"electronic"}],"subject":[],"published":{"date-parts":[[2020]]}}}