{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,8,1]],"date-time":"2026-08-01T21:12:42Z","timestamp":1785618762631,"version":"3.56.0"},"reference-count":208,"publisher":"Institute of Electrical and Electronics Engineers (IEEE)","issue":"3","license":[{"start":{"date-parts":[[2021,3,1]],"date-time":"2021-03-01T00:00:00Z","timestamp":1614556800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/legalcode"}],"funder":[{"name":"Institute of Information & Communications Technology Planning & Evaluation (IITP) grants funded by the Korea Government","award":["2017-0-00451"],"award-info":[{"award-number":["2017-0-00451"]}]},{"name":"Institute of Information & Communications Technology Planning & Evaluation (IITP) grants funded by the Korea Government","award":["2019-0-00079"],"award-info":[{"award-number":["2019-0-00079"]}]},{"DOI":"10.13039\/501100002347","name":"German Ministry for Education and Research","doi-asserted-by":"publisher","award":["01IS14013A-E"],"award-info":[{"award-number":["01IS14013A-E"]}],"id":[{"id":"10.13039\/501100002347","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100002347","name":"German Ministry for Education and Research","doi-asserted-by":"publisher","award":["01GQ1115"],"award-info":[{"award-number":["01GQ1115"]}],"id":[{"id":"10.13039\/501100002347","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100002347","name":"German Ministry for Education and Research","doi-asserted-by":"publisher","award":["01GQ0850"],"award-info":[{"award-number":["01GQ0850"]}],"id":[{"id":"10.13039\/501100002347","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100002347","name":"German Ministry for Education and Research","doi-asserted-by":"publisher","award":["01IS18025A"],"award-info":[{"award-number":["01IS18025A"]}],"id":[{"id":"10.13039\/501100002347","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100002347","name":"German Ministry for Education and Research","doi-asserted-by":"publisher","award":["01IS18037A"],"award-info":[{"award-number":["01IS18037A"]}],"id":[{"id":"10.13039\/501100002347","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100001659","name":"German Research Foundation","doi-asserted-by":"publisher","award":["Math+"],"award-info":[{"award-number":["Math+"]}],"id":[{"id":"10.13039\/501100001659","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100001659","name":"German Research Foundation","doi-asserted-by":"publisher","award":["EXC 2046\/1"],"award-info":[{"award-number":["EXC 2046\/1"]}],"id":[{"id":"10.13039\/501100001659","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100001659","name":"German Research Foundation","doi-asserted-by":"publisher","award":["390685689"],"award-info":[{"award-number":["390685689"]}],"id":[{"id":"10.13039\/501100001659","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["Proc. IEEE"],"published-print":{"date-parts":[[2021,3]]},"DOI":"10.1109\/jproc.2021.3060483","type":"journal-article","created":{"date-parts":[[2021,3,4]],"date-time":"2021-03-04T20:32:09Z","timestamp":1614889929000},"page":"247-278","source":"Crossref","is-referenced-by-count":1265,"title":["Explaining Deep Neural Networks and Beyond: A Review of Methods and Applications"],"prefix":"10.1109","volume":"109","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-6283-3265","authenticated-orcid":false,"given":"Wojciech","family":"Samek","sequence":"first","affiliation":[{"name":"Department of Artificial Intelligence, Fraunhofer Heinrich Hertz Institute, Berlin, Germany"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-7243-6186","authenticated-orcid":false,"given":"Gregoire","family":"Montavon","sequence":"additional","affiliation":[{"name":"BIFOLD\u2714Berlin Institute for the Foundations of Learning and Data, Berlin, Germany"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-0762-7258","authenticated-orcid":false,"given":"Sebastian","family":"Lapuschkin","sequence":"additional","affiliation":[{"name":"Department of Artificial Intelligence, Fraunhofer Heinrich Hertz Institute, Berlin, Germany"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Christopher J.","family":"Anders","sequence":"additional","affiliation":[{"name":"BIFOLD\u2714Berlin Institute for the Foundations of Learning and Data, Berlin, Germany"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-3861-7685","authenticated-orcid":false,"given":"Klaus-Robert","family":"Muller","sequence":"additional","affiliation":[{"name":"BIFOLD\u2714Berlin Institute for the Foundations of Learning and Data, Berlin, Germany"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"263","reference":[{"key":"ref170","first-page":"3145","article-title":"Learning important features through propagating activation differences","author":"shrikumar","year":"2017","journal-title":"Proc 34th Int Conf Mach Learn (ICML)"},{"key":"ref172","article-title":"Computationally efficient measures of internal neuron importance","author":"shrikumar","year":"2018","journal-title":"arXiv 1807 09946"},{"key":"ref171","article-title":"Not just a black box: Learning important features through propagating activation differences","author":"shrikumar","year":"2016","journal-title":"arXiv 1605 01713"},{"key":"ref174","article-title":"Deep inside convolutional networks: Visualising image classification models and saliency maps","author":"simonyan","year":"2014","journal-title":"Proc Int Conf Learn Represent Workshops"},{"key":"ref173","doi-asserted-by":"publisher","DOI":"10.1109\/TPAMI.2018.2885764"},{"key":"ref176","article-title":"SmoothGrad: Removing noise by adding noise","author":"smilkov","year":"2017","journal-title":"arXiv 1706 03825"},{"key":"ref175","article-title":"Very deep convolutional networks for large-scale image recognition","author":"simonyan","year":"2015","journal-title":"Proc 3rd Int Conf Learn Represent"},{"key":"ref178","article-title":"Striving for simplicity: The all convolutional net","author":"springenberg","year":"2015","journal-title":"Proc Int Conf Learn Represent (ICLR)"},{"key":"ref177","doi-asserted-by":"publisher","DOI":"10.1371\/journal.pone.0100335"},{"key":"ref168","doi-asserted-by":"publisher","DOI":"10.1145\/1878039.1878045"},{"key":"ref169","doi-asserted-by":"publisher","DOI":"10.1515\/9781400881970-018"},{"key":"ref39","first-page":"13567","article-title":"Explanations can be manipulated and geometry is to blame","author":"dombrowski","year":"2019","journal-title":"Proc Adv Neural Inf Process Syst"},{"key":"ref38","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/P17-1106"},{"key":"ref33","doi-asserted-by":"publisher","DOI":"10.1111\/j.2517-6161.1972.tb00899.x"},{"key":"ref32","article-title":"Plan operation NHANES I epidemiologic followup study, 1987","author":"cox","year":"1992"},{"key":"ref31","doi-asserted-by":"publisher","DOI":"10.1109\/SPW.2018.00011"},{"key":"ref30","author":"chollet","year":"2015","journal-title":"Keras"},{"key":"ref37","doi-asserted-by":"publisher","DOI":"10.1145\/1014052.1014118"},{"key":"ref36","article-title":"How important is a neuron?","author":"dhamdhere","year":"2018","journal-title":"arXiv 1805 12233"},{"key":"ref35","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2009.5206848"},{"key":"ref34","first-page":"1087","article-title":"Learning global pairwise interactions with Bayesian neural networks","volume":"325","author":"cui","year":"2020","journal-title":"Proc 24th Eur Conf Artif Intell"},{"key":"ref181","doi-asserted-by":"publisher","DOI":"10.23915\/distill.00022"},{"key":"ref180","doi-asserted-by":"publisher","DOI":"10.1016\/j.jneumeth.2016.10.008"},{"key":"ref185","article-title":"Intriguing properties of neural networks","author":"szegedy","year":"2014","journal-title":"Proc Int Conf Learn Represent (ICLR)"},{"key":"ref184","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-77927-5_24"},{"key":"ref183","first-page":"3319","article-title":"Axiomatic attribution for deep networks","author":"sundararajan","year":"2017","journal-title":"Proc 34th Int Conf Mach Learn"},{"key":"ref182","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-01258-8_39"},{"key":"ref189","doi-asserted-by":"publisher","DOI":"10.1145\/3287560.3287566"},{"key":"ref188","article-title":"Detecting statistical interactions from neural network weights","author":"tsang","year":"2018","journal-title":"Proc 6th Int Conf Learn Represent"},{"key":"ref187","article-title":"The frequency range of the voice fundamental in the speech of male and female adults","author":"traunm\u00fcller","year":"1995"},{"key":"ref186","doi-asserted-by":"publisher","DOI":"10.3389\/fnins.2019.01321"},{"key":"ref28","doi-asserted-by":"publisher","DOI":"10.1145\/2783258.2788613"},{"key":"ref27","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-01264-9_9"},{"key":"ref179","first-page":"1","article-title":"An efficient explanation of individual classifications using game theory","volume":"11","author":"strumbelj","year":"2010","journal-title":"J Mach Learn Res"},{"key":"ref29","doi-asserted-by":"publisher","DOI":"10.1016\/j.cor.2008.04.004"},{"key":"ref20","article-title":"Interpreting and explaining deep neural networks for classification of audio signals","author":"becker","year":"2018","journal-title":"arXiv 1807 03418"},{"key":"ref22","doi-asserted-by":"crossref","DOI":"10.1038\/s42256-021-00303-4","article-title":"Morphological and molecular breast cancer profiling through explainable machine learning","author":"binder","year":"2021","journal-title":"Nature Mach Intell"},{"key":"ref21","first-page":"908","article-title":"Safe model-based reinforcement learning with stability guarantees","author":"berkenkamp","year":"2017","journal-title":"Proc Adv Neural Inf Process Syst"},{"key":"ref24","doi-asserted-by":"publisher","DOI":"10.1109\/MSP.2008.4408441"},{"key":"ref23","author":"bishop","year":"1996","journal-title":"Neural Networks for Pattern Recognition"},{"key":"ref26","doi-asserted-by":"publisher","DOI":"10.1109\/SP.2017.49"},{"key":"ref25","article-title":"Approximating CNNs with bag-of-local-features models works surprisingly well on imagenet","author":"brendel","year":"2019","journal-title":"Proc 7th Int Conf Learn Represent"},{"key":"ref50","doi-asserted-by":"publisher","DOI":"10.1073\/pnas.1716999115"},{"key":"ref51","first-page":"315","article-title":"Deep sparse rectifier neural networks","volume":"15","author":"glorot","year":"2011","journal-title":"Proc 14th Int Conf Artif Intell Statist"},{"key":"ref154","doi-asserted-by":"publisher","DOI":"10.1038\/s42256-019-0048-x"},{"key":"ref153","doi-asserted-by":"publisher","DOI":"10.1109\/ICCVW.2015.41"},{"key":"ref156","first-page":"4393","article-title":"Deep one-class classification","author":"ruff","year":"2018","journal-title":"Proc Int Conf Mach Learn"},{"key":"ref155","doi-asserted-by":"publisher","DOI":"10.1109\/JPROC.2021.3052449"},{"key":"ref150","doi-asserted-by":"publisher","DOI":"10.1109\/ACCESS.2020.2976199"},{"key":"ref152","doi-asserted-by":"publisher","DOI":"10.24963\/ijcai.2017\/371"},{"key":"ref151","doi-asserted-by":"publisher","DOI":"10.1037\/h0042519"},{"key":"ref146","doi-asserted-by":"publisher","DOI":"10.1038\/s41591-020-0870-z"},{"key":"ref147","doi-asserted-by":"publisher","DOI":"10.1145\/2939672.2939778"},{"key":"ref148","first-page":"1527","article-title":"Anchors: High-precision model-agnostic explanations","volume":"18","author":"ribeiro","year":"2018","journal-title":"Proc AAAI Conf Artif Intell"},{"key":"ref149","first-page":"115","article-title":"Structuring neural networks for more explainable predictions","author":"rieger","year":"2019","journal-title":"Explainable and Interpretable Models in Computer Vision and Machine Learning"},{"key":"ref59","doi-asserted-by":"publisher","DOI":"10.1145\/3236009"},{"key":"ref58","article-title":"Local rule-based explanations of black box decision systems","author":"guidotti","year":"2018","journal-title":"arXiv 1805 10820"},{"key":"ref57","first-page":"119","article-title":"Understanding individual decisions of CNNs via contrastive backpropagation","volume":"11363","author":"gu","year":"2018","journal-title":"Proc 14th Asian Conf Comput Vis in Lecture Notes in Computer Science"},{"key":"ref56","article-title":"Contextual prediction difference analysis for explaining individual image classifications","author":"gu","year":"2019","journal-title":"arXiv 1910 09086"},{"key":"ref55","doi-asserted-by":"publisher","DOI":"10.1007\/s001820050125"},{"key":"ref54","doi-asserted-by":"publisher","DOI":"10.1609\/aimag.v38i3.2741"},{"key":"ref53","article-title":"Explaining and harnessing adversarial examples","author":"goodfellow","year":"2015","journal-title":"Proc 3rd Int Conf Learn Represent"},{"key":"ref52","author":"goodfellow","year":"2016","journal-title":"Deep Learning"},{"key":"ref40","doi-asserted-by":"publisher","DOI":"10.7551\/mitpress\/7493.001.0001"},{"key":"ref167","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2017.74"},{"key":"ref166","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-28954-6_17"},{"key":"ref165","doi-asserted-by":"publisher","DOI":"10.1038\/ncomms13890"},{"key":"ref164","doi-asserted-by":"publisher","DOI":"10.1162\/089976698300017467"},{"key":"ref163","author":"sch\u00f6lkopf","year":"2002","journal-title":"Learning With Kernels Support Vector Machines Regularization Optimization and Beyond"},{"key":"ref162","article-title":"Higher-order explanations of graph neural networks via relevant walks","author":"schnake","year":"2020","journal-title":"arXiv 2006 03589"},{"key":"ref161","doi-asserted-by":"publisher","DOI":"10.1016\/j.neunet.2014.09.003"},{"key":"ref160","doi-asserted-by":"publisher","DOI":"10.1109\/TNN.2008.2005605"},{"key":"ref4","article-title":"Towards better understanding of gradient-based attribution methods for deep neural networks","author":"ancona","year":"2018","journal-title":"Proc Int Conf Learn Represent (ICLR)"},{"key":"ref3","first-page":"93:1","article-title":"iNNvestigate neural networks","volume":"20","author":"alber","year":"2019","journal-title":"J Mach Learn Res"},{"key":"ref6","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-28954-6_16"},{"key":"ref5","first-page":"272","article-title":"Explaining deep neural networks with a polynomial time algorithm for Shapley value approximation","volume":"97","author":"ancona","year":"2019","journal-title":"Proc 36th Int Conf Mach Learn"},{"key":"ref8","first-page":"13544","article-title":"Rudder: Return decomposition for delayed rewards","author":"arjona-medina","year":"2019","journal-title":"Proc Adv Neural Inf Process Syst"},{"key":"ref159","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-28954-6"},{"key":"ref7","article-title":"Finding and removing clever hans: Using explanation methods to debug and improve deep models","author":"anders","year":"2019","journal-title":"arXiv 1912 11425"},{"key":"ref49","doi-asserted-by":"publisher","DOI":"10.1145\/2523813"},{"key":"ref157","doi-asserted-by":"publisher","DOI":"10.1007\/s11263-015-0816-y"},{"key":"ref9","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-28954-6_11"},{"key":"ref158","doi-asserted-by":"publisher","DOI":"10.1109\/TNNLS.2016.2599820"},{"key":"ref46","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2019.00304"},{"key":"ref45","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-98131-4"},{"key":"ref48","doi-asserted-by":"publisher","DOI":"10.1109\/ISBI.2019.8759236"},{"key":"ref47","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2017.371"},{"key":"ref42","doi-asserted-by":"publisher","DOI":"10.1109\/TPAMI.2020.3020738"},{"key":"ref41","article-title":"Towards a rigorous science of interpretable machine learning","author":"doshi-velez","year":"2017","journal-title":"arXiv 1702 08608"},{"key":"ref44","doi-asserted-by":"publisher","DOI":"10.1109\/TIFS.2014.2359646"},{"key":"ref43","doi-asserted-by":"publisher","DOI":"10.1175\/BAMS-D-20-0097.1"},{"key":"ref73","doi-asserted-by":"publisher","DOI":"10.1109\/DISA.2018.8490530"},{"key":"ref72","doi-asserted-by":"publisher","DOI":"10.1016\/j.patcog.2006.07.009"},{"key":"ref71","doi-asserted-by":"publisher","DOI":"10.1016\/j.jmgm.2018.06.005"},{"key":"ref70","doi-asserted-by":"publisher","DOI":"10.1162\/neco.1997.9.8.1735"},{"key":"ref76","first-page":"9734","article-title":"A benchmark for interpretability methods in deep neural networks","author":"hooker","year":"2019","journal-title":"Proc Adv Neural Inf Process Syst"},{"key":"ref77","doi-asserted-by":"publisher","DOI":"10.1038\/s41598-019-38748-8"},{"key":"ref74","doi-asserted-by":"publisher","DOI":"10.1002\/widm.1312"},{"key":"ref75","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-28954-6_5"},{"key":"ref78","author":"huang","year":"2007"},{"key":"ref79","article-title":"SqueezeNet: AlexNet-level accuracy with 50x fewer parameters and <1mb model size","author":"iandola","year":"2016","journal-title":"arXiv 1602 07360"},{"key":"ref60","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2018.00240"},{"key":"ref62","doi-asserted-by":"publisher","DOI":"10.1002\/minf.201100059"},{"key":"ref61","doi-asserted-by":"publisher","DOI":"10.1038\/s41598-020-62724-2"},{"key":"ref63","doi-asserted-by":"publisher","DOI":"10.1016\/j.neuroimage.2013.10.067"},{"key":"ref64","doi-asserted-by":"publisher","DOI":"10.2307\/2285654"},{"key":"ref65","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2016.90"},{"key":"ref66","doi-asserted-by":"publisher","DOI":"10.1145\/3038912.3052569"},{"key":"ref67","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-46493-0_1"},{"key":"ref68","first-page":"2921","article-title":"Fooling neural network interpretations via adversarial model manipulation","volume":"32","author":"heo","year":"2019","journal-title":"Proc Adv Neural Inf Process Syst"},{"key":"ref69","doi-asserted-by":"publisher","DOI":"10.1093\/aje\/kwp293"},{"key":"ref197","doi-asserted-by":"publisher","DOI":"10.1016\/S0008-8846(98)00165-3"},{"key":"ref198","first-page":"9240","article-title":"GNNExplainer: Generating explanations for graph neural networks","author":"ying","year":"2019","journal-title":"Proc Adv Neural Inf Process Syst 32"},{"key":"ref199","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-33850-3_6"},{"key":"ref193","article-title":"Evaluating explanation methods for deep learning in security","author":"warnecke","year":"2019","journal-title":"arXiv 1906 02108"},{"key":"ref194","author":"williams","year":"2006","journal-title":"Gaussian Processes for Machine Learning"},{"key":"ref195","first-page":"2048","article-title":"Show, attend and tell: Neural image caption generation with visual attention","volume":"37","author":"xu","year":"2015","journal-title":"Proc 32nd Int Conf Mach Learn"},{"key":"ref196","doi-asserted-by":"publisher","DOI":"10.1109\/ICHI.2018.00025"},{"key":"ref95","author":"kokhlikyan","year":"2019","journal-title":"Pytorch Captum"},{"key":"ref94","doi-asserted-by":"publisher","DOI":"10.1109\/IJCNN48605.2020.9206975"},{"key":"ref190","doi-asserted-by":"publisher","DOI":"10.1007\/978-1-4757-2440-0"},{"key":"ref93","doi-asserted-by":"publisher","DOI":"10.1016\/j.semcancer.2018.07.001"},{"key":"ref191","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-23525-7_9"},{"key":"ref92","volume":"8","author":"kittel","year":"2004","journal-title":"Introduction to Solid State Physics"},{"key":"ref192","doi-asserted-by":"publisher","DOI":"10.1016\/0959-4388(95)80014-X"},{"key":"ref91","article-title":"Semi-supervised classification with graph convolutional networks","author":"kipf","year":"2017","journal-title":"Proc 5th Int Conf Learn Represent"},{"key":"ref90","article-title":"Learning how to explain neural networks: PatternNet and PatternAttribution","author":"kindermans","year":"2018","journal-title":"Proc 6th Int Conf Learn Represent"},{"key":"ref98","doi-asserted-by":"publisher","DOI":"10.1093\/bioinformatics\/btw252"},{"key":"ref99","first-page":"1106","article-title":"ImageNet classification with deep convolutional neural networks","author":"krizhevsky","year":"2012","journal-title":"Proc NIPS"},{"key":"ref96","doi-asserted-by":"publisher","DOI":"10.1016\/j.csbj.2014.11.005"},{"key":"ref97","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-28954-6_19"},{"key":"ref82","doi-asserted-by":"publisher","DOI":"10.1016\/j.bushor.2018.03.007"},{"key":"ref81","article-title":"Explaining explanations: Axiomatic feature interactions for deep networks","author":"janizek","year":"2020","journal-title":"arXiv 2002 04138"},{"key":"ref84","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-63387-9_5"},{"key":"ref83","doi-asserted-by":"publisher","DOI":"10.1145\/2647868.2654889"},{"key":"ref80","doi-asserted-by":"publisher","DOI":"10.1109\/ICCVW.2019.00513"},{"key":"ref89","first-page":"2673","article-title":"Interpretability beyond feature attribution: Quantitative testing with concept activation vectors (TCAV)","volume":"80","author":"kim","year":"2018","journal-title":"Proc 35th Int Conf Mach Learn"},{"key":"ref85","article-title":"From clustering to cluster explanations via neural networks","author":"kauffmann","year":"2019","journal-title":"arXiv 1906 07633"},{"key":"ref86","doi-asserted-by":"publisher","DOI":"10.1016\/j.patcog.2020.107198"},{"key":"ref87","doi-asserted-by":"publisher","DOI":"10.1101\/gr.227819.117"},{"key":"ref88","doi-asserted-by":"publisher","DOI":"10.1038\/89044"},{"key":"ref200","first-page":"1899","article-title":"Graying the black box: Understanding DQNs","volume":"48","author":"zahavy","year":"2016","journal-title":"Proc 33nd Int Conf Mach Learn"},{"key":"ref101","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2016.318"},{"key":"ref100","doi-asserted-by":"publisher","DOI":"10.1109\/CIDM.2013.6597214"},{"key":"ref203","doi-asserted-by":"publisher","DOI":"10.1109\/TPAMI.2020.2992207"},{"key":"ref204","doi-asserted-by":"publisher","DOI":"10.1038\/s42256-019-0052-1"},{"key":"ref201","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-10590-1_53"},{"key":"ref202","doi-asserted-by":"publisher","DOI":"10.1007\/s11263-017-1059-x"},{"key":"ref207","doi-asserted-by":"publisher","DOI":"10.1007\/s10509-019-3540-1"},{"key":"ref208","article-title":"Visualizing deep neural network decisions: Prediction difference analysis","author":"zintgraf","year":"2017","journal-title":"Proc Int Conf Learn Represent (ICLR)"},{"key":"ref205","doi-asserted-by":"publisher","DOI":"10.1109\/TPAMI.2018.2858759"},{"key":"ref206","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2016.319"},{"key":"ref127","doi-asserted-by":"publisher","DOI":"10.1016\/j.patcog.2016.11.008"},{"key":"ref126","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-28954-6_10"},{"key":"ref125","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-28954-6_13"},{"key":"ref124","doi-asserted-by":"publisher","DOI":"10.1038\/nature14236"},{"key":"ref129","first-page":"2924","article-title":"On the number of linear regions of deep neural networks","volume":"27","author":"mont\u00fafar","year":"2014","journal-title":"Proc Adv Neural Inf Process Syst"},{"key":"ref128","doi-asserted-by":"publisher","DOI":"10.1016\/j.dsp.2017.10.011"},{"key":"ref130","doi-asserted-by":"publisher","DOI":"10.1109\/ICNN.1995.488997"},{"key":"ref133","article-title":"How do humans understand explanations from machine learning systems? An evaluation of the human-interpretability of explanation","author":"narayanan","year":"2018","journal-title":"arXiv 1802 00682"},{"key":"ref134","first-page":"3387","article-title":"Synthesizing the preferred inputs for neurons in neural networks via deep generator networks","volume":"29","author":"nguyen","year":"2016","journal-title":"Proc Adv Neural Inf Process Syst"},{"key":"ref131","article-title":"Inceptionism: Going deeper into neural networks","author":"mordvintsev","year":"2015","journal-title":"Google Research Blog"},{"key":"ref132","doi-asserted-by":"publisher","DOI":"10.1109\/72.914517"},{"key":"ref136","article-title":"Multifaceted feature visualization: Uncovering the different types of features learned by each neuron in deep neural networks","author":"nguyen","year":"2016","journal-title":"arXiv 1602 03616"},{"key":"ref135","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2015.7298640"},{"key":"ref138","volume":"1","author":"oliphant","year":"2006","journal-title":"A Guide to NumPy"},{"key":"ref137","first-page":"151","article-title":"CuPy: A numpy-compatible library for NVIDIA GPU calculations","author":"okuta","year":"2017","journal-title":"Proc Adv Neural Inf Process Syst"},{"key":"ref139","doi-asserted-by":"publisher","DOI":"10.1109\/72.963764"},{"key":"ref140","doi-asserted-by":"publisher","DOI":"10.1016\/j.envpol.2019.113395"},{"key":"ref141","doi-asserted-by":"publisher","DOI":"10.1214\/aoms\/1177704472"},{"key":"ref142","first-page":"151","article-title":"RISE: Randomized input sampling for explanation of black-box models","author":"petsiuk","year":"2018","journal-title":"Proc Brit Mach Vis Conf"},{"key":"ref143","first-page":"1822","article-title":"Visual explanation of evidence with additive classifiers","author":"poulin","year":"2006","journal-title":"Proc 21st Nat Conf Artif Intell 18th Innov Appl Artif Intell Conf"},{"key":"ref2","article-title":"Explaining image classifiers by removing input features using generative models","author":"agarwal","year":"2020","journal-title":"Proc Asian Conf Comput Vis"},{"key":"ref144","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-28954-6_18"},{"key":"ref1","article-title":"Local explanation methods for deep neural networks lack sensitivity to parameter values","author":"adebayo","year":"2018","journal-title":"Proc Int Conf Learn Represent"},{"key":"ref145","doi-asserted-by":"publisher","DOI":"10.1016\/j.media.2017.04.012"},{"key":"ref109","first-page":"1","article-title":"Applying machine learning methods to detect convection using GOES-16 ABI data","volume":"2020","author":"lee","year":"2020","journal-title":"Atmospheric Measurement Techniques Discussions"},{"key":"ref108","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-35289-8_3"},{"key":"ref107","doi-asserted-by":"crossref","first-page":"436","DOI":"10.1038\/nature14539","article-title":"Deep learning","volume":"521","author":"lecun","year":"2015","journal-title":"Nature"},{"key":"ref106","doi-asserted-by":"publisher","DOI":"10.1038\/s41467-020-17431-x"},{"key":"ref105","first-page":"1243","article-title":"Learning to combine foveal glimpses with a third-order Boltzmann machine","volume":"23","author":"larochelle","year":"2010","journal-title":"Proc Adv Neural Inf Process Syst"},{"key":"ref104","doi-asserted-by":"publisher","DOI":"10.1038\/s41467-019-08987-4"},{"key":"ref103","doi-asserted-by":"publisher","DOI":"10.1109\/ICCVW.2017.191"},{"key":"ref102","first-page":"1","article-title":"The layer-wise relevance propagation toolbox for artificial neural networks","volume":"17","author":"lapuschkin","year":"2016","journal-title":"J Mach Learn Res"},{"key":"ref111","article-title":"Network in network","author":"lin","year":"2014","journal-title":"Proc Int Conf Learn Represent (ICLR)"},{"key":"ref112","doi-asserted-by":"publisher","DOI":"10.1145\/3236386.3241340"},{"key":"ref110","doi-asserted-by":"publisher","DOI":"10.1109\/TEST.2018.8624792"},{"key":"ref10","doi-asserted-by":"publisher","DOI":"10.1371\/journal.pone.0181142"},{"key":"ref11","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/W17-5221"},{"key":"ref12","article-title":"Ground truth evaluation of neural network explanations with CLEVR-XAI","author":"arras","year":"2020","journal-title":"arXiv 2003 07258"},{"key":"ref13","doi-asserted-by":"publisher","DOI":"10.1371\/journal.pone.0130140"},{"key":"ref14","first-page":"1803","article-title":"How to explain individual classification decisions","volume":"11","author":"baehrens","year":"2010","journal-title":"J Mach Learn Res"},{"key":"ref15","article-title":"Neural machine translation by jointly learning to align and translate","author":"bahdanau","year":"2015","journal-title":"Proc 3rd Int Conf Learn Represent"},{"key":"ref16","first-page":"342","article-title":"The shattered gradients problem: If resnets are the answer, then what is the question","volume":"70","author":"balduzzi","year":"2017","journal-title":"Proc 34th Int Conf Mach Learn"},{"key":"ref118","article-title":"A rate-distortion framework for explaining neural network decisions","author":"macdonald","year":"2019","journal-title":"arXiv 1905 11092"},{"key":"ref17","article-title":"GAN dissection: Visualizing and understanding generative adversarial networks","author":"bau","year":"2019","journal-title":"Proc 7th Int Conf Learn Represent"},{"key":"ref117","doi-asserted-by":"publisher","DOI":"10.1186\/1471-2105-8-60"},{"key":"ref18","doi-asserted-by":"publisher","DOI":"10.1073\/pnas.1907375117"},{"key":"ref19","author":"bazen","year":"2013","journal-title":"The Taylor decomposition A unified generalization of the Oaxaca method to nonlinear models"},{"key":"ref119","first-page":"281","article-title":"Some methods for classification and analysis of multivariate observations","volume":"1","author":"macqueen","year":"1967","journal-title":"Proc 5th Berkeley Symp Math Statist Probab"},{"key":"ref114","doi-asserted-by":"publisher","DOI":"10.1093\/bioinformatics\/btaa506"},{"key":"ref113","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/P19-1560"},{"key":"ref116","first-page":"4765","article-title":"A unified approach to interpreting model predictions","volume":"30","author":"lundberg","year":"2017","journal-title":"Proc Adv Neural Inf Process Syst"},{"key":"ref115","doi-asserted-by":"publisher","DOI":"10.1038\/s42256-019-0138-9"},{"key":"ref120","doi-asserted-by":"publisher","DOI":"10.1016\/S0304-3975(00)00251-6"},{"key":"ref121","doi-asserted-by":"publisher","DOI":"10.1175\/BAMS-D-18-0195.1"},{"key":"ref122","doi-asserted-by":"publisher","DOI":"10.1109\/TPAMI.2013.53"},{"key":"ref123","doi-asserted-by":"publisher","DOI":"10.1016\/j.artint.2018.07.007"}],"container-title":["Proceedings of the IEEE"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx7\/5\/9369414\/09369420.pdf?arnumber=9369420","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2021,12,17]],"date-time":"2021-12-17T19:57:30Z","timestamp":1639771050000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/9369420\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2021,3]]},"references-count":208,"journal-issue":{"issue":"3"},"URL":"https:\/\/doi.org\/10.1109\/jproc.2021.3060483","relation":{},"ISSN":["0018-9219","1558-2256"],"issn-type":[{"value":"0018-9219","type":"print"},{"value":"1558-2256","type":"electronic"}],"subject":[],"published":{"date-parts":[[2021,3]]}}}