{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,11,2]],"date-time":"2026-11-02T15:52:10Z","timestamp":1793634730091,"version":"4.5.1"},"reference-count":208,"publisher":"Institute of Electrical and Electronics Engineers (IEEE)","issue":"3","license":[{"start":{"date-parts":[[2021,3,1]],"date-time":"2021-03-01T00:00:00Z","timestamp":1614556800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/legalcode"}],"funder":[{"name":"Institute of Information & Communications Technology Planning & Evaluation (IITP) grants funded by the Korea Government","award":["2017-0-00451"],"award-info":[{"award-number":["2017-0-00451"]}],"id":[{"id":"https:\/\/ror.org\/01g0hqq23","id-type":"ROR","asserted-by":"crossref"}]},{"name":"Institute of Information & Communications Technology Planning & Evaluation (IITP) grants funded by the Korea Government","award":["2019-0-00079"],"award-info":[{"award-number":["2019-0-00079"]}],"id":[{"id":"https:\/\/ror.org\/01g0hqq23","id-type":"ROR","asserted-by":"crossref"}]},{"DOI":"10.13039\/501100002347","name":"German Ministry for Education and Research","doi-asserted-by":"publisher","award":["01IS14013A-E"],"award-info":[{"award-number":["01IS14013A-E"]}],"id":[{"id":"10.13039\/501100002347","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100002347","name":"German Ministry for Education and Research","doi-asserted-by":"publisher","award":["01GQ1115"],"award-info":[{"award-number":["01GQ1115"]}],"id":[{"id":"10.13039\/501100002347","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100002347","name":"German Ministry for Education and Research","doi-asserted-by":"publisher","award":["01GQ0850"],"award-info":[{"award-number":["01GQ0850"]}],"id":[{"id":"10.13039\/501100002347","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100002347","name":"German Ministry for Education and Research","doi-asserted-by":"publisher","award":["01IS18025A"],"award-info":[{"award-number":["01IS18025A"]}],"id":[{"id":"10.13039\/501100002347","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100002347","name":"German Ministry for Education and Research","doi-asserted-by":"publisher","award":["01IS18037A"],"award-info":[{"award-number":["01IS18037A"]}],"id":[{"id":"10.13039\/501100002347","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100001659","name":"German Research Foundation","doi-asserted-by":"publisher","award":["Math+"],"award-info":[{"award-number":["Math+"]}],"id":[{"id":"10.13039\/501100001659","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100001659","name":"German Research Foundation","doi-asserted-by":"publisher","award":["EXC 2046\/1"],"award-info":[{"award-number":["EXC 2046\/1"]}],"id":[{"id":"10.13039\/501100001659","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100001659","name":"German Research Foundation","doi-asserted-by":"publisher","award":["390685689"],"award-info":[{"award-number":["390685689"]}],"id":[{"id":"10.13039\/501100001659","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["Proc. IEEE"],"published-print":{"date-parts":[[2021,3]]},"DOI":"10.1109\/jproc.2021.3060483","type":"journal-article","created":{"date-parts":[[2021,3,4]],"date-time":"2021-03-04T15:32:09Z","timestamp":1614871929000},"page":"247-278","source":"Crossref","is-referenced-by-count":1405,"title":["Explaining Deep Neural Networks and Beyond: A Review of Methods and Applications"],"prefix":"10.1109","volume":"109","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-6283-3265","authenticated-orcid":false,"given":"Wojciech","family":"Samek","sequence":"first","affiliation":[{"name":"Department of Artificial Intelligence, Fraunhofer Heinrich Hertz Institute, Berlin, Germany"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-7243-6186","authenticated-orcid":false,"given":"Gregoire","family":"Montavon","sequence":"additional","affiliation":[{"name":"BIFOLD\u2714Berlin Institute for the Foundations of Learning and Data, Berlin, Germany"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-0762-7258","authenticated-orcid":false,"given":"Sebastian","family":"Lapuschkin","sequence":"additional","affiliation":[{"name":"Department of Artificial Intelligence, Fraunhofer Heinrich Hertz Institute, Berlin, Germany"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Christopher J.","family":"Anders","sequence":"additional","affiliation":[{"name":"BIFOLD\u2714Berlin Institute for the Foundations of Learning and Data, Berlin, Germany"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-3861-7685","authenticated-orcid":false,"given":"Klaus-Robert","family":"Muller","sequence":"additional","affiliation":[{"name":"BIFOLD\u2714Berlin Institute for the Foundations of Learning and Data, Berlin, Germany"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"263","reference":[{"key":"ref1","article-title":"Local explanation methods for deep neural networks lack sensitivity to parameter values","volume-title":"Proc. Int. Conf. Learn. Represent.","author":"Adebayo"},{"key":"ref2","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-69544-6_7"},{"issue":"93","key":"ref3","first-page":"93:1","article-title":"iNNvestigate neural networks","volume":"20","author":"Alber","year":"2019","journal-title":"J. Mach. Learn. Res."},{"key":"ref4","article-title":"Towards better understanding of gradient-based attribution methods for deep neural networks","volume-title":"Proc. Int. Conf. Learn. Represent. (ICLR)","author":"Ancona"},{"key":"ref5","first-page":"272","article-title":"Explaining deep neural networks with a polynomial time algorithm for Shapley value approximation","volume-title":"Proc. 36th Int. Conf. Mach. Learn.","volume":"97","author":"Ancona"},{"key":"ref6","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-28954-6_16"},{"key":"ref7","article-title":"Finding and removing clever hans: Using explanation methods to debug and improve deep models","author":"Anders","year":"2019","journal-title":"arXiv:1912.11425"},{"key":"ref8","first-page":"13544","article-title":"Rudder: Return decomposition for delayed rewards","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","author":"Arjona-Medina"},{"key":"ref9","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-28954-6_11"},{"key":"ref10","doi-asserted-by":"publisher","DOI":"10.1371\/journal.pone.0181142"},{"key":"ref11","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/W17-5221"},{"key":"ref12","article-title":"Ground truth evaluation of neural network explanations with CLEVR-XAI","author":"Arras","year":"2020","journal-title":"arXiv:2003.07258"},{"key":"ref13","doi-asserted-by":"publisher","DOI":"10.1371\/journal.pone.0130140"},{"key":"ref14","first-page":"1803","article-title":"How to explain individual classification decisions","volume":"11","author":"Baehrens","year":"2010","journal-title":"J. Mach. Learn. Res."},{"key":"ref15","doi-asserted-by":"publisher","DOI":"10.1146\/annurev.neuro.26.041002.131047"},{"key":"ref16","first-page":"342","article-title":"The shattered gradients problem: If resnets are the answer, then what is the question","volume-title":"Proc. 34th Int. Conf. Mach. Learn.","volume":"70","author":"Balduzzi"},{"key":"ref17","article-title":"GAN dissection: Visualizing and understanding generative adversarial networks","volume-title":"Proc. 7th Int. Conf. Learn. Represent.","author":"Bau"},{"key":"ref18","doi-asserted-by":"publisher","DOI":"10.1073\/pnas.1907375117"},{"key":"ref19","volume-title":"The Taylor decomposition: A unified generalization of the Oaxaca method to nonlinear models","author":"Bazen","year":"2013"},{"key":"ref20","article-title":"Interpreting and explaining deep neural networks for classification of audio signals","author":"Becker","year":"2018","journal-title":"arXiv:1807.03418"},{"key":"ref21","first-page":"908","article-title":"Safe model-based reinforcement learning with stability guarantees","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","author":"Berkenkamp"},{"key":"ref22","doi-asserted-by":"publisher","DOI":"10.1038\/s42256-021-00303-4"},{"key":"ref23","doi-asserted-by":"publisher","DOI":"10.1093\/oso\/9780198538493.001.0001"},{"key":"ref24","doi-asserted-by":"publisher","DOI":"10.1109\/MSP.2008.4408441"},{"key":"ref25","article-title":"Approximating CNNs with bag-of-local-features models works surprisingly well on imagenet","volume-title":"Proc. 7th Int. Conf. Learn. Represent.","author":"Brendel"},{"key":"ref26","doi-asserted-by":"publisher","DOI":"10.1109\/SP.2017.49"},{"key":"ref27","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-01264-9_9"},{"key":"ref28","doi-asserted-by":"publisher","DOI":"10.1145\/2783258.2788613"},{"key":"ref29","doi-asserted-by":"publisher","DOI":"10.1016\/j.cor.2008.04.004"},{"key":"ref30","volume-title":"Keras","author":"Chollet","year":"2015"},{"key":"ref31","doi-asserted-by":"publisher","DOI":"10.1109\/SPW.2018.00011"},{"key":"ref32","article-title":"Plan operation NHANES I epidemiologic followup study, 1987","author":"Cox","year":"1992"},{"key":"ref33","doi-asserted-by":"publisher","DOI":"10.1111\/j.2517-6161.1972.tb00899.x"},{"key":"ref34","first-page":"1087","article-title":"Learning global pairwise interactions with Bayesian neural networks","volume-title":"Proc. 24th Eur. Conf. Artif. Intell.","volume":"325","author":"Cui"},{"key":"ref35","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2009.5206848"},{"key":"ref36","article-title":"How important is a neuron?","author":"Dhamdhere","year":"2018","journal-title":"arXiv:1805.12233"},{"key":"ref37","doi-asserted-by":"publisher","DOI":"10.1145\/1014052.1014118"},{"key":"ref38","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/P17-1106"},{"key":"ref39","first-page":"13567","article-title":"Explanations can be manipulated and geometry is to blame","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","author":"Dombrowski"},{"key":"ref40","doi-asserted-by":"publisher","DOI":"10.7551\/mitpress\/7493.001.0001"},{"key":"ref41","article-title":"Towards a rigorous science of interpretable machine learning","author":"Doshi-Velez","year":"2017","journal-title":"arXiv:1702.08608"},{"key":"ref42","doi-asserted-by":"publisher","DOI":"10.1109\/TPAMI.2020.3020738"},{"key":"ref43","doi-asserted-by":"publisher","DOI":"10.1175\/BAMS-D-20-0097.1"},{"key":"ref44","doi-asserted-by":"publisher","DOI":"10.1109\/TIFS.2014.2359646"},{"key":"ref45","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-98131-4"},{"key":"ref46","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2019.00304"},{"key":"ref47","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2017.371"},{"key":"ref48","doi-asserted-by":"publisher","DOI":"10.1109\/ISBI.2019.8759236"},{"key":"ref49","doi-asserted-by":"publisher","DOI":"10.1145\/2523813"},{"key":"ref50","doi-asserted-by":"publisher","DOI":"10.1073\/pnas.1716999115"},{"key":"ref51","first-page":"315","article-title":"Deep sparse rectifier neural networks","volume-title":"Proc. 14th Int. Conf. Artif. Intell. Statist.","volume":"15","author":"Glorot"},{"key":"ref52","volume-title":"Deep Learning","author":"Goodfellow","year":"2016"},{"key":"ref53","article-title":"Explaining and harnessing adversarial examples","volume-title":"Proc. 3rd Int. Conf. Learn. Represent.","author":"Goodfellow"},{"key":"ref54","doi-asserted-by":"publisher","DOI":"10.1609\/aimag.v38i3.2741"},{"key":"ref55","doi-asserted-by":"publisher","DOI":"10.1007\/s001820050125"},{"key":"ref56","article-title":"Contextual prediction difference analysis for explaining individual image classifications","author":"Gu","year":"2019","journal-title":"arXiv:1910.09086"},{"key":"ref57","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-20893-6_8"},{"key":"ref58","article-title":"Local rule-based explanations of black box decision systems","author":"Guidotti","year":"2018","journal-title":"arXiv:1805.10820"},{"key":"ref59","doi-asserted-by":"publisher","DOI":"10.1145\/3236009"},{"key":"ref60","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2018.00240"},{"key":"ref61","doi-asserted-by":"publisher","DOI":"10.1038\/s41598-020-62724-2"},{"key":"ref62","doi-asserted-by":"publisher","DOI":"10.1002\/minf.201100059"},{"key":"ref63","doi-asserted-by":"publisher","DOI":"10.1016\/j.neuroimage.2013.10.067"},{"key":"ref64","doi-asserted-by":"publisher","DOI":"10.1080\/01621459.1974.10482950"},{"key":"ref65","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2016.90"},{"key":"ref66","doi-asserted-by":"publisher","DOI":"10.1145\/3038912.3052569"},{"key":"ref67","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-46493-0_1"},{"key":"ref68","first-page":"2921","article-title":"Fooling neural network interpretations via adversarial model manipulation","volume-title":"Proc. Adv. Neural Inf. Process. Syst","volume":"32","author":"Heo"},{"key":"ref69","doi-asserted-by":"publisher","DOI":"10.1093\/aje\/kwp293"},{"key":"ref70","doi-asserted-by":"publisher","DOI":"10.1162\/neco.1997.9.8.1735"},{"key":"ref71","doi-asserted-by":"publisher","DOI":"10.1016\/j.jmgm.2018.06.005"},{"key":"ref72","doi-asserted-by":"publisher","DOI":"10.1016\/j.patcog.2006.07.009"},{"key":"ref73","doi-asserted-by":"publisher","DOI":"10.1109\/DISA.2018.8490530"},{"key":"ref74","doi-asserted-by":"publisher","DOI":"10.1002\/widm.1312"},{"key":"ref75","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-28954-6_5"},{"key":"ref76","first-page":"9734","article-title":"A benchmark for interpretability methods in deep neural networks","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","author":"Hooker"},{"key":"ref77","doi-asserted-by":"publisher","DOI":"10.1038\/s41598-019-38748-8"},{"key":"ref78","article-title":"Labeled faces in the wild: A database for studying face recognition in unconstrained environments","author":"Huang","year":"2007"},{"key":"ref79","article-title":"SqueezeNet: AlexNet-level accuracy with 50x fewer parameters and <1mb model size","author":"Iandola","year":"2016","journal-title":"arXiv:1602.07360"},{"key":"ref80","doi-asserted-by":"publisher","DOI":"10.1109\/ICCVW.2019.00513"},{"key":"ref81","article-title":"Explaining explanations: Axiomatic feature interactions for deep networks","author":"Janizek","year":"2020","journal-title":"arXiv:2002.04138"},{"key":"ref82","doi-asserted-by":"publisher","DOI":"10.1016\/j.bushor.2018.03.007"},{"key":"ref83","doi-asserted-by":"publisher","DOI":"10.1145\/2647868.2654889"},{"key":"ref84","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-63387-9_5"},{"key":"ref85","article-title":"From clustering to cluster explanations via neural networks","author":"Kauffmann","year":"2019","journal-title":"arXiv:1906.07633"},{"key":"ref86","doi-asserted-by":"publisher","DOI":"10.1016\/j.patcog.2020.107198"},{"key":"ref87","doi-asserted-by":"publisher","DOI":"10.1101\/gr.227819.117"},{"key":"ref88","doi-asserted-by":"publisher","DOI":"10.1038\/89044"},{"key":"ref89","first-page":"2673","article-title":"Interpretability beyond feature attribution: Quantitative testing with concept activation vectors (TCAV)","volume-title":"Proc. 35th Int. Conf. Mach. Learn.","volume":"80","author":"Kim"},{"key":"ref90","article-title":"Learning how to explain neural networks: PatternNet and PatternAttribution","volume-title":"Proc. 6th Int. Conf. Learn. Represent.","author":"Kindermans"},{"key":"ref91","article-title":"Semi-supervised classification with graph convolutional networks","volume-title":"Proc. 5th Int. Conf. Learn. Represent.","author":"Kipf"},{"key":"ref92","volume":"8","author":"Kittel","year":"2004","journal-title":"Introduction to Solid State Physics"},{"key":"ref93","doi-asserted-by":"publisher","DOI":"10.1016\/j.semcancer.2018.07.001"},{"key":"ref94","doi-asserted-by":"publisher","DOI":"10.1109\/IJCNN48605.2020.9206975"},{"key":"ref95","volume-title":"Pytorch Captum","author":"Kokhlikyan","year":"2019"},{"key":"ref96","doi-asserted-by":"publisher","DOI":"10.1016\/j.csbj.2014.11.005"},{"key":"ref97","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-28954-6_19"},{"key":"ref98","doi-asserted-by":"publisher","DOI":"10.1093\/bioinformatics\/btw252"},{"key":"ref99","first-page":"1106","article-title":"ImageNet classification with deep convolutional neural networks","volume-title":"Proc. NIPS","author":"Krizhevsky"},{"key":"ref100","doi-asserted-by":"publisher","DOI":"10.1109\/CIDM.2013.6597214"},{"key":"ref101","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2016.318"},{"issue":"114","key":"ref102","first-page":"1","article-title":"The layer-wise relevance propagation toolbox for artificial neural networks","volume":"17","author":"Lapuschkin","year":"2016","journal-title":"J. Mach. Learn. Res."},{"key":"ref103","doi-asserted-by":"publisher","DOI":"10.1109\/ICCVW.2017.191"},{"key":"ref104","doi-asserted-by":"publisher","DOI":"10.1038\/s41467-019-08987-4"},{"key":"ref105","first-page":"1243","article-title":"Learning to combine foveal glimpses with a third-order Boltzmann machine","volume-title":"Proc. Adv. Neural Inf. Process. Syst","volume":"23","author":"Larochelle"},{"key":"ref106","doi-asserted-by":"publisher","DOI":"10.1038\/s41467-020-17431-x"},{"key":"ref107","doi-asserted-by":"publisher","DOI":"10.1038\/nature14539"},{"key":"ref108","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-35289-8_3"},{"key":"ref109","first-page":"1","article-title":"Applying machine learning methods to detect convection using GOES-16 ABI data","volume":"2020","author":"Lee","year":"2020","journal-title":"Atmos. Meas. Techn. Discuss."},{"key":"ref110","doi-asserted-by":"publisher","DOI":"10.1109\/TEST.2018.8624792"},{"key":"ref111","doi-asserted-by":"publisher","DOI":"10.48550\/arXiv.1312.4400"},{"key":"ref112","doi-asserted-by":"publisher","DOI":"10.1145\/3236386.3241340"},{"key":"ref113","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/P19-1560"},{"key":"ref114","doi-asserted-by":"publisher","DOI":"10.1093\/bioinformatics\/btaa506"},{"key":"ref115","doi-asserted-by":"publisher","DOI":"10.1038\/s42256-019-0138-9"},{"key":"ref116","first-page":"4765","article-title":"A unified approach to interpreting model predictions","volume-title":"Proc. Adv. Neural Inf. Process. Syst","volume":"30","author":"Lundberg"},{"key":"ref117","doi-asserted-by":"publisher","DOI":"10.1186\/1471-2105-8-60"},{"key":"ref118","article-title":"A rate-distortion framework for explaining neural network decisions","author":"Macdonald","year":"2019","journal-title":"arXiv:1905.11092"},{"key":"ref119","first-page":"281","article-title":"Some methods for classification and analysis of multivariate observations","volume-title":"Proc. 5th Berkeley Symp. Math. Statist. Probab.","volume":"1","author":"MacQueen"},{"key":"ref120","doi-asserted-by":"publisher","DOI":"10.1016\/S0304-3975(00)00251-6"},{"key":"ref121","doi-asserted-by":"publisher","DOI":"10.1175\/BAMS-D-18-0195.1"},{"key":"ref122","doi-asserted-by":"publisher","DOI":"10.1109\/TPAMI.2013.53"},{"key":"ref123","doi-asserted-by":"publisher","DOI":"10.1016\/j.artint.2018.07.007"},{"key":"ref124","doi-asserted-by":"publisher","DOI":"10.1038\/nature14236"},{"key":"ref125","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-28954-6_13"},{"key":"ref126","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-28954-6_10"},{"key":"ref127","doi-asserted-by":"publisher","DOI":"10.1016\/j.patcog.2016.11.008"},{"key":"ref128","doi-asserted-by":"publisher","DOI":"10.1016\/j.dsp.2017.10.011"},{"key":"ref129","doi-asserted-by":"publisher","DOI":"10.5555\/2969033.2969153"},{"key":"ref130","doi-asserted-by":"publisher","DOI":"10.1109\/ICNN.1995.488997"},{"key":"ref131","article-title":"Inceptionism: Going deeper into neural networks","author":"Mordvintsev","year":"2015","journal-title":"Google Res. Blog"},{"key":"ref132","doi-asserted-by":"publisher","DOI":"10.1109\/72.914517"},{"key":"ref133","article-title":"How do humans understand explanations from machine learning systems? An evaluation of the human-interpretability of explanation","author":"Narayanan","year":"2018","journal-title":"arXiv:1802.00682"},{"key":"ref134","doi-asserted-by":"publisher","DOI":"10.5555\/3157382.3157477"},{"key":"ref135","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2015.7298640"},{"key":"ref136","article-title":"Multifaceted feature visualization: Uncovering the different types of features learned by each neuron in deep neural networks","author":"Nguyen","year":"2016","journal-title":"arXiv:1602.03616"},{"key":"ref137","first-page":"151","article-title":"CuPy: A numpy-compatible library for NVIDIA GPU calculations","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","author":"Okuta"},{"key":"ref138","volume-title":"A Guide to NumPy","volume":"1","author":"Oliphant","year":"2006"},{"key":"ref139","doi-asserted-by":"publisher","DOI":"10.1109\/72.963764"},{"key":"ref140","doi-asserted-by":"publisher","DOI":"10.1016\/j.envpol.2019.113395"},{"key":"ref141","doi-asserted-by":"publisher","DOI":"10.1214\/aoms\/1177704472"},{"key":"ref142","first-page":"151","article-title":"RISE: Randomized input sampling for explanation of black-box models","volume-title":"Proc. Brit. Mach. Vis. Conf.","author":"Petsiuk"},{"key":"ref143","first-page":"1822","article-title":"Visual explanation of evidence with additive classifiers","volume-title":"Proc. 21st Nat. Conf. Artif. Intell. 18th Innov. Appl. Artif. Intell. Conf.","author":"Poulin"},{"key":"ref144","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-28954-6_18"},{"key":"ref145","doi-asserted-by":"publisher","DOI":"10.1016\/j.media.2017.04.012"},{"key":"ref146","doi-asserted-by":"publisher","DOI":"10.1038\/s41591-020-0870-z"},{"key":"ref147","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/N16-3020"},{"key":"ref148","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v32i1.11491"},{"key":"ref149","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-98131-4_5"},{"key":"ref150","doi-asserted-by":"publisher","DOI":"10.1109\/ACCESS.2020.2976199"},{"key":"ref151","doi-asserted-by":"publisher","DOI":"10.1037\/h0042519"},{"key":"ref152","doi-asserted-by":"publisher","DOI":"10.24963\/ijcai.2017\/371"},{"key":"ref153","doi-asserted-by":"publisher","DOI":"10.1109\/ICCVW.2015.41"},{"key":"ref154","doi-asserted-by":"publisher","DOI":"10.1038\/s42256-019-0048-x"},{"key":"ref155","doi-asserted-by":"publisher","DOI":"10.1109\/jproc.2021.3052449"},{"key":"ref156","first-page":"4393","article-title":"Deep one-class classification","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Ruff"},{"key":"ref157","doi-asserted-by":"publisher","DOI":"10.1007\/s11263-015-0816-y"},{"key":"ref158","doi-asserted-by":"publisher","DOI":"10.1109\/TNNLS.2016.2599820"},{"key":"ref159","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-28954-6"},{"key":"ref160","doi-asserted-by":"publisher","DOI":"10.1109\/TNN.2008.2005605"},{"key":"ref161","doi-asserted-by":"publisher","DOI":"10.48550\/ARXIV.1404.7828"},{"key":"ref162","article-title":"Higher-order explanations of graph neural networks via relevant walks","author":"Schnake","year":"2020","journal-title":"arXiv:2006.03589"},{"key":"ref163","doi-asserted-by":"publisher","DOI":"10.7551\/mitpress\/4175.001.0001"},{"key":"ref164","doi-asserted-by":"publisher","DOI":"10.1162\/089976698300017467"},{"key":"ref165","doi-asserted-by":"publisher","DOI":"10.1038\/ncomms13890"},{"key":"ref166","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-28954-6_17"},{"key":"ref167","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2017.74"},{"key":"ref168","doi-asserted-by":"publisher","DOI":"10.1145\/1878039.1878045"},{"key":"ref169","doi-asserted-by":"publisher","DOI":"10.1515\/9781400881970-018"},{"key":"ref170","first-page":"3145","article-title":"Learning important features through propagating activation differences","volume-title":"Proc. 34th Int. Conf. Mach. Learn. (ICML)","author":"Shrikumar"},{"key":"ref171","article-title":"Not just a black box: Learning important features through propagating activation differences","author":"Shrikumar","year":"2016","journal-title":"arXiv:1605.01713"},{"key":"ref172","article-title":"Computationally efficient measures of internal neuron importance","author":"Shrikumar","year":"2018","journal-title":"arXiv:1807.09946"},{"key":"ref173","doi-asserted-by":"publisher","DOI":"10.1109\/TPAMI.2018.2885764"},{"key":"ref174","article-title":"Deep inside convolutional networks: Visualising image classification models and saliency maps","volume-title":"Proc. Int. Conf. Learn. Represent. Workshops","author":"Simonyan"},{"key":"ref175","article-title":"Very deep convolutional networks for large-scale image recognition","volume-title":"Proc. 3rd Int. Conf. Learn. Represent.","author":"Simonyan"},{"key":"ref176","article-title":"SmoothGrad: Removing noise by adding noise","author":"Smilkov","year":"2017","journal-title":"arXiv:1706.03825"},{"key":"ref177","doi-asserted-by":"publisher","DOI":"10.1371\/journal.pone.0100335"},{"key":"ref178","article-title":"Striving for simplicity: The all convolutional net","volume-title":"Proc. Int. Conf. Learn. Represent. (ICLR)","author":"Springenberg"},{"key":"ref179","first-page":"1","article-title":"An efficient explanation of individual classifications using game theory","volume":"11","author":"Strumbelj","year":"2010","journal-title":"J. Mach. Learn. Res."},{"key":"ref180","doi-asserted-by":"publisher","DOI":"10.1016\/j.jneumeth.2016.10.008"},{"key":"ref181","doi-asserted-by":"publisher","DOI":"10.23915\/distill.00022"},{"key":"ref182","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-01258-8_39"},{"key":"ref183","first-page":"3319","article-title":"Axiomatic attribution for deep networks","volume-title":"Proc. 34th Int. Conf. Mach. Learn.","author":"Sundararajan"},{"key":"ref184","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-77927-5_24"},{"key":"ref185","article-title":"Intriguing properties of neural networks","volume-title":"Proc. Int. Conf. Learn. Represent. (ICLR)","author":"Szegedy"},{"key":"ref186","doi-asserted-by":"publisher","DOI":"10.3389\/fnins.2019.01321"},{"key":"ref187","article-title":"The frequency range of the voice fundamental in the speech of male and female adults","author":"Traunm\u00fcller","year":"1995"},{"key":"ref188","article-title":"Detecting statistical interactions from neural network weights","volume-title":"Proc. 6th Int. Conf. Learn. Represent.","author":"Tsang"},{"key":"ref189","doi-asserted-by":"publisher","DOI":"10.1145\/3287560.3287566"},{"key":"ref190","doi-asserted-by":"publisher","DOI":"10.1007\/978-1-4757-2440-0"},{"key":"ref191","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-23525-7_9"},{"key":"ref192","doi-asserted-by":"publisher","DOI":"10.1016\/0959-4388(95)80014-X"},{"key":"ref193","article-title":"Evaluating explanation methods for deep learning in security","author":"Warnecke","year":"2019","journal-title":"arXiv:1906.02108"},{"key":"ref194","volume-title":"Gaussian Processes for Machine Learning","author":"Williams","year":"2006"},{"key":"ref195","doi-asserted-by":"publisher","DOI":"10.5555\/3045118.3045336"},{"key":"ref196","doi-asserted-by":"publisher","DOI":"10.1109\/ICHI.2018.00025"},{"key":"ref197","doi-asserted-by":"publisher","DOI":"10.1016\/S0008-8846(98)00165-3"},{"key":"ref198","first-page":"9240","article-title":"GNNExplainer: Generating explanations for graph neural networks","volume-title":"Proc. Adv. Neural Inf. Process. Syst. 32","author":"Ying"},{"key":"ref199","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-33850-3_6"},{"key":"ref200","first-page":"1899","article-title":"Graying the black box: Understanding DQNs","volume-title":"Proc. 33nd Int. Conf. Mach. Learn.","volume":"48","author":"Zahavy"},{"key":"ref201","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-10590-1_53"},{"key":"ref202","doi-asserted-by":"publisher","DOI":"10.1007\/s11263-017-1059-x"},{"key":"ref203","doi-asserted-by":"publisher","DOI":"10.1109\/tpami.2020.2992207"},{"key":"ref204","doi-asserted-by":"publisher","DOI":"10.1038\/s42256-019-0052-1"},{"key":"ref205","doi-asserted-by":"publisher","DOI":"10.1109\/TPAMI.2018.2858759"},{"key":"ref206","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2016.319"},{"key":"ref207","doi-asserted-by":"publisher","DOI":"10.1007\/s10509-019-3540-1"},{"key":"ref208","article-title":"Visualizing deep neural network decisions: Prediction difference analysis","volume-title":"Proc. Int. Conf. Learn. Represent. (ICLR)","author":"Zintgraf"}],"container-title":["Proceedings of the IEEE"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx7\/5\/9369414\/09369420.pdf?arnumber=9369420","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,10,25]],"date-time":"2026-10-25T07:37:19Z","timestamp":1792913839000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/9369420\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2021,3]]},"references-count":208,"journal-issue":{"issue":"3"},"URL":"https:\/\/doi.org\/10.1109\/jproc.2021.3060483","relation":{},"ISSN":["0018-9219","1558-2256"],"issn-type":[{"value":"0018-9219","type":"print"},{"value":"1558-2256","type":"electronic"}],"subject":[],"published":{"date-parts":[[2021,3]]}}}