{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,8]],"date-time":"2026-04-08T16:31:55Z","timestamp":1775665915646,"version":"3.50.1"},"reference-count":106,"publisher":"Institute of Electrical and Electronics Engineers (IEEE)","issue":"11","license":[{"start":{"date-parts":[[2024,11,1]],"date-time":"2024-11-01T00:00:00Z","timestamp":1730419200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/legalcode"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["IEEE Trans. Pattern Anal. Mach. Intell."],"published-print":{"date-parts":[[2024,11]]},"DOI":"10.1109\/tpami.2024.3388275","type":"journal-article","created":{"date-parts":[[2024,4,12]],"date-time":"2024-04-12T17:27:38Z","timestamp":1712942858000},"page":"7283-7299","source":"Crossref","is-referenced-by-count":17,"title":["Disentangled Explanations of Neural Network Predictions by Finding Relevant Subspaces"],"prefix":"10.1109","volume":"46","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-7582-4667","authenticated-orcid":false,"given":"Pattarawat","family":"Chormai","sequence":"first","affiliation":[{"name":"Machine Learning Group, Technische Universität Berlin (TU Berlin), Berlin, Germany"}]},{"ORCID":"https:\/\/orcid.org\/0009-0001-7209-0848","authenticated-orcid":false,"given":"Jan","family":"Herrmann","sequence":"additional","affiliation":[{"name":"BASF SE, Statistics and Machine Learning, Ludwigshafen am Rhein, Germany"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-3861-7685","authenticated-orcid":false,"given":"Klaus-Robert","family":"M\u00fcller","sequence":"additional","affiliation":[{"name":"Machine Learning Group, Technische Universität Berlin (TU Berlin), Berlin, Germany"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-7243-6186","authenticated-orcid":false,"given":"Gr\u00e9goire","family":"Montavon","sequence":"additional","affiliation":[{"name":"Department of Mathematics and Computer Science, Freie Universität Berlin (FU Berlin), Berlin, Germany"}]}],"member":"263","reference":[{"key":"ref1","first-page":"1803","article-title":"How to explain individual classification decisions","volume":"11","author":"Baehrens","year":"2010","journal-title":"J. Mach. Learn. Res."},{"key":"ref2","doi-asserted-by":"publisher","DOI":"10.1109\/JPROC.2021.3060483"},{"key":"ref3","doi-asserted-by":"publisher","DOI":"10.1016\/j.inffus.2019.12.012"},{"key":"ref4","doi-asserted-by":"publisher","DOI":"10.1145\/3301275.3308446"},{"key":"ref5","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-28954-6"},{"key":"ref6","doi-asserted-by":"publisher","DOI":"10.1109\/ACCESS.2020.2976199"},{"key":"ref7","first-page":"1","article-title":"An efficient explanation of individual classifications using game theory","volume":"11","author":"Strumbelj","year":"2010","journal-title":"J. Mach. Learn. Res."},{"key":"ref8","doi-asserted-by":"publisher","DOI":"10.1371\/journal.pone.0130140"},{"key":"ref9","first-page":"3319","article-title":"Axiomatic attribution for deep networks","volume-title":"Proc. Mach. Learn. Res.","author":"Sundararajan"},{"key":"ref10","first-page":"4765","article-title":"A unified approach to interpreting model predictions","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Lundberg"},{"key":"ref11","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-28954-6_19"},{"key":"ref12","doi-asserted-by":"publisher","DOI":"10.1175\/BAMS-D-20-0097.1"},{"key":"ref13","doi-asserted-by":"publisher","DOI":"10.1038\/s42256-019-0138-9"},{"key":"ref14","doi-asserted-by":"publisher","DOI":"10.1038\/s41698-022-00278-4"},{"key":"ref15","doi-asserted-by":"publisher","DOI":"10.1146\/annurev-pathmechdis-051222-113147"},{"key":"ref16","doi-asserted-by":"publisher","DOI":"10.1038\/s41598-020-62724-2"},{"key":"ref17","doi-asserted-by":"publisher","DOI":"10.1016\/j.inffus.2021.07.015"},{"key":"ref18","doi-asserted-by":"publisher","DOI":"10.1109\/TPAMI.2018.2885764"},{"key":"ref19","doi-asserted-by":"publisher","DOI":"10.1109\/TPAMI.2020.3020738"},{"key":"ref20","doi-asserted-by":"publisher","DOI":"10.1109\/TPAMI.2021.3115452"},{"key":"ref21","doi-asserted-by":"publisher","DOI":"10.1109\/TPAMI.2020.2992207"},{"key":"ref22","article-title":"This looks more like that: Enhancing self-explaining models by prototypical relevance propagation","volume":"136","author":"Gautam","year":"2022","journal-title":"Pattern Recognit."},{"key":"ref23","doi-asserted-by":"publisher","DOI":"10.1038\/s42256-023-00711-8"},{"key":"ref24","first-page":"2673","article-title":"Interpretability beyond feature attribution: Quantitative testing with concept activation vectors (TCAV)","volume-title":"Proc. Mach. Learn. Res.","author":"Kim"},{"key":"ref25","article-title":"Multi-dimensional concept discovery (MCD): A unifying framework with completeness guarantees","author":"Vielhaben","year":"2023","journal-title":"Trans. Mach. Learn. Res."},{"key":"ref26","article-title":"Very deep convolutional networks for large-scale image recognition","volume-title":"Proc. 3rd Int. Conf. Learn. Representations","author":"Simonyan"},{"key":"ref27","article-title":"Torchvision: PyTorchs computer vision library.","year":"2016"},{"key":"ref28","first-page":"1875","article-title":"On relevant dimensions in kernel feature spaces","volume":"9","author":"Braun","year":"2008","journal-title":"J. Mach. Learn. Res."},{"key":"ref29","doi-asserted-by":"publisher","DOI":"10.1016\/S0893-6080(00)00026-5"},{"key":"ref30","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2011.5995496"},{"key":"ref31","volume-title":"Natural Image Statistics - A Probabilistic Approach to Early Computational Vision","volume":"39","author":"Hyv\u00e4rinen","year":"2009"},{"key":"ref32","doi-asserted-by":"publisher","DOI":"10.1515\/9781400881970-018"},{"key":"ref33","doi-asserted-by":"publisher","DOI":"10.1016\/j.patcog.2016.11.008"},{"key":"ref34","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-28954-6_10"},{"key":"ref35","first-page":"435","article-title":"XAI for transformers: Better explanations through conservative propagation","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Ali"},{"key":"ref36","doi-asserted-by":"publisher","DOI":"10.1038\/s41467-019-08987-4"},{"key":"ref37","first-page":"13567","article-title":"Explanations can be manipulated and geometry is to blame","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Dombrowski"},{"key":"ref38","doi-asserted-by":"publisher","DOI":"10.1016\/j.patcog.2021.108194"},{"key":"ref39","doi-asserted-by":"publisher","DOI":"10.1016\/j.dsp.2017.10.011"},{"key":"ref40","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-04083-2"},{"key":"ref41","doi-asserted-by":"publisher","DOI":"10.1002\/widm.1312"},{"key":"ref42","doi-asserted-by":"publisher","DOI":"10.3390\/e23010018"},{"key":"ref43","doi-asserted-by":"publisher","DOI":"10.1145\/3233231"},{"key":"ref44","first-page":"2563","article-title":"Kernel analysis of deep networks","volume":"12","author":"Montavon","year":"2011","journal-title":"J. Mach. Learn. Res."},{"key":"ref45","doi-asserted-by":"publisher","DOI":"10.1371\/journal.pcbi.1003963"},{"key":"ref46","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-10590-1_53"},{"key":"ref47","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-01237-3_8"},{"key":"ref48","first-page":"9273","article-title":"Towards automatic concept-based explanations","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Ghorbani"},{"key":"ref49","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v35i13.17389"},{"key":"ref50","doi-asserted-by":"publisher","DOI":"10.1016\/j.patcog.2024.110309"},{"key":"ref51","doi-asserted-by":"publisher","DOI":"10.1109\/TPAMI.2018.2858759"},{"key":"ref52","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2017.354"},{"key":"ref53","doi-asserted-by":"publisher","DOI":"10.1145\/3563039"},{"key":"ref54","article-title":"Concept-based explainable artificial intelligence: A survey","author":"Poeta","year":"2023"},{"key":"ref55","article-title":"Recovering pairwise interactions using neural networks","author":"Cui","year":"2019"},{"key":"ref56","first-page":"104:1","article-title":"Explaining explanations: Axiomatic feature interactions for deep networks","volume":"22","author":"Janizek","year":"2021","journal-title":"J. Mach. Learn. Res."},{"key":"ref57","doi-asserted-by":"publisher","DOI":"10.1007\/s11263-022-01746-x"},{"key":"ref58","article-title":"Hierarchical interpretations for neural network predictions","volume-title":"Proc. 7th Int. Conf. Learn. Representations","author":"Singh"},{"key":"ref59","article-title":"Towards hierarchical importance attribution: Explaining compositional semantics for neural sequence models","volume-title":"Proc. 8th Int. Conf. Learn. Representations","author":"Jin"},{"key":"ref60","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.acl-main.494"},{"key":"ref61","article-title":"Less is more: Fewer interpretable region via submodular subset selection","volume-title":"Proc. 12th Int. Conf. Learn. Representations","author":"Chen"},{"key":"ref62","article-title":"DORA: Exploring outlier representations in deep neural networks","author":"Bykov","year":"2023","journal-title":"Trans. Mach. Learn. Res."},{"key":"ref63","doi-asserted-by":"publisher","DOI":"10.1016\/j.inffus.2023.102094"},{"key":"ref64","article-title":"Last layer re-training is sufficient for robustness to spurious correlations","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Kirichenko"},{"key":"ref65","doi-asserted-by":"publisher","DOI":"10.1109\/TPAMI.2013.50"},{"key":"ref66","doi-asserted-by":"publisher","DOI":"10.1016\/j.patcog.2010.12.015"},{"key":"ref67","first-page":"1207","article-title":"When are post-hoc conceptual explanations identifiable?","volume-title":"Proc. 39th Conf. Uncertainty Artif. Intell.","author":"Leemann"},{"key":"ref68","article-title":"A framework for the quantitative evaluation of disentangled representations","volume-title":"Proc. 6th Int. Conf. Learn. Representations","author":"Eastwood"},{"key":"ref69","doi-asserted-by":"publisher","DOI":"10.1109\/TBME.2002.805480"},{"key":"ref70","doi-asserted-by":"publisher","DOI":"10.1109\/ITW.2015.7133169"},{"key":"ref71","doi-asserted-by":"publisher","DOI":"10.1016\/j.neunet.2020.03.001"},{"key":"ref72","doi-asserted-by":"publisher","DOI":"10.1007\/s10994-021-06074-8"},{"key":"ref73","doi-asserted-by":"publisher","DOI":"10.1016\/j.patcog.2022.109172"},{"key":"ref74","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52729.2023.00266"},{"key":"ref75","article-title":"Intriguing properties of neural networks","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Szegedy"},{"key":"ref76","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2018.00910"},{"key":"ref77","doi-asserted-by":"publisher","DOI":"10.1109\/TNNLS.2022.3185901"},{"key":"ref78","first-page":"3145","article-title":"Learning important features through propagating activation differences","volume-title":"Proc. 34th Int. Conf. Mach. Learn.","author":"Shrikumar"},{"key":"ref79","article-title":"The use of principal components for creating improved imagery for geometric control point selection","volume-title":"Proc. Marshall Univ. Proc. Nat. Conf. Energy Resour. Manage.","author":"Imhoff"},{"key":"ref80","doi-asserted-by":"publisher","DOI":"10.1080\/01431168708948659"},{"key":"ref81","doi-asserted-by":"publisher","DOI":"10.1007\/978-0-387-21606-5"},{"key":"ref82","first-page":"3409","article-title":"$L_{\\text{p}}$Lp-nested symmetric distributions","volume":"11","author":"Sinz","year":"2010","journal-title":"J. Mach. Learn. Res."},{"key":"ref83","doi-asserted-by":"publisher","DOI":"10.1038\/s41592-019-0686-2"},{"key":"ref84","doi-asserted-by":"publisher","DOI":"10.1002\/0471221317"},{"key":"ref85","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2009.5206848"},{"key":"ref86","doi-asserted-by":"publisher","DOI":"10.1109\/TPAMI.2017.2723009"},{"key":"ref87","doi-asserted-by":"publisher","DOI":"10.1145\/2647868.2654889"},{"key":"ref88","first-page":"1059","article-title":"High-performance large-scale image recognition without normalization","volume-title":"Proc. 38th Int. Conf. Mach. Learn.","author":"Brock"},{"key":"ref89","article-title":"PyTorch image models.","author":"Wightman","year":"2019"},{"key":"ref90","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2016.90"},{"key":"ref91","doi-asserted-by":"publisher","DOI":"10.1109\/TNNLS.2016.2599820"},{"key":"ref92","article-title":"Captum: A unified and generic model interpretability library for PyTorch","author":"Kokhlikyan","year":"2020"},{"key":"ref93","article-title":"Software for dataset-wide XAI: From local explanations to global insights with Zennit, CoRelAy, and ViRelAy","author":"Anders","year":"2021"},{"key":"ref94","doi-asserted-by":"publisher","DOI":"10.1016\/j.patcog.2021.107899"},{"key":"ref95","first-page":"151","article-title":"RISE: Randomized input sampling for explanation of black-box models","volume-title":"Proc. Brit. Mach. Vis. Conf.","author":"Petsiuk"},{"key":"ref96","doi-asserted-by":"publisher","DOI":"10.1080\/10867651.2004.10487596"},{"key":"ref97","first-page":"122","article-title":"The OpenCV library","volume":"120","author":"Bradski","year":"2000","journal-title":"Dr Dobbs J. Softw. Tools"},{"key":"ref98","first-page":"20554","article-title":"On completeness-aware concept-based explanations in deep neural networks","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Yeh"},{"key":"ref99","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-28954-6_18"},{"key":"ref100","doi-asserted-by":"publisher","DOI":"10.1038\/ncomms13890"},{"key":"ref101","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-28954-6_17"},{"key":"ref102","doi-asserted-by":"publisher","DOI":"10.1016\/j.jneumeth.2016.10.008"},{"key":"ref103","doi-asserted-by":"publisher","DOI":"10.3389\/fnins.2019.01321"},{"key":"ref104","doi-asserted-by":"publisher","DOI":"10.1038\/s42256-021-00303-4"},{"key":"ref105","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v33i01.33013681"},{"key":"ref106","doi-asserted-by":"publisher","DOI":"10.1109\/tpami.2023.3339980"}],"container-title":["IEEE Transactions on Pattern Analysis and Machine Intelligence"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx7\/34\/10704791\/10497845.pdf?arnumber=10497845","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,10,4]],"date-time":"2024-10-04T04:16:18Z","timestamp":1728015378000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/10497845\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,11]]},"references-count":106,"journal-issue":{"issue":"11"},"URL":"https:\/\/doi.org\/10.1109\/tpami.2024.3388275","relation":{},"ISSN":["0162-8828","2160-9292","1939-3539"],"issn-type":[{"value":"0162-8828","type":"print"},{"value":"2160-9292","type":"electronic"},{"value":"1939-3539","type":"electronic"}],"subject":[],"published":{"date-parts":[[2024,11]]}}}