{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,17]],"date-time":"2026-07-17T03:59:11Z","timestamp":1784260751280,"version":"3.55.0"},"reference-count":69,"publisher":"Institute of Electrical and Electronics Engineers (IEEE)","issue":"12","license":[{"start":{"date-parts":[[2022,12,1]],"date-time":"2022-12-01T00:00:00Z","timestamp":1669852800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/ieeexplore.ieee.org\/Xplorehelp\/downloads\/license-information\/IEEE.html"},{"start":{"date-parts":[[2022,12,1]],"date-time":"2022-12-01T00:00:00Z","timestamp":1669852800000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2022,12,1]],"date-time":"2022-12-01T00:00:00Z","timestamp":1669852800000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"}],"funder":[{"DOI":"10.13039\/501100018537","name":"National Science and Technology Major Project","doi-asserted-by":"publisher","award":["2018AAA0100701"],"award-info":[{"award-number":["2018AAA0100701"]}],"id":[{"id":"10.13039\/501100018537","id-type":"DOI","asserted-by":"publisher"}]},{"name":"Major Research and Development Project of Guangdong Province","award":["2020B1111500002"],"award-info":[{"award-number":["2020B1111500002"]}]},{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["61906106"],"award-info":[{"award-number":["61906106"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["62022048"],"award-info":[{"award-number":["62022048"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["IEEE Trans. Pattern Anal. Mach. Intell."],"published-print":{"date-parts":[[2022,12,1]]},"DOI":"10.1109\/tpami.2021.3133717","type":"journal-article","created":{"date-parts":[[2021,12,9]],"date-time":"2021-12-09T21:13:23Z","timestamp":1639084403000},"page":"10222-10235","source":"Crossref","is-referenced-by-count":20,"title":["Temporal-Spatial Causal Interpretations for Vision-Based Reinforcement Learning"],"prefix":"10.1109","volume":"44","author":[{"ORCID":"https:\/\/orcid.org\/0000-0001-8677-0011","authenticated-orcid":false,"given":"Wenjie","family":"Shi","sequence":"first","affiliation":[{"name":"Department of Automation \/ Beijing National Research Center for Information Science and Technology (BNRist), Tsinghua University, Beijing, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-7251-0988","authenticated-orcid":false,"given":"Gao","family":"Huang","sequence":"additional","affiliation":[{"name":"Department of Automation \/ Beijing National Research Center for Information Science and Technology (BNRist), Tsinghua University, Beijing, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-7361-9283","authenticated-orcid":false,"given":"Shiji","family":"Song","sequence":"additional","affiliation":[{"name":"Department of Automation \/ Beijing National Research Center for Information Science and Technology (BNRist), Tsinghua University, Beijing, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-8611-2665","authenticated-orcid":false,"given":"Cheng","family":"Wu","sequence":"additional","affiliation":[{"name":"Department of Automation \/ Beijing National Research Center for Information Science and Technology (BNRist), Tsinghua University, Beijing, China"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"263","reference":[{"key":"ref39","doi-asserted-by":"publisher","DOI":"10.1145\/3278721.3278776"},{"key":"ref38","first-page":"272","article-title":"Explaining deep neural networks with a polynomial time algorithm for shapley values approximation","author":"ancona","year":"2019","journal-title":"Proc Int Conf Mach Learn"},{"key":"ref33","doi-asserted-by":"publisher","DOI":"10.1109\/ICIP.2019.8803153"},{"key":"ref32","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v33i01.33013681"},{"key":"ref31","doi-asserted-by":"publisher","DOI":"10.1007\/s11263-017-1059-x"},{"key":"ref30","first-page":"3319","article-title":"Axiomatic attribution for deep networks","author":"sundararajan","year":"2017","journal-title":"Proc Int Conf Mach Learn"},{"key":"ref37","doi-asserted-by":"publisher","DOI":"10.1515\/9781400881970-018"},{"key":"ref36","first-page":"818","article-title":"Visualizing and understanding convolutional networks","author":"zeiler","year":"2014","journal-title":"Proc Eur Conf Comput Vis"},{"key":"ref35","first-page":"6967","article-title":"Real time image saliency for black box classifiers","author":"dabkowski","year":"2017","journal-title":"Proc Adv Neural Inf Process Syst"},{"key":"ref34","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2017.371"},{"key":"ref60","doi-asserted-by":"publisher","DOI":"10.3115\/v1\/W14-4012"},{"key":"ref62","first-page":"1925","article-title":"Refinenet: Multi-path refinement networks for high-resolution semantic segmentation","author":"lin","year":"2017","journal-title":"Proc IEEE Conf Comput Vis Pattern Recognit"},{"key":"ref61","first-page":"234","article-title":"U-Net: Convolutional networks for biomedical image segmentation","author":"ronneberger","year":"2015","journal-title":"Proc Int Conf Med Image Comput Comput -Assist Interv"},{"key":"ref63","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2016.438"},{"key":"ref28","doi-asserted-by":"publisher","DOI":"10.1109\/ACCESS.2020.3023394"},{"key":"ref64","doi-asserted-by":"publisher","DOI":"10.1162\/neco.1991.3.1.88"},{"key":"ref27","doi-asserted-by":"publisher","DOI":"10.1613\/jair.3912"},{"key":"ref65","article-title":"Intriguing properties of neural networks","author":"szegedy","year":"2014","journal-title":"Proc Int Conf Learn Representations"},{"key":"ref66","first-page":"12469","article-title":"Generalization of reinforcement learners with working and episodic memory","volume":"32","author":"fortunato","year":"2019","journal-title":"Proc Adv Neural Inf Process Syst"},{"key":"ref29","article-title":"Deep inside convolutional networks: Visualising image classification models and saliency maps","author":"simonyan","year":"2014","journal-title":"Workshop Proc Int Conf Learn Representations"},{"key":"ref67","article-title":"Unsupervised predictive memory in a goal-directed agent","author":"wayne","year":"2018"},{"key":"ref68","article-title":"Towards mental time travel: A hierarchical memory for reinforcement learning agents","author":"lampinen","year":"2021","journal-title":"Proc Adv Neural Inf Process Syst"},{"key":"ref69","article-title":"Duckietown environments for openai gym","author":"chevalier-boisvert","year":"2018"},{"key":"ref2","first-page":"11012","article-title":"Designing optimal dynamic treatment regimes: A causal reinforcement learning approach","author":"zhang","year":"2020","journal-title":"Proc Int Conf Mach Learn"},{"key":"ref1","doi-asserted-by":"publisher","DOI":"10.1038\/nature14236"},{"key":"ref20","article-title":"Visualizing and understanding recurrent networks","author":"karpathy","year":"2016","journal-title":"In International Conference on Learning Representations Workshop"},{"key":"ref22","doi-asserted-by":"publisher","DOI":"10.2307\/1912791"},{"key":"ref21","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2018.00156"},{"key":"ref24","first-page":"1898","article-title":"Discovering temporal causal relations from subsampled data","author":"gong","year":"2015","journal-title":"Proc Int Conf Mach Learn"},{"key":"ref23","doi-asserted-by":"publisher","DOI":"10.1145\/1281192.1281203"},{"key":"ref26","first-page":"10220","article-title":"CXPlain: Causal explanations for model interpretation under uncertainty","author":"schwab","year":"2019","journal-title":"Proc Adv Neural Inf Process Syst"},{"key":"ref25","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v33i01.33014846"},{"key":"ref50","article-title":"Self-supervised discovering of interpretable features for reinforcement learning","author":"shi","year":"2020","journal-title":"IEEE Trans Pattern Anal and Mach Intell"},{"key":"ref51","article-title":"Graphical modelling of multivariate time series with latent variables","author":"eichler","year":"2006","journal-title":"Preprint Universiteit Maastricht"},{"key":"ref59","doi-asserted-by":"crossref","first-page":"436","DOI":"10.1038\/nature14539","article-title":"Deep learning","volume":"521","author":"lecun","year":"2015","journal-title":"Nature"},{"key":"ref58","doi-asserted-by":"publisher","DOI":"10.1109\/IROS.2012.6386109"},{"key":"ref57","first-page":"214","article-title":"Wasserstein generative adversarial networks","author":"arjovsky","year":"2017","journal-title":"Proc Int Conf Mach Learn"},{"key":"ref56","article-title":"Proximal policy optimization algorithms","author":"schulman","year":"2017"},{"key":"ref55","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-27645-3"},{"key":"ref54","doi-asserted-by":"publisher","DOI":"10.1186\/1471-2105-8-S2-S3"},{"key":"ref53","doi-asserted-by":"publisher","DOI":"10.1098\/rstb.2005.1654"},{"key":"ref52","doi-asserted-by":"publisher","DOI":"10.1016\/j.jspi.2007.05.035"},{"key":"ref10","first-page":"3145","article-title":"Learning important features through propagating activation differences","author":"shrikumar","year":"2017","journal-title":"Proc Int Conf Mach Learn"},{"key":"ref11","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2017.74"},{"key":"ref40","article-title":"Exploratory not explanatory: Counterfactual analysis of saliency maps for deep reinforcement learning","author":"atrey","year":"2020","journal-title":"Proc Int Conf Learn Representations"},{"key":"ref12","first-page":"4765","article-title":"A unified approach to interpreting model predictions","author":"lundberg","year":"2017","journal-title":"Proc Adv Neural Inf Process Syst"},{"key":"ref13","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2017.354"},{"key":"ref14","doi-asserted-by":"publisher","DOI":"10.1109\/TPAMI.2018.2858759"},{"key":"ref15","first-page":"1899","article-title":"Graying the black box: Understanding DQNs","author":"zahavy","year":"2016","journal-title":"Proc Int Conf Mach Learn"},{"key":"ref16","first-page":"1995","article-title":"Dueling network architectures for deep reinforcement learning","volume":"48","author":"wang","year":"2016","journal-title":"Proc Int Conf Mach Learn"},{"key":"ref17","first-page":"1792","article-title":"Visualizing and understanding atari agents","author":"greydanus","year":"2018","journal-title":"Proc Int Conf Mach Learn"},{"key":"ref18","article-title":"Explain your move: Understanding agent actions using specific and relevant feature attribution","author":"puri","year":"2020","journal-title":"Proc Int Conf Learn Representations"},{"key":"ref19","first-page":"12350","article-title":"Towards interpretable reinforcement learning using attention augmented agents","author":"mott","year":"2019","journal-title":"Proc Adv Neural Inf Process Syst"},{"key":"ref4","doi-asserted-by":"publisher","DOI":"10.5204\/lthj.v1i0.1261"},{"key":"ref3","first-page":"147","article-title":"Continuous state-space models for optimal sepsis treatment: A deep reinforcement learning approach","author":"raghu","year":"2017","journal-title":"Proc Mach Learn Healthcare Conf"},{"key":"ref6","doi-asserted-by":"publisher","DOI":"10.1109\/TPAMI.2019.2901464"},{"key":"ref5","doi-asserted-by":"publisher","DOI":"10.1109\/TPAMI.2019.2943456"},{"key":"ref8","doi-asserted-by":"publisher","DOI":"10.1145\/2939672.2939778"},{"key":"ref7","doi-asserted-by":"publisher","DOI":"10.1109\/TPAMI.2019.2954501"},{"key":"ref49","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v34i04.6161"},{"key":"ref9","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-44781-0_8"},{"key":"ref46","article-title":"Multi-focus attention network for efficient deep reinforcement learning","author":"choi","year":"2017","journal-title":"Proc Workshops 31st AAAI Conf Artif Intell"},{"key":"ref45","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v33i01.33014561"},{"key":"ref48","first-page":"2494","article-title":"Verifiable reinforcement learning via policy extraction","author":"bastani","year":"2018","journal-title":"Proc Adv Neural Inf Process Syst"},{"key":"ref47","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v34i03.5631"},{"key":"ref42","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-36802-9_25"},{"key":"ref41","doi-asserted-by":"publisher","DOI":"10.1109\/TPAMI.2020.2966453"},{"key":"ref44","article-title":"Deep attention recurrent Q-network","author":"sorokin","year":"2015"},{"key":"ref43","doi-asserted-by":"publisher","DOI":"10.1109\/ICCVW.2019.00522"}],"container-title":["IEEE Transactions on Pattern Analysis and Machine Intelligence"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx7\/34\/9940445\/09645253.pdf?arnumber=9645253","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2022,12,5]],"date-time":"2022-12-05T22:36:57Z","timestamp":1670279817000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/9645253\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2022,12,1]]},"references-count":69,"journal-issue":{"issue":"12"},"URL":"https:\/\/doi.org\/10.1109\/tpami.2021.3133717","relation":{},"ISSN":["0162-8828","2160-9292","1939-3539"],"issn-type":[{"value":"0162-8828","type":"print"},{"value":"2160-9292","type":"electronic"},{"value":"1939-3539","type":"electronic"}],"subject":[],"published":{"date-parts":[[2022,12,1]]}}}