{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,17]],"date-time":"2026-07-17T03:59:11Z","timestamp":1784260751654,"version":"3.55.0"},"reference-count":223,"publisher":"Institute of Electrical and Electronics Engineers (IEEE)","issue":"4","license":[{"start":{"date-parts":[[2025,4,1]],"date-time":"2025-04-01T00:00:00Z","timestamp":1743465600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/ieeexplore.ieee.org\/Xplorehelp\/downloads\/license-information\/IEEE.html"},{"start":{"date-parts":[[2025,4,1]],"date-time":"2025-04-01T00:00:00Z","timestamp":1743465600000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2025,4,1]],"date-time":"2025-04-01T00:00:00Z","timestamp":1743465600000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"}],"funder":[{"DOI":"10.13039\/501100018537","name":"National Science and Technology Major Project","doi-asserted-by":"publisher","award":["2018AAA0102900"],"award-info":[{"award-number":["2018AAA0102900"]}],"id":[{"id":"10.13039\/501100018537","id-type":"DOI","asserted-by":"publisher"}]},{"name":"\u201cNew Generation Artificial Intelligence\u201d Key Field Research and Development Plan of Guangdong Province","award":["2021B0101410002"],"award-info":[{"award-number":["2021B0101410002"]}]},{"DOI":"10.13039\/100014717","name":"National Outstanding Youth Science Fund Project of National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["62122022"],"award-info":[{"award-number":["62122022"]}],"id":[{"id":"10.13039\/100014717","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100018537","name":"National Science and Technology Major Project","doi-asserted-by":"publisher","award":["2018AAA0102900"],"award-info":[{"award-number":["2018AAA0102900"]}],"id":[{"id":"10.13039\/501100018537","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["61876043"],"award-info":[{"award-number":["61876043"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["61976052"],"award-info":[{"award-number":["61976052"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["62306019"],"award-info":[{"award-number":["62306019"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]},{"name":"Guangdong Provincial Science and Technology Innovation Strategy Fund","award":["2019B121203012"],"award-info":[{"award-number":["2019B121203012"]}]},{"name":"Beijing Natural Science Foundation","award":["4244098"],"award-info":[{"award-number":["4244098"]}]},{"DOI":"10.13039\/501100002858","name":"China Postdoctoral Science Foundation","doi-asserted-by":"publisher","award":["2022M711812"],"award-info":[{"award-number":["2022M711812"]}],"id":[{"id":"10.13039\/501100002858","id-type":"DOI","asserted-by":"publisher"}]},{"name":"Tsinghua University Initiative Scientific Research Program"},{"name":"Research and Development Program of Beijing Municipal Education Commission","award":["KM202410011016"],"award-info":[{"award-number":["KM202410011016"]}]}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["IEEE Trans. Neural Netw. Learning Syst."],"published-print":{"date-parts":[[2025,4]]},"DOI":"10.1109\/tnnls.2024.3403001","type":"journal-article","created":{"date-parts":[[2024,11,28]],"date-time":"2024-11-28T13:55:55Z","timestamp":1732802155000},"page":"5942-5962","source":"Crossref","is-referenced-by-count":36,"title":["A Survey on Causal Reinforcement Learning"],"prefix":"10.1109","volume":"36","author":[{"ORCID":"https:\/\/orcid.org\/0000-0001-7721-2560","authenticated-orcid":false,"given":"Yan","family":"Zeng","sequence":"first","affiliation":[{"name":"School of Mathematics and Statistics, Beijing Technology and Business University, Beijing, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-8972-167X","authenticated-orcid":false,"given":"Ruichu","family":"Cai","sequence":"additional","affiliation":[{"name":"School of Computer Science, Guangdong University of Technology, Guangzhou, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-3546-6305","authenticated-orcid":false,"given":"Fuchun","family":"Sun","sequence":"additional","affiliation":[{"name":"Department of Computer Science and Technology, Tsinghua University, Beijing, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-4479-5840","authenticated-orcid":false,"given":"Libo","family":"Huang","sequence":"additional","affiliation":[{"name":"Chinese Academy of Sciences, Institute of Computing Technology, Beijing, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-9731-1504","authenticated-orcid":false,"given":"Zhifeng","family":"Hao","sequence":"additional","affiliation":[{"name":"College of Science, Shantou University, Shantou, China"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"263","reference":[{"key":"ref1","volume-title":"Reinforcement Learning: An Introduction","author":"Sutton","year":"2018"},{"key":"ref2","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v32i1.11694"},{"key":"ref3","doi-asserted-by":"publisher","DOI":"10.1038\/nature14236"},{"key":"ref4","article-title":"Playing Atari with deep reinforcement learning","volume-title":"Proc. Adv. Neural Inf. Process. Syst. (NIPS)","author":"Mnih"},{"key":"ref5","doi-asserted-by":"publisher","DOI":"10.1126\/science.aau6249"},{"key":"ref6","doi-asserted-by":"publisher","DOI":"10.1038\/s41586-019-1724-z"},{"key":"ref7","doi-asserted-by":"publisher","DOI":"10.1177\/0278364913495721"},{"key":"ref8","first-page":"49","article-title":"Guided cost learning: Deep inverse optimal control via policy optimization","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Finn"},{"key":"ref9","doi-asserted-by":"publisher","DOI":"10.1109\/tnnls.2023.3280161"},{"key":"ref10","doi-asserted-by":"publisher","DOI":"10.1145\/3442381.3449985"},{"key":"ref11","doi-asserted-by":"publisher","DOI":"10.1145\/3477600"},{"key":"ref12","doi-asserted-by":"publisher","DOI":"10.1109\/JPROC.2021.3058954"},{"key":"ref13","first-page":"1342","article-title":"Bandits with unobserved confounders: A causal approach","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","volume":"28","author":"Bareinboim"},{"key":"ref14","volume-title":"Elements of Causal Inference: Foundations and Learning Algorithms","author":"Jonas","year":"2017"},{"key":"ref15","volume-title":"Causality: Models, Reasoning, and Inference","author":"Pearl","year":"2000"},{"key":"ref16","doi-asserted-by":"publisher","DOI":"10.3389\/fgene.2019.00524"},{"key":"ref17","doi-asserted-by":"publisher","DOI":"10.1186\/s40535-016-0018-x"},{"key":"ref18","doi-asserted-by":"publisher","DOI":"10.7551\/mitpress\/1754.001.0001"},{"key":"ref19","doi-asserted-by":"publisher","DOI":"10.1017\/CBO9781139025751"},{"key":"ref20","article-title":"Policy learning for balancing short-term and long-term rewards","volume-title":"arXiv:2405.03329","author":"Wu","year":"2024"},{"key":"ref21","volume-title":"The Book of Why: The New Science of Cause and Effect","author":"Pearl","year":"2018"},{"key":"ref22","article-title":"Causal reinforcement learning","volume-title":"Proc. 39th Int. Conf. Mach. Learn.","author":"Bareinboim"},{"key":"ref23","article-title":"Introduction to causal reinforcement learning","author":"Lu","year":"2018","journal-title":"Blogpost"},{"key":"ref24","article-title":"Causal multi-agent reinforcement learning: Review and open problems","volume-title":"Proc. Cooperat. AI Workshop Adv. Neural Inf. Process. Syst.","author":"Grimbly"},{"key":"ref25","article-title":"Causality and batch reinforcement learning: Complementary approaches to planning in unknown domains","volume-title":"arXiv:2006.02579","author":"Bannon","year":"2020"},{"key":"ref26","article-title":"Causal machine learning: A survey and open problems","volume-title":"arXiv:2206.15475","author":"Kaddour","year":"2022"},{"key":"ref27","volume-title":"Causal Inference: What If","author":"Hern\u00e1n","year":"2020"},{"key":"ref28","first-page":"2003","article-title":"A linear non-Gaussian acyclic model for causal discovery","volume":"7","author":"Shimizu","year":"2006","journal-title":"J. Mach. Learn. Res."},{"key":"ref29","article-title":"Causal inference in the presence of latent variables and selection bias","volume-title":"arXiv:1302.4983","author":"Spirtes","year":"2013"},{"key":"ref30","doi-asserted-by":"publisher","DOI":"10.1177\/089443939100900106"},{"key":"ref31","first-page":"121","article-title":"On causal discovery from time series data using FCI","volume-title":"Proc. 5th Eur. Workshop Probabilistic Graph. Models","author":"Entner"},{"key":"ref32","first-page":"23","article-title":"Causal structure learning from multivariate time series in settings with unmeasured confounding","volume-title":"Proc. ACM SIGKDD Workshop Causal Discovery","author":"Malinsky"},{"key":"ref33","doi-asserted-by":"publisher","DOI":"10.1145\/1390156.1390210"},{"key":"ref34","first-page":"12615","article-title":"High-recall causal discovery for autocorrelated time series with latent confounders","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","volume":"33","author":"Gerhardus"},{"key":"ref35","doi-asserted-by":"publisher","DOI":"10.2307\/1912791.1969"},{"issue":"89","key":"ref36","first-page":"1","article-title":"Causal discovery from heterogeneous\/nonstationary data","volume":"21","author":"Huang","year":"2020","journal-title":"J. Mach. Learn. Res."},{"key":"ref37","doi-asserted-by":"publisher","DOI":"10.1145\/3219819.3220104"},{"key":"ref38","first-page":"507","article-title":"Optimal structure identification with greedy search","volume":"3","author":"Chickering","year":"2002","journal-title":"J. Mach. Learn. Res."},{"key":"ref39","doi-asserted-by":"publisher","DOI":"10.1007\/BF00994016"},{"key":"ref40","doi-asserted-by":"publisher","DOI":"10.1214\/aos\/1176344136"},{"key":"ref41","first-page":"1225","article-title":"DirectLiNGAM: A direct method for learning a linear non-Gaussian structural equation model","volume":"12","author":"Shimizu","year":"2011","journal-title":"J. Mach. Learn. Res."},{"issue":"14","key":"ref42","first-page":"1","article-title":"Python package for causal discovery based on lingam","volume":"24","author":"Ikeuchi","year":"2023","journal-title":"J. Mach. Learn. Res."},{"key":"ref43","first-page":"689","article-title":"Nonlinear causal discovery with additive noise models","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","volume":"21","author":"Hoyer"},{"key":"ref44","first-page":"647","article-title":"On the identifiability of the post-nonlinear causal model","volume-title":"Proc. 25th Conf. Uncertainty Artif. Intell.","author":"Zhang"},{"key":"ref45","doi-asserted-by":"publisher","DOI":"10.1016\/j.ijar.2008.02.006"},{"key":"ref46","doi-asserted-by":"publisher","DOI":"10.1162\/NECO_a_00533"},{"key":"ref47","first-page":"735","article-title":"RCD: Repetitive causal discovery of linear non-Gaussian acyclic models with latent confounders","volume-title":"Proc. Int. Conf. Artif. Intell. Statist.","author":"Maeda"},{"key":"ref48","doi-asserted-by":"publisher","DOI":"10.1109\/TNNLS.2020.3045812"},{"key":"ref49","first-page":"14891","article-title":"Generalized independent noise condition for estimating latent variable causal graphs","volume-title":"Proc. Adv. neural Inf. Process. Syst.","volume":"33","author":"Xie"},{"key":"ref50","first-page":"24370","article-title":"Identification of linear non-Gaussian latent hierarchical structure","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Xie"},{"key":"ref51","doi-asserted-by":"publisher","DOI":"10.24963\/ijcai.2021\/289"},{"key":"ref52","doi-asserted-by":"publisher","DOI":"10.1109\/TNNLS.2022.3205128"},{"key":"ref53","doi-asserted-by":"publisher","DOI":"10.1109\/TNNLS.2019.2921613"},{"key":"ref54","doi-asserted-by":"publisher","DOI":"10.1109\/TNNLS.2021.3106111"},{"key":"ref55","doi-asserted-by":"publisher","DOI":"10.1109\/TNNLS.2017.2734804"},{"key":"ref56","doi-asserted-by":"publisher","DOI":"10.1037\/h0037350"},{"key":"ref57","doi-asserted-by":"publisher","DOI":"10.1093\/oxfordhb\/9780199286546.003.0011"},{"key":"ref58","doi-asserted-by":"publisher","DOI":"10.4135\/9781412952637.n77"},{"key":"ref59","doi-asserted-by":"publisher","DOI":"10.1093\/biomet\/70.1.41"},{"key":"ref60","doi-asserted-by":"publisher","DOI":"10.1111\/j.1467-6419.2007.00527.x"},{"key":"ref61","doi-asserted-by":"publisher","DOI":"10.1198\/jcgs.2010.08162"},{"key":"ref62","first-page":"3020","article-title":"Learning representations for counterfactual inference","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Johansson"},{"key":"ref63","doi-asserted-by":"publisher","DOI":"10.1073\/pnas.1804597116"},{"key":"ref64","doi-asserted-by":"publisher","DOI":"10.1093\/biomet\/asaa076"},{"key":"ref65","doi-asserted-by":"publisher","DOI":"10.1007\/978-1-4612-1284-3_1"},{"key":"ref66","doi-asserted-by":"publisher","DOI":"10.1198\/016214506000000023"},{"issue":"302","key":"ref67","first-page":"1","article-title":"On instrumental variable regression for deep offline policy evaluation","volume":"23","author":"Chen","year":"2022","journal-title":"J. Mach. Learn. Res."},{"key":"ref68","doi-asserted-by":"publisher","DOI":"10.2139\/ssrn.3792824"},{"key":"ref69","first-page":"26264","article-title":"Deep proxy causal learning and its application to confounded bandit policy evaluation","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","volume":"34","author":"Xu"},{"key":"ref70","doi-asserted-by":"publisher","DOI":"10.1093\/biomet\/asy038"},{"key":"ref71","doi-asserted-by":"publisher","DOI":"10.1126\/science.153.3731.34"},{"key":"ref72","doi-asserted-by":"publisher","DOI":"10.1016\/j.compchemeng.2020.106886"},{"key":"ref73","doi-asserted-by":"publisher","DOI":"10.1038\/058390b0"},{"issue":"1","key":"ref74","first-page":"1","article-title":"The epoch-greedy algorithm for contextual multi-armed bandits","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","volume":"20","author":"Langford"},{"key":"ref75","first-page":"11698","article-title":"Causal confusion in imitation learning","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","volume":"32","author":"De Haan"},{"key":"ref76","doi-asserted-by":"publisher","DOI":"10.1111\/1467-9868.00389"},{"key":"ref77","first-page":"11012","article-title":"Designing optimal dynamic treatment regimes: A causal reinforcement learning approach","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Zhang"},{"key":"ref78","doi-asserted-by":"publisher","DOI":"10.1561\/2200000086"},{"key":"ref79","first-page":"1889","article-title":"Trust region policy optimization","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Schulman"},{"key":"ref80","article-title":"Proximal policy optimization algorithms","volume-title":"arXiv:1707.06347","author":"Schulman","year":"2017"},{"key":"ref81","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v30i1.10295"},{"key":"ref82","first-page":"1995","article-title":"Dueling network architectures for deep reinforcement learning","volume-title":"Proc. Int. Conf. Int. Conf. Mach. Learn.","volume":"48","author":"Wang"},{"key":"ref83","first-page":"1008","article-title":"Actor-critic algorithms","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","volume":"12","author":"Konda"},{"key":"ref84","first-page":"1928","article-title":"Asynchronous methods for deep reinforcement learning","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Mnih"},{"key":"ref85","first-page":"1861","article-title":"Soft actor-critic: Off-policy maximum entropy deep reinforcement learning with a stochastic actor","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Haarnoja"},{"key":"ref86","article-title":"Continuous control with deep reinforcement learning","volume-title":"Proc. Int. Conf. Learn. Represent. (ICLR)","author":"Lillicrap"},{"key":"ref87","first-page":"1109","article-title":"VIME: Variational information maximizing exploration","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","volume":"29","author":"Houthooft"},{"key":"ref88","doi-asserted-by":"publisher","DOI":"10.1007\/s11432-022-3696-5"},{"key":"ref89","doi-asserted-by":"publisher","DOI":"10.1109\/ICRA.2018.8463189"},{"key":"ref90","first-page":"4754","article-title":"Deep reinforcement learning in a handful of trials using probabilistic dynamics models","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","volume":"31","author":"Chua"},{"key":"ref91","doi-asserted-by":"publisher","DOI":"10.1609\/aiide.v4i1.18700"},{"key":"ref92","doi-asserted-by":"publisher","DOI":"10.13140\/RG.2.2.18893.74727"},{"key":"ref93","doi-asserted-by":"publisher","DOI":"10.1038\/nature24270"},{"key":"ref94","article-title":"Model-ensemble trust-region policy optimization","volume-title":"Proc. Int. Conf. Learn. Represent.","author":"Kurutach"},{"key":"ref95","first-page":"12519","article-title":"When to trust your model: Model-based policy optimization","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","volume":"32","author":"Janner"},{"key":"ref96","first-page":"1","article-title":"Guided policy search","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Levine"},{"key":"ref97","first-page":"465","article-title":"PILCO: A model-based and data-efficient approach to policy search","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Deisenroth"},{"key":"ref98","first-page":"14129","article-title":"MOPO: Model-based offline policy optimization","volume-title":"Proc. Int. Conf. Adv. Neural Inf. Process. Syst.","volume":"33","author":"Yu"},{"key":"ref99","first-page":"28954","article-title":"Combo: Conservative offline model-based policy optimization","volume-title":"Proc. Int. Conf. Adv. Neural Inf. Process. Syst.","volume":"34","author":"Yu"},{"key":"ref100","doi-asserted-by":"publisher","DOI":"10.2307\/jj.20206644.79"},{"key":"ref101","first-page":"16728","article-title":"Model-based offline reinforcement learning with count-based conservatism","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Kim"},{"key":"ref102","doi-asserted-by":"publisher","DOI":"10.1561\/2300000021"},{"key":"ref103","article-title":"Instrumental variable value iteration for causal offline reinforcement learning","volume-title":"arXiv:2102.09907","author":"Liao","year":"2021"},{"key":"ref104","article-title":"Sample-efficient reinforcement learning via counterfactual-based data augmentation","volume-title":"Proc. Offline Reinforcement Learn. Workshop Neural Inf. Process. Syst.","author":"Lu"},{"key":"ref105","volume-title":"CausalDyna: Improving generalization of dyna-style reinforcement learning via counterfactual-based data augmentation","author":"Zhu","year":"2021"},{"key":"ref106","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v36i8.20913"},{"key":"ref107","volume-title":"Markov decision processes with unobserved confounders: A causal approach","author":"Zhang","year":"2016"},{"key":"ref108","first-page":"21164","article-title":"Provably efficient causal reinforcement learning with confounded observational data","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","volume":"34","author":"Wang"},{"key":"ref109","first-page":"1116","article-title":"Model-free and model-based policy evaluation when causality is uncertain","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Bruns-Smith"},{"key":"ref110","doi-asserted-by":"publisher","DOI":"10.1080\/01621459.2022.2110878"},{"key":"ref111","first-page":"18819","article-title":"Off-policy policy evaluation for sequential decisions under unobserved confounding","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","volume":"33","author":"Namkoong"},{"key":"ref112","article-title":"A relational intervention approach for unsupervised dynamics generalization in model-based reinforcement learning","volume-title":"Proc. Int. Conf. Learn. Represent.","author":"Guo"},{"key":"ref113","article-title":"Woulda, coulda, shoulda: Counterfactually-guided policy search","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Buesing"},{"key":"ref114","first-page":"4881","article-title":"Counterfactual off-policy evaluation with Gumbel-max structural causal models","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Oberst"},{"key":"ref115","first-page":"5","article-title":"Counterfactually guided policy transfer in clinical settings","volume-title":"Proc. Conf. Health, Inference, Learn.","author":"Killian"},{"key":"ref116","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v34i06.6590"},{"key":"ref117","article-title":"Proximal reinforcement learning: Efficient off-policy evaluation in partially observed Markov decision processes","volume-title":"arXiv:2110.15332","author":"Bennett","year":"2021"},{"key":"ref118","article-title":"Off-policy evaluation in partially observed Markov decision processes under sequential ignorability","volume-title":"arXiv:2110.12343","author":"Hu","year":"2021"},{"key":"ref119","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v32i1.11794"},{"key":"ref120","first-page":"3040","article-title":"Social influence as intrinsic motivation for multi-agent deep reinforcement learning","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Jaques"},{"key":"ref121","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-02053-8_64"},{"key":"ref122","first-page":"20057","article-title":"A minimax learning approach to off-policy evaluation in confounded partially observable Markov decision processes","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Shi"},{"key":"ref123","first-page":"1156","article-title":"Counterfactual data-fusion for online reinforcement learners","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Forney"},{"key":"ref124","first-page":"3057","article-title":"Identifying best interventions through online importance sampling","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Sen"},{"key":"ref125","first-page":"5512","article-title":"Causal bandits with propagating inference","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Yabe"},{"key":"ref126","first-page":"1181","article-title":"Causal bandits: Learning good interventions via causal inference","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","volume":"29","author":"Lattimore"},{"key":"ref127","first-page":"141","article-title":"Regret analysis of bandit problems with causal background knowledge","volume-title":"Proc. 36th Conf. Uncertainty Artif. Intell. (UAI)","author":"Lu"},{"key":"ref128","doi-asserted-by":"publisher","DOI":"10.1145\/3442381.3449982"},{"key":"ref129","doi-asserted-by":"publisher","DOI":"10.24963\/ijcai.2017\/186"},{"key":"ref130","first-page":"2568","article-title":"Structural causal bandits: Where to intervene?","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","volume":"31","author":"Lee"},{"key":"ref131","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v33i01.33014164"},{"key":"ref132","article-title":"Causal contextual bandits with targeted interventions","volume-title":"Proc. Int. Conf. Learn. Represent.","author":"Subramanian"},{"key":"ref133","first-page":"4826","article-title":"Policy evaluation with latent confounders via optimal balance","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","volume":"32","author":"Bennett"},{"key":"ref134","first-page":"9269","article-title":"Confounding-robust policy improvement","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","volume":"31","author":"Kallus"},{"key":"ref135","doi-asserted-by":"publisher","DOI":"10.1287\/mnsc.2020.3699"},{"key":"ref136","first-page":"13401","article-title":"Near-optimal reinforcement learning in dynamic treatment regimes","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","volume":"32","author":"Zhang"},{"key":"ref137","first-page":"3191","article-title":"Online reinforcement learning for mixed policy scopes","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","volume":"35","author":"Zhang"},{"key":"ref138","article-title":"Estimating and improving dynamic treatment regimes with a time-varying instrumental variable","volume-title":"arXiv:2104.07822","author":"Chen","year":"2021"},{"key":"ref139","first-page":"12263","article-title":"Causal imitation learning with unobserved confounders","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","volume":"33","author":"Zhang"},{"key":"ref140","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v34i06.6571"},{"key":"ref141","first-page":"14669","article-title":"Sequential causal imitation learning with unobserved confounders","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","volume":"34","author":"Kumor"},{"key":"ref142","article-title":"On covariate shift of latent confounders in imitation and reinforcement learning","volume-title":"Proc. Int. Conf. Learn. Represent.","author":"Tennenholtz"},{"key":"ref143","first-page":"20877","article-title":"Causal imitation learning under temporally correlated noise","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Swamy"},{"key":"ref144","doi-asserted-by":"publisher","DOI":"10.1109\/TAC.2017.2775960"},{"key":"ref145","article-title":"Causal induction from visual observations for goal directed tasks","volume-title":"arXiv:1910.01751","author":"Nair","year":"2019"},{"key":"ref146","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-89817-5_16"},{"key":"ref147","first-page":"526","article-title":"Efficient reinforcement learning with prior causal knowledge","volume-title":"Proc. 1st Conf. Causal Learn. Reasoning","author":"Lu"},{"key":"ref148","first-page":"2493","article-title":"Explainable reinforcement learning through a causal lens","volume-title":"Proc. AAAI Conf. Artif. Intell.","volume":"34","author":"Madumal"},{"key":"ref149","first-page":"22293","article-title":"Confounding-robust policy evaluation in infinite-horizon reinforcement learning","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","volume":"33","author":"Kallus"},{"key":"ref150","first-page":"1999","article-title":"Off-policy evaluation in infinite-horizon reinforcement learning with latent confounders","volume-title":"Proc. Int. Conf. Artif. Intell. Statist.","author":"Bennett"},{"key":"ref151","article-title":"A spectral approach to off-policy evaluation for pomdps","volume-title":"arXiv:2109.10502","author":"Nair","year":"2021"},{"key":"ref152","first-page":"2017","article-title":"Budgeted and non-budgeted causal bandits","volume-title":"Proc. Int. Conf. Artif. Intell. Statist.","author":"Nair"},{"key":"ref153","first-page":"8565","article-title":"Characterizing optimal mixed policies: Where to intervene and what to observe","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","volume":"33","author":"Lee"},{"key":"ref154","doi-asserted-by":"publisher","DOI":"10.1145\/3054912"},{"key":"ref155","doi-asserted-by":"publisher","DOI":"10.1561\/2300000053"},{"key":"ref156","doi-asserted-by":"publisher","DOI":"10.1145\/1015330.1015430"},{"key":"ref157","first-page":"1255","article-title":"Modeling interaction via the principle of maximum causal entropy","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Ziebart"},{"key":"ref158","doi-asserted-by":"publisher","DOI":"10.1109\/TIT.2012.2234824"},{"key":"ref159","article-title":"Offline reinforcement learning with causal structured world models","volume-title":"arXiv:2206.01474","author":"Zhu","year":"2022"},{"key":"ref160","first-page":"8733","article-title":"Learning plannable representations with causal infogan","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","volume":"31","author":"Kurutach"},{"key":"ref161","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v36i6.20648"},{"key":"ref162","article-title":"Learning invariant representations for reinforcement learning without reconstruction","volume-title":"Proc. Int. Conf. Learn. Represent.","author":"Zhang"},{"key":"ref163","article-title":"Intervention design for effective Sim2Real transfer","volume-title":"arXiv:2012.02055","author":"Mozifian","year":"2020"},{"key":"ref164","first-page":"3976","article-title":"Counterfactual data augmentation using locally factored dynamics","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","volume":"33","author":"Pitis"},{"key":"ref165","first-page":"22905","article-title":"Causal influence detection for improving efficiency in reinforcement learning","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","volume":"34","author":"Seitzer"},{"key":"ref166","article-title":"Factored adaptation for non-stationary reinforcement learning","volume-title":"arXiv:2203.16582","author":"Feng","year":"2022"},{"key":"ref167","first-page":"20064","article-title":"Causality-driven hierarchical structure discovery for reinforcement learning","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","author":"Peng"},{"key":"ref168","first-page":"23151","article-title":"Causal dynamics learning for task-independent state abstraction","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Wang"},{"key":"ref169","first-page":"9848","article-title":"Causal curiosity: RL agents discovering self-supervised experiments for causal representation learning","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Sontakke"},{"key":"ref170","article-title":"Deconfounding reinforcement learning in observational settings","volume-title":"arXiv:1812.10576","author":"Lu","year":"2018"},{"key":"ref171","first-page":"9260","article-title":"Action-sufficient state representation learning for control with structural constraints","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Huang"},{"key":"ref172","article-title":"AdaRL: What, where, and how to adapt in transfer reinforcement learning","volume-title":"Proc. Int. Conf. Learn. Represent.","author":"Huang"},{"key":"ref173","article-title":"Learning causal state representations of partially observable environments","volume-title":"arXiv:1906.10437","author":"Zhang","year":"2019"},{"key":"ref174","first-page":"21","article-title":"Invariant policy optimization: Towards stronger generalization in reinforcement learning","volume-title":"Proc. Learn. Dyn. Control","author":"Sonar"},{"key":"ref175","first-page":"24817","article-title":"Causal bandits with unknown graph structure","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","volume":"34","author":"Lu"},{"key":"ref176","first-page":"8606","article-title":"Invariant policy learning: A causal perspective","volume-title":"Proc. Workshop Reinforcement Learn. Theory 38th Int. Conf. Mach. Learn.","author":"Saengkyongam"},{"key":"ref177","first-page":"430","article-title":"Bandits with partially observable confounded data","volume-title":"Uncertainty in Artificial Intelligence","author":"Tennenholtz","year":"2021"},{"key":"ref178","article-title":"Causal imitative model for autonomous driving","volume-title":"arXiv:2112.03908","author":"Reza Samsami","year":"2021"},{"key":"ref179","volume-title":"A cognitive robotic imitation learning system based on cause-effect reasoning","author":"Katz","year":"2017"},{"key":"ref180","article-title":"Learning what-if explanations for sequential decision-making","volume-title":"Proc. Int. Conf. Learn. Represent.","author":"Bica"},{"key":"ref181","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2019.00942"},{"key":"ref182","first-page":"3952","article-title":"Invariant causal imitation learning for generalizable policies","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","volume":"34","author":"Bica"},{"key":"ref183","doi-asserted-by":"publisher","DOI":"10.1109\/TCDS.2017.2651643"},{"key":"ref184","first-page":"2564","article-title":"Fighting copycat agents in behavioral cloning from observation histories","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","volume":"33","author":"Wen"},{"key":"ref185","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-19842-7_23"},{"key":"ref186","first-page":"11123","article-title":"Keyframe-focused visual imitation learning","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Wen"},{"key":"ref187","first-page":"23723","article-title":"Fighting fire with fire: Avoiding DNN shortcuts through priming","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Wen"},{"key":"ref188","doi-asserted-by":"publisher","DOI":"10.24963\/ijcai.2021\/684"},{"key":"ref189","first-page":"26532","article-title":"Generalizing goal-conditioned reinforcement learning with variational causal reasoning","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","author":"Ding"},{"key":"ref190","volume-title":"Structural Causal Models for Reinforcement Learning","author":"Luczkow","year":"2021"},{"key":"ref191","doi-asserted-by":"publisher","DOI":"10.1109\/TII.2023.3333921"},{"key":"ref192","first-page":"11214","article-title":"Invariant causal prediction for block MDPs","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Zhang"},{"key":"ref193","article-title":"Model-invariant state abstractions for model-based reinforcement learning","volume-title":"arXiv:2102.09850","author":"Tomar","year":"2021"},{"key":"ref194","doi-asserted-by":"publisher","DOI":"10.1109\/TNNLS.2023.3296642"},{"key":"ref195","article-title":"Resolving causal confusion in reinforcement learning via robust exploration","volume-title":"Proc. Self-Supervision Reinforcement Learn. Workshop ICLR","author":"Lyle"},{"key":"ref196","article-title":"Causal reasoning from meta-reinforcement learning","volume-title":"arXiv:1901.08162","author":"Dasgupta","year":"2019"},{"key":"ref197","doi-asserted-by":"publisher","DOI":"10.1109\/TNNLS.2022.3165114"},{"key":"ref198","article-title":"Causally correct partial models for reinforcement learning","volume-title":"arXiv:2002.02836","author":"Rezende","year":"2020"},{"key":"ref199","doi-asserted-by":"publisher","DOI":"10.1109\/TNNLS.2021.3107375"},{"key":"ref200","first-page":"1809","article-title":"Schema networks: Zero-shot transfer with a generative causal model of intuitive physics","volume-title":"Proc. 34th Int. Conf. Mach. Learn.","author":"Kansky"},{"key":"ref201","article-title":"Causal reinforcement learning using observational and interventional data","volume-title":"arXiv:2106.14421","author":"Gasse","year":"2021"},{"key":"ref202","doi-asserted-by":"publisher","DOI":"10.1109\/IROS45743.2020.9340809"},{"key":"ref203","doi-asserted-by":"publisher","DOI":"10.24963\/ijcai.2021\/268"},{"key":"ref204","doi-asserted-by":"publisher","DOI":"10.1093\/jrsssb\/qkad018"},{"key":"ref205","article-title":"Resolving spurious correlations in causal models of environments via interventions","volume-title":"Proc. Causal Learn. Decis. Making Workshop ICLR","author":"Volodin"},{"key":"ref206","doi-asserted-by":"publisher","DOI":"10.1109\/ICRA48506.2021.9561439"},{"key":"ref207","article-title":"Invariant causal representation learning for generalization in imitation and reinforcement learning","volume-title":"Proc. ICLR Workshop Elements Reasoning: Objects, Struct. Causality","author":"Lu"},{"key":"ref208","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-41649-6_7"},{"key":"ref209","article-title":"Causalworld: A robotic manipulation benchmark for causal structure and transfer learning","volume-title":"Proc. Int. Conf. Learn. Represent.","author":"Ahmed"},{"key":"ref210","article-title":"Deep reinforcement learning: An overview","volume-title":"arXiv:1701.07274","author":"Li","year":"2017"},{"key":"ref211","doi-asserted-by":"publisher","DOI":"10.1109\/IROS.2012.6386109"},{"key":"ref212","article-title":"Ace: Off-policy actor-critic with causality-aware entropy regularization","volume-title":"Proc. 41st Int. Conf. Mach. Learn.","author":"Ji"},{"key":"ref213","first-page":"7156","article-title":"Hierarchical reinforcement learning for zero-shot generalization with subtask dependencies","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","volume":"31","author":"Sohn"},{"key":"ref214","article-title":"Eden: A unified environment framework for booming reinforcement learning algorithms","author":"Chen","year":"2021","journal-title":"arXiv:2109.01768"},{"key":"ref215","article-title":"Offline reinforcement learning: Tutorial, review, and perspectives on open problems","volume-title":"arXiv:2005.01643","author":"Levine","year":"2020"},{"key":"ref216","article-title":"Playing against nature: Causal discovery for decision making under uncertainty","volume-title":"Proc. CausalML Workshop 35th Int. Conf. Mach. Learn.","author":"Gonzalez-Soto"},{"key":"ref217","doi-asserted-by":"publisher","DOI":"10.1109\/TKDE.2018.2827047"},{"key":"ref218","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v33i01.33014902"},{"key":"ref219","doi-asserted-by":"publisher","DOI":"10.1145\/3292500.3330933"},{"key":"ref220","doi-asserted-by":"crossref","DOI":"10.1109\/MSP.2017.2743240","article-title":"A brief survey of deep reinforcement learning","volume-title":"arXiv:1708.05866","author":"Arulkumaran","year":"2017"},{"key":"ref221","first-page":"2827","article-title":"Neural episodic control","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Pritzel"},{"key":"ref222","first-page":"1","article-title":"CARLA: An open urban driving simulator","volume-title":"Proc. 1st Annu. Conf. Robot Learn.","author":"Dosovitskiy"},{"key":"ref223","first-page":"264","article-title":"Smarts: An open-source scalable multi-agent RL training school for autonomous driving","volume-title":"Proc. Conf. Robot Learn. (CoRL)","author":"Zhou"}],"container-title":["IEEE Transactions on Neural Networks and Learning Systems"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx8\/5962385\/10949581\/10771589.pdf?arnumber=10771589","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,12,5]],"date-time":"2025-12-05T18:39:25Z","timestamp":1764959965000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/10771589\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,4]]},"references-count":223,"journal-issue":{"issue":"4"},"URL":"https:\/\/doi.org\/10.1109\/tnnls.2024.3403001","relation":{},"ISSN":["2162-237X","2162-2388"],"issn-type":[{"value":"2162-237X","type":"print"},{"value":"2162-2388","type":"electronic"}],"subject":[],"published":{"date-parts":[[2025,4]]}}}