{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,11,3]],"date-time":"2025-11-03T02:05:14Z","timestamp":1762135514800,"version":"build-2065373602"},"reference-count":59,"publisher":"Springer Science and Business Media LLC","issue":"2","license":[{"start":{"date-parts":[[2025,10,29]],"date-time":"2025-10-29T00:00:00Z","timestamp":1761696000000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2025,10,29]],"date-time":"2025-10-29T00:00:00Z","timestamp":1761696000000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Sci. China Inf. Sci."],"published-print":{"date-parts":[[2026,2]]},"DOI":"10.1007\/s11432-024-4403-7","type":"journal-article","created":{"date-parts":[[2025,11,3]],"date-time":"2025-11-03T00:45:02Z","timestamp":1762130702000},"update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":0,"title":["Learning by doing: an online causal reinforcement learning framework with causal-aware policy"],"prefix":"10.1007","volume":"69","author":[{"given":"Ruichu","family":"Cai","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Siyang","family":"Huang","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Jie","family":"Qiao","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Wei","family":"Chen","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Yan","family":"Zeng","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Keli","family":"Zhang","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Fuchun","family":"Sun","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Yang","family":"Yu","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Zhifeng","family":"Hao","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2025,10,29]]},"reference":[{"key":"4403_CR1","doi-asserted-by":"crossref","first-page":"229","DOI":"10.1017\/S0263574799281520","volume":"17","author":"R S Sutton","year":"1999","unstructured":"Sutton R S, Barto A G. Reinforcement learning: an introduction. Robotica, 1999, 17:229\u2013235","journal-title":"Robotica"},{"key":"4403_CR2","doi-asserted-by":"publisher","first-page":"1238","DOI":"10.1177\/0278364913495721","volume":"32","author":"J Kober","year":"2013","unstructured":"Kober J, Bagnell J A, Peters J. Reinforcement learning in robotics: a survey. Int J Robot Res, 2013, 32: 1238\u20131274","journal-title":"Int J Robot Res"},{"key":"4403_CR3","doi-asserted-by":"publisher","first-page":"484","DOI":"10.1038\/nature16961","volume":"529","author":"D Silver","year":"2016","unstructured":"Silver D, Huang A, Maddison C J, et al. Mastering the game of Go with deep neural networks and tree search. Nature, 2016, 529: 484\u2013489","journal-title":"Nature"},{"key":"4403_CR4","unstructured":"Shalev-Shwartz S, Shammah S, Shashua A. Safe, multi-agent, reinforcement learning for autonomous driving. 2016. ArXiv:161003295"},{"key":"4403_CR5","doi-asserted-by":"publisher","first-page":"1035","DOI":"10.1109\/TNNLS.2021.3107375","volume":"34","author":"Y Sun","year":"2023","unstructured":"Sun Y, Zhang K, Sun C. Model-based transfer reinforcement learning based on graphical model representations. IEEE Trans Neural Netw Learn Syst, 2023, 34: 1035\u20131048","journal-title":"IEEE Trans Neural Netw Learn Syst"},{"key":"4403_CR6","unstructured":"Zhu Z M, Chen X H, Tian H L, et al. Offline reinforcement learning with causal structured world models. 2022. ArXiv:220601474"},{"key":"4403_CR7","first-page":"9848","volume":"139","author":"S A Sontakke","year":"2021","unstructured":"Sontakke S A, Mehrjou A, Itti L, et al. Causal curiosity: RL agents discovering self-supervised experiments for causal representation learning. In: Proceedings of International Conference on Machine Learning, 2021. 139: 9848\u20139858","journal-title":"Proceedings of International Conference on Machine Learning"},{"key":"4403_CR8","volume-title":"Proceedings of the 9th International Conference on Learning Representations","author":"A Zhang","year":"2021","unstructured":"Zhang A, McAllister R T, Calandra R, et al. Learning invariant representations for reinforcement learning without reconstruction. In: Proceedings of the 9th International Conference on Learning Representations, 2021"},{"key":"4403_CR9","unstructured":"Tomar M, Zhang A, Calandra R, et al. Model-invariant state abstractions for model-based reinforcement learning. 2021. ArXiv:210209850"},{"key":"4403_CR10","first-page":"3952","volume-title":"Proceedings of Advances in Neural Information Processing Systems","author":"I Bica","year":"2021","unstructured":"Bica I, Jarrett D, van der Schaar M. Invariant causal imitation learning for generalizable policies. In: Proceedings of Advances in Neural Information Processing Systems, 2021. 3952\u20133964"},{"key":"4403_CR11","volume-title":"Proceedings of ICLR2022 Workshop on the Elements of Reasoning: Objects, Structure and Causality","author":"S Sodhani","year":"2022","unstructured":"Sodhani S, Levine S, Zhang A. Improving generalization with approximate factored value functions. In: Proceedings of ICLR2022 Workshop on the Elements of Reasoning: Objects, Structure and Causality, 2022"},{"key":"4403_CR12","volume-title":"Proceedings of the 35th International Conference on Neural Information Processing Systems, Robot Learning workshop","author":"Z Wang","year":"2021","unstructured":"Wang Z, Xiao X, Zhu Y, et al. Task-independent causal state abstraction. In: Proceedings of the 35th International Conference on Neural Information Processing Systems, Robot Learning workshop. 2021"},{"key":"4403_CR13","first-page":"26532","volume-title":"Proceedings of Advances in Neural Information Processing Systems","author":"W Ding","year":"2022","unstructured":"Ding W, Lin H, Li B, et al. Generalizing goal-conditioned reinforcement learning with variational causal reasoning. In: Proceedings of Advances in Neural Information Processing Systems, 2022. 26532\u201326548"},{"key":"4403_CR14","first-page":"22905","volume":"34","author":"M Seitzer","year":"2021","unstructured":"Seitzer M, Sch\u00f6lkopf B, Martius G. Causal influence detection for improving efficiency in reinforcement learning. In: Proceedings of Advances in Neural Information Processing Systems, 2021. 34: 22905\u201322918","journal-title":"Proceedings of Advances in Neural Information Processing Systems"},{"key":"4403_CR15","volume-title":"Proceedings of the 10th International Conference on Learning Representations","author":"B Huang","year":"2022","unstructured":"Huang B, Feng F, Lu C, et al. Adarl: what, where, and how to adapt in transfer reinforcement learning. In: Proceedings of the 10th International Conference on Learning Representations, 2022"},{"key":"4403_CR16","first-page":"9260","volume-title":"Proceedings of International Conference on Machine Learning","author":"B Huang","year":"2022","unstructured":"Huang B, Lu C, Leqi L, et al. Action-sufficient state representation learning for control with structural constraints. In: Proceedings of International Conference on Machine Learning, 2022. 9260\u20139279"},{"key":"4403_CR17","first-page":"21164","volume-title":"Proceedings of Advances in Neural Information Processing Systems","author":"L Wang","year":"2021","unstructured":"Wang L, Yang Z, Wang Z. Provably efficient causal reinforcement learning with confounded observational data. In: Proceedings of Advances in Neural Information Processing Systems, 2021. 21164\u201321175"},{"key":"4403_CR18","unstructured":"Liao L, Fu Z, Yang Z, et al. Instrumental variable value iteration for causal offline reinforcement learning. 2021. ArXiv:2102.09907"},{"key":"4403_CR19","unstructured":"Volodin S, Wichers N, Nixon J. Resolving spurious correlations in causal models of environments via interventions. 2020. ArXiv:2002.05217"},{"key":"4403_CR20","unstructured":"Zhang A, Lipton Z C, Pineda L, et al. Learning causal state representations of partially observable environments. 2019. ArXiv:1906.10437"},{"key":"4403_CR21","first-page":"4776","volume-title":"Proceedings of 2021 IEEE International Conference on Robotics and Automation (ICRA)","author":"T E Lee","year":"2021","unstructured":"Lee T E, Zhao J A, Sawhney A S, et al. Causal reasoning in simulation for structure and transfer learning of robot manipulation policies. In: Proceedings of 2021 IEEE International Conference on Robotics and Automation (ICRA), 2021. 4776\u20134782"},{"key":"4403_CR22","unstructured":"Mnih V, Kavukcuoglu K, Silver D, et al. Playing Atari with deep reinforcement learning. 2013. ArXiv:13125602"},{"key":"4403_CR23","volume-title":"Proceedings of the 4th International Conference on Learning Representations","author":"T P Lillicrap","year":"2016","unstructured":"Lillicrap T P, Hunt J J, Pritzel A, et al. Continuous control with deep reinforcement learning. In: Proceedings of the 4th International Conference on Learning Representations, San Juan, 2016"},{"key":"4403_CR24","first-page":"1995","volume-title":"Proceedings of International Conference on Machine Learning","author":"Z Wang","year":"2016","unstructured":"Wang Z, Schaul T, Hessel M, et al. Dueling network architectures for deep reinforcement learning. In: Proceedings of International Conference on Machine Learning, 2016. 1995\u20132003"},{"key":"4403_CR25","unstructured":"Schulman J, Wolski F, Dhariwal P, et al. Proximal policy optimization algorithms. 2017. ArXiv:170706347"},{"key":"4403_CR26","first-page":"1889","volume-title":"Proceedings of International Conference on Machine Learning","author":"J Schulman","year":"2015","unstructured":"Schulman J, Levine S, Abbeel P, et al. Trust region policy optimization. In: Proceedings of International Conference on Machine Learning, 2015. 1889\u20131897"},{"key":"4403_CR27","first-page":"1928","volume-title":"Proceedings of the 33rd International Conference on Machine Learning","author":"V Mnih","year":"2016","unstructured":"Mnih V, Badia A P, Mirza M, et al. Asynchronous methods for deep reinforcement learning. In: Proceedings of the 33rd International Conference on Machine Learning, 2016. 1928\u20131937"},{"key":"4403_CR28","first-page":"1861","volume-title":"Proceedings of International Conference on Machine Learning","author":"T Haarnoja","year":"2018","unstructured":"Haarnoja T, Zhou A, Abbeel P, et al. Soft actor-critic: off-policy maximum entropy deep reinforcement learning with a stochastic actor. In: Proceedings of International Conference on Machine Learning, 2018. 1861\u20131870"},{"key":"4403_CR29","unstructured":"Kaiser L, Babaeizadeh M, Milos P, et al. Model-based reinforcement learning for Atari. 2019. ArXiv:190300374"},{"key":"4403_CR30","doi-asserted-by":"publisher","first-page":"160","DOI":"10.1145\/122344.122377","volume":"2","author":"R S Sutton","year":"1991","unstructured":"Sutton R S. Dyna, an integrated architecture for learning, planning, and reacting. ACM SIGART Bull, 1991, 2: 160\u2013163","journal-title":"ACM SIGART Bull"},{"key":"4403_CR31","volume-title":"Proceedings of Advances in Neural Information Processing Systems","author":"M Janner","year":"2019","unstructured":"Janner M, Fu J, Zhang M, et al. When to trust your model: model-based policy optimization. In: Proceedings of Advances in Neural Information Processing Systems, 2019"},{"key":"4403_CR32","doi-asserted-by":"publisher","first-page":"335","DOI":"10.1016\/0005-1098(89)90002-2","volume":"25","author":"C E Garc\u00eda","year":"1989","unstructured":"Garc\u00eda C E, Prett D M, Morari M. Model predictive control: theory and practice-a survey. Automatica, 1989, 25: 335\u2013348","journal-title":"Automatica"},{"key":"4403_CR33","doi-asserted-by":"publisher","first-page":"121101","DOI":"10.1007\/s11432-022-3696-5","volume":"67","author":"F M Luo","year":"2024","unstructured":"Luo F M, Xu T, Lai H, et al. A survey on model-based reinforcement learning. Sci China Inf Sci, 2024, 67: 121101","journal-title":"Sci China Inf Sci"},{"key":"4403_CR34","unstructured":"Zeng Y, Cai R, Sun F, et al. A survey on causal reinforcement learning. 2023. ArXiv:2302.05209"},{"key":"4403_CR35","volume-title":"Proceedings of Advances in Neural Information Processing Systems","author":"P de Haan","year":"2019","unstructured":"de Haan P, Jayaraman D, Levine S. Causal confusion in imitation learning. In: Proceedings of Advances in Neural Information Processing Systems, 2019"},{"key":"4403_CR36","first-page":"21","volume-title":"Proceedings of the 3rd Annual Conference on Learning for Dynamics and Control","author":"A Sonar","year":"2021","unstructured":"Sonar A, Pacelli V, Majumdar A. Invariant policy optimization: towards stronger generalization in reinforcement learning. In: Proceedings of the 3rd Annual Conference on Learning for Dynamics and Control, 2021. 21\u201333"},{"key":"4403_CR37","unstructured":"Lu C, Huang B, Wang K, et al. Sample-efficient reinforcement learning via counterfactual-based data augmentation. 2020. ArXiv:201209092"},{"key":"4403_CR38","first-page":"3976","volume":"33","author":"S Pitis","year":"2020","unstructured":"Pitis S, Creager E, Garg A. Counterfactual data augmentation using locally factored dynamics. In: Proceedings of Advances in Neural Information Processing Systems, 2020. 33: 3976\u20133990","journal-title":"Proceedings of Advances in Neural Information Processing Systems"},{"key":"4403_CR39","first-page":"23151","volume-title":"Proceedings of International Conference on Machine Learning","author":"Z Wang","year":"2022","unstructured":"Wang Z, Xiao X, Xu Z, et al. Causal dynamics learning for task-independent state abstraction. In: Proceedings of International Conference on Machine Learning, 2022. 23151\u201323180"},{"key":"4403_CR40","doi-asserted-by":"publisher","DOI":"10.7551\/mitpress\/1754.001.0001","volume-title":"Causation, Prediction, and Search","author":"P Spirtes","year":"2001","unstructured":"Spirtes P, Glymour C, Scheines R. Causation, Prediction, and Search. Cambridge: MIT Press, 2001"},{"key":"4403_CR41","first-page":"507","volume":"3","author":"D M Chickering","year":"2002","unstructured":"Chickering D M. Optimal structure identification with greedy search. J Mach Learn Res, 2002, 3: 507\u2013554","journal-title":"J Mach Learn Res"},{"key":"4403_CR42","doi-asserted-by":"publisher","first-page":"121","DOI":"10.1007\/s41060-016-0032-z","volume":"3","author":"J Ramsey","year":"2017","unstructured":"Ramsey J, Glymour M, Sanchez-Romero R, et al. A million variables and more: the Fast Greedy Equivalence Search algorithm for learning high-dimensional graphical causal models, with an application to functional magnetic resonance images. Int J Data Sci Anal, 2017, 3: 121\u2013129","journal-title":"Int J Data Sci Anal"},{"key":"4403_CR43","doi-asserted-by":"publisher","first-page":"1551","DOI":"10.1145\/3219819.3220104","volume-title":"Proceedings of the 24th ACM SIGKDD International Conference on Knowledge Discovery & Data Mining","author":"B Huang","year":"2018","unstructured":"Huang B, Zhang K, Lin Y, et al. Generalized score functions for causal discovery. In: Proceedings of the 24th ACM SIGKDD International Conference on Knowledge Discovery & Data Mining, 2018. 1551\u20131560"},{"key":"4403_CR44","first-page":"2003","volume":"7","author":"S Shimizu","year":"2006","unstructured":"Shimizu S, Hoyer P O, Hyv\u00e4rinen A, et al. A linear non-Gaussian acyclic model for causal discovery. J Mach Learn Res, 2006. 7: 2003\u20132030","journal-title":"J Mach Learn Res"},{"key":"4403_CR45","volume-title":"Proceedings of Advances in Neural Information Processing Systems","author":"P Hoyer","year":"2008","unstructured":"Hoyer P, Janzing D, Mooij J M, et al. Nonlinear causal discovery with additive noise models. In: Proceedings of Advances in Neural Information Processing Systems, 2008"},{"key":"4403_CR46","first-page":"2009","volume":"15","author":"J Peters","year":"2014","unstructured":"Peters J, Mooij J M, Janzing D, et al. Causal discovery with continuous additive noise models. J Mach Learn Res, 2014, 15: 2009\u20132053","journal-title":"J Mach Learn Res"},{"key":"4403_CR47","first-page":"1787","volume-title":"Proceedings of the AAAI Conference on Artificial Intelligence","author":"R Cai","year":"2018","unstructured":"Cai R, Qiao J, Zhang Z, et al. Self: structural equational likelihood framework for causal discovery. In: Proceedings of the AAAI Conference on Artificial Intelligence, 2018. 1787\u20131794"},{"key":"4403_CR48","first-page":"21865","volume":"33","author":"P Brouillard","year":"2020","unstructured":"Brouillard P, Lachapelle S, Lacoste A, et al. Differentiable causal discovery from interventional data. In: Proceedings of Advances in Neural Information Processing Systems, 2020. 33: 21865\u201321877","journal-title":"Proceedings of Advances in Neural Information Processing Systems"},{"key":"4403_CR49","first-page":"24130","volume":"35","author":"P Tigas","year":"2022","unstructured":"Tigas P, Annadani Y, Jesson A, et al. Interventions, where and how? Experimental design for causal models at scale. In: Proceedings of Advances in Neural Information Processing Systems, 2022. 35: 24130\u201324143","journal-title":"Proceedings of Advances in Neural Information Processing Systems"},{"key":"4403_CR50","unstructured":"Abel D. A theory of abstraction in reinforcement learning. 2022. ArXiv:2203.00397"},{"key":"4403_CR51","volume-title":"Elements of Causal Inference: Foundations and Learning Algorithms","author":"J Peters","year":"2017","unstructured":"Peters J, Janzing D, Sch\u00f6lkopf B. Elements of Causal Inference: Foundations and Learning Algorithms. Cambridge: MIT Press, 2017"},{"key":"4403_CR52","doi-asserted-by":"publisher","DOI":"10.1017\/CBO9780511803161","volume-title":"Causality","author":"J Pearl","year":"2009","unstructured":"Pearl J. Causality. Cambridge: Cambridge University Press, 2009"},{"key":"4403_CR53","doi-asserted-by":"publisher","first-page":"41","DOI":"10.1093\/biomet\/70.1.41","volume":"70","author":"P R Rosenbaum","year":"1983","unstructured":"Rosenbaum P R, Rubin D B. The central role of the propensity score in observational studies for causal effects. Biometrika, 1983, 70: 41\u201355","journal-title":"Biometrika"},{"key":"4403_CR54","doi-asserted-by":"publisher","first-page":"432","DOI":"10.1214\/17-STS612","volume":"32","author":"M J Lopez","year":"2017","unstructured":"Lopez M J, Gutman R. Estimation of causal effects with multiple treatments: a review and new ideas. Stat Sci, 2017, 32: 432\u2013454","journal-title":"Stat Sci"},{"key":"4403_CR55","series-title":"Technical Report","volume-title":"Difference-in-differences with a continuous treatment","author":"B Callaway","year":"2024","unstructured":"Callaway B, Goodman-Bacon A, Sant\u2019Anna P H. Difference-in-differences with a continuous treatment. Technical Report, National Bureau of Economic Research, 2024"},{"key":"4403_CR56","unstructured":"Samsami M R, Bahari M, Salehkaleybar S, et al. Causal imitative model for autonomous driving. 2021. ArXiv:211203908"},{"key":"4403_CR57","doi-asserted-by":"publisher","first-page":"597","DOI":"10.1111\/rssb.12268","volume":"80","author":"S Athey","year":"2018","unstructured":"Athey S, Imbens G W, Wager S. Approximate residual balancing: debiased inference of average treatment effects in high dimensions. J R Stat Soc Ser B-Stat Methodol, 2018, 80: 597\u2013623","journal-title":"J R Stat Soc Ser B-Stat Methodol"},{"key":"4403_CR58","first-page":"31","volume-title":"Proceedings of Advances in Neural Information Processing Systems","author":"X Zheng","year":"2018","unstructured":"Zheng X, Aragam B, Ravikumar P K, et al. Dags with no tears: continuous optimization for structure learning. In: Proceedings of Advances in Neural Information Processing Systems, 2018. 31"},{"key":"4403_CR59","doi-asserted-by":"publisher","first-page":"479","DOI":"10.1109\/TNNLS.2022.3175622","volume":"35","author":"R Cai","year":"2022","unstructured":"Cai R, Wu S, Qiao J, et al. THPs: topological Hawkes processes for learning causal structure on event sequences. IEEE Trans Neural Netw Learn Syst, 2022, 35: 479\u2013493","journal-title":"IEEE Trans Neural Netw Learn Syst"}],"container-title":["Science China Information Sciences"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s11432-024-4403-7.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s11432-024-4403-7\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s11432-024-4403-7.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,11,3]],"date-time":"2025-11-03T02:02:21Z","timestamp":1762135341000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s11432-024-4403-7"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,10,29]]},"references-count":59,"journal-issue":{"issue":"2","published-print":{"date-parts":[[2026,2]]}},"alternative-id":["4403"],"URL":"https:\/\/doi.org\/10.1007\/s11432-024-4403-7","relation":{},"ISSN":["1674-733X","1869-1919"],"issn-type":[{"type":"print","value":"1674-733X"},{"type":"electronic","value":"1869-1919"}],"subject":[],"published":{"date-parts":[[2025,10,29]]},"assertion":[{"value":"2 August 2024","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"7 February 2025","order":2,"name":"revised","label":"Revised","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"23 April 2025","order":3,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"29 October 2025","order":4,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}}],"article-number":"122104"}}