{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,2,10]],"date-time":"2026-02-10T03:49:53Z","timestamp":1770695393612,"version":"3.49.0"},"reference-count":59,"publisher":"Springer Science and Business Media LLC","issue":"1","license":[{"start":{"date-parts":[[2026,1,1]],"date-time":"2026-01-01T00:00:00Z","timestamp":1767225600000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2026,1,1]],"date-time":"2026-01-01T00:00:00Z","timestamp":1767225600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"funder":[{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"crossref","award":["62076028"],"award-info":[{"award-number":["62076028"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"crossref"}]},{"name":"National Key Research and Development Plan of China","award":["2018AAA0101000"],"award-info":[{"award-number":["2018AAA0101000"]}]}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Int. J. Mach. Learn. & Cyber."],"published-print":{"date-parts":[[2026,1]]},"DOI":"10.1007\/s13042-025-02963-9","type":"journal-article","created":{"date-parts":[[2026,1,17]],"date-time":"2026-01-17T07:21:37Z","timestamp":1768634497000},"update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":0,"title":["An offline actor-critic policy improvement algorithm with historical state-action pairs"],"prefix":"10.1007","volume":"17","author":[{"given":"Huaqing","family":"Zhang","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Xiaofei","family":"Zhang","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Jixiang","family":"Jiang","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Mingrui","family":"Hao","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Hongbin","family":"Ma","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Ning","family":"Zhang","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2026,1,17]]},"reference":[{"key":"2963_CR1","doi-asserted-by":"publisher","first-page":"746","DOI":"10.1016\/j.ins.2022.08.019","volume":"610","author":"X Lou","year":"2022","unstructured":"Lou X, Yin Q, Zhang J, Yu C, He Z, Cheng N, Huang K (2022) Offline reinforcement learning with representations for actions. Inf Sci 610:746\u2013758","journal-title":"Inf Sci"},{"key":"2963_CR2","doi-asserted-by":"publisher","first-page":"221","DOI":"10.1016\/j.ins.2023.03.019","volume":"632","author":"J Deng","year":"2023","unstructured":"Deng J, Sierla S, Sun J, Vyatkin V (2023) Offline reinforcement learning for industrial process control: a case study from steel industry. Inf Sci 632:221\u2013231","journal-title":"Inf Sci"},{"issue":"6","key":"2963_CR3","doi-asserted-by":"publisher","first-page":"26","DOI":"10.1109\/MSP.2017.2743240","volume":"34","author":"K Arulkumaran","year":"2017","unstructured":"Arulkumaran K, Deisenroth MP, Brundage M, Bharath AA (2017) Deep reinforcement learning: a brief survey. IEEE Signal Process Mag 34(6):26\u201338","journal-title":"IEEE Signal Process Mag"},{"key":"2963_CR4","unstructured":"Fujimoto S, Hoof H, Meger D (2018) Addressing function approximation error in actor-critic methods. In: International conference on machine learning, pp. 1587\u20131596"},{"key":"2963_CR5","unstructured":"Haarnoja T, Zhou A, Abbeel P, Levine S (2018) Soft actor-critic: off-policy maximum entropy deep reinforcement learning with a stochastic actor. In: International conference on machine learning, pp 1861\u20131870"},{"key":"2963_CR6","doi-asserted-by":"publisher","DOI":"10.1016\/j.ins.2023.119401","volume":"646","author":"X Zhou","year":"2023","unstructured":"Zhou X, Wu L, Zhang Y, Chen Z-S, Jiang S (2023) A robust deep reinforcement learning approach to driverless taxi dispatching under uncertain demand. Inf Sci 646:119401","journal-title":"Inf Sci"},{"key":"2963_CR7","doi-asserted-by":"publisher","first-page":"110","DOI":"10.1016\/j.ins.2020.03.105","volume":"532","author":"J Li","year":"2020","unstructured":"Li J, Yao L, Xu X, Cheng B, Ren J (2020) Deep reinforcement learning for pedestrian collision avoidance and human-machine cooperative driving. Inf Sci 532:110\u2013124","journal-title":"Inf Sci"},{"key":"2963_CR8","doi-asserted-by":"publisher","first-page":"309","DOI":"10.1016\/j.ins.2012.12.021","volume":"232","author":"B Fernandez-Gauna","year":"2013","unstructured":"Fernandez-Gauna B, Marques I, Gra\u00f1a M (2013) Undesired state-action prediction in multi-agent reinforcement learning for linked multi-component robotic system control. Inf Sci 232:309\u2013324","journal-title":"Inf Sci"},{"key":"2963_CR9","doi-asserted-by":"publisher","DOI":"10.1016\/j.ins.2023.119154","volume":"642","author":"X Liu","year":"2023","unstructured":"Liu X, Chai Z-Y, Li Y-L, Cheng Y-Y, Zeng Y (2023) Multi-objective deep reinforcement learning for computation offloading in UAV-assisted multi-access edge computing. Inf Sci 642:119154","journal-title":"Inf Sci"},{"key":"2963_CR10","doi-asserted-by":"publisher","first-page":"480","DOI":"10.1016\/j.ins.2022.10.071","volume":"614","author":"L Lin","year":"2022","unstructured":"Lin L, Pan L, Liu S (2022) Learning to make auto-scaling decisions with heterogeneous spot and on-demand instances via reinforcement learning. Inf Sci 614:480\u2013496","journal-title":"Inf Sci"},{"key":"2963_CR11","unstructured":"Sinha S, Mandlekar A, Garg A (2022) S4RL: surprisingly simple self-supervision for offline reinforcement learning in robotics. In: Conference on robot learning, pp 907\u2013917"},{"key":"2963_CR12","doi-asserted-by":"crossref","unstructured":"Fang X, Zhang Q, Gao Y, Zhao D (2022) Offline reinforcement learning for autonomous driving with real world driving data. In: IEEE 25th international conference on intelligent transportation systems, pp 3417\u20133422","DOI":"10.1109\/ITSC55140.2022.9922100"},{"key":"2963_CR13","unstructured":"Singh A, Yu A, Yang J, Zhang J, Kumar A, Levine S (2020) COG: connecting new skills to past experience with offline reinforcement learning. arXiv preprint. arXiv:2010.14500"},{"key":"2963_CR14","unstructured":"Arnob SY, Islam R, Precup D (2021) Importance of empirical sample complexity analysis for offline reinforcement learning. arXiv preprint. arXiv:arXiv"},{"key":"2963_CR15","unstructured":"Schweighofer K, Hofmarcher M, Dinu MC, Renz P, Bitto-Nemling A, Patil V, Hochreiter S (2021) Understanding the effects of dataset characteristics on offline reinforcement learning. arXiv preprint"},{"key":"2963_CR16","unstructured":"Fujimoto S, Meger D, Precup D (2019) Off-policy deep reinforcement learning without exploration. In: International conference on machine learning, pp 2052\u20132062"},{"key":"2963_CR17","first-page":"1","volume":"32","author":"A Kumar","year":"2019","unstructured":"Kumar A, Fu J, Soh M, Tucker G, Levine S (2019) Stabilizing off-policy Q-learning via bootstrapping error reduction. Adv Neural Inf Process Syst 32:1\u201312","journal-title":"Adv Neural Inf Process Syst"},{"issue":"3","key":"2963_CR18","doi-asserted-by":"publisher","first-page":"796","DOI":"10.3390\/s21030796","volume":"21","author":"X Yu","year":"2021","unstructured":"Yu X, Wang P, Zhang Z (2021) Learning-based end-to-end path planning for lunar rovers with safety constraints. Sensors 21(3):796","journal-title":"Sensors"},{"issue":"4","key":"2963_CR19","doi-asserted-by":"publisher","first-page":"253","DOI":"10.3390\/aerospace11040253","volume":"11","author":"S Lu","year":"2024","unstructured":"Lu S, Xu R, Li Z, Wang B, Zhao Z (2024) Lunar rover collaborated path planning with artificial potential field-based heuristic on deep reinforcement learning. Aerospace 11(4):253","journal-title":"Aerospace"},{"issue":"8","key":"2963_CR20","doi-asserted-by":"publisher","first-page":"10237","DOI":"10.1109\/TNNLS.2023.3250269","volume":"35","author":"RF Prudencio","year":"2024","unstructured":"Prudencio RF, Maximo MR, Colombini EL (2024) A survey on offline reinforcement learning: taxonomy, review, and open problems. IEEE Trans Neural Netw Learn Syst 35(8):10237\u201310257","journal-title":"IEEE Trans Neural Netw Learn Syst"},{"issue":"10","key":"2963_CR21","doi-asserted-by":"publisher","first-page":"7391","DOI":"10.1109\/TNNLS.2022.3142822","volume":"34","author":"Z Huang","year":"2023","unstructured":"Huang Z, Wu J, Lv C (2023) Efficient deep reinforcement learning with imitative expert priors for autonomous driving. IEEE Trans Neural Netw Learn Syst 34(10):7391\u20137403","journal-title":"IEEE Trans Neural Netw Learn Syst"},{"key":"2963_CR22","unstructured":"Lillicrap TP, Hunt JJ, Pritzel A, Heess N, Erez T, Tassa Y, Silver D, Wierstra D (2015) Continuous control with deep reinforcement learning. arXiv preprint"},{"key":"2963_CR23","unstructured":"Laroche R, Trichelair P, Des\u00a0Combes RT (2019) Safe policy improvement with baseline bootstrapping. In: International conference on machine learning, pp 3652\u20133661"},{"issue":"1","key":"2963_CR24","first-page":"723","volume":"13","author":"A Gretton","year":"2012","unstructured":"Gretton A, Borgwardt KM, Rasch MJ, Sch\u00f6lkopf B, Smola A (2012) A kernel two-sample test. J Mach Learn Res 13(1):723\u2013773","journal-title":"J Mach Learn Res"},{"key":"2963_CR25","unstructured":"Jaques N, Ghandeharioun A, Shen JH, Ferguson C, Lapedriza A, Jones N, Gu S, Picard R (2019) Way off-policy batch deep reinforcement learning of implicit human preferences in dialog. arXiv preprint. arXiv:1907.00456"},{"key":"2963_CR26","unstructured":"Wu Y, Tucker G, Nachum O (2019) Behavior regularized offline reinforcement learning. arXiv preprint. arXiv:1911.11361"},{"key":"2963_CR27","first-page":"20132","volume":"34","author":"S Fujimoto","year":"2021","unstructured":"Fujimoto S, Gu SS (2021) A minimalist approach to offline reinforcement learning. Adv Neural Inf Process Syst 34:20132\u201320145","journal-title":"Adv Neural Inf Process Syst"},{"key":"2963_CR28","unstructured":"Paine TL, Paduraru C, Michi A, Gulcehre C, Zolna K, Novikov A, Wang Z, Freitas N (2020) Hyperparameter selection for ofline reinforcement learning. arXiv preprint. arXiv:2007.09055"},{"key":"2963_CR29","first-page":"12864","volume":"34","author":"S Zhang","year":"2021","unstructured":"Zhang S, Jiang N (2021) Towards hyperparameter-free policy selection for offline reinforcement learning. Adv Neural Inf Process Syst 34:12864\u201312875","journal-title":"Adv Neural Inf Process Syst"},{"key":"2963_CR30","first-page":"1264","volume":"33","author":"Y Liu","year":"2020","unstructured":"Liu Y, Swaminathan A, Agarwal A, Brunskill E (2020) Provably good batch off-policy reinforcement learning without great exploration. Adv Neural Inf Process Syst 33:1264\u20131274","journal-title":"Adv Neural Inf Process Syst"},{"key":"2963_CR31","unstructured":"Uehara M, Zhang X, Sun W (2022) Representation learning for online and offline RL in low-rank MDPs. arXiv preprint. arXiv:2110.04652"},{"key":"2963_CR32","first-page":"1179","volume":"33","author":"A Kumar","year":"2020","unstructured":"Kumar A, Zhou A, Tucker G, Levine S (2020) Conservative Q-learning for offline reinforcement learning. Adv Neural Inf Process Syst 33:1179\u20131191","journal-title":"Adv Neural Inf Process Syst"},{"key":"2963_CR33","unstructured":"Jin Y, Yang Z, Wang Z (2021) Is pessimism provably efficient for offline RL? In: International conference on machine learning, pp 5084\u20135096"},{"key":"2963_CR34","first-page":"6683","volume":"34","author":"T Xie","year":"2021","unstructured":"Xie T, Cheng C, Jiang N, Mineiro P, Agarwal A (2021) Bellman-consistent pessimism for offline reinforcement learning. Adv Neural Inf Process Syst 34:6683\u20136694","journal-title":"Adv Neural Inf Process Syst"},{"key":"2963_CR35","unstructured":"Gulcehre C, Colmenarejo SG, Sygnowski J, Paine T, Zolna K, Chen Y, Hoffman M, Pascanu R, Freitas N, et al (2020) Addressing extrapolation error in deep offline reinforcement learning"},{"key":"2963_CR36","unstructured":"Kostrikov I, Nair A, Levine S (2021) Offline reinforcement learning with implicit Q-Learning. arXiv preprint. arXiv:2110.06169"},{"key":"2963_CR37","first-page":"4933","volume":"34","author":"D Brandfonbrener","year":"2021","unstructured":"Brandfonbrener D, Whitney W, Ranganath R, Bruna J (2021) Offline RL without off-policy evaluation. Adv Neural Inf Process Syst 34:4933\u20134946","journal-title":"Adv Neural Inf Process Syst"},{"key":"2963_CR38","unstructured":"Nair A, Gupta A, Dalal M, Levine S (2021) AWAC: Accelerating online reinforcement learning with offline datasets. arXiv preprint arXiv:2006.09359"},{"key":"2963_CR39","unstructured":"Yang M, Nachum O (2021) Representation matters: offline pretraining for sequential decision making. In: International Conference on Machine Learning, pp. 11784\u201311794"},{"key":"2963_CR40","unstructured":"Cheng C, Xie T, Jiang N, Agarwal A (2022) Adversarially trained actor critic for offline reinforcement learning. arXiv preprint. arXiv:2202.02446"},{"key":"2963_CR41","doi-asserted-by":"crossref","unstructured":"Mao L, Xu H, Zhan X, Zhang W, Zhang A (2024) Diffusion-DICE: in-sample diffusion guidance for offline reinforcement learning. arXiv preprint. arXiv:2407.20109","DOI":"10.52202\/079017-3136"},{"key":"2963_CR42","unstructured":"Kim Y, Park J, Kim H, Kim S, Lee BJ, Kim S (2024) Diffusion-based offline RL for improved decision-making in augmented ARC task. arXiv preprint. arXiv:2410.11324"},{"key":"2963_CR43","doi-asserted-by":"crossref","unstructured":"Zeng X, Peng H, Li A (2023) Effective and stable role-based multi-agent collaboration by structural information principles. In: Proceedings of the AAAI conference on artificial intelligence, pp 11772\u201311780","DOI":"10.1609\/aaai.v37i10.26390"},{"key":"2963_CR44","doi-asserted-by":"crossref","unstructured":"Zeng X, Peng H, Li A, Liu C, He L, Yu PS (2023) Hierarchical state abstraction based on structural information principles. In: International joint conference on artificial intelligence, pp 4549\u20134557","DOI":"10.24963\/ijcai.2023\/506"},{"key":"2963_CR45","first-page":"1","volume":"21","author":"X Zeng","year":"2024","unstructured":"Zeng X, Peng H, Li A (2024) Effective exploration based on the structural information principles. Adv Neural Inf Process Syst 21:1\u201314","journal-title":"Adv Neural Inf Process Syst"},{"issue":"3","key":"2963_CR46","first-page":"279","volume":"8","author":"CJ Watkins","year":"1992","unstructured":"Watkins CJ, Dayan P (1992) Q-learning. Mach Learn 8(3):279\u2013292","journal-title":"Mach Learn"},{"key":"2963_CR47","unstructured":"Sutton RS, McAllester D, Singh S, Mansour Y (1999) Policy gradient methods for reinforcement learning with function approximation. Advances in neural information processing systems, 12"},{"key":"2963_CR48","doi-asserted-by":"publisher","first-page":"229","DOI":"10.1023\/A:1022672621406","volume":"8","author":"RJ Williams","year":"1992","unstructured":"Williams RJ (1992) Simple statistical gradient-following algorithms for connectionist reinforcement learning. Mach Learn 8:229\u2013256","journal-title":"Mach Learn"},{"key":"2963_CR49","unstructured":"Schulman J, Wolski F, Dhariwal P, Radford A, Klimov O (2017) Proximal policy optimization algorithms. arXiv preprint. arXiv:1707.06347"},{"key":"2963_CR50","first-page":"169","volume":"41","author":"RM Sakia","year":"1992","unstructured":"Sakia RM (1992) The Box-Cox transformation technique: a review. J R Stat Soc Ser D 41:169\u2013178","journal-title":"J R Stat Soc Ser D"},{"issue":"1","key":"2963_CR51","first-page":"12","volume":"15","author":"J Osborne","year":"2010","unstructured":"Osborne J (2010) Improving your data transformations: applying the box-cox transformation. Pract Assess Res Eval 15(1):12","journal-title":"Pract Assess Res Eval"},{"key":"2963_CR52","unstructured":"Brockman G, Cheung V, Pettersson L, Schneider J, Schulman J, Tang J, Zaremba W (2016) OpenAI Gym. arXiv preprint. arXiv:1606.01540"},{"key":"2963_CR53","unstructured":"Coumans E, Bai Y (2016) Pybullet, a python module for physics simulation for games, robotics and machine learning"},{"key":"2963_CR54","unstructured":"Fu J, Kumar A, Nachum O, Tucker G, Levine S (2021) D4RL: datasets for deep data-driven reinforcement learning. arXiv preprint. arXiv:2004.07219"},{"key":"2963_CR55","first-page":"21810","volume":"33","author":"R Kidambi","year":"2020","unstructured":"Kidambi R, Rajeswaran A, Netrapalli P, Joachims T (2020) Morel: model-based offline reinforcement learning. Adv Neural Inf Process Syst 33:21810\u201321823","journal-title":"Adv Neural Inf Process Syst"},{"key":"2963_CR56","first-page":"14129","volume":"33","author":"T Yu","year":"2020","unstructured":"Yu T, Thomas G, Yu L, Ermon S, Zou JY, Levine S, Finn C, Ma T (2020) MOPO: model-based offline policy optimization. Adv Neural Inf Process Syst 33:14129\u201314142","journal-title":"Adv Neural Inf Process Syst"},{"key":"2963_CR57","doi-asserted-by":"publisher","DOI":"10.1016\/j.engappai.2021.104366","volume":"104","author":"P Swazinna","year":"2021","unstructured":"Swazinna P, Udluft S, Runkler T (2021) Overcoming model bias for robust offline deep reinforcement learning. Eng Appl Artif Intell 104:104366","journal-title":"Eng Appl Artif Intell"},{"key":"2963_CR58","first-page":"28954","volume":"34","author":"T Yu","year":"2021","unstructured":"Yu T, Kumar A, Rafailov R, Rajeswaran A, Levine S, Finn C (2021) COMBO: conservative offline model-based policy optimization. Adv Neural Inf Process Syst 34:28954\u201328967","journal-title":"Adv Neural Inf Process Syst"},{"issue":"15","key":"2963_CR59","doi-asserted-by":"publisher","first-page":"19","DOI":"10.1016\/j.ifacol.2022.07.602","volume":"55","author":"P Swazinna","year":"2022","unstructured":"Swazinna P, Udluft S, Hein D, Runkler T (2022) Comparing model-free and model-based algorithms for offline reinforcement learning. IFAC-PapersOnLine 55(15):19\u201326","journal-title":"IFAC-PapersOnLine"}],"container-title":["International Journal of Machine Learning and Cybernetics"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s13042-025-02963-9.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s13042-025-02963-9","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s13042-025-02963-9.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,2,9]],"date-time":"2026-02-09T09:39:54Z","timestamp":1770629994000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s13042-025-02963-9"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026,1]]},"references-count":59,"journal-issue":{"issue":"1","published-print":{"date-parts":[[2026,1]]}},"alternative-id":["2963"],"URL":"https:\/\/doi.org\/10.1007\/s13042-025-02963-9","relation":{},"ISSN":["1868-8071","1868-808X"],"issn-type":[{"value":"1868-8071","type":"print"},{"value":"1868-808X","type":"electronic"}],"subject":[],"published":{"date-parts":[[2026,1]]},"assertion":[{"value":"2 December 2024","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"19 September 2025","order":2,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"17 January 2026","order":3,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Declarations"}},{"value":"The authors declare no Conflict of interest.","order":2,"name":"Ethics","group":{"name":"EthicsHeading","label":"Conflict of interest"}},{"value":"This article does not contain any studies with human participants or animals performed by any of the authors.","order":3,"name":"Ethics","group":{"name":"EthicsHeading","label":"Research involving human participants and\/or animals"}}],"article-number":"8"}}