{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,30]],"date-time":"2026-04-30T03:18:49Z","timestamp":1777519129724,"version":"3.51.4"},"reference-count":44,"publisher":"Springer Science and Business Media LLC","issue":"21","license":[{"start":{"date-parts":[[2024,9,9]],"date-time":"2024-09-09T00:00:00Z","timestamp":1725840000000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2024,9,9]],"date-time":"2024-09-09T00:00:00Z","timestamp":1725840000000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"funder":[{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["62076028"],"award-info":[{"award-number":["62076028"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]},{"name":"National Key Research and Development Program of China","award":["2018AAA0101000"],"award-info":[{"award-number":["2018AAA0101000"]}]}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Appl Intell"],"published-print":{"date-parts":[[2024,11]]},"DOI":"10.1007\/s10489-024-05508-9","type":"journal-article","created":{"date-parts":[[2024,9,9]],"date-time":"2024-09-09T05:02:10Z","timestamp":1725858130000},"page":"11144-11159","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":1,"title":["A multi-step on-policy deep reinforcement learning method assisted by off-policy policy evaluation"],"prefix":"10.1007","volume":"54","author":[{"given":"Huaqing","family":"Zhang","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-5734-3157","authenticated-orcid":false,"given":"Hongbin","family":"Ma","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Bemnet Wondimagegnehu","family":"Mersha","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Ying","family":"Jin","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2024,9,9]]},"reference":[{"issue":"12","key":"5508_CR1","doi-asserted-by":"publisher","first-page":"10222","DOI":"10.1109\/TPAMI.2021.3133717","volume":"44","author":"W Shi","year":"2021","unstructured":"Shi W, Huang G, Song S, Wu C (2021) Temporal-spatial causal interpretations for vision-based reinforcement learning. IEEE Trans Pattern Anal Mach Intell 44(12):10222\u201310235. https:\/\/doi.org\/10.1109\/TPAMI.2021.3133717","journal-title":"IEEE Trans Pattern Anal Mach Intell"},{"issue":"23","key":"5508_CR2","doi-asserted-by":"publisher","first-page":"28832","DOI":"10.1007\/s10489-023-05065-7","volume":"53","author":"Z Chai","year":"2023","unstructured":"Chai Z, Hou H, Li Y (2023) A dynamic queuing model based distributed task offloading algorithm using deep reinforcement learning in mobile edge computing. Appl Intell 53(23):28832\u201328847. https:\/\/doi.org\/10.1007\/s10489-023-05065-7","journal-title":"Appl Intell"},{"issue":"23","key":"5508_CR3","doi-asserted-by":"publisher","first-page":"28730","DOI":"10.1007\/s10489-023-05050-0","volume":"53","author":"C Ma","year":"2023","unstructured":"Ma C, Cao Y, Dong D (2023) Reinforcement learning based time-varying formation control for quadrotor unmanned aerial vehicles system with input saturation. Appl Intell 53(23):28730\u201328744. https:\/\/doi.org\/10.1007\/s10489-023-05050-0","journal-title":"Appl Intell"},{"issue":"23","key":"5508_CR4","doi-asserted-by":"publisher","first-page":"28207","DOI":"10.1007\/s10489-023-05007-3","volume":"53","author":"Y Gong","year":"2023","unstructured":"Gong Y, Xiong H, Li M, Wang H, Nian X (2023) Reinforcement learning for multi-agent formation navigation with scalability. Appl Intell 53(23):28207\u201328225. https:\/\/doi.org\/10.1007\/s10489-023-05007-3","journal-title":"Appl Intell"},{"issue":"22","key":"5508_CR5","doi-asserted-by":"publisher","first-page":"27128","DOI":"10.1007\/s10489-023-04911-y","volume":"53","author":"Z Sun","year":"2023","unstructured":"Sun Z, Jing C, Guo S, An L (2023) PAC-Bayesian offline Meta-reinforcement learning. Appl Intell 53(22):27128\u201327147. https:\/\/doi.org\/10.1007\/s10489-023-04911-y","journal-title":"Appl Intell"},{"issue":"7782","key":"5508_CR6","doi-asserted-by":"publisher","first-page":"350","DOI":"10.1038\/s41586-019-1724-z","volume":"575","author":"O Vinyals","year":"2019","unstructured":"Vinyals O, Babuschkin I, Czarnecki WM, Mathieu M, Dudzik A, Chung J, Choi DH, Powell R, Ewalds T, Georgiev P et al (2019) Grandmaster level in StarCraft II using multi-agent reinforcement learning. Nature 575(7782):350\u2013354. https:\/\/doi.org\/10.1038\/s41586-019-1724-z","journal-title":"Nature"},{"key":"5508_CR7","doi-asserted-by":"publisher","unstructured":"Li Q, Peng Z, Feng L, Zhang Q, Xue Z, Zhou B (2022) Metadrive: composing diverse driving scenarios for generalizable reinforcement learning. IEEE Trans Pattern Anal Mach Intell 45(3):3461\u20133475. https:\/\/doi.org\/10.1109\/TPAMI.2022.3190471","DOI":"10.1109\/TPAMI.2022.3190471"},{"issue":"23","key":"5508_CR8","doi-asserted-by":"publisher","first-page":"28555","DOI":"10.1007\/s10489-023-05022-4","volume":"53","author":"W Hu","year":"2023","unstructured":"Hu W, Wang H, He M, Wang N (2023) Uncertainty-aware hierarchical reinforcement learning for long-horizon tasks. Appl Intell 53(23):28555\u201328569. https:\/\/doi.org\/10.1007\/s10489-023-05022-4","journal-title":"Appl Intell"},{"issue":"22","key":"5508_CR9","doi-asserted-by":"publisher","first-page":"27249","DOI":"10.1007\/s10489-023-04691-5","volume":"53","author":"A Ballou","year":"2023","unstructured":"Ballou A, Alameda-Pineda X, Reinke C (2023) Variational meta reinforcement learning for social robotics. Appl Intell 53(22):27249\u201327268. https:\/\/doi.org\/10.1007\/s10489-023-04691-5","journal-title":"Appl Intell"},{"issue":"1","key":"5508_CR10","doi-asserted-by":"publisher","first-page":"5721","DOI":"10.1038\/s41467-021-25874-z","volume":"12","author":"A Gupta","year":"2021","unstructured":"Gupta A, Savarese S, Ganguli S, Fei-Fei L (2021) Embodied intelligence via learning and evolution. Nat Commun 12(1):5721. https:\/\/doi.org\/10.1038\/s41467-021-25874-z","journal-title":"Nat Commun"},{"issue":"9","key":"5508_CR11","doi-asserted-by":"publisher","first-page":"9326","DOI":"10.1109\/TCYB.2021.3053414","volume":"52","author":"Q Wei","year":"2021","unstructured":"Wei Q, Ma H, Chen C, Dong D (2021) Deep reinforcement learning with quantum-inspired experience replay. IEEE Trans Cybern 52(9):9326\u20139338. https:\/\/doi.org\/10.1109\/TCYB.2021.3053414","journal-title":"IEEE Trans Cybern"},{"issue":"6","key":"5508_CR12","doi-asserted-by":"publisher","first-page":"7258","DOI":"10.1109\/TPAMI.2022.3223872","volume":"45","author":"H Huang","year":"2023","unstructured":"Huang H, Ye D, Shen L, Liu W (2023) Curriculum-based asymmetric multi-task reinforcement learning. IEEE Trans Pattern Anal Mach Intell 45(6):7258\u20137269. https:\/\/doi.org\/10.1109\/TPAMI.2022.3223872","journal-title":"IEEE Trans Pattern Anal Mach Intell"},{"key":"5508_CR13","doi-asserted-by":"publisher","unstructured":"Wang Y, Tang H, Huang L, Pan L, Yang L, Yang H, Mu F, Yang M (2023) Author correction: self-play reinforcement learning guides protein engineering. Nat Mach Intell 5(8):947\u2013947. https:\/\/doi.org\/10.1038\/s42256-023-00713-6","DOI":"10.1038\/s42256-023-00713-6"},{"issue":"2","key":"5508_CR14","doi-asserted-by":"publisher","first-page":"145","DOI":"10.1038\/s42256-023-00610-y","volume":"5","author":"Z Cao","year":"2023","unstructured":"Cao Z, Jiang K, Zhou W, Xu S, Peng H, Yang D (2023) Continuous improvement of self-driving cars using dynamic confidence-aware reinforcement learning. Nat Mach Intell 5(2):145\u2013158. https:\/\/doi.org\/10.1038\/s42256-023-00610-y","journal-title":"Nat Mach Intell"},{"key":"5508_CR15","unstructured":"Hausknecht M, Stone P: Deep recurrent Q-learning for partially observable MDPs. In: 2015 AAAI fall symposium series, pp 29\u201337 (2015)"},{"key":"5508_CR16","unstructured":"Lin L (1992) Reinforcement learning for robots using neural networks. PhD thesis, Carnegie Mellon University"},{"issue":"5","key":"5508_CR17","doi-asserted-by":"publisher","first-page":"220","DOI":"10.1016\/j.tins.2010.01.006","volume":"33","author":"J O\u2019Neill","year":"2010","unstructured":"O\u2019Neill J, Pleydell-Bouverie B, Dupret D, Csicsvari J (2010) Play it again: reactivation of waking experience and memory. Trends Neurosci 33(5):220\u2013229. https:\/\/doi.org\/10.1016\/j.tins.2010.01.006","journal-title":"Trends Neurosci"},{"issue":"7540","key":"5508_CR18","doi-asserted-by":"publisher","first-page":"529","DOI":"10.1038\/nature14236","volume":"518","author":"V Mnih","year":"2015","unstructured":"Mnih V, Kavukcuoglu K, Silver D, Rusu AA, Veness J, Bellemare MG, Graves A, Riedmiller M, Fidjeland AK, Ostrovski G et al (2015) Human-level control through deep reinforcement learning. Nature 518(7540):529\u2013533. https:\/\/doi.org\/10.1038\/nature14236","journal-title":"Nature"},{"issue":"6","key":"5508_CR19","doi-asserted-by":"publisher","first-page":"1820","DOI":"10.1109\/TNNLS.2019.2927227","volume":"31","author":"X Wang","year":"2019","unstructured":"Wang X, Gu Y, Cheng Y, Liu A, Chen CP (2019) Approximate policy-based accelerated deep reinforcement learning. IEEE Trans Neural Netw Learn Syst 31(6):1820\u20131830. https:\/\/doi.org\/10.1109\/TNNLS.2019.2927227","journal-title":"IEEE Trans Neural Netw Learn Syst"},{"key":"5508_CR20","doi-asserted-by":"publisher","unstructured":"Schulman J, Levine S, Abbeel P, Jordan M, Moritz P (2015) Trust region policy optimization. In: International conference on machine learning, pp 1889\u20131897. https:\/\/doi.org\/10.48550\/arXiv.1502.05477. PMLR","DOI":"10.48550\/arXiv.1502.05477"},{"key":"5508_CR21","doi-asserted-by":"publisher","unstructured":"Mnih V, Badia A.P, Mirza M, Graves A, Lillicrap T, Harley T, Silver D, Kavukcuoglu K (2017) Asynchronous methods for deep reinforcement learning. In: International conference on machine learning, pp 1928\u20131937. https:\/\/doi.org\/10.48550\/arXiv.1602.01783. PMLR","DOI":"10.48550\/arXiv.1602.01783"},{"key":"5508_CR22","doi-asserted-by":"publisher","unstructured":"Schulman J, Wolski F, Dhariwal P, Radford A, Klimov O (2017) Proximal policy optimization algorithms. https:\/\/doi.org\/10.48550\/arXiv.1707.06347","DOI":"10.48550\/arXiv.1707.06347"},{"key":"5508_CR23","doi-asserted-by":"publisher","unstructured":"Espeholt L, Soyer H, Munos R, Simonyan K, Mnih V, Ward T, Doron Y, Firoiu V, Harley T, Dunning I et al (2018) Impala: scalable distributed deep-rl with importance weighted actor-learner architectures. In: International conference on machine learning, pp 1407\u20131416. https:\/\/doi.org\/10.48550\/arXiv.1802.01561. PMLR","DOI":"10.48550\/arXiv.1802.01561"},{"key":"5508_CR24","doi-asserted-by":"publisher","unstructured":"Assran M, Romoff J, Ballas N, Pineau J, Rabbat M (2019) Gossip-based actor-learner architectures for deep reinforcement learning. In: Advances in neural information processing systems, vol 32. https:\/\/doi.org\/10.48550\/arXiv.1906.04585","DOI":"10.48550\/arXiv.1906.04585"},{"key":"5508_CR25","doi-asserted-by":"publisher","unstructured":"Peng J, Williams RJ (1994) Incremental multi-step Q-learning. In: Machine learning proceedings, pp 226\u2013232. https:\/\/doi.org\/10.1007\/BF00114731. Elsevier","DOI":"10.1007\/BF00114731"},{"key":"5508_CR26","doi-asserted-by":"publisher","unstructured":"Heess N, Tb D, Sriram S, Lemmon J, Merel J, Wayne G, Tassa Y, Erez T, Wang Z, Eslami S et al (2017) Emergence of locomotion behaviours in rich environments. https:\/\/doi.org\/10.48550\/arXiv.1707.02286","DOI":"10.48550\/arXiv.1707.02286"},{"key":"5508_CR27","doi-asserted-by":"publisher","unstructured":"Lillicrap TP, Hunt JJ, Pritzel A, Heess N, Erez T, Tassa Y, Silver D, Wierstra D (2015) Continuous control with deep reinforcement learning. https:\/\/doi.org\/10.48550\/arXiv.1509.02971","DOI":"10.48550\/arXiv.1509.02971"},{"key":"5508_CR28","doi-asserted-by":"publisher","unstructured":"Fujimoto S, Hoof H, Meger D (2018) Addressing function approximation error in actor-critic methods. In: International conference on machine learning, pp 1587\u20131596. https:\/\/doi.org\/10.48550\/arXiv.1802.09477. PMLR","DOI":"10.48550\/arXiv.1802.09477"},{"key":"5508_CR29","doi-asserted-by":"publisher","unstructured":"Van Hasselt H, Guez A, Silver D (2016) Deep reinforcement learning with double Q-learning. In: Proceedings of the AAAI conference on artificial intelligence, vol 30. https:\/\/doi.org\/10.48550\/arXiv.1509.06461","DOI":"10.48550\/arXiv.1509.06461"},{"key":"5508_CR30","unstructured":"Haarnoja T, Zhou A, Abbeel P, Levine S (2018) Soft actor-critic: off-policy maximum entropy deep reinforcement learning with a stochastic actor. In: International conference on machine learning, pp 1861\u20131870. PMLR"},{"issue":"6","key":"5508_CR31","doi-asserted-by":"publisher","first-page":"1820","DOI":"10.1109\/TNNLS.2019.2927227","volume":"31","author":"X Wang","year":"2020","unstructured":"Wang X, Gu Y, Cheng Y, Liu A, Chen CP (2020) Approximate policy-based accelerated deep reinforcement learning. IEEE Trans Neural Netw Learn Syst 31(6):1820\u20131830. https:\/\/doi.org\/10.1109\/TNNLS.2019.2927227","journal-title":"IEEE Trans Neural Netw Learn Syst"},{"key":"5508_CR32","doi-asserted-by":"publisher","unstructured":"Kuznetsov A, Shvechikov P, Grishin A, Vetrov D (2020) Controlling overestimation bias with truncated mixture of continuous distributional quantile critics. In: International conference on machine learning, pp 5556\u20135566. https:\/\/doi.org\/10.48550\/arXiv.2005.04269. PMLR","DOI":"10.48550\/arXiv.2005.04269"},{"key":"5508_CR33","doi-asserted-by":"publisher","first-page":"106736","DOI":"10.1016\/j.knosys.2020.106736","volume":"214","author":"S Han","year":"2021","unstructured":"Han S, Zhou W, L\u00fc S, Yu J (2021) Regularly updated deterministic policy gradient algorithm. Knowl-Based Syst 214:106736. https:\/\/doi.org\/10.1016\/j.knosys.2020.106736","journal-title":"Knowl-Based Syst"},{"key":"5508_CR34","doi-asserted-by":"publisher","first-page":"63","DOI":"10.1016\/j.neunet.2020.12.023","volume":"136","author":"T Kobayashi","year":"2021","unstructured":"Kobayashi T, Ilboudo WEL (2021) T-soft update of target network for deep reinforcement learning. Neural Netw 136:63\u201371. https:\/\/doi.org\/10.1016\/j.neunet.2020.12.023","journal-title":"Neural Netw"},{"key":"5508_CR35","doi-asserted-by":"publisher","first-page":"94","DOI":"10.1016\/j.neucom.2021.12.039","volume":"474","author":"H Wu","year":"2022","unstructured":"Wu H, Zhang J, Wang Z, Lin Y, Li H (2022) Sub-AVG: Overestimation reduction for cooperative multi-agent reinforcement learning. Neurocomputing 474:94\u2013106. https:\/\/doi.org\/10.1016\/j.neucom.2021.12.039","journal-title":"Neurocomputing"},{"key":"5508_CR36","doi-asserted-by":"publisher","first-page":"119494","DOI":"10.1016\/j.ins.2023.119494","volume":"647","author":"L Li","year":"2023","unstructured":"Li L, Li Y, Wei W, Zhang Y, Liang J (2023) Multi-actor mechanism for actor-critic reinforcement learning. Inf Sci 647:119494. https:\/\/doi.org\/10.1016\/j.ins.2023.119494","journal-title":"Inf Sci"},{"key":"5508_CR37","doi-asserted-by":"publisher","unstructured":"Gu S, Lillicrap T, Ghahramani Z, Turner RE, Levine S (2016) Q-prop: sample-efficient policy gradient with an off-policy critic. https:\/\/doi.org\/10.48550\/arXiv.1611.02247","DOI":"10.48550\/arXiv.1611.02247"},{"key":"5508_CR38","doi-asserted-by":"publisher","unstructured":"Gu SS, Lillicrap T, Turner RE, Ghahramani Z, Sch\u00f6lkopf B, Levine S (2017) Interpolated policy gradient: merging on-policy and off-policy gradient estimation for deep reinforcement learning. In: Advances in neural information processing systems, vol 30. https:\/\/doi.org\/10.48550\/arXiv.1706.00387","DOI":"10.48550\/arXiv.1706.00387"},{"key":"5508_CR39","doi-asserted-by":"publisher","unstructured":"Fakoor R, Chaudhari P, Smola AJ (2020) P3O: Policy-on policy-off policy optimization. In: Uncertainty in artificial intelligence, pp 1017\u20131027. https:\/\/doi.org\/10.48550\/arXiv.1905.01756. PMLR","DOI":"10.48550\/arXiv.1905.01756"},{"key":"5508_CR40","doi-asserted-by":"publisher","unstructured":"Wu YF, ZHANG W, Xu P, Gu Q (2020) A finite-time analysis of two time-scale actor-critic methods. In: Advances in neural information processing systems, vol 33, pp 17617\u201317628. https:\/\/doi.org\/10.48550\/arXiv.2005.01350","DOI":"10.48550\/arXiv.2005.01350"},{"issue":"2","key":"5508_CR41","doi-asserted-by":"publisher","first-page":"652","DOI":"10.1109\/JSAIT.2021.3078754","volume":"2","author":"S Qiu","year":"2021","unstructured":"Qiu S, Yang Z, Ye J, Wang Z (2021) On finite-time convergence of actor-critic algorithm. IEEE J Sel Areas Inf Theory 2(2):652\u2013664. https:\/\/doi.org\/10.1109\/JSAIT.2021.3078754","journal-title":"IEEE J Sel Areas Inf Theory"},{"key":"5508_CR42","doi-asserted-by":"publisher","unstructured":"Bellemare MG, Naddaf Y, Veness J, Bowling M (2013) The arcade learning environment: an evaluation platform for general agents. J Artif Intell Res 47:253\u2013279. https:\/\/doi.org\/10.1613\/jair.3912","DOI":"10.1613\/jair.3912"},{"key":"5508_CR43","doi-asserted-by":"publisher","unstructured":"Machado MC, Bellemare MG, Talvitie E, Veness J, Hausknecht M, Bowling M (2018) Revisiting the arcade learning environment: evaluation protocols and open problems for general agents. J Artif Intell Res 61:523\u2013562. https:\/\/doi.org\/10.1613\/jair.5699","DOI":"10.1613\/jair.5699"},{"key":"5508_CR44","unstructured":"Hu M (2022) Deep RL Zoo: A collections of deep RL algorithms implemented with PyTorch. https:\/\/github.com\/michaelnny\/deep_rl_zoo"}],"container-title":["Applied Intelligence"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10489-024-05508-9.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s10489-024-05508-9\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10489-024-05508-9.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,9,18]],"date-time":"2024-09-18T14:30:46Z","timestamp":1726669846000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s10489-024-05508-9"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,9,9]]},"references-count":44,"journal-issue":{"issue":"21","published-print":{"date-parts":[[2024,11]]}},"alternative-id":["5508"],"URL":"https:\/\/doi.org\/10.1007\/s10489-024-05508-9","relation":{},"ISSN":["0924-669X","1573-7497"],"issn-type":[{"value":"0924-669X","type":"print"},{"value":"1573-7497","type":"electronic"}],"subject":[],"published":{"date-parts":[[2024,9,9]]},"assertion":[{"value":"5 May 2024","order":1,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"9 September 2024","order":2,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Declarations"}},{"value":"The authors declare that they have no personal and financial relationships with other organizations or people that can inappropriately influence the work, reported in this paper.","order":2,"name":"Ethics","group":{"name":"EthicsHeading","label":"Competing of interest"}}]}}