{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,6,6]],"date-time":"2026-06-06T16:44:40Z","timestamp":1780764280122,"version":"3.54.1"},"reference-count":64,"publisher":"Institute of Electrical and Electronics Engineers (IEEE)","issue":"10","license":[{"start":{"date-parts":[[2025,10,1]],"date-time":"2025-10-01T00:00:00Z","timestamp":1759276800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/ieeexplore.ieee.org\/Xplorehelp\/downloads\/license-information\/IEEE.html"},{"start":{"date-parts":[[2025,10,1]],"date-time":"2025-10-01T00:00:00Z","timestamp":1759276800000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2025,10,1]],"date-time":"2025-10-01T00:00:00Z","timestamp":1759276800000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"}],"funder":[{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["U21A20518"],"award-info":[{"award-number":["U21A20518"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["U24A20279"],"award-info":[{"award-number":["U24A20279"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["62306325"],"award-info":[{"award-number":["62306325"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100019081","name":"Science and Technology Innovation Program of Hunan Province","doi-asserted-by":"publisher","award":["2024RC3145"],"award-info":[{"award-number":["2024RC3145"]}],"id":[{"id":"10.13039\/501100019081","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["IEEE Trans. Neural Netw. Learning Syst."],"published-print":{"date-parts":[[2025,10]]},"DOI":"10.1109\/tnnls.2025.3586603","type":"journal-article","created":{"date-parts":[[2025,7,24]],"date-time":"2025-07-24T17:56:16Z","timestamp":1753379776000},"page":"17990-18004","source":"Crossref","is-referenced-by-count":4,"title":["Game-Theoretic Constrained Policy Optimization for Safe Reinforcement Learning"],"prefix":"10.1109","volume":"36","author":[{"ORCID":"https:\/\/orcid.org\/0000-0001-8829-1049","authenticated-orcid":false,"given":"Changxin","family":"Zhang","sequence":"first","affiliation":[{"name":"College of Intelligence Science and Technology, National University of Defense Technology, Changsha, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-0587-2487","authenticated-orcid":false,"given":"Xinglong","family":"Zhang","sequence":"additional","affiliation":[{"name":"College of Intelligence Science and Technology, National University of Defense Technology, Changsha, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-4503-643X","authenticated-orcid":false,"given":"Yixing","family":"Lan","sequence":"additional","affiliation":[{"name":"College of Intelligence Science and Technology, National University of Defense Technology, Changsha, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-6974-4997","authenticated-orcid":false,"given":"Hao","family":"Gao","sequence":"additional","affiliation":[{"name":"College of Intelligence Science and Technology, National University of Defense Technology, Changsha, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-3238-745X","authenticated-orcid":false,"given":"Xin","family":"Xu","sequence":"additional","affiliation":[{"name":"College of Intelligence Science and Technology, National University of Defense Technology, Changsha, China"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"263","reference":[{"key":"ref1","article-title":"StarCraft II: A new challenge for reinforcement learning","author":"Vinyals","year":"2017","journal-title":"arXiv:1708.04782"},{"key":"ref2","doi-asserted-by":"publisher","DOI":"10.1109\/TNNLS.2023.3280161"},{"key":"ref3","doi-asserted-by":"publisher","DOI":"10.1109\/JIOT.2020.3037641"},{"key":"ref4","doi-asserted-by":"publisher","DOI":"10.1177\/0278364913495721"},{"key":"ref5","doi-asserted-by":"publisher","DOI":"10.1109\/TITS.2021.3054625"},{"key":"ref6","article-title":"Concrete problems in AI safety","author":"Amodei","year":"2016","journal-title":"arXiv:1606.06565"},{"issue":"1","key":"ref7","first-page":"1437","article-title":"A comprehensive survey on safe reinforcement learning","volume":"16","author":"Garc\u00eda","year":"2015","journal-title":"J. Mach. Learn. Res."},{"issue":"12","key":"ref8","first-page":"2090","article-title":"Safe reinforcement learning and its applications in robotics: A survey","volume":"40","author":"Zhang","year":"2023","journal-title":"Control Theory Appl."},{"key":"ref9","doi-asserted-by":"publisher","DOI":"10.1109\/TNNLS.2023.3331304"},{"key":"ref10","first-page":"22","article-title":"Constrained policy optimization","volume-title":"Proc. 34th Int. Conf. Mach. Learn.","author":"Achiam"},{"issue":"167","key":"ref11","first-page":"1","article-title":"Risk-constrained reinforcement learning with percentile risk criteria","volume":"18","author":"Chow","year":"2017","journal-title":"J. Mach. Learn. Res."},{"key":"ref12","first-page":"25636","article-title":"Reachability constrained reinforcement learning","volume-title":"Proc. 39th Int. Conf. Mach. Learn.","volume":"162","author":"Yu"},{"key":"ref13","doi-asserted-by":"publisher","DOI":"10.1201\/9781315140223"},{"key":"ref14","doi-asserted-by":"crossref","first-page":"331","DOI":"10.1016\/S0927-0507(05)80172-0","article-title":"Markov decision processes","volume":"2","author":"Puterman","year":"1990","journal-title":"Handbooks in Operations Research and Management Science"},{"key":"ref15","doi-asserted-by":"publisher","DOI":"10.1145\/2318716.2318723"},{"key":"ref16","first-page":"387","article-title":"Policy gradients with variance related risk criteria","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Castro"},{"key":"ref17","doi-asserted-by":"publisher","DOI":"10.1016\/0377-2217(84)90224-8"},{"key":"ref18","article-title":"Accelerated primal-dual policy optimization for safe reinforcement learning","author":"Liang","year":"2018","journal-title":"arXiv:1802.06480"},{"key":"ref19","first-page":"7553","article-title":"Constrained reinforcement learning has zero duality gap","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","volume":"32","author":"Paternain"},{"key":"ref20","first-page":"1","article-title":"Reward constrained policy optimization","volume-title":"Proc. Int. Conf. Learn. Represent.","author":"Tessler"},{"key":"ref21","first-page":"267","article-title":"Approximately optimal approximate reinforcement learning","volume-title":"Proc. 19th Int. Conf. Mach. Learn.","author":"Kakade"},{"key":"ref22","first-page":"1889","article-title":"Trust region policy optimization","volume-title":"Proc. 32nd Int. Conf. Mach. Learn.","author":"Schulman"},{"key":"ref23","first-page":"1","article-title":"Projection-based constrained policy optimization","volume-title":"Proc. Int. Conf. Learn. Represent.","author":"Yang"},{"key":"ref24","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v34i04.5932"},{"key":"ref25","first-page":"15338","article-title":"First order constrained optimization in policy space","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","author":"Zhang"},{"key":"ref26","doi-asserted-by":"publisher","DOI":"10.24963\/ijcai.2022\/520"},{"key":"ref27","first-page":"9111","article-title":"Constrained update projection approach to safe policy optimization","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","author":"Long"},{"key":"ref28","first-page":"25303","article-title":"ReLOAD: Reinforcement learning with optimistic ascent-descent for last-iterate convergence in constrained MDPs","volume-title":"Proc. 40th Int. Conf. Mach. Learn.","author":"Moskovitz"},{"key":"ref29","first-page":"25","article-title":"Gradient shaping for multi-constraint safe reinforcement learning","volume-title":"Proc. 6th Annu. Learn. Dyn. Control Conf.","author":"Yao"},{"key":"ref30","doi-asserted-by":"publisher","DOI":"10.1109\/TCYB.2022.3163816"},{"key":"ref31","first-page":"11480","article-title":"CRPO: A new approach for safe reinforcement learning with convergence guarantee","volume-title":"Proc. Int. Conf. Mach. Learn. (ICML)","author":"Xu"},{"key":"ref32","doi-asserted-by":"publisher","DOI":"10.1109\/ICRA.2019.8794107"},{"key":"ref33","doi-asserted-by":"publisher","DOI":"10.1109\/TAC.2018.2876389"},{"key":"ref34","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v32i1.11797"},{"key":"ref35","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v37i12.26723"},{"key":"ref36","article-title":"Safe exploration in continuous action spaces","author":"Dalal","year":"2018","journal-title":"arXiv:1801.08757"},{"key":"ref37","doi-asserted-by":"publisher","DOI":"10.1109\/TNNLS.2020.2967871"},{"key":"ref38","doi-asserted-by":"publisher","DOI":"10.1109\/TIE.2023.3317853"},{"key":"ref39","first-page":"908","article-title":"Safe model-based reinforcement learning with stability guarantees","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","volume":"30","author":"Berkenkamp"},{"key":"ref40","first-page":"1","article-title":"A Lyapunov-based approach to safe reinforcement learning","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","author":"Chow"},{"key":"ref41","article-title":"Lyapunov-based safe policy optimization for continuous control","author":"Chow","year":"2019","journal-title":"arXiv:1901.10031"},{"key":"ref42","doi-asserted-by":"publisher","DOI":"10.1109\/TRO.2020.2992981"},{"key":"ref43","doi-asserted-by":"publisher","DOI":"10.1109\/TNNLS.2021.3106818"},{"key":"ref44","article-title":"Trial without error: Towards safe reinforcement learning via human intervention","author":"Saunders","year":"2017","journal-title":"arXiv:1707.05173"},{"key":"ref45","article-title":"DESTA: A framework for safe reinforcement learning with Markov games of intervention","author":"Mguni","year":"2021","journal-title":"arXiv:2110.14468"},{"key":"ref46","doi-asserted-by":"publisher","DOI":"10.1109\/TPAMI.2024.3443916"},{"key":"ref47","doi-asserted-by":"publisher","DOI":"10.1201\/9781315140223"},{"key":"ref48","article-title":"Trustworthy reinforcement learning against intrinsic vulnerabilities: Robustness, safety, and generalizability","author":"Xu","year":"2022","journal-title":"arXiv:2209.08025"},{"key":"ref49","doi-asserted-by":"publisher","DOI":"10.1073\/pnas.39.10.1953"},{"key":"ref50","doi-asserted-by":"publisher","DOI":"10.4171\/icm2022\/152"},{"key":"ref51","doi-asserted-by":"publisher","DOI":"10.1145\/779928.779935"},{"key":"ref52","first-page":"5571","article-title":"Mean field multi-agent reinforcement learning","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Yang"},{"key":"ref53","first-page":"1039","article-title":"Nash q-learning for general-sum stochastic games","volume":"4","author":"Hu","year":"2003","journal-title":"J. Mach. Learn. Res."},{"key":"ref54","doi-asserted-by":"publisher","DOI":"10.1109\/TRO.2025.3531818"},{"key":"ref55","doi-asserted-by":"publisher","DOI":"10.32917\/hmj\/1206139508"},{"key":"ref56","article-title":"Continuous control with deep reinforcement learning","author":"Lillicrap","year":"2015","journal-title":"arXiv:1509.02971"},{"key":"ref57","first-page":"1861","article-title":"Soft actor-critic: Off-policy maximum entropy deep reinforcement learning with a stochastic actor","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Haarnoja"},{"key":"ref58","article-title":"Proximal policy optimization algorithms","author":"Schulman","year":"2017","journal-title":"arXiv:1707.06347"},{"key":"ref59","doi-asserted-by":"publisher","DOI":"10.1109\/LRA.2022.3196132"},{"key":"ref60","volume-title":"Benchmarking Safe Exploration in Deep Reinforcement Learning","author":"Ray","year":"2023"},{"key":"ref61","doi-asserted-by":"publisher","DOI":"10.1109\/IROS.2012.6386109"},{"key":"ref62","first-page":"18964","article-title":"Safety gymnasium: A unified safe reinforcement learning benchmark","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","volume":"36","author":"Ji"},{"key":"ref63","article-title":"Isaac Gym: High performance GPU-based physics simulation for robot learning","author":"Makoviychuk","year":"2021","journal-title":"arXiv:2108.10470"},{"key":"ref64","doi-asserted-by":"publisher","DOI":"10.1162\/089976699300016070"}],"container-title":["IEEE Transactions on Neural Networks and Learning Systems"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx8\/5962385\/11195929\/11095811.pdf?arnumber=11095811","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,10,8]],"date-time":"2025-10-08T17:38:39Z","timestamp":1759945119000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/11095811\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,10]]},"references-count":64,"journal-issue":{"issue":"10"},"URL":"https:\/\/doi.org\/10.1109\/tnnls.2025.3586603","relation":{},"ISSN":["2162-237X","2162-2388"],"issn-type":[{"value":"2162-237X","type":"print"},{"value":"2162-2388","type":"electronic"}],"subject":[],"published":{"date-parts":[[2025,10]]}}}