{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,3,20]],"date-time":"2025-03-20T04:17:05Z","timestamp":1742444225014,"version":"3.40.1"},"reference-count":29,"publisher":"Springer Science and Business Media LLC","issue":"1","license":[{"start":{"date-parts":[[2025,2,1]],"date-time":"2025-02-01T00:00:00Z","timestamp":1738368000000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2025,2,1]],"date-time":"2025-02-01T00:00:00Z","timestamp":1738368000000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["J Syst Sci Complex"],"published-print":{"date-parts":[[2025,2]]},"DOI":"10.1007\/s11424-025-4572-y","type":"journal-article","created":{"date-parts":[[2025,3,19]],"date-time":"2025-03-19T16:05:05Z","timestamp":1742400305000},"page":"421-435","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":0,"title":["On-Policy and Off-Policy Value Iteration Algorithms for Stochastic Zero-Sum Dynamic Games"],"prefix":"10.1007","volume":"38","author":[{"given":"Liangyuan","family":"Guo","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Bing-Chang","family":"Wang","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Ji-Feng","family":"Zhang","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2025,3,19]]},"reference":[{"key":"4572_CR1","doi-asserted-by":"publisher","first-page":"151","DOI":"10.1007\/s10957-010-9661-x","volume":"146","author":"M Pachter","year":"2010","unstructured":"Pachter M and Pham K D, Discrete-time linear-quadratic dynamic games, Journal of Optimization Theory and Applications, 2010, 146): 151\u2013179.","journal-title":"Journal of Optimization Theory and Applications"},{"key":"4572_CR2","doi-asserted-by":"publisher","DOI":"10.1515\/9781400829460","volume-title":"Theory of Games and Economic Behavior: 60th Anniversary Commemorative Edition","author":"J Von Neumann","year":"2007","unstructured":"Von Neumann J and Oskar M, Theory of Games and Economic Behavior: 60th Anniversary Commemorative Edition, Princeton University Press, Princeton, America, 2007."},{"key":"4572_CR3","doi-asserted-by":"publisher","first-page":"213","DOI":"10.1016\/j.automatica.2018.05.027","volume":"95","author":"S A A Rizvi","year":"2018","unstructured":"Rizvi S A A and Lin Z, Output feedback Q-learning for discrete-time linear zero-sum games with application to the H-infinity control, Automatica, 2018, 95): 213\u2013221.","journal-title":"Automatica"},{"issue":"6","key":"4572_CR4","doi-asserted-by":"publisher","first-page":"2042","DOI":"10.1109\/TNNLS.2017.2773458","volume":"29","author":"B Kiumarsi","year":"2017","unstructured":"Kiumarsi B, Vamvoudakis K G, Modares H, et al., Optimal and autonomous control using reinforcement learning: A survey, IEEE Transactions on Neural Networks and Learning Systems, 2017, 29(6): 2042\u20132062.","journal-title":"IEEE Transactions on Neural Networks and Learning Systems"},{"issue":"1","key":"4572_CR5","doi-asserted-by":"publisher","first-page":"14","DOI":"10.1109\/TSMCB.2010.2043839","volume":"41","author":"F L Lewis","year":"2010","unstructured":"Lewis F L and Vamvoudakis K G, Reinforcement learning for partially observable dynamic processes: Adaptive dynamic programming using measured output data, IEEE Transactions on Systems, Man, and Cybernetics, Part B (Cybernetics), 2010, 41(1): 14\u201325.","journal-title":"IEEE Transactions on Systems, Man, and Cybernetics, Part B (Cybernetics)"},{"key":"4572_CR6","volume-title":"Handbook of Intelligent Control","author":"P Werbos","year":"1992","unstructured":"Werbos P, Approximate Dynamic Programming for Real-Time Control and Neural Modeling, Handbook of Intelligent Control, Springer, London, 1992."},{"issue":"1\u20132","key":"4572_CR7","doi-asserted-by":"publisher","first-page":"81","DOI":"10.1016\/0004-3702(94)00011-O","volume":"72","author":"A G Barto","year":"1995","unstructured":"Barto A G, Bradtke S J, and Singh S P, Learning to act using real-time dynamic programming, Artificial Intelligence, 1995, 72(1\u20132): 81\u2013138.","journal-title":"Artificial Intelligence"},{"key":"4572_CR8","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-50815-3","volume-title":"Adaptive Dynamic Programming with Applications in Optimal Control","author":"D Liu","year":"2017","unstructured":"Liu D, Wei Q, Wang D, et al., Adaptive Dynamic Programming with Applications in Optimal Control, Springer, New York, 2017."},{"key":"4572_CR9","volume-title":"Reinforcement Learning and Approximate Dynamic Programming for Feedback Control","author":"F L Lewis","year":"2013","unstructured":"Lewis F L and Liu D, Reinforcement Learning and Approximate Dynamic Programming for Feedback Control, John Wiley & Sons, Hoboken, 2013."},{"issue":"3","key":"4572_CR10","doi-asserted-by":"publisher","first-page":"473","DOI":"10.1016\/j.automatica.2006.09.019","volume":"43","author":"A Al-Tamimi","year":"2007","unstructured":"Al-Tamimi A, Lewis F L, and Abu-Khalaf M, Model-free Q-learning designs for linear discrete-time zero-sum games with application to H-infinity control, Automatica, 2007, 43(3): 473\u2013481.","journal-title":"Automatica"},{"issue":"9","key":"4572_CR11","doi-asserted-by":"publisher","first-page":"4522","DOI":"10.1109\/TNNLS.2017.2755501","volume":"29","author":"A Heydari","year":"2017","unstructured":"Heydari A, Stability analysis of optimal adaptive control under value iteration using a stabilizing initial policy, IEEE Transactions on Neural Networks and Learning Systems, 2017, 29(9): 4522\u20134527.","journal-title":"IEEE Transactions on Neural Networks and Learning Systems"},{"issue":"4","key":"4572_CR12","doi-asserted-by":"publisher","first-page":"943","DOI":"10.1109\/TSMCB.2008.926614","volume":"38","author":"A Al-Tamimi","year":"2008","unstructured":"Al-Tamimi A, Lewis F L, and Abu-Khalaf M, Discrete-time nonlinear HJB solution using approximate dynamic programming: Convergence proof, IEEE Transactions on Systems, Man, and Cybernetics, Part B (Cybernetics), 2008, 38(4): 943\u2013949.","journal-title":"IEEE Transactions on Systems, Man, and Cybernetics, Part B (Cybernetics)"},{"issue":"2","key":"4572_CR13","doi-asserted-by":"publisher","first-page":"477","DOI":"10.1016\/j.automatica.2008.08.017","volume":"45","author":"D Vrabie","year":"2009","unstructured":"Vrabie D, Pastravanu O, Abu-Khalaf M, et al., Adaptive optimal control for continuous-time linear systems based on policy iteration, Automatica, 2009, 45(2): 477\u2013484.","journal-title":"Automatica"},{"issue":"1","key":"4572_CR14","doi-asserted-by":"publisher","first-page":"240","DOI":"10.1109\/TSMCB.2006.880135","volume":"37","author":"A Al-Tamimi","year":"2007","unstructured":"Al-Tamimi A, Abu-Khalaf M, and Lewis F L, Adaptive critic designs for discrete-time zero-sum games with application to H\u221e control, IEEE Transactions on Systems, Man, and Cybernetics, Part B (Cybernetics), 2007, 37(1): 240\u2013247.","journal-title":"IEEE Transactions on Systems, Man, and Cybernetics, Part B (Cybernetics)"},{"key":"4572_CR15","doi-asserted-by":"publisher","first-page":"110685","DOI":"10.1016\/j.automatica.2022.110685","volume":"147","author":"J Lai","year":"2023","unstructured":"Lai J, Xiong J, and Shu Z, Model-free optimal control of discrete-time systems with additive and multiplicative noises, Automatica, 2023, 147): 110685.","journal-title":"Automatica"},{"issue":"5","key":"4572_CR16","doi-asserted-by":"publisher","first-page":"1907","DOI":"10.1007\/s11424-024-3343-5","volume":"37","author":"B Q Zhang","year":"2024","unstructured":"Zhang B Q, Wang B C, and Cao Y, An online Q-learning method for linear-quadratic nonzero-sum stochastic differential games with completely unknown dynamics, Journal of Systems Science & Complexity, 2024, 37(5): 1907\u20131922.","journal-title":"Journal of Systems Science & Complexity"},{"issue":"4","key":"4572_CR17","doi-asserted-by":"publisher","first-page":"1520","DOI":"10.1109\/TNNLS.2020.3042589","volume":"33","author":"Z Xu","year":"2022","unstructured":"Xu Z, Shen T, and Cheng D, Model-free reinforcement learning by embedding an auxiliary system for optimal control of nonlinear systems, IEEE Transactions on Neural Networks and Learning Systems, 2022, 33(4): 1520\u20131534.","journal-title":"IEEE Transactions on Neural Networks and Learning Systems"},{"key":"4572_CR18","doi-asserted-by":"publisher","first-page":"111162","DOI":"10.1016\/j.automatica.2023.111162","volume":"155","author":"Z Xu","year":"2023","unstructured":"Xu Z, Shen T, and Huang M, Model-free policy iteration approach to NCE-based strategy design for linear quadratic Gaussian games, Automatica, 2023, 155): 111162.","journal-title":"Automatica"},{"key":"4572_CR19","doi-asserted-by":"publisher","first-page":"111924","DOI":"10.1016\/j.automatica.2024.111924","volume":"172","author":"Z Xu","year":"2025","unstructured":"Xu Z, Wang B-C, and Shen T, Mean field LQG social optimization: A reinforcement learning approach, Automatica, 2025, 172): 111924.","journal-title":"Automatica"},{"key":"4572_CR20","doi-asserted-by":"publisher","unstructured":"Zhang K, Yang Z, and Basar T, Policy optimization provably converges to Nash equilibria in zero-sum linear quadratic games, Advances in Neural Information Processing Systems, 2019, DOI: https:\/\/doi.org\/10.48550\/arXiv.1906.00729.","DOI":"10.48550\/arXiv.1906.00729"},{"issue":"10","key":"4572_CR21","doi-asserted-by":"publisher","first-page":"2699","DOI":"10.1016\/j.automatica.2012.06.096","volume":"48","author":"Y Jiang","year":"2012","unstructured":"Jiang Y and Jiang Z-P, Computational adaptive optimal control for continuous-time linear systems with completely unknown dynamics, Automatica, 2012, 48(10): 2699\u20132704.","journal-title":"Automatica"},{"issue":"7","key":"4572_CR22","doi-asserted-by":"publisher","first-page":"3630","DOI":"10.1109\/TCYB.2020.2970969","volume":"51","author":"B Luo","year":"2020","unstructured":"Luo B, Yang Y, and Liu D, Policy iteration Q-learning for data-based two-player zero-sum game of linear discrete-time systems, IEEE Transactions on Cybernetics, 2020, 51(7): 3630\u20133640.","journal-title":"IEEE Transactions on Cybernetics"},{"issue":"12","key":"4572_CR23","doi-asserted-by":"publisher","first-page":"4170","DOI":"10.1109\/TAC.2016.2550518","volume":"61","author":"T Bian","year":"2016","unstructured":"Bian T, Jiang Y, and Jiang Z-P, Adaptive dynamic programming for stochastic systems with state and control dependent noise, IEEE Transactions on Automatic Control, 2016, 61(12): 4170\u20134175.","journal-title":"IEEE Transactions on Automatic Control"},{"issue":"6","key":"4572_CR24","doi-asserted-by":"publisher","first-page":"4150","DOI":"10.1137\/18M1214147","volume":"57","author":"T Bian","year":"2019","unstructured":"Bian T and Jiang Z-P, Continuous-time robust dynamic programming, SIAM Journal on Control and Optimization, 2019, 57(6): 4150\u20134174.","journal-title":"SIAM Journal on Control and Optimization"},{"key":"4572_CR25","doi-asserted-by":"publisher","first-page":"140203","DOI":"10.1007\/s11432-023-3962-0","volume":"67","author":"B-C Wang","year":"2024","unstructured":"Wang B-C, Li S, and Cao Y, An online value iteration method for linear-quadratic mean field social control with unknown dynamics, Science China Information Sciences, 2024, 67): 140203.","journal-title":"Science China Information Sciences"},{"issue":"9","key":"4572_CR26","doi-asserted-by":"publisher","first-page":"5009","DOI":"10.1109\/TAC.2022.3181248","volume":"67","author":"N Li","year":"2022","unstructured":"Li N, Li X, Peng J, et al., Stochastic linear quadratic optimal control problem: A reinforcement learning method, IEEE Transactions on Automatic Control, 2022, 67(9): 5009\u20135016.","journal-title":"IEEE Transactions on Automatic Control"},{"key":"4572_CR27","doi-asserted-by":"publisher","first-page":"95","DOI":"10.1007\/BF01218397","volume":"26","author":"S Chen","year":"1992","unstructured":"Chen S, Necessary and sufficient conditions for the existence of positive solutions to algebraic Riccati equations with indefinite quadratic term, Applied Mathematics and Optimization, 1992, 26): 95\u2013110.","journal-title":"Applied Mathematics and Optimization"},{"key":"4572_CR28","doi-asserted-by":"publisher","first-page":"92","DOI":"10.1016\/j.neucom.2012.11.021","volume":"110","author":"D Liu","year":"2013","unstructured":"Liu D, Li H, and Wang D, Neural-network-based zero-sum game for discrete-time nonlinear systems via iterative adaptive dynamic programming algorithm, Neurocomputing, 2013, 110): 92\u2013100.","journal-title":"Neurocomputing"},{"key":"4572_CR29","doi-asserted-by":"publisher","first-page":"73","DOI":"10.1007\/s11768-019-8168-8","volume":"17","author":"B Pang","year":"2019","unstructured":"Pang B, Bian T, and Jiang Z-P, Adaptive dynamic programming for finite-horizon optimal control of linear time-varying discrete-time systems, Control Theory and Technology, 2019, 17): 73\u201384.","journal-title":"Control Theory and Technology"}],"container-title":["Journal of Systems Science and Complexity"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s11424-025-4572-y.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s11424-025-4572-y\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s11424-025-4572-y.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,3,19]],"date-time":"2025-03-19T16:05:08Z","timestamp":1742400308000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s11424-025-4572-y"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,2]]},"references-count":29,"journal-issue":{"issue":"1","published-print":{"date-parts":[[2025,2]]}},"alternative-id":["4572"],"URL":"https:\/\/doi.org\/10.1007\/s11424-025-4572-y","relation":{},"ISSN":["1009-6124","1559-7067"],"issn-type":[{"value":"1009-6124","type":"print"},{"value":"1559-7067","type":"electronic"}],"subject":[],"published":{"date-parts":[[2025,2]]},"assertion":[{"value":"11 November 2024","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"9 January 2025","order":2,"name":"revised","label":"Revised","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"19 March 2025","order":3,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"ZHANG Ji-Feng is a guest editorial board member for Journal of Systems Science & Complexity and was not involved in the editorial review or the decision to publish this article. All authors declare that there are no competing interests.","order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Conflict of Interest"}}]}}