{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,1,8]],"date-time":"2026-01-08T20:45:41Z","timestamp":1767905141544,"version":"3.49.0"},"reference-count":45,"publisher":"Springer Science and Business Media LLC","issue":"3","license":[{"start":{"date-parts":[[2023,4,5]],"date-time":"2023-04-05T00:00:00Z","timestamp":1680652800000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2023,4,5]],"date-time":"2023-04-05T00:00:00Z","timestamp":1680652800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"funder":[{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["11831010"],"award-info":[{"award-number":["11831010"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["61821004"],"award-info":[{"award-number":["61821004"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["61925306"],"award-info":[{"award-number":["61925306"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100007129","name":"Natural Science Foundation of Shandong Province","doi-asserted-by":"publisher","award":["ZR2019ZD42"],"award-info":[{"award-number":["ZR2019ZD42"]}],"id":[{"id":"10.13039\/501100007129","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100007129","name":"Natural Science Foundation of Shandong Province","doi-asserted-by":"publisher","award":["ZR2020ZD24"],"award-info":[{"award-number":["ZR2020ZD24"]}],"id":[{"id":"10.13039\/501100007129","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100012166","name":"National Key R &D Program of China","doi-asserted-by":"crossref","award":["2022YFA1006103"],"award-info":[{"award-number":["2022YFA1006103"]}],"id":[{"id":"10.13039\/501100012166","id-type":"DOI","asserted-by":"crossref"}]}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["J. Appl. Math. Comput."],"published-print":{"date-parts":[[2023,6]]},"DOI":"10.1007\/s12190-023-01857-9","type":"journal-article","created":{"date-parts":[[2023,4,5]],"date-time":"2023-04-05T20:24:49Z","timestamp":1680726289000},"page":"2741-2760","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":16,"title":["An adaptive dynamic programming-based algorithm for infinite-horizon linear quadratic stochastic optimal control problems"],"prefix":"10.1007","volume":"69","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-2508-1137","authenticated-orcid":false,"given":"Heng","family":"Zhang","sequence":"first","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2023,4,5]]},"reference":[{"key":"1857_CR1","doi-asserted-by":"publisher","first-page":"1296","DOI":"10.1137\/S0363012900371083","volume":"40","author":"M Ait Rami","year":"2001","unstructured":"Ait Rami, M., Moore, J.B., Zhou, X.: Indefinite stochastic linear quadratic control and generilized differential Riccati equation. SIAM J. Control Optim. 40, 1296\u20131311 (2001)","journal-title":"SIAM J. Control Optim."},{"issue":"6","key":"1857_CR2","doi-asserted-by":"publisher","first-page":"1131","DOI":"10.1109\/9.863597","volume":"45","author":"M Ait Rami","year":"2000","unstructured":"Ait Rami, M., Zhou, X.: Linear matrix inequalities, Riccati equations, and indefinite stochastic linear quadratic controls. IEEE Trans. Autom. Control 45(6), 1131\u20131143 (2000)","journal-title":"IEEE Trans. Autom. Control"},{"issue":"2","key":"1857_CR3","doi-asserted-by":"publisher","first-page":"123","DOI":"10.1016\/S0167-6911(00)00046-3","volume":"41","author":"M Ait Rami","year":"2000","unstructured":"Ait Rami, M., Zhou, X., Moore, J.B.: Well-posedness and attainability of indefinite stochastic linear quadratic control in infinite time horizon. Syst. Control Lett. 41(2), 123\u2013133 (2000)","journal-title":"Syst. Control Lett."},{"issue":"3","key":"1857_CR4","doi-asserted-by":"publisher","first-page":"473","DOI":"10.1016\/j.automatica.2006.09.019","volume":"43","author":"A Al-Tamimi","year":"2007","unstructured":"Al-Tamimi, A., Lewis, F.L., Abu-Khalaf, M.: Model-free Q-learning designs for linear discrete-time zero-sum games with application to H-infinity control. Automatica 43(3), 473\u2013481 (2007)","journal-title":"Automatica"},{"issue":"12","key":"1857_CR5","doi-asserted-by":"publisher","first-page":"4170","DOI":"10.1109\/TAC.2016.2550518","volume":"61","author":"T Bian","year":"2016","unstructured":"Bian, T., Jiang, Y., Jiang, Z.P.: Adaptive dynamic programming for stochastic systems with state and control dependent noise. IEEE Trans. Autom. Control 61(12), 4170\u20134175 (2016)","journal-title":"IEEE Trans. Autom. Control"},{"key":"1857_CR6","doi-asserted-by":"crossref","unstructured":"Bian, T., Jiang, Z.P.: Stochastic adaptive dynamic programming for robust optimal control design, in Control of Complex Systems: Theory and Applications, K.G. Vamvoudakis and S. Jagannathan, eds. Butterworth-Heinemann, Cambridge, MA, pp. 211\u2013245 (2016)","DOI":"10.1016\/B978-0-12-805246-4.00007-0"},{"issue":"6","key":"1857_CR7","doi-asserted-by":"publisher","first-page":"4150","DOI":"10.1137\/18M1214147","volume":"57","author":"T Bian","year":"2019","unstructured":"Bian, T., Jiang, Z.P.: Continuous-time robust dynamic programming. SIAM J. Control Optim. 57(6), 4150\u20134174 (2019)","journal-title":"SIAM J. Control Optim."},{"key":"1857_CR8","first-page":"295","volume":"5","author":"SJ Bradtke","year":"1993","unstructured":"Bradtke, S.J.: Reinforcement learning applied to linear quadratic regulation. Adv. Neural Inf. Process. Syst. 5, 295\u2013302 (1993)","journal-title":"Adv. Neural Inf. Process. Syst."},{"key":"1857_CR9","doi-asserted-by":"publisher","first-page":"81","DOI":"10.1016\/S0024-3795(00)00144-0","volume":"332\u2013334","author":"T Damm","year":"2001","unstructured":"Damm, T., Hinrichsen, D.: Newton\u2019s method for a rational matrix equation occuring in stochastic control. Linear Algebra Appl. 332\u2013334, 81\u2013109 (2001)","journal-title":"Linear Algebra Appl."},{"issue":"1\u20133","key":"1857_CR10","doi-asserted-by":"publisher","first-page":"43","DOI":"10.1016\/S0024-3795(02)00651-1","volume":"379","author":"G Freiling","year":"2004","unstructured":"Freiling, G., Hochhaus, A.: On a class of rational matrix differential equations arising in stochastic control. Linear Algebra Appl. 379(1\u20133), 43\u201368 (2004)","journal-title":"Linear Algebra Appl."},{"issue":"6","key":"1857_CR11","doi-asserted-by":"publisher","first-page":"1137","DOI":"10.1016\/S0898-1221(03)00073-7","volume":"45","author":"G Freiling","year":"2003","unstructured":"Freiling, G., Hochhaus, A.: Properties of the solutions of rational matrix difference equations. Comput. Math. Appl. 45(6), 1137\u20131154 (2003)","journal-title":"Comput. Math. Appl."},{"issue":"15","key":"1857_CR12","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1080\/00207721.2021.1929554","volume":"52","author":"Y Ge","year":"2021","unstructured":"Ge, Y., Liu, X., Li, Y.: Optimal control for unknown mean-field discrete-time system based on Q-Learning. Int. J. Syst. Sci. 52(15), 1\u201315 (2021)","journal-title":"Int. J. Syst. Sci."},{"issue":"6","key":"1857_CR13","doi-asserted-by":"publisher","first-page":"977","DOI":"10.1016\/j.camwa.2006.12.009","volume":"53","author":"IG Ivanov","year":"2007","unstructured":"Ivanov, I.G.: Iterations for solving a rational Riccati equation arising in stochastic control. Comput. Math. Appl. 53(6), 977\u2013988 (2007)","journal-title":"Comput. Math. Appl."},{"issue":"10","key":"1857_CR14","doi-asserted-by":"publisher","first-page":"2699","DOI":"10.1016\/j.automatica.2012.06.096","volume":"48","author":"Y Jiang","year":"2012","unstructured":"Jiang, Y., Jiang, Z.P.: Computational adaptive optimal control for continuous-time linear systems with completely unknown dynamics. Automatica 48(10), 2699\u20132704 (2012)","journal-title":"Automatica"},{"issue":"1","key":"1857_CR15","doi-asserted-by":"publisher","first-page":"882","DOI":"10.1002\/mma.6796","volume":"44","author":"T Jin","year":"2021","unstructured":"Jin, T., Xia, H., Chen, H.: Optimal control problem of the uncertain second-order circuit based on first hitting criteria. Math. Method. Appl. Sci. 44(1), 882\u2013900 (2021)","journal-title":"Math. Method. Appl. Sci."},{"issue":"12","key":"1857_CR16","doi-asserted-by":"publisher","first-page":"5955","DOI":"10.1007\/s00034-021-01761-2","volume":"40","author":"T Jin","year":"2021","unstructured":"Jin, T., Xia, H., Deng, W., Li, Y., Chen, H.: Uncertain fractional-order multi-objective optimization based on reliability analysis and application to fractional-order circuit with caputo type. Circ. Syst. Signal Process. 40(12), 5955\u20135982 (2021)","journal-title":"Circ. Syst. Signal Process."},{"issue":"5","key":"1857_CR17","doi-asserted-by":"publisher","first-page":"2615","DOI":"10.1002\/mma.7943","volume":"45","author":"T Jin","year":"2022","unstructured":"Jin, T., Xia, H., Gao, S.: Reliability analysis of the uncertain fractional-order dynamic system with state constraint. Math. Method. Appl. Sci. 45(5), 2615\u20132637 (2022)","journal-title":"Math. Method. Appl. Sci."},{"issue":"6","key":"1857_CR18","doi-asserted-by":"publisher","first-page":"2042","DOI":"10.1109\/TNNLS.2017.2773458","volume":"29","author":"B Kiumarsi","year":"2017","unstructured":"Kiumarsi, B., Vamvoudakis, K.G., Modares, H., Lewis, F.L.: Optimal and autonomous control using reinforcement learning: a survey. IEEE Trans. Neural Netw. Learn. Syst. 29(6), 2042\u20132062 (2017)","journal-title":"IEEE Trans. Neural Netw. Learn. Syst."},{"issue":"6","key":"1857_CR19","doi-asserted-by":"publisher","first-page":"673","DOI":"10.1109\/TAC.1969.1099303","volume":"14","author":"D Kleinman","year":"1969","unstructured":"Kleinman, D.: Optimal stationary control of linear systems with control-dependent noise. IEEE Trans. Autom. Control. 14(6), 673\u2013677 (1969)","journal-title":"IEEE Trans. Autom. Control."},{"key":"1857_CR20","unstructured":"Kolmanovsky, V.B., Shaikhet, L.E.: Control of Systems with Aftereffect. Trans. Math. Monogr. (1996)"},{"key":"1857_CR21","volume-title":"Reinforcement Learning and Approximate Dynamic Programming for Feedback Control","author":"FL Lewis","year":"2013","unstructured":"Lewis, F.L., Liu, D.: Reinforcement Learning and Approximate Dynamic Programming for Feedback Control. Wiley, New York, NY, USA (2013)"},{"key":"1857_CR22","doi-asserted-by":"publisher","DOI":"10.1016\/j.knosys.2022.109683","volume":"255","author":"B Li","year":"2022","unstructured":"Li, B., Xu, J., Jin, T., Shu, Y.: Piecewise parameterization for multifactor uncertain system and uncertain inventory-promotion optimization. Knowl-based Syst. 255, 109683 (2022)","journal-title":"Knowl-based Syst."},{"key":"1857_CR23","doi-asserted-by":"publisher","DOI":"10.1016\/j.automatica.2022.110751","volume":"147","author":"B Li","year":"2023","unstructured":"Li, B., Zhang, R., Sun, Y.: Multi-period portfolio selection based on uncertainty theory with bankruptcy control and liquidity. Automatica 147, 110751 (2023)","journal-title":"Automatica"},{"key":"1857_CR24","doi-asserted-by":"publisher","DOI":"10.1016\/j.automatica.2022.110253","volume":"140","author":"M Li","year":"2022","unstructured":"Li, M., Qin, J., Zheng, W., Wang, Y., Kang, Y.: Model-free design of stochastic LQR controller from a primal-dual optimization perspective. Automatica 140, 110253 (2022)","journal-title":"Automatica"},{"key":"1857_CR25","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-50815-3","volume-title":"Adaptive Dynamic Programming with Applications in Optimal Control","author":"D Liu","year":"2017","unstructured":"Liu, D., Wei, Q., Wang, D., Yang, X., Li, H.: Adaptive Dynamic Programming with Applications in Optimal Control. Springer, Cham, Switzerland (2017)"},{"key":"1857_CR26","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1016\/j.camwa.2019.03.005","volume":"363","author":"X Liu","year":"2019","unstructured":"Liu, X., Ge, Y., Li, Y.: Stackelberg games for model-free continuous-time stochastic systems based on adaptive dynamic programming. Appl. Math. Comput. 363, 1\u201319 (2019)","journal-title":"Appl. Math. Comput."},{"issue":"10","key":"1857_CR27","doi-asserted-by":"publisher","first-page":"3341","DOI":"10.1109\/TCYB.2016.2623859","volume":"47","author":"B Luo","year":"2017","unstructured":"Luo, B., Liu, D., Wu, H., Wang, D., Lewis, F.L.: Policy gradient adaptive dynamic programming for data-based optimal control. IEEE Trans. Cybern. 47(10), 3341\u20133354 (2017)","journal-title":"IEEE Trans. Cybern."},{"key":"1857_CR28","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1016\/j.sysconle.2021.104983","volume":"154","author":"S Mukherjee","year":"2021","unstructured":"Mukherjee, S., Bai, H., Chakrabortty, A.: Model-based and model-free designs for an extended continuous-time LQR with exogenous inputs. Syst. Control Lett. 154, 1\u20139 (2021)","journal-title":"Syst. Control Lett."},{"issue":"5","key":"1857_CR29","doi-asserted-by":"publisher","first-page":"3844","DOI":"10.1137\/11085267X","volume":"51","author":"Y Ni","year":"2013","unstructured":"Ni, Y., Fang, H.: Policy iteration algorithm for singular controlled diffusion processes. SIAM J. Control Optim. 51(5), 3844\u20133862 (2013)","journal-title":"SIAM J. Control Optim."},{"key":"1857_CR30","unstructured":"\u00d8ksendal, B. (sixth ed.): Stochastic Differential Equations: An Introduction with Applications. Springer Berlin (2014)"},{"issue":"2","key":"1857_CR31","doi-asserted-by":"publisher","first-page":"165","DOI":"10.1109\/TCYB.2014.2322116","volume":"45","author":"M Palanisamy","year":"2015","unstructured":"Palanisamy, M., Modares, H., Lewis, F.L., Aurangzeb, M.: Continuous-time q-learning for infinite-horizon discounted cost linear quadratic regulator problems. IEEE Trans. Cybern. 45(2), 165\u2013176 (2015)","journal-title":"IEEE Trans. Cybern."},{"issue":"1","key":"1857_CR32","doi-asserted-by":"publisher","first-page":"504","DOI":"10.1109\/TAC.2021.3085510","volume":"67","author":"B Pang","year":"2022","unstructured":"Pang, B., Bian, T., Jiang, Z.P.: Robust policy iteration for continuous-time linear quadratic regulation. IEEE Trans. Autom. Control 67(1), 504\u2013511 (2022)","journal-title":"IEEE Trans. Autom. Control"},{"key":"1857_CR33","doi-asserted-by":"crossref","unstructured":"Pang, B., Jiang, Z.P.: Reinforcement learning for adaptive optimal stationary control of linear stochastic systems. IEEE Trans. Autom. Control, Early Access (2022)","DOI":"10.1109\/TAC.2022.3172250"},{"key":"1857_CR34","doi-asserted-by":"publisher","first-page":"14","DOI":"10.1016\/j.sysconle.2016.12.003","volume":"100","author":"KG Vamvoudakis","year":"2017","unstructured":"Vamvoudakis, K.G.: Q-learning for continuous-time linear systems: a model-free infinite horizon optimal control approach. Syst. Control Lett. 100, 14\u201320 (2017)","journal-title":"Syst. Control Lett."},{"issue":"2","key":"1857_CR35","doi-asserted-by":"publisher","first-page":"477","DOI":"10.1016\/j.automatica.2008.08.017","volume":"45","author":"D Vrabie","year":"2009","unstructured":"Vrabie, D., Pastravanu, O., Abu-Khalaf, M., Lewis, F.L.: Adaptive optimal control for continuous-time linear systems based on policy iteration. Automatica 45(2), 477\u2013484 (2009)","journal-title":"Automatica"},{"key":"1857_CR36","doi-asserted-by":"publisher","first-page":"379","DOI":"10.1016\/j.neucom.2015.06.053","volume":"171","author":"T Wang","year":"2016","unstructured":"Wang, T., Zhang, H., Luo, Y.: Infinite-time stochastic linear quadratic optimal control for unknown discrete-time systems using adaptive dynamic programming approach. Neurocomputing 171, 379\u2013386 (2016)","journal-title":"Neurocomputing"},{"key":"1857_CR37","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1016\/j.neucom.2018.04.018","volume":"312","author":"T Wang","year":"2018","unstructured":"Wang, T., Zhang, H., Luo, Y.: Stochastic linear quadratic optimal control for model-free discrete-time systems based on Q-learning algorithm. Neurocomputing 312, 1\u20138 (2018)","journal-title":"Neurocomputing"},{"key":"1857_CR38","unstructured":"Wang, Y., Ni, Y., Chen, Z., Zhang, J.: Probabilistic Framework of Howard\u2019s Policy Iteration: BML Evaluation and Robust Convergence Analysis. arXiv:2210.07473"},{"issue":"3","key":"1857_CR39","doi-asserted-by":"publisher","first-page":"840","DOI":"10.1109\/TCYB.2015.2492242","volume":"46","author":"Q Wei","year":"2016","unstructured":"Wei, Q., Liu, D., Lin, H.: Value iteration adaptive dynamic programming for optimal control of discrete-time nonlinear systems. IEEE Trans. Cybern. 46(3), 840\u2013853 (2016)","journal-title":"IEEE Trans. Cybern."},{"key":"1857_CR40","unstructured":"Werbos, P.J.: Beyond regression: new tools for prediction and analysis in the behavioural sciences. Ph.D. Thesis, Harvard University (1974)"},{"issue":"4","key":"1857_CR41","doi-asserted-by":"publisher","first-page":"681","DOI":"10.1137\/0306044","volume":"6","author":"WM Wonham","year":"1968","unstructured":"Wonham, W.M.: On a matrix Riccati equation of stochastic control. SIAM J. Control 6(4), 681\u2013697 (1968)","journal-title":"SIAM J. Control"},{"key":"1857_CR42","doi-asserted-by":"publisher","DOI":"10.1016\/j.automatica.2022.110564","volume":"146","author":"K Xie","year":"2022","unstructured":"Xie, K., Yu, X., Lan, W.: Optimal output regulation for unknown continuous-time linear systems by internal model and adaptive dynamic programming. Automatica 146, 110564 (2022)","journal-title":"Automatica"},{"issue":"6","key":"1857_CR43","doi-asserted-by":"publisher","first-page":"1017","DOI":"10.1016\/j.automatica.2012.03.007","volume":"48","author":"H Xu","year":"2012","unstructured":"Xu, H., Jagannathan, S., Lewis, F.L.: Stochastic optimal control of unknown linear networked control system in the presence of random delays and packet losses. Automatica 48(6), 1017\u20131030 (2012)","journal-title":"Automatica"},{"key":"1857_CR44","unstructured":"Zhang, W.: Study on Algebraic Riccati Equation Arising from Infinite Horizon Stochastic LQ Optimal Control. Ph.D. Thesis, Zhejiang University (1998)"},{"issue":"1","key":"1857_CR45","doi-asserted-by":"publisher","first-page":"19","DOI":"10.1007\/s002450010003","volume":"42","author":"X Zhou","year":"2000","unstructured":"Zhou, X., Li, D.: Continuous-time mean-variance portfolio selection: a stochastic LQ framework. Appl. Math. Optim. 42(1), 19\u201333 (2000)","journal-title":"Appl. Math. Optim."}],"container-title":["Journal of Applied Mathematics and Computing"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s12190-023-01857-9.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s12190-023-01857-9\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s12190-023-01857-9.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2023,5,27]],"date-time":"2023-05-27T01:11:20Z","timestamp":1685149880000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s12190-023-01857-9"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023,4,5]]},"references-count":45,"journal-issue":{"issue":"3","published-print":{"date-parts":[[2023,6]]}},"alternative-id":["1857"],"URL":"https:\/\/doi.org\/10.1007\/s12190-023-01857-9","relation":{},"ISSN":["1598-5865","1865-2085"],"issn-type":[{"value":"1598-5865","type":"print"},{"value":"1865-2085","type":"electronic"}],"subject":[],"published":{"date-parts":[[2023,4,5]]},"assertion":[{"value":"27 October 2022","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"23 March 2023","order":2,"name":"revised","label":"Revised","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"25 March 2023","order":3,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"5 April 2023","order":4,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Declarations"}},{"value":"The author declares that he has no conflicts of interest.","order":2,"name":"Ethics","group":{"name":"EthicsHeading","label":"Conflict of interest"}}]}}