{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,14]],"date-time":"2026-07-14T14:51:59Z","timestamp":1784040719398,"version":"3.55.0"},"reference-count":85,"publisher":"Institute of Electrical and Electronics Engineers (IEEE)","issue":"6","content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["IEEE Trans. Syst., Man, Cybern. C"],"published-print":{"date-parts":[[2012,11]]},"DOI":"10.1109\/tsmcc.2012.2218595","type":"journal-article","created":{"date-parts":[[2012,12,21]],"date-time":"2012-12-21T19:04:54Z","timestamp":1356116694000},"page":"1291-1307","source":"Crossref","is-referenced-by-count":876,"title":["A Survey of Actor-Critic Reinforcement Learning: Standard and Natural Policy Gradients"],"prefix":"10.1109","volume":"42","author":[{"given":"Ivo","family":"Grondman","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Lucian","family":"Busoniu","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Gabriel A. D.","family":"Lopes","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Robert","family":"Babuska","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"263","reference":[{"key":"ref73","doi-asserted-by":"publisher","DOI":"10.1109\/ICASSP.1998.675489"},{"key":"ref72","doi-asserted-by":"publisher","DOI":"10.1109\/ACC.2006.1656451"},{"key":"ref71","doi-asserted-by":"publisher","DOI":"10.1109\/9.119632"},{"key":"ref70","doi-asserted-by":"publisher","DOI":"10.1109\/21.52551"},{"key":"ref76","doi-asserted-by":"crossref","first-page":"66","DOI":"10.1007\/978-3-540-87481-2_5","article-title":"Fitted natural actor-critic: A new algorithm for continuous state-action MDPs","author":"melo","year":"2008","journal-title":"Proc Eur Conf Mach Learn Knowl Discovery Databases"},{"key":"ref77","first-page":"105","article-title":"Incremental natural actor-critic algorithms","author":"bhatnagar","year":"2008","journal-title":"Proc Adv Neural Inf Process Syst"},{"key":"ref74","first-page":"1019","article-title":"Covariant policy search","author":"bagnell","year":"2003","journal-title":"Proc 18th Int Joint Conf Artif Intell"},{"key":"ref39","author":"jacobson","year":"1970","journal-title":"Differential Dynamic Programming (ser Modern Analytic and Computational Methods in Science and Mathematics vol 24)"},{"key":"ref75","doi-asserted-by":"publisher","DOI":"10.1162\/089976698300017746"},{"key":"ref38","doi-asserted-by":"publisher","DOI":"10.1016\/j.neunet.2008.02.003"},{"key":"ref78","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-540-87481-2_6"},{"key":"ref79","doi-asserted-by":"publisher","DOI":"10.1162\/neco.2009.12-08-922"},{"key":"ref33","doi-asserted-by":"publisher","DOI":"10.1007\/BF00992696"},{"key":"ref32","doi-asserted-by":"publisher","DOI":"10.1016\/0893-6080(90)90056-Q"},{"key":"ref31","first-page":"615","article-title":"Finite-sample analysis of LSTD","author":"lazaric","year":"2010","journal-title":"Proc 27th Int Conf Mach Learn"},{"key":"ref30","first-page":"1555","article-title":"Optimality of reinforcement learning algorithms with linear function approximation","author":"schoknecht","year":"2003","journal-title":"Advances in Neural Information Processing Systems 15"},{"key":"ref37","first-page":"366","article-title":"Likelihood ratio gradient estimation: An overview","author":"glynn","year":"1987","journal-title":"Proc Winter Simul Conf"},{"key":"ref36","first-page":"11","article-title":"Stochastic optimization","volume":"5","author":"aleksandrov","year":"1968","journal-title":"Eng Cybern"},{"key":"ref35","doi-asserted-by":"publisher","DOI":"10.1145\/1390156.1390214"},{"key":"ref34","article-title":"Policy search in kernel Hilbert space","author":"bagnell","year":"2003"},{"key":"ref60","doi-asserted-by":"publisher","DOI":"10.1007\/11596448_9"},{"key":"ref62","doi-asserted-by":"publisher","DOI":"10.1109\/SICE.2008.4654995"},{"key":"ref61","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-540-89722-4_9"},{"key":"ref63","doi-asserted-by":"crossref","first-page":"433","DOI":"10.1109\/TSMCB.2009.2026289","article-title":"Impedance learning for robotic contact tasks using natural actor-critic algorithm","volume":"40","author":"kim","year":"2010","journal-title":"IEEE Trans Syst Man Cybern B Cybern"},{"key":"ref28","doi-asserted-by":"publisher","DOI":"10.1109\/9.580874"},{"key":"ref64","doi-asserted-by":"publisher","DOI":"10.1016\/j.neunet.2007.01.002"},{"key":"ref27","doi-asserted-by":"publisher","DOI":"10.1007\/BF00114724"},{"key":"ref65","doi-asserted-by":"publisher","DOI":"10.1109\/ROBOT.2010.5509751"},{"key":"ref66","doi-asserted-by":"publisher","DOI":"10.1109\/TSMCB.2005.846001"},{"key":"ref29","doi-asserted-by":"publisher","DOI":"10.1145\/1390156.1390240"},{"key":"ref67","doi-asserted-by":"publisher","DOI":"10.1109\/TSMCB.2006.886173"},{"key":"ref68","first-page":"1312","article-title":"A generalized natural actor-critic algorithm","author":"morimura","year":"2009","journal-title":"Advances in Neural Information Processing Systems 22"},{"key":"ref69","doi-asserted-by":"publisher","DOI":"10.1147\/rd.33.0210"},{"key":"ref2","first-page":"1169","article-title":"Natural actor-critic for road traffic optimisation","author":"richter","year":"2007","journal-title":"Advances in Neural Information Processing Systems 19"},{"key":"ref1","doi-asserted-by":"publisher","DOI":"10.1137\/S0363012901385691"},{"key":"ref20","doi-asserted-by":"publisher","DOI":"10.1016\/j.automatica.2009.07.008"},{"key":"ref22","doi-asserted-by":"publisher","DOI":"10.1007\/BF00992698"},{"key":"ref21","author":"watkins","year":"1989","journal-title":"Learning from delayed rewards"},{"key":"ref24","article-title":"On-line Q-learning using connectionist systems","author":"rummery","year":"1994","journal-title":"Technical Report CUED\/F-INFENG\/TR291"},{"key":"ref23","doi-asserted-by":"publisher","DOI":"10.1109\/ACC.1994.735224"},{"key":"ref26","doi-asserted-by":"publisher","DOI":"10.1016\/B978-1-55860-377-6.50040-2"},{"key":"ref25","doi-asserted-by":"publisher","DOI":"10.1016\/B978-1-55860-377-6.50013-X"},{"key":"ref50","doi-asserted-by":"publisher","DOI":"10.1109\/CDC.2009.5400592"},{"key":"ref51","first-page":"1531","article-title":"Natural policy gradient","author":"kakade","year":"2001","journal-title":"Advances in Neural Information Processing Systems 14"},{"key":"ref59","doi-asserted-by":"publisher","DOI":"10.1109\/COEC.2003.1210269"},{"key":"ref58","doi-asserted-by":"publisher","DOI":"10.1109\/CDC.2001.980135"},{"key":"ref57","first-page":"368","article-title":"Urban traffic signal learning control using fuzzy actor-critic methods","author":"li","year":"2009","journal-title":"Proc 5th Int Conf Natural Comput"},{"key":"ref56","doi-asserted-by":"publisher","DOI":"10.1016\/j.sysconle.2010.08.013"},{"key":"ref55","first-page":"37","article-title":"A consolidated actor-critic model with function approximation for high-dimensional POMDPs","author":"niedzwiedz","year":"2008","journal-title":"Proc AAAI Workshop Adv POMDP Solvers"},{"key":"ref54","doi-asserted-by":"publisher","DOI":"10.1016\/j.ins.2007.03.012"},{"key":"ref53","doi-asserted-by":"crossref","first-page":"2985","DOI":"10.1109\/ICMLC.2004.1378544","article-title":"Application of actor-critic learning to adaptive state space construction","author":"cheng","year":"2004","journal-title":"Proc 3rd Int Conf Mach Learn Cybern"},{"key":"ref52","article-title":"Reinforcement learning for humanoid robotics","author":"peters","year":"0","journal-title":"IEEE-RAS Int Conf on Hum Robot"},{"key":"ref10","doi-asserted-by":"publisher","DOI":"10.2200\/S00268ED1V01Y201005AIM009"},{"key":"ref11","doi-asserted-by":"publisher","DOI":"10.1109\/TNN.2006.889499"},{"key":"ref40","author":"dyer","year":"1970","journal-title":"The Computation and Theory of Optimal Control (ser Mathematics in Science and Engineering vol 65)"},{"key":"ref12","doi-asserted-by":"publisher","DOI":"10.1016\/j.automatica.2010.02.018"},{"key":"ref13","doi-asserted-by":"publisher","DOI":"10.1109\/TAC.2009.2037462"},{"key":"ref14","first-page":"878","article-title":"A multi-agent reinforcement learning using actor-critic methods","author":"li","year":"2008","journal-title":"Proc Int Conf Mach Learn Cybern"},{"key":"ref15","author":"bu?oniu","year":"2010","journal-title":"Reinforcement Learning and Dynamic Programming Using Function Approximators"},{"key":"ref82","doi-asserted-by":"publisher","DOI":"10.1016\/S0921-8890(97)00043-2"},{"key":"ref16","doi-asserted-by":"publisher","DOI":"10.1016\/j.neucom.2007.11.026"},{"key":"ref81","first-page":"327","article-title":"Learning control under extreme uncertainty","author":"gullapalli","year":"1993","journal-title":"Advances in Neural Information Processing Systems 5"},{"key":"ref17","doi-asserted-by":"publisher","DOI":"10.1016\/S0167-6911(01)00152-9"},{"key":"ref84","doi-asserted-by":"publisher","DOI":"10.1007\/s10994-010-5223-6"},{"key":"ref18","author":"bertsekas","year":"2007","journal-title":"Dynamic Programming and Optimal Control Volume II"},{"key":"ref83","doi-asserted-by":"publisher","DOI":"10.1137\/S036301299731669X"},{"key":"ref19","first-page":"1057","article-title":"Policy gradient methods for reinforcement learning with function approximation","author":"sutton","year":"2000","journal-title":"Advances in Neural Information Processing Systems 12"},{"key":"ref80","doi-asserted-by":"publisher","DOI":"10.1109\/IJCNN.1992.287219"},{"key":"ref4","doi-asserted-by":"crossref","first-page":"319","DOI":"10.1613\/jair.806","article-title":"Infinite-horizon policy-gradient estimation","volume":"15","author":"baxter","year":"2001","journal-title":"J Artif Intell Res"},{"key":"ref3","doi-asserted-by":"publisher","DOI":"10.1023\/A:1017936530646"},{"key":"ref6","doi-asserted-by":"publisher","DOI":"10.1007\/BF00115009"},{"key":"ref5","doi-asserted-by":"publisher","DOI":"10.1109\/TFUZZ.2003.814834"},{"key":"ref85","doi-asserted-by":"publisher","DOI":"10.1007\/s10514-009-9132-0"},{"key":"ref8","doi-asserted-by":"crossref","first-page":"237","DOI":"10.1613\/jair.301","article-title":"Reinforcement learning: A survey","volume":"4","author":"kaelbling","year":"1996","journal-title":"J Artif Intell Res"},{"key":"ref7","author":"sutton","year":"1998","journal-title":"Reinforcement Learning An Introduction"},{"key":"ref49","doi-asserted-by":"publisher","DOI":"10.1016\/S0005-1098(99)00099-0"},{"key":"ref9","doi-asserted-by":"publisher","DOI":"10.1287\/ijoc.1080.0305"},{"key":"ref46","doi-asserted-by":"publisher","DOI":"10.1109\/TSMC.1983.6313077"},{"key":"ref45","doi-asserted-by":"publisher","DOI":"10.1016\/S0019-9958(77)90354-0"},{"key":"ref48","article-title":"Gradient descent for general reinforcement learning","author":"baird","year":"1999","journal-title":"Advances in Neural Information Processing Systems 11"},{"key":"ref47","doi-asserted-by":"publisher","DOI":"10.1007\/BF00114723"},{"key":"ref42","first-page":"465","article-title":"PILCO: A model-based and data-efficient approach to policy search","author":"deisenroth","year":"2011","journal-title":"Proc 28th Int Conf Mach Learn"},{"key":"ref41","author":"hasdorff","year":"1976","journal-title":"Gradient Optimization and Nonlinear Control"},{"key":"ref44","first-page":"1607","article-title":"Relative entropy policy search","author":"peters","year":"2010","journal-title":"Proc 24th AAAI Conf Artif Intell"},{"key":"ref43","doi-asserted-by":"publisher","DOI":"10.1109\/ADPRL.2007.368196"}],"container-title":["IEEE Transactions on Systems, Man, and Cybernetics, Part C (Applications and Reviews)"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx5\/5326\/6330018\/06392457.pdf?arnumber=6392457","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2023,6,28]],"date-time":"2023-06-28T02:40:49Z","timestamp":1687920049000},"score":1,"resource":{"primary":{"URL":"http:\/\/ieeexplore.ieee.org\/document\/6392457\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2012,11]]},"references-count":85,"journal-issue":{"issue":"6"},"URL":"https:\/\/doi.org\/10.1109\/tsmcc.2012.2218595","relation":{},"ISSN":["1094-6977","1558-2442"],"issn-type":[{"value":"1094-6977","type":"print"},{"value":"1558-2442","type":"electronic"}],"subject":[],"published":{"date-parts":[[2012,11]]}}}