{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,1,31]],"date-time":"2026-01-31T13:12:59Z","timestamp":1769865179356,"version":"3.49.0"},"reference-count":51,"publisher":"Institute of Electrical and Electronics Engineers (IEEE)","issue":"8","license":[{"start":{"date-parts":[[2025,8,1]],"date-time":"2025-08-01T00:00:00Z","timestamp":1754006400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/ieeexplore.ieee.org\/Xplorehelp\/downloads\/license-information\/IEEE.html"},{"start":{"date-parts":[[2025,8,1]],"date-time":"2025-08-01T00:00:00Z","timestamp":1754006400000},"content-version":"am","delay-in-days":0,"URL":"https:\/\/ieeexplore.ieee.org\/Xplorehelp\/downloads\/license-information\/IEEE.html"},{"start":{"date-parts":[[2025,8,1]],"date-time":"2025-08-01T00:00:00Z","timestamp":1754006400000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2025,8,1]],"date-time":"2025-08-01T00:00:00Z","timestamp":1754006400000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"}],"funder":[{"name":"U. S. Army Research Laboratory and the U.S. Army Research Office","award":["W911NF2010219"],"award-info":[{"award-number":["W911NF2010219"]}]},{"DOI":"10.13039\/100000006","name":"Office of Naval Research","doi-asserted-by":"publisher","award":["N000142412673"],"award-info":[{"award-number":["N000142412673"]}],"id":[{"id":"10.13039\/100000006","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000181","name":"Air Force Office of Scientific Research","doi-asserted-by":"publisher","id":[{"id":"10.13039\/100000181","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000001","name":"National Science Foundation","doi-asserted-by":"publisher","id":[{"id":"10.13039\/100000001","id-type":"DOI","asserted-by":"publisher"}]},{"name":"UC Noyce Initiative"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["IEEE Trans. Automat. Contr."],"published-print":{"date-parts":[[2025,8]]},"DOI":"10.1109\/tac.2025.3540965","type":"journal-article","created":{"date-parts":[[2025,2,11]],"date-time":"2025-02-11T13:33:14Z","timestamp":1739280794000},"page":"5129-5144","source":"Crossref","is-referenced-by-count":1,"title":["Beyond Exact Gradients: Convergence of Stochastic Soft-Max Policy Gradient Methods With Entropy Regularization"],"prefix":"10.1109","volume":"70","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-1521-7566","authenticated-orcid":false,"given":"Yuhao","family":"Ding","sequence":"first","affiliation":[{"name":"Department of Industrial Engineering and Operations Research, University of California, Berkeley, Berkeley, CA, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-5086-0063","authenticated-orcid":false,"given":"Junzi","family":"Zhang","sequence":"additional","affiliation":[{"name":"Citadel Securities, Chicago, IL, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-1718-9594","authenticated-orcid":false,"given":"Hyunin","family":"Lee","sequence":"additional","affiliation":[{"name":"Department of Mechanical Engineering, University of California, Berkeley, Berkeley, CA, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-4294-1338","authenticated-orcid":false,"given":"Javad","family":"Lavaei","sequence":"additional","affiliation":[{"name":"Department of Industrial Engineering and Operations Research, University of California, Berkeley, Berkeley, CA, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"263","reference":[{"key":"ref1","doi-asserted-by":"publisher","DOI":"10.1080\/09540099108946587"},{"key":"ref2","first-page":"1928","article-title":"Asynchronous methods for deep reinforcement learning","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Mnih","year":"2016"},{"key":"ref3","first-page":"1861","article-title":"Soft actorcritic: Off-policy maximum entropy deep reinforcement learning with a stochastic actor","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Haarnoja","year":"2018"},{"key":"ref4","article-title":"Combining policy gradient and Q-learning","author":"O\u2019Donoghue","year":"2016"},{"key":"ref5","first-page":"1352","article-title":"Reinforcement learning with deep energy-based policies","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Haarnoja","year":"2017"},{"key":"ref6","article-title":"TEAC: Intergrating trust region and max entropy actor critic for continuous control","author":"Zang","year":"2020"},{"key":"ref7","volume-title":"Modeling Purposeful Adaptive Behavior With the Principle of Maximum Causal Entropy","author":"Ziebart","year":"2010"},{"key":"ref8","article-title":"Equivalence between policy gradients and soft Q-learning","author":"Schulman","year":"2017"},{"key":"ref9","first-page":"6820","article-title":"On the global convergence rates of softmax policy gradient methods","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Mei","year":"2020"},{"key":"ref10","doi-asserted-by":"publisher","DOI":"10.1007\/s10107-022-01816-5"},{"key":"ref11","doi-asserted-by":"publisher","DOI":"10.1287\/opre.2021.2151"},{"key":"ref12","first-page":"1999","article-title":"Beyond variance reduction: Understanding the true impact of baselines on policy optimization","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Chung","year":"2021"},{"key":"ref13","first-page":"19339","article-title":"Understanding the effect of stochasticity in policy optimization","volume-title":"Adv. Neural Inf. Process. Syst.","volume":"34","author":"Mei","year":"2021"},{"key":"ref14","doi-asserted-by":"publisher","DOI":"10.1007\/BF00992696"},{"key":"ref15","first-page":"151","article-title":"Understanding the impact of entropy on policy optimization","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Ahmed","year":"2019"},{"issue":"98","key":"ref16","first-page":"1","article-title":"On the theory of policy gradient methods: Optimality, approximation, and distribution shift","volume":"22","author":"Agarwal","year":"2021","journal-title":"J. Mach. Learn. Res."},{"issue":"282","key":"ref17","first-page":"1","article-title":"On the convergence rates of policy gradient methods","volume":"23","author":"Xiao","year":"2022","journal-title":"J. Mach. Learn. Res."},{"key":"ref18","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v34i04.6021"},{"key":"ref19","doi-asserted-by":"publisher","DOI":"10.1287\/opre.2021.0014"},{"key":"ref20","first-page":"2228","article-title":"On the convergence and sample efficiency of variance-reduced policy gradient method","volume-title":"Proc. Int. Conf. Adv. Neural Inf. Process. Syst.","volume":"34","author":"Zhang","year":"2021"},{"key":"ref21","first-page":"4572","article-title":"Variational policy gradient method for reinforcement learning with general utilities","volume-title":"Proc. Int. Conf. Adv. Neural Inf. Process. Syst.","volume":"33","author":"Zhang","year":"2020"},{"key":"ref22","first-page":"3107","article-title":"Softmax policy gradient methods can take exponential time to converge","volume-title":"Proc. Conf. Learn. Theory","author":"Li","year":"2021"},{"key":"ref23","first-page":"7555","article-title":"Leveraging non-uniformity in first-order non-convex optimization","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Mei","year":"2021"},{"key":"ref24","first-page":"7624","article-title":"An improved analysis of (variance-reduced) policy gradient and natural policy gradient methods","volume-title":"Proc. Int. Conf. Adv. Neural Inf. Process. Syst.","volume":"33","author":"Liu","year":"2020"},{"key":"ref25","first-page":"1910","article-title":"On the global optimum convergence of momentum-based policy gradient","volume-title":"Proc. Int. Conf. Artif. Intell. Statist.","author":"Ding","year":"2022"},{"key":"ref26","article-title":"If MaxEnt RL is the answer, What is the question?","author":"Eysenbach","year":"2019"},{"key":"ref27","doi-asserted-by":"publisher","DOI":"10.1109\/tnn.1998.712192"},{"key":"ref28","first-page":"2","article-title":"Linear convergence of entropy-regularized natural policy gradient with linear function approximation","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Cayci","year":"2021"},{"key":"ref29","doi-asserted-by":"publisher","DOI":"10.1137\/19M1288012"},{"key":"ref30","first-page":"1057","article-title":"Policy gradient methods for reinforcement learning with function approximation","volume-title":"Proc. Int. Conf. Adv. Neural Inf. Process. Syst.","volume":"99","author":"Sutton","year":"1999"},{"key":"ref31","doi-asserted-by":"publisher","DOI":"10.1613\/jair.806"},{"key":"ref32","doi-asserted-by":"publisher","DOI":"10.1007\/BFb0096509"},{"key":"ref33","volume-title":"Stochastic Approximation: A Dynamical Systems Viewpoint","volume":"48","author":"Borkar","year":"2009"},{"key":"ref34","doi-asserted-by":"publisher","DOI":"10.1137\/S1052623497331063"},{"key":"ref35","doi-asserted-by":"publisher","DOI":"10.1007\/BF02218617"},{"key":"ref36","doi-asserted-by":"publisher","DOI":"10.1007\/978-1-4684-9352-8"},{"key":"ref37","first-page":"1117","article-title":"On the almost sure convergence of stochastic gradient descent in non-convex problems","volume-title":"Proc. Int. Conf. Adv. Neural Inf. Process. Syst.","volume":"33","author":"Mertikopoulos","year":"2020"},{"key":"ref38","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v35i12.17300"},{"key":"ref39","first-page":"21130","article-title":"Escaping the gravitational pull of softmax","volume-title":"Proc. Int. Conf. Adv. Neural Inf. Process. Syst.","volume":"33","author":"Mei","year":"2020"},{"issue":"7","key":"ref40","first-page":"2121","article-title":"Adaptive subgradient methods for online learning and stochastic optimization","volume":"12","author":"Duchi","year":"2011","journal-title":"J. Mach. Learn. Res."},{"key":"ref41","article-title":"Adadelta: An adaptive learning rate method","author":"Zeiler","year":"2012"},{"key":"ref42","doi-asserted-by":"publisher","DOI":"10.4135\/9781412983907.n1717"},{"key":"ref43","first-page":"1","article-title":"Adam: A method for stochastic optimization","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Kingma","year":"2015"},{"key":"ref44","volume-title":"Markov Decision Processes: Discrete Stochastic Dynamic Programming","author":"Puterman","year":"2014"},{"key":"ref45","first-page":"1","article-title":"On the convergence of single-call stochastic extra-gradient methods","volume-title":"Proc. Int. Conf. Adv. Neural Inf. Process. Syst.","volume":"32","author":"Hsieh","year":"2019"},{"key":"ref46","first-page":"16223","article-title":"Explore aggressively, update conservatively: Stochastic extragradient methods with variable stepsize scaling","volume-title":"Proc. Int. Conf. Adv. Neural Inf. Process. Syst.","volume":"33","author":"Hsieh","year":"2020"},{"key":"ref47","doi-asserted-by":"publisher","DOI":"10.1007\/s10107-018-1254-8"},{"key":"ref48","doi-asserted-by":"publisher","DOI":"10.1002\/0471200611"},{"key":"ref49","first-page":"2772","article-title":"Bridging the gap between value and policy based reinforcement learning","volume-title":"Proc. Int. Conf. Adv. Neural Inf. Process. Syst.","volume":"30","author":"Nachum","year":"2017"},{"key":"ref50","article-title":"Beyond exact gradients: Convergence of stochastic soft-max policy gradient methods with entropy regularization","author":"Ding","year":"2022"},{"key":"ref51","first-page":"3332","article-title":"A general sample complexity analysis of vanilla policy gradient","volume-title":"Proc. Int. Conf. Artif. Intell. Statist.","author":"Yuan","year":"2022"}],"container-title":["IEEE Transactions on Automatic Control"],"original-title":[],"link":[{"URL":"https:\/\/ieeexplore.ieee.org\/ielam\/9\/11099039\/10882911-aam.pdf","content-type":"application\/pdf","content-version":"am","intended-application":"syndication"},{"URL":"http:\/\/xplorestaging.ieee.org\/ielx8\/9\/11099039\/10882911.pdf?arnumber=10882911","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,1,30]],"date-time":"2026-01-30T21:06:58Z","timestamp":1769807218000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/10882911\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,8]]},"references-count":51,"journal-issue":{"issue":"8"},"URL":"https:\/\/doi.org\/10.1109\/tac.2025.3540965","relation":{},"ISSN":["0018-9286","1558-2523","2334-3303"],"issn-type":[{"value":"0018-9286","type":"print"},{"value":"1558-2523","type":"electronic"},{"value":"2334-3303","type":"electronic"}],"subject":[],"published":{"date-parts":[[2025,8]]}}}