{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2024,10,31]],"date-time":"2024-10-31T02:52:25Z","timestamp":1730343145110,"version":"3.28.0"},"reference-count":36,"publisher":"IEEE","license":[{"start":{"date-parts":[[2023,6,13]],"date-time":"2023-06-13T00:00:00Z","timestamp":1686614400000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2023,6,13]],"date-time":"2023-06-13T00:00:00Z","timestamp":1686614400000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2023,6,13]]},"DOI":"10.23919\/ecc57647.2023.10178123","type":"proceedings-article","created":{"date-parts":[[2023,8,1]],"date-time":"2023-08-01T18:00:50Z","timestamp":1690912850000},"page":"1-8","source":"Crossref","is-referenced-by-count":0,"title":["Local Analysis of Entropy-Regularized Stochastic Soft-Max Policy Gradient Methods"],"prefix":"10.23919","author":[{"given":"Yuhao","family":"Ding","sequence":"first","affiliation":[{"name":"University of California at Berkeley,Department of Industrail Engineering and Operations Research"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Junzi","family":"Zhang","sequence":"additional","affiliation":[{"name":"Citadel Securities (work done prior to joining Citadel Securities),Chicago,IL,USA,60603"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Javad","family":"Lavaei","sequence":"additional","affiliation":[{"name":"University of California at Berkeley,Department of Industrail Engineering and Operations Research"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"263","reference":[{"key":"ref1","doi-asserted-by":"publisher","DOI":"10.1080\/09540099108946587"},{"key":"ref2","first-page":"1928","article-title":"Asynchronous methods for deep reinforcement learning","volume-title":"In International conference on machine learning","author":"Mnih"},{"key":"ref3","first-page":"1861","article-title":"Soft actor-critic: Off-policy maximum entropy deep reinforcement learning with a stochastic actor","volume-title":"In International conference on machine learning","author":"Haarnoja"},{"key":"ref4","article-title":"Combining policy gradient and q-learning","author":"O\u2019Donoghue","year":"2016","journal-title":"arXiv preprint arXiv:1611.01626"},{"key":"ref5","first-page":"1352","article-title":"Reinforcement learning with deep energy-based policies","volume-title":"In International Conference on Machine Learning","author":"Haarnoja"},{"volume-title":"Teac: Intergrating trust region and max entropy actor critic for continuous control","year":"2020","author":"Zang","key":"ref6"},{"issue":"98","key":"ref7","first-page":"1","article-title":"On the theory of policy gradient methods: Optimality, approximation, and distribution shift","volume":"22","author":"Agarwal","year":"2021","journal-title":"J. Mach. Learn. Res."},{"key":"ref8","doi-asserted-by":"publisher","DOI":"10.1007\/s10107-022-01816-5"},{"key":"ref9","article-title":"On the convergence rates of policy gradient methods","author":"Xiao","year":"2022","journal-title":"arXiv preprint arXiv:2201.07443"},{"key":"ref10","first-page":"6820","article-title":"On the global convergence rates of softmax policy gradient methods","volume-title":"In International Conference on Machine Learning","author":"Mei"},{"key":"ref11","doi-asserted-by":"publisher","DOI":"10.1287\/opre.2021.2151"},{"key":"ref12","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v34i04.6021"},{"key":"ref13","article-title":"Global optimality guarantees for policy gradient methods","author":"Bhandari","year":"2019","journal-title":"arXiv preprint arXiv:1906.01786"},{"key":"ref14","first-page":"2228","article-title":"On the convergence and sample efficiency of variance-reduced policy gradient method","volume":"34","author":"Zhang","year":"2021","journal-title":"Advances in Neural Information Processing Systems"},{"key":"ref15","first-page":"4572","article-title":"Variational policy gradient method for reinforcement learning with general utilities","volume":"33","author":"Zhang","year":"2020","journal-title":"Advances in Neural Information Processing Systems"},{"key":"ref16","first-page":"3107","article-title":"Softmax policy gradient methods can take exponential time to converge","volume-title":"In Conference on Learning Theory","author":"Li"},{"key":"ref17","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v37i9.26299"},{"key":"ref18","first-page":"1887","article-title":"A dual approach to constrained markov decision processes with entropy regularization","volume-title":"In International Conference on Artificial Intelligence and Statistics","author":"Ying"},{"article-title":"A cmdp-within-online framework for meta-safe reinforcement learning","volume-title":"In The Eleventh International Conference on Learning Representations","author":"Khattar","key":"ref19"},{"article-title":"Leveraging non-uniformity in first-order non-convex optimization","volume-title":"International Conference on Machine Learning","author":"Mei","key":"ref20"},{"key":"ref21","article-title":"An improved analysis of (variance-reduced) policy gradient and natural policy gradient methods","volume":"33","author":"Liu","year":"2020","journal-title":"Advances in Neural Information Processing Systems"},{"key":"ref22","first-page":"1910","article-title":"On the global optimum convergence of momentum-based policy gradient","volume-title":"In International Conference on Artificial Intelligence and Statistics","author":"Ding"},{"key":"ref23","first-page":"1999","article-title":"Beyond variance reduction: Understanding the true impact of baselines on policy optimization","volume-title":"In International Conference on Machine Learning","author":"Chung"},{"key":"ref24","article-title":"Understanding the effect of stochasticity in policy optimization","volume":"34","author":"Mei","year":"2021","journal-title":"Advances in Neural Information Processing Systems"},{"key":"ref25","article-title":"Equivalence between policy gradients and soft q-learning","volume":"abs\/1704.06440","author":"Schulman","year":"2017","journal-title":"CoRR"},{"key":"ref26","article-title":"If MaxEnt RL is the answer, what is the question?","author":"Eysenbach","year":"2019","journal-title":"arXiv preprint arXiv:1910.01913"},{"key":"ref27","doi-asserted-by":"publisher","DOI":"10.1109\/tnn.1998.712192"},{"key":"ref28","doi-asserted-by":"crossref","DOI":"10.1137\/22M1540156","article-title":"Linear convergence of entropy-regularized natural policy gradient with linear function approximation","volume-title":"In International conference on machine learning","author":"Cayci"},{"key":"ref29","doi-asserted-by":"publisher","DOI":"10.1137\/19M1288012"},{"key":"ref30","article-title":"On the convergence of single-call stochastic extra-gradient methods","volume":"32","author":"Hsieh","year":"2019","journal-title":"Advances in Neural Information Processing Systems"},{"key":"ref31","first-page":"16223","article-title":"Explore aggressively, update conservatively: Stochastic extragradient methods with variable stepsize scaling","volume":"33","author":"Hsieh","year":"2020","journal-title":"Advances in Neural Information Processing Systems"},{"key":"ref32","doi-asserted-by":"publisher","DOI":"10.1007\/s10107-018-1254-8"},{"key":"ref33","first-page":"1117","article-title":"On the almost sure convergence of stochastic gradient descent in nonconvex problems","volume":"33","author":"Mertikopoulos","year":"2020","journal-title":"Advances in Neural Information Processing Systems"},{"key":"ref34","first-page":"6820","article-title":"On the global convergence rates of softmax policy gradient methods","volume-title":"In International Conference on Machine Learning","author":"Mei"},{"key":"ref35","doi-asserted-by":"publisher","DOI":"10.1002\/0471200611"},{"key":"ref36","article-title":"Bridging the gap between value and policy based reinforcement learning","volume":"30","author":"Nachum","year":"2017","journal-title":"Advances in neural information processing systems"}],"event":{"name":"2023 European Control Conference (ECC)","start":{"date-parts":[[2023,6,13]]},"location":"Bucharest, Romania","end":{"date-parts":[[2023,6,16]]}},"container-title":["2023 European Control Conference (ECC)"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx7\/10178092\/10178115\/10178123.pdf?arnumber=10178123","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,10,25]],"date-time":"2024-10-25T14:04:49Z","timestamp":1729865089000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/10178123\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023,6,13]]},"references-count":36,"URL":"https:\/\/doi.org\/10.23919\/ecc57647.2023.10178123","relation":{},"subject":[],"published":{"date-parts":[[2023,6,13]]}}}