{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,2,21]],"date-time":"2025-02-21T23:48:18Z","timestamp":1740181698378,"version":"3.37.3"},"reference-count":53,"publisher":"Institute of Electrical and Electronics Engineers (IEEE)","issue":"2","funder":[{"DOI":"10.13039\/100006754","name":"Army Research Laboratory","doi-asserted-by":"publisher","award":["W911NF-16-2-0008"],"award-info":[{"award-number":["W911NF-16-2-0008"]}],"id":[{"id":"10.13039\/100006754","id-type":"DOI","asserted-by":"publisher"}]},{"name":"SMART Scholarship for Service"},{"name":"ARL DCIST CRA"},{"name":"ORAU Journeyman Fellowship"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["IEEE J. Sel. Areas Inf. Theory"],"published-print":{"date-parts":[[2021,6]]},"DOI":"10.1109\/jsait.2021.3081108","type":"journal-article","created":{"date-parts":[[2021,5,17]],"date-time":"2021-05-17T21:48:27Z","timestamp":1621288107000},"page":"611-626","source":"Crossref","is-referenced-by-count":3,"title":["Cautious Reinforcement Learning via Distributional Risk in the Dual Domain"],"prefix":"10.1109","volume":"2","author":[{"given":"Junyu","family":"Zhang","sequence":"first","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0002-8807-2695","authenticated-orcid":false,"given":"Amrit Singh","family":"Bedi","sequence":"additional","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0002-2101-9507","authenticated-orcid":false,"given":"Mengdi","family":"Wang","sequence":"additional","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0003-2447-2873","authenticated-orcid":false,"given":"Alec","family":"Koppel","sequence":"additional","affiliation":[]}],"member":"263","reference":[{"journal-title":"Stochastic Optimal Control The Discrete-Time Case","year":"2004","author":"bertsekas","key":"ref39"},{"key":"ref38","doi-asserted-by":"publisher","DOI":"10.1137\/140969221"},{"key":"ref33","first-page":"449","article-title":"A distributional perspective on reinforcement learning","author":"bellemare","year":"2017","journal-title":"Proc 34th Int Conf Mach Learn (ICML)"},{"key":"ref32","doi-asserted-by":"publisher","DOI":"10.1561\/2200000049"},{"key":"ref31","first-page":"1073","article-title":"A block coordinate ascent algorithm for mean-variance optimization","author":"xie","year":"2018","journal-title":"Proc 32nd Int Conf Neural Inf Process Syst"},{"key":"ref30","doi-asserted-by":"publisher","DOI":"10.1007\/s10994-016-5569-5"},{"key":"ref37","first-page":"8103","article-title":"A Lyapunov-based approach to safe reinforcement learning","author":"chow","year":"2018","journal-title":"Proc 32nd Int Conf Neural Inf Process Syst"},{"key":"ref36","first-page":"5251","article-title":"Nonlinear distributional gradient temporal-difference learning","author":"qu","year":"2019","journal-title":"Proc Int Conf Mach Learn (ICML)"},{"key":"ref35","first-page":"6190","article-title":"Fully parameterized quantile function for distributional reinforcement learning","author":"yang","year":"2019","journal-title":"Advances in Neural Information Processing Systems (NIPS)"},{"key":"ref34","first-page":"1096","article-title":"Implicit quantile networks for distributional reinforcement learning","volume":"80","author":"dabney","year":"2018","journal-title":"Proc 34th Int Conf Mach Learn (ICML)"},{"article-title":"Mean-variance optimization in Markov decision processes","year":"2011","author":"mannor","key":"ref28"},{"key":"ref27","doi-asserted-by":"publisher","DOI":"10.2307\/3213832"},{"article-title":"Policy gradients with variance related risk criteria","year":"2012","author":"di castro","key":"ref29"},{"key":"ref2","doi-asserted-by":"publisher","DOI":"10.1109\/TNN.1998.712192"},{"key":"ref1","article-title":"’Beyond cumulative returns via reinforcement learning over state-action occupancy measures","author":"zhang","year":"2021","journal-title":"Proc IEEE Amer Control Conf (ACC)"},{"article-title":"Risk-averse learning by temporal difference methods","year":"2020","author":"kose","key":"ref20"},{"key":"ref22","doi-asserted-by":"publisher","DOI":"10.1109\/CDC.2003.1272362"},{"key":"ref21","first-page":"1468","article-title":"Policy gradient for coherent risk measures","author":"tamar","year":"2015","journal-title":"Advances in Neural Information Processing Systems (NIPS)"},{"key":"ref24","doi-asserted-by":"publisher","DOI":"10.1109\/CDC40024.2019.9029423"},{"key":"ref23","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-11662-4_12"},{"key":"ref26","doi-asserted-by":"publisher","DOI":"10.1137\/050622328"},{"key":"ref25","first-page":"3121","article-title":"Convergent policy optimization for safe reinforcement learning","volume":"32","author":"yu","year":"2019","journal-title":"Advances in Neural Information Processing Systems (NIPS)"},{"key":"ref50","doi-asserted-by":"publisher","DOI":"10.1287\/moor.2019.1000"},{"article-title":"Projection onto the probability simplex: An efficient algorithm with a simple proof, and an application","year":"2013","author":"wang","key":"ref51"},{"key":"ref53","first-page":"164","article-title":"A universal algorithm for variational inequalities adaptive to smoothness and noise","author":"bach","year":"2019","journal-title":"Proc 32nd Conf Learn Theory (COLT)"},{"key":"ref52","first-page":"1651","article-title":"Policy gradients with variance related risk criteria","author":"tamar","year":"2012","journal-title":"Proc 29th Int Conf Mach Learn"},{"key":"ref10","doi-asserted-by":"publisher","DOI":"10.1126\/science.1134239"},{"key":"ref11","first-page":"22","article-title":"Constrained policy optimization","author":"achiam","year":"2017","journal-title":"Proc 34th Int Conf Mach Learn (ICML)"},{"journal-title":"Markov Decision Processes Discrete Stochastic Dynamic Programming","year":"2014","author":"puterman","key":"ref40"},{"key":"ref12","doi-asserted-by":"publisher","DOI":"10.2307\/2975974"},{"key":"ref13","doi-asserted-by":"publisher","DOI":"10.1016\/S0378-4266(02)00271-6"},{"key":"ref14","doi-asserted-by":"publisher","DOI":"10.1111\/1467-9965.00068"},{"key":"ref15","first-page":"6070","article-title":"Risk-constrained reinforcement learning with percentile risk criteria","volume":"18","author":"chow","year":"2017","journal-title":"J Mach Learn Res"},{"key":"ref16","doi-asserted-by":"publisher","DOI":"10.1287\/moor.2017.0872"},{"key":"ref17","doi-asserted-by":"publisher","DOI":"10.1007\/s00780-014-0234-y"},{"key":"ref18","doi-asserted-by":"publisher","DOI":"10.1007\/s10107-010-0393-3"},{"article-title":"Being optimistic to be conservative: Quickly learning a CVaR policy","year":"2019","author":"keramati","key":"ref19"},{"article-title":"Playing Atari with deep reinforcement learning","year":"2013","author":"mnih","key":"ref4"},{"key":"ref3","doi-asserted-by":"publisher","DOI":"10.1145\/2507157.2508063"},{"journal-title":"High-dimensional continuous control using generalized advantage estimation","year":"2016","author":"schulman","key":"ref6"},{"key":"ref5","first-page":"2","author":"vinyals","year":"2019","journal-title":"Alphastar Mastering the real-time strategy game StarCraft II"},{"key":"ref8","doi-asserted-by":"publisher","DOI":"10.1007\/s10339-011-0392-1"},{"key":"ref7","doi-asserted-by":"publisher","DOI":"10.1007\/s00170-008-1715-y"},{"article-title":"Primal-dual $pi$\n learning: Sample complexity and sublinear run time for ergodic Markov decision problems","year":"2017","author":"wang","key":"ref49"},{"key":"ref9","doi-asserted-by":"publisher","DOI":"10.1016\/S0304-3932(03)00029-1"},{"article-title":"Scalable bilinear $pi$\n learning using state and action features","year":"2018","author":"chen","key":"ref46"},{"article-title":"Stochastic primal–dual methods and sample complexity of reinforcement learning","year":"2016","author":"chen","key":"ref45"},{"key":"ref48","doi-asserted-by":"publisher","DOI":"10.1016\/S0167-6377(02)00231-6"},{"key":"ref47","first-page":"272","article-title":"Exponentiated gradient methods for reinforcement learning","author":"precup","year":"1997","journal-title":"Proc Int Conf Machine Learn (ICML)"},{"key":"ref42","doi-asserted-by":"publisher","DOI":"10.1287\/mnsc.6.3.259"},{"key":"ref41","doi-asserted-by":"publisher","DOI":"10.1287\/opre.51.6.850.24925"},{"key":"ref44","first-page":"1569","article-title":"Stable dual dynamic programming","author":"wang","year":"2008","journal-title":"Advances in neural information processing systems"},{"key":"ref43","doi-asserted-by":"publisher","DOI":"10.1287\/mnsc.10.1.98"}],"container-title":["IEEE Journal on Selected Areas in Information Theory"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx7\/8700143\/9459757\/09432963.pdf?arnumber=9432963","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2021,11,8]],"date-time":"2021-11-08T22:36:58Z","timestamp":1636411018000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/9432963\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2021,6]]},"references-count":53,"journal-issue":{"issue":"2"},"URL":"https:\/\/doi.org\/10.1109\/jsait.2021.3081108","relation":{},"ISSN":["2641-8770"],"issn-type":[{"type":"electronic","value":"2641-8770"}],"subject":[],"published":{"date-parts":[[2021,6]]}}}