{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,5,3]],"date-time":"2026-05-03T06:02:40Z","timestamp":1777788160544,"version":"3.51.4"},"reference-count":41,"publisher":"Institute of Electrical and Electronics Engineers (IEEE)","issue":"6","license":[{"start":{"date-parts":[[2026,6,1]],"date-time":"2026-06-01T00:00:00Z","timestamp":1780272000000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/ieeexplore.ieee.org\/Xplorehelp\/downloads\/license-information\/IEEE.html"},{"start":{"date-parts":[[2026,6,1]],"date-time":"2026-06-01T00:00:00Z","timestamp":1780272000000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2026,6,1]],"date-time":"2026-06-01T00:00:00Z","timestamp":1780272000000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"}],"funder":[{"name":"State Key Laboratory (SKL) of Biobased Transportation Fuel Technology"},{"name":"Industrial Technology Development"},{"name":"Cyber-Physical Control System Design","award":["K20243390"],"award-info":[{"award-number":["K20243390"]}]}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["IEEE Robot. Autom. Lett."],"published-print":{"date-parts":[[2026,6]]},"DOI":"10.1109\/lra.2026.3678143","type":"journal-article","created":{"date-parts":[[2026,3,26]],"date-time":"2026-03-26T19:51:48Z","timestamp":1774554708000},"page":"6648-6655","source":"Crossref","is-referenced-by-count":0,"title":["Where-to-Learn: Analytical Policy Gradient Directed Exploration for On-Policy Robotic Reinforcement Learning"],"prefix":"10.1109","volume":"11","author":[{"ORCID":"https:\/\/orcid.org\/0009-0003-8979-2162","authenticated-orcid":false,"given":"Leixin","family":"Chang","sequence":"first","affiliation":[{"name":"Zhejiang University-University of Illinois Urbana-Champaign Institute (ZJUI), Haining, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0009-0002-0501-0390","authenticated-orcid":false,"given":"Xinchen","family":"Yao","sequence":"additional","affiliation":[{"name":"Zhejiang University-University of Illinois Urbana-Champaign Institute (ZJUI), Haining, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0009-0008-6879-6259","authenticated-orcid":false,"given":"Ben","family":"Liu","sequence":"additional","affiliation":[{"name":"Southern University of Science and Technology, Shenzhen, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-3294-0879","authenticated-orcid":false,"given":"Liangjing","family":"Yang","sequence":"additional","affiliation":[{"name":"Zhejiang University-University of Illinois Urbana-Champaign Institute (ZJUI), Haining, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-4252-8693","authenticated-orcid":false,"given":"Hua","family":"Chen","sequence":"additional","affiliation":[{"name":"Zhejiang University-University of Illinois Urbana-Champaign Institute (ZJUI), Haining, China"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"263","reference":[{"key":"ref1","doi-asserted-by":"publisher","DOI":"10.1126\/scirobotics.aau5872"},{"key":"ref2","doi-asserted-by":"publisher","DOI":"10.1177\/0278364913495721"},{"key":"ref3","first-page":"91","article-title":"Learning to walk in minutes using massively parallel deep reinforcement learning","volume-title":"Proc. 5th Conf. Robot Learn.","volume":"164","author":"Rudin","year":"2022"},{"key":"ref4","doi-asserted-by":"publisher","DOI":"10.1145\/3197517.3201311"},{"key":"ref5","article-title":"Proximal policy optimization algorithms","author":"Schulman","year":"2017"},{"key":"ref6","doi-asserted-by":"publisher","DOI":"10.1007\/BF00992696"},{"key":"ref7","first-page":"1861","article-title":"Soft actor-critic: Off-policy maximum entropy deep reinforcement learning with a stochastic actor","volume-title":"Proc. 35th Int. Conf. Mach. Learn.","volume":"80","author":"Haarnoja","year":"2018"},{"key":"ref8","article-title":"Parameter space noise for exploration","author":"Plappert","year":"2018"},{"key":"ref9","article-title":"Model-ensemble trust-region policy optimization","author":"Kurutach","year":"2018"},{"key":"ref10","first-page":"28954","article-title":"COMBO: Conservative offline model-based policy optimization","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","volume":"34","author":"Yu","year":"2021"},{"key":"ref11","article-title":"Model-based value estimation for efficient model-free reinforcement learning","author":"Feinberg","year":"2018"},{"key":"ref12","first-page":"465","article-title":"PILCO: A model-based and data-efficient approach to policy search","volume-title":"Proc. 28th Int. Conf. Mach. Learn.","author":"Deisenroth","year":"2011"},{"key":"ref13","first-page":"1433","article-title":"Maximum entropy inverse reinforcement learning","volume-title":"Proc. 23rd Nat. Conf. Artif. Intell.","volume":"3","author":"Ziebart","year":"2008"},{"key":"ref14","article-title":"Exploration by random network distillation","author":"Burda","year":"2019"},{"key":"ref15","doi-asserted-by":"publisher","DOI":"10.1109\/CVPRW.2017.70"},{"key":"ref16","first-page":"1117","article-title":"VIME: Variational information maximizing exploration","author":"Houthooft","year":"2016"},{"key":"ref17","article-title":"Brax - a differentiable physics engine for large scale rigid body simulation","volume-title":"Proc. Neural Inf. Process. Syst. Track Datasets Benchmarks","volume":"1","author":"Freeman"},{"key":"ref18","volume-title":"A Focused Backpropagation Algorithm for Temporal Pattern Recognition","author":"Mozer","year":"1995"},{"key":"ref19","article-title":"Gradients are not all you need","author":"Metz","year":"2022"},{"key":"ref20","first-page":"258","article-title":"Learning quadruped locomotion using differentiable simulation","volume-title":"Proc. 8th Conf. Rob. Learn.","volume":"270","author":"Song","year":"2025"},{"key":"ref21","doi-asserted-by":"publisher","DOI":"10.1109\/ICRA55743.2025.11127448"},{"key":"ref22","first-page":"3665","article-title":"Learning deployable locomotion control via differentiable simulation","volume-title":"Proc. 9th Conf. Robot Learn.","volume":"305","author":"Schwarke","year":"2025"},{"key":"ref23","doi-asserted-by":"publisher","DOI":"10.1038\/s42256-025-01048-0"},{"key":"ref24","article-title":"Accelerated policy learning with parallel differentiable simulation","author":"Xu","year":"2022"},{"key":"ref25","article-title":"Model-augmented actor-critic: Backpropagating through paths","author":"Clavera","year":"2020"},{"key":"ref26","article-title":"Learning robot control: From reinforcement learning to differentiable simulation","author":"Song","year":"2024"},{"key":"ref27","first-page":"20668","article-title":"Do differentiable simulators give better policy gradients","volume-title":"Proc. 39th Int. Conf. Mach. Learn.","volume":"162","author":"Suh","year":"2022"},{"key":"ref28","first-page":"1587","article-title":"Addressing function approximation error in actor-critic methods","volume-title":"Proc. 35th Int. Conf. Mach. Learn.","volume":"80","author":"Fujimoto","year":"2018"},{"key":"ref29","article-title":"Advantage-weighted regression: Simple and scalable off-policy reinforcement learning","author":"Peng","year":"2019"},{"key":"ref30","doi-asserted-by":"publisher","DOI":"10.1145\/122344.122377"},{"key":"ref31","first-page":"101","article-title":"Model-based reinforcement learning with an approximate, learned model","volume-title":"Proc. 9th Yale Workshop Adaptive Learn. Syst.","author":"Kuvayev","year":"1996"},{"key":"ref32","first-page":"12498","article-title":"When to trust your model: Model-based policy optimization","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","volume":"32","author":"Janner","year":"2019"},{"key":"ref33","article-title":"Probabilistic planning with sequential Monte Carlo methods","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Pich","year":"2019"},{"key":"ref34","first-page":"4759","article-title":"Deep reinforcement learning in a handful of trials using probabilistic dynamics models","volume-title":"Proc. 32nd Int. Conf. Neural Inf. Process. Syst.","author":"Chua","year":"2018"},{"key":"ref35","first-page":"8661","article-title":"Efficient differentiable simulation of articulated bodies","volume-title":"Proc. 38th Int. Conf. Mach. Learn.","volume":"139","author":"Qiao","year":"2021"},{"key":"ref36","article-title":"Differentiable physics simulations with contacts: Do they have correct gradients w.r.t. position, velocity and control?","author":"Zhong","year":"2022"},{"key":"ref37","first-page":"267","article-title":"Approximately optimal approximate reinforcement learning","volume-title":"Proc. 19th Int. Conf. Mach. Learn.","author":"Kakade","year":"2002"},{"key":"ref38","first-page":"1889","article-title":"Trust region policy optimization","volume-title":"Proc. 32nd Int. Conf. Mach. Learn.","volume":"37","author":"Schulman","year":"2015"},{"key":"ref39","doi-asserted-by":"publisher","DOI":"10.15607\/RSS.2025.XXI.020"},{"key":"ref40","article-title":"MuJoCo XLA (MJX) - MuJoCo documentation","year":"2024"},{"key":"ref41","doi-asserted-by":"publisher","DOI":"10.1109\/IROS.2012.6386109"}],"container-title":["IEEE Robotics and Automation Letters"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx8\/7083369\/11481819\/11456663.pdf?arnumber=11456663","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,4,29]],"date-time":"2026-04-29T19:52:59Z","timestamp":1777492379000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/11456663\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026,6]]},"references-count":41,"journal-issue":{"issue":"6"},"URL":"https:\/\/doi.org\/10.1109\/lra.2026.3678143","relation":{},"ISSN":["2377-3766","2377-3774"],"issn-type":[{"value":"2377-3766","type":"electronic"},{"value":"2377-3774","type":"electronic"}],"subject":[],"published":{"date-parts":[[2026,6]]}}}