{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,3,7]],"date-time":"2026-03-07T19:54:02Z","timestamp":1772913242704,"version":"3.50.1"},"reference-count":82,"publisher":"Institute of Electrical and Electronics Engineers (IEEE)","issue":"2","license":[{"start":{"date-parts":[[2025,4,1]],"date-time":"2025-04-01T00:00:00Z","timestamp":1743465600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/ieeexplore.ieee.org\/Xplorehelp\/downloads\/license-information\/IEEE.html"},{"start":{"date-parts":[[2025,4,1]],"date-time":"2025-04-01T00:00:00Z","timestamp":1743465600000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2025,4,1]],"date-time":"2025-04-01T00:00:00Z","timestamp":1743465600000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"}],"funder":[{"name":"National Key Research and Development Program of China","award":["2020YFB1600200"],"award-info":[{"award-number":["2020YFB1600200"]}]},{"name":"Tsinghua-Toyota Joint Research Fund"},{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["52221005"],"award-info":[{"award-number":["52221005"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["IEEE Trans. Evol. Computat."],"published-print":{"date-parts":[[2025,4]]},"DOI":"10.1109\/tevc.2025.3529503","type":"journal-article","created":{"date-parts":[[2025,1,14]],"date-time":"2025-01-14T19:52:30Z","timestamp":1736884350000},"page":"555-569","source":"Crossref","is-referenced-by-count":4,"title":["Zeroth-Order Actor\u2013Critic: An Evolutionary Framework for Sequential Decision Problems"],"prefix":"10.1109","volume":"29","author":[{"ORCID":"https:\/\/orcid.org\/0009-0006-1940-3573","authenticated-orcid":false,"given":"Yuheng","family":"Lei","sequence":"first","affiliation":[{"name":"Department of Computer Science, The University of Hong Kong, Hong Kong, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-1539-472X","authenticated-orcid":false,"given":"Yao","family":"Lyu","sequence":"additional","affiliation":[{"name":"School of Vehicle and Mobility, Tsinghua University, Beijing, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-1246-4860","authenticated-orcid":false,"given":"Guojian","family":"Zhan","sequence":"additional","affiliation":[{"name":"School of Vehicle and Mobility, Tsinghua University, Beijing, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Tao","family":"Zhang","sequence":"additional","affiliation":[{"name":"SunRising AI Ltd., Beijing, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Jiangtao","family":"Li","sequence":"additional","affiliation":[{"name":"SunRising AI Ltd., Beijing, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-0282-8621","authenticated-orcid":false,"given":"Jianyu","family":"Chen","sequence":"additional","affiliation":[{"name":"Institute for Interdisciplinary Information Sciences, Tsinghua University, Beijing, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-4923-3633","authenticated-orcid":false,"given":"Shengbo Eben","family":"Li","sequence":"additional","affiliation":[{"name":"School of Vehicle and Mobility, Tsinghua University, Beijing, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-5160-1365","authenticated-orcid":false,"given":"Sifa","family":"Zheng","sequence":"additional","affiliation":[{"name":"School of Vehicle and Mobility, Tsinghua University, Beijing, China"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"263","reference":[{"key":"ref1","doi-asserted-by":"publisher","DOI":"10.1007\/978-981-19-7784-8"},{"key":"ref2","doi-asserted-by":"publisher","DOI":"10.1038\/nature14236"},{"key":"ref3","doi-asserted-by":"publisher","DOI":"10.1109\/TCYB.2022.3163816"},{"key":"ref4","first-page":"91","article-title":"From pixels to legs: Hierarchical learning of quadruped locomotion","volume-title":"Proc. Conf. Robot Learn.","author":"Jain"},{"key":"ref5","doi-asserted-by":"publisher","DOI":"10.1016\/j.neunet.2019.01.011"},{"key":"ref6","doi-asserted-by":"publisher","DOI":"10.1109\/CEC.2014.6900316"},{"key":"ref7","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-31129-2_27"},{"key":"ref8","doi-asserted-by":"publisher","DOI":"10.31763\/ijrcs.v2i2.476"},{"key":"ref9","doi-asserted-by":"publisher","DOI":"10.23919\/ACC50511.2021.9483417"},{"key":"ref10","first-page":"2916","article-title":"Derivative-free methods for policy optimization: Guarantees for linear quadratic systems","volume-title":"Proc. 22nd Int. Conf. Artif. Intell. Statist.","author":"Malik"},{"key":"ref11","doi-asserted-by":"publisher","DOI":"10.1109\/LCSYS.2020.3006256"},{"key":"ref12","first-page":"1805","article-title":"Simple random search of static linear policies is competitive for reinforcement learning","volume-title":"Proc. 32nd Int. Conf. Neural Inf. Process. Syst.","author":"Mania"},{"key":"ref13","doi-asserted-by":"publisher","DOI":"10.1109\/TEVC.2023.3338952"},{"key":"ref14","doi-asserted-by":"publisher","DOI":"10.1109\/TPWRS.2021.3095179"},{"key":"ref15","doi-asserted-by":"publisher","DOI":"10.1109\/TCST.2021.3123605"},{"key":"ref16","first-page":"1205","article-title":"Optimal cost design for model predictive control","volume-title":"Proc. 3rd Annu. Learn. Dyn. Control","author":"Jain"},{"key":"ref17","doi-asserted-by":"publisher","DOI":"10.1016\/j.robot.2018.03.006"},{"key":"ref18","doi-asserted-by":"publisher","DOI":"10.1016\/j.engappai.2017.07.005"},{"key":"ref19","doi-asserted-by":"publisher","DOI":"10.1115\/DSCC2017-5209"},{"key":"ref20","doi-asserted-by":"publisher","DOI":"10.1016\/j.robot.2020.103568"},{"key":"ref21","doi-asserted-by":"publisher","DOI":"10.1109\/TCYB.2022.3180664"},{"key":"ref22","doi-asserted-by":"publisher","DOI":"10.1145\/3321707.3321817"},{"key":"ref23","doi-asserted-by":"publisher","DOI":"10.1109\/TEVC.2021.3083362"},{"key":"ref24","article-title":"Deep neuroevolution: Genetic algorithms are a competitive alternative for training deep neural networks for reinforcement learning","author":"Such","year":"2017","journal-title":"arXiv:1712.06567"},{"key":"ref25","article-title":"ES-ENAS: Blackbox optimization over hybrid spaces via combinatorial and continuous evolution","author":"Song","year":"2021","journal-title":"arXiv:2101.07415"},{"key":"ref26","article-title":"Evolution strategies as a scalable alternative to reinforcement learning","author":"Salimans","year":"2017","journal-title":"arXiv:1703.03864"},{"key":"ref27","doi-asserted-by":"publisher","DOI":"10.1038\/s42256-018-0006-z"},{"key":"ref28","doi-asserted-by":"publisher","DOI":"10.1016\/j.neunet.2009.12.004"},{"key":"ref29","first-page":"970","article-title":"Structured evolution with compact architectures for scalable policy optimization","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Choromanski"},{"key":"ref30","first-page":"1","article-title":"Backpropagation through the void: Optimizing control variates for black-box gradient estimation","volume-title":"Proc. Int. Conf. Learn. Represent.","author":"Grathwohl"},{"key":"ref31","first-page":"4264","article-title":"Guided evolutionary strategies: Augmenting random search with surrogate gradients","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Maheswaranathan"},{"key":"ref32","first-page":"10299","article-title":"From complexity to simplicity: Adaptive ES-active subspaces for blackbox optimization","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","volume":"32","author":"Choromanski"},{"key":"ref33","first-page":"1","article-title":"Learning to guide random search","volume-title":"Proc. Int. Conf. Learn. Represent.","author":"Sener"},{"key":"ref34","volume-title":"Reinforcement Learning: An Introduction","author":"Sutton","year":"2018"},{"key":"ref35","first-page":"1928","article-title":"Asynchronous methods for deep reinforcement learning","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Mnih"},{"key":"ref36","article-title":"Proximal policy optimization algorithms","author":"Schulman","year":"2017","journal-title":"arXiv:1707.06347"},{"key":"ref37","doi-asserted-by":"publisher","DOI":"10.1007\/BF00992696"},{"key":"ref38","first-page":"1196","article-title":"Evolution-guided policy gradient in reinforcement learning","volume-title":"Proc. 32nd Int. Conf. Neural Inf. Process. Syst.","author":"Khadka"},{"key":"ref39","first-page":"1","article-title":"CEM-RL: Combining evolutionary and gradient-based methods for policy search","volume-title":"Proc. Int. Conf. Learn. Represent.","author":"Pourchot"},{"key":"ref40","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v34i04.5728"},{"key":"ref41","first-page":"1","article-title":"Parameter space noise for exploration","volume-title":"Proc. Int. Conf. Learn. Represent.","author":"Plappert"},{"key":"ref42","first-page":"1","article-title":"Noisy networks for exploration","volume-title":"Proc. Int. Conf. Learn. Represent.","author":"Fortunato"},{"key":"ref43","doi-asserted-by":"publisher","DOI":"10.5555\/3291168.3291210"},{"key":"ref44","article-title":"Gymnasium: A standard interface for reinforcement learning environments","author":"Towers","year":"2024","journal-title":"arXiv:2407.17032"},{"key":"ref45","first-page":"387","article-title":"Deterministic policy gradient algorithms","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Silver"},{"issue":"1","key":"ref46","first-page":"949","article-title":"Natural evolution strategies","volume":"15","author":"Wierstra","year":"2014","journal-title":"J. Mach. Learn. Res."},{"key":"ref47","doi-asserted-by":"crossref","DOI":"10.1007\/978-1-4757-4321-0","volume":"133","author":"Rubinstein","year":"2004","journal-title":"The Cross-Entropy Method: A Unified Approach to Combinatorial Optimization, Monte-Carlo Simulation, and Machine Learning"},{"key":"ref48","doi-asserted-by":"publisher","DOI":"10.1162\/106365601750190398"},{"key":"ref49","doi-asserted-by":"publisher","DOI":"10.1109\/ICRA40945.2020.9197080"},{"key":"ref50","doi-asserted-by":"publisher","DOI":"10.1109\/TEVC.2008.924423"},{"key":"ref51","first-page":"1","article-title":"Continuous control with deep reinforcement learning","volume-title":"Proc. Int. Conf. Learn. Represent.","author":"Lillicrap"},{"key":"ref52","first-page":"1587","article-title":"Addressing function approximation error in actor-critic methods","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Fujimoto"},{"key":"ref53","first-page":"1634","article-title":"Smooth exploration for robotic reinforcement learning","volume-title":"Proc. Conf. Robot Learn.","author":"Raffin"},{"key":"ref54","doi-asserted-by":"publisher","DOI":"10.1007\/s10208-021-09513-z"},{"key":"ref55","doi-asserted-by":"publisher","DOI":"10.1145\/3205455.3205474"},{"key":"ref56","first-page":"1057","article-title":"Policy gradient methods for reinforcement learning with function approximation","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","author":"Sutton"},{"key":"ref57","first-page":"12535","article-title":"On-policy deep reinforcement learning for the average-reward criterion","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Zhang"},{"key":"ref58","article-title":"High-dimensional continuous control using generalized advantage estimation","author":"Schulman","year":"2015","journal-title":"arXiv:1506.02438"},{"key":"ref59","first-page":"262","article-title":"Analysis and improvement of policy gradient estimation","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","volume":"24","author":"Zhao"},{"key":"ref60","first-page":"36014","article-title":"Global convergence and stability of stochastic gradient descent","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","volume":"35","author":"Patel"},{"key":"ref61","doi-asserted-by":"publisher","DOI":"10.1016\/0893-6080(91)90009-T"},{"key":"ref62","doi-asserted-by":"publisher","DOI":"10.1007\/s11042-020-10139-6"},{"key":"ref63","doi-asserted-by":"publisher","DOI":"10.1016\/j.ifacol.2020.12.923"},{"key":"ref64","doi-asserted-by":"publisher","DOI":"10.1109\/CEC.2019.8790260"},{"key":"ref65","first-page":"1","article-title":"What matters for on-policy deep actor-critic methods? A large-scale study","volume-title":"Proc. Int. Conf. Learn. Represent.","author":"Andrychowicz"},{"key":"ref66","doi-asserted-by":"publisher","DOI":"10.1109\/ITSC48978.2021.9564880"},{"key":"ref67","doi-asserted-by":"publisher","DOI":"10.1109\/MCI.2024.3364428"},{"key":"ref68","doi-asserted-by":"publisher","DOI":"10.23919\/ACC55779.2023.10155792"},{"key":"ref69","doi-asserted-by":"publisher","DOI":"10.1109\/IVWorkshops54471.2021.9669260"},{"issue":"3","key":"ref70","first-page":"128","article-title":"Recent development and applications of SUMO-simulation of urban MObility","volume":"5","author":"Krajzewicz","year":"2012","journal-title":"Int. J. Adv. Syst. Meas."},{"key":"ref71","doi-asserted-by":"publisher","DOI":"10.1016\/j.trd.2011.05.008"},{"key":"ref72","doi-asserted-by":"publisher","DOI":"10.3141\/1999-10"},{"key":"ref73","doi-asserted-by":"publisher","DOI":"10.1103\/PhysRevE.62.1805"},{"key":"ref74","doi-asserted-by":"publisher","DOI":"10.3166\/ejc.11.335-352"},{"key":"ref75","doi-asserted-by":"publisher","DOI":"10.1007\/s12532-018-0139-4"},{"key":"ref76","volume-title":"Nevergrad\u2014A gradient-free optimization platform","author":"Rapin","year":"2018"},{"key":"ref77","doi-asserted-by":"publisher","DOI":"10.1109\/ICNN.1995.488968"},{"key":"ref78","doi-asserted-by":"publisher","DOI":"10.1109\/IROS40897.2019.8968102"},{"key":"ref79","doi-asserted-by":"publisher","DOI":"10.1023\/A:1008202821328"},{"key":"ref80","doi-asserted-by":"publisher","DOI":"10.1007\/s12065-019-00215-8"},{"key":"ref81","doi-asserted-by":"publisher","DOI":"10.1016\/j.commtr.2023.100096"},{"key":"ref82","article-title":"Trust region policy optimization","author":"Schulman","year":"2015","journal-title":"arXiv:1502.05477"}],"container-title":["IEEE Transactions on Evolutionary Computation"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx8\/4235\/10947080\/10841436.pdf?arnumber=10841436","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,4,2]],"date-time":"2025-04-02T08:19:06Z","timestamp":1743581946000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/10841436\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,4]]},"references-count":82,"journal-issue":{"issue":"2"},"URL":"https:\/\/doi.org\/10.1109\/tevc.2025.3529503","relation":{},"ISSN":["1089-778X","1089-778X","1941-0026"],"issn-type":[{"value":"1089-778X","type":"print"},{"value":"1089-778X","type":"print"},{"value":"1941-0026","type":"electronic"}],"subject":[],"published":{"date-parts":[[2025,4]]}}}