{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,5,18]],"date-time":"2026-05-18T10:49:57Z","timestamp":1779101397231,"version":"3.51.4"},"reference-count":117,"publisher":"Institute of Electrical and Electronics Engineers (IEEE)","issue":"7","license":[{"start":{"date-parts":[[2023,7,1]],"date-time":"2023-07-01T00:00:00Z","timestamp":1688169600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/ieeexplore.ieee.org\/Xplorehelp\/downloads\/license-information\/IEEE.html"},{"start":{"date-parts":[[2023,7,1]],"date-time":"2023-07-01T00:00:00Z","timestamp":1688169600000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2023,7,1]],"date-time":"2023-07-01T00:00:00Z","timestamp":1688169600000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"}],"funder":[{"DOI":"10.13039\/501100001691","name":"Japan Society for the Promotion of Science (JSPS) KAKENHI","doi-asserted-by":"publisher","award":["JP20KK0256"],"award-info":[{"award-number":["JP20KK0256"]}],"id":[{"id":"10.13039\/501100001691","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["IEEE Trans. Neural Netw. Learning Syst."],"published-print":{"date-parts":[[2023,7]]},"DOI":"10.1109\/tnnls.2021.3112718","type":"journal-article","created":{"date-parts":[[2021,9,29]],"date-time":"2021-09-29T21:24:51Z","timestamp":1632950691000},"page":"3444-3459","source":"Crossref","is-referenced-by-count":69,"title":["A Survey of Sim-to-Real Transfer Techniques Applied to Reinforcement Learning for Bioinspired Robots"],"prefix":"10.1109","volume":"34","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-4251-1741","authenticated-orcid":false,"given":"Wei","family":"Zhu","sequence":"first","affiliation":[{"name":"Department of Robotics, Graduate School of Engineering, Tohoku University, Sendai, Japan"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-3793-1207","authenticated-orcid":false,"given":"Xian","family":"Guo","sequence":"additional","affiliation":[{"name":"College of Artificial Intelligence, Institute of Robotics and Automatic Information System, Nankai University, Tianjin, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Dai","family":"Owaki","sequence":"additional","affiliation":[{"name":"Department of Robotics, Graduate School of Engineering, Tohoku University, Sendai, Japan"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-5326-7847","authenticated-orcid":false,"given":"Kyo","family":"Kutsuzawa","sequence":"additional","affiliation":[{"name":"Department of Robotics, Graduate School of Engineering, Tohoku University, Sendai, Japan"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-6179-5706","authenticated-orcid":false,"given":"Mitsuhiro","family":"Hayashibe","sequence":"additional","affiliation":[{"name":"Department of Robotics, Graduate School of Engineering, Tohoku University, Sendai, Japan"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"263","reference":[{"key":"ref57","doi-asserted-by":"publisher","DOI":"10.1109\/ICRA.2015.7139509"},{"key":"ref56","doi-asserted-by":"publisher","DOI":"10.1109\/ROBOT.2010.5509336"},{"key":"ref59","doi-asserted-by":"publisher","DOI":"10.1109\/TCYB.2015.2412554"},{"key":"ref58","first-page":"3137","article-title":"A generalized path integral control approach to reinforcement learning","volume":"11","author":"theodorou","year":"2010","journal-title":"J Mach Learn Res"},{"key":"ref53","first-page":"1","article-title":"Guided policy search","author":"levine","year":"2013","journal-title":"Proc Int Conf Mach Learn"},{"key":"ref52","article-title":"Combining model-based and model-free updates for trajectory-centric reinforcement learning","author":"chebotar","year":"2017","journal-title":"arXiv 1703 03078 [cs]"},{"key":"ref55","doi-asserted-by":"publisher","DOI":"10.1523\/JNEUROSCI.13-02-00467.1993"},{"key":"ref54","doi-asserted-by":"publisher","DOI":"10.1109\/IROS.2012.6386025"},{"key":"ref51","first-page":"4565","article-title":"Generative adversarial imitation learning","author":"ho","year":"2016","journal-title":"Proc Adv Neural Inform Process Syst"},{"key":"ref50","doi-asserted-by":"publisher","DOI":"10.1109\/HUMANOIDS.2013.7030010"},{"key":"ref46","doi-asserted-by":"publisher","DOI":"10.1016\/j.robot.2017.11.010"},{"key":"ref45","doi-asserted-by":"publisher","DOI":"10.1109\/IROS.2017.8202137"},{"key":"ref48","first-page":"2538","article-title":"Synthesizing neural network controllers with probabilistic model-based reinforcement learning","author":"higuera","year":"2018","journal-title":"Proc IEEE\/RSJ Int Conf Intell Robots Syst"},{"key":"ref47","doi-asserted-by":"publisher","DOI":"10.1109\/ICRA.2018.8461083"},{"key":"ref42","first-page":"5026","article-title":"Sample-efficient adversarial imitation learning from observation","author":"torabi","year":"2019","journal-title":"Proc Int Conf Mach Learn"},{"key":"ref41","first-page":"1","article-title":"Data efficient reinforcement learning for legged robots","author":"yang","year":"2020","journal-title":"Proc Conf Robot Learn"},{"key":"ref44","doi-asserted-by":"publisher","DOI":"10.1109\/ICRA.2017.7989622"},{"key":"ref43","first-page":"465","article-title":"Pilco: A model-based and data-efficient approach to policy search","author":"deisenroth","year":"2011","journal-title":"Proc Int Conf Mach Learn"},{"key":"ref49","doi-asserted-by":"publisher","DOI":"10.1109\/IROS40897.2019.8967834"},{"key":"ref8","doi-asserted-by":"publisher","DOI":"10.1109\/TNNLS.2017.2773458"},{"key":"ref7","doi-asserted-by":"publisher","DOI":"10.1038\/nature16961"},{"key":"ref9","doi-asserted-by":"publisher","DOI":"10.1109\/TNNLS.2019.2927869"},{"key":"ref4","first-page":"1329","article-title":"Benchmarking deep reinforcement learning for continuous control","author":"duan","year":"2016","journal-title":"Proc Int Conf Mach Learn"},{"key":"ref3","doi-asserted-by":"publisher","DOI":"10.1038\/s41586-019-1724-z"},{"key":"ref6","author":"sutton","year":"1998","journal-title":"Reinforcement Learning An Introduction"},{"key":"ref5","doi-asserted-by":"publisher","DOI":"10.1613\/jair.301"},{"key":"ref100","doi-asserted-by":"publisher","DOI":"10.1109\/ICRAE48301.2019.9043822"},{"key":"ref101","article-title":"Learning agile robotic locomotion skills by imitating animals","author":"peng","year":"2020","journal-title":"arXiv 2004 00784"},{"key":"ref40","article-title":"DIRL: Domain-invariant representation learning for sim-to-real transfer","author":"tanwani","year":"2020","journal-title":"arXiv 2011 07589"},{"key":"ref35","doi-asserted-by":"publisher","DOI":"10.1177\/0278364919887447"},{"key":"ref34","doi-asserted-by":"publisher","DOI":"10.15607\/RSS.2018.XIV.010"},{"key":"ref37","doi-asserted-by":"publisher","DOI":"10.1109\/LRA.2020.2977835"},{"key":"ref36","doi-asserted-by":"publisher","DOI":"10.1109\/IROS40897.2019.8968053"},{"key":"ref31","doi-asserted-by":"crossref","first-page":"63","DOI":"10.1109\/MCS.2010.939963","article-title":"Identification of a bipedal robot with a compliant drivetrain","volume":"31","author":"park","year":"2011","journal-title":"IEEE Control Syst Mag"},{"key":"ref30","doi-asserted-by":"publisher","DOI":"10.1109\/TIE.2009.2015752"},{"key":"ref33","doi-asserted-by":"publisher","DOI":"10.1126\/scirobotics.abc5986"},{"key":"ref32","article-title":"Robust recovery controller for a quadrupedal robot using deep reinforcement learning","author":"lee","year":"2019","journal-title":"arXiv 1901 07517"},{"key":"ref39","doi-asserted-by":"publisher","DOI":"10.1177\/0278364919870227"},{"key":"ref38","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-43089-4_44"},{"key":"ref24","doi-asserted-by":"publisher","DOI":"10.1126\/scirobotics.aau5872"},{"key":"ref23","doi-asserted-by":"publisher","DOI":"10.1109\/LRA.2018.2792536"},{"key":"ref26","doi-asserted-by":"publisher","DOI":"10.1109\/ICRA.2015.7140085"},{"key":"ref25","doi-asserted-by":"publisher","DOI":"10.1109\/SYSCON.2015.7116790"},{"key":"ref20","doi-asserted-by":"publisher","DOI":"10.1109\/ICRA.2018.8460528"},{"key":"ref22","doi-asserted-by":"publisher","DOI":"10.1109\/IROS.2012.6386109"},{"key":"ref21","doi-asserted-by":"publisher","DOI":"10.1109\/IROS.2004.1389727"},{"key":"ref28","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-35699-6_1"},{"key":"ref27","doi-asserted-by":"publisher","DOI":"10.1109\/ROBOT.2010.5509181"},{"key":"ref29","doi-asserted-by":"publisher","DOI":"10.1109\/ICARSC.2019.8733632"},{"key":"ref13","article-title":"Soft actor-critic: Off-policy maximum entropy deep reinforcement learning with a stochastic actor","author":"haarnoja","year":"2018","journal-title":"arXiv 1801 01290"},{"key":"ref12","article-title":"Proximal policy optimization algorithms","author":"schulman","year":"2017","journal-title":"arXiv 1707 06347"},{"key":"ref15","article-title":"Sample efficient actor-critic with experience replay","author":"wang","year":"2016","journal-title":"arXiv 1611 01224"},{"key":"ref14","article-title":"Prioritized experience replay","author":"schaul","year":"2015","journal-title":"arXiv 1511 05952"},{"key":"ref97","doi-asserted-by":"publisher","DOI":"10.1109\/ICRA.2013.6631333"},{"key":"ref96","doi-asserted-by":"publisher","DOI":"10.3390\/app9040799"},{"key":"ref11","article-title":"High-dimensional continuous control using generalized advantage estimation","author":"schulman","year":"2015","journal-title":"arXiv 1506 02438 [cs]"},{"key":"ref99","doi-asserted-by":"publisher","DOI":"10.1109\/IROS.2018.8593722"},{"key":"ref10","article-title":"Continuous control with deep reinforcement learning","author":"lillicrap","year":"2015","journal-title":"arXiv 1509 02971"},{"key":"ref98","doi-asserted-by":"publisher","DOI":"10.1109\/IRC.2019.00102"},{"key":"ref17","doi-asserted-by":"publisher","DOI":"10.1109\/ICRA.2019.8793864"},{"key":"ref16","first-page":"1928","article-title":"Asynchronous methods for deep reinforcement learning","author":"mnih","year":"2016","journal-title":"Proc Int Conf Mach Learn"},{"key":"ref19","doi-asserted-by":"publisher","DOI":"10.1109\/IROS40897.2019.8968251"},{"key":"ref18","doi-asserted-by":"publisher","DOI":"10.1109\/LRA.2020.2979660"},{"key":"ref93","article-title":"Learning to sequence robot behaviors for visual navigation","author":"salman","year":"2018","journal-title":"arXiv 1803 01446"},{"key":"ref92","doi-asserted-by":"publisher","DOI":"10.1109\/LRA.2018.2851148"},{"key":"ref95","doi-asserted-by":"publisher","DOI":"10.1109\/ICRA.2019.8794207"},{"key":"ref94","doi-asserted-by":"publisher","DOI":"10.1007\/s10514-013-9328-1"},{"key":"ref91","doi-asserted-by":"publisher","DOI":"10.1109\/CYBER46603.2019.9066720"},{"key":"ref90","doi-asserted-by":"publisher","DOI":"10.1109\/RO-MAN46459.2019.8956332"},{"key":"ref89","article-title":"Gait library synthesis for quadruped robots via augmented random search","author":"tirumala","year":"2019","journal-title":"arXiv 1912 12907"},{"key":"ref86","first-page":"1515","article-title":"Automatic goal generation for reinforcement learning agents","author":"florensa","year":"2018","journal-title":"Proc Int Conf Mach Learn"},{"key":"ref85","article-title":"Learning and transfer of modulated locomotor controllers","author":"heess","year":"2016","journal-title":"arXiv 1610 05182"},{"key":"ref88","doi-asserted-by":"publisher","DOI":"10.1109\/IROS.2011.6095131"},{"key":"ref87","doi-asserted-by":"publisher","DOI":"10.1109\/Humanoids.2011.6100896"},{"key":"ref82","doi-asserted-by":"publisher","DOI":"10.1109\/IROS.2011.6094427"},{"key":"ref81","doi-asserted-by":"publisher","DOI":"10.1109\/IROS40897.2019.8967913"},{"key":"ref84","first-page":"166","article-title":"Modular multitask reinforcement learning with policy sketches","author":"andreas","year":"2017","journal-title":"Proc Int Conf Mach Learn"},{"key":"ref83","first-page":"1","article-title":"Accelerating reinforcement learning on a robot by using subgoals in a hierarchical framework","author":"van","year":"2011","journal-title":"Proc BeNeLux Conf Artif Intell"},{"key":"ref80","doi-asserted-by":"publisher","DOI":"10.1109\/ICRA.2019.8794179"},{"key":"ref79","doi-asserted-by":"publisher","DOI":"10.1109\/LRA.2020.2968067"},{"key":"ref108","article-title":"Learning human behaviors from motion capture by adversarial imitation","author":"merel","year":"2017","journal-title":"arXiv 1707 02201"},{"key":"ref78","article-title":"Learning generalizable locomotion skills with hierarchical reinforcement learning","author":"li","year":"2019","journal-title":"arXiv 1909 12324"},{"key":"ref109","first-page":"24","article-title":"Transfer learning for reinforcement learning on a physical robot","author":"barrett","year":"2010","journal-title":"Proc Int Conf Auton Agents Multiagent Syst -Adapt Learn Agents Workshop"},{"key":"ref106","first-page":"317","article-title":"Learning locomotion skills for cassie: Iterative design and sim-to-real","author":"xie","year":"2020","journal-title":"Proc Conf Robot Learn"},{"key":"ref107","doi-asserted-by":"publisher","DOI":"10.1109\/Humanoids43949.2019.9034991"},{"key":"ref75","doi-asserted-by":"publisher","DOI":"10.1109\/RAAD.2014.7002234"},{"key":"ref104","doi-asserted-by":"publisher","DOI":"10.1109\/ICRA.2019.8794102"},{"key":"ref74","doi-asserted-by":"publisher","DOI":"10.1109\/IROS.2015.7353919"},{"key":"ref105","article-title":"Iterative reinforcement learning based design of dynamic locomotion skills for cassie","author":"xie","year":"2019","journal-title":"arXiv 1903 09537"},{"key":"ref77","first-page":"1","article-title":"Learning compliant locomotion on a quadruped robot","author":"ponton","year":"2014","journal-title":"Proc IEEE Int Workshop Intell Robots Syst"},{"key":"ref102","doi-asserted-by":"publisher","DOI":"10.1016\/j.robot.2012.05.019"},{"key":"ref76","doi-asserted-by":"publisher","DOI":"10.1109\/TRO.2019.2922493"},{"key":"ref103","article-title":"Learning complex dexterous manipulation with deep reinforcement learning and demonstrations","author":"rajeswaran","year":"2017","journal-title":"arXiv 1709 10087"},{"key":"ref2","doi-asserted-by":"publisher","DOI":"10.1126\/science.aar6404"},{"key":"ref1","doi-asserted-by":"crossref","first-page":"529","DOI":"10.1038\/nature14236","article-title":"Human-level control through deep reinforcement learning","volume":"518","author":"mnih","year":"2015","journal-title":"Nature"},{"key":"ref71","doi-asserted-by":"publisher","DOI":"10.1109\/ICRA.2017.7989384"},{"key":"ref111","article-title":"MOReL: Model-based offline reinforcement learning","author":"kidambi","year":"2020","journal-title":"arXiv 2005 05951"},{"key":"ref70","doi-asserted-by":"publisher","DOI":"10.1109\/IROS.2017.8202141"},{"key":"ref112","first-page":"651","article-title":"Scalable deep reinforcement learning for vision-based robotic manipulation","author":"kalashnikov","year":"2018","journal-title":"Proc Conf Robot Learn"},{"key":"ref73","doi-asserted-by":"publisher","DOI":"10.1109\/Humanoids.2011.6100908"},{"key":"ref72","doi-asserted-by":"publisher","DOI":"10.7551\/mitpress\/9123.003.0009"},{"key":"ref110","article-title":"Offline reinforcement learning: Tutorial, review, and perspectives on open problems","author":"levine","year":"2020","journal-title":"arXiv 2005 01643"},{"key":"ref68","doi-asserted-by":"publisher","DOI":"10.1109\/TRO.2012.2210294"},{"key":"ref67","doi-asserted-by":"publisher","DOI":"10.1109\/ICRA.2011.5980200"},{"key":"ref117","doi-asserted-by":"publisher","DOI":"10.1145\/3197517.3201311"},{"key":"ref69","doi-asserted-by":"publisher","DOI":"10.1109\/IROS.2015.7354296"},{"key":"ref64","doi-asserted-by":"publisher","DOI":"10.15607\/RSS.2010.VI.020"},{"key":"ref115","doi-asserted-by":"publisher","DOI":"10.1109\/TSMC.2015.2418321"},{"key":"ref63","doi-asserted-by":"publisher","DOI":"10.1177\/0278364911402527"},{"key":"ref116","doi-asserted-by":"publisher","DOI":"10.1109\/TII.2017.2647993"},{"key":"ref66","doi-asserted-by":"publisher","DOI":"10.1109\/IROS.2011.6095096"},{"key":"ref113","article-title":"Visual foresight: Model-based deep reinforcement learning for vision-based robotic control","author":"ebert","year":"2018","journal-title":"arXiv 1812 00568"},{"key":"ref65","doi-asserted-by":"publisher","DOI":"10.1109\/IROS.2011.6094877"},{"key":"ref114","article-title":"RoboNet: Large-scale multi-robot learning","author":"dasari","year":"2019","journal-title":"arXiv 1910 11215"},{"key":"ref60","doi-asserted-by":"publisher","DOI":"10.1109\/TIE.2017.2694391"},{"key":"ref62","doi-asserted-by":"publisher","DOI":"10.1109\/ICHR.2010.5686320"},{"key":"ref61","doi-asserted-by":"publisher","DOI":"10.1109\/HUMANOIDS.2015.7363478"}],"container-title":["IEEE Transactions on Neural Networks and Learning Systems"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx7\/5962385\/10175014\/09552429.pdf?arnumber=9552429","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2023,7,31]],"date-time":"2023-07-31T17:33:41Z","timestamp":1690824821000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/9552429\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023,7]]},"references-count":117,"journal-issue":{"issue":"7"},"URL":"https:\/\/doi.org\/10.1109\/tnnls.2021.3112718","relation":{},"ISSN":["2162-237X","2162-2388"],"issn-type":[{"value":"2162-237X","type":"print"},{"value":"2162-2388","type":"electronic"}],"subject":[],"published":{"date-parts":[[2023,7]]}}}