{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,9]],"date-time":"2026-07-09T05:07:34Z","timestamp":1783573654801,"version":"3.55.0"},"reference-count":88,"publisher":"Springer Science and Business Media LLC","issue":"12","license":[{"start":{"date-parts":[[2022,12,14]],"date-time":"2022-12-14T00:00:00Z","timestamp":1670976000000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2022,12,14]],"date-time":"2022-12-14T00:00:00Z","timestamp":1670976000000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"funder":[{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["51809246"],"award-info":[{"award-number":["51809246"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Nat Mach Intell"],"DOI":"10.1038\/s42256-022-00573-6","type":"journal-article","created":{"date-parts":[[2022,12,14]],"date-time":"2022-12-14T17:03:35Z","timestamp":1671037415000},"page":"1077-1087","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":130,"title":["Transferring policy of deep reinforcement learning from simulation to reality for robotics"],"prefix":"10.1038","volume":"4","author":[{"given":"Hao","family":"Ju","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Rongshun","family":"Juan","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Randy","family":"Gomez","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Keisuke","family":"Nakamura","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-1728-5711","authenticated-orcid":false,"given":"Guangliang","family":"Li","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2022,12,14]]},"reference":[{"key":"573_CR1","unstructured":"Sutton, R. & Barto, A. Reinforcement Learning: an Introduction (MIT Press, 2018)."},{"key":"573_CR2","doi-asserted-by":"publisher","first-page":"1238","DOI":"10.1177\/0278364913495721","volume":"32","author":"J Kober","year":"2013","unstructured":"Kober, J., Bagnell, J. A. & Peters, J. Reinforcement learning in robotics: a survey. Int. J. Robot. Res. 32, 1238\u20131274 (2013).","journal-title":"Int. J. Robot. Res."},{"key":"573_CR3","doi-asserted-by":"publisher","first-page":"185","DOI":"10.1016\/j.conb.2008.08.003","volume":"18","author":"P Dayan","year":"2008","unstructured":"Dayan, P. & Niv, Y. Reinforcement learning: the good, the bad and the ugly. Curr. Opin. Neurobiol. 18, 185\u2013196 (2008).","journal-title":"Curr. Opin. Neurobiol."},{"key":"573_CR4","doi-asserted-by":"publisher","first-page":"445","DOI":"10.1038\/nature14540","volume":"521","author":"ML Littman","year":"2015","unstructured":"Littman, M. L. Reinforcement learning improves behaviour from evaluative feedback. Nature 521, 445\u2013451 (2015).","journal-title":"Nature"},{"key":"573_CR5","doi-asserted-by":"publisher","first-page":"436","DOI":"10.1038\/nature14539","volume":"521","author":"Y LeCun","year":"2015","unstructured":"LeCun, Y., Bengio, Y. & Hinton, G. Deep learning. Nature 521, 436\u2013444 (2015).","journal-title":"Nature"},{"key":"573_CR6","doi-asserted-by":"publisher","first-page":"185","DOI":"10.1016\/j.neunet.2020.07.010","volume":"130","author":"P Angelov","year":"2020","unstructured":"Angelov, P. & Soares, E. Towards explainable deep neural networks (xDNN). Neural Netw. 130, 185\u2013194 (2020).","journal-title":"Neural Netw."},{"key":"573_CR7","doi-asserted-by":"publisher","first-page":"529","DOI":"10.1038\/nature14236","volume":"518","author":"V Mnih","year":"2015","unstructured":"Mnih, V. et al. Human-level control through deep reinforcement learning. Nature 518, 529\u2013533 (2015).","journal-title":"Nature"},{"key":"573_CR8","doi-asserted-by":"publisher","first-page":"486","DOI":"10.1038\/518486a","volume":"518","author":"B Sch\u00f6lkopf","year":"2015","unstructured":"Sch\u00f6lkopf, B. Learning to see and act. Nature 518, 486\u2013487 (2015).","journal-title":"Nature"},{"key":"573_CR9","unstructured":"Google DeepMind. AlphaStar: Mastering the Real-Time Strategy Game StarCraft II https:\/\/www.deepmind.com\/blog\/alphastar-mastering-the-real-time-strategy-game-starcraft-ii (2019)."},{"key":"573_CR10","doi-asserted-by":"publisher","first-page":"484","DOI":"10.1038\/nature16961","volume":"529","author":"D Silver","year":"2016","unstructured":"Silver, D. et al. Mastering the game of Go with deep neural networks and tree search. Nature 529, 484\u2013489 (2016).","journal-title":"Nature"},{"key":"573_CR11","unstructured":"Berner, C. et al. Dota 2 with large scale deep reinforcement learning. Preprint at https:\/\/arxiv.org\/abs\/1912.06680 (2019)."},{"key":"573_CR12","unstructured":"Heess, N. et al. Emergence of locomotion behaviours in rich environments. Preprint at https:\/\/arxiv.org\/abs\/1707.02286 (2017)."},{"key":"573_CR13","unstructured":"Florensa, C., Duan, Y. & Abbeel, P. Stochastic neural networks for hierarchical reinforcement learning. In Proc. International Conference on Learning Representations (ICLR) 1\u201310 (OpenReview.net, 2017)."},{"key":"573_CR14","doi-asserted-by":"crossref","unstructured":"Rajeswaran, A. et al. Learning complex dexterous manipulation with deep reinforcement learning and demonstrations. In Proc. Robotics: Science and Systems (RSS) 1\u20139 (RSS foundation, 2018).","DOI":"10.15607\/RSS.2018.XIV.049"},{"key":"573_CR15","doi-asserted-by":"publisher","first-page":"3","DOI":"10.1177\/0278364919887447","volume":"39","author":"M Andrychowicz","year":"2020","unstructured":"Andrychowicz, M. et al. Learning dexterous in-hand manipulation. Int. J. Robot. Res. 39, 3\u201320 (2020).","journal-title":"Int. J. Robot. Res."},{"key":"573_CR16","doi-asserted-by":"crossref","unstructured":"Peng, X. B., Andrychowicz, M., Zaremba, W. & Abbeel, P. Sim-to-real transfer of robotic control with dynamics randomization. In Proc. IEEE International Conference on Robotics and Automation (ICRA) 3803\u20133810 (IEEE, 2018).","DOI":"10.1109\/ICRA.2018.8460528"},{"key":"573_CR17","doi-asserted-by":"crossref","unstructured":"Tan, J. et al. Sim-to-real: learning agile locomotion for quadruped robots. In Proc. Robotics: Science and Systems (RSS) 1\u201311 (RSS foundation, 2018).","DOI":"10.15607\/RSS.2018.XIV.010"},{"key":"573_CR18","doi-asserted-by":"crossref","unstructured":"Wang, J. & Jiang, J. Learning across tasks for zero-shot domain adaptation from a single source domain. IEEE Trans. Pattern. Anal. Mach. Intell. 44, 6264\u20136279 (2021).","DOI":"10.1109\/TPAMI.2021.3088859"},{"key":"573_CR19","unstructured":"Daum\u00e9, H. III. Frustratingly easy domain adaptation. In Proc. 45th Annual Meeting of the Association of Computational Linguistics 256\u2013263(2007)."},{"key":"573_CR20","unstructured":"Ben-David, S. et al. Analysis of representations for domain adaptation. Adv. Neural Inf. Process. Syst. 19, 137\u2013144 (2007)."},{"key":"573_CR21","doi-asserted-by":"crossref","unstructured":"Tremblay, J. et al. Training deep networks with synthetic data: bridging the reality gap by domain randomization. In Proc. IEEE Conference on Computer Vision and Pattern Recognition Workshop 969\u2013977 (IEEE, 2018).","DOI":"10.1109\/CVPRW.2018.00143"},{"key":"573_CR22","doi-asserted-by":"crossref","unstructured":"Tobin, J. et al. Domain randomization and generative models for robotic grasping. In Proc. International Conference on Intelligent Robots and Systems (IROS) 3482\u20133489 (IEEE, 2018).","DOI":"10.1109\/IROS.2018.8593933"},{"key":"573_CR23","unstructured":"Christiano, P. et al. Transfer from simulation to real world through learning deep inverse dynamics model. Preprint at https:\/\/arxiv.org\/abs\/1610.03518 (2016)."},{"key":"573_CR24","doi-asserted-by":"publisher","first-page":"2469","DOI":"10.1007\/s10994-021-05982-z","volume":"110","author":"JP Hanna","year":"2021","unstructured":"Hanna, J. P., Desai, S., Karnan, H., Warnell, G. & Stone, P. Grounded action transformation for sim-to-real reinforcement learning. Mach. Learn. 110, 2469\u20132499 (2021).","journal-title":"Mach. Learn."},{"key":"573_CR25","unstructured":"Rusu, A. A. et al. Progressive neural networks. Preprint at https:\/\/arxiv.org\/abs\/1606.04671 (2016)."},{"key":"573_CR26","unstructured":"Zhang, Z. et al. Progressive neural networks for image classification. Preprint at https:\/\/arxiv.org\/abs\/1804.09803 (2018)."},{"key":"573_CR27","unstructured":"Mishra, N., Rohaninejad, M., Chen, X. & Abbeel, P. A simple neural attentive meta-learner. In Proc. International Conference on Learning Representations (ICLR) 1-17 (OpenReview.net, 2018)."},{"key":"573_CR28","unstructured":"Xu, Z., van Hasselt, H. & Silver, D. Meta-gradient reinforcement learning. Adv. Neural Inf. Process. Syst. 31, 2402\u20132413 (Neural Information Processing Systems Foundation, 2018)."},{"key":"573_CR29","unstructured":"Clavera, I. et al. Model-based reinforcement learning via meta-policy optimization. In Proc. 2nd Annual Conference on Robot Learning 87, 617\u2013629 (PMLR, 2018)."},{"key":"573_CR30","unstructured":"Finn, C., Abbeel, P. & Levine, S. Model-agnostic meta-learning for fast adaptation of deep networks. In Proc. International Conference on Machine Learning (ICML) (eds Precup, D. & Teh, Y. W.) 1126\u20131135 (JMLR.org, 2017)."},{"key":"573_CR31","doi-asserted-by":"crossref","unstructured":"Zhao, W., Queralta, J. P. & Westerlund, T. Sim-to-real transfer in deep reinforcement learning for robotics: a survey. In IEEE Symposium Series on Computational Intelligence (SSCI) 737\u2013744 (IEEE, 2020).","DOI":"10.1109\/SSCI47803.2020.9308468"},{"key":"573_CR32","first-page":"1633","volume":"10","author":"ME Taylor","year":"2009","unstructured":"Taylor, M. E. & Stone, P. H. Transfer learning for reinforcement learning domains: a survey. J. Mach. Learn. Res. 10, 1633\u20131685 (2009).","journal-title":"J. Mach. Learn. Res."},{"key":"573_CR33","doi-asserted-by":"publisher","first-page":"237","DOI":"10.1613\/jair.301","volume":"4","author":"LP Kaelbling","year":"1996","unstructured":"Kaelbling, L. P., Littman, M. L. & Moore, A. W. Reinforcement learning: a survey. J. Artif. Intell. Res. 4, 237\u2013285 (1996).","journal-title":"J. Artif. Intell. Res."},{"key":"573_CR34","doi-asserted-by":"publisher","unstructured":"Wu, J., Huang, Z. & Lv, C. Uncertainty-aware model-based reinforcement learning: methodology and application in autonomous driving. In IEEE Trans. Intell. Veh. https:\/\/doi.org\/10.1109\/TIV.2022.3185159 (2022).","DOI":"10.1109\/TIV.2022.3185159"},{"key":"573_CR35","doi-asserted-by":"publisher","first-page":"279","DOI":"10.1007\/BF00992698","volume":"8","author":"CJ Watkins","year":"1992","unstructured":"Watkins, C. J. & Dayan, P. Q-learning. Mach. Learn. 8, 279\u2013292 (1992).","journal-title":"Mach. Learn."},{"key":"573_CR36","doi-asserted-by":"crossref","unstructured":"Li, S. et al. Robust multi-agent reinforcement learning via minimax deep deterministic policy gradient. In Proc. AAAI Conference on Artificial Intelligence Vol. 33, 4213\u20134220 (AAAI Press, 2019).","DOI":"10.1609\/aaai.v33i01.33014213"},{"key":"573_CR37","doi-asserted-by":"crossref","unstructured":"Tobin, J. et al. Domain randomization for transferring deep neural networks from simulation to the real world. In Proc. International Conference on Intelligent Robots and Systems (IROS) 23\u201330 (IEEE, 2017).","DOI":"10.1109\/IROS.2017.8202133"},{"key":"573_CR38","unstructured":"Tzeng, E., Hoffman, J., Zhang, N., Saenko, K. & Darrell, T. Deep domain confusion: maximizing for domain invariance. Preprint at https:\/\/arxiv.org\/abs\/1412.3474 (2014)."},{"key":"573_CR39","unstructured":"Long, M., Cao, Y., Wang, J. & Jordan, M. Learning transferable features with deep adaptation networks. In Proc. International Conference on Machine Learning (ICML) 37, 97\u2013105 (JMLR.org, 2015)."},{"key":"573_CR40","doi-asserted-by":"crossref","unstructured":"Sun, B., Feng, J. & Saenko, K. Return of frustratingly easy domain adaptation. In Proc. AAAI Conference on Artificial Intelligence Vol. 30, 2058\u20132065 (AAAI Press, 2016).","DOI":"10.1609\/aaai.v30i1.10306"},{"key":"573_CR41","first-page":"2096","volume":"17","author":"Y Ganin","year":"2016","unstructured":"Ganin, Y. et al. Domain-adversarial training of neural networks. J. Mach. Learn. Res. 17, 2096\u20132030 (2016).","journal-title":"J. Mach. Learn. Res."},{"key":"573_CR42","doi-asserted-by":"crossref","unstructured":"Tzeng, E., Hoffman, J., Darrell, T. & Saenko, K. Simultaneous deep transfer across domains and tasks. In Proc. International Conference on Computer Vision (ICCV) 4068\u20134076 (IEEE, 2015).","DOI":"10.1109\/ICCV.2015.463"},{"key":"573_CR43","doi-asserted-by":"crossref","unstructured":"Bousmalis, K., Silberman, N., Dohan, D., Erhan, D. & Krishnan, D. Unsupervised pixel-level domain adaptation with generative adversarial networks. In Proc. IEEE Conference on Computer Vision and Pattern Recognition (CVPR) 3722\u20133731 (IEEE, 2017).","DOI":"10.1109\/CVPR.2017.18"},{"key":"573_CR44","first-page":"343","volume":"29","author":"K Bousmalis","year":"2016","unstructured":"Bousmalis, K., Trigeorgis, G., Silberman, N., Krishnan, D. & Erhan, D. Domain separation networks. Adv. Neural Inf. Process. Syst. 29, 343\u2013351 (2016).","journal-title":"Adv. Neural Inf. Process. Syst."},{"key":"573_CR45","unstructured":"Carr, T., Chli, M. & Vogiatzis, G. Domain adaptation for reinforcement learning on the Atari. In Proc. 18th International Conference on Autonomous Agents and MultiAgent Systems 1859\u20131861 (International Foundation for Autonomous Agents and Multiagent Systems, 2019)."},{"key":"573_CR46","doi-asserted-by":"publisher","first-page":"253","DOI":"10.1613\/jair.3912","volume":"47","author":"MG Bellemare","year":"2013","unstructured":"Bellemare, M. G., Naddaf, Y., Veness, J. & Bowling, M. The Arcade Learning Environment: an evaluation platform for general agents. J. Artif. Intell. Res. 47, 253\u2013279 (2013).","journal-title":"J. Artif. Intell. Res."},{"key":"573_CR47","doi-asserted-by":"crossref","unstructured":"Tzeng, E. et al. Adapting deep visuomotor representations with weak pairwise constraints. In Algorithmic Foundations of Robotics XII (eds Goldberg, K. et al.) 688\u2013703 (Springer, 2020).","DOI":"10.1007\/978-3-030-43089-4_44"},{"key":"573_CR48","unstructured":"Wise, M., Ferguson, M., King, D., Diehr, E. & Dymesich, D. Fetch and Freight: standard platforms for service robot applications. In Workshop on Autonomous Mobile Service Robots 1-6 (2016)."},{"key":"573_CR49","doi-asserted-by":"crossref","unstructured":"Xu, Y. & Vatankhah, H. SimSpark: an open source robot simulator developed by the RoboCup community. In RoboCup 2013: Robot World Cup XVII (eds S. Behnke et al.) 632\u2013639 (Springer, 2013).","DOI":"10.1007\/978-3-662-44468-9_59"},{"key":"573_CR50","doi-asserted-by":"crossref","unstructured":"Koenig, N. & Howard, A. Design and use paradigms for Gazebo, an open-source multi-robot simulator. In Proc. International Conference on Intelligent Robots and Systems (IROS) Vol. 3, 2149\u20132154 (IEEE, 2004).","DOI":"10.1109\/IROS.2004.1389727"},{"key":"573_CR51","first-page":"3917","volume":"33","author":"S Desai","year":"2020","unstructured":"Desai, S. et al. An imitation from observation approach to transfer learning with dynamics mismatch. Adv. Neural Inf. Process. Syst. 33, 3917\u20133929 (2020).","journal-title":"Adv. Neural Inf. Process. Syst."},{"key":"573_CR52","doi-asserted-by":"crossref","unstructured":"Karnan, H., Desai, S., Hanna, J. P., Warnell, G. & Stone, P. Reinforced grounded action transformation for sim-to-real transfer. In Proc. International Conference on Intelligent Robots and Systems (IROS) 4397\u20134402 (IEEE, 2020).","DOI":"10.1109\/IROS45743.2020.9341149"},{"key":"573_CR53","unstructured":"Rusu, A. A. et al. Sim-to-real robot learning from pixels with progressive nets. In Proc. 1st Annual Conference on Robot Learning 78, 262\u2013270 (PMLR, 2017)."},{"key":"573_CR54","doi-asserted-by":"publisher","first-page":"860","DOI":"10.1038\/s41593-018-0147-8","volume":"21","author":"JX Wang","year":"2018","unstructured":"Wang, J. X. et al. Prefrontal cortex as a meta-reinforcement learning system. Nat. Neurosci. 21, 860\u2013868 (2018).","journal-title":"Nat. Neurosci."},{"key":"573_CR55","unstructured":"Schulman, J., Wolski, F., Dhariwal, P., Radford, A. & Klimov, O. Proximal policy optimization algorithms. Preprint at https:\/\/arxiv.org\/abs\/1707.06347 (2017)."},{"key":"573_CR56","doi-asserted-by":"crossref","unstructured":"Arndt, K., Hazara, M., Ghadirzadeh, A. & Kyrki, V. Meta reinforcement learning for sim-to-real domain adaptation. In Proc. IEEE International Conference on Robotics and Automation 2725\u20132731 (IEEE, 2020).","DOI":"10.1109\/ICRA40945.2020.9196540"},{"key":"573_CR57","unstructured":"Nagabandi, A. et al. Learning to adapt in dynamic, real-world environments through meta-reinforcement learning. In Proc. International Conference on Learning Representations 1-17 (OpenReview.net, 2019)."},{"key":"573_CR58","doi-asserted-by":"crossref","unstructured":"Chebotar, Y. et al. Closing the sim-to-real loop: adapting simulation randomization with real world experience. In Proc. International Conference on Robotics and Automation (ICRA) 8973\u20138979 (IEEE, 2019).","DOI":"10.1109\/ICRA.2019.8793789"},{"key":"573_CR59","unstructured":"Mehta, B., Diaz, M., Golemo, F., Pal, C. J. & Paull, L. Active domain randomization. In Proc. 4th Annual Conference on Robot Learning 100, 1162\u20131176 (PMLR, 2020)."},{"key":"573_CR60","doi-asserted-by":"publisher","first-page":"1172","DOI":"10.1109\/TPAMI.2019.2952353","volume":"43","author":"F Muratore","year":"2021","unstructured":"Muratore, F., Gienger, M. & Peters, J. Assessing transferability from simulation to reality for reinforcement learning. IEEE Trans. Pattern Anal. Mach. Intell. 43, 1172\u20131183 (2021).","journal-title":"IEEE Trans. Pattern Anal. Mach. Intell."},{"key":"573_CR61","unstructured":"Rusu, A. A. et al. Policy distillation. In Proc. International Conference on Learning Representations (ICLR) 1-13 (OpenReview.net, 2016)."},{"key":"573_CR62","unstructured":"Traor\u00e9, R. et al. DisCoRL: continual reinforcement learning via policy distillation. In NeurIPS Workshop on Deep Reinforcement Learning 1-15 (2019)."},{"key":"573_CR63","doi-asserted-by":"crossref","unstructured":"James, S. et al. Sim-to-real via sim-to-sim: data-efficient robotic grasping via randomized-to-canonical adaptation networks. In Proc. IEEE Conference on Computer Vision and Pattern Recognition 12627\u201312637 (IEEE, 2019).","DOI":"10.1109\/CVPR.2019.01291"},{"key":"573_CR64","unstructured":"Kalashnikov, D. et al. QT-Opt: scalable deep reinforcement learning for vision-based robotic manipulation. In Proc. 2nd Annual Conference on Robot Learning Vol. 87, 651\u2013673 (PMLR, 2018)."},{"key":"573_CR65","doi-asserted-by":"crossref","unstructured":"Ljung, L. System identification. In Signal Analysis and Prediction (eds A. Proch\u00e1zka et al.) 163\u2013173 (Springer, 1998).","DOI":"10.1007\/978-1-4612-1768-8_11"},{"key":"573_CR66","doi-asserted-by":"publisher","first-page":"123","DOI":"10.1016\/0005-1098(71)90059-8","volume":"7","author":"KJ \u00c5str\u00f6m","year":"1971","unstructured":"\u00c5str\u00f6m, K. J. & Eykhoff, P. System identification\u2014a survey. Automatica 7, 123\u2013162 (1971).","journal-title":"Automatica"},{"key":"573_CR67","doi-asserted-by":"crossref","unstructured":"Lowrey, K., Kolev, S., Dao, J., Rajeswaran, A. & Todorov, E. Reinforcement learning for non-prehensile manipulation: transfer from simulation to physical system. In Proc. IEEE International Conference on Simulation, Modeling, and Programming for Autonomous Robots (SIMPAR) 35\u201342 (IEEE, 2018).","DOI":"10.1109\/SIMPAR.2018.8376268"},{"key":"573_CR68","unstructured":"Antonova, R., Cruciani, S., Smith, C. & Kragic, D. Reinforcement learning for pivoting task. Preprint at https:\/\/arxiv.org\/abs\/1703.00472 (2017)."},{"key":"573_CR69","doi-asserted-by":"crossref","unstructured":"Shah, S., Dey, D., Lovett, C. & Kapoor, A. AirSim: high-fidelity visual and physical simulation for autonomous vehicles. In Field and Service Robotics (eds M. Hutter & R. Siegwart) 621\u2013635 (Springer Proceedings in Advanced Robotics Vol. 5, Springer, 2018).","DOI":"10.1007\/978-3-319-67361-5_40"},{"key":"573_CR70","unstructured":"Dosovitskiy, A., Ros, G., Codevilla, F., Lopez, A. & Koltun, V. CARLA: an open urban driving simulator. In Proc. 1st Annual Conference on Robot Learning 78, 1\u201316 (2017)."},{"key":"573_CR71","doi-asserted-by":"publisher","first-page":"1281","DOI":"10.1021\/cr0300993","volume":"105","author":"GS Kottas","year":"2005","unstructured":"Kottas, G. S., Clarke, L. I., Horinek, D. & Michl, J. Artificial molecular rotors. Chem. Rev. 105, 1281\u20131376 (2005).","journal-title":"Chem. Rev."},{"key":"573_CR72","doi-asserted-by":"crossref","unstructured":"McCord, C., Queralta, J. P., Gia, T. N. & Westerlund, T. Distributed progressive formation control for multi-agent systems: 2D and 3D deployment of UAVs in ROS\/Gazebo with RotorS. In Proc. European Conference on Mobile Robots (ECMR) 1\u20136 (IEEE, 2019).","DOI":"10.1109\/ECMR.2019.8870934"},{"key":"573_CR73","unstructured":"Coumans, E. & Bai, Y. PyBullet, a Python Module for Physics Simulation for Games, Robotics and Machine Learning https:\/\/pybullet.org\/wordpress\/ (2016)."},{"key":"573_CR74","doi-asserted-by":"crossref","unstructured":"Todorov, E., Erez, T. & Tassa, Y. MuJoCo: a physics engine for model-based control. In Proc. International Conference on Intelligent Robots and Systems (IROS) 5026\u20135033 (IEEE, 2012).","DOI":"10.1109\/IROS.2012.6386109"},{"key":"573_CR75","doi-asserted-by":"publisher","first-page":"335","DOI":"10.1162\/0899766053011528","volume":"17","author":"J Morimoto","year":"2005","unstructured":"Morimoto, J. & Doya, K. Robust reinforcement learning. Neural Comput. 17, 335\u2013359 (2005).","journal-title":"Neural Comput."},{"key":"573_CR76","unstructured":"Tessler, C., Efroni, Y. & Mannor, S. Action robust reinforcement learning and applications in continuous control. In Proc. International Conference on Machine Learning (ICML) 97, 6215\u20136224 (JMLR.org, 2019)."},{"key":"573_CR77","unstructured":"Mankowitz, D. J. et al. Robust reinforcement learning for continuous control with model misspecification. In Proc. International Conference on Learning Representations (ICLR)1-11 (OpenReview.net, 2020)."},{"key":"573_CR78","first-page":"1437","volume":"16","author":"J Garc\u0131a","year":"2015","unstructured":"Garc\u0131a, J. & Fern\u00e1ndez, F. A comprehensive survey on safe reinforcement learning. J. Mach. Learn. Res. 16, 1437\u20131480 (2015).","journal-title":"J. Mach. Learn. Res."},{"key":"573_CR79","unstructured":"Saunders, W., Sastry, G., Stuhlm\u00fcller, A. & Evans, O. Trial without error: towards safe reinforcement learning via human intervention. In Proc. 17th International Conference on Autonomous Agents and MultiAgent Systems 2067\u20132069 (International Foundation for Autonomous Agents and Multiagent Systems, 2018)."},{"key":"573_CR80","first-page":"27395","volume":"34","author":"T Xie","year":"2021","unstructured":"Xie, T., Jiang, N., Wang, H., Xiong, C. & Bai, Y. Policy finetuning: bridging sample-efficient offline and online reinforcement learning. Adv. Neural Inf. Process. Syst. 34, 27395\u201327407 (2021).","journal-title":"Adv. Neural Inf. Process. Syst."},{"key":"573_CR81","unstructured":"Lee, S., Seo, Y., Lee, K., Abbeel, P. & Shin, J. Offline-to-online reinforcement learning via balanced replay and pessimistic Q-ensemble. In Proc. 6th Annual Conference on Robot Learning 164, 1702\u20131712 (2022)."},{"key":"573_CR82","unstructured":"Christiano, P. F. et al. Deep reinforcement learning from human preferences. Adv. Neural Inf. Process. Syst. 30, 4302\u20134310 (2017)."},{"key":"573_CR83","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1007\/s10458-017-9374-8","volume":"32","author":"G Li","year":"2018","unstructured":"Li, G., Whiteson, S., Knox, W. B. & Hung, H. Social interaction for efficient agent learning from human reward. Auton. Agent Multi Agent Syst. 32, 1\u201325 (2018).","journal-title":"Auton. Agent Multi Agent Syst."},{"key":"573_CR84","doi-asserted-by":"crossref","unstructured":"Li, G., He, B., Gomez, R. & Nakamura, K. Interactive reinforcement learning from demonstration and human evaluative feedback. In Proc. 27th IEEE International Symposium on Robot and Human Interactive Communication (RO-MAN) 1156\u20131162 (IEEE, 2018).","DOI":"10.1109\/ROMAN.2018.8525837"},{"key":"573_CR85","doi-asserted-by":"publisher","first-page":"103500","DOI":"10.1016\/j.artint.2021.103500","volume":"297","author":"S Arora","year":"2021","unstructured":"Arora, S. & Doshi, P. A survey of inverse reinforcement learning: challenges, methods and progress. Artif. Intell. 297, 103500 (2021).","journal-title":"Artif. Intell."},{"key":"573_CR86","doi-asserted-by":"crossref","unstructured":"Juan, R. et al. Shaping progressive net of reinforcement learning for policy transfer with human evaluative feedback. In Proc. IEEE International Conference on Intelligent Robots and Systems (IROS) 1281\u20131288 (IEEE, 2021).","DOI":"10.1109\/IROS51168.2021.9636061"},{"key":"573_CR87","doi-asserted-by":"publisher","first-page":"337","DOI":"10.1109\/THMS.2019.2912447","volume":"49","author":"G Li","year":"2019","unstructured":"Li, G., Gomez, R., Nakamura, K. & He, B. Human-centered reinforcement learning: a survey. IEEE Trans. Hum. Mach. Syst. 49, 337\u2013349 (2019).","journal-title":"IEEE Trans. Hum. Mach. Syst."},{"key":"573_CR88","doi-asserted-by":"publisher","first-page":"133","DOI":"10.1038\/s42256-019-0025-4","volume":"1","author":"EO Neftci","year":"2019","unstructured":"Neftci, E. O. & Averbeck, B. B. Reinforcement learning in artificial and biological systems. Nat. Mach. Intell. 1, 133\u2013143 (2019).","journal-title":"Nat. Mach. Intell."}],"container-title":["Nature Machine Intelligence"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/www.nature.com\/articles\/s42256-022-00573-6.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/www.nature.com\/articles\/s42256-022-00573-6","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/www.nature.com\/articles\/s42256-022-00573-6.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,10,10]],"date-time":"2024-10-10T10:39:51Z","timestamp":1728556791000},"score":1,"resource":{"primary":{"URL":"https:\/\/www.nature.com\/articles\/s42256-022-00573-6"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2022,12,14]]},"references-count":88,"journal-issue":{"issue":"12","published-online":{"date-parts":[[2022,12]]}},"alternative-id":["573"],"URL":"https:\/\/doi.org\/10.1038\/s42256-022-00573-6","relation":{},"ISSN":["2522-5839"],"issn-type":[{"value":"2522-5839","type":"electronic"}],"subject":[],"published":{"date-parts":[[2022,12,14]]},"assertion":[{"value":"5 December 2021","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"21 October 2022","order":2,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"14 December 2022","order":3,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"The authors declare no competing interests.","order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Competing interests"}}]}}