{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,7,7]],"date-time":"2025-07-07T23:10:10Z","timestamp":1751929810292,"version":"3.41.2"},"reference-count":87,"publisher":"Institute of Electrical and Electronics Engineers (IEEE)","issue":"8","license":[{"start":{"date-parts":[[2025,8,1]],"date-time":"2025-08-01T00:00:00Z","timestamp":1754006400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/ieeexplore.ieee.org\/Xplorehelp\/downloads\/license-information\/IEEE.html"},{"start":{"date-parts":[[2025,8,1]],"date-time":"2025-08-01T00:00:00Z","timestamp":1754006400000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2025,8,1]],"date-time":"2025-08-01T00:00:00Z","timestamp":1754006400000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"}],"funder":[{"name":"Guangxi Key Laboratory of Brain-Inspired Computing and Intelligent Chips, Guangxi Normal University","award":["BCIC-23-K9"],"award-info":[{"award-number":["BCIC-23-K9"]}]}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["IEEE Trans. Pattern Anal. Mach. Intell."],"published-print":{"date-parts":[[2025,8]]},"DOI":"10.1109\/tpami.2025.3560628","type":"journal-article","created":{"date-parts":[[2025,4,21]],"date-time":"2025-04-21T17:39:01Z","timestamp":1745257141000},"page":"6647-6659","source":"Crossref","is-referenced-by-count":0,"title":["On the Value of Myopic Behavior in Policy Reuse"],"prefix":"10.1109","volume":"47","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-8379-9385","authenticated-orcid":false,"given":"Chenjia","family":"Bai","sequence":"first","affiliation":[{"name":"Institute of Artificial Intelligence (TeleAI), China Telecom, Shanghai, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-6040-3002","authenticated-orcid":false,"given":"Kang","family":"Xu","sequence":"additional","affiliation":[{"name":"Academy for Engineering and Technology, Fudan University, Shanghai, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-9651-1061","authenticated-orcid":false,"given":"Shuang","family":"Qiu","sequence":"additional","affiliation":[{"name":"City University of Hong Kong, Hong Kong, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Haoran","family":"He","sequence":"additional","affiliation":[{"name":"Hong Kong University of Science and Technology, Hong Kong, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-0294-8538","authenticated-orcid":false,"given":"Bin","family":"Zhao","sequence":"additional","affiliation":[{"name":"School of Artificial Intelligence, Optics and Electronics (iOPEN), Northwestern Polytechnical University, Xi’an, Shanxi, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-8182-2852","authenticated-orcid":false,"given":"Zhen","family":"Wang","sequence":"additional","affiliation":[{"name":"School of Artificial Intelligence, Optics and Electronics (iOPEN), Northwestern Polytechnical University, Xi’an, Shanxi, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-9786-585X","authenticated-orcid":false,"given":"Wei","family":"Li","sequence":"additional","affiliation":[{"name":"Academy for Engineering and Technology, Fudan University, Shanghai, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-2924-946X","authenticated-orcid":false,"given":"Xuelong","family":"Li","sequence":"additional","affiliation":[{"name":"Institute of Artificial Intelligence (TeleAI), China Telecom, Shanghai, China"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"263","reference":[{"article-title":"Solving Rubik\u2019s cube with a robot hand","year":"2019","author":"Akkaya","key":"ref1"},{"key":"ref2","doi-asserted-by":"publisher","DOI":"10.1126\/scirobotics.abc5986"},{"article-title":"Dota 2 with large scale deep reinforcement learning","year":"2019","author":"Berner","key":"ref3"},{"key":"ref4","doi-asserted-by":"publisher","DOI":"10.1145\/1160633.1160762"},{"key":"ref5","doi-asserted-by":"publisher","DOI":"10.1007\/s10994-016-5547-y"},{"key":"ref6","first-page":"5587","article-title":"Policy improvement via imitation of multiple oracles","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Cheng"},{"article-title":"CUP: Critic-guided policy reuse","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Zhang","key":"ref7"},{"article-title":"Multi-advisor reinforcement learning","year":"2017","author":"Laroche","key":"ref8"},{"key":"ref9","first-page":"1787","article-title":"Contextual policy transfer in reinforcement learning domains via deep mixtures-of-experts","volume-title":"Proc. Int. Conf. Uncertainty Artif. Intell.","author":"Gimelfarb"},{"key":"ref10","doi-asserted-by":"publisher","DOI":"10.24963\/ijcai.2020\/430"},{"key":"ref11","first-page":"989","article-title":"Context-aware policy reuse","volume-title":"Proc. 18th Int. Conf. Auton. Agents MultiAgent Syst.","author":"Li"},{"article-title":"SkillS: Adaptive skill sequencing for efficient temporally-extended exploration","year":"2022","author":"Vezzani","key":"ref12"},{"key":"ref13","first-page":"2053","article-title":"Efficient deep reinforcement learning through policy transfer","volume-title":"Proc. 19th Int. Conf. Auton. Agents Multiagent Syst.","author":"Yang"},{"key":"ref14","first-page":"717","article-title":"AC-Teach: A Bayesian actor-critic method for policy learning with an ensemble of suboptimal teachers","volume-title":"Proc. Conf. Robot Learn.","author":"Kurenkov"},{"key":"ref15","first-page":"9549","article-title":"Reinforcement learning with multiple experts: A Bayesian model combination approach","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Gimelfarb"},{"article-title":"Jump-start reinforcement learning","year":"2022","author":"Uchendu","key":"ref16"},{"key":"ref17","first-page":"28955","article-title":"Reincarnating reinforcement learning: Reusing prior computation to accelerate progress","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Agarwal"},{"article-title":"Policy expansion for bridging offline-to-online reinforcement learning","volume-title":"Proc. 11th Int. Conf. Learn. Representations","author":"Zhang","key":"ref18"},{"key":"ref19","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v32i1.11718"},{"article-title":"Beyond fine-tuning: Transferring behavior in reinforcement learning","volume-title":"Proc. ICML 2021 Workshop Unsupervised Reinforcement Learn.","author":"Campos","key":"ref20"},{"article-title":"Efficient multi-task reinforcement learning via selective behavior sharing","year":"2023","author":"Zhang","key":"ref21"},{"article-title":"Diversity is all you need: Learning skills without a reward function","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Eysenbach","key":"ref22"},{"article-title":"Neural probabilistic motor primitives for humanoid control","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Merel","key":"ref23"},{"article-title":"Learning more skills through optimistic exploration","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Strouse","key":"ref24"},{"article-title":"OPAL: Offline primitive discovery for accelerating offline reinforcement learning","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Ajay","key":"ref25"},{"article-title":"Variational intrinsic control","year":"2016","author":"Gregor","key":"ref26"},{"article-title":"Dynamics-aware unsupervised discovery of skills","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Sharma","key":"ref27"},{"key":"ref28","first-page":"39183","article-title":"Behavior contrastive learning for unsupervised skill discovery","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Yang"},{"article-title":"Composing task-agnostic policies with deep reinforcement learning","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Qureshi","key":"ref29"},{"key":"ref30","first-page":"21847","article-title":"Accelerating robotic reinforcement learning via parameterized action primitives","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Dalal"},{"key":"ref31","first-page":"3681","article-title":"MCP: Learning composable hierarchical control with multiplicative compositional policies","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Peng"},{"key":"ref32","first-page":"188","article-title":"Accelerating reinforcement learning with learned skill priors","volume-title":"Proc. Int. Conf. Robot Learn.","author":"Pertsch"},{"article-title":"Parrot: Data-driven behavioral priors for reinforcement learning","year":"2020","author":"Singh","key":"ref33"},{"key":"ref34","first-page":"1652","article-title":"Don\u2019t start from scratch: Leveraging prior data to automate robotic reinforcement learning","volume-title":"Proc. Int. Conf. Robot Learn.","author":"Walke"},{"article-title":"Skill-based meta-reinforcement learning","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Nam","key":"ref35"},{"key":"ref36","first-page":"2262","article-title":"Skill-based model-based reinforcement learning","volume-title":"Proc. Annu. Conf. Robot Learn.","author":"Shi"},{"key":"ref37","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v31i1.10916"},{"article-title":"Meta learning shared hierarchies","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Frans","key":"ref38"},{"article-title":"Composing complex skills by learning transition policies","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Lee","key":"ref39"},{"key":"ref40","first-page":"21672","article-title":"Hierarchical programmatic reinforcement learning via learning to compose programs","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Liu"},{"key":"ref41","doi-asserted-by":"publisher","DOI":"10.1512\/iumj.1957.6.56038"},{"volume-title":"Reinforcement Learning: An Introduction","year":"2018","author":"Sutton","key":"ref42"},{"key":"ref43","first-page":"1265","article-title":"Biasing approximate dynamic programming with a lower discount factor","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Petrik"},{"key":"ref44","first-page":"1181","article-title":"The dependence of effective planning horizon on model accuracy","volume-title":"Proc. Int. Conf. Auton. Agents Multiagent Syst.","author":"Jiang"},{"key":"ref45","first-page":"269","article-title":"Discount factor as a regularizer in reinforcement learning","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Amit"},{"key":"ref46","doi-asserted-by":"publisher","DOI":"10.1109\/IROS.2018.8594418"},{"key":"ref47","first-page":"9072","article-title":"On the role of discount factor in offline reinforcement learning","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Hu"},{"key":"ref48","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-70087-8_81"},{"key":"ref49","first-page":"5468","article-title":"Separating value functions across time-scales","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Romoff"},{"key":"ref50","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v34i04.6027"},{"key":"ref51","first-page":"10130","article-title":"Taylor expansion of discount factors","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Tang"},{"article-title":"Composite Q-learning: Multi-scale Q-function decomposition and separable optimization","year":"2019","author":"Kalweit","key":"ref52"},{"key":"ref53","first-page":"2402","article-title":"Meta-gradient reinforcement learning","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Xu"},{"key":"ref54","doi-asserted-by":"publisher","DOI":"10.1007\/BF00992698"},{"key":"ref55","volume-title":"On-Line Q-Learning Using Connectionist Systems","volume":"37","author":"Rummery","year":"1994"},{"key":"ref56","first-page":"449","article-title":"A distributional perspective on reinforcement learning","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Bellemare"},{"key":"ref57","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v32i1.11791"},{"key":"ref58","first-page":"1096","article-title":"Implicit quantile networks for distributional reinforcement learning","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Dabney"},{"key":"ref59","doi-asserted-by":"publisher","DOI":"10.1109\/TNNLS.2022.3217189"},{"key":"ref60","first-page":"1312","article-title":"Universal value function approximators","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Schaul"},{"key":"ref61","first-page":"5055","article-title":"Hindsight experience replay","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Andrychowicz"},{"key":"ref62","first-page":"14814","article-title":"Planning with goal-conditioned policies","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Nasiriany"},{"key":"ref63","first-page":"5392","article-title":"Hybrid reward architecture for reinforcement learning","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Van Seijen"},{"key":"ref64","first-page":"11298","article-title":"RD$^{2}$2: Reward decomposition with representation decomposition","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Lin"},{"key":"ref65","first-page":"1519","article-title":"Distributional reinforcement learning for multi-dimensional reward functions","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Zhang"},{"key":"ref66","first-page":"501","article-title":"Transfer in deep reinforcement learning using successor features and generalised policy improvement","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Barreto"},{"article-title":"Universal successor features approximators","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Borsa","key":"ref67"},{"article-title":"Dynamic programming and Markov processes","year":"1960","author":"Howard","key":"ref68"},{"key":"ref69","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v30i1.10295"},{"key":"ref70","first-page":"14322","article-title":"When to use parametric models in reinforcement learning?","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Van Hasselt"},{"key":"ref71","doi-asserted-by":"publisher","DOI":"10.12794\/metadc1505267"},{"key":"ref72","first-page":"1861","article-title":"Soft actor-critic: Off-policy maximum entropy deep reinforcement learning with a stochastic actor","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Haarnoja"},{"key":"ref73","first-page":"5331","article-title":"Efficient off-policy meta-reinforcement learning via probabilistic context variables","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Rakelly"},{"key":"ref74","first-page":"4759","article-title":"Deep reinforcement learning in a handful of trials using probabilistic dynamics models","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Chua"},{"key":"ref75","first-page":"8583","article-title":"Planning to explore via self-supervised world models","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Sekar"},{"key":"ref76","first-page":"2555","article-title":"Learning latent dynamics for planning from pixels","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Hafner"},{"issue":"5","key":"ref77","first-page":"815","article-title":"Finite-time bounds for fitted value iteration","volume":"9","author":"Munos","year":"2008","journal-title":"J. Mach. Learn. Res."},{"key":"ref78","first-page":"1587","article-title":"Addressing function approximation error in actor-critic methods","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Fujimoto"},{"article-title":"Reinforcement learning: Theory and algorithms","year":"2019","author":"Agarwal","key":"ref79"},{"key":"ref80","first-page":"397","article-title":"Using confidence bounds for exploitation-exploration trade-offs","volume":"3","author":"Auer","year":"2002","journal-title":"J. Mach. Learn. Res."},{"key":"ref81","first-page":"1094","article-title":"Meta-world: A benchmark and evaluation for multi-task and meta reinforcement learning","volume-title":"Proc. Int. Conf. Robot Learn.","author":"Yu"},{"article-title":"D4RL: Datasets for deep data-driven reinforcement learning","year":"2020","author":"Fu","key":"ref82"},{"key":"ref83","first-page":"4767","article-title":"Multi-task reinforcement learning with soft modularization","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Yang"},{"article-title":"Conservative safety critics for exploration","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Bharadhwaj","key":"ref84"},{"key":"ref85","first-page":"1522","article-title":"Risk-sensitive and robust decision-making: A CVaR optimization approach","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Chow"},{"key":"ref86","first-page":"22","article-title":"Constrained policy optimization","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Achiam"},{"key":"ref87","first-page":"6304","article-title":"Disentangling transfer in continual reinforcement learning","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Wolczyk"}],"container-title":["IEEE Transactions on Pattern Analysis and Machine Intelligence"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx8\/34\/11068886\/10971952.pdf?arnumber=10971952","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,7,7]],"date-time":"2025-07-07T22:57:43Z","timestamp":1751929063000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/10971952\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,8]]},"references-count":87,"journal-issue":{"issue":"8"},"URL":"https:\/\/doi.org\/10.1109\/tpami.2025.3560628","relation":{},"ISSN":["0162-8828","2160-9292","1939-3539"],"issn-type":[{"type":"print","value":"0162-8828"},{"type":"electronic","value":"2160-9292"},{"type":"electronic","value":"1939-3539"}],"subject":[],"published":{"date-parts":[[2025,8]]}}}