{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,1,17]],"date-time":"2026-01-17T19:58:20Z","timestamp":1768679900281,"version":"3.49.0"},"reference-count":40,"publisher":"Springer Science and Business Media LLC","issue":"23","license":[{"start":{"date-parts":[[2024,9,11]],"date-time":"2024-09-11T00:00:00Z","timestamp":1726012800000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2024,9,11]],"date-time":"2024-09-11T00:00:00Z","timestamp":1726012800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"funder":[{"DOI":"10.13039\/100014718","name":"Innovative Research Group Project of the National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["62062033"],"award-info":[{"award-number":["62062033"]}],"id":[{"id":"10.13039\/100014718","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100004479","name":"Natural Science Foundation of Jiangxi Province","doi-asserted-by":"publisher","award":["20232BAB202018"],"award-info":[{"award-number":["20232BAB202018"]}],"id":[{"id":"10.13039\/501100004479","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Appl Intell"],"published-print":{"date-parts":[[2024,12]]},"DOI":"10.1007\/s10489-024-05806-2","type":"journal-article","created":{"date-parts":[[2024,9,11]],"date-time":"2024-09-11T03:30:51Z","timestamp":1726025451000},"page":"12246-12271","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":3,"title":["An offline-to-online reinforcement learning approach based on multi-action evaluation with policy extension"],"prefix":"10.1007","volume":"54","author":[{"given":"Xuebo","family":"Cheng","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-7269-4484","authenticated-orcid":false,"given":"Xiaohui","family":"Huang","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Zhichao","family":"Huang","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Nan","family":"Jiang","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2024,9,11]]},"reference":[{"key":"5806_CR1","unstructured":"Fu J, Kumar A, Nachum O, Tucker G, Levine S (2020) D4rl: Datasets for deep data-driven reinforcement learning. arXiv preprint arXiv:2004.07219"},{"key":"5806_CR2","unstructured":"McDonald MJ, Hadfield-Menell D (2022) Guided imitation of task and motion planning. Proceedings of the 5th conference on robot learning. 164:630\u2013640"},{"key":"5806_CR3","doi-asserted-by":"publisher","DOI":"10.1016\/j.knosys.2023.110335","volume":"264","author":"X Chen","year":"2023","unstructured":"Chen X, Yao L, McAuley J, Zhou G, Wang X (2023) Deep reinforcement learning in recommender systems: A survey and new perspectives. Knowl-Based Syst 264:110335","journal-title":"Knowl-Based Syst"},{"key":"5806_CR4","doi-asserted-by":"crossref","unstructured":"Sharif\u00a0Razavian A, Azizpour H, Sullivan J, Carlsson S (2014) CNN features off-the-shelf: An astounding baseline for recognition. Proceedings of the IEEE conference on computer vision and pattern recognition (CVPR) workshops","DOI":"10.1109\/CVPRW.2014.131"},{"key":"5806_CR5","first-page":"1025","volume":"100","author":"A Gupta","year":"2020","unstructured":"Gupta A, Kumar V, Lynch C, Levine S, Hausman K (2020) Relay policy learning: Solving long-horizon tasks via imitation and reinforcement learning. Proceedings of the Conference on Robot Learning. 100:1025\u20131037","journal-title":"Proceedings of the Conference on Robot Learning."},{"key":"5806_CR6","unstructured":"Fujimoto S, Meger D, Precup D (2019) Off-policy deep reinforcement learning without exploration. Proceedings of the 36th international conference on machine learning. 97:2052\u20132062"},{"key":"5806_CR7","first-page":"12686","volume":"34","author":"M Schwarzer","year":"2021","unstructured":"Schwarzer M, Rajkumar N, Noukhovitch M, Anand A, Charlin L, Hjelm RD, Bachman P, Courville AC (2021) Pretraining representations for data-efficient reinforcement learning. Adv Neural Inf Process Syst 34:12686\u201312699","journal-title":"Adv Neural Inf Process Syst"},{"key":"5806_CR8","unstructured":"Kenton JDM-WC, Toutanova LK (2019) Bert: Pre-training of deep bidirectional transformers for language understanding. Proceedings of North American chapter of the association for computational linguistics, pp 4171\u20134186"},{"key":"5806_CR9","unstructured":"Campos V, Sprechmann P, Hansen S, Barreto A, Kapturowski S, Vitvitskyi A, Badia AP, Blundell C (2021) Beyond fine-tuning: Transferring behavior in reinforcement learning. Proceedings of the international conference on machine learning 2021 workshop on unsupervised reinforcement learning"},{"key":"5806_CR10","unstructured":"Nair A, Gupta A, Dalal M, Levine S (2020) Awac: Accelerating online reinforcement learning with offline datasets. arXiv preprint arXiv:2006.09359"},{"key":"5806_CR11","first-page":"62244","volume":"36","author":"M Nakamoto","year":"2023","unstructured":"Nakamoto M, Zhai S, Singh A, Sobol Mark M, Ma Y, Finn C, Kumar A, Levine S (2023) Cal-ql: Calibrated offline rl pre-training for efficient online fine-tuning. Adv Neural Inf Process Syst 36:62244\u201362269","journal-title":"Adv Neural Inf Process Syst"},{"key":"5806_CR12","unstructured":"Kumar A, Fu J, Soh M, Tucker G, Levine S (2019) Stabilizing off-policy q-learning via bootstrapping error reduction. Adv Neural Inform Process Syst 32"},{"key":"5806_CR13","unstructured":"Zhang H, Xu W, Yu H (2023) Policy expansion for bridging offline-to-online reinforcement learning. Proceedings of the eleventh international conference on learning representations"},{"key":"5806_CR14","unstructured":"Seo Y, Lee K, James SL, Abbeel P (2022) Reinforcement learning with action-free pre-training from videos. Proceedings of the 39th international conference on machine learning, 162:19561\u201319579"},{"key":"5806_CR15","first-page":"8788","volume":"36","author":"S Son","year":"2023","unstructured":"Son S, Zheng L, Sullivan R, Qiao Y-L, Lin M (2023) Gradient informed proximal policy optimization. Adv Neural Inf Process Syst 36:8788\u20138814","journal-title":"Adv Neural Inf Process Syst"},{"key":"5806_CR16","first-page":"61573","volume":"36","author":"S Fujimoto","year":"2023","unstructured":"Fujimoto S, Chang W-D, Smith E, Gu SS, Precup D, Meger D (2023) For sale: State-action representation learning for deep reinforcement learning. Adv Neural Inf Process Syst 36:61573\u201361624","journal-title":"Adv Neural Inf Process Syst"},{"key":"5806_CR17","unstructured":"Bhatt A, Palenicek D, Belousov B, Argus M, Amiranashvili A, Brox T, Peters J (2024) Crossq: Batch normalization in deep reinforcement learning for greater sample efficiency and simplicity. Proceedings of the international conference on learning representations (ICLR)"},{"issue":"21","key":"5806_CR18","doi-asserted-by":"publisher","first-page":"24847","DOI":"10.1007\/s10489-023-04867-z","volume":"53","author":"R Li","year":"2023","unstructured":"Li R, Shang Z, Zheng C, Li H, Liang Q, Cui Y (2023) Efficient distributional reinforcement learning with kullback-leibler divergence regularization. Appl Intell 53(21):24847\u201324863","journal-title":"Appl Intell"},{"key":"5806_CR19","doi-asserted-by":"crossref","unstructured":"Shang Z, Li R, Zheng C, Li H, Cui Y (2023) Relative entropy regularized sample-efficient reinforcement learning with continuous actions. IEEE Trans Neural Netw Learn Syst pp 1\u201311","DOI":"10.1109\/TNNLS.2023.3329513"},{"key":"5806_CR20","unstructured":"Hiraoka T, Imagawa T, Hashimoto T, Onishi T, Tsuruoka Y (2022) Dropout q-functions for doubly efficient reinforcement learning. Proceedings of the tenth international conference on learning representations, ICLR 2022, Virtual Event, April 25\u201329 2022"},{"issue":"2","key":"5806_CR21","doi-asserted-by":"publisher","first-page":"581","DOI":"10.1007\/s10489-018-1296-x","volume":"49","author":"X Zhao","year":"2019","unstructured":"Zhao X, Ding S, An Y, Jia W (2019) Applications of asynchronous deep reinforcement learning based on dynamic updating weights. Appl Intell 49(2):581\u2013591","journal-title":"Appl Intell"},{"issue":"12","key":"5806_CR22","doi-asserted-by":"publisher","first-page":"4303","DOI":"10.1007\/s10489-019-01501-9","volume":"49","author":"S Ding","year":"2019","unstructured":"Ding S, Zhao X, Xu X, Sun T, Jia W (2019) An effective asynchronous framework for small scale reinforcement learning problems. Appl Intell 49(12):4303\u20134318","journal-title":"Appl Intell"},{"key":"5806_CR23","doi-asserted-by":"publisher","DOI":"10.1016\/j.patcog.2023.109917","volume":"145","author":"X Du","year":"2024","unstructured":"Du X, Chen H, Wang C, Xing Y, Yang J, Yu PS, Chang Y, He L (2024) Robust multi-agent reinforcement learning via bayesian distributional value estimation. Pattern Recogn 145:109917","journal-title":"Pattern Recogn"},{"key":"5806_CR24","unstructured":"Ciosek K, Vuong Q, Loftin R, Hofmann K (2019) Better exploration with optimistic actor critic. Adv Neural Inf Process Syst 32:103368"},{"key":"5806_CR25","first-page":"31278","volume":"35","author":"J Wu","year":"2022","unstructured":"Wu J, Wu H, Qiu Z, Wang J, Long M (2022) Supported policy optimization for offline reinforcement learning. Adv Neural Inf Process Syst 35:31278\u201331291","journal-title":"Adv Neural Inf Process Syst"},{"key":"5806_CR26","first-page":"20132","volume":"34","author":"S Fujimoto","year":"2021","unstructured":"Fujimoto S, Gu SS (2021) A minimalist approach to offline reinforcement learning. Adv Neural Inf Process Syst 34:20132\u201320145","journal-title":"Adv Neural Inf Process Syst"},{"key":"5806_CR27","first-page":"1179","volume":"33","author":"A Kumar","year":"2020","unstructured":"Kumar A, Zhou A, Tucker G, Levine S (2020) Conservative q-learning for offline reinforcement learning. Adv Neural Inf Process Syst 33:1179\u20131191","journal-title":"Adv Neural Inf Process Syst"},{"key":"5806_CR28","unstructured":"Kostrikov I, Fergus R, Tompson J, Nachum O (2021) Offline reinforcement learning with fisher divergence critic regularization. Proceedings of the 38th international conference on machine learning, 139:5774\u20135783"},{"key":"5806_CR29","unstructured":"Kidambi R, Rajeswaran A, Netrapalli P, Joachims T (2020) Morel: model-based offline reinforcement learning. Proceedings of the 34th international conference on neural information processing systems"},{"key":"5806_CR30","first-page":"14129","volume":"33","author":"T Yu","year":"2020","unstructured":"Yu T, Thomas G, Yu L, Ermon S, Zou JY, Levine S, Finn C, Ma T (2020) Mopo: Model-based offline policy optimization. Adv Neural Inf Process Syst 33:14129\u201314142","journal-title":"Adv Neural Inf Process Syst"},{"key":"5806_CR31","first-page":"18459","volume":"34","author":"H Liu","year":"2021","unstructured":"Liu H, Abbeel P (2021) Behavior from the void: Unsupervised active pre-training. Adv Neural Inf Process Syst 34:18459\u201318473","journal-title":"Adv Neural Inf Process Syst"},{"key":"5806_CR32","first-page":"31278","volume":"35","author":"J Wu","year":"2022","unstructured":"Wu J, Wu H, Qiu Z, Wang J, Long M (2022) Supported policy optimization for offline reinforcement learning. Adv Neural Inf Process Syst 35:31278\u201331291","journal-title":"Adv Neural Inf Process Syst"},{"key":"5806_CR33","unstructured":"Lee S, Seo Y, Lee K, Abbeel P, Shin J (2022) Offline-to-online reinforcement learning via balanced replay and pessimistic q-ensemble. Proceedings of the 5th conference on robot learning, 164:1702\u20131712"},{"key":"5806_CR34","unstructured":"Yang M, Nachum O (2021) Representation matters: Offline pretraining for sequential decision making. Proceedings of the 38th international conference on machine learning. 139:11784\u201311794"},{"key":"5806_CR35","unstructured":"Uchendu I, Xiao T, Lu Y, Zhu B, Yan M, Simon J, Bennice M, Fu C, Ma C, Jiao J, Levine S, Hausman K (2023) Jump-start reinforcement learning. Proceedings of the 40th international conference on machine learning, 202:34556\u201334583"},{"key":"5806_CR36","unstructured":"Kostrikov I, Nair A, Levine S (2021) Offline reinforcement learning with implicit q-learning. Advances in deep reinforcement learning workshop conference on neural information processing systems"},{"key":"5806_CR37","unstructured":"Sundhar\u00a0Ramesh S, Giuseppe\u00a0Sessa P, Hu Y, Krause A, Bogunovic I (2024) Distributionally robust model-based reinforcement learning with large state spaces. Proceedings of the 27th international conference on artificial intelligence and statistics, 238:100\u2013108"},{"issue":"3","key":"5806_CR38","doi-asserted-by":"publisher","first-page":"1299","DOI":"10.1109\/TKDE.2023.3302804","volume":"36","author":"S Guo","year":"2024","unstructured":"Guo S, Zou L, Chen H, Qu B, Chi H, Yu PS, Chang Y (2024) Sample efficient offline-to-online reinforcement learning. IEEE Trans Knowl Data Eng 36(3):1299\u20131310","journal-title":"IEEE Trans Knowl Data Eng"},{"key":"5806_CR39","unstructured":"Li P, Tang H, Yang T, Hao X, Sang T, Zheng Y, Hao J, Taylor ME, Tao W, Wang Z (2022) PMIC: Improving multi-agent reinforcement learning with progressive mutual information collaboration. Proceedings of the 39th international conference on machine learning, 162:12979\u201312997"},{"key":"5806_CR40","unstructured":"Haarnoja T, Zhou A, Abbeel P, Levine S (2018) Soft actor-critic: Off-policy maximum entropy deep reinforcement learning with a stochastic actor. Proceedings of the 35th international conference on machine learning, 80:1861\u20131870"}],"container-title":["Applied Intelligence"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10489-024-05806-2.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s10489-024-05806-2\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10489-024-05806-2.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,9,30]],"date-time":"2024-09-30T13:09:03Z","timestamp":1727701743000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s10489-024-05806-2"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,9,11]]},"references-count":40,"journal-issue":{"issue":"23","published-print":{"date-parts":[[2024,12]]}},"alternative-id":["5806"],"URL":"https:\/\/doi.org\/10.1007\/s10489-024-05806-2","relation":{},"ISSN":["0924-669X","1573-7497"],"issn-type":[{"value":"0924-669X","type":"print"},{"value":"1573-7497","type":"electronic"}],"subject":[],"published":{"date-parts":[[2024,9,11]]},"assertion":[{"value":"22 August 2024","order":1,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"11 September 2024","order":2,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Declarations"}},{"value":"Relevant ethical guidelines and regulations were followed in conducting this study.","order":2,"name":"Ethics","group":{"name":"EthicsHeading","label":"Ethical and informed consent for data used"}},{"value":"The authors declare that there are no potential competing interests or conflicting relationships in this study.","order":3,"name":"Ethics","group":{"name":"EthicsHeading","label":"Competing Interests"}}]}}