{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,1,14]],"date-time":"2026-01-14T13:17:08Z","timestamp":1768396628185,"version":"3.49.0"},"publisher-location":"Singapore","reference-count":39,"publisher":"Springer Nature Singapore","isbn-type":[{"value":"9789819669479","type":"print"},{"value":"9789819669486","type":"electronic"}],"license":[{"start":{"date-parts":[[2025,1,1]],"date-time":"2025-01-01T00:00:00Z","timestamp":1735689600000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2025,1,1]],"date-time":"2025-01-01T00:00:00Z","timestamp":1735689600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2025]]},"DOI":"10.1007\/978-981-96-6948-6_25","type":"book-chapter","created":{"date-parts":[[2025,6,21]],"date-time":"2025-06-21T13:44:25Z","timestamp":1750513465000},"page":"354-367","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":0,"title":["Contrastive Novelty for\u00a0Anti-exploration in\u00a0Offline Reinforcement Learning"],"prefix":"10.1007","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-3918-8461","authenticated-orcid":false,"given":"Xuemin","family":"Hu","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0009-0006-8636-2038","authenticated-orcid":false,"given":"Shen","family":"Li","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0009-0000-1251-7574","authenticated-orcid":false,"given":"Yingfen","family":"Xu","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0009-0008-9043-5863","authenticated-orcid":false,"given":"Yongsheng","family":"Pan","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-5708-766X","authenticated-orcid":false,"given":"Bo","family":"Tang","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-4925-0572","authenticated-orcid":false,"given":"Long","family":"Chen","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2025,6,22]]},"reference":[{"issue":"11","key":"25_CR1","doi-asserted-by":"publisher","first-page":"1238","DOI":"10.1177\/0278364913495721","volume":"32","author":"J Kober","year":"2013","unstructured":"Kober, J., Bagnell, J.A., Peters, J.: Reinforcement learning in robotics: a survey. Int. J. Robot. Res. 32(11), 1238\u20131274 (2013)","journal-title":"Int. J. Robot. Res."},{"key":"25_CR2","doi-asserted-by":"crossref","unstructured":"Agand, P., Iskrov, A., Chen, M.: Deep reinforcement learning-based intelligent traffic signal controls with optimized co2 emissions. In: Proceedings of the IEEE\/RSJ International Conf. Intelligent Robots Syst. (IROS), pp. 5495\u20135500. IEEE (2023)","DOI":"10.1109\/IROS55552.2023.10341972"},{"key":"25_CR3","doi-asserted-by":"crossref","unstructured":"B\u00fchrer, N., Zhang, Z., Liniger, A., Yu, F., Van\u00a0Gool, L.: A multiplicative value function for safe and efficient reinforcement learning. In: Proceedings of the IEEE\/RSJ International Conference Intelligent Robots System (IROS), pp. 5582\u20135589. IEEE (2023)","DOI":"10.1109\/IROS55552.2023.10342288"},{"key":"25_CR4","unstructured":"Badia, A.P., et al.: Agent57: Outperforming the atari human benchmark. In: Proceedings of the International Conference Machine Learning, (ICML), pp. 507\u2013517. PMLR (2020)"},{"issue":"12","key":"25_CR5","doi-asserted-by":"publisher","first-page":"7585","DOI":"10.1109\/TITS.2020.3004984","volume":"22","author":"X Hu","year":"2020","unstructured":"Hu, X., Tang, B., Chen, L., Song, S., Tong, X.: Learning a deep cascaded neural network for multiple motion commands prediction in autonomous driving. IEEE Trans. Intell. Transp. Syst. 22(12), 7585\u20137596 (2020)","journal-title":"IEEE Trans. Intell. Transp. Syst."},{"issue":"1","key":"25_CR6","doi-asserted-by":"publisher","first-page":"593","DOI":"10.1109\/TIV.2023.3312777","volume":"9","author":"X Hu","year":"2024","unstructured":"Hu, X., Li, S., Huang, T., Tang, B., Huai, R., Chen, L.: How simulation helps autonomous driving: a survey of sim2real, digital twins, and parallel intelligence. IEEE Trans. Intell. Vehicles. 9(1), 593\u2013612 (2024)","journal-title":"IEEE Trans. Intell. Vehicles."},{"issue":"1","key":"25_CR7","doi-asserted-by":"publisher","first-page":"236","DOI":"10.1109\/JAS.2018.7511186","volume":"6","author":"L Chen","year":"2018","unstructured":"Chen, L., Hu, X., Tian, W., Wang, H., Cao, D., Wang, F.Y.: Parallel planning: a new motion planning framework for autonomous driving. IEEE\/CAA J. Autom. Sinica 6(1), 236\u2013246 (2018)","journal-title":"IEEE\/CAA J. Autom. Sinica"},{"issue":"3","key":"25_CR8","doi-asserted-by":"publisher","first-page":"362","DOI":"10.1002\/rob.21918","volume":"37","author":"S Grigorescu","year":"2020","unstructured":"Grigorescu, S., Trasnea, B., Cocias, T., Macesanu, G.: A survey of deep learning techniques for autonomous driving. J. Field Robot. 37(3), 362\u2013386 (2020)","journal-title":"J. Field Robot."},{"key":"25_CR9","doi-asserted-by":"publisher","DOI":"10.1109\/TITS.2023.3287223","volume-title":"Learning Dynamic Graph for Overtaking Strategy in Autonomous Driving","author":"X Hu","year":"2023","unstructured":"Hu, X., Liu, Y., Tang, B., Yan, J., Chen, L.: Learning Dynamic Graph for Overtaking Strategy in Autonomous Driving. IEEE Trans. Intell. Transp, Syst (2023)"},{"issue":"1","key":"25_CR10","first-page":"36","volume":"11","author":"L Chen","year":"2018","unstructured":"Chen, L., Hu, X., Tang, B., Cao, D.: Parallel motion planning: learning a deep planning model against emergencies. IEEE Intell. Trans. Syst. Mag. 11(1), 36\u201341 (2018)","journal-title":"IEEE Intell. Trans. Syst. Mag."},{"key":"25_CR11","doi-asserted-by":"crossref","unstructured":"Boase, D., Gueaieb, W., Miah, M.S.: Underactuated mimo airship control based on online data-driven reinforcement learning. In: Proceedings of the IEEE\/RSJ International Conference Intelligent Robots System, (IROS), pp. 9464\u20139471. IEEE (2023)","DOI":"10.1109\/IROS55552.2023.10341752"},{"key":"25_CR12","doi-asserted-by":"crossref","unstructured":"Chen, C., Xiang, P., Lu, H., Wang, Y., Xiong, R.: C 2: Co-design of robots via concurrent-network coupling online and offline reinforcement learning. In: Proc. IEEE\/RSJ International Conference Intelligent Robots System, (IROS), pp. 7487\u20137494. IEEE (2023)","DOI":"10.1109\/IROS55552.2023.10341983"},{"key":"25_CR13","doi-asserted-by":"crossref","unstructured":"Ye, R., et\u00a0al.: Reinforcement learning for robot navigation with adaptive forward simulation time (afst) in a semi-markov model. In: Proceedings of the IEEE\/RSJ International Conference Intelligent Robots Syst. (IROS), pp. 3597\u20133604. IEEE (2023)","DOI":"10.1109\/IROS55552.2023.10341985"},{"key":"25_CR14","doi-asserted-by":"crossref","unstructured":"Rezaeifar, S., et al.: Offline reinforcement learning as anti-exploration. In: Proceedings of the AAAI, vol.\u00a036, pp. 8106\u20138114 (2022)","DOI":"10.1609\/aaai.v36i7.20783"},{"key":"25_CR15","unstructured":"Kumar, A., Fu, J., Soh, M., Tucker, G., Levine, S.: Stabilizing off-policy q-learning via bootstrapping error reduction. In: Proceedings of the Advance Neural Information Processing System (NeurIPS), vol. 32 (2019)"},{"key":"25_CR16","unstructured":"Fujimoto, S., Hoof, H., Meger, D.: Addressing function approximation error in actor-critic methods. In: Proceedings of the International Conference Machine Learning (ICML), pp. 1587\u20131596. PMLR (2018)"},{"key":"25_CR17","unstructured":"Fujimoto, S., Meger, D., Precup, D.: Off-policy deep reinforcement learning without exploration. In: Proceedings of the International Conference Machine Learning (ICML), pp. 2052\u20132062. PMLR (2019)"},{"key":"25_CR18","unstructured":"Wu, Y., et al.: Uncertainty weighted actor-critic for offline reinforcement learning. In: Proceedings of the International Conference Machine Learning (ICML), pp. 11319\u201311328. PMLR (2021)"},{"key":"25_CR19","first-page":"20132","volume":"34","author":"S Fujimoto","year":"2021","unstructured":"Fujimoto, S., Gu, S.S.: A minimalist approach to offline reinforcement learning. Proc. Adv. Neural Inf. Process. Syst. (NeurIPS) 34, 20132\u201320145 (2021)","journal-title":"Proc. Adv. Neural Inf. Process. Syst. (NeurIPS)"},{"key":"25_CR20","unstructured":"Nair, A., Gupta, A., Dalal, M., Levine, S.: Awac: Accelerating online reinforcement learning with offline datasets. arXiv preprint arXiv:2006.09359 (2020)"},{"key":"25_CR21","volume-title":"A Survey on Offline Reinforcement Learning: Taxonomy, Review, and Open Problems","author":"RF Prudencio","year":"2023","unstructured":"Prudencio, R.F., Maximo, M.R., Colombini, E.L.: A Survey on Offline Reinforcement Learning: Taxonomy, Review, and Open Problems. IEEE Trans. Neural Netw. Learn, Syst (2023)"},{"key":"25_CR22","first-page":"1179","volume":"33","author":"A Kumar","year":"2020","unstructured":"Kumar, A., Zhou, A., Tucker, G., Levine, S.: Conservative q-learning for offline reinforcement learning. Proc. Adv. Neural Inf. Process. Syst. (NeurIPS) 33, 1179\u20131191 (2020)","journal-title":"Proc. Adv. Neural Inf. Process. Syst. (NeurIPS)"},{"key":"25_CR23","first-page":"18267","volume":"35","author":"K Ghasemipour","year":"2022","unstructured":"Ghasemipour, K., Gu, S.S., Nachum, O.: Why so pessimistic? estimating uncertainties for offline RL through ensembles, and why their independence matters. Proc. Adv. Neural Inf. Process. Syst. (NeurIPS) 35, 18267\u201318281 (2022)","journal-title":"Proc. Adv. Neural Inf. Process. Syst. (NeurIPS)"},{"key":"25_CR24","unstructured":"Kostrikov, I., Nair, A., Levine, S.: Offline reinforcement learning with implicit q-learning. arXiv preprint arXiv:2110.06169 (2021)"},{"key":"25_CR25","unstructured":"Hansen-Estruch, P., Kostrikov, I., Janner, M., Kuba, J.G., Levine, S.: Idql: implicit q-learning as an actor-critic method with diffusion policies. arXiv preprint arXiv:2304.10573 (2023)"},{"key":"25_CR26","first-page":"7436","volume":"34","author":"G An","year":"2021","unstructured":"An, G., Moon, S., Kim, J.H., Song, H.O.: Uncertainty-based offline reinforcement learning with diversified q-ensemble. Proc. Adv. Neural Inf. Process. Syst. (NeurIPS) 34, 7436\u20137447 (2021)","journal-title":"Proc. Adv. Neural Inf. Process. Syst. (NeurIPS)"},{"key":"25_CR27","first-page":"14129","volume":"33","author":"T Yu","year":"2020","unstructured":"Yu, T., et al.: MOPO: Model-based offline policy optimization. Proc. Adv. Neural Inf. Process. Syst. (NeurIPS) 33, 14129\u201314142 (2020)","journal-title":"Proc. Adv. Neural Inf. Process. Syst. (NeurIPS)"},{"key":"25_CR28","unstructured":"Feng, Y., Ren, T., Tang, Z., Liu, Q.: Accountable off-policy evaluation with kernel bellman statistics. In: Proceedings of the International Conference Machine Learning (ICML), pp. 3102\u20133111. PMLR (2020)"},{"key":"25_CR29","first-page":"9398","volume":"33","author":"B Dai","year":"2020","unstructured":"Dai, B., Nachum, O., Chow, Y., Li, L., Szepesv\u00e1ri, C., Schuurmans, D.: CoinDICE: off-policy confidence interval estimation. Proc. Adv. Neural Inf. Process. Syst. (NeurIPS) 33, 9398\u20139411 (2020)","journal-title":"Proc. Adv. Neural Inf. Process. Syst. (NeurIPS)"},{"key":"25_CR30","unstructured":"Yang, M., Dai, B., Nachum, O., Tucker, G., Schuurmans, D.: Offline policy selection under uncertainty. In: International Conference on Artificial Intelligence and Statistics pp. 4376\u20134396. PMLR (2022)"},{"key":"25_CR31","unstructured":"Kostrikov, I., Nachum, O.: Statistical bootstrapping for uncertainty estimation in off-policy evaluation. arXiv preprint arXiv:2007.13609 (2020)"},{"key":"25_CR32","unstructured":"Nikulin, A., Kurenkov, V., Tarasov, D., Kolesnikov, S.: Anti-exploration by random network distillation. In: Proceedings of the Internatinal Conference Machine Learning (ICML), PMLR (2023)"},{"key":"25_CR33","unstructured":"Fu, J., Kumar, A., Nachum, O., Tucker, G., Levine, S.: D4rl: datasets for deep data-driven reinforcement learning. arXiv preprint arXiv:2004.07219 (2020)"},{"key":"25_CR34","unstructured":"Haarnoja, T., Zhou, A., Abbeel, P., Levine, S.: Soft actor-critic: off-policy maximum entropy deep reinforcement learning with a stochastic actor. In: Proceedings of the International Conference Machine Learning. (ICML), pp. 1861\u20131870. PMLR (2018)"},{"key":"25_CR35","first-page":"15084","volume":"34","author":"L Chen","year":"2021","unstructured":"Chen, L., et al.: Decision transformer: reinforcement learning via sequence modeling. Proc. Adv. Neural Inf. Process. Syst. (NeurIPS) 34, 15084\u201315097 (2021)","journal-title":"Proc. Adv. Neural Inf. Process. Syst. (NeurIPS)"},{"key":"25_CR36","unstructured":"Bai, C., et al.: Pessimistic bootstrapping for uncertainty-driven offline reinforcement learning. arXiv preprint arXiv:2202.11566 (2022)"},{"key":"25_CR37","first-page":"31278","volume":"35","author":"J Wu","year":"2022","unstructured":"Wu, J., Wu, H., Qiu, Z., Wang, J., Long, M.: Supported policy optimization for offline reinforcement learning. Proc. Adv. Neural Inf. Process. Syst. (NeurIPS) 35, 31278\u201331291 (2022)","journal-title":"Proc. Adv. Neural Inf. Process. Syst. (NeurIPS)"},{"key":"25_CR38","unstructured":"Tarasov, D., Kurenkov, V., Nikulin, A., Kolesnikov, S.: Revisiting the minimalist approach to offline reinforcement learning. Proc. Adv. Neural Inf. Process. Syst. (NeurIPS) 36 (2024)"},{"key":"25_CR39","unstructured":"Mao, Y., Zhang, H., Chen, C., Xu, Y., Ji, X.: Supported trust region optimization for offline reinforcement learning. In: Proceedings International Conference Maching Learning (ICML), pp. 23829\u201323851. PMLR (2023)"}],"container-title":["Communications in Computer and Information Science","Neural Information Processing"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-981-96-6948-6_25","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,1,14]],"date-time":"2026-01-14T08:06:53Z","timestamp":1768378013000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-981-96-6948-6_25"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025]]},"ISBN":["9789819669479","9789819669486"],"references-count":39,"URL":"https:\/\/doi.org\/10.1007\/978-981-96-6948-6_25","relation":{},"ISSN":["1865-0929","1865-0937"],"issn-type":[{"value":"1865-0929","type":"print"},{"value":"1865-0937","type":"electronic"}],"subject":[],"published":{"date-parts":[[2025]]},"assertion":[{"value":"22 June 2025","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"ICONIP","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"International Conference on Neural Information Processing","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Auckland","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"New Zealand","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2024","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2 December 2024","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"6 December 2024","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"31","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"iconip2024","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"http:\/\/iconip2024.org","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}}]}}