{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,8,22]],"date-time":"2025-08-22T14:10:08Z","timestamp":1755871808690,"version":"3.44.0"},"publisher-location":"New York, NY, USA","reference-count":44,"publisher":"ACM","license":[{"start":{"date-parts":[[2024,1,4]],"date-time":"2024-01-04T00:00:00Z","timestamp":1704326400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2024,1,4]]},"DOI":"10.1145\/3632410.3632423","type":"proceedings-article","created":{"date-parts":[[2024,1,3]],"date-time":"2024-01-03T18:15:16Z","timestamp":1704305716000},"page":"82-90","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":1,"title":["Guiding Offline Reinforcement Learning Using a Safety Expert"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0001-9579-0559","authenticated-orcid":false,"given":"Richa","family":"Verma","sequence":"first","affiliation":[{"name":"Tata Consultancy Services Ltd, India"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-6392-7266","authenticated-orcid":false,"given":"Durgesh","family":"Kalwar","sequence":"additional","affiliation":[{"name":"Tata Consultancy Services Ltd, India"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-3601-778X","authenticated-orcid":false,"given":"Harshad","family":"Khadilkar","sequence":"additional","affiliation":[{"name":"Tata Consultancy Services Ltd, India"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-5364-7639","authenticated-orcid":false,"given":"Balaraman","family":"Ravindran","sequence":"additional","affiliation":[{"name":"Indian Institute of Technology Madras, India, India"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2024,1,4]]},"reference":[{"key":"e_1_3_2_1_1_1","volume-title":"International conference on machine learning. PMLR, 22\u201331","author":"Achiam Joshua","year":"2017","unstructured":"Joshua Achiam, David Held, Aviv Tamar, and Pieter Abbeel. 2017. Constrained policy optimization. In International conference on machine learning. PMLR, 22\u201331."},{"key":"e_1_3_2_1_2_1","volume-title":"The importance of pessimism in fixed-dataset policy optimization. arXiv preprint arXiv:2009.06799","author":"Buckman Jacob","year":"2020","unstructured":"Jacob Buckman, Carles Gelada, and Marc\u00a0G Bellemare. 2020. The importance of pessimism in fixed-dataset policy optimization. arXiv preprint arXiv:2009.06799 (2020)."},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10458-019-09430-0"},{"key":"e_1_3_2_1_4_1","volume-title":"D4rl: Datasets for deep data-driven reinforcement learning. arXiv preprint arXiv:2004.07219","author":"Fu Justin","year":"2020","unstructured":"Justin Fu, Aviral Kumar, Ofir Nachum, George Tucker, and Sergey Levine. 2020. D4rl: Datasets for deep data-driven reinforcement learning. arXiv preprint arXiv:2004.07219 (2020)."},{"key":"e_1_3_2_1_5_1","volume-title":"Benchmarking batch deep reinforcement learning algorithms. arXiv preprint arXiv:1910.01708","author":"Fujimoto Scott","year":"2019","unstructured":"Scott Fujimoto, Edoardo Conti, Mohammad Ghavamzadeh, and Joelle Pineau. 2019. Benchmarking batch deep reinforcement learning algorithms. arXiv preprint arXiv:1910.01708 (2019)."},{"key":"e_1_3_2_1_6_1","volume-title":"A minimalist approach to offline reinforcement learning. Advances in neural information processing systems 34","author":"Fujimoto Scott","year":"2021","unstructured":"Scott Fujimoto and Shixiang\u00a0Shane Gu. 2021. A minimalist approach to offline reinforcement learning. Advances in neural information processing systems 34 (2021), 20132\u201320145."},{"key":"e_1_3_2_1_7_1","volume-title":"International conference on machine learning. PMLR","author":"Fujimoto Scott","year":"2019","unstructured":"Scott Fujimoto, David Meger, and Doina Precup. 2019. Off-policy deep reinforcement learning without exploration. In International conference on machine learning. PMLR, 2052\u20132062."},{"key":"e_1_3_2_1_8_1","volume-title":"Conference on Robot Learning. PMLR, 1259\u20131277","author":"Kamyar\u00a0Seyed Ghasemipour Seyed","year":"2020","unstructured":"Seyed Kamyar\u00a0Seyed Ghasemipour, Richard Zemel, and Shixiang Gu. 2020. A divergence minimization perspective on imitation learning methods. In Conference on Robot Learning. PMLR, 1259\u20131277."},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICRA.2017.7989385"},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v32i1.11757"},{"key":"e_1_3_2_1_11_1","volume-title":"ThriftyDAgger: Budget-aware novelty and risk gating for interactive imitation learning. arXiv preprint arXiv:2109.08273","author":"Hoque Ryan","year":"2021","unstructured":"Ryan Hoque, Ashwin Balakrishna, Ellen Novoseller, Albert Wilcox, Daniel\u00a0S Brown, and Ken Goldberg. 2021. ThriftyDAgger: Budget-aware novelty and risk gating for interactive imitation learning. arXiv preprint arXiv:2109.08273 (2021)."},{"key":"e_1_3_2_1_12_1","volume-title":"Way off-policy batch deep reinforcement learning of implicit human preferences in dialog. arXiv preprint arXiv:1907.00456","author":"Jaques Natasha","year":"2019","unstructured":"Natasha Jaques, Asma Ghandeharioun, Judy\u00a0Hanwen Shen, Craig Ferguson, Agata Lapedriza, Noah Jones, Shixiang Gu, and Rosalind Picard. 2019. Way off-policy batch deep reinforcement learning of implicit human preferences in dialog. arXiv preprint arXiv:1907.00456 (2019)."},{"key":"e_1_3_2_1_13_1","volume-title":"Offline reinforcement learning with implicit q-learning. arXiv preprint arXiv:2110.06169","author":"Kostrikov Ilya","year":"2021","unstructured":"Ilya Kostrikov, Ashvin Nair, and Sergey Levine. 2021. Offline reinforcement learning with implicit q-learning. arXiv preprint arXiv:2110.06169 (2021)."},{"key":"e_1_3_2_1_14_1","volume-title":"Stabilizing off-policy q-learning via bootstrapping error reduction. Advances in Neural Information Processing Systems 32","author":"Kumar Aviral","year":"2019","unstructured":"Aviral Kumar, Justin Fu, Matthew Soh, George Tucker, and Sergey Levine. 2019. Stabilizing off-policy q-learning via bootstrapping error reduction. Advances in Neural Information Processing Systems 32 (2019)."},{"key":"e_1_3_2_1_15_1","volume-title":"Offline Robot Reinforcement Learning with Uncertainty-Guided Human Expert Sampling. arXiv preprint arXiv:2212.08232","author":"Kumar Ashish","year":"2022","unstructured":"Ashish Kumar and Ilya Kuzovkin. 2022. Offline Robot Reinforcement Learning with Uncertainty-Guided Human Expert Sampling. arXiv preprint arXiv:2212.08232 (2022)."},{"key":"e_1_3_2_1_16_1","first-page":"1179","article-title":"Conservative q-learning for offline reinforcement learning","volume":"33","author":"Kumar Aviral","year":"2020","unstructured":"Aviral Kumar, Aurick Zhou, George Tucker, and Sergey Levine. 2020. Conservative q-learning for offline reinforcement learning. Advances in Neural Information Processing Systems 33 (2020), 1179\u20131191.","journal-title":"Advances in Neural Information Processing Systems"},{"volume-title":"Reinforcement learning","author":"Lange Sascha","key":"e_1_3_2_1_17_1","unstructured":"Sascha Lange, Thomas Gabel, and Martin Riedmiller. 2012. Batch reinforcement learning. In Reinforcement learning. Springer, 45\u201373."},{"key":"e_1_3_2_1_18_1","volume-title":"International Conference on Machine Learning. PMLR, 3652\u20133661","author":"Laroche Romain","year":"2019","unstructured":"Romain Laroche, Paul Trichelair, and Remi\u00a0Tachet Des\u00a0Combes. 2019. Safe policy improvement with baseline bootstrapping. In International Conference on Machine Learning. PMLR, 3652\u20133661."},{"key":"e_1_3_2_1_19_1","unstructured":"Seunghyun Lee Younggyo Seo Kimin Lee Pieter Abbeel and Jinwoo Shin. 2020. Addressing Distribution Shift in Online Reinforcement Learning with Offline Datasets. (2020)."},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.5555\/2946645.2946684"},{"key":"e_1_3_2_1_21_1","volume-title":"Continuous control with deep reinforcement learning. arXiv preprint arXiv:1509.02971","author":"Lillicrap P","year":"2015","unstructured":"Timothy\u00a0P Lillicrap, Jonathan\u00a0J Hunt, Alexander Pritzel, Nicolas Heess, Tom Erez, Yuval Tassa, David Silver, and Daan Wierstra. 2015. Continuous control with deep reinforcement learning. arXiv preprint arXiv:1509.02971 (2015)."},{"key":"e_1_3_2_1_22_1","volume-title":"Learning to generalize across long-horizon tasks from human demonstrations. arXiv preprint arXiv:2003.06085","author":"Mandlekar Ajay","year":"2020","unstructured":"Ajay Mandlekar, Danfei Xu, Roberto Mart\u00edn-Mart\u00edn, Silvio Savarese, and Li Fei-Fei. 2020. Learning to generalize across long-horizon tasks from human demonstrations. arXiv preprint arXiv:2003.06085 (2020)."},{"key":"e_1_3_2_1_23_1","volume-title":"What matters in learning from offline human demonstrations for robot manipulation. arXiv preprint arXiv:2108.03298","author":"Mandlekar Ajay","year":"2021","unstructured":"Ajay Mandlekar, Danfei Xu, Josiah Wong, Soroush Nasiriany, Chen Wang, Rohun Kulkarni, Li Fei-Fei, Silvio Savarese, Yuke Zhu, and Roberto Mart\u00edn-Mart\u00edn. 2021. What matters in learning from offline human demonstrations for robot manipulation. arXiv preprint arXiv:2108.03298 (2021)."},{"key":"e_1_3_2_1_24_1","volume-title":"Playing atari with deep reinforcement learning. arXiv preprint arXiv:1312.5602","author":"Mnih Volodymyr","year":"2013","unstructured":"Volodymyr Mnih, Koray Kavukcuoglu, David Silver, Alex Graves, Ioannis Antonoglou, Daan Wierstra, and Martin Riedmiller. 2013. Playing atari with deep reinforcement learning. arXiv preprint arXiv:1312.5602 (2013)."},{"key":"e_1_3_2_1_25_1","volume-title":"Human-level control through deep reinforcement learning. nature 518, 7540","author":"Mnih Volodymyr","year":"2015","unstructured":"Volodymyr Mnih, Koray Kavukcuoglu, David Silver, Andrei\u00a0A Rusu, Joel Veness, Marc\u00a0G Bellemare, Alex Graves, Martin Riedmiller, Andreas\u00a0K Fidjeland, Georg Ostrovski, 2015. Human-level control through deep reinforcement learning. nature 518, 7540 (2015), 529\u2013533."},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-46133-1_4"},{"key":"e_1_3_2_1_27_1","volume-title":"Awac: Accelerating online reinforcement learning with offline datasets. arXiv preprint arXiv:2006.09359","author":"Nair Ashvin","year":"2020","unstructured":"Ashvin Nair, Abhishek Gupta, Murtaza Dalal, and Sergey Levine. 2020. Awac: Accelerating online reinforcement learning with offline datasets. arXiv preprint arXiv:2006.09359 (2020)."},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICRA.2018.8463162"},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v29i1.9722"},{"key":"e_1_3_2_1_30_1","volume-title":"Actor-mimic: Deep multitask and transfer reinforcement learning. arXiv preprint arXiv:1511.06342","author":"Parisotto Emilio","year":"2015","unstructured":"Emilio Parisotto, Jimmy\u00a0Lei Ba, and Ruslan Salakhutdinov. 2015. Actor-mimic: Deep multitask and transfer reinforcement learning. arXiv preprint arXiv:1511.06342 (2015)."},{"key":"e_1_3_2_1_31_1","volume-title":"Advantage-weighted regression: Simple and scalable off-policy reinforcement learning. arXiv preprint arXiv:1910.00177","author":"Peng Xue\u00a0Bin","year":"2019","unstructured":"Xue\u00a0Bin Peng, Aviral Kumar, Grace Zhang, and Sergey Levine. 2019. Advantage-weighted regression: Simple and scalable off-policy reinforcement learning. arXiv preprint arXiv:1910.00177 (2019)."},{"key":"e_1_3_2_1_32_1","volume-title":"Observe and look further: Achieving consistent performance on atari. arXiv preprint arXiv:1805.11593","author":"Pohlen Tobias","year":"2018","unstructured":"Tobias Pohlen, Bilal Piot, Todd Hester, Mohammad\u00a0Gheshlaghi Azar, Dan Horgan, David Budden, Gabriel Barth-Maron, Hado Van\u00a0Hasselt, John Quan, Mel Ve\u010der\u00edk, 2018. Observe and look further: Achieving consistent performance on atari. arXiv preprint arXiv:1805.11593 (2018)."},{"key":"e_1_3_2_1_33_1","volume-title":"Benchmarking safe exploration in deep reinforcement learning. arXiv preprint arXiv:1910.01708 7, 1","author":"Ray Alex","year":"2019","unstructured":"Alex Ray, Joshua Achiam, and Dario Amodei. 2019. Benchmarking safe exploration in deep reinforcement learning. arXiv preprint arXiv:1910.01708 7, 1 (2019), 2."},{"key":"e_1_3_2_1_34_1","volume-title":"Rambo-rl: Robust adversarial model-based offline reinforcement learning. arXiv preprint arXiv:2204.12581","author":"Rigter Marc","year":"2022","unstructured":"Marc Rigter, Bruno Lacerda, and Nick Hawes. 2022. Rambo-rl: Robust adversarial model-based offline reinforcement learning. arXiv preprint arXiv:2204.12581 (2022)."},{"key":"e_1_3_2_1_35_1","volume-title":"Policy distillation. arXiv preprint arXiv:1511.06295","author":"Rusu A","year":"2015","unstructured":"Andrei\u00a0A Rusu, Sergio\u00a0Gomez Colmenarejo, Caglar Gulcehre, Guillaume Desjardins, James Kirkpatrick, Razvan Pascanu, Volodymyr Mnih, Koray Kavukcuoglu, and Raia Hadsell. 2015. Policy distillation. arXiv preprint arXiv:1511.06295 (2015)."},{"key":"e_1_3_2_1_36_1","volume-title":"Proximal policy optimization algorithms. arXiv preprint arXiv:1707.06347","author":"Schulman John","year":"2017","unstructured":"John Schulman, Filip Wolski, Prafulla Dhariwal, Alec Radford, and Oleg Klimov. 2017. Proximal policy optimization algorithms. arXiv preprint arXiv:1707.06347 (2017)."},{"key":"e_1_3_2_1_37_1","first-page":"18967","article-title":"Expert-supervised reinforcement learning for offline policy learning and evaluation","volume":"33","author":"Sonabend Aaron","year":"2020","unstructured":"Aaron Sonabend, Junwei Lu, Leo\u00a0Anthony Celi, Tianxi Cai, and Peter Szolovits. 2020. Expert-supervised reinforcement learning for offline policy learning and evaluation. Advances in Neural Information Processing Systems 33 (2020), 18967\u201318977.","journal-title":"Advances in Neural Information Processing Systems"},{"key":"e_1_3_2_1_38_1","volume-title":"Reinforcement Learning: An Introduction","author":"Sutton S.","year":"2018","unstructured":"Richard\u00a0S. Sutton and Andrew\u00a0G. Barto. 2018. Reinforcement Learning: An Introduction (second ed.). The MIT Press. http:\/\/incompleteideas.net\/book\/the-book-2nd.html"},{"key":"e_1_3_2_1_39_1","volume-title":"A study of count-based exploration for deep reinforcement learning. Advances in neural information processing systems 30","author":"Tang Haoran","year":"2017","unstructured":"Haoran Tang, Rein Houthooft, Davis Foote, Adam Stooke, OpenAI Xi\u00a0Chen, Yan Duan, John Schulman, Filip DeTurck, and Pieter Abbeel. 2017. # exploration: A study of count-based exploration for deep reinforcement learning. Advances in neural information processing systems 30 (2017)."},{"key":"e_1_3_2_1_40_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-01882-4"},{"key":"e_1_3_2_1_41_1","volume-title":"Risk-averse offline reinforcement learning. arXiv preprint arXiv:2102.05371","author":"Urp\u00ed N\u00faria\u00a0Armengol","year":"2021","unstructured":"N\u00faria\u00a0Armengol Urp\u00ed, Sebastian Curi, and Andreas Krause. 2021. Risk-averse offline reinforcement learning. arXiv preprint arXiv:2102.05371 (2021)."},{"key":"e_1_3_2_1_42_1","volume-title":"Leveraging demonstrations for deep reinforcement learning on robotics problems with sparse rewards. arXiv preprint arXiv:1707.08817","author":"Vecerik Mel","year":"2017","unstructured":"Mel Vecerik, Todd Hester, Jonathan Scholz, Fumin Wang, Olivier Pietquin, Bilal Piot, Nicolas Heess, Thomas Roth\u00f6rl, Thomas Lampe, and Martin Riedmiller. 2017. Leveraging demonstrations for deep reinforcement learning on robotics problems with sparse rewards. arXiv preprint arXiv:1707.08817 (2017)."},{"key":"e_1_3_2_1_43_1","volume-title":"Uncertainty weighted actor-critic for offline reinforcement learning. arXiv preprint arXiv:2105.08140","author":"Wu Yue","year":"2021","unstructured":"Yue Wu, Shuangfei Zhai, Nitish Srivastava, Joshua Susskind, Jian Zhang, Ruslan Salakhutdinov, and Hanlin Goh. 2021. Uncertainty weighted actor-critic for offline reinforcement learning. arXiv preprint arXiv:2105.08140 (2021)."},{"key":"e_1_3_2_1_44_1","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v36i8.20855"}],"event":{"name":"CODS-COMAD 2024: 7th Joint International Conference on Data Science & Management of Data (11th ACM IKDD CODS and 29th COMAD)","acronym":"CODS-COMAD 2024","location":"Bangalore India"},"container-title":["Proceedings of the 7th Joint International Conference on Data Science & Management of Data (11th ACM IKDD CODS and 29th COMAD)"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3632410.3632423","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3632410.3632423","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,8,22]],"date-time":"2025-08-22T13:36:06Z","timestamp":1755869766000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3632410.3632423"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,1,4]]},"references-count":44,"alternative-id":["10.1145\/3632410.3632423","10.1145\/3632410"],"URL":"https:\/\/doi.org\/10.1145\/3632410.3632423","relation":{},"subject":[],"published":{"date-parts":[[2024,1,4]]},"assertion":[{"value":"2024-01-04","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}