{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,8,27]],"date-time":"2025-08-27T00:19:37Z","timestamp":1756253977679,"version":"3.44.0"},"publisher-location":"New York, NY, USA","reference-count":30,"publisher":"ACM","license":[{"start":{"date-parts":[[2023,11,30]],"date-time":"2023-11-30T00:00:00Z","timestamp":1701302400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"DOI":"10.13039\/501100006374","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["62276142, 62206133, 62202240, 62192783"],"award-info":[{"award-number":["62276142, 62206133, 62202240, 62192783"]}],"id":[{"id":"10.13039\/501100006374","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2023,11,30]]},"DOI":"10.1145\/3627676.3627687","type":"proceedings-article","created":{"date-parts":[[2023,12,30]],"date-time":"2023-12-30T06:05:14Z","timestamp":1703916314000},"page":"1-7","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":0,"title":["Backtracking Exploration for Reinforcement Learning"],"prefix":"10.1145","author":[{"given":"Xingguo","family":"Chen","sequence":"first","affiliation":[{"name":"Nanjing University of Posts and Telecommunications, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Zening","family":"Chen","sequence":"additional","affiliation":[{"name":"Nanjing University of Posts and Telecommunications, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Dingyuanhao","family":"Sun","sequence":"additional","affiliation":[{"name":"Nanjing University of Posts and Telecommunications, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Yang","family":"Gao","sequence":"additional","affiliation":[{"name":"Nanjing University, China"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2023,12,30]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"publisher","DOI":"10.1023\/A:1013689704352"},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.1016\/B978-1-55860-377-6.50013-X"},{"volume-title":"Neuro-Dynamic Programming","author":"Bertsekas D.","key":"e_1_3_2_1_3_1","unstructured":"D. Bertsekas and J.\u00a0N. Tsitsiklis. 1996. Neuro-Dynamic Programming. Athena Scientific."},{"key":"e_1_3_2_1_4_1","unstructured":"Qi Cai Zhuoran Yang Jason\u00a0D Lee and Zhaoran Wang. 2019. Neural temporal-difference learning converges to global optima. In Advances in Neural Information Processing Systems. 11312\u201311322."},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.ins.2021.08.082"},{"key":"e_1_3_2_1_6_1","unstructured":"Xingguo Chen Xingzhou Ma Yang Li Guang Yang Shangdong Yang and Yang Gao. 2023. Modified Retrace for Off-Policy Temporal Difference Learning. In Uncertainty in Artificial Intelligence. PMLR 303\u2013312."},{"key":"e_1_3_2_1_7_1","volume-title":"International Conference on Learning Representations.","author":"Chung Wesley","year":"2019","unstructured":"Wesley Chung, Somjit Nath, Ajin Joseph, and Martha White. 2019. Two-timescale networks for nonlinear value function approximation. In International Conference on Learning Representations."},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v32i1.12079"},{"key":"e_1_3_2_1_9_1","volume-title":"International Conference on Machine Learning. PMLR, 4666\u20134689","author":"Dann Chris","year":"2022","unstructured":"Chris Dann, Yishay Mansour, Mehryar Mohri, Ayush Sekhari, and Karthik Sridharan. 2022. Guarantees for epsilon-greedy reinforcement learning with function approximation. In International Conference on Machine Learning. PMLR, 4666\u20134689."},{"key":"e_1_3_2_1_10_1","first-page":"1","article-title":"Learning Rates for Q-learning","volume":"5","author":"Even-Dar Eyal","year":"2003","unstructured":"Eyal Even-Dar and Yishay Mansour. 2003. Learning Rates for Q-learning. Journal of Machine Learning Research 5 (2003), 1\u201325.","journal-title":"Journal of Machine Learning Research"},{"key":"e_1_3_2_1_11_1","volume-title":"Proceedings of the 31st International Conference on Neural Information Processing Systems. 3138\u20133148","author":"Levine Nir","year":"2017","unstructured":"Nir Levine, Tom Zahavy, Daniel\u00a0J Mankowitz, Aviv Tamar, and Shie Mannor. 2017. Shallow updates for deep reinforcement learning. In Proceedings of the 31st International Conference on Neural Information Processing Systems. 3138\u20133148."},{"key":"e_1_3_2_1_12_1","volume-title":"Is Q-learning minimax optimal? a tight sample complexity analysis. Operations Research","author":"Li Gen","year":"2023","unstructured":"Gen Li, Changxiao Cai, Yuxin Chen, Yuting Wei, and Yuejie Chi. 2023. Is Q-learning minimax optimal? a tight sample complexity analysis. Operations Research (2023)."},{"key":"e_1_3_2_1_13_1","first-page":"16671","article-title":"Sample-Efficient Reinforcement Learning Is Feasible for Linearly Realizable MDPs with Limited Revisiting","volume":"34","author":"Li Gen","year":"2021","unstructured":"Gen Li, Yuxin Chen, Yuejie Chi, Yuantao Gu, and Yuting Wei. 2021. Sample-Efficient Reinforcement Learning Is Feasible for Linearly Realizable MDPs with Limited Revisiting. Advances in Neural Information Processing Systems 34 (2021), 16671\u201316685.","journal-title":"Advances in Neural Information Processing Systems"},{"key":"e_1_3_2_1_14_1","first-page":"7031","article-title":"Sample complexity of asynchronous Q-learning: Sharper analysis and variance reduction","volume":"33","author":"Li Gen","year":"2020","unstructured":"Gen Li, Yuting Wei, Yuejie Chi, Yuantao Gu, and Yuxin Chen. 2020. Sample complexity of asynchronous Q-learning: Sharper analysis and variance reduction. Advances in Neural Information Processing Systems 33 (2020), 7031\u20137043.","journal-title":"Advances in Neural Information Processing Systems"},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v34i04.5955"},{"key":"e_1_3_2_1_16_1","unstructured":"Hamid\u00a0Reza Maei Csaba Szepesvari Shalabh Bhatnagar Doina Precup David Silver and Richard\u00a0S Sutton. 2009. Convergent temporal-difference learning with arbitrary smooth function approximation. In Advances in Neural Information Processing Systems. 1204\u20131212."},{"key":"e_1_3_2_1_17_1","volume-title":"Proceedings of 2nd Asian Conference on Machine Learning. JMLR Workshop and Conference Proceedings, 299\u2013314","author":"Maillard Odalric-Ambrym","year":"2010","unstructured":"Odalric-Ambrym Maillard, R\u00e9mi Munos, Alessandro Lazaric, and Mohammad Ghavamzadeh. 2010. Finite-sample analysis of Bellman residual minimization. In Proceedings of 2nd Asian Conference on Machine Learning. JMLR Workshop and Conference Proceedings, 299\u2013314."},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.24963\/ijcai.2017\/344"},{"key":"e_1_3_2_1_19_1","volume-title":"Convergence of Q-learning: A simple proof","author":"Melo S","year":"2001","unstructured":"Francisco\u00a0S Melo. 2001. Convergence of Q-learning: A simple proof. Institute Of Systems and Robotics, Tech. Rep (2001), 1\u20134."},{"volume-title":"Proceedings of the 26th annual international conference on machine learning.993\u20131000","author":"Sutton R.S.","key":"e_1_3_2_1_20_1","unstructured":"R.S. Sutton, H.R. Maei, D. Precup, S. Bhatnagar, D. Silver, C. Szepesv\u00e1ri, and E. Wiewiora. 2009. Fast gradient-descent methods for temporal-difference learning with linear function approximation. In Proceedings of the 26th annual international conference on machine learning.993\u20131000."},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.1023\/A:1022633531479"},{"volume-title":"Reinforcement learning: An introduction","author":"Sutton S","key":"e_1_3_2_1_22_1","unstructured":"Richard\u00a0S Sutton and Andrew\u00a0G Barto. 2018. Reinforcement learning: An introduction. MIT press."},{"volume-title":"Advances in Neural Information Processing Systems","author":"Sutton S","key":"e_1_3_2_1_23_1","unstructured":"Richard\u00a0S Sutton, Hamid\u00a0R Maei, and Csaba Szepesv\u00e1ri. 2008. A Convergent O(n) Temporal-difference Algorithm for Off-policy Learning with Linear Function Approximation. In Advances in Neural Information Processing Systems. Cambridge, MA: MIT Press, 1609\u20131616."},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"publisher","DOI":"10.5555\/2946645.3007026"},{"key":"e_1_3_2_1_25_1","unstructured":"Sebastian Thrun. 1992. Efficient exploration in reinforcement learning. Technical Report. Carnegie Mellon University (1992)."},{"key":"e_1_3_2_1_26_1","unstructured":"John\u00a0N Tsitsiklis and Benjamin Van\u00a0Roy. 1997. Analysis of temporal-difference learning with function approximation. In Advances in Neural Information Processing Systems. 1075\u20131081."},{"key":"e_1_3_2_1_27_1","volume-title":"International Conference on Artificial Intelligence and Statistics. PMLR, 3370\u20133398","author":"Wang Shengbo","year":"2023","unstructured":"Shengbo Wang, Nian Si, Jose Blanchet, and Zhengyuan Zhou. 2023. A finite sample complexity bound for distributionally robust q-learning. In International Conference on Artificial Intelligence and Statistics. PMLR, 3370\u20133398."},{"key":"e_1_3_2_1_28_1","first-page":"16628","article-title":"Finite-time analysis for double Q-learning","volume":"33","author":"Xiong Huaqing","year":"2020","unstructured":"Huaqing Xiong, Lin Zhao, Yingbin Liang, and Wei Zhang. 2020. Finite-time analysis for double Q-learning. Advances in Neural Information Processing Systems 33 (2020), 16628\u201316638.","journal-title":"Advances in Neural Information Processing Systems"},{"key":"e_1_3_2_1_29_1","volume-title":"International Conference on Artificial Intelligence and Statistics. PMLR, 811\u2013819","author":"Xu Tengyu","year":"2021","unstructured":"Tengyu Xu and Yingbin Liang. 2021. Sample complexity bounds for two timescale value-based reinforcement learning algorithms. In International Conference on Artificial Intelligence and Statistics. PMLR, 811\u2013819."},{"key":"e_1_3_2_1_30_1","unstructured":"Shaofeng Zou Tengyu Xu and Yingbin Liang. 2019. Finite-sample analysis for SARSA with linear function approximation. In Advances in Neural Information Processing Systems. 8668\u20138678."}],"event":{"name":"DAI '23: The Fifth International Conference on Distributed Artificial Intelligence","acronym":"DAI '23","location":"Singapore Singapore"},"container-title":["The Fifth International Conference on Distributed Artificial Intelligence"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3627676.3627687","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3627676.3627687","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,8,26]],"date-time":"2025-08-26T19:26:12Z","timestamp":1756236372000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3627676.3627687"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023,11,30]]},"references-count":30,"alternative-id":["10.1145\/3627676.3627687","10.1145\/3627676"],"URL":"https:\/\/doi.org\/10.1145\/3627676.3627687","relation":{},"subject":[],"published":{"date-parts":[[2023,11,30]]},"assertion":[{"value":"2023-12-30","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}