{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,6,9]],"date-time":"2026-06-09T07:08:08Z","timestamp":1780988888768,"version":"3.54.1"},"reference-count":45,"publisher":"Springer Science and Business Media LLC","issue":"3","license":[{"start":{"date-parts":[[2026,2,3]],"date-time":"2026-02-03T00:00:00Z","timestamp":1770076800000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2026,2,3]],"date-time":"2026-02-03T00:00:00Z","timestamp":1770076800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["J Optim Theory Appl"],"published-print":{"date-parts":[[2026,3]]},"DOI":"10.1007\/s10957-025-02903-1","type":"journal-article","created":{"date-parts":[[2026,2,3]],"date-time":"2026-02-03T11:46:50Z","timestamp":1770119210000},"update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":0,"title":["UVIP: Model-Free Approach to Evaluate Reinforcement Learning Algorithms"],"prefix":"10.1007","volume":"208","author":[{"given":"Denis","family":"Belomestny","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-9708-8189","authenticated-orcid":false,"given":"Ilya","family":"Levin","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Alexey","family":"Naumov","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Sergey","family":"Samsonov","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2026,2,3]]},"reference":[{"key":"2903_CR1","unstructured":"Antos, A., Szepesv\u00e1ri, C., Munos, R.: Fitted Q-Iteration in Continuous Action-Space MDPs. In: J.\u00a0Platt, D.\u00a0Koller, Y.\u00a0Singer, S.\u00a0Roweis (eds.) Advances in Neural Information Processing Systems, vol.\u00a020. Curran Associates, Inc. (2007)"},{"issue":"3","key":"2903_CR2","doi-asserted-by":"publisher","first-page":"354","DOI":"10.1057\/jors.1995.50","volume":"46","author":"TW Archibald","year":"1995","unstructured":"Archibald, T.W., McKinnon, K.I.M., Thomas, L.C.: On the generation of markov decision processes. J. Operational Res. Soc. 46(3), 354\u2013361 (1995)","journal-title":"J. Operational Res. Soc."},{"key":"2903_CR3","unstructured":"Azar, M.G., Osband, I., Munos, R.: Minimax Regret Bounds for Reinforcement Learning. In: D.\u00a0Precup, Y.W. Teh (eds.) Proceedings of the 34th International Conference on Machine Learning, vol.\u00a070, pp. 263\u2013272. PMLR (2017)"},{"issue":"67","key":"2903_CR4","first-page":"1","volume":"17","author":"AM Barreto","year":"2016","unstructured":"Barreto, A.M., Precup, D., Pineau, J.: Practical kernel-based reinforcement learning. J. Mach. Learn. Res. 17(67), 1\u201370 (2016)","journal-title":"J. Mach. Learn. Res."},{"issue":"1","key":"2903_CR5","doi-asserted-by":"publisher","first-page":"20","DOI":"10.1016\/j.cam.2005.08.011","volume":"196","author":"G Beliakov","year":"2006","unstructured":"Beliakov, G.: Interpolation of Lipschitz functions. J. Comput. Appl. Math. 196(1), 20\u201344 (2006)","journal-title":"J. Comput. Appl. Math."},{"key":"2903_CR6","doi-asserted-by":"crossref","unstructured":"Belomestny, D., Schoenmakers, J.: Advanced Simulation-Based Methods for Optimal Stopping and Control: With Applications in Finance. Springer (2018)","DOI":"10.1057\/978-1-137-03351-2"},{"key":"2903_CR7","unstructured":"Bertsekas, D., Shreve, S.E.: Stochastic Optimal Control: the Discrete-Time Case, vol.\u00a05. Athena Scientific (1996)"},{"key":"2903_CR8","doi-asserted-by":"crossref","unstructured":"Bertsekas, D.P., Tsitsiklis, J.N.: Neuro-Dynamic Programming, 1st edn. Athena Scientific (1996)","DOI":"10.1007\/978-3-030-54621-2_440-1"},{"key":"2903_CR9","unstructured":"Bourel, H., Maillard, O., Talebi, M.S.: Tightening Exploration in Upper Confidence Reinforcement Learning. In: H.D. III, A.\u00a0Singh (eds.) Proceedings of the 37th International Conference on Machine Learning, vol. 119, pp. 1056\u20131066. PMLR (2020)"},{"key":"2903_CR10","unstructured":"Brockman, G., Cheung, V., Pettersson, L., Schneider, J., Schulman, J., Tang, J., Zaremba, W.: OpenAI Gym. arXiv preprint arXiv:1606.01540 (2016)"},{"issue":"3","key":"2903_CR11","doi-asserted-by":"publisher","first-page":"246","DOI":"10.1561\/2400000027","volume":"5","author":"DB Brown","year":"2022","unstructured":"Brown, D.B., Smith, J.E.: Information relaxations and duality in stochastic dynamic programs: a review and tutorial. Foundations and Trends in Optimization 5(3), 246\u2013339 (2022)","journal-title":"Foundations and Trends in Optimization"},{"key":"2903_CR12","doi-asserted-by":"crossref","unstructured":"Chen, N., Liu, M., Wang, X., Zhang, N.: Adversarial Reinforcement Learning: A Duality-Based Approach To Solving Optimal Control Problems. arXiv preprint arXiv:2506.00801 (2025)","DOI":"10.1109\/WSC68292.2025.11339041"},{"issue":"52","key":"2903_CR13","first-page":"1","volume":"21","author":"K Ciosek","year":"2020","unstructured":"Ciosek, K., Whiteson, S.: Expected policy gradients for reinforcement learning. J. Mach. Learn. Res. 21(52), 1\u201351 (2020)","journal-title":"J. Mach. Learn. Res."},{"key":"2903_CR14","unstructured":"Dann, C., Li, L., Wei, W., Brunskill, E.: Policy Certificates: Towards Accountable Reinforcement Learning. In: K.\u00a0Chaudhuri, R.\u00a0Salakhutdinov (eds.) Proceedings of the 36th International Conference on Machine Learning, vol.\u00a097, pp. 1507\u20131516. PMLR (2019)"},{"key":"2903_CR15","unstructured":"Domingues, O.D., Flet-Berliac, Y., Leurent, E., M\u00e9nard, P., Shang, X., Valko, M.: rlberry - A Reinforcement Learning Library for Research and Education (2021). https:\/\/github.com\/rlberry-py\/rlberry"},{"key":"2903_CR16","unstructured":"Domingues, O.D., Menard, P., Pirotta, M., Kaufmann, E., Valko, M.: Kernel-Based Reinforcement Learning: A Finite-Time Analysis. In: M.\u00a0Meila, T.\u00a0Zhang (eds.) Proceedings of the 38th International Conference on Machine Learning, vol. 139, pp. 2783\u20132792. PMLR (2021)"},{"key":"2903_CR17","doi-asserted-by":"crossref","unstructured":"Douc, R., Moulines, E., Priouret, P., Soulier, P.: Markov Chains. Springer (2018)","DOI":"10.1007\/978-3-319-97704-1"},{"key":"2903_CR18","unstructured":"Efroni, Y., Merlis, N., Ghavamzadeh, M., Mannor, S.: Tight Regret Bounds for Model-Based Reinforcement Learning with Greedy Policies. In: H.\u00a0Wallach, H.\u00a0Larochelle, A.\u00a0Beygelzimer, F.\u00a0d\u2019Alch\u00e9-Buc, E.\u00a0Fox, R.\u00a0Garnett (eds.) Advances in Neural Information Processing Systems, vol.\u00a032. Curran Associates, Inc. (2019)"},{"key":"2903_CR19","doi-asserted-by":"crossref","unstructured":"El\u00a0Shar, I., Jiang, D.: Weakly Coupled Deep Q-Networks. In: A.\u00a0Oh, T.\u00a0Naumann, A.\u00a0Globerson, K.\u00a0Saenko, M.\u00a0Hardt, S.\u00a0Levine (eds.) Advances in Neural Information Processing Systems, vol.\u00a036, pp. 43931\u201343950 (2023)","DOI":"10.52202\/075280-1906"},{"key":"2903_CR20","unstructured":"Heess, N., Wayne, G., Silver, D., Lillicrap, T., Erez, T., Tassa, Y.: Learning Continuous Control Policies by Stochastic Value Gradients. In: C.\u00a0Cortes, N.\u00a0Lawrence, D.\u00a0Lee, M.\u00a0Sugiyama, R.\u00a0Garnett (eds.) Advances in Neural Information Processing Systems, vol.\u00a028. Curran Associates, Inc. (2015)"},{"issue":"51","key":"2903_CR21","first-page":"1563","volume":"11","author":"T Jaksch","year":"2010","unstructured":"Jaksch, T., Ortner, R., Auer, P.: Near-optimal regret bounds for reinforcement learning. J. Mach. Learn. Res. 11(51), 1563\u20131600 (2010)","journal-title":"J. Mach. Learn. Res."},{"issue":"2","key":"2903_CR22","doi-asserted-by":"publisher","first-page":"494","DOI":"10.1239\/jap\/996986758","volume":"38","author":"SF Jarner","year":"2001","unstructured":"Jarner, S.F., Tweedie, R.L.: Locally contracting iterated functions and stability of markov chains. J. Appl. Probab. 38(2), 494\u2013507 (2001)","journal-title":"J. Appl. Probab."},{"key":"2903_CR23","unstructured":"Jin, C., Allen-Zhu, Z., Bubeck, S., Jordan, M.I.: LSQ-Learning Provably Efficient? In: S.\u00a0Bengio, H.\u00a0Wallach, H.\u00a0Larochelle, K.\u00a0Grauman, N.\u00a0Cesa-Bianchi, R.\u00a0Garnett (eds.) Advances in Neural Information Processing Systems, vol.\u00a031. Curran Associates, Inc. (2018)"},{"key":"2903_CR24","doi-asserted-by":"crossref","unstructured":"Kostenetskiy, P., Chulkevich, R., Kozyrev, V.: HPC Resources of the Higher School of Economics. In: Journal of Physics: Conference Series, vol. 1740, p. 012050. IOP Publishing (2021)","DOI":"10.1088\/1742-6596\/1740\/1\/012050"},{"key":"2903_CR25","doi-asserted-by":"crossref","unstructured":"Kveton, B., Theocharous, G.: Kernel-Based Reinforcement Learning on Representative States. In: J.\u00a0Hoffmann, B.\u00a0Selman (eds.) Proceedings of the AAAI Conference on Artificial Intelligence, vol.\u00a026, pp. 977\u2013983 (2012)","DOI":"10.1609\/aaai.v26i1.8294"},{"key":"2903_CR26","unstructured":"Liu, H., Feng, Y., Mao, Y., Zhou, D., Peng, J., Liu, Q.: Action-Dependent Control Variates for Policy Optimization via Stein\u2019s Identity. arXiv preprint arXiv:1710.11198 (2017)"},{"key":"2903_CR27","doi-asserted-by":"crossref","unstructured":"Lyle, C., Bellemare, M.G., Castro, P.S.: A Comparative Analysis of Expected and Distributional Reinforcement Learning. In: P.V. Hentenryck, Z.H. Zhou (eds.) Proceedings of the AAAI Conference on Artificial Intelligence, vol.\u00a033, pp. 4504\u20134511 (2019)","DOI":"10.1609\/aaai.v33i01.33014504"},{"key":"2903_CR28","doi-asserted-by":"crossref","unstructured":"Merlis, N.: Reinforcement Learning with Lookahead Information. In: A.\u00a0Globerson, L.\u00a0Mackey, D.\u00a0Belgrave, A.\u00a0Fan, U.\u00a0Paquet, J.\u00a0Tomczak, C.\u00a0Zhang (eds.) Advances in Neural Information Processing Systems, vol.\u00a037, pp. 64523\u201364581 (2024)","DOI":"10.52202\/079017-2059"},{"key":"2903_CR29","unstructured":"Mnih, V., Badia, A.P., Mirza, M., Graves, A., Lillicrap, T., Harley, T., Silver, D., Kavukcuoglu, K.: Asynchronous Methods for Deep Reinforcement Learning. In: M.F. Balcan, K.Q. Weinberger (eds.) Proceedings of The 33rd International Conference on Machine Learning, vol.\u00a048, pp. 1928\u20131937. PMLR, New York, New York, USA (2016)"},{"issue":"7540","key":"2903_CR30","doi-asserted-by":"publisher","first-page":"529","DOI":"10.1038\/nature14236","volume":"518","author":"V Mnih","year":"2015","unstructured":"Mnih, V., Kavukcuoglu, K., Silver, D., Rusu, A.A., Veness, J., Bellemare, M.G., Graves, A., Riedmiller, M.A., Fidjeland, A., Ostrovski, G., Petersen, S., Beattie, C., Sadik, A., Antonoglou, I., King, H., Kumaran, D., Wierstra, D., Legg, S., Hassabis, D.: Human-level control through deep reinforcement learning. Nature 518(7540), 529\u2013533 (2015)","journal-title":"Nature"},{"key":"2903_CR31","first-page":"121","volume-title":"29th Annual Conference on Learning Theory","author":"B \u00c1vila Pires","year":"2016","unstructured":"\u00c1vila Pires, B., Szepesv\u00e1ri, C.: Policy error bounds for model-based reinforcement learning with factored linear models. In: Feldman, V., Rakhlin, A., Shamir, O. (eds.) 29th Annual Conference on Learning Theory, vol. 49, pp. 121\u2013151. Columbia University, New York, New York, USA, PMLR (2016)"},{"key":"2903_CR32","unstructured":"Puterman, M.L.: Markov Decision Processes: Discrete Stochastic Dynamic Programming. John Wiley & Sons (2014)"},{"issue":"19","key":"2903_CR33","doi-asserted-by":"publisher","first-page":"6065","DOI":"10.1093\/imrn\/rnv342","volume":"2016","author":"A Reznikov","year":"2015","unstructured":"Reznikov, A., Saff, E.B.: The covering radius of randomly distributed points on a manifold. Int. Math. Res. Not. 2016(19), 6065\u20136094 (2015)","journal-title":"Int. Math. Res. Not."},{"issue":"3","key":"2903_CR34","doi-asserted-by":"publisher","first-page":"1116","DOI":"10.1137\/050642885","volume":"46","author":"LCG Rogers","year":"2007","unstructured":"Rogers, L.C.G.: Pathwise stochastic optimal control. SIAM J. Control. Optim. 46(3), 1116\u20131132 (2007)","journal-title":"SIAM J. Control. Optim."},{"key":"2903_CR35","doi-asserted-by":"crossref","unstructured":"Rosenberg, A., Hallak, A., Mannor, S., Chechik, G., Dalal, G.: Planning and Learning with Adaptive Lookahead. In: Y.\u00a0Chen, J.\u00a0Neville (eds.) Proceedings of the AAAI Conference on Artificial Intelligence, vol.\u00a037, pp. 9606\u20139613 (2023)","DOI":"10.1609\/aaai.v37i8.26149"},{"key":"2903_CR36","unstructured":"Rowland, M., Harutyunyan, A., van Hasselt, H., Borsa, D., Schaul, T., Munos, R., Dabney, W.: Conditional Importance Sampling for Off-Policy Learning. In: S.\u00a0Chiappa, R.\u00a0Calandra (eds.) Proceedings of the Twenty Third International Conference on Artificial Intelligence and Statistics, vol. 108, pp. 45\u201355. PMLR (2020)"},{"key":"2903_CR37","unstructured":"Shar, I.E., Jiang, D.: Lookahead-Bounded Q-learning. In: H.D. III, A.\u00a0Singh (eds.) Proceedings of the 37th International Conference on Machine Learning, vol. 119, pp. 8665\u20138675. PMLR (2020)"},{"key":"2903_CR38","doi-asserted-by":"publisher","first-page":"9","DOI":"10.1023\/A:1022633531479","volume":"3","author":"R Sutton","year":"1988","unstructured":"Sutton, R.: Learning to predict by the method of temporal differences. Mach. Learn. 3, 9\u201344 (1988)","journal-title":"Mach. Learn."},{"key":"2903_CR39","unstructured":"Sutton, R.S., Barto, A.G.: Reinforcement Learning: An Introduction, second edn. The MIT Press (2018)"},{"issue":"1","key":"2903_CR40","first-page":"1","volume":"4","author":"C Szepesv\u00e1ri","year":"2010","unstructured":"Szepesv\u00e1ri, C.: Algorithms for reinforcement learning. Synth. Lect. Artif. Intell. Mach. Learn. 4(1), 1\u2013103 (2010)","journal-title":"Synth. Lect. Artif. Intell. Mach. Learn."},{"key":"2903_CR41","unstructured":"Tsitsiklis, J., Van\u00a0Roy, B.: Analysis of Temporal-Diffference Learning with Function Approximation. In: M.\u00a0Mozer, M.\u00a0Jordan, T.\u00a0Petsche (eds.) Advances in Neural Information Processing Systems, vol.\u00a09. MIT Press (1996)"},{"key":"2903_CR42","volume-title":"High-Dimensional Probability","author":"R Vershynin","year":"2018","unstructured":"Vershynin, R.: High-Dimensional Probability, vol. 47. Cambridge University Press, Cambridge (2018)"},{"key":"2903_CR43","unstructured":"Wainwright, M.J.: Variance-Reduced Q-learning is Minimax Optimal. arXiv preprint arXiv:1906.04697 (2019)"},{"key":"2903_CR44","unstructured":"Wang, Z., Schaul, T., Hessel, M., Hasselt, H., Lanctot, M., Freitas, N.: Dueling Network Architectures for Deep Reinforcement Learning. In: M.F. Balcan, K.Q. Weinberger (eds.) International Conference on Machine Learning, vol.\u00a048, pp. 1995\u20132003. PMLR (2016)"},{"key":"2903_CR45","doi-asserted-by":"publisher","first-page":"229","DOI":"10.1023\/A:1022672621406","volume":"8","author":"RJ Williams","year":"1992","unstructured":"Williams, R.J.: Simple statistical gradient-following algorithms for connectionist reinforcement learning. Mach. Learn. 8, 229\u2013256 (1992)","journal-title":"Mach. Learn."}],"container-title":["Journal of Optimization Theory and Applications"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10957-025-02903-1.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s10957-025-02903-1","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10957-025-02903-1.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,6,9]],"date-time":"2026-06-09T06:48:10Z","timestamp":1780987690000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s10957-025-02903-1"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026,2,3]]},"references-count":45,"journal-issue":{"issue":"3","published-print":{"date-parts":[[2026,3]]}},"alternative-id":["2903"],"URL":"https:\/\/doi.org\/10.1007\/s10957-025-02903-1","relation":{},"ISSN":["0022-3239","1573-2878"],"issn-type":[{"value":"0022-3239","type":"print"},{"value":"1573-2878","type":"electronic"}],"subject":[],"published":{"date-parts":[[2026,2,3]]},"assertion":[{"value":"10 January 2025","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"7 December 2025","order":2,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"3 February 2026","order":3,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}}],"article-number":"89"}}