{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,3,9]],"date-time":"2026-03-09T01:45:24Z","timestamp":1773020724409,"version":"3.50.1"},"reference-count":68,"publisher":"Informa UK Limited","issue":"551","funder":[{"name":"NSF FRGMS","award":["1952373"],"award-info":[{"award-number":["1952373"]}]}],"content-domain":{"domain":["www.tandfonline.com"],"crossmark-restriction":true},"short-container-title":["Journal of the American Statistical Association"],"published-print":{"date-parts":[[2025,7,3]]},"DOI":"10.1080\/01621459.2025.2506197","type":"journal-article","created":{"date-parts":[[2025,6,2]],"date-time":"2025-06-02T15:05:31Z","timestamp":1748876731000},"page":"1517-1530","update-policy":"https:\/\/doi.org\/10.1080\/tandf_crossmark_01","source":"Crossref","is-referenced-by-count":1,"title":["Distributional Off-Policy Evaluation in Reinforcement Learning"],"prefix":"10.1080","volume":"120","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-0270-7969","authenticated-orcid":false,"given":"Zhengling","family":"Qi","sequence":"first","affiliation":[{"name":"Department of Decision Sciences, The George Washington University","place":["Washington"]}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-8379-9385","authenticated-orcid":false,"given":"Chenjia","family":"Bai","sequence":"additional","affiliation":[{"name":"Shanghai AI Laboratory","place":["Shanghai, China"]}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Zhaoran","family":"Wang","sequence":"additional","affiliation":[{"name":"Departments of Industrial Engineering, The Northwestern University","place":["Evanston"]}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-3217-0202","authenticated-orcid":false,"given":"Lan","family":"Wang","sequence":"additional","affiliation":[{"name":"Department of Management Science, The University of Miami","place":["Coral Gables"]}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"301","published-online":{"date-parts":[[2025,8,5]]},"reference":[{"key":"e_1_3_4_2_1","volume-title":"Advances in Neural Information Processing Systems","author":"Agarwal R.","year":"2021","unstructured":"Agarwal, R., Schwarzer, M., Castro, P. S., Courville, A. C., and Bellemare, M. (2021), \u201cDeep Reinforcement Learning at the Edge of the Statistical Precipice,\u201d in Advances in Neural Information Processing Systems (Vol. 34)."},{"key":"e_1_3_4_3_1","doi-asserted-by":"publisher","DOI":"10.1111\/1468-0262.00470"},{"key":"e_1_3_4_4_1","first-page":"9","volume-title":"Advances in Neural Information Processing Systems","author":"Antos A.","year":"2008","unstructured":"Antos, A., Szepesv\u00e1ri, C., and Munos, R. (2008), \u201cFitted q-iteration in Continuous Action-Space MDPs,\u201d in Advances in Neural Information Processing Systems, pp. 9\u201316."},{"key":"e_1_3_4_5_1","first-page":"214","volume-title":"International Conference on Machine Learning,","author":"Arjovsky M.","year":"2017","unstructured":"Arjovsky, M., Chintala, S., and Bottou, L. (2017), \u201cWasserstein Generative Adversarial Networks,\u201d in International Conference on Machine Learning, PMLR, pp. 214\u2013223."},{"key":"e_1_3_4_6_1","first-page":"449","volume-title":"International Conference on Machine Learning,","author":"Bellemare M. G.","year":"2017","unstructured":"Bellemare, M. G., Dabney, W., and Munos, R. (2017), \u201cA Distributional Perspective on Reinforcement Learning,\u201d in International Conference on Machine Learning, PMLR, pp. 449\u2013458."},{"key":"e_1_3_4_7_1","first-page":"2203","volume-title":"The 22nd International Conference on Artificial Intelligence and Statistics,","author":"Bellemare M. G.","year":"2019","unstructured":"Bellemare, M. G., Le Roux, N., Castro, P. S., and Moitra, S. (2019), \u201cDistributional Reinforcement Learning with Linear Function Approximation,\u201d in The 22nd International Conference on Artificial Intelligence and Statistics, PMLR, pp. 2203\u20132211."},{"key":"e_1_3_4_8_1","doi-asserted-by":"publisher","DOI":"10.1287\/mnsc.32.11.1445"},{"key":"e_1_3_4_9_1","volume-title":"Dynamic Programming and Optimal Control","author":"Bertsekas D. P.","year":"1995","unstructured":"Bertsekas, D. P. (1995), Dynamic Programming and Optimal Control (Vol. 1), Belmont, MA: Athena Scientific."},{"key":"e_1_3_4_10_1","first-page":"1","article-title":"\u201cSome Theoretical Insights into Wasserstein GANs,\u201d","volume":"22","author":"Biau G.","year":"2021","unstructured":"Biau, G., Sangnier, M., and Tanielian, U. (2021), \u201cSome Theoretical Insights into Wasserstein GANs,\u201d Journal of Machine Learning Research, 22, 1\u201345.","journal-title":"Journal of Machine Learning Research"},{"key":"e_1_3_4_11_1","doi-asserted-by":"publisher","DOI":"10.1111\/j.1468-0262.2007.00808.x"},{"key":"e_1_3_4_12_1","doi-asserted-by":"publisher","DOI":"10.1287\/moor.27.2.294.324"},{"key":"e_1_3_4_13_1","unstructured":"Chen M. Liao W. Zha H. and Zhao T. (2020) \u201cStatistical Guarantees of Generative Adversarial Networks for Distribution Estimation \u201d arXiv preprint arXiv:2002.03938."},{"key":"e_1_3_4_14_1","doi-asserted-by":"publisher","DOI":"10.3982\/QE722"},{"key":"e_1_3_4_15_1","doi-asserted-by":"publisher","DOI":"10.3982\/ECTA10771"},{"key":"e_1_3_4_16_1","doi-asserted-by":"crossref","unstructured":"Chen Y. Gao Q. and Wang X. (2021) \u201cInferential Wasserstein Generative Adversarial Networks \u201d arXiv preprint arXiv:2109.05652.","DOI":"10.1111\/rssb.12476"},{"key":"e_1_3_4_17_1","first-page":"6070","article-title":"\u201cRisk-Constrained Reinforcement Learning with Percentile Risk Criteria,\u201d","volume":"18","author":"Chow Y.","year":"2017","unstructured":"Chow, Y., Ghavamzadeh, M., Janson, L., and Pavone, M. (2017), \u201cRisk-Constrained Reinforcement Learning with Percentile Risk Criteria,\u201d The Journal of Machine Learning Research, 18, 6070\u20136120.","journal-title":"The Journal of Machine Learning Research"},{"key":"e_1_3_4_18_1","doi-asserted-by":"publisher","DOI":"10.1137\/0325004"},{"key":"e_1_3_4_19_1","doi-asserted-by":"publisher","DOI":"10.1137\/1.9781611976748"},{"key":"e_1_3_4_20_1","first-page":"1096","volume-title":"International Conference on Machine Learning,","author":"Dabney W.","year":"2018","unstructured":"Dabney, W., Ostrovski, G., Silver, D., and Munos, R. (2018), \u201cImplicit Quantile Networks for Distributional Reinforcement Learning,\u201d in International Conference on Machine Learning, PMLR, pp. 1096\u20131105."},{"key":"e_1_3_4_21_1","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v32i1.11791"},{"key":"e_1_3_4_22_1","first-page":"1665","volume-title":"International Conference on Machine Learning","author":"Du S.","year":"2019","unstructured":"Du, S., Krishnamurthy, A., Jiang, N., Agarwal, A., Dudik, M., and Langford, J. (2019), \u201cProvably Efficient RL with Rich Observations via Latent State Decoding,\u201d in International Conference on Machine Learning, PMLR, pp. 1665\u20131674."},{"key":"e_1_3_4_23_1","unstructured":"Frankle J. and Carbin M. (2018) \u201cThe Lottery Ticket Hypothesis: Finding Sparse Trainable Neural Networks \u201d arXiv preprint arXiv:1803.03635."},{"key":"e_1_3_4_24_1","unstructured":"Fu J. Norouzi M. Nachum O. Tucker G. Wang Z. Novikov A. Yang M. Zhang M. R. Chen Y. Kumar A. et\u00a0al. (2021) \u201cBenchmarks for Deep Off-Policy Evaluation \u201d arXiv preprint arXiv:2103.16596."},{"key":"e_1_3_4_25_1","first-page":"1437","article-title":"\u201cA Comprehensive Survey on Safe Reinforcement Learning,\u201d","volume":"16","author":"Garc\u0131a J.","year":"2015","unstructured":"Garc\u0131a, J., and Fern\u00e1ndez, F. (2015), \u201cA Comprehensive Survey on Safe Reinforcement Learning,\u201d Journal of Machine Learning Research, 16, 1437\u20131480.","journal-title":"Journal of Machine Learning Research"},{"key":"e_1_3_4_26_1","article-title":"\u201cGenerative Adversarial Nets,\u201d","volume":"27","author":"Goodfellow I.","year":"2014","unstructured":"Goodfellow, I., Pouget-Abadie, J., Mirza, M., Xu, B., Warde-Farley, D., Ozair, S., Courville, A., and Bengio, Y. (2014), \u201cGenerative Adversarial Nets,\u201d in Advances in Neural Information Processing Systems (Vol. 27).","journal-title":"in Advances in Neural Information Processing Systems"},{"key":"e_1_3_4_27_1","article-title":"\u201cImproved Training of Wasserstein GANs,\u201d in","author":"Gulrajani I.","year":"2017","unstructured":"Gulrajani, I., Ahmed, F., Arjovsky, M., Dumoulin, V., and Courville, A. C. (2017), \u201cImproved Training of Wasserstein GANs,\u201d in Neural Information Processing Systems.","journal-title":"Neural Information Processing Systems"},{"key":"e_1_3_4_28_1","unstructured":"Haas M. and Richter S. (2020) \u201cStatistical Analysis of Wasserstein GANs with Applications to Time Series Forecasting \u201d arXiv preprint arXiv:2011.03074."},{"key":"e_1_3_4_29_1","doi-asserted-by":"publisher","DOI":"10.2307\/1912775"},{"key":"e_1_3_4_30_1","first-page":"2681","volume-title":"International Conference on Machine Learning, PMLR","author":"Hazan E.","year":"2019","unstructured":"Hazan, E., Kakade, S., Singh, K., and Van Soest, A. (2019), \u201cProvably Efficient Maximum Entropy Exploration,\u201d in International Conference on Machine Learning, PMLR, pp. 2681\u20132691."},{"key":"e_1_3_4_31_1","unstructured":"Hubbs C. D. Perez H. D. Sarwar O. Sahinidis N. V. Grossmann I. E. and Wassick J. M. (2020) \u201cOr-gym: A Reinforcement Learning Library for Operations Research Problems \u201d arXiv preprint arXiv:2008.06319."},{"key":"e_1_3_4_32_1","doi-asserted-by":"publisher","DOI":"10.1214\/aos\/1176342415"},{"key":"e_1_3_4_33_1","first-page":"4870","volume-title":"International Conference on Machine Learning, PMLR","author":"Jin C.","year":"2020","unstructured":"Jin, C., Krishnamurthy, A., Simchowitz, M., and Yu, T. (2020), \u201cReward-Free Exploration for Reinforcement Learning,\u201d in International Conference on Machine Learning, PMLR, pp. 4870\u20134879."},{"key":"e_1_3_4_34_1","unstructured":"Kallus N. and Uehara M. (2019) \u201cEfficiently Breaking the Curse of Horizon in Off-Policy Evaluation with Double Reinforcement Learning \u201d arXiv preprint arXiv:1909.05850."},{"key":"e_1_3_4_35_1","doi-asserted-by":"publisher","DOI":"10.1287\/opre.2021.2249"},{"key":"e_1_3_4_36_1","doi-asserted-by":"publisher","DOI":"10.1146\/annurev-statistics-030718-105251"},{"key":"e_1_3_4_37_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-97146-4"},{"key":"e_1_3_4_38_1","unstructured":"Levine S. Kumar A. Tucker G. and Fu J. (2020) \u201cOffline Reinforcement Learning: Tutorial Review and Perspectives on Open Problems \u201d arXiv preprint arXiv:2005.01643."},{"key":"e_1_3_4_39_1","unstructured":"Liao P. Qi Z. and Murphy S. (2020) \u201cBatch Policy Learning in Average Reward Markov Decision Processes \u201d arXiv preprint arXiv:2007.11771."},{"key":"e_1_3_4_40_1","unstructured":"Liu J. Zhang Y. Wang X. Deng Y. and Wu X. (2019) \u201cDynamic Pricing on e-commerce Platform with Deep Reinforcement Learning \u201d arXiv preprint arXiv:1912.02572."},{"key":"e_1_3_4_41_1","volume-title":"ICML","author":"Lizotte D. J.","year":"2010","unstructured":"Lizotte, D. J., Bowling, M. H., and Murphy, S. A. (2010), \u201cEfficient Reinforcement Learning with Multiple Reward Functions for Randomized Controlled Trial Analysis,\u201d in ICML."},{"key":"e_1_3_4_42_1","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v33i01.33014504"},{"key":"e_1_3_4_43_1","article-title":"\u201cConditional Generative Adversarial Nets,\u201d","author":"Mirza M.","year":"2014","unstructured":"Mirza, M., and Osindero, S. (2014), \u201cConditional Generative Adversarial Nets,\u201d arXiv preprint arXiv:1411.1784.","journal-title":"arXiv preprint"},{"key":"e_1_3_4_44_1","first-page":"6961","volume-title":"International Conference on Machine Learning,","author":"Misra D.","year":"2020","unstructured":"Misra, D., Henaff, M., Krishnamurthy, A., and Langford, J. (2020), \u201cKinematic State Abstraction and Provably Efficient Rich-Observation Reinforcement Learning,\u201d in International Conference on Machine Learning, PMLR, pp. 6961\u20136971."},{"key":"e_1_3_4_45_1","unstructured":"Mnih V. Kavukcuoglu K. Silver D. Graves A. Antonoglou I. Wierstra D. and Riedmiller M. (2013) \u201cPlaying Atari with Deep Reinforcement Learning \u201d arXiv preprint arXiv:1312.5602."},{"key":"e_1_3_4_46_1","article-title":"\u201cVisual Reinforcement Learning with Imagined Goals,\u201d in","volume":"31","author":"Nair A. V.","year":"2018","unstructured":"Nair, A. V., Pong, V., Dalal, M., Bahl, S., Lin, S., and Levine, S. (2018), \u201cVisual Reinforcement Learning with Imagined Goals,\u201d in Advances in Neural Information Processing Systems (Vol. 31).","journal-title":"Advances in Neural Information Processing Systems"},{"key":"e_1_3_4_47_1","doi-asserted-by":"publisher","DOI":"10.1111\/1468-0262.00459"},{"key":"e_1_3_4_48_1","first-page":"278","volume-title":"Icml","author":"Ng A. Y.","year":"1999","unstructured":"Ng, A. Y., Harada, D., and Russell, S. (1999), \u201cPolicy Invariance Under Reward Transformations: Theory and Application to Reward Shaping,\u201d in Icml (Vol. 99), pp. 278\u2013287, Citeseer."},{"key":"e_1_3_4_49_1","doi-asserted-by":"publisher","DOI":"10.1523\/JNEUROSCI.5498-10.2012"},{"key":"e_1_3_4_50_1","first-page":"252","volume-title":"Proceedings of the 27th International Conference on Neural Information Processing Systems (NIPS\u201913), Vol","author":"Prashanth L.","year":"2013","unstructured":"Prashanth, L., and Ghavamzadeh, M. (2013), \u201cActor-Critic Algorithms for Risk-Sensitive MDPs,\u201d in Proceedings of the 27th International Conference on Neural Information Processing Systems (NIPS\u201913), Vol. 1. Red Hook, NY: Curran Associates Inc., pp. 252\u2013260."},{"key":"e_1_3_4_51_1","first-page":"80","volume-title":"Computer Science Department Faculty Publication Series","author":"Precup D.","year":"2000","unstructured":"Precup, D. (2000), \u201cEligibility Traces for Off-Policy Policy Evaluation,\u201d Computer Science Department Faculty Publication Series p. 80."},{"key":"e_1_3_4_52_1","doi-asserted-by":"publisher","DOI":"10.21314\/JOR.2000.038"},{"key":"e_1_3_4_53_1","doi-asserted-by":"publisher","DOI":"10.1613\/jair.3987"},{"key":"e_1_3_4_54_1","first-page":"29","volume-title":"International Conference on Artificial Intelligence and Statistics","author":"Rowland M.","year":"2018","unstructured":"Rowland, M., Bellemare, M., Dabney, W., Munos, R., and Teh, Y. W. (2018), \u201cAn Analysis of Categorical Distributional Reinforcement Learning,\u201d in International Conference on Artificial Intelligence and Statistics, PMLR, pp. 29\u201337."},{"key":"e_1_3_4_55_1","doi-asserted-by":"publisher","DOI":"10.2307\/1911259"},{"key":"e_1_3_4_56_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10107-010-0393-3"},{"key":"e_1_3_4_57_1","first-page":"1875","article-title":"\u201cNonparametric Regression Using Deep Neural Networks with Relu Activation Function,\u201d","volume":"48","author":"Schmidt-Hieber J.","year":"2020","unstructured":"Schmidt-Hieber, J. (2020), \u201cNonparametric Regression Using Deep Neural Networks with Relu Activation Function,\u201d The Annals of Statistics, 48, 1875\u20131897.","journal-title":"The Annals of Statistics"},{"key":"e_1_3_4_58_1","doi-asserted-by":"publisher","DOI":"10.1162\/NECO_a_00600"},{"key":"e_1_3_4_59_1","doi-asserted-by":"crossref","unstructured":"Shi C. Qi Z. Wang J. and Zhou F. (2023) \u201cValue Enhancement of Reinforcement Learning via Efficient and Robust Trust Region Optimization \u201d arXiv preprint arXiv:2301.02220.","DOI":"10.1080\/01621459.2023.2238942"},{"key":"e_1_3_4_60_1","doi-asserted-by":"publisher","DOI":"10.1214\/aos\/1176345969"},{"key":"e_1_3_4_61_1","volume-title":"Reinforcement Learning: An Introduction","author":"Sutton R. S.","year":"2018","unstructured":"Sutton, R. S., and Barto, A. G. (2018), Reinforcement Learning: An Introduction, Cambridge, MA: MIT Press."},{"key":"e_1_3_4_62_1","volume-title":"Twenty-Ninth IAAI Conference","author":"Thomas P. S.","year":"2017","unstructured":"Thomas, P. S., Theocharous, G., Ghavamzadeh, M., Durugkar, I., and Brunskill, E. (2017), \u201cPredictive Off-Policy Policy Evaluation for Nonstationary Decision Problems, with Applications to Digital Marketing,\u201d in Twenty-Ninth IAAI Conference."},{"key":"e_1_3_4_63_1","doi-asserted-by":"publisher","DOI":"10.1111\/mafi.12281"},{"key":"e_1_3_4_64_1","doi-asserted-by":"publisher","DOI":"10.1080\/01621459.2017.1330204"},{"key":"e_1_3_4_65_1","article-title":"\u201cUnsupervised Control through Non-parametric Discriminative Rewards,\u201d","author":"Warde-Farley D.","year":"2018","unstructured":"Warde-Farley, D., Van de Wiele, T., Kulkarni, T., Ionescu, C., Hansen, S., and Mnih, V. (2018), \u201cUnsupervised Control through Non-parametric Discriminative Rewards,\u201d arXiv preprint arXiv:1811.11359.","journal-title":"arXiv preprint"},{"key":"e_1_3_4_66_1","first-page":"6193","article-title":"\u201cFully Parameterized Quantile Function for Distributional Reinforcement Learning,\u201d in","volume":"32","author":"Yang D.","year":"2019","unstructured":"Yang, D., Zhao, L., Lin, Z., Qin, T., Bian, J., and Liu, T.-Y. (2019), \u201cFully Parameterized Quantile Function for Distributional Reinforcement Learning,\u201d in Advances in Neural Information Processing Systems (Vol. 32), pp. 6193\u20136202.","journal-title":"Advances in Neural Information Processing Systems"},{"key":"e_1_3_4_67_1","unstructured":"Zhang J. Koppel A. Bedi A. S. Szepesvari C. and Wang M. (2020) \u201cVariational Policy Gradient Method for Reinforcement Learning with General Utilities \u201d arXiv preprint arXiv:2007.02151."},{"key":"e_1_3_4_68_1","unstructured":"Zhong H. Fang E. X. Yang Z. and Wang Z. (2020) \u201cRisk-Sensitive Deep rl: Variance-Constrained Actor-Critic Provably Finds Globally Optimal Policy \u201d arXiv preprint arXiv:2012.14098."},{"key":"e_1_3_4_69_1","doi-asserted-by":"publisher","DOI":"10.1070\/SM1976v030n03ABEH002280"}],"container-title":["Journal of the American Statistical Association"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/www.tandfonline.com\/doi\/pdf\/10.1080\/01621459.2025.2506197","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,2,10]],"date-time":"2026-02-10T01:46:49Z","timestamp":1770688009000},"score":1,"resource":{"primary":{"URL":"https:\/\/www.tandfonline.com\/doi\/full\/10.1080\/01621459.2025.2506197"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,7,3]]},"references-count":68,"journal-issue":{"issue":"551","published-print":{"date-parts":[[2025,7,3]]}},"alternative-id":["10.1080\/01621459.2025.2506197"],"URL":"https:\/\/doi.org\/10.1080\/01621459.2025.2506197","relation":{},"ISSN":["0162-1459","1537-274X"],"issn-type":[{"value":"0162-1459","type":"print"},{"value":"1537-274X","type":"electronic"}],"subject":[],"published":{"date-parts":[[2025,7,3]]},"assertion":[{"value":"The publishing and review policy for this title is described in its Aims & Scope.","order":1,"name":"peerreview_statement","label":"Peer Review Statement"},{"value":"http:\/\/www.tandfonline.com\/action\/journalInformation?show=aimsScope&journalCode=uasa20","URL":"http:\/\/www.tandfonline.com\/action\/journalInformation?show=aimsScope&journalCode=uasa20","order":2,"name":"aims_and_scope_url","label":"Aim & Scope"},{"value":"2022-04-12","order":0,"name":"received","label":"Received","group":{"name":"publication_history","label":"Publication History"}},{"value":"2024-12-25","order":1,"name":"revised","label":"Revised","group":{"name":"publication_history","label":"Publication History"}},{"value":"2025-02-26","order":2,"name":"accepted","label":"Accepted","group":{"name":"publication_history","label":"Publication History"}},{"value":"2025-08-05","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}