{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,6,11]],"date-time":"2026-06-11T15:36:09Z","timestamp":1781192169197,"version":"3.54.1"},"reference-count":51,"publisher":"Springer Science and Business Media LLC","issue":"9","license":[{"start":{"date-parts":[[2021,4,14]],"date-time":"2021-04-14T00:00:00Z","timestamp":1618358400000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"},{"start":{"date-parts":[[2021,4,14]],"date-time":"2021-04-14T00:00:00Z","timestamp":1618358400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Mach Learn"],"published-print":{"date-parts":[[2021,9]]},"DOI":"10.1007\/s10994-021-05969-w","type":"journal-article","created":{"date-parts":[[2021,4,15]],"date-time":"2021-04-15T06:28:47Z","timestamp":1618468127000},"page":"2603-2640","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":8,"title":["Partially observable environment estimation with uplift inference for reinforcement learning based recommendation"],"prefix":"10.1007","volume":"110","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-9331-4062","authenticated-orcid":false,"given":"Wenjie","family":"Shang","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Qingyang","family":"Li","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Zhiwei","family":"Qin","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Yang","family":"Yu","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Yiping","family":"Meng","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Jieping","family":"Ye","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2021,4,14]]},"reference":[{"key":"5969_CR1","unstructured":"Agarwal, A., Bird, S., Cozowicz, M., Hoang, L., Langford, J., Lee, S., et al. (2016). Making contextual decisions with low technical debt. CoRR (abs\/1606.03966)."},{"issue":"5","key":"5969_CR2","doi-asserted-by":"publisher","first-page":"469","DOI":"10.1016\/j.robot.2008.10.024","volume":"57","author":"B Argall","year":"2009","unstructured":"Argall, B., Chernova, S., Veloso, M. M., & Browning, B. (2009). A survey of robot learning from demonstration. Robotics and Autonomous Systems, 57(5), 469\u2013483.","journal-title":"Robotics and Autonomous Systems"},{"issue":"1","key":"5969_CR3","doi-asserted-by":"publisher","first-page":"174","DOI":"10.1016\/0022-247X(65)90154-X","volume":"10","author":"KJ Astrom","year":"1965","unstructured":"Astrom, K. J. (1965). Optimal control of Markov processes with incomplete state information. Journal of Mathematical Analysis and Applications, 10(1), 174\u2013205.","journal-title":"Journal of Mathematical Analysis and Applications"},{"issue":"5","key":"5969_CR4","first-page":"1","volume":"1050","author":"S Athey","year":"2015","unstructured":"Athey, S., & Imbens, G. W. (2015). Machine learning methods for estimating heterogeneous causal effects. Stat, 1050(5), 1\u201326.","journal-title":"Stat"},{"issue":"1","key":"5969_CR5","doi-asserted-by":"publisher","first-page":"5","DOI":"10.1023\/A:1010933404324","volume":"45","author":"L Breiman","year":"2001","unstructured":"Breiman, L. (2001). Random forests. Machine Learning, 45(1), 5\u201332.","journal-title":"Machine Learning"},{"key":"5969_CR6","unstructured":"Brown, N., & Sandholm, T., (2017). Safe and nested subgame solving for imperfect-information games. In Advances in neural information processing systems 30: annual conference on neural information processing systems 2017, pp. 689\u2013699."},{"key":"5969_CR7","doi-asserted-by":"crossref","unstructured":"Cassandra, A., Nodine, M., Bondale, S., Ford, S., & Wells, D. (2005). Using pomdp-based state estimation to enhance agent system survivability. In IEEE 2nd symposium on multi-agent security and survivability, pp. 11\u201320.","DOI":"10.1145\/1082473.1082663"},{"key":"5969_CR8","doi-asserted-by":"crossref","unstructured":"Chen, T., & Guestrin, C. (2016). Xgboost: A scalable tree boosting system. In Proceedings of the 22nd ACM SIGKDD international conference on knowledge discovery and data mining, pp. 785\u2013794.","DOI":"10.1145\/2939672.2939785"},{"key":"5969_CR9","unstructured":"Chen, X., Li, S., Li, H., Jiang, S., Qi, Y., & Song, L. (2019). Generative adversarial user model for reinforcement learning based recommendation system. In Proceedings of the 36th international conference on machine learning, ICML, Vol.\u00a097, pp. 1052\u20131061."},{"key":"5969_CR10","unstructured":"Finn, C., Christiano, P. F., Abbeel, P., & Levine, S. (2016). A connection between generative adversarial networks, inverse reinforcement learning, and energy-based models. CoRR abs\/1611.03852."},{"key":"5969_CR11","first-page":"2672","volume":"27","author":"I Goodfellow","year":"2014","unstructured":"Goodfellow, I., Pouget-Abadie, J., Mirza, M., Xu, B., Warde-Farley, D., Ozair, S., et al. (2014). Generative adversarial nets. Advances in Neural Information Processing Systems, 27, 2672\u20132680.","journal-title":"Advances in Neural Information Processing Systems"},{"key":"5969_CR12","unstructured":"Guelman, L., Guill\u00e9n, M., & P\u00e9rez\u00a0Mar\u00edn, A. M. (2014). Optimal personalized treatment rules for marketing interventions: A review of methods, a new proposal, and an insurance case study. UB Riskcenter Working Paper Series."},{"issue":"3\u20134","key":"5969_CR13","doi-asserted-by":"publisher","first-page":"230","DOI":"10.1080\/01969722.2015.1012892","volume":"46","author":"L Guelman","year":"2015","unstructured":"Guelman, L., Guill\u00e9n, M., & P\u00e9rez-Mar\u00edn, A. M. (2015). Uplift random forests. Cybernetics and Systems, 46(3\u20134), 230\u2013248.","journal-title":"Cybernetics and Systems"},{"key":"5969_CR14","unstructured":"Gutierrez, P., & G\u00e9rardy, J. Y. (2017). Causal inference and uplift modelling: A review of the literature. In International conference on predictive applications and APIs, pp. 1\u201313."},{"issue":"3","key":"5969_CR15","doi-asserted-by":"publisher","first-page":"35","DOI":"10.1002\/dir.10035","volume":"16","author":"B Hansotia","year":"2002","unstructured":"Hansotia, B., & Rukstales, B. (2002). Incremental value modeling. Journal of Interactive Marketing, 16(3), 35.","journal-title":"Journal of Interactive Marketing"},{"key":"5969_CR16","first-page":"4565","volume":"29","author":"J Ho","year":"2016","unstructured":"Ho, J., & Ermon, S. (2016). Generative adversarial imitation learning. Advances in Neural Information Processing Systems, 29, 4565\u20134573.","journal-title":"Advances in Neural Information Processing Systems"},{"key":"5969_CR17","unstructured":"Jaskowski, M., & Jaroszewicz, S. (2012). Uplift modeling for clinical trial data. In ICML Workshop on clinical data analysis."},{"key":"5969_CR18","unstructured":"Johansson, F. D., Shalit, U., & Sontag, D. A. (2016). Learning representations for counterfactual inference. In Proceedings of the 33nd international conference on machine learning, ICML, Vol.\u00a048, pp. 3020\u20133029."},{"issue":"1\u20132","key":"5969_CR19","doi-asserted-by":"publisher","first-page":"99","DOI":"10.1016\/S0004-3702(98)00023-X","volume":"101","author":"LP Kaelbling","year":"1998","unstructured":"Kaelbling, L. P., Littman, M. L., & Cassandra, A. R. (1998). Planning and acting in partially observable stochastic domains. Artificial Intelligence, 101(1\u20132), 99\u2013134.","journal-title":"Artificial Intelligence"},{"issue":"10","key":"5969_CR20","doi-asserted-by":"publisher","first-page":"4156","DOI":"10.1073\/pnas.1804597116","volume":"116","author":"SR K\u00fcnzel","year":"2019","unstructured":"K\u00fcnzel, S. R., Sekhon, J. S., Bickel, P. J., & Yu, B. (2019). Metalearners for estimating heterogeneous treatment effects using machine learning. Proceedings of the National Academy of Sciences, 116(10), 4156\u20134165.","journal-title":"Proceedings of the National Academy of Sciences"},{"issue":"2","key":"5969_CR21","doi-asserted-by":"publisher","first-page":"78","DOI":"10.1145\/772862.772872","volume":"4","author":"VSY Lo","year":"2002","unstructured":"Lo, V. S. Y. (2002). The true lift model\u2014a novel data mining approach to response modeling in database marketing. ACM SIGKDD Explorations Newsletter, 4(2), 78\u201386.","journal-title":"ACM SIGKDD Explorations Newsletter"},{"key":"5969_CR22","unstructured":"Menick, J., & Kalchbrenner, N. (2018). Generating high fidelity images with subscale pixel networks and multidimensional upscaling. CoRR abs\/1812.01608."},{"key":"5969_CR23","doi-asserted-by":"crossref","unstructured":"Michel, R., Schnakenburg, I., & Martens, T. (2019). Targeting uplift: An introduction to net scores. Springer","DOI":"10.1007\/978-3-030-22625-1"},{"key":"5969_CR24","doi-asserted-by":"publisher","first-page":"529","DOI":"10.1038\/nature14236","volume":"518","author":"V Mnih","year":"2015","unstructured":"Mnih, V., Kavukcuoglu, K., Silver, D., Rusu, A. A., Veness, J., Bellemare, M. G., et al. (2015). Human-level control through deep reinforcement learning. Nature, 518, 529\u2013533.","journal-title":"Nature"},{"key":"5969_CR25","unstructured":"Nassif, H., Kuusisto, F., Burnside, E. S., & Shavlik, J. W. (2013). Uplift modeling with ROC: An SRL case study. In Late breaking papers of the 23rd international conference on inductive logic programming, CEUR workshop proceedings, Vol. 1187, pp. 40\u201345."},{"key":"5969_CR26","first-page":"06680","volume":"1912","author":"BC OpenAI","year":"2019","unstructured":"OpenAI, B. C., Brockman, G., Chan, B., Cheung, V., D\u0229biak, P., Dennison, C., et al. (2019). Dota 2 with large scale deep reinforcement learning. CoRR, 1912, 06680.","journal-title":"CoRR"},{"key":"5969_CR27","unstructured":"Pineau, J., Gordon, G. J., & Thrun, S. (2003). Point-based value iteration: An anytime algorithm for pomdps. In Proceedings of the eighteenth international joint conference on artificial intelligence, pp. 1025\u20131032."},{"issue":"1","key":"5969_CR28","doi-asserted-by":"publisher","first-page":"88","DOI":"10.1162\/neco.1991.3.1.88","volume":"3","author":"D Pomerleau","year":"1991","unstructured":"Pomerleau, D. (1991). Efficient training of artificial neural networks for autonomous navigation. Neural Computation, 3(1), 88\u201397.","journal-title":"Neural Computation"},{"issue":"5","key":"5969_CR29","doi-asserted-by":"publisher","first-page":"272","DOI":"10.1287\/inte.2020.1047","volume":"50","author":"ZT Qin","year":"2020","unstructured":"Qin, Z. T., Tang, X., Jiao, Y., Zhang, F., Xu, Z., Zhu, H., & Ye, J. (2020). Ride-hailing order dispatching at didi via reinforcement learning. INFORMS Journal on Applied Analytics, 50(5), 272\u2013286.","journal-title":"INFORMS Journal on Applied Analytics"},{"key":"5969_CR30","first-page":"1421","volume":"1","author":"NJ Radcliffe","year":"2007","unstructured":"Radcliffe, N. J. (2007). Using control groups to target on predicted lift: Building and assessing uplift models. Direct Marketing Analytics Journal, 1, 1421.","journal-title":"Direct Marketing Analytics Journal"},{"key":"5969_CR31","unstructured":"Radcliffe, N. J., & Surry, P. D. (2011). Real-world uplift modelling with significance-based uplift trees. White Paper TR-2011-1, Stochastic Solutions pp. 1\u201333."},{"issue":"1","key":"5969_CR32","doi-asserted-by":"publisher","first-page":"41","DOI":"10.1093\/biomet\/70.1.41","volume":"70","author":"PR Rosenbaum","year":"1983","unstructured":"Rosenbaum, P. R., & Rubin, D. B. (1983). The central role of the propensity score in observational studies for causal effects. Biometrika, 70(1), 41\u201355.","journal-title":"Biometrika"},{"key":"5969_CR33","unstructured":"Ross, S., Gordon, G. J., & Bagnell, D. (2011). A reduction of imitation learning and structured prediction to no-regret online learning. In Proceedings of the fourteenth international conference on artificial intelligence and statistics, pp. 627\u2013635."},{"key":"5969_CR34","doi-asserted-by":"crossref","unstructured":"Russell, S. J. (1998). Learning agents for uncertain environments (extended abstract). In Proceedings of the eleventh annual conference on computational learning theory, pp. 101\u2013103.","DOI":"10.1145\/279943.279964"},{"issue":"2","key":"5969_CR35","doi-asserted-by":"publisher","first-page":"303","DOI":"10.1007\/s10115-011-0434-0","volume":"32","author":"P Rzepakowski","year":"2012","unstructured":"Rzepakowski, P., & Jaroszewicz, S. (2012). Decision trees for uplift modeling with single and multiple treatments. Knowledge and Information Systems, 32(2), 303\u2013327.","journal-title":"Knowledge and Information Systems"},{"key":"5969_CR36","doi-asserted-by":"crossref","unstructured":"Sadeghi, F., & Levine, S. (2016). CAD2RL: Real single-image flight without a single real image. CoRR abs\/1611.04201.","DOI":"10.15607\/RSS.2017.XIII.034"},{"key":"5969_CR37","first-page":"1050","volume":"12","author":"B Sallans","year":"1999","unstructured":"Sallans, B. (1999). Learning factored representations for partially observable Markov decision processes. Neural Information Processing Systems, 12, 1050\u20131056.","journal-title":"Neural Information Processing Systems"},{"issue":"6","key":"5969_CR38","doi-asserted-by":"publisher","first-page":"233","DOI":"10.1016\/S1364-6613(99)01327-3","volume":"3","author":"S Schaal","year":"1999","unstructured":"Schaal, S. (1999). Is imitation learning the route to humanoid robots? Trends in Cognitive Sciences, 3(6), 233\u2013242.","journal-title":"Trends in Cognitive Sciences"},{"key":"5969_CR39","unstructured":"Schulman, J., Levine, S., Abbeel, P., Jordan, M. I., & Moritz, P. (2015). Trust region policy optimization. In Proceedings of the 32nd international conference on machine learning, pp. 1889\u20131897."},{"key":"5969_CR40","doi-asserted-by":"crossref","unstructured":"Shang, W., Yu, Y., Li, Q., Qin, Z., Meng. Y., & Ye, J. (2019). Environment reconstruction with hidden confounders for reinforcement learning based recommendation. In Proceedings of the 25th ACM SIGKDD international conference on knowledge discovery & data mining, pp. 566\u2013576.","DOI":"10.1145\/3292500.3330933"},{"key":"5969_CR41","unstructured":"Shi, J., Yu, Y., Da, Q., Chen, S., & Zeng, A (2018) Virtual-taobao: Virtualizing real-world online retail environment for reinforcement learning. CoRR abs\/1805.10000."},{"issue":"7587","key":"5969_CR42","doi-asserted-by":"publisher","first-page":"484","DOI":"10.1038\/nature16961","volume":"529","author":"D Silver","year":"2016","unstructured":"Silver, D., Huang, A., Maddison, C. J., Guez, A., Sifre, L., van den Driessche, G., et al. (2016). Mastering the game of go with deep neural networks and tree search. Nature, 529(7587), 484\u2013489.","journal-title":"Nature"},{"key":"5969_CR43","doi-asserted-by":"crossref","unstructured":"Singh, S. P., Jaakkola, T. S., & Jordan, M. I. (1994). Learning without state-estimation in partially observable markovian decision processes. In Proceedings of the eleventh international conference on machine learning, pp. 284\u2013292.","DOI":"10.1016\/B978-1-55860-335-6.50042-8"},{"key":"5969_CR44","unstructured":"Su, X., Kang, J., Fan, J., Levine, R. A., & Yan, X. (2012). Facilitating score and causal inference trees for large observational studies. Journal of Machine Learning Research, 13(Oct):2955\u20132994."},{"key":"5969_CR45","unstructured":"Sutton, R. S., & Barto, A. G. (2018). Reinforcement learning: An introduction (2nd ed.). MIT Press."},{"key":"5969_CR46","doi-asserted-by":"crossref","unstructured":"Tzeng, E., Devin, C., Hoffman, J., Finn, C., Abbeel, P., Levine, S., et al. (2016). Adapting deep visuomotor representations with weak pairwise constraints. Algorithmic foundations of robotics XII, Proceedings of the twelfth workshop on the algorithmic foundations of robotics, WAFR, Vol. 13, pp. 688\u2013703.","DOI":"10.1007\/978-3-030-43089-4_44"},{"issue":"523","key":"5969_CR47","doi-asserted-by":"publisher","first-page":"1228","DOI":"10.1080\/01621459.2017.1319839","volume":"113","author":"S Wager","year":"2018","unstructured":"Wager, S., & Athey, S. (2018). Estimation and inference of heterogeneous treatment effects using random forests. Journal of the American Statistical Association, 113(523), 1228\u20131242.","journal-title":"Journal of the American Statistical Association"},{"key":"5969_CR48","doi-asserted-by":"crossref","unstructured":"Ye, Z., Zhang, L., Xiao, K., Zhou, W., Ge, Y., & Deng, Y. (2018). Multi-user mobile sequential recommendation: An efficient parallel computing paradigm. In Proceedings of the 24th ACM SIGKDD international conference on knowledge discovery & data mining, pp. 2624\u20132633.","DOI":"10.1145\/3219819.3220111"},{"issue":"2","key":"5969_CR49","doi-asserted-by":"publisher","first-page":"243","DOI":"10.1109\/TKDE.2018.2827047","volume":"31","author":"Z Ye","year":"2019","unstructured":"Ye, Z., Xiao, K., Ge, Y., & Deng, Y. (2019). Applying simulated annealing and parallel computing to the mobile sequential recommendation. IEEE Transactions on Knowledge and Data Engineering, 31(2), 243\u2013256.","journal-title":"IEEE Transactions on Knowledge and Data Engineering"},{"key":"5969_CR50","doi-asserted-by":"crossref","unstructured":"Zaniewicz, L., & Jaroszewicz, S. (2013). Support vector machines for uplift modeling. In 13th IEEE international conference on data mining workshops, ICDM workshops, pp. 131\u2013138.","DOI":"10.1109\/ICDMW.2013.23"},{"issue":"3","key":"5969_CR51","doi-asserted-by":"publisher","first-page":"533","DOI":"10.1007\/s10994-019-05835-w","volume":"109","author":"P Zhao","year":"2020","unstructured":"Zhao, P., Cai, L. W., & Zhou, Z. H. (2020). Handling concept drift via model reuse. Machine Learning, 109(3), 533\u2013568.","journal-title":"Machine Learning"}],"container-title":["Machine Learning"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10994-021-05969-w.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s10994-021-05969-w\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10994-021-05969-w.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2022,4,14]],"date-time":"2022-04-14T00:12:03Z","timestamp":1649895123000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s10994-021-05969-w"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2021,4,14]]},"references-count":51,"journal-issue":{"issue":"9","published-print":{"date-parts":[[2021,9]]}},"alternative-id":["5969"],"URL":"https:\/\/doi.org\/10.1007\/s10994-021-05969-w","relation":{},"ISSN":["0885-6125","1573-0565"],"issn-type":[{"value":"0885-6125","type":"print"},{"value":"1573-0565","type":"electronic"}],"subject":[],"published":{"date-parts":[[2021,4,14]]},"assertion":[{"value":"15 May 2020","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"6 February 2021","order":2,"name":"revised","label":"Revised","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"4 March 2021","order":3,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"14 April 2021","order":4,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"This content has been made available to all.","name":"free","label":"Free to read"}]}}