{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,2,9]],"date-time":"2026-02-09T05:53:09Z","timestamp":1770616389755,"version":"3.49.0"},"reference-count":24,"publisher":"Springer Science and Business Media LLC","issue":"9-10","license":[{"start":{"date-parts":[[2017,7,12]],"date-time":"2017-07-12T00:00:00Z","timestamp":1499817600000},"content-version":"unspecified","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Mach Learn"],"published-print":{"date-parts":[[2017,10]]},"DOI":"10.1007\/s10994-017-5650-8","type":"journal-article","created":{"date-parts":[[2017,7,12]],"date-time":"2017-07-12T14:14:09Z","timestamp":1499868849000},"page":"1569-1598","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":14,"title":["Offline reinforcement learning with task hierarchies"],"prefix":"10.1007","volume":"106","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-0172-9744","authenticated-orcid":false,"given":"Devin","family":"Schwab","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Soumya","family":"Ray","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2017,7,12]]},"reference":[{"issue":"13","key":"5650_CR1","doi-asserted-by":"crossref","first-page":"1608","DOI":"10.1177\/0278364910371999","volume":"29","author":"P Abbeel","year":"2010","unstructured":"Abbeel, P., Coates, A., & Ng, A. Y. (2010). Autonomous helicopter aerobatics through apprenticeship learning. The International Journal of Robotics Research, 29(13), 1608\u20131639.","journal-title":"The International Journal of Robotics Research"},{"key":"5650_CR2","volume-title":"Reversible Markov chains and random walks on graphs","author":"D Aldous","year":"2014","unstructured":"Aldous, D., & Fill, J. (2014). Reversible Markov chains and random walks on graphs. Berkeley: University of California Berkeley."},{"key":"5650_CR3","unstructured":"Andre, D., & Russell, S.\u00a0J. (2002). State abstraction for programmable reinforcement learning agents. In Association for the advancement of artificial intelligence\/innovative applications of artificial intelligence conference (pp. 119\u2013125)."},{"issue":"4","key":"5650_CR4","doi-asserted-by":"crossref","first-page":"341","DOI":"10.1023\/A:1025696116075","volume":"13","author":"AG Barto","year":"2003","unstructured":"Barto, A. G., & Mahadevan, S. (2003). Recent advances in hierarchical reinforcement learning. Discrete Event Dynamic Systems, 13(4), 341\u2013379.","journal-title":"Discrete Event Dynamic Systems"},{"key":"5650_CR5","unstructured":"Cao, F., & Ray, S. (2012). Bayesian hierarchical reinforcement learning. In Advances in neural information processing systems."},{"key":"5650_CR6","unstructured":"Comanici, G., & Precup, D. (2010). Optimal policy switching algorithms for reinforcement learning. In Proceedings of the 9th international conference on autonomous agents and multiagent systems (Vol. 1, pp. 709\u2013714). International Foundation for Autonomous Agents and Multiagent Systems."},{"key":"5650_CR7","doi-asserted-by":"crossref","first-page":"227","DOI":"10.1613\/jair.639","volume":"13","author":"TG Dietterich","year":"2000","unstructured":"Dietterich, T. G. (2000). Hierarchical reinforcement learning with MAXQ value function decomposition. Journal of Artificial Intelligence Research, 13, 227\u2013303.","journal-title":"Journal of Artificial Intelligence Research"},{"key":"5650_CR8","first-page":"2259","volume":"7","author":"A Jonsson","year":"2006","unstructured":"Jonsson, A. (2006). Causal graph based decomposition of factored MDPs. Journal of Machine Learning Research, 7, 2259\u20132301.","journal-title":"Journal of Machine Learning Research"},{"key":"5650_CR9","unstructured":"Kulkarni, T. D., Narasimhan, K., Saeedi, A., & Tenenbaum, J. (2016). Hierarchical deep reinforcement learning: Integrating temporal abstraction and intrinsic motivation. In Advances in neural information processing systems (Vol. 29, pp. 3675\u20133683)."},{"key":"5650_CR10","first-page":"1107","volume":"4","author":"MG Lagoudakis","year":"2003","unstructured":"Lagoudakis, M. G., & Parr, R. (2003). Least-Squares Policy Iteration. Journal of Machine Learning Research, 4, 1107\u20131149.","journal-title":"Journal of Machine Learning Research"},{"key":"5650_CR11","first-page":"3041","volume":"13","author":"A Lazaric","year":"2012","unstructured":"Lazaric, A., Ghavamzadeh, M., & Munos, R. (2012). Finite-sample analysis of least-squares policy iteration. Journal of Machine Learning Research, 13, 3041\u20133074.","journal-title":"Journal of Machine Learning Research"},{"key":"5650_CR12","unstructured":"Lov\u00e1sz, L. (1996). Random walks on graphs: A survey. In D. Mikl\u00f3s, V. T. Sos, & T. Szonyin (Eds.), Combinatorics: Paul Erd\u0151s is eighty, Bolyai society mathematical studies (pp. 353\u2013348). Budapest: J\u00e1nos Bolyai Mathematical Society."},{"key":"5650_CR13","unstructured":"Marthi, B., Russell, S., & Latham, D. (2005). Writing stratagus-playing agents in concurrent ALISP. In Reasoning: Representation, and learning in computer games (p. 67)."},{"issue":"7540","key":"5650_CR14","doi-asserted-by":"crossref","first-page":"529","DOI":"10.1038\/nature14236","volume":"518","author":"V Mnih","year":"2015","unstructured":"Mnih, V., Kavukcuoglu, K., Silver, D., Rusu, A. A., Veness, J., Bellemare, M. G., et al. (2015). Human-level control through deep reinforcement learning. Nature, 518(7540), 529\u2013533.","journal-title":"Nature"},{"key":"5650_CR15","unstructured":"Parr, R., & Russell, S. (1998). Reinforcement learning with hierarchies of machines. In Advances in neural information processing systems."},{"key":"5650_CR16","volume-title":"Markov decision processes: Discrete stochastic dynamic programming","author":"ML Puterman","year":"2005","unstructured":"Puterman, M. L. (2005). Markov decision processes: Discrete stochastic dynamic programming. Hoboken, NJ: Wiley."},{"key":"5650_CR17","unstructured":"Schultink, E., Cavallo, R., & Parkes, D.\u00a0C. (2008). Economic hierarchical q-learning. In Proceedings of the 23rd national conference on artificial intelligence."},{"issue":"7587","key":"5650_CR18","doi-asserted-by":"crossref","first-page":"484","DOI":"10.1038\/nature16961","volume":"529","author":"D Silver","year":"2016","unstructured":"Silver, D., Huang, A., Maddison, C. J., Guez, A., Sifre, L., van den Driessche, G., et al. (2016). Mastering the game of go with deep neural networks and tree search. Nature, 529(7587), 484\u2013489.","journal-title":"Nature"},{"key":"5650_CR19","doi-asserted-by":"crossref","unstructured":"Stolle, M., & Precup, D. (2002). Learning options in reinforcement learning. In International Symposium on abstraction, reformulation, and approximation (pp. 212\u2013223). Springer.","DOI":"10.1007\/3-540-45622-8_16"},{"issue":"1\u20132","key":"5650_CR20","doi-asserted-by":"crossref","first-page":"181","DOI":"10.1016\/S0004-3702(99)00052-1","volume":"112","author":"RS Sutton","year":"1999","unstructured":"Sutton, R. S., Precup, D., & Singh, S. (1999). Between MDPs and semi-MDPs: A framework for temporal abstraction in reinforcement learning. Artificial Intelligence, 112(1\u20132), 181\u2013211.","journal-title":"Artificial Intelligence"},{"key":"5650_CR21","unstructured":"van Seijen, H., & Sutton, R. (2015). A deeper look at planning as learning from replay. In International conference on machine learning (pp. 2314\u20132322)."},{"issue":"3","key":"5650_CR22","first-page":"279","volume":"8","author":"CJCH Watkins","year":"1992","unstructured":"Watkins, C. J. C. H., & Dayan, P. (1992). Q-learning. Machine Learning, 8(3), 279\u2013292.","journal-title":"Machine Learning"},{"key":"5650_CR23","doi-asserted-by":"crossref","unstructured":"Wiering, M., & van Otterlo, M. (2012). Reinforcement learning state-of-the-art. Berlin: Springer.","DOI":"10.1007\/978-3-642-27645-3"},{"key":"5650_CR24","unstructured":"Zhang, W., & Dietterich, T.\u00a0G. (1995). A reinforcement learning approach to job-shop scheduling. In International joint conference on artificial intelligence (Vol. 95)."}],"container-title":["Machine Learning"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/link.springer.com\/article\/10.1007\/s10994-017-5650-8\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s10994-017-5650-8.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s10994-017-5650-8.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2019,9,29]],"date-time":"2019-09-29T16:02:28Z","timestamp":1569772948000},"score":1,"resource":{"primary":{"URL":"http:\/\/link.springer.com\/10.1007\/s10994-017-5650-8"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2017,7,12]]},"references-count":24,"journal-issue":{"issue":"9-10","published-print":{"date-parts":[[2017,10]]}},"alternative-id":["5650"],"URL":"https:\/\/doi.org\/10.1007\/s10994-017-5650-8","relation":{},"ISSN":["0885-6125","1573-0565"],"issn-type":[{"value":"0885-6125","type":"print"},{"value":"1573-0565","type":"electronic"}],"subject":[],"published":{"date-parts":[[2017,7,12]]}}}