{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,3,7]],"date-time":"2026-03-07T02:26:02Z","timestamp":1772850362446,"version":"3.50.1"},"reference-count":63,"publisher":"Institute of Electrical and Electronics Engineers (IEEE)","issue":"9","license":[{"start":{"date-parts":[[2022,9,1]],"date-time":"2022-09-01T00:00:00Z","timestamp":1661990400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/ieeexplore.ieee.org\/Xplorehelp\/downloads\/license-information\/IEEE.html"},{"start":{"date-parts":[[2022,9,1]],"date-time":"2022-09-01T00:00:00Z","timestamp":1661990400000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2022,9,1]],"date-time":"2022-09-01T00:00:00Z","timestamp":1661990400000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"}],"funder":[{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["61671266"],"award-info":[{"award-number":["61671266"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["61836004"],"award-info":[{"award-number":["61836004"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["U19B2034"],"award-info":[{"award-number":["U19B2034"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["61836014"],"award-info":[{"award-number":["61836014"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]},{"name":"Tsinghua-Guoqiang research program","award":["2019GQG0006"],"award-info":[{"award-number":["2019GQG0006"]}]}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["IEEE Trans. Pattern Anal. Mach. Intell."],"published-print":{"date-parts":[[2022,9,1]]},"DOI":"10.1109\/tpami.2021.3069005","type":"journal-article","created":{"date-parts":[[2021,3,25]],"date-time":"2021-03-25T20:37:52Z","timestamp":1616704672000},"page":"5572-5589","source":"Crossref","is-referenced-by-count":6,"title":["State-Temporal Compression in Reinforcement Learning With the Reward-Restricted Geodesic Metric"],"prefix":"10.1109","volume":"44","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-3181-6881","authenticated-orcid":false,"given":"Shangqi","family":"Guo","sequence":"first","affiliation":[{"name":"Department of Automation, Tsinghua University, Beijing, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-1633-2320","authenticated-orcid":false,"given":"Qi","family":"Yan","sequence":"additional","affiliation":[{"name":"Department of Automation, Tsinghua University, Beijing, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-5979-8702","authenticated-orcid":false,"given":"Xin","family":"Su","sequence":"additional","affiliation":[{"name":"Department of Automation, Tsinghua University, Beijing, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-4907-7354","authenticated-orcid":false,"given":"Xiaolin","family":"Hu","sequence":"additional","affiliation":[{"name":"Department of Computer Science and Technology, Institute for Artificial Intelligence, Beijing National Research Center for Information Science and Technology, State Key Laboratory of Intelligent Technology and Systems, Tsinghua University, Beijing, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-4813-2494","authenticated-orcid":false,"given":"Feng","family":"Chen","sequence":"additional","affiliation":[{"name":"Department of Automation, Tsinghua University, Beijing, China"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"263","reference":[{"key":"ref1","doi-asserted-by":"publisher","DOI":"10.1038\/nature14236"},{"key":"ref2","doi-asserted-by":"publisher","DOI":"10.1109\/CIG.2016.7860433"},{"key":"ref3","article-title":"On the sample complexity of reinforcement learning,","author":"Kakade","year":"2003"},{"key":"ref4","first-page":"2413","article-title":"Reinforcement learning in finite MDPs: PAC analysis,","volume":"10","author":"Strehl","year":"2009","journal-title":"J. Mach. Learn. Res."},{"key":"ref5","doi-asserted-by":"publisher","DOI":"10.1162\/153244303765208377"},{"key":"ref6","first-page":"3675","article-title":"Hierarchical deep reinforcement learning: Integrating temporal abstraction and intrinsic motivation,","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Kulkarni"},{"key":"ref7","first-page":"13 544","article-title":"RUDDER: Return decomposition for delayed rewards,","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Arjona-Medina"},{"key":"ref8","volume-title":"Reinforcement Learning: An Introduction","author":"Sutton","year":"2018"},{"key":"ref9","first-page":"531","article-title":"Towards a unified theory of state abstraction for MDPs,","volume-title":"Proc. Int. Symp. Artif. Intell. Math.","author":"Li"},{"key":"ref10","first-page":"10","article-title":"State abstractions for lifelong reinforcement learning,","volume-title":"Proc. 35th Int. Conf. Mach. Learn.","author":"Abel"},{"key":"ref11","article-title":"An algebraic approach to abstraction in reinforcement learning,","author":"Ravindran","year":"2004"},{"key":"ref12","first-page":"1011","article-title":"SMDP homomorphisms: An algebraic approach to abstraction in semi-Markov decision processes,","volume-title":"Proc. 12th Int. Joint Conf. Artif. Intell.","author":"Ravindran"},{"key":"ref13","doi-asserted-by":"publisher","DOI":"10.1016\/S0004-3702(99)00052-1"},{"key":"ref14","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v31i1.10916"},{"key":"ref15","first-page":"9414","article-title":"Language as an abstraction for hierarchical deep reinforcement learning,","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Jiang"},{"key":"ref16","article-title":"Approximate homomorphisms: A framework for non-exact minimization in Markov decision processes,","volume-title":"Proc. 5th Int. Conf. Knowl. Based Comput. Syst.","author":"Ravindran"},{"key":"ref17","article-title":"Approximate exploration through state abstraction,","author":"Ta\u00efga","year":"2018"},{"key":"ref18","first-page":"3540","article-title":"Feudal networks for hierarchical reinforcement learning,","volume-title":"Proc. 34th Int. Conf. Mach. Learn.","author":"Vezhnevets"},{"key":"ref19","first-page":"3303","article-title":"Data-efficient hierarchical reinforcement learning,","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Nachum"},{"key":"ref20","first-page":"162","article-title":"Metrics for finite Markov decision processes,","volume-title":"Proc. 20th Conf. Uncertainty Artif. Intell.","author":"Ferns"},{"key":"ref21","first-page":"166","article-title":"Modular multitask reinforcement learning with policy sketches,","volume-title":"Proc. 34th Int. Conf. Mach. Learn.","author":"Andreas"},{"key":"ref22","first-page":"2661","article-title":"Zero-shot task generalization with multi-task deep reinforcement learning,","volume-title":"Proc. 34th Int. Conf. Mach. Learn.","author":"Oh"},{"key":"ref23","doi-asserted-by":"publisher","DOI":"10.1109\/CVPRW.2017.70"},{"key":"ref24","doi-asserted-by":"publisher","DOI":"10.1613\/jair.639"},{"key":"ref25","article-title":"On planning, prediction and knowledge transfer in fully and partially observable Markov decision processes,","author":"Castro","year":"2011"},{"key":"ref26","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-29946-9_16"},{"key":"ref27","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v24i1.7751"},{"key":"ref28","first-page":"201","article-title":"Metrics for Markov decision processes with infinite state spaces,","volume-title":"Proc. 21th Conf. Uncertainty Artif. Intell.","author":"Ferns"},{"key":"ref29","doi-asserted-by":"publisher","DOI":"10.1111\/j.1751-5823.2002.tb00178.x"},{"key":"ref30","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v34i06.6564"},{"key":"ref31","first-page":"1649","article-title":"Bounding performance loss in approximate MDP homomorphisms,","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Taylor"},{"key":"ref32","first-page":"1639","article-title":"Value preserving state-action abstractions,","volume-title":"Proc. 23rd Int. Conf. Artif. Intell. Statist.","author":"Abel"},{"key":"ref33","doi-asserted-by":"publisher","DOI":"10.1145\/1015330.1015355"},{"key":"ref34","first-page":"994","article-title":"State abstraction in MAXQ hierarchical reinforcement learning,","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Dietterich"},{"key":"ref35","first-page":"2295","article-title":"A Laplacian framework for option discovery in reinforcement learning,","volume-title":"Proc. 34th Int. Conf. Mach. Learn.","author":"Machado"},{"key":"ref36","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v33i01.33015175"},{"key":"ref37","first-page":"2915","article-title":"Near optimal behavior via approximate state abstraction,","volume-title":"Proc. 33rd Int. Conf. Mach. Learn.","author":"Abel"},{"key":"ref38","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-540-75225-7_30"},{"key":"ref39","first-page":"179","article-title":"Abstraction selection in model-based reinforcement learning,","volume-title":"Proc. 32nd Int. Conf. Mach. Learn.","author":"Jiang"},{"key":"ref40","first-page":"2169","article-title":"Proto-value functions: A Laplacian framework for learning representation and control in Markov decision processes,","volume":"8","author":"Mahadevan","year":"2007","journal-title":"J. Mach. Learn. Res."},{"key":"ref41","doi-asserted-by":"publisher","DOI":"10.1016\/S0550-3213(03)00355-9"},{"key":"ref42","doi-asserted-by":"publisher","DOI":"10.1162\/neco.1993.5.4.613"},{"key":"ref43","first-page":"4055","article-title":"Successor features for transfer in reinforcement learning,","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Barreto"},{"key":"ref44","doi-asserted-by":"publisher","DOI":"10.1103\/PhysRevE.73.031108"},{"key":"ref45","doi-asserted-by":"publisher","DOI":"10.1103\/PhysRevE.86.031143"},{"key":"ref46","article-title":"Distance metric learning: A comprehensive survey,","author":"Yang"},{"key":"ref47","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2006.100"},{"key":"ref48","article-title":"Near-optimal representation learning for hierarchical reinforcement learning,","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Nachum"},{"issue":"1","key":"ref49","first-page":"1633","article-title":"Transfer learning for reinforcement learning domains: A survey,","volume":"10","author":"Taylor","year":"2009","journal-title":"J. Mach. Learn. Res."},{"key":"ref50","article-title":"Semi-parametric topological memory for navigation,","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Savinov"},{"key":"ref51","article-title":"Episodic curiosity through reachability,","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Savinov"},{"key":"ref52","first-page":"695","article-title":"Adaptive k-means clustering,","volume-title":"Proc. Int. Florida Artif. Intell. Res. Soc. Conf.","author":"Bhatia"},{"key":"ref53","first-page":"3619","article-title":"Symbol acquisition for probabilistic high-level planning,","volume-title":"Proc. 24th Int. Joint Conf. Artif. Intell.","author":"Konidaris"},{"key":"ref54","doi-asserted-by":"publisher","DOI":"10.1613\/jair.5575"},{"key":"ref55","article-title":"Adam: A method for stochastic optimization,","author":"Kingma","year":"2014"},{"key":"ref56","article-title":"Eigenoption discovery through the deep successor representation,","author":"Machado","year":"2018"},{"key":"ref57","article-title":"OpenAI gym,","author":"Brockman","year":"2016"},{"key":"ref58","article-title":"Temporal difference models: Model-free deep RL for model-based control,","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Pong"},{"key":"ref59","doi-asserted-by":"publisher","DOI":"10.1016\/j.cobeha.2018.11.005"},{"key":"ref60","doi-asserted-by":"publisher","DOI":"10.1016\/j.conb.2012.05.008"},{"key":"ref61","article-title":"Unsupervised subgoal discovery method for learning hierarchical representations,","volume-title":"Proc. 7th Int. Conf. Learn. Representations","author":"Rafati"},{"key":"ref62","doi-asserted-by":"publisher","DOI":"10.1109\/TAMD.2011.2160943"},{"key":"ref63","article-title":"Option discovery in hierarchical reinforcement learning using spatio-temporal clustering,","author":"Srinivas","year":"2020"}],"container-title":["IEEE Transactions on Pattern Analysis and Machine Intelligence"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx7\/34\/9893109\/09387144.pdf?arnumber=9387144","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,1,9]],"date-time":"2024-01-09T23:24:27Z","timestamp":1704842667000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/9387144\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2022,9,1]]},"references-count":63,"journal-issue":{"issue":"9"},"URL":"https:\/\/doi.org\/10.1109\/tpami.2021.3069005","relation":{},"ISSN":["0162-8828","2160-9292","1939-3539"],"issn-type":[{"value":"0162-8828","type":"print"},{"value":"2160-9292","type":"electronic"},{"value":"1939-3539","type":"electronic"}],"subject":[],"published":{"date-parts":[[2022,9,1]]}}}