{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,25]],"date-time":"2026-04-25T09:51:16Z","timestamp":1777110676270,"version":"3.51.4"},"update-to":[{"DOI":"10.1371\/journal.pcbi.1008317","type":"new_version","label":"New version","source":"publisher","updated":{"date-parts":[[2020,10,27]],"date-time":"2020-10-27T00:00:00Z","timestamp":1603756800000}}],"reference-count":56,"publisher":"Public Library of Science (PLoS)","issue":"10","license":[{"start":{"date-parts":[[2020,10,15]],"date-time":"2020-10-15T00:00:00Z","timestamp":1602720000000},"content-version":"vor","delay-in-days":0,"URL":"http:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"content-domain":{"domain":["www.ploscompbiol.org"],"crossmark-restriction":false},"short-container-title":["PLoS Comput Biol"],"DOI":"10.1371\/journal.pcbi.1008317","type":"journal-article","created":{"date-parts":[[2020,10,15]],"date-time":"2020-10-15T15:00:57Z","timestamp":1602774057000},"page":"e1008317","update-policy":"https:\/\/doi.org\/10.1371\/journal.pcbi.corrections_policy","source":"Crossref","is-referenced-by-count":34,"title":["Reward-predictive representations generalize across tasks in reinforcement learning"],"prefix":"10.1371","volume":"16","author":[{"ORCID":"https:\/\/orcid.org\/0000-0001-5897-499X","authenticated-orcid":true,"given":"Lucas","family":"Lehnert","sequence":"first","affiliation":[]},{"given":"Michael L.","family":"Littman","sequence":"additional","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0001-8451-0523","authenticated-orcid":true,"given":"Michael J.","family":"Frank","sequence":"additional","affiliation":[]}],"member":"340","published-online":{"date-parts":[[2020,10,15]]},"reference":[{"key":"pcbi.1008317.ref001","volume-title":"Reinforcement learning: An introduction","author":"RS Sutton","year":"2018"},{"issue":"7540","key":"pcbi.1008317.ref002","doi-asserted-by":"crossref","first-page":"529","DOI":"10.1038\/nature14236","article-title":"Human-level control through deep reinforcement learning","volume":"518","author":"V Mnih","year":"2015","journal-title":"Nature"},{"issue":"4","key":"pcbi.1008317.ref003","doi-asserted-by":"crossref","first-page":"e1006116","DOI":"10.1371\/journal.pcbi.1006116","article-title":"Compositional clustering in task structure learning","volume":"14","author":"NT Franklin","year":"2018","journal-title":"PLoS computational biology"},{"key":"pcbi.1008317.ref004","doi-asserted-by":"crossref","first-page":"160","DOI":"10.1016\/j.cognition.2016.04.002","article-title":"Neural signature of hierarchically structured expectations predicts clustering and transfer of rule sets in reinforcement learning","volume":"152","author":"AGE Collins","year":"2016","journal-title":"Cognition"},{"key":"pcbi.1008317.ref005","unstructured":"Abel D, Arumugam D, Lehnert L, Littman M. State Abstractions for Lifelong Reinforcement Learning. In: Dy J, Krause A, editors. Proceedings of the 35th International Conference on Machine Learning. vol. 80 of Proceedings of Machine Learning Research. Stockholmsm\u00e4ssan, Stockholm Sweden: PMLR; 2018. p. 10\u201319. Available from: http:\/\/proceedings.mlr.press\/v80\/abel18a.html."},{"key":"pcbi.1008317.ref006","unstructured":"Li L, Walsh TJ, Littman ML. Towards a Unified Theory of State Abstraction for MDPs. In: ISAIM; 2006."},{"key":"pcbi.1008317.ref007","doi-asserted-by":"crossref","DOI":"10.1002\/9780470316887","volume-title":"Markov decision processes: discrete stochastic dynamic programming","author":"ML Puterman","year":"1994"},{"issue":"Jul","key":"pcbi.1008317.ref008","first-page":"1633","article-title":"Transfer learning for reinforcement learning domains: A survey","volume":"10","author":"ME Taylor","year":"2009","journal-title":"Journal of Machine Learning Research"},{"key":"pcbi.1008317.ref009","unstructured":"Lehnert L, Littman ML. Successor Features Combine Elements of Model-Free and Model-based Reinforcement Learning. arXiv preprint arXiv:190111437v2. 2019;."},{"issue":"4","key":"pcbi.1008317.ref010","doi-asserted-by":"crossref","first-page":"613","DOI":"10.1162\/neco.1993.5.4.613","article-title":"Improving generalization for temporal difference learning: The successor representation","volume":"5","author":"P Dayan","year":"1993","journal-title":"Neural Computation"},{"issue":"9","key":"pcbi.1008317.ref011","doi-asserted-by":"crossref","first-page":"680","DOI":"10.1038\/s41562-017-0180-8","article-title":"The successor representation in human reinforcement learning","volume":"1","author":"I Momennejad","year":"2017","journal-title":"Nature Human Behaviour"},{"key":"pcbi.1008317.ref012","first-page":"4055","article-title":"Successor features for transfer in reinforcement learning","author":"A Barreto","year":"2017","journal-title":"Advances in neural information processing systems"},{"key":"pcbi.1008317.ref013","unstructured":"Barreto A, Borsa D, Quan J, Schaul T, Silver D, Hessel M, et al. Transfer in Deep Reinforcement Learning Using Successor Features and Generalised Policy Improvement. In: Proceedings of the 35th International Conference on Machine Learning. vol. 80 of Proceedings of Machine Learning Research. PMLR; 2018. p. 501\u2013510."},{"key":"pcbi.1008317.ref014","unstructured":"Lehnert L, Tellex S, Littman ML. Advantages and Limitations of using Successor Features for Transfer in Reinforcement Learning. arXiv preprint arXiv:170800102. 2017;."},{"issue":"3","key":"pcbi.1008317.ref015","doi-asserted-by":"crossref","first-page":"527","DOI":"10.1093\/cercor\/bhr117","article-title":"Mechanisms of hierarchical reinforcement learning in cortico\u2013striatal circuits 2: Evidence from fMRI","volume":"22","author":"D Badre","year":"2011","journal-title":"Cerebral cortex"},{"key":"pcbi.1008317.ref016","article-title":"Generalizing to generalize: when (and when not) to be compositional in task structure learning","author":"NT Franklin","year":"2019","journal-title":"bioRxiv"},{"key":"pcbi.1008317.ref017","doi-asserted-by":"crossref","first-page":"1643","DOI":"10.1038\/nn.4650","article-title":"The hippocampus as a predictive map","volume":"20","author":"KL Stachenfeld","year":"2017","journal-title":"Nature Neuroscience"},{"issue":"9","key":"pcbi.1008317.ref018","doi-asserted-by":"crossref","first-page":"e1005768","DOI":"10.1371\/journal.pcbi.1005768","article-title":"Predictive representations can link model-based reinforcement learning to model-free mechanisms","volume":"13","author":"EM Russek","year":"2017","journal-title":"PLoS computational biology"},{"issue":"3","key":"pcbi.1008317.ref019","doi-asserted-by":"crossref","first-page":"279","DOI":"10.1007\/BF00992698","article-title":"Q-learning","volume":"8","author":"CJCH Watkins","year":"1992","journal-title":"Machine Learning"},{"key":"pcbi.1008317.ref020","doi-asserted-by":"crossref","unstructured":"Sutton RS. Integrated architectures for learning, planning, and reacting based on approximating dynamic programming. In: Proceedings of the seventh international conference on machine learning; 1990. p. 216\u2013224.","DOI":"10.1016\/B978-1-55860-141-3.50030-4"},{"issue":"Oct","key":"pcbi.1008317.ref021","first-page":"213","article-title":"R-max-a general polynomial time algorithm for near-optimal reinforcement learning","volume":"3","author":"RI Brafman","year":"2002","journal-title":"Journal of Machine Learning Research"},{"key":"pcbi.1008317.ref022","article-title":"The Arcade Learning Environment: An Evaluation Platform for General Agents","author":"MG Bellemare","year":"2012","journal-title":"CoRR"},{"key":"pcbi.1008317.ref023","author":"B Ravindran","year":"2004","journal-title":"Approximate Homomorphisms: A framework for non-exact minimization in Markov Decision Processes"},{"key":"pcbi.1008317.ref024","volume-title":"Fascicle 3: Generating All Combinations and Partitions","author":"DE Knuth","year":"2005"},{"key":"pcbi.1008317.ref025","article-title":"Successor Features for Transfer in Reinforcement Learning","author":"A Barreto","year":"2016","journal-title":"CoRR"},{"issue":"1","key":"pcbi.1008317.ref026","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1016\/j.jmp.2011.08.004","article-title":"A tutorial on Bayesian nonparametric models","volume":"56","author":"SJ Gershman","year":"2012","journal-title":"Journal of Mathematical Psychology"},{"issue":"476","key":"pcbi.1008317.ref027","doi-asserted-by":"crossref","first-page":"1566","DOI":"10.1198\/016214506000000302","article-title":"Hierarchical Dirichlet Processes","volume":"101","author":"YW Teh","year":"2006","journal-title":"Journal of the American Statistical Association"},{"issue":"3","key":"pcbi.1008317.ref028","doi-asserted-by":"crossref","first-page":"509","DOI":"10.1093\/cercor\/bhr114","article-title":"Mechanisms of hierarchical reinforcement learning in corticostriatal circuits 1: computational analysis","volume":"22","author":"MJ Frank","year":"2012","journal-title":"Cerebral cortex"},{"key":"pcbi.1008317.ref029","doi-asserted-by":"crossref","unstructured":"Zhang J, Springenberg JT, Boedecker J, Burgard W. Deep reinforcement learning with successor features for navigation across similar environments. In: 2017 IEEE\/RSJ International Conference on Intelligent Robots and Systems (IROS). IEEE; 2017. p. 2371\u20132378.","DOI":"10.1109\/IROS.2017.8206049"},{"key":"pcbi.1008317.ref030","unstructured":"Kulkarni TD, Saeedi A, Gautam S, Gershman SJ. Deep successor reinforcement learning. arXiv preprint arXiv:160602396. 2016;."},{"key":"pcbi.1008317.ref031","author":"TJ Madarasz","year":"2019","journal-title":"Better Transfer Learning Through Inferred Successor Maps"},{"issue":"3","key":"pcbi.1008317.ref032","doi-asserted-by":"crossref","first-page":"784","DOI":"10.1037\/0033-295X.114.3.784","article-title":"Reconciling reinforcement learning models with behavioral extinction and renewal: implications for addiction, relapse, and problem gambling","volume":"114","author":"AD Redish","year":"2007","journal-title":"Psychological review"},{"issue":"2","key":"pcbi.1008317.ref033","doi-asserted-by":"crossref","first-page":"267","DOI":"10.1016\/j.neuron.2013.11.005","article-title":"Orbitofrontal cortex as a cognitive map of task space","volume":"81","author":"RC Wilson","year":"2014","journal-title":"Neuron"},{"issue":"6","key":"pcbi.1008317.ref034","doi-asserted-by":"crossref","first-page":"1402","DOI":"10.1016\/j.neuron.2016.08.019","article-title":"Human orbitofrontal cortex represents a cognitive map of state space","volume":"91","author":"NW Schuck","year":"2016","journal-title":"Neuron"},{"issue":"5","key":"pcbi.1008317.ref035","doi-asserted-by":"crossref","first-page":"1936","DOI":"10.1523\/JNEUROSCI.16-05-01936.1996","article-title":"A framework for mesencephalic dopamine systems based on predictive Hebbian learning","volume":"16","author":"PR Montague","year":"1996","journal-title":"Journal of neuroscience"},{"issue":"3","key":"pcbi.1008317.ref036","doi-asserted-by":"crossref","first-page":"337","DOI":"10.1037\/a0037015","article-title":"Opponent actor learning (OpAL): Modeling interactive effects of striatal dopamine on reinforcement learning and choice incentive","volume":"121","author":"AG Collins","year":"2014","journal-title":"Psychological review"},{"issue":"5","key":"pcbi.1008317.ref037","doi-asserted-by":"crossref","first-page":"735","DOI":"10.1038\/nn.4538","article-title":"Dopamine transients are sufficient and necessary for acquisition of model-based associations","volume":"20","author":"MJ Sharpe","year":"2017","journal-title":"Nature Neuroscience"},{"key":"pcbi.1008317.ref038","doi-asserted-by":"crossref","first-page":"244","DOI":"10.3389\/fpsyg.2017.00244","article-title":"The dopamine prediction error: contributions to associative models of reward learning","volume":"8","author":"HM Nasser","year":"2017","journal-title":"Frontiers in psychology"},{"issue":"11","key":"pcbi.1008317.ref039","doi-asserted-by":"crossref","first-page":"1609","DOI":"10.1038\/s41593-018-0232-z","article-title":"Prioritized memory access explains planning and hippocampal replay","volume":"21","author":"MG Mattar","year":"2018","journal-title":"Nature neuroscience"},{"issue":"3","key":"pcbi.1008317.ref040","doi-asserted-by":"crossref","first-page":"640","DOI":"10.1016\/j.cell.2019.06.012","article-title":"Human replay spontaneously reorganizes experience","volume":"178","author":"Y Liu","year":"2019","journal-title":"Cell"},{"issue":"6447","key":"pcbi.1008317.ref041","doi-asserted-by":"crossref","first-page":"eaaw5181","DOI":"10.1126\/science.aaw5181","article-title":"Sequential replay of nonspatial task states in the human hippocampus","volume":"364","author":"NW Schuck","year":"2019","journal-title":"Science"},{"issue":"5","key":"pcbi.1008317.ref042","doi-asserted-by":"crossref","first-page":"1124","DOI":"10.1016\/j.neuron.2016.07.047","article-title":"Reverse replay of hippocampal place cells is uniquely modulated by changing reward","volume":"91","author":"RE Ambrose","year":"2016","journal-title":"Neuron"},{"issue":"3","key":"pcbi.1008317.ref043","doi-asserted-by":"crossref","first-page":"253","DOI":"10.1016\/j.nlm.2004.06.004","article-title":"Hippocampus, cortex, and basal ganglia: Insights from computational models of complementary learning systems","volume":"82","author":"HE Atallah","year":"2004","journal-title":"Neurobiology of learning and memory"},{"issue":"1","key":"pcbi.1008317.ref044","doi-asserted-by":"crossref","first-page":"3","DOI":"10.1002\/hipo.22523","article-title":"Statistical learning of temporal community structure in the hippocampus","volume":"26","author":"AC Schapiro","year":"2016","journal-title":"Hippocampus"},{"key":"pcbi.1008317.ref045","doi-asserted-by":"crossref","first-page":"e17086","DOI":"10.7554\/eLife.17086","article-title":"A map of abstract relational knowledge in the human hippocampal\u2013entorhinal cortex","volume":"6","author":"MM Garvert","year":"2017","journal-title":"Elife"},{"issue":"9","key":"pcbi.1008317.ref046","doi-asserted-by":"crossref","first-page":"1450","DOI":"10.1038\/s41593-019-0464-6","article-title":"Reward revaluation biases hippocampal replay content away from the preferred outcome","volume":"22","author":"AA Carey","year":"2019","journal-title":"Nature Neuroscience"},{"key":"pcbi.1008317.ref047","first-page":"8484","volume-title":"Advances in Neural Information Processing Systems 31","author":"J Whittington","year":"2018"},{"key":"pcbi.1008317.ref048","first-page":"770495","article-title":"The Tolman-Eichenbaum Machine: Unifying space and relational memory through generalisation in the hippocampal formation","author":"JC Whittington","year":"2019","journal-title":"bioRxiv"},{"issue":"2","key":"pcbi.1008317.ref049","doi-asserted-by":"crossref","first-page":"490","DOI":"10.1016\/j.neuron.2018.10.002","article-title":"What is a cognitive map? Organizing knowledge for flexible behavior","volume":"100","author":"TE Behrens","year":"2018","journal-title":"Neuron"},{"key":"pcbi.1008317.ref050","unstructured":"Borsa D, Barreto A, Quan J, Mankowitz D, Munos R, van Hasselt H, et al. Universal successor features approximators. arXiv preprint arXiv:181207626. 2018;."},{"issue":"1","key":"pcbi.1008317.ref051","doi-asserted-by":"crossref","first-page":"163","DOI":"10.1016\/S0004-3702(02)00376-4","article-title":"Equivalence notions and model minimization in Markov decision processes","volume":"147","author":"R Givan","year":"2003","journal-title":"Artificial Intelligence"},{"key":"pcbi.1008317.ref052","doi-asserted-by":"crossref","unstructured":"Fran\u00e7ois-Lavet V, Bengio Y, Precup D, Pineau J. Combined Reinforcement Learning via Abstract Representations. arXiv preprint arXiv:180904506. 2018;.","DOI":"10.1609\/aaai.v33i01.33013582"},{"key":"pcbi.1008317.ref053","first-page":"2899","article-title":"Basis refinement strategies for linear value function approximation in MDPs","author":"G Comanici","year":"2015","journal-title":"Advances in Neural Information Processing Systems"},{"key":"pcbi.1008317.ref054","unstructured":"Ferns N, Precup D. Bisimulation Metrics are Optimal Value Functions. In: UAI. Citeseer; 2014. p. 210\u2013219."},{"issue":"6","key":"pcbi.1008317.ref055","doi-asserted-by":"crossref","first-page":"1662","DOI":"10.1137\/10080484X","article-title":"Bisimulation metrics for continuous Markov decision processes","volume":"40","author":"N Ferns","year":"2011","journal-title":"SIAM Journal on Computing"},{"key":"pcbi.1008317.ref056","doi-asserted-by":"crossref","first-page":"362","DOI":"10.1016\/B978-1-55860-377-6.50052-9","volume-title":"Machine Learning Proceedings 1995","author":"ML Littman","year":"1995"}],"updated-by":[{"DOI":"10.1371\/journal.pcbi.1008317","type":"new_version","label":"New version","source":"publisher","updated":{"date-parts":[[2020,10,27]],"date-time":"2020-10-27T00:00:00Z","timestamp":1603756800000}}],"container-title":["PLOS Computational Biology"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/dx.plos.org\/10.1371\/journal.pcbi.1008317","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2020,10,27]],"date-time":"2020-10-27T14:06:15Z","timestamp":1603807575000},"score":1,"resource":{"primary":{"URL":"https:\/\/dx.plos.org\/10.1371\/journal.pcbi.1008317"}},"subtitle":[],"editor":[{"given":"Samuel J.","family":"Gershman","sequence":"first","affiliation":[]}],"short-title":[],"issued":{"date-parts":[[2020,10,15]]},"references-count":56,"journal-issue":{"issue":"10","published-online":{"date-parts":[[2020,10,15]]}},"URL":"https:\/\/doi.org\/10.1371\/journal.pcbi.1008317","relation":{"has-preprint":[{"id-type":"doi","id":"10.1101\/653493","asserted-by":"object"}]},"ISSN":["1553-7358"],"issn-type":[{"value":"1553-7358","type":"electronic"}],"subject":[],"published":{"date-parts":[[2020,10,15]]}}}