{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,3,22]],"date-time":"2026-03-22T13:29:14Z","timestamp":1774186154003,"version":"3.50.1"},"reference-count":93,"publisher":"Springer Science and Business Media LLC","issue":"4","license":[{"start":{"date-parts":[[2018,3,23]],"date-time":"2018-03-23T00:00:00Z","timestamp":1521763200000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0"}],"funder":[{"DOI":"10.13039\/100004440","name":"Wellcome Trust","doi-asserted-by":"publisher","award":["088130\/Z\/09\/Z"],"award-info":[{"award-number":["088130\/Z\/09\/Z"]}],"id":[{"id":"10.13039\/100004440","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Biol Cybern"],"published-print":{"date-parts":[[2018,8]]},"DOI":"10.1007\/s00422-018-0753-2","type":"journal-article","created":{"date-parts":[[2018,3,23]],"date-time":"2018-03-23T06:25:39Z","timestamp":1521786339000},"page":"323-343","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":163,"title":["Planning and navigation as active inference"],"prefix":"10.1007","volume":"112","author":[{"given":"Raphael","family":"Kaplan","sequence":"first","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0001-7984-8909","authenticated-orcid":false,"given":"Karl J.","family":"Friston","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2018,3,23]]},"reference":[{"key":"753_CR1","unstructured":"Attias H (2003) Planning by probabilistic inference. Proc. of the 9th Int. Workshop on Artificial Intelligence and Statistics"},{"key":"753_CR2","first-page":"217","volume-title":"Sensory communication","author":"H Barlow","year":"1961","unstructured":"Barlow H (1961) Possible principles underlying the transformations of sensory messages. In: Rosenblith W (ed) Sensory communication. MIT Press, Cambridge, pp 217\u2013234"},{"key":"753_CR3","doi-asserted-by":"publisher","first-page":"716","DOI":"10.1073\/pnas.38.8.716","volume":"38","author":"R Bellman","year":"1952","unstructured":"Bellman R (1952) On the theory of dynamic programming. Proc Natl Acad Sci USA 38:716\u2013719","journal-title":"Proc Natl Acad Sci USA"},{"key":"753_CR4","doi-asserted-by":"publisher","first-page":"309","DOI":"10.1016\/S0165-0173(98)00019-8","volume":"28","author":"KC Berridge","year":"1998","unstructured":"Berridge KC, Robinson TE (1998) What is the role of dopamine in reward: hedonic impact, reward learning, or incentive salience? Brain Res Rev 28:309\u2013369","journal-title":"Brain Res Rev"},{"key":"753_CR5","doi-asserted-by":"publisher","first-page":"485","DOI":"10.1016\/j.tics.2012.08.006","volume":"16","author":"M Botvinick","year":"2012","unstructured":"Botvinick M, Toussaint M (2012) Planning as inference. Trends Cogn Sci 16:485\u2013488","journal-title":"Trends Cogn Sci"},{"key":"753_CR6","doi-asserted-by":"publisher","first-page":"262","DOI":"10.1016\/j.cognition.2008.08.011","volume":"113","author":"MM Botvinick","year":"2009","unstructured":"Botvinick MM, Niv Y, Barto AC (2009) Hierarchically organized behavior and its neural foundations: a reinforcement learning perspective. Cognition 113:262\u2013280","journal-title":"Cognition"},{"key":"753_CR7","doi-asserted-by":"crossref","unstructured":"Braun DA, Ortega PA, Theodorou E, Schaal S (2011) Path integral control and bounded rationality. In: 2011 IEEE symposium on adaptive dynamic programming and reinforcement learning (ADPRL), pp 202\u2013209. IEEE, Paris","DOI":"10.1109\/ADPRL.2011.5967366"},{"key":"753_CR8","doi-asserted-by":"publisher","first-page":"369","DOI":"10.1016\/j.neuron.2006.06.021","volume":"51","author":"N Bunzeck","year":"2006","unstructured":"Bunzeck N, Duzel E (2006) Absolute coding of stimulus novelty in the human substantia nigra\/VTA. Neuron 51:369\u2013379","journal-title":"Neuron"},{"key":"753_CR9","doi-asserted-by":"publisher","first-page":"301","DOI":"10.1007\/s004220000172","volume":"83","author":"N Burgess","year":"2000","unstructured":"Burgess N, Jackson A, Hartley T, O\u2019Keefe J (2000) Predictions derived from modelling the hippocampal role in navigation. Biol Cybern 83:301\u2013312","journal-title":"Biol Cybern"},{"key":"753_CR10","doi-asserted-by":"publisher","first-page":"625","DOI":"10.1016\/S0896-6273(02)00830-9","volume":"35","author":"N Burgess","year":"2002","unstructured":"Burgess N, Maguire EA, O\u2019Keefe J (2002) The human hippocampus and spatial and episodic memory. Neuron 35:625\u2013641","journal-title":"Neuron"},{"key":"753_CR11","doi-asserted-by":"publisher","first-page":"507","DOI":"10.1016\/j.neuron.2015.07.006","volume":"87","author":"D Bush","year":"2015","unstructured":"Bush D, Barry C, Manson D, Burgess N (2015) Using grid cells for navigation. Neuron 87:507\u2013520","journal-title":"Neuron"},{"key":"753_CR12","doi-asserted-by":"publisher","first-page":"130","DOI":"10.1038\/nn.3304","volume":"16","author":"G Buzsaki","year":"2013","unstructured":"Buzsaki G, Moser EI (2013) Memory, navigation and theta rhythm in the hippocampal\u2013entorhinal system. Nat Neurosci 16:130\u2013138","journal-title":"Nat Neurosci"},{"key":"753_CR13","doi-asserted-by":"publisher","first-page":"353","DOI":"10.1038\/nature08573","volume":"462","author":"LL Colgin","year":"2009","unstructured":"Colgin LL, Denninger T, Fyhn M, Hafting T, Bonnevie T, Jensen O, Moser MB, Moser EI (2009) Frequency of gamma oscillations routes flow of information in the hippocampus. Nature 462:353\u2013357","journal-title":"Nature"},{"key":"753_CR14","doi-asserted-by":"publisher","first-page":"160","DOI":"10.1016\/j.cognition.2016.04.002","volume":"152","author":"AGE Collins","year":"2016","unstructured":"Collins AGE, Frank MJ (2016) Neural signature of hierarchically structured expectations predicts clustering and transfer of rule sets in reinforcement learning. Cognition 152:160\u2013169","journal-title":"Cognition"},{"key":"753_CR15","doi-asserted-by":"publisher","first-page":"1264","DOI":"10.1126\/science.1150605","volume":"319","author":"K D\u2019Ardenne","year":"2008","unstructured":"D\u2019Ardenne K, McClure SM, Nystrom LE, Cohen JD (2008) BOLD responses reflecting dopaminergic signals in the human ventral tegmental area. Science 319:1264\u20131267","journal-title":"Science"},{"key":"753_CR16","doi-asserted-by":"publisher","first-page":"1704","DOI":"10.1038\/nn1560","volume":"8","author":"ND Daw","year":"2005","unstructured":"Daw ND, Niv Y, Dayan P (2005) Uncertainty-based competition between prefrontal and dorsolateral striatal systems for behavioral control. Nat Neurosci 8:1704\u20131711","journal-title":"Nat Neurosci"},{"key":"753_CR17","doi-asserted-by":"publisher","first-page":"1637","DOI":"10.1162\/neco.2006.18.7.1637","volume":"18","author":"ND Daw","year":"2006","unstructured":"Daw ND, Courville AC, Touretzky DS (2006) Representation and timing in theories of the dopamine system. Neural Comput 18:1637\u20131677","journal-title":"Neural Comput"},{"key":"753_CR18","doi-asserted-by":"publisher","first-page":"1204","DOI":"10.1016\/j.neuron.2011.02.027","volume":"69","author":"ND Daw","year":"2011","unstructured":"Daw ND, Gershman SJ, Seymour B, Dayan P, Dolan RJ (2011) Model-based influences on humans\u2019 choices and striatal prediction errors. Neuron 69:1204\u20131215","journal-title":"Neuron"},{"key":"753_CR19","doi-asserted-by":"publisher","first-page":"613","DOI":"10.1162\/neco.1993.5.4.613","volume":"5","author":"P Dayan","year":"1993","unstructured":"Dayan P (1993) Improving generalization for temporal difference learning: the successor representation. Neural Comput 5:613\u2013624","journal-title":"Neural Comput"},{"key":"753_CR20","doi-asserted-by":"publisher","first-page":"269","DOI":"10.1007\/BF01386390","volume":"1","author":"EW Dijkstra","year":"1959","unstructured":"Dijkstra EW (1959) A note on two problems in connexion with graphs. Numer Math 1:269\u2013271","journal-title":"Numer Math"},{"key":"753_CR21","doi-asserted-by":"publisher","first-page":"e1004864","DOI":"10.1371\/journal.pcbi.1004864","volume":"12","author":"F Donnarumma","year":"2016","unstructured":"Donnarumma F, Maisto D, Pezzulo G (2016) Problem solving as probabilistic inference with subgoaling: explaining human successes and pitfalls in the tower of hanoi. PLoS Comput Biol 12:e1004864","journal-title":"PLoS Comput Biol"},{"key":"753_CR22","doi-asserted-by":"publisher","first-page":"145","DOI":"10.1016\/j.neuron.2006.02.023","volume":"50","author":"G Dragoi","year":"2006","unstructured":"Dragoi G, Buzsaki G (2006) Temporal encoding of place sequences by hippocampal cell assemblies. Neuron 50:145\u2013157","journal-title":"Neuron"},{"key":"753_CR23","volume-title":"Optimal learning: computational procedure for bayes-adaptive Markov decision processes","author":"M Duff","year":"2002","unstructured":"Duff M (2002) Optimal learning: computational procedure for bayes-adaptive Markov decision processes. University of Massachusetts, Amherst"},{"key":"753_CR24","doi-asserted-by":"publisher","first-page":"732","DOI":"10.1038\/nrn3827","volume":"15","author":"H Eichenbaum","year":"2014","unstructured":"Eichenbaum H (2014) Time cells in the hippocampus: a new dimension for mapping memories. Nat Rev Neurosci 15:732\u2013744","journal-title":"Nat Rev Neurosci"},{"key":"753_CR25","doi-asserted-by":"publisher","first-page":"916","DOI":"10.1111\/j.1460-9568.2012.08015.x","volume":"35","author":"UM Erdem","year":"2012","unstructured":"Erdem UM, Hasselmo M (2012) A goal-directed spatial navigation model using forward trajectory planning based on grid cells. Eur J Neurosci 35:916\u2013931","journal-title":"Eur J Neurosci"},{"key":"753_CR26","doi-asserted-by":"publisher","first-page":"136","DOI":"10.3389\/fncom.2015.00136","volume":"9","author":"TH FitzGerald","year":"2015","unstructured":"FitzGerald TH, Dolan RJ, Friston K (2015) Dopamine, reward learning, and active inference. Front Comput Neurosci 9:136","journal-title":"Front Comput Neurosci"},{"key":"753_CR27","doi-asserted-by":"publisher","first-page":"e1004592","DOI":"10.1371\/journal.pcbi.1004592","volume":"11","author":"J Fonollosa","year":"2015","unstructured":"Fonollosa J, Neftci E, Rabinovich M (2015) Learning of chunking sequences in cognition and behavior. PLoS Comput Biol 11:e1004592","journal-title":"PLoS Comput Biol"},{"issue":"7","key":"753_CR28","doi-asserted-by":"publisher","first-page":"500","DOI":"10.1016\/j.tics.2016.05.001","volume":"20","author":"K Friston","year":"2016","unstructured":"Friston K, Buzsaki G (2016) The functional anatomy of time: what and when in the brain. Trends Cogn Sci 20(7):500\u2013511","journal-title":"Trends Cogn Sci"},{"issue":"1655","key":"753_CR29","doi-asserted-by":"publisher","first-page":"20130481","DOI":"10.1098\/rstb.2013.0481","volume":"369","author":"K Friston","year":"2014","unstructured":"Friston K, Schwartenbeck P, FitzGerald T, Moutoussis M, Behrens T, Dolan RJ (2014) The anatomy of choice: dopamine and decision-making. Philos Trans R Soc Lond Ser B Biol Sci 369(1655):20130481","journal-title":"Philos Trans R Soc Lond Ser B Biol Sci"},{"issue":"4","key":"753_CR30","first-page":"1","volume":"6","author":"K Friston","year":"2015","unstructured":"Friston K, Rigoli F, Ognibene D, Mathys C, Fitzgerald T, Pezzulo G (2015) Active inference and epistemic value. Cogn Neurosci 6(4):1\u201328","journal-title":"Cogn Neurosci"},{"key":"753_CR31","doi-asserted-by":"publisher","first-page":"862","DOI":"10.1016\/j.neubiorev.2016.06.022","volume":"68","author":"K Friston","year":"2016","unstructured":"Friston K, FitzGerald T, Rigoli F, Schwartenbeck P, O\u2019Doherty J, Pezzulo G (2016) Active inference and learning. Neurosci Biobehav Rev 68:862\u2013879","journal-title":"Neurosci Biobehav Rev"},{"key":"753_CR32","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1162\/NECO_a_00912","volume":"29","author":"K Friston","year":"2017","unstructured":"Friston K, FitzGerald T, Rigoli F, Schwartenbeck P, Pezzulo G (2017a) Active inference: a process theory. Neural Comput 29:1\u201349","journal-title":"Neural Comput"},{"issue":"10","key":"753_CR33","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1162\/NECO_a_00912","volume":"29","author":"KJ Friston","year":"2017","unstructured":"Friston KJ, Lin M, Frith CD, Pezzulo G, Hobson JA, Ondobaka S (2017b) Active inference, curiosity and insight. Neural Comput 29(10):1\u201351","journal-title":"Neural Comput"},{"key":"753_CR34","doi-asserted-by":"crossref","unstructured":"Gauthier JL, Tank DW (2017) Context-invariant encoding of reward location in a distinct hippocampal population. bioRxiv","DOI":"10.1101\/207043"},{"key":"753_CR35","doi-asserted-by":"publisher","first-page":"7","DOI":"10.1016\/j.cobeha.2017.05.025","volume":"17","author":"SJ Gershman","year":"2017","unstructured":"Gershman SJ (2017) Predicting the past, remembering the future. Curr Opin Behav Sci 17:7\u201313","journal-title":"Curr Opin Behav Sci"},{"key":"753_CR36","doi-asserted-by":"publisher","first-page":"101","DOI":"10.1146\/annurev-psych-122414-033625","volume":"68","author":"SJ Gershman","year":"2017","unstructured":"Gershman SJ, Daw ND (2017) Reinforcement learning and episodic memory in humans and animals: an integrative framework. Ann Rev Psychol 68:101\u2013128","journal-title":"Ann Rev Psychol"},{"key":"753_CR37","doi-asserted-by":"publisher","first-page":"85","DOI":"10.3389\/fnins.2011.00085","volume":"5","author":"C Guger","year":"2011","unstructured":"Guger C, Gener T, Pennartz CM, Brotons-Mas JR, Edlinger G, Bermudez IBS, Verschure P, Schaffelhofer S, Sanchez-Vives MV (2011) Real-time position reconstruction with hippocampal place cells. Front Neurosci 5:85","journal-title":"Front Neurosci"},{"key":"753_CR38","doi-asserted-by":"publisher","first-page":"801","DOI":"10.1038\/nature03721","volume":"436","author":"T Hafting","year":"2005","unstructured":"Hafting T, Fyhn M, Molden S, Moser MB, Moser EI (2005) Microstructure of a spatial map in the entorhinal cortex. Nature 436:801\u2013806","journal-title":"Nature"},{"key":"753_CR39","doi-asserted-by":"publisher","first-page":"369","DOI":"10.1002\/1098-1063(2000)10:4<369::AID-HIPO3>3.0.CO;2-0","volume":"10","author":"T Hartley","year":"2000","unstructured":"Hartley T, Burgess N, Lever C, Cacucci F, O\u2019Keefe J (2000) Modeling place fields in terms of the cortical inputs to the hippocampus. Hippocampus 10:369\u2013379","journal-title":"Hippocampus"},{"key":"753_CR40","doi-asserted-by":"publisher","first-page":"1263","DOI":"10.1098\/rstb.2008.0296","volume":"364","author":"D Hassabis","year":"2009","unstructured":"Hassabis D, Maguire EA (2009) The construction system of the brain. Philos Trans R Soc Lond Ser B Biol Sci 364:1263\u20131271","journal-title":"Philos Trans R Soc Lond Ser B Biol Sci"},{"key":"753_CR41","doi-asserted-by":"publisher","first-page":"33","DOI":"10.1613\/jair.678","volume":"13","author":"M Hauskrecht","year":"2000","unstructured":"Hauskrecht M (2000) Value-function approximations for partially observable Markov decision processes. J Artif Intell Res 13:33\u201394","journal-title":"J Artif Intell Res"},{"key":"753_CR42","doi-asserted-by":"publisher","first-page":"259","DOI":"10.1111\/nous.12062","volume":"50","author":"J Hohwy","year":"2016","unstructured":"Hohwy J (2016) The self-evidencing brain. No\u00fbs 50:259\u2013285","journal-title":"No\u00fbs"},{"key":"753_CR43","first-page":"1109","volume":"29","author":"Ra Houthooft","year":"2016","unstructured":"Houthooft Ra, Xi Chen X, Duan Y, Schulman J, De Turck F, Abbeel P (2016) VIME: variational information maximizing exploration. Adv Neural Inf Process Syst 29:1109\u20131117","journal-title":"Adv Neural Inf Process Syst"},{"key":"753_CR44","doi-asserted-by":"publisher","first-page":"22","DOI":"10.1109\/TSSC.1966.300074","volume":"2","author":"R Howard","year":"1966","unstructured":"Howard R (1966) Information value theory. IEEE Trans Syst Sci Cybern SSC 2:22\u201326","journal-title":"IEEE Trans Syst Sci Cybern SSC"},{"key":"753_CR45","doi-asserted-by":"publisher","first-page":"385","DOI":"10.1016\/j.pneurobio.2009.11.003","volume":"90","author":"MD Humphries","year":"2010","unstructured":"Humphries MD, Prescott TJ (2010) The ventral basal ganglia, a selection mechanism at the crossroads of space, strategy, and reward. Prog Neurobiol 90:385\u2013417","journal-title":"Prog Neurobiol"},{"key":"753_CR46","doi-asserted-by":"publisher","first-page":"e1002410","DOI":"10.1371\/journal.pcbi.1002410","volume":"8","author":"QJ Huys","year":"2012","unstructured":"Huys QJ, Eshel N, O\u2019Nions E, Sheridan L, Dayan P, Roiser JP (2012) Bonsai trees in your head: how the pavlovian system sculpts goal-directed choices by pruning decision trees. PLoS Comput Biol 8:e1002410","journal-title":"PLoS Comput Biol"},{"key":"753_CR47","doi-asserted-by":"publisher","first-page":"1295","DOI":"10.1016\/j.visres.2008.09.007","volume":"49","author":"L Itti","year":"2009","unstructured":"Itti L, Baldi P (2009) Bayesian surprise attracts human attention. Vis Res 49:1295\u20131306","journal-title":"Vis Res"},{"key":"753_CR48","doi-asserted-by":"publisher","first-page":"246","DOI":"10.1038\/nature10439","volume":"478","author":"K Jezek","year":"2011","unstructured":"Jezek K, Henriksen E, Treves A, Moser E, Moser M-B (2011) Theta-paced flickering between place-cell maps in the hippocampus. Nature 478:246\u2013249","journal-title":"Nature"},{"key":"753_CR49","doi-asserted-by":"publisher","first-page":"692","DOI":"10.1016\/j.conb.2008.01.003","volume":"17","author":"A Johnson","year":"2007","unstructured":"Johnson A, van der Meer MAA, Redish AD (2007) Integrating hippocampus and striatum in decision-making. Curr Opin Neurobiol 17:692\u2013697","journal-title":"Curr Opin Neurobiol"},{"key":"753_CR50","doi-asserted-by":"publisher","first-page":"e1002588","DOI":"10.1371\/journal.pbio.1002588","volume":"15","author":"R Kaplan","year":"2017","unstructured":"Kaplan R, King J, Koster R, Penny WD, Burgess N, Friston KJ (2017a) The neural representation of prospective choice during spatial planning and decisions. PLoS Biol 15:e1002588","journal-title":"PLoS Biol"},{"key":"753_CR51","doi-asserted-by":"publisher","first-page":"256","DOI":"10.1016\/j.tins.2017.03.002","volume":"40","author":"R Kaplan","year":"2017","unstructured":"Kaplan R, Schuck NW, Doeller CF (2017b) The role of mental maps in decision-making. Trends Neurosci 40:256\u2013259","journal-title":"Trends Neurosci"},{"key":"753_CR52","doi-asserted-by":"publisher","first-page":"159","DOI":"10.1007\/s10994-012-5278-7","volume":"87","author":"HJ Kappen","year":"2012","unstructured":"Kappen HJ, Gomez Y, Opper M (2012) Optimal control as a graphical model inference problem. Mach Learn 87:159\u2013182","journal-title":"Mach Learn"},{"key":"753_CR53","doi-asserted-by":"publisher","first-page":"846","DOI":"10.1177\/0278364911406761","volume":"30","author":"S Karaman","year":"2011","unstructured":"Karaman S, Frazzoli E (2011) Sampling-based algorithms for optimal motion planning. Int J Robot Res 30:846\u2013894","journal-title":"Int J Robot Res"},{"key":"753_CR54","doi-asserted-by":"publisher","first-page":"e1002055","DOI":"10.1371\/journal.pcbi.1002055","volume":"7","author":"M Keramati","year":"2011","unstructured":"Keramati M, Dezfouli A, Piray P (2011) Speed\/accuracy trade-off between the habitual and the goal-directed processes. PLoS Comput Biol 7:e1002055","journal-title":"PLoS Comput Biol"},{"issue":"45","key":"753_CR55","doi-asserted-by":"publisher","first-page":"12868","DOI":"10.1073\/pnas.1609094113","volume":"113","author":"M Keramati","year":"2016","unstructured":"Keramati M, Smittenaar P, Dolan RJ, Dayan P (2016) Adaptive integration of habits into depth-limited planning defines a habitual-goal-directed spectrum. Proc Natl Acad Sci USA 113(45):12868\u201312873","journal-title":"Proc Natl Acad Sci USA"},{"key":"753_CR56","doi-asserted-by":"publisher","first-page":"20130369","DOI":"10.1098\/rstb.2013.0369","volume":"369","author":"JJ Knierim","year":"2014","unstructured":"Knierim JJ, Neunuebel JP, Deshmukh SS (2014) Functional correlates of the lateral and medial entorhinal cortex: objects, path integration and local-global reference frames. Philos Trans R Soc Lond Ser B Biol Sci 369:20130369","journal-title":"Philos Trans R Soc Lond Ser B Biol Sci"},{"key":"753_CR57","doi-asserted-by":"publisher","first-page":"e1005753","DOI":"10.1371\/journal.pcbi.1005753","volume":"13","author":"JJ Lee","year":"2017","unstructured":"Lee JJ, Keramati M (2017) Flexibility to contingency changes distinguishes habitual and goal-directed strategies in humans. PLoS Comput Biol 13:e1005753","journal-title":"PLoS Comput Biol"},{"key":"753_CR58","doi-asserted-by":"publisher","first-page":"257","DOI":"10.1146\/annurev.ne.13.030190.001353","volume":"13","author":"R Linsker","year":"1990","unstructured":"Linsker R (1990) Perceptual neural organization: some approaches based on network models and information theory. Annu Rev Neurosci 13:257\u2013281","journal-title":"Annu Rev Neurosci"},{"key":"753_CR59","doi-asserted-by":"publisher","first-page":"1193","DOI":"10.1098\/rstb.2008.0316","volume":"364","author":"J Lisman","year":"2009","unstructured":"Lisman J, Redish AD (2009) Prediction, sequences and the hippocampus. Philos Trans R Soc Lond Ser B Biol Sci 364:1193\u20131201","journal-title":"Philos Trans R Soc Lond Ser B Biol Sci"},{"key":"753_CR60","doi-asserted-by":"publisher","first-page":"20141335","DOI":"10.1098\/rsif.2014.1335","volume":"12","author":"D Maisto","year":"2015","unstructured":"Maisto D, Donnarumma F, Pezzulo G (2015) Divide et impera: subgoaling reduces the complexity of probabilistic inference and problem solving. J R Soc Interface 12:20141335","journal-title":"J R Soc Interface"},{"key":"753_CR61","doi-asserted-by":"publisher","first-page":"e1002045","DOI":"10.1371\/journal.pcbi.1002045","volume":"7","author":"LE Martinet","year":"2011","unstructured":"Martinet LE, Sheynikhovich D, Benchenane K, Arleo A (2011) Spatial learning and action planning in a prefrontal cortical network model. PLoS Comput Biol 7:e1002045","journal-title":"PLoS Comput Biol"},{"key":"753_CR62","first-page":"4511","volume":"29","author":"D McNamee","year":"2016","unstructured":"McNamee D, Wolpert DM, Lengyel M (2016) Efficient state-space modularization for planning: theory, behavioral and neural signatures. Adv Neural Inf Process Syst 29:4511\u20134519","journal-title":"Adv Neural Inf Process Syst"},{"key":"753_CR63","doi-asserted-by":"publisher","first-page":"56","DOI":"10.3389\/fncom.2016.00056","volume":"10","author":"MB Mirza","year":"2016","unstructured":"Mirza MB, Adams RA, Mathys CD, Friston KJ (2016) Scene construction, visual foraging, and active inference. Front Comput Neurosci 10:56","journal-title":"Front Comput Neurosci"},{"key":"753_CR64","doi-asserted-by":"publisher","first-page":"680","DOI":"10.1038\/s41562-017-0180-8","volume":"1","author":"I Momennejad","year":"2017","unstructured":"Momennejad I, Russek EM, Cheong JH, Botvinick MM, Daw ND, Gershman SJ (2017) The successor representation in human reinforcement learning. Nat Hum Behav 1:680\u2013692","journal-title":"Nat Hum Behav"},{"key":"753_CR65","unstructured":"Oliehoek F, Spaan MTJ, Vlassis N (2005) Best-response play in partially observable card games. In: Proceedings of the 14th annual machine learning conference of Belgium and the Netherlands, pp 45\u201350"},{"key":"753_CR66","doi-asserted-by":"publisher","first-page":"132","DOI":"10.1152\/jn.1987.57.1.162","volume":"57","author":"L Optican","year":"1987","unstructured":"Optican L, Richmond BJ (1987) Temporal encoding of two-dimensional patterns by single units in primate inferior cortex. II. Information theoretic analysis. J Neurophysiol 57:132\u2013146","journal-title":"J Neurophysiol"},{"key":"753_CR67","doi-asserted-by":"publisher","first-page":"1322","DOI":"10.1126\/science.1159775","volume":"321","author":"E Pastalkova","year":"2008","unstructured":"Pastalkova E, Itskov V, Amarasingham A, Buzsaki G (2008) Internally generated cell assembly sequences in the rat hippocampus. Science 321:1322\u20131327","journal-title":"Science"},{"key":"753_CR68","doi-asserted-by":"publisher","first-page":"74","DOI":"10.1038\/nature12112","volume":"497","author":"BE Pfeiffer","year":"2013","unstructured":"Pfeiffer BE, Foster DJ (2013) Hippocampal place cell sequences depict future paths to remembered goals. Nature 497:74\u201379","journal-title":"Nature"},{"key":"753_CR69","doi-asserted-by":"publisher","first-page":"146","DOI":"10.3389\/fncom.2010.00146","volume":"4","author":"R Rao","year":"2010","unstructured":"Rao R (2010) Decision making under uncertainty: a neural model based on partially observable Markov decision processes. Front Comput Neurosci 4:146","journal-title":"Front Comput Neurosci"},{"key":"753_CR70","doi-asserted-by":"publisher","first-page":"370","DOI":"10.1016\/j.neuron.2011.05.042","volume":"71","author":"JJ Ribas-Fernandes","year":"2011","unstructured":"Ribas-Fernandes JJ, Solway A, Diuk C, McGuire JT, Barto AG, Niv Y, Botvinick MM (2011) A neural signature of hierarchical reinforcement learning. Neuron 71:370\u2013379","journal-title":"Neuron"},{"key":"753_CR71","doi-asserted-by":"publisher","first-page":"e1005768","DOI":"10.1371\/journal.pcbi.1005768","volume":"13","author":"EM Russek","year":"2017","unstructured":"Russek EM, Momennejad I, Botvinick MM, Gershman SJ, Daw ND (2017) Predictive representations can link model-based reinforcement learning to model-free mechanisms. PLoS Comput Biol 13:e1005768","journal-title":"PLoS Comput Biol"},{"key":"753_CR72","doi-asserted-by":"publisher","first-page":"3801","DOI":"10.1098\/rstb.2008.0152","volume":"363","author":"W Schultz","year":"2008","unstructured":"Schultz W, Preuschoff K, Camerer C, Hsu M, Fiorillo CD, Tobler PN, Bossaerts P (2008) Explicit neural signals reflecting reward uncertainty. Philos Trans R Soc Lond B Biol Sci 363:3801\u20133811","journal-title":"Philos Trans R Soc Lond B Biol Sci"},{"key":"753_CR73","doi-asserted-by":"crossref","unstructured":"Schwartenbeck P, Friston K (2016) Computational phenotyping in psychiatry: a worked example. eNeuro 3","DOI":"10.1523\/ENEURO.0049-16.2016"},{"key":"753_CR74","doi-asserted-by":"publisher","first-page":"3434","DOI":"10.1093\/cercor\/bhu159","volume":"25","author":"P Schwartenbeck","year":"2015","unstructured":"Schwartenbeck P, FitzGerald TH, Mathys C, Dolan R, Friston K (2015) The dopaminergic midbrain encodes the expected certainty about desired outcomes. Cereb Cortex 25:3434\u20133445","journal-title":"Cereb Cortex"},{"key":"753_CR75","unstructured":"Silver D, Veness J (2010) Monte-Carlo Planning in Large POMDPs. Advances in Neural Information Processing Systems 23 (NIPS 2010)"},{"key":"753_CR76","doi-asserted-by":"publisher","first-page":"1865","DOI":"10.1126\/science.1166466","volume":"322","author":"T Solstad","year":"2008","unstructured":"Solstad T, Boccara CN, Kropff E, Moser MB, Moser EI (2008) Representation of geometric borders in the entorhinal cortex. Science 322:1865\u20131868","journal-title":"Science"},{"key":"753_CR77","doi-asserted-by":"publisher","first-page":"120","DOI":"10.1037\/a0026435","volume":"119","author":"A Solway","year":"2012","unstructured":"Solway A, Botvinick M (2012) Goal-directed decision making as probabilistic inference: a computational framework and potential neural correlates. Psychol Rev 119:120\u2013154","journal-title":"Psychol Rev"},{"key":"753_CR78","doi-asserted-by":"publisher","first-page":"11708","DOI":"10.1073\/pnas.1505483112","volume":"112","author":"A Solway","year":"2015","unstructured":"Solway A, Botvinick MM (2015) Evidence integration in model-based tree search. Proc Natl Acad Sci USA 112:11708\u201311713","journal-title":"Proc Natl Acad Sci USA"},{"key":"753_CR79","doi-asserted-by":"publisher","first-page":"e1003779","DOI":"10.1371\/journal.pcbi.1003779","volume":"10","author":"A Solway","year":"2014","unstructured":"Solway A, Diuk C, Cordova N, Yee D, Barto AG, Niv Y, Botvinick MM (2014) Optimal behavioral hierarchy. PLoS Comput Biol 10:e1003779","journal-title":"PLoS Comput Biol"},{"key":"753_CR80","unstructured":"Stachenfeld KL, Botvinick MM, Gershman SJ (2014) Design principles of the hippocampal cognitive map. In: Proceedings of the 27th international conference on neural information processing systems, vol 2, pp 2528\u20132536. MIT Press, Montreal, Canada"},{"key":"753_CR81","doi-asserted-by":"publisher","first-page":"1643","DOI":"10.1038\/nn.4650","volume":"20","author":"KL Stachenfeld","year":"2017","unstructured":"Stachenfeld KL, Botvinick MM, Gershman SJ (2017) The hippocampus as a predictive map. Nat Neurosci 20:1643\u20131653","journal-title":"Nat Neurosci"},{"key":"753_CR82","doi-asserted-by":"publisher","first-page":"581","DOI":"10.1038\/nn.4520","volume":"20","author":"CK Starkweather","year":"2017","unstructured":"Starkweather CK, Babayan BM, Uchida N, Gershman SJ (2017) Dopamine reward prediction errors reflect hidden-state inference across time. Nat Neurosci 20:581\u2013589","journal-title":"Nat Neurosci"},{"key":"753_CR83","doi-asserted-by":"crossref","unstructured":"Sun Y, Gomez F, Schmidhuber J (2011a) Planning to be surprised: optimal Bayesian exploration in dynamic environments. In: Proceedings of the 4th international conference on artificial general intelligence, pp 41\u201351. Springer, Mountain View, CA","DOI":"10.1007\/978-3-642-22887-2_5"},{"key":"753_CR84","doi-asserted-by":"crossref","unstructured":"Sun Y, Gomez F, Schmidhuber J (2011b) Planning to be surprised: optimal Bayesian exploration in dynamic environments. In: Schmidhuber J et al (eds) Artificial general intelligence: 4th international conference, AGI 2011, Mountain View, CA, USA, August 3\u20136, 2011 Proceedings, pp 41\u201351. Springer, Berlin","DOI":"10.1007\/978-3-642-22887-2_5"},{"key":"753_CR85","volume-title":"Reinforcement learning: an introduction","author":"RS Sutton","year":"1998","unstructured":"Sutton RS, Barto AG (1998) Reinforcement learning: an introduction. MIT Press, Cambridge"},{"key":"753_CR86","doi-asserted-by":"publisher","first-page":"181","DOI":"10.1016\/S0004-3702(99)00052-1","volume":"112","author":"RS Sutton","year":"1999","unstructured":"Sutton RS, Precup D, Singh S (1999) Between MDPs and semi-MDPs: a framework for temporal abstraction in reinforcement learning. Artif Intell 112:181\u2013211","journal-title":"Artif Intell"},{"key":"753_CR87","doi-asserted-by":"publisher","first-page":"181","DOI":"10.1146\/annurev.neuro.29.051605.112854","volume":"30","author":"JS Taube","year":"2007","unstructured":"Taube JS (2007) The head direction signal: origins and sensory-motor integration. Annu Rev Neurosci 30:181\u2013207","journal-title":"Annu Rev Neurosci"},{"key":"753_CR88","doi-asserted-by":"crossref","unstructured":"Todorov E (2006) Linearly-solvable Markov decision problems. In: Advances in neural information processing systems, vol 19, pp 1369\u20131376. MIT Press, Boston","DOI":"10.7551\/mitpress\/7503.003.0176"},{"key":"753_CR89","first-page":"1","volume":"6","author":"JL Broek van den","year":"2010","unstructured":"van den Broek JL, Wiegerinck WAJJ, Kappen HJ (2010) Risk-sensitive path integral control. UAI 6:1\u20138","journal-title":"UAI"},{"key":"753_CR90","doi-asserted-by":"crossref","unstructured":"van Dijk SG, Polani D (2011) Grounding subgoals in information transitions. In: 2011 IEEE symposium on adaptive dynamic programming and reinforcement learning (ADPRL), Paris, pp 105\u2013111","DOI":"10.1109\/ADPRL.2011.5967384"},{"key":"753_CR91","doi-asserted-by":"publisher","first-page":"1350016","DOI":"10.1142\/S0219525913500161","volume":"16","author":"SG Dijk Van","year":"2013","unstructured":"Van Dijk SG, Polani D (2013) Informational constraints-driven organization in goal-directed behavior. Adv Complex Syst 16:1350016","journal-title":"Adv Complex Syst"},{"key":"753_CR92","doi-asserted-by":"publisher","first-page":"132","DOI":"10.1109\/TAMD.2010.2050205","volume":"2","author":"CM Vigorito","year":"2010","unstructured":"Vigorito CM, Barto AG (2010) Intrinsically motivated hierarchical skill learning in structured environments. IEEE Trans Auton Ment Dev 2:132\u2013143","journal-title":"IEEE Trans Auton Ment Dev"},{"key":"753_CR93","doi-asserted-by":"publisher","first-page":"220","DOI":"10.1016\/j.visres.2009.11.013","volume":"50","author":"Z Wang","year":"2010","unstructured":"Wang Z, Klein RM (2010) Searching for inhibition of return in visual search: a review. Vis Res 50:220\u2013228","journal-title":"Vis Res"}],"container-title":["Biological Cybernetics"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/link.springer.com\/article\/10.1007\/s00422-018-0753-2\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s00422-018-0753-2.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s00422-018-0753-2.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2023,9,1]],"date-time":"2023-09-01T15:08:34Z","timestamp":1693580914000},"score":1,"resource":{"primary":{"URL":"http:\/\/link.springer.com\/10.1007\/s00422-018-0753-2"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2018,3,23]]},"references-count":93,"journal-issue":{"issue":"4","published-print":{"date-parts":[[2018,8]]}},"alternative-id":["753"],"URL":"https:\/\/doi.org\/10.1007\/s00422-018-0753-2","relation":{"has-preprint":[{"id-type":"doi","id":"10.1101\/230599","asserted-by":"object"}]},"ISSN":["0340-1200","1432-0770"],"issn-type":[{"value":"0340-1200","type":"print"},{"value":"1432-0770","type":"electronic"}],"subject":[],"published":{"date-parts":[[2018,3,23]]},"assertion":[{"value":"17 January 2017","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"7 March 2018","order":2,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"23 March 2018","order":3,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Compliance with Ethical Standards"}},{"value":"The authors have no disclosures or conflict of interest.","order":2,"name":"Ethics","group":{"name":"EthicsHeading","label":"Disclosure statement"}}]}}