{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,9]],"date-time":"2026-07-09T20:06:42Z","timestamp":1783627602320,"version":"3.55.0"},"reference-count":58,"publisher":"Public Library of Science (PLoS)","issue":"8","license":[{"start":{"date-parts":[[2016,8,26]],"date-time":"2016-08-26T00:00:00Z","timestamp":1472169600000},"content-version":"vor","delay-in-days":0,"URL":"http:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"funder":[{"name":"Office of Naval Research (US)","award":["N00014-14-1-0800"],"award-info":[{"award-number":["N00014-14-1-0800"]}]},{"DOI":"10.13039\/100000001","name":"National Science Foundation (US)","doi-asserted-by":"crossref","award":["CCF-1231216"],"award-info":[{"award-number":["CCF-1231216"]}],"id":[{"id":"10.13039\/100000001","id-type":"DOI","asserted-by":"crossref"}]}],"content-domain":{"domain":["www.ploscompbiol.org"],"crossmark-restriction":false},"short-container-title":["PLoS Comput Biol"],"DOI":"10.1371\/journal.pcbi.1005090","type":"journal-article","created":{"date-parts":[[2016,8,26]],"date-time":"2016-08-26T17:29:03Z","timestamp":1472232543000},"page":"e1005090","update-policy":"https:\/\/doi.org\/10.1371\/journal.pcbi.corrections_policy","source":"Crossref","is-referenced-by-count":197,"title":["When Does Model-Based Control Pay Off?"],"prefix":"10.1371","volume":"12","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-4792-7044","authenticated-orcid":true,"given":"Wouter","family":"Kool","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-6929-9982","authenticated-orcid":false,"given":"Fiery A.","family":"Cushman","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Samuel J.","family":"Gershman","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"340","published-online":{"date-parts":[[2016,8,26]]},"reference":[{"key":"ref1","doi-asserted-by":"crossref","first-page":"67","DOI":"10.1098\/rstb.1985.0010","article-title":"Actions and habits: The development of behavioural autonomy","volume":"308","author":"A Dickinson","year":"1985","journal-title":"Philosophical Transactions of the Royal Society B: Biological Sciences"},{"key":"ref2","doi-asserted-by":"crossref","first-page":"3","DOI":"10.1037\/0033-2909.119.1.3","article-title":"The empirical case for two systems of reasoning","volume":"119","author":"SA Sloman","year":"1996","journal-title":"Psychological Bulletin"},{"key":"ref3","doi-asserted-by":"crossref","first-page":"697","DOI":"10.1037\/0003-066X.58.9.697","article-title":"A perspective on judgment and choice: Mapping bounded rationality","volume":"58","author":"D Kahneman","year":"2003","journal-title":"American Psychologist"},{"key":"ref4","doi-asserted-by":"crossref","first-page":"1449","DOI":"10.1257\/aer.96.5.1449","article-title":"A dual self model of impulse control","volume":"96","author":"D Fudenberg","year":"2006","journal-title":"American Economic Review"},{"key":"ref5","doi-asserted-by":"crossref","first-page":"48","DOI":"10.1038\/npp.2009.131","article-title":"Human and rodent homologies in action control: Corticostrialtal determinants of goal-directed and habitual action","volume":"35","author":"BW Balleine","year":"2009","journal-title":"Neuropsychopharmacology"},{"key":"ref6","doi-asserted-by":"crossref","first-page":"312","DOI":"10.1016\/j.neuron.2013.09.007","article-title":"Goals and habits in the brain","volume":"80","author":"RJ Dolan","year":"2013","journal-title":"Neuron"},{"key":"ref7","doi-asserted-by":"crossref","first-page":"1704","DOI":"10.1038\/nn1560","article-title":"Uncertainty-based competition between prefrontal and dorsolateral striatal systems for behavioral control","volume":"8","author":"ND Daw","year":"2005","journal-title":"Nature Neuroscience"},{"key":"ref8","doi-asserted-by":"crossref","first-page":"1204","DOI":"10.1016\/j.neuron.2011.02.027","article-title":"Model-based influences on humans' choices and striatal prediction errors","volume":"69","author":"ND Daw","year":"2011","journal-title":"Neuron"},{"key":"ref9","doi-asserted-by":"crossref","first-page":"e1004648","DOI":"10.1371\/journal.pcbi.1004648","article-title":"Simple Plans or Sophisticated Habits? State, Transition and Learning Interactions in the Two-Step Task","volume":"11","author":"T Akam","year":"2015","journal-title":"PLoS computational biology"},{"key":"ref10","doi-asserted-by":"crossref","first-page":"914","DOI":"10.1016\/j.neuron.2013.08.009","article-title":"Disruption of dorsolateral prefrontal cortex decreases model-based in favor of model-free control in humans","volume":"80","author":"P Smittenaar","year":"2013","journal-title":"Neuron"},{"key":"ref11","first-page":"1","article-title":"Valence-dependent influence of serotonin depletion on model-based choice strategy","author":"Y Worbe","year":"2015","journal-title":"Molecular Psychiatry"},{"key":"ref12","doi-asserted-by":"crossref","first-page":"418","DOI":"10.1016\/j.neuron.2012.03.042","article-title":"Dopamine enhances model-based over model-free choice behavior","volume":"75","author":"K Wunderlich","year":"2012","journal-title":"Neuron"},{"key":"ref13","doi-asserted-by":"crossref","first-page":"1595","DOI":"10.1073\/pnas.1417219112","article-title":"Ventral striatal dopamine reflects behavioral and neural signatures of model-based control during sequential decision making","volume":"112","author":"L Deserno","year":"2015","journal-title":"Proceedings of the National Academy of Sciences"},{"key":"ref14","doi-asserted-by":"crossref","first-page":"1211","DOI":"10.1523\/JNEUROSCI.1901-15.2016","article-title":"Variability in dopamine genes dissociates model-based and model-free reinforcement kearning","volume":"36","author":"BB Doll","year":"2016","journal-title":"Journal of Neuroscience"},{"key":"ref15","doi-asserted-by":"crossref","first-page":"767","DOI":"10.1038\/nn.3981","article-title":"Model-based choices involve prospective neural activity","volume":"18","author":"BB Doll","year":"2015","journal-title":"Nature Neuroscience"},{"key":"ref16","doi-asserted-by":"crossref","first-page":"6188","DOI":"10.1523\/JNEUROSCI.6486-10.2011","article-title":"Dopaminergic genes predict individual differences in susceptibility to confirmation bias","volume":"31","author":"BB Doll","year":"2011","journal-title":"Journal of Neuroscience"},{"key":"ref17","doi-asserted-by":"crossref","first-page":"118","DOI":"10.1016\/j.cortex.2015.11.004","article-title":"Fronto-striatal organization: Defining functional and microstructural substrates of behavioural flexibility","volume":"74","author":"LS Morris","year":"2016","journal-title":"CORTEX"},{"key":"ref18","doi-asserted-by":"crossref","first-page":"e86850","DOI":"10.1371\/journal.pone.0086850","article-title":"Transcranial direct current stimulation of right dorsolateral prefrontal cortex does not affect model-based or model-free reinforcement learning in humans","volume":"9","author":"P Smittenaar","year":"2014","journal-title":"PLoS ONE"},{"key":"ref19","doi-asserted-by":"crossref","first-page":"e1004463","DOI":"10.1371\/journal.pcbi.1004463","article-title":"Model-based reasoning in humans becomes automatic with training","volume":"11","author":"M Economides","year":"2015","journal-title":"PLOS Computational Biology"},{"key":"ref20","doi-asserted-by":"crossref","first-page":"253","DOI":"10.3389\/fnins.2013.00253","article-title":"Of goals and habits: age-related and individual differences in goal-directed decision-making","volume":"7","author":"B Eppinger","year":"2013","journal-title":"Frontiers in Neuroscience"},{"key":"ref21","doi-asserted-by":"crossref","first-page":"523","DOI":"10.3758\/s13415-015-0347-6","article-title":"Model-based learning protects against forming habits","volume":"15","author":"CM Gillan","year":"2015","journal-title":"Cognitive, Affective, & Behavioral Neuroscience"},{"key":"ref22","doi-asserted-by":"crossref","first-page":"751","DOI":"10.1177\/0956797612463080","article-title":"The curse of planning: Dissecting multiple reinforcement-learning systems by taxing the central executive","volume":"24","author":"AR Otto","year":"2013","journal-title":"Psychological Science"},{"key":"ref23","doi-asserted-by":"crossref","first-page":"20941","DOI":"10.1073\/pnas.1312011110","article-title":"Working-memory capacity protects model-based learning from stress","volume":"110","author":"AR Otto","year":"2013","journal-title":"Proceedings of the National Academy of Sciences USA"},{"key":"ref24","doi-asserted-by":"crossref","first-page":"319","DOI":"10.1162\/jocn_a_00709","article-title":"Cognitive control predicts use of model-based reinforcement learning","volume":"27","author":"AR Otto","year":"2015","journal-title":"Journal of Cognitive Neuroscience"},{"key":"ref25","doi-asserted-by":"crossref","first-page":"e1003364","DOI":"10.1371\/journal.pcbi.1003364","article-title":"Actions, action sequences and habits: Evidence that goal-directed and habitual action control are hierarchically organized","volume":"9","author":"A Dezfouli","year":"2013","journal-title":"PLOS Computational Biology"},{"key":"ref26","doi-asserted-by":"crossref","first-page":"20130482","DOI":"10.1098\/rstb.2013.0482","article-title":"Habits as action sequences: hierarchical action control and changes in outcome value","volume":"369","author":"A Dezfouli","year":"2014","journal-title":"Philosophical Transactions of the Royal Society of London Series B, Biological sciences"},{"key":"ref27","doi-asserted-by":"crossref","first-page":"587","DOI":"10.3389\/fnhum.2014.00587","article-title":"Devaluation and sequential decisions: linking goal-directed and model-based behavior","volume":"8","author":"E Friedel","year":"2014","journal-title":"Frontiers in Human Neuroscience"},{"key":"ref28","doi-asserted-by":"crossref","first-page":"268","DOI":"10.1016\/j.psyneuen.2014.12.017","article-title":"The interaction of acute and chronic stress impairs model-based behavioral control","volume":"53","author":"C Radenbach","year":"2015","journal-title":"Psychoneuroendocrinology"},{"key":"ref29","doi-asserted-by":"crossref","first-page":"1450","DOI":"10.3389\/fpsyg.2014.01450","article-title":"Processing speed enhances model-based over model-free reinforcement learning in the presence of high working memory functioning","volume":"5","author":"DJ Schad","year":"2014","journal-title":"Frontiers in Psychology"},{"key":"ref30","first-page":"525","article-title":"Extraversion differentiates between model-based and model-free strategies in a reinforcement learning task","volume":"7","author":"A Skatova","year":"2015","journal-title":"Frontiers in Human Neuroscience"},{"key":"ref31","article-title":"From creatures of habit to goal-directed learners: Tracking the developmental emergence of model-based reinforcement learning","author":"JH Decker","journal-title":"Psychological Science"},{"key":"ref32","doi-asserted-by":"crossref","first-page":"355","DOI":"10.1093\/brain\/awv347","article-title":"Dopamine selectively remediates &model-based& reward learning: a computational approach","volume":"139","author":"ME Sharp","year":"2015","journal-title":"Brain"},{"key":"ref33","doi-asserted-by":"crossref","first-page":"e670","DOI":"10.1038\/tp.2015.165","article-title":"Motivation and value influences in the relative balance of goal-directed and habitual behaviours in obsessive-compulsive disorder","volume":"5","author":"V Voon","year":"2015","journal-title":"Translational Psychiatry"},{"key":"ref34","doi-asserted-by":"crossref","first-page":"345","DOI":"10.1038\/mp.2014.44","article-title":"Disorders of compulsivity: a common bias towards learning habits","volume":"20","author":"V Voon","year":"2014","journal-title":"Molecular Psychiatry"},{"key":"ref35","article-title":"Characterizing a psychiatric symptom dimension related to deficits in goal-directed control","author":"CM Gillan","journal-title":"eLife"},{"key":"ref36","doi-asserted-by":"crossref","first-page":"1593","DOI":"10.1126\/science.275.5306.1593","article-title":"A neural substrate of prediction and reward","volume":"275","author":"W Schultz","year":"1997","journal-title":"Science"},{"key":"ref37","doi-asserted-by":"crossref","DOI":"10.1109\/TNN.1998.712192","article-title":"Reinforcement Learning: An Introduction","author":"RS Sutton","year":"1998"},{"key":"ref38","doi-asserted-by":"crossref","first-page":"92","DOI":"10.3389\/fpsyg.2013.00092","article-title":"The Mixed Instrumental Controller: Using Value of Information to combine habitual choice and mental simulation","volume":"4","author":"G Pezzulo","year":"2013","journal-title":"Frontiers in Psychology"},{"key":"ref39","doi-asserted-by":"crossref","first-page":"e659","DOI":"10.1038\/tp.2015.139","article-title":"Lateral prefrontal model-based signatures are reduced in healthy individuals with high trait impulsivity","volume":"5","author":"L Deserno","year":"2015","journal-title":"Translational Psychiatry"},{"key":"ref40","doi-asserted-by":"crossref","first-page":"585","DOI":"10.1016\/j.neuron.2010.04.016","article-title":"States versus rewards: dissociable neural prediction error signals underlying model-based and model-free reinforcement learning","volume":"66","author":"J Gl\u00e4scher","year":"2010","journal-title":"Neuron"},{"key":"ref41","doi-asserted-by":"crossref","first-page":"182","DOI":"10.1037\/a0030844","article-title":"Retrospective revaluation in sequential decision making: A tale of two systems","volume":"143","author":"SJ Gershman","year":"2014","journal-title":"Journal of Experimental Psychology: General"},{"key":"ref42","article-title":"Habitual control of goal selection in humans","author":"F Cushman","year":"2015","journal-title":"Proceedings of the National Academy of Science"},{"key":"ref43","doi-asserted-by":"crossref","first-page":"665","DOI":"10.1037\/a0020198","article-title":"Decision making and the avoidance of cognitive demand","volume":"139","author":"W Kool","year":"2010","journal-title":"Journal of Experimental Psychology: General"},{"key":"ref44","doi-asserted-by":"crossref","first-page":"661","DOI":"10.1017\/S0140525X12003196","article-title":"An opportunity cost model of subjective effort and task performance","volume":"36","author":"R Kurzban","year":"2013","journal-title":"Behavioral and Brain Sciences"},{"key":"ref45","doi-asserted-by":"crossref","first-page":"e68210","DOI":"10.1371\/journal.pone.0068210","article-title":"What is the subjective cost of cognitive effort? Load, trait, and aging effects revealed by economic preference","volume":"22","author":"A Westbrook","year":"2013","journal-title":"PLOS ONE"},{"key":"ref46","doi-asserted-by":"crossref","first-page":"e1002055","DOI":"10.1371\/journal.pcbi.1002055","article-title":"Speed\/accuracy trade-off between the habitual and the goal-directed processes","volume":"7","author":"M Keramati","year":"2011","journal-title":"PLOS Computational Biology"},{"key":"ref47","doi-asserted-by":"crossref","first-page":"273","DOI":"10.1126\/science.aac6076","article-title":"Computational rationality: A converging paradigm for intelligence in brains, minds, and machines","volume":"349","author":"SJ Gershman","year":"2015","journal-title":"Science"},{"key":"ref48","doi-asserted-by":"crossref","first-page":"217","DOI":"10.1111\/tops.12142","article-title":"Rational use of cognitive resources: Levels of analysis between the computational and the algorithmic","volume":"7","author":"TL Griffiths","year":"2015","journal-title":"Topics in Cognitive Science"},{"key":"ref49","first-page":"534","article-title":"Adaptive strategy selection in decision making","volume":"14","author":"JW Payne","year":"1988","journal-title":"Journal of Experimental Psychology: Learning, Memory, and Cognition"},{"key":"ref50","doi-asserted-by":"crossref","first-page":"207","DOI":"10.1037\/0096-3445.135.2.207","article-title":"SSL: A theory of how people learn to select strategies","volume":"135","author":"J Rieskamp","year":"2006","journal-title":"Journal of Experimental Psychology: General"},{"key":"ref51","doi-asserted-by":"crossref","first-page":"687","DOI":"10.1016\/j.neuron.2013.11.028","article-title":"Neural computations underlying arbitration between model-based and model-free Learning","volume":"81","author":"SW Lee","year":"2014","journal-title":"Neuron"},{"key":"ref52","article-title":"On-line Q-learning using connectionist systems","author":"G Rummery","year":"1994"},{"key":"ref53","unstructured":"Simon DA, Daw ND. Environmental statistics and the trade-off between model-based and TD learning in humans. In: Shawe-Taylor J, Zemel R, Bartlett P, Pereira F, Weinberger K, editors. Advances in Neural Information Processing Systems. 242011. p. 127\u201335."},{"key":"ref54","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1016\/j.jmp.2016.01.006","article-title":"Empirical priors for reinforcement learning models","volume":"71","author":"SJ Gershman","year":"2016","journal-title":"Journal of Mathematical Psychology"},{"key":"ref55","doi-asserted-by":"crossref","first-page":"1214","DOI":"10.1038\/nn1954","article-title":"Learning the value of information in an uncertain world","volume":"10","author":"TEJ Behrens","year":"2007","journal-title":"Nature Neuroscience"},{"key":"ref56","doi-asserted-by":"crossref","first-page":"83","DOI":"10.1146\/annurev-psych-010814-015044","article-title":"Motivation and cognitive control: From behavior to neural mechanism","volume":"66","author":"MM Botvinick","year":"2015","journal-title":"Annual Review of Psychology"},{"key":"ref57","doi-asserted-by":"crossref","first-page":"593","DOI":"10.1521\/soco.2008.26.5.593","article-title":"The cognitive neuroscience of motivation and learning","volume":"26","author":"ND Daw","year":"2008","journal-title":"Social Cognition"},{"key":"ref58","doi-asserted-by":"crossref","first-page":"1024","DOI":"10.1111\/j.1460-9568.2011.07980.x","article-title":"How much of reinforcement learning is working memory, not reinforcement learning? A behavioral, computational, and neurogenetic analysis","volume":"35","author":"AGE Collins","year":"2012","journal-title":"European Journal of Neuroscience"}],"container-title":["PLOS Computational Biology"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/dx.plos.org\/10.1371\/journal.pcbi.1005090","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2020,5,9]],"date-time":"2020-05-09T02:44:57Z","timestamp":1588992297000},"score":1,"resource":{"primary":{"URL":"https:\/\/dx.plos.org\/10.1371\/journal.pcbi.1005090"}},"subtitle":[],"editor":[{"given":"Jill X","family":"O'Reilly","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"editor"}]}],"short-title":[],"issued":{"date-parts":[[2016,8,26]]},"references-count":58,"journal-issue":{"issue":"8","published-online":{"date-parts":[[2016,8,26]]}},"URL":"https:\/\/doi.org\/10.1371\/journal.pcbi.1005090","relation":{},"ISSN":["1553-7358"],"issn-type":[{"value":"1553-7358","type":"electronic"}],"subject":[],"published":{"date-parts":[[2016,8,26]]}}}