{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,2,23]],"date-time":"2026-02-23T16:09:29Z","timestamp":1771862969900,"version":"3.50.1"},"reference-count":137,"publisher":"Public Library of Science (PLoS)","issue":"12","license":[{"start":{"date-parts":[[2015,12,23]],"date-time":"2015-12-23T00:00:00Z","timestamp":1450828800000},"content-version":"vor","delay-in-days":0,"URL":"http:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"content-domain":{"domain":["www.ploscompbiol.org"],"crossmark-restriction":false},"short-container-title":["PLoS Comput Biol"],"DOI":"10.1371\/journal.pcbi.1004622","type":"journal-article","created":{"date-parts":[[2015,12,23]],"date-time":"2015-12-23T18:58:20Z","timestamp":1450897100000},"page":"e1004622","update-policy":"https:\/\/doi.org\/10.1371\/journal.pcbi.corrections_policy","source":"Crossref","is-referenced-by-count":54,"title":["Tamping Ramping: Algorithmic, Implementational, and Computational Explanations of Phasic Dopamine Signals in the Accumbens"],"prefix":"10.1371","volume":"11","author":[{"given":"Kevin","family":"Lloyd","sequence":"first","affiliation":[]},{"given":"Peter","family":"Dayan","sequence":"additional","affiliation":[]}],"member":"340","published-online":{"date-parts":[[2015,12,23]]},"reference":[{"issue":"5","key":"ref1","doi-asserted-by":"crossref","first-page":"1936","DOI":"10.1523\/JNEUROSCI.16-05-01936.1996","article-title":"A framework for mesencephalic dopamine systems based on predictive hebbian learning","volume":"16","author":"PR Montague","year":"1996","journal-title":"The Journal of Neuroscience"},{"key":"ref2","doi-asserted-by":"crossref","first-page":"1593","DOI":"10.1126\/science.275.5306.1593","article-title":"A neural substrate of prediction and reward","volume":"275","author":"W Schultz","year":"1997","journal-title":"Science"},{"issue":"1","key":"ref3","doi-asserted-by":"crossref","first-page":"9","DOI":"10.1007\/BF00115009","article-title":"Learning to predict by the methods of temporal differences","volume":"3","author":"RS Sutton","year":"1988","journal-title":"Machine Learning"},{"key":"ref4","doi-asserted-by":"crossref","DOI":"10.1109\/TNN.1998.712192","article-title":"Reinforcement learning: An introduction","author":"RS Sutton","year":"1998"},{"issue":"8","key":"ref5","doi-asserted-by":"crossref","first-page":"3526","DOI":"10.1523\/JNEUROSCI.5119-12.2013","article-title":"Dopamine encoding of Pavlovian incentive stimuli diminishes with extended training","volume":"33","author":"JJ Clark","year":"2013","journal-title":"The Journal of Neuroscience"},{"issue":"8","key":"ref6","doi-asserted-by":"crossref","first-page":"1020","DOI":"10.1038\/nn1923","article-title":"Associative learning mediates dynamic shifts in dopamine signaling in the nucleus accumbens","volume":"10","author":"JJ Day","year":"2007","journal-title":"Nature Neuroscience"},{"issue":"7328","key":"ref7","doi-asserted-by":"crossref","first-page":"53","DOI":"10.1038\/nature09588","article-title":"A selective role for dopamine in stimulus-reward learning","volume":"469","author":"SB Flagel","year":"2011","journal-title":"Nature"},{"key":"ref8","doi-asserted-by":"crossref","first-page":"1117","DOI":"10.1111\/j.1460-9568.2009.06916.x","article-title":"Neural encoding of cocaine-seeking behavior is coincident with phasic dopamine release in the accumbens core and shell","volume":"30","author":"CA Owesson-White","year":"2009","journal-title":"European Journal of Neuroscience"},{"key":"ref9","doi-asserted-by":"crossref","first-page":"614","DOI":"10.1038\/nature01476","article-title":"Subsecond dopamine release promotes cocaine seeking","volume":"422","author":"PE Phillips","year":"2003","journal-title":"Nature"},{"issue":"6","key":"ref10","doi-asserted-by":"crossref","first-page":"1265","DOI":"10.1523\/JNEUROSCI.3823-03.2004","article-title":"Dopamine operates as a subsecond modulator of food seeking","volume":"24","author":"MF Roitman","year":"2004","journal-title":"The Journal of Neuroscience"},{"key":"ref11","doi-asserted-by":"crossref","first-page":"853","DOI":"10.1038\/sj.npp.1300619","article-title":"Rapid dopamine signaling in the nucleus accumbens during contingent and noncontingent cocaine administration","volume":"30","author":"GD Stuber","year":"2005","journal-title":"Neuropsychopharmacology"},{"issue":"4","key":"ref12","doi-asserted-by":"crossref","first-page":"661","DOI":"10.1016\/j.neuron.2005.04.036","article-title":"Extinction of cocaine self-administration reveals functionally and temporally distinct dopaminergic signals in the nucleus accumbens","volume":"46","author":"GD Stuber","year":"2005","journal-title":"Neuron"},{"key":"ref13","doi-asserted-by":"crossref","first-page":"846","DOI":"10.1016\/j.biopsych.2011.12.019","article-title":"Phasic mesolimbic dopamine signaling precedes and predicts performance of a self-initiated action sequence task","volume":"71","author":"KM Wassum","year":"2012","journal-title":"Biological Psychiatry"},{"key":"ref14","doi-asserted-by":"crossref","first-page":"575","DOI":"10.1038\/nature12475","article-title":"Prolonged dopamine signalling in striatum signals proximity and value of distant rewards","volume":"500","author":"MW Howe","year":"2013","journal-title":"Nature"},{"key":"ref15","doi-asserted-by":"crossref","first-page":"533","DOI":"10.1038\/500533a","article-title":"Neuroscience: Dopamine ramps up","volume":"500","author":"Y Niv","year":"2013","journal-title":"Nature"},{"key":"ref16","first-page":"249","article-title":"Models of Information Processing in the Basal Ganglia","author":"JC Houk","year":"1995"},{"key":"ref17","doi-asserted-by":"crossref","first-page":"211","DOI":"10.1147\/rd.33.0210","article-title":"Some studies in machine learning using the game of checkers","volume":"3","author":"AL Samuel","year":"1959","journal-title":"IBM Journal on Research and Development"},{"key":"ref18","unstructured":"Watkins, CJCH. Learning from Delayed Rewards. PhD Thesis, University of Cambridge. 1989."},{"issue":"6","key":"ref19","doi-asserted-by":"crossref","first-page":"1735","DOI":"10.1523\/JNEUROSCI.5562-08.2009","article-title":"Synaptic overflow of dopamine in the nucleus accumbens arises from neuronal activity in the ventral tegmental area","volume":"29","author":"LA Sombers","year":"2009","journal-title":"The Journal of Neuroscience"},{"issue":"7","key":"ref20","doi-asserted-by":"crossref","first-page":"1754","DOI":"10.1523\/JNEUROSCI.4279-03.2004","article-title":"Dynamic gain control of dopamine delivery in freely moving animals","volume":"24","author":"PR Montague","year":"2004","journal-title":"The Journal of Neuroscience"},{"issue":"3","key":"ref21","doi-asserted-by":"crossref","first-page":"698","DOI":"10.1523\/JNEUROSCI.2489-13.2014","article-title":"Phasic dopamine release in the rat nucleus accumbens symmetrically encodes a reward prediction error term","volume":"34","author":"AS Hart","year":"2014","journal-title":"The Journal of Neuroscience"},{"key":"ref22","doi-asserted-by":"crossref","first-page":"84","DOI":"10.1016\/j.nlm.2014.07.010","article-title":"Dynamic shaping of dopamine signals during probabilistic Pavlovian conditioning","volume":"117","author":"A Hart","year":"2015","journal-title":"Neurobiology of Learning and Memory"},{"issue":"1","key":"ref23","doi-asserted-by":"crossref","first-page":"129","DOI":"10.1016\/j.neuron.2005.05.020","article-title":"Midbrain dopamine neurons encode a quantitative reward prediction error signal","volume":"47","author":"HM Bayer","year":"2005","journal-title":"Neuron"},{"key":"ref24","doi-asserted-by":"crossref","first-page":"1428","DOI":"10.1152\/jn.01140.2006","article-title":"Statistics of midbrain dopamine neuron spike trains in the awake primate","volume":"98","author":"HM Bayer","year":"2007","journal-title":"Journal of Neurophysiology"},{"issue":"1","key":"ref25","doi-asserted-by":"crossref","first-page":"6","DOI":"10.1186\/1744-9081-1-6","article-title":"Dopamine, uncertainty and TD learning","volume":"1","author":"Y Niv","year":"2005","journal-title":"Behavioral and Brain Functions"},{"issue":"5","key":"ref26","doi-asserted-by":"crossref","first-page":"220","DOI":"10.1016\/j.tins.2007.03.003","article-title":"Regulation of firing of dopaminergic neurons and control of goal-directed behaviors","volume":"30","author":"AA Grace","year":"2007","journal-title":"Trends in Neurosciences"},{"key":"ref27","doi-asserted-by":"crossref","first-page":"1","DOI":"10.3389\/fnbeh.2014.00188","article-title":"Local control of striatal dopamine release","volume":"8","author":"R Cachope","year":"2014","journal-title":"Frontiers in Behavioral Neuroscience"},{"key":"ref28","doi-asserted-by":"crossref","first-page":"112","DOI":"10.1016\/j.neuroscience.2011.08.066","article-title":"Dopamine release in the basal ganglia","volume":"198","author":"ME Rice","year":"2011","journal-title":"Neuroscience"},{"key":"ref29","article-title":"Dynamic Programming","author":"RE Bellman","year":"1957"},{"key":"ref30","article-title":"Dynamic Programming and Markov Processes","author":"R Howard","year":"1960"},{"key":"ref31","first-page":"835","article-title":"Neuronlike adaptive elements that can solve difficult learning control problems","volume":"13","author":"AG Barto","year":"1983","journal-title":"IEEE Transactions on Systems, Man, and Cybernetics"},{"key":"ref32","first-page":"215","article-title":"Models of Information Processing in the Basal Ganglia","author":"AG Barto","year":"1995"},{"key":"ref33","doi-asserted-by":"crossref","first-page":"535","DOI":"10.1016\/S0893-6080(02)00047-3","article-title":"Actor-critic models of the basal ganglia: new anatomical and computational perspectives","volume":"15","author":"D Joel","year":"2002","journal-title":"Neural Networks"},{"key":"ref34","doi-asserted-by":"crossref","first-page":"841","DOI":"10.1162\/089976601300014376","article-title":"Temporal difference model reproduces predictive neural activity","volume":"13","author":"R Suri","year":"2001","journal-title":"Neural Computation"},{"key":"ref35","first-page":"91","article-title":"Recent Breakthroughs in Basal Ganglia Research","author":"ND Daw","year":"2006"},{"key":"ref36","doi-asserted-by":"crossref","first-page":"452","DOI":"10.1126\/science.1094285","article-title":"Dissociable roles of ventral and dorsal striatum in instrumental conditioning","volume":"304","author":"J O\u2019Doherty","year":"2004","journal-title":"Science"},{"issue":"3","key":"ref37","doi-asserted-by":"crossref","first-page":"321","DOI":"10.1016\/S0149-7634(02)00007-6","article-title":"Emotion and motivation: the role of the amygdala, ventral striatum, and prefrontal cortex","volume":"26","author":"RN Cardinal","year":"2002","journal-title":"Neuroscience & Biobehavioral Reviews"},{"issue":"1","key":"ref38","doi-asserted-by":"crossref","first-page":"563","DOI":"10.1146\/annurev.neuro.25.112701.142937","article-title":"Learning and memory functions of the basal ganglia","volume":"25","author":"MG Packard","year":"2002","journal-title":"Annual Review of Neuroscience"},{"issue":"3","key":"ref39","doi-asserted-by":"crossref","first-page":"507","DOI":"10.1007\/s00213-006-0502-4","article-title":"Tonic dopamine: opportunity costs and the control of response vigor","volume":"191","author":"Y Niv","year":"2007","journal-title":"Psychopharmacology"},{"key":"ref40","doi-asserted-by":"crossref","first-page":"20130969","DOI":"10.1098\/rsif.2013.0969","article-title":"Optimal indolence: a normative microscopic approach to work and leisure","volume":"11","author":"RK Niyogi","year":"2014","journal-title":"Journal of the Royal Society Interface"},{"issue":"7","key":"ref41","doi-asserted-by":"crossref","first-page":"1152","DOI":"10.1111\/j.1460-9568.2012.08026.x","article-title":"Instrumental vigour in punishment and reward","volume":"35","author":"P Dayan","year":"2012","journal-title":"European Journal of Neuroscience"},{"key":"ref42","doi-asserted-by":"crossref","DOI":"10.1002\/9780470316887","article-title":"Markov decision processes: Discrete stochastic dynamic programming","author":"ML Puterman","year":"1994"},{"key":"ref43","doi-asserted-by":"crossref","first-page":"159","DOI":"10.1007\/BF00114727","article-title":"Average reward reinforcement learning: Foundations, algorithms, and empirical results","volume":"22","author":"S Mahadevan","year":"1996","journal-title":"Machine Learning"},{"issue":"8","key":"ref44","doi-asserted-by":"crossref","first-page":"375","DOI":"10.1016\/j.tics.2006.06.010","article-title":"A normative perspective on motivation","volume":"10","author":"Y Niv","year":"2006","journal-title":"Trends in Cognitive Sciences"},{"key":"ref45","doi-asserted-by":"crossref","first-page":"470","DOI":"10.1016\/j.neuron.2012.10.021","article-title":"The mysterious motivational functions of mesolimbic dopamine","volume":"76","author":"JD Salamone","year":"2012","journal-title":"Neuron"},{"key":"ref46","doi-asserted-by":"crossref","first-page":"1495","DOI":"10.1038\/npp.2013.48","article-title":"Dopamine modulates reward-related vigor","volume":"38","author":"U Beierholm","year":"2013","journal-title":"Neuropsychopharmacology"},{"issue":"12","key":"ref47","doi-asserted-by":"crossref","first-page":"3933","DOI":"10.1162\/jocn_a_00090","article-title":"Vigor in the face of fluctuating rates of reward: an experimental examination","volume":"23","author":"M Guitart-Masip","year":"2011","journal-title":"Journal of Cognitive Neuroscience"},{"key":"ref48","doi-asserted-by":"crossref","first-page":"1637","DOI":"10.1162\/neco.2006.18.7.1637","article-title":"Representation and timing in theories of the dopamine system","volume":"18","author":"ND Daw","year":"2006","journal-title":"Neural Computation"},{"key":"ref49","doi-asserted-by":"crossref","first-page":"3062","DOI":"10.1162\/NECO_a_00049","article-title":"Internal-time temporal difference model for neural value-based decision making","volume":"22","author":"H Nakahara","year":"2010","journal-title":"Neural Computation"},{"issue":"3","key":"ref50","doi-asserted-by":"crossref","first-page":"623","DOI":"10.1093\/brain\/106.3.623","article-title":"Time of conscious intention to act in relation to onset of cerebral activity (readiness-potential)","volume":"106","author":"B Libet","year":"1983","journal-title":"Brain"},{"issue":"1","key":"ref51","doi-asserted-by":"crossref","first-page":"177","DOI":"10.1007\/BF00231144","article-title":"Morphological and electrophysiological characteristics of pyramidal tract neurons in the rat","volume":"57","author":"P Landry","year":"1984","journal-title":"Experimental Brain Research"},{"issue":"2","key":"ref52","doi-asserted-by":"crossref","first-page":"311","DOI":"10.1016\/0006-8993(95)01333-4","article-title":"Corticostriatal projections from layer V cells in rat are collaterals of long-range corticofugal axons","volume":"709","author":"M L\u00e9vesque","year":"1996","journal-title":"Brain Research"},{"issue":"38","key":"ref53","doi-asserted-by":"crossref","first-page":"8289","DOI":"10.1523\/JNEUROSCI.1990-04.2004","article-title":"Evidence for differential cortical input to direct pathway versus indirect pathway striatal projection neurons in rats","volume":"24","author":"W Lei","year":"2004","journal-title":"The Journal of Neuroscience"},{"key":"ref54","doi-asserted-by":"crossref","first-page":"357","DOI":"10.1146\/annurev.ne.09.030186.002041","article-title":"Parallel organization of functionally segregated circuits linking basal ganglia and cortex","volume":"9","author":"GE Alexander","year":"1986","journal-title":"Annual Review of Neuroscience"},{"issue":"3","key":"ref55","doi-asserted-by":"crossref","first-page":"396","DOI":"10.1007\/BF00227836","article-title":"Role of primate basal ganglia and frontal cortex in the internal generation of movements III. Neuronal activity in the supplementary motor area","volume":"91","author":"R Romo","year":"1992","journal-title":"Experimental Brain Research"},{"key":"ref56","doi-asserted-by":"crossref","first-page":"317","DOI":"10.1016\/j.jchemneu.2003.10.003","article-title":"The primate basal ganglia: parallel and integrative networks","volume":"26","author":"SN Haber","year":"2003","journal-title":"Journal of Chemical Neuroanatomy"},{"issue":"1","key":"ref57","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1007\/BF00412364","article-title":"Hirnpotential\u00e4nderungen bei Willk\u00fcrbewegungen und passiven Bewegungen des Menschen: Bereitschaftspotential und reafferente Potentiale","volume":"284","author":"HH Kornhuber","year":"1965","journal-title":"Pfl\u00fcger\u2019s Archiv f\u00fcr die gesamte Physiologie des Menschen und der Tiere"},{"key":"ref58","doi-asserted-by":"crossref","first-page":"322","DOI":"10.1016\/0013-4694(82)90181-X","article-title":"Readiness potential preceding unrestricted \u2018spontaneous\u2019 vs. pre-planned voluntary acts","volume":"54","author":"B Libet","year":"1982","journal-title":"Electroencephalography and Clinical Neurophysiology"},{"issue":"3","key":"ref59","doi-asserted-by":"crossref","first-page":"279","DOI":"10.1037\/0033-295X.84.3.279","article-title":"Scalar expectancy theory and Weber\u2019s law in animal timing","volume":"84","author":"J Gibbon","year":"1977","journal-title":"Psychological Review"},{"key":"ref60","doi-asserted-by":"crossref","first-page":"603","DOI":"10.1016\/S0893-6080(02)00052-7","article-title":"Opponent interactions between serotonin and dopamine","volume":"15","author":"ND Daw","year":"2002","journal-title":"Neural Networks"},{"key":"ref61","doi-asserted-by":"crossref","first-page":"144","DOI":"10.1016\/j.neuron.2010.06.016","article-title":"Distinct tonic and phasic anticipatory activity in lateral habenula and dopamine neurons","volume":"67","author":"ES Bromberg-Martin","year":"2010","journal-title":"Neuron"},{"key":"ref62","doi-asserted-by":"crossref","first-page":"966","DOI":"10.1038\/nn.2159","article-title":"The temporal precision of reward prediction in dopamine neurons","volume":"11","author":"CD Fiorillo","year":"2008","journal-title":"Nature Neuroscience"},{"issue":"4","key":"ref63","doi-asserted-by":"crossref","first-page":"1110","DOI":"10.1152\/jn.00401.2014","article-title":"Dopamine neurons encode errors in predicting movement trigger occurrence","volume":"113","author":"B Pasquereau","year":"2015","journal-title":"Journal of Neurophysiology"},{"issue":"49","key":"ref64","doi-asserted-by":"crossref","first-page":"16585","DOI":"10.1523\/JNEUROSCI.3958-10.2010","article-title":"The flexible approach hypothesis: unification of effort and cue-responding hypotheses for the role of nucleus accumbens dopamine in the activation of reward-seeking behavior","volume":"30","author":"SM Nicola","year":"2010","journal-title":"The Journal of Neuroscience"},{"issue":"8","key":"ref65","doi-asserted-by":"crossref","first-page":"423","DOI":"10.1016\/S0166-2236(03)00177-2","article-title":"A computational substrate for incentive salience","volume":"26","author":"SM McClure","year":"2003","journal-title":"Trends in Neurosciences"},{"key":"ref66","doi-asserted-by":"crossref","first-page":"725","DOI":"10.1038\/377725a0","article-title":"Bee foraging in uncertain environments using predictive Hebbian learning","volume":"377","author":"PR Montague","year":"1995","journal-title":"Nature"},{"key":"ref67","doi-asserted-by":"crossref","first-page":"760","DOI":"10.1038\/nature03015","article-title":"Computational roles for dopamine in behavioural control","volume":"431","author":"PR Montague","year":"2004","journal-title":"Nature"},{"issue":"1","key":"ref68","doi-asserted-by":"crossref","first-page":"1","DOI":"10.2307\/2983610","article-title":"Sequential tests in industrial statistics","volume":"8","author":"GA Barnard","year":"1946","journal-title":"Supplement to the Journal of the Royal Statistical Society"},{"key":"ref69","article-title":"Sequential analysis","author":"A Wald","year":"1947"},{"key":"ref70","doi-asserted-by":"crossref","first-page":"535","DOI":"10.1146\/annurev.neuro.29.051605.113038","article-title":"The neural basis of decision making","volume":"30","author":"JI Gold","year":"2007","journal-title":"Annual Reviews Neuroscience"},{"key":"ref71","doi-asserted-by":"crossref","first-page":"59","DOI":"10.1037\/0033-295X.85.2.59","article-title":"A theory of memory retrieval","volume":"85","author":"R Ratcliff","year":"1978","journal-title":"Psychological Review"},{"issue":"33","key":"ref72","doi-asserted-by":"crossref","first-page":"13852","DOI":"10.1073\/pnas.1101328108","article-title":"Multialternative drift-diffusion model predicts the relationship between visual fixations and choice in value-based decisions","volume":"108","author":"I Krajbich","year":"2011","journal-title":"Proceedings of the National Academy of Sciences of the USA"},{"issue":"4","key":"ref73","doi-asserted-by":"crossref","first-page":"700","DOI":"10.1037\/0033-295X.113.4.700","article-title":"The physics of optimal decision making: A formal analysis of models of performance in two-alternative forced-choice tasks","volume":"113","author":"R Bogacz","year":"2006","journal-title":"Psychological Review"},{"key":"ref74","article-title":"Information theory of choice-reaction times","author":"DRJ Laming","year":"1968"},{"key":"ref75","doi-asserted-by":"crossref","first-page":"251","DOI":"10.1007\/BF02289729","article-title":"Models for choice reaction time","volume":"25","author":"M Stone","year":"1960","journal-title":"Psychometrika"},{"issue":"3","key":"ref76","doi-asserted-by":"crossref","first-page":"550","DOI":"10.1037\/0033-295X.108.3.550","article-title":"The time course of perceptual choice: the leaky, competing accumulator model","volume":"108","author":"M Usher","year":"2001","journal-title":"Psychological Review"},{"issue":"5","key":"ref77","doi-asserted-by":"crossref","first-page":"955","DOI":"10.1016\/S0896-6273(02)01092-9","article-title":"Probabilistic decision making by slow reverberation in cortical circuits","volume":"36","author":"XJ Wang","year":"2002","journal-title":"Neuron"},{"key":"ref78","doi-asserted-by":"crossref","first-page":"185","DOI":"10.1146\/annurev.neuro.23.1.185","article-title":"Dopaminergic modulation of neuronal excitability in the striatum and nucleus accumbens","volume":"23","author":"SM Nicola","year":"2000","journal-title":"Annual Review of Neuroscience"},{"key":"ref79","doi-asserted-by":"crossref","first-page":"892","DOI":"10.1126\/science.2392679","article-title":"A network model of catecholamine effects: Gain, signal-to-noise ratio, and behavior","volume":"249","author":"D Servan-Schreiber","year":"1990","journal-title":"Science"},{"key":"ref80","doi-asserted-by":"crossref","first-page":"2863","DOI":"10.1162\/neco.2008.03-07-487","article-title":"Optimization of decision making in multilayer networks: the role of the locus coeruleus","volume":"20","author":"E Shea-Brown","year":"2008","journal-title":"Neural Computation"},{"issue":"1","key":"ref81","doi-asserted-by":"crossref","first-page":"51","DOI":"10.1162\/0898929052880093","article-title":"Dynamic dopamine modulation in the basal ganglia: A neurocomputational account of cognitive deficits in medicated and nonmedicated Parkinsonism","volume":"17","author":"MJ Frank","year":"2005","journal-title":"Journal of Cognitive Neuroscience"},{"key":"ref82","article-title":"Optimizing reward rate in two alternative choice tasks: mathematical formalism","author":"J Moehlis","year":"2004"},{"issue":"30","key":"ref83","doi-asserted-by":"crossref","first-page":"9913","DOI":"10.1523\/JNEUROSCI.23-30-09913.2003","article-title":"Correlated coding of motivation and outcome of decision by dopamine neurons","volume":"23","author":"T Satoh","year":"2003","journal-title":"The Journal of Neuroscience"},{"issue":"12","key":"ref84","doi-asserted-by":"crossref","first-page":"1704","DOI":"10.1038\/nn1560","article-title":"Uncertainty-based competition between prefrontal and dorsolateral striatal systems for behavioral control","volume":"8","author":"ND Daw","year":"2005","journal-title":"Nature Neuroscience"},{"key":"ref85","article-title":"Mesolimbic dopamine signals the value of work","author":"AA Hamid","year":"2015","journal-title":"Nature Neuroscience"},{"key":"ref86","article-title":"Dynamic Programming and Optimal Control","volume":"Vol.II","author":"DP Bertsekas","year":"2007"},{"key":"ref87","doi-asserted-by":"crossref","first-page":"219","DOI":"10.1162\/089976600300015961","article-title":"Reinforcement learning in continuous time and space","volume":"12","author":"K Doya","year":"2000","journal-title":"Neural Computation"},{"issue":"7","key":"ref88","doi-asserted-by":"crossref","first-page":"2046","DOI":"10.1111\/j.1460-9568.2007.05772.x","article-title":"Dopamine release is heterogeneous within microenvironments of the rat nucleus accumbens","volume":"26","author":"RM Wightman","year":"2007","journal-title":"European Journal of Neuroscience"},{"issue":"31","key":"ref89","doi-asserted-by":"crossref","first-page":"7837","DOI":"10.1523\/JNEUROSCI.1600-08.2008","article-title":"Influence of reward delays on responses of dopamine neurons","volume":"28","author":"S Kobayashi","year":"2008","journal-title":"The Journal of Neuroscience"},{"issue":"1","key":"ref90","doi-asserted-by":"crossref","first-page":"145","DOI":"10.1152\/jn.1992.67.1.145","article-title":"Responses of monkey dopamine neurons during learning of behavioural reactions","volume":"67","author":"T Ljungberg","year":"1992","journal-title":"Journal of Neurophysiology"},{"issue":"3","key":"ref91","doi-asserted-by":"crossref","first-page":"592","DOI":"10.1152\/jn.1990.63.3.592","article-title":"Dopamine neurons of the monkey midbrain: contingencies of response to active touch during self-initiated arm movements","volume":"63","author":"R Romo","year":"1990","journal-title":"Journal of Neurophysiology"},{"key":"ref92","doi-asserted-by":"crossref","first-page":"431","DOI":"10.1007\/BF00247503","article-title":"Neuronal activity in the monkey striatum during the initiation of movements","volume":"71","author":"W Schultz","year":"1988","journal-title":"Experimental Brain Research"},{"issue":"3","key":"ref93","doi-asserted-by":"crossref","first-page":"363","DOI":"10.1007\/BF00227834","article-title":"Role of primate basal ganglia and frontal cortex in the internal generation of movements I. Preparatory activity in the anterior striatum","volume":"91","author":"W Schultz","year":"1992","journal-title":"Experimental Brain Research"},{"key":"ref94","doi-asserted-by":"crossref","first-page":"1849","DOI":"10.1111\/j.1460-9568.2008.06480.x","article-title":"Anticipatory reward signals in ventral striatal neurons of behaving rats","volume":"28","author":"M Khamassi","year":"2008","journal-title":"European Journal of Neuroscience"},{"issue":"8","key":"ref95","doi-asserted-by":"crossref","first-page":"2843","DOI":"10.1523\/JNEUROSCI.4869-10.2011","article-title":"Theta phase precession in rat ventral striatum links place and reward information","volume":"31","author":"MAA van der Meer","year":"2011","journal-title":"The Journal of Neuroscience"},{"issue":"52","key":"ref96","doi-asserted-by":"crossref","first-page":"19150","DOI":"10.1073\/pnas.0509607102","article-title":"Simultaneous dopamine and single-unit recordings reveal accumbens GABAergic responses: implications for intracranial self-stimulation","volume":"102","author":"JF Cheer","year":"2005","journal-title":"Proceedings of the National Academy of Sciences of the United States of America"},{"key":"ref97","doi-asserted-by":"crossref","first-page":"237","DOI":"10.1016\/j.neuron.2007.03.021","article-title":"Coordinated accumbal dopamine release and neural activity drive goal-directed behavior","volume":"54","author":"JF Cheer","year":"2007","journal-title":"Neuron"},{"issue":"39","key":"ref98","doi-asserted-by":"crossref","first-page":"13860","DOI":"10.1523\/JNEUROSCI.1340-11.2011","article-title":"Rapid dopamine signaling differentially modulates distinct microcircuits within the nucleus accumbens during sucrose-directed behavior","volume":"31","author":"F Cacciapaglia","year":"2011","journal-title":"The Journal of Neuroscience"},{"key":"ref99","doi-asserted-by":"crossref","first-page":"3034","DOI":"10.1162\/neco.2008.11-07-654","article-title":"Stimulus representation and the timing of reward-prediction errors in models of the dopamine system","volume":"20","author":"EA Ludvig","year":"2008","journal-title":"Neural Computation"},{"issue":"1","key":"ref100","doi-asserted-by":"crossref","first-page":"107","DOI":"10.1007\/s10827-009-0191-1","article-title":"Alternative time representation in dopamine models","volume":"28","author":"F Rivest","year":"2010","journal-title":"Journal of Computational Neuroscience"},{"issue":"3","key":"ref101","doi-asserted-by":"crossref","first-page":"871","DOI":"10.1016\/S0306-4522(98)00697-6","article-title":"A neural network model with dopamine-like reinforcement signal that learns a spatial delayed response task","volume":"91","author":"RE Suri","year":"1999","journal-title":"Neuroscience"},{"key":"ref102","first-page":"194","article-title":"Time representation in reinforcement learning models of the basal ganglia","volume":"7","author":"SJ Gershman","year":"2013","journal-title":"Frontiers in Computational Neuroscience"},{"key":"ref103","doi-asserted-by":"crossref","first-page":"755","DOI":"10.1038\/nrn1764","article-title":"What makes us tick? Functional and neural mechanisms of interval timing","volume":"6","author":"CV Buhusi","year":"2005","journal-title":"Nature Reviews Neuroscience"},{"key":"ref104","doi-asserted-by":"crossref","first-page":"9914","DOI":"10.1523\/JNEUROSCI.2446-04.2004","article-title":"Phasic activation of monkey locus ceruleus neurons by simple decisions in a forced-choice task","volume":"24","author":"E Clayton","year":"2004","journal-title":"The Journal of Neuroscience"},{"key":"ref105","doi-asserted-by":"crossref","first-page":"1186","DOI":"10.1162\/NECO_a_00270","article-title":"Reinforcement-based decision making in corticostriatal circuits: mutual constraints by neurocomputational and diffusion models","volume":"24","author":"R Ratcliff","year":"2012","journal-title":"Neural Computation"},{"key":"ref106","unstructured":"Wiecki TV. Computational psychiatry: Combining multiple levels of analysis to understand brain disorders. PhD Thesis, Brown University; 2015."},{"issue":"3","key":"ref107","doi-asserted-by":"crossref","first-page":"337","DOI":"10.1037\/a0037015","article-title":"Opponent Actor Learning (OpAL): modeling interactive effects of striatal dopamine on reinforcement learning and choice incentive","volume":"121","author":"AGE Collins","year":"2014","journal-title":"Psychological Review"},{"issue":"3","key":"ref108","doi-asserted-by":"crossref","first-page":"485","DOI":"10.1901\/jeab.1974.21-485","article-title":"Impulse control in pigeons","volume":"21","author":"GW Ainslie","year":"1974","journal-title":"Journal of the Experimental Analysis of Behavior"},{"key":"ref109","doi-asserted-by":"crossref","DOI":"10.1017\/CBO9781139164191","article-title":"Breakdown of Will","author":"G Ainslie","year":"2001"},{"key":"ref110","unstructured":"Sorg J, Singh SP, Lewis RL. Internal rewards mitigate agent boundedness. In: Proceedings of the 27th International Conference on Machine Learning; 2010. p. 1007\u20131014."},{"issue":"Suppl 4","key":"ref111","doi-asserted-by":"crossref","first-page":"S177","DOI":"10.1098\/rsbl.2003.0134","article-title":"Do pretty women inspire men to discount the future?","volume":"271","author":"M Wilson","year":"2004","journal-title":"Proceedings of the Royal Society of London B: Biological Sciences"},{"issue":"11","key":"ref112","doi-asserted-by":"crossref","first-page":"e152","DOI":"10.1371\/journal.pcbi.0020152","article-title":"Humans can adopt optimal discounting strategy under real-time constraints","volume":"2","author":"N Schweighofer","year":"2006","journal-title":"PLOS Computational Biology"},{"issue":"8","key":"ref113","doi-asserted-by":"crossref","first-page":"887","DOI":"10.1038\/nn1279","article-title":"Prediction of immediate and future rewards differentially recruits cortico-basal ganglia loops","volume":"7","author":"SC Tanaka","year":"2004","journal-title":"Nature Neuroscience"},{"issue":"12","key":"ref114","doi-asserted-by":"crossref","first-page":"e1333","DOI":"10.1371\/journal.pone.0001333","article-title":"Serotonin differentially regulates short-and long-term prediction of rewards in the ventral and dorsal striatum","volume":"2","author":"SC Tanaka","year":"2007","journal-title":"PLOS One"},{"issue":"9","key":"ref115","doi-asserted-by":"crossref","first-page":"968","DOI":"10.1038\/nn1103","article-title":"Afferent modulation of dopamine neuron firing differentially regulates tonic and phasic dopamine transmission","volume":"6","author":"SB Floresco","year":"2003","journal-title":"Nature Neuroscience"},{"issue":"1","key":"ref116","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1016\/0306-4522(91)90196-U","article-title":"Phasic versus tonic dopamine release and the modulation of dopamine system responsivity: a hypothesis for the etiology of schizophrenia","volume":"41","author":"AA Grace","year":"1991","journal-title":"Neuroscience"},{"key":"ref117","doi-asserted-by":"crossref","first-page":"1898","DOI":"10.1126\/science.1077349","article-title":"Discrete coding of reward probability and uncertainty by dopamine neurons","volume":"299","author":"CD Fiorillo","year":"2003","journal-title":"Science"},{"key":"ref118","doi-asserted-by":"crossref","unstructured":"Fiorillo CD, Tobler PN, Schultz W. Evidence that the delay-period activity of dopamine neurons corresponds to reward uncertainty rather than backpropagating TD errors. Behavioral and Brain Functions. 2005;.","DOI":"10.1186\/1744-9081-1-7"},{"key":"ref119","doi-asserted-by":"crossref","first-page":"58","DOI":"10.1016\/j.neuron.2012.04.038","article-title":"Striatal dopamine release is triggered by synchronized activity in cholinergic interneurons","volume":"75","author":"S Threlfell","year":"2012","journal-title":"Neuron"},{"issue":"4","key":"ref120","doi-asserted-by":"crossref","first-page":"751","DOI":"10.1016\/0306-4522(92)90202-D","article-title":"On the significance of subterritories in the \u201caccumbens\u201d part of the rat ventral striatum","volume":"50","author":"DS Zahm","year":"1992","journal-title":"Neuroscience"},{"issue":"1","key":"ref121","doi-asserted-by":"crossref","first-page":"113","DOI":"10.1111\/j.1749-6632.1999.tb09264.x","article-title":"Functional-anatomical implications of the nucleus accumbens core and shell subterritories","volume":"877","author":"DS Zahm","year":"1999","journal-title":"Annals of the New York Academy of Sciences"},{"issue":"1","key":"ref122","doi-asserted-by":"crossref","first-page":"75","DOI":"10.1016\/S0166-4328(02)00286-3","article-title":"Nucleus accumbens shell and core dopamine: differential role in behavior and addiction","volume":"137","author":"G Di Chiara","year":"2002","journal-title":"Behavioural Brain Research"},{"issue":"33","key":"ref123","doi-asserted-by":"crossref","first-page":"11786","DOI":"10.1523\/JNEUROSCI.2711-11.2011","article-title":"The general and outcome-specific forms of pavlovian-instrumental transfer are differentially mediated by the nucleus accumbens core and shell","volume":"31","author":"LH Corbit","year":"2011","journal-title":"The Journal of Neuroscience"},{"key":"ref124","doi-asserted-by":"crossref","first-page":"27","DOI":"10.1016\/j.brainresrev.2007.05.004","article-title":"Dopamine reward circuitry: Two projection systems from the ventral midbrain to the nucleus accumbens-olfactory tubercle complex","volume":"56","author":"S Ikemoto","year":"2007","journal-title":"Brain Research Reviews"},{"issue":"6","key":"ref125","doi-asserted-by":"crossref","first-page":"2401","DOI":"10.1523\/JNEUROSCI.19-06-02401.1999","article-title":"Dissociation in effects of lesions of the nucleus accumbens core and shell on appetitive pavlovian approach behavior and the potentiation of conditioned reinforcement and locomotor activity by d-amphetamine","volume":"19","author":"JA Parkinson","year":"1999","journal-title":"The Journal of Neuroscience"},{"issue":"4","key":"ref126","doi-asserted-by":"crossref","first-page":"385","DOI":"10.1016\/j.pneurobio.2009.11.003","article-title":"The ventral basal ganglia, a selection mechanism at the crossroads of space, strategy, and reward","volume":"90","author":"MD Humphries","year":"2010","journal-title":"Progress in Neurobiology"},{"key":"ref127","doi-asserted-by":"crossref","first-page":"273","DOI":"10.2741\/E615","article-title":"Rapid dopamine dynamics in the accumbens core and shell: learning and action","volume":"5","author":"MP Saddoris","year":"2013","journal-title":"Frontiers in Bioscience"},{"key":"ref128","doi-asserted-by":"crossref","first-page":"467","DOI":"10.1162\/NECO_a_00559","article-title":"Dopamine ramps are a consequence of reward prediction errors","volume":"26","author":"SJ Gershman","year":"2014","journal-title":"Neural Computation"},{"issue":"1","key":"ref129","doi-asserted-by":"crossref","first-page":"171","DOI":"10.1016\/0006-8993(71)90358-1","article-title":"The hippocampus as a spatial map. Preliminary evidence from unit activity in the freely-moving rat","volume":"34","author":"J O\u2019Keefe","year":"1971","journal-title":"Brain Research"},{"key":"ref130","article-title":"The hippocampus as a cognitive map","author":"J O\u2019Keefe","year":"1978"},{"key":"ref131","doi-asserted-by":"crossref","first-page":"69","DOI":"10.1146\/annurev.neuro.31.061307.090723","article-title":"Place cells, grid cells, and the brain\u2019s spatial representation system","volume":"31","author":"EI Moser","year":"2008","journal-title":"Annual Review of Neuroscience"},{"key":"ref132","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1002\/(SICI)1098-1063(2000)10:1<1::AID-HIPO1>3.0.CO;2-1","article-title":"A model of hippocampally dependent navigation, using the temporal difference learning rule","volume":"10","author":"DJ Foster","year":"2000","journal-title":"Hippocampus"},{"issue":"2","key":"ref133","doi-asserted-by":"crossref","first-page":"131","DOI":"10.1109\/TNSRE.2005.847368","article-title":"An analysis of hippocampal spatio-temporal representations using a Bayesian algorithm for neural spike train decoding","volume":"13","author":"R Barbieri","year":"2005","journal-title":"IEEE Transactions on Neural Systems and Rehabilitation Engineering"},{"key":"ref134","first-page":"1","article-title":"Striatal dopamine ramping may indicate flexible reinforcement learning with forgetting in the cortical-basal ganglia circuits","volume":"8","author":"K Morita","year":"2014","journal-title":"Frontiers in Neural Circuits"},{"issue":"2","key":"ref135","doi-asserted-by":"crossref","first-page":"160","DOI":"10.1089\/cap.2005.15.160","article-title":"Dopamine, learning, and impulsivity: A biological account of attention-deficit\/hyperactivity disorder","volume":"15","author":"J Williams","year":"2005","journal-title":"Journal of Child & Adolescent Psychopharmacology"},{"issue":"8","key":"ref136","doi-asserted-by":"crossref","first-page":"1163","DOI":"10.1021\/cn4000692","article-title":"Two-photon optical interrogation of individual dendritic spines with caged dopamine","volume":"4","author":"R Araya","year":"2013","journal-title":"ACS Chemical Neuroscience"},{"issue":"6","key":"ref137","doi-asserted-by":"crossref","first-page":"816","DOI":"10.1038\/nn.3100","article-title":"Distinct roles for direct and indirect pathway striatal neurons in reinforcement","volume":"15","author":"AV Kravitz","year":"2012","journal-title":"Nature Neuroscience"}],"container-title":["PLOS Computational Biology"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/dx.plos.org\/10.1371\/journal.pcbi.1004622","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2019,9,2]],"date-time":"2019-09-02T22:33:38Z","timestamp":1567463618000},"score":1,"resource":{"primary":{"URL":"https:\/\/dx.plos.org\/10.1371\/journal.pcbi.1004622"}},"subtitle":[],"editor":[{"given":"Olaf","family":"Sporns","sequence":"first","affiliation":[]}],"short-title":[],"issued":{"date-parts":[[2015,12,23]]},"references-count":137,"journal-issue":{"issue":"12","published-online":{"date-parts":[[2015,12,23]]}},"URL":"https:\/\/doi.org\/10.1371\/journal.pcbi.1004622","relation":{},"ISSN":["1553-7358"],"issn-type":[{"value":"1553-7358","type":"electronic"}],"subject":[],"published":{"date-parts":[[2015,12,23]]}}}