{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,10,12]],"date-time":"2025-10-12T04:57:09Z","timestamp":1760245029004},"reference-count":56,"publisher":"Springer Science and Business Media LLC","issue":"3","license":[{"start":{"date-parts":[[2008,1,15]],"date-time":"2008-01-15T00:00:00Z","timestamp":1200355200000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by-nc\/2.0"},{"start":{"date-parts":[[2008,1,15]],"date-time":"2008-01-15T00:00:00Z","timestamp":1200355200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by-nc\/2.0"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Biol Cybern"],"published-print":{"date-parts":[[2008,3]]},"DOI":"10.1007\/s00422-007-0209-6","type":"journal-article","created":{"date-parts":[[2008,1,14]],"date-time":"2008-01-14T14:47:51Z","timestamp":1200322071000},"update-policy":"http:\/\/dx.doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":12,"title":["Mathematical properties of neuronal TD-rules and differential Hebbian learning: a comparison"],"prefix":"10.1007","volume":"98","author":[{"given":"Christoph","family":"Kolodziejski","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Bernd","family":"Porr","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Florentin","family":"W\u00f6rg\u00f6tter","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2008,1,15]]},"reference":[{"issue":"3","key":"209_CR1","doi-asserted-by":"publisher","first-page":"287","DOI":"10.1007\/s004220000171","volume":"83","author":"A Arleo","year":"2000","unstructured":"Arleo A, Gerstner W (2000) Spatial cognition and neuro-mimetic navigation: a model of hippocampal place cell activity. Biol Cybern 83(3): 287\u201399","journal-title":"Biol Cybern"},{"key":"209_CR2","first-page":"215","volume-title":"Models of information processing in the basal ganglia","author":"A Barto","year":"1995","unstructured":"Barto A (1995) Adaptive critics and the basal ganglia. In: Houk JC, Davis JL, Beiser DG(eds) Models of information processing in the basal ganglia. MIT Press, Cambridge, pp 215\u201332"},{"key":"209_CR3","doi-asserted-by":"publisher","first-page":"139","DOI":"10.1146\/annurev.neuro.24.1.139","volume":"24","author":"GQ Bi","year":"2001","unstructured":"Bi GQ, Poo M (2001) Synaptic modification by correlated activity: Hebb\u2019s postulate revisited. Annu Rev Neurosci 24: 139\u201366","journal-title":"Annu Rev Neurosci"},{"issue":"5","key":"209_CR4","doi-asserted-by":"publisher","first-page":"462","DOI":"10.1119\/1.1557302","volume":"71","author":"TB Boykina","year":"2003","unstructured":"Boykina TB (2003) Derivatives of the dirac delta function by explicit construction of sequences. Am J Phys 71(5): 462\u201368","journal-title":"Am J Phys"},{"issue":"3\/4","key":"209_CR5","doi-asserted-by":"publisher","first-page":"341","DOI":"10.1023\/A:1022632907294","volume":"8","author":"P Dayan","year":"1992","unstructured":"Dayan P (1992) The convergence of TD(\u03bb). Mach Learn 8(3\/4): 341\u201362","journal-title":"Mach Learn"},{"key":"209_CR6","volume-title":"Theoretical Neuroscience, Computational and mathematical modeling of neural systems","author":"P Dayan","year":"2003","unstructured":"Dayan P, Abbott L (2003) Theoretical Neuroscience, Computational and mathematical modeling of neural systems. MIT Press, Cambridge"},{"key":"209_CR7","first-page":"295","volume":"14","author":"P Dayan","year":"1994","unstructured":"Dayan P, Seynowski T (1994) TD(\u03bb) converges with probability 1. Mach Learn 14: 295\u201301","journal-title":"Mach Learn"},{"key":"209_CR8","doi-asserted-by":"publisher","first-page":"1468","DOI":"10.1162\/neco.2007.19.6.1468","volume":"19","author":"RV Florian","year":"2007","unstructured":"Florian RV (2007) Reinforcement learning through modulation of spike-timing-dependent synaptic plasticity. Neural Comput 19: 1468\u2013502","journal-title":"Neural Comput"},{"key":"209_CR9","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1002\/(SICI)1098-1063(2000)10:1<1::AID-HIPO1>3.0.CO;2-1","volume":"10","author":"DJ Foster","year":"2000","unstructured":"Foster DJ, Morris RGM, Dayan P (2000) A model of hippocampally dependent navigation, using the temporal difference learning rule. Hippocampus 10: 1\u20136","journal-title":"Hippocampus"},{"key":"209_CR10","doi-asserted-by":"publisher","first-page":"76","DOI":"10.1038\/383076a0","volume":"383","author":"W Gerstner","year":"1996","unstructured":"Gerstner W, Kempter R, van Hemmen JL, Wagner H (1996) A neuronal learning rule for sub-millisecond temporal coding. Nature 383: 76\u20138","journal-title":"Nature"},{"key":"209_CR11","doi-asserted-by":"publisher","first-page":"9","DOI":"10.1037\/h0054032","volume":"46","author":"CL Hull","year":"1939","unstructured":"Hull CL (1939) The problem of stimulus equivalence in behavior theory. Psychol Rev 46: 9\u20130","journal-title":"Psychol Rev"},{"key":"209_CR12","volume-title":"Principles of behavior","author":"CL Hull","year":"1943","unstructured":"Hull CL (1943) Principles of behavior. Appleton Century Crofts, New York"},{"issue":"6968","key":"209_CR13","doi-asserted-by":"publisher","first-page":"841","DOI":"10.1038\/nature02194","volume":"426","author":"Y Humeau","year":"2003","unstructured":"Humeau Y, Shaban H, Bissiere S, Luthi A (2003) Presynaptic induction of heterosynaptic associative plasticity in the mammalian brain. Nature 426(6968): 841\u201345","journal-title":"Nature"},{"key":"209_CR14","doi-asserted-by":"crossref","unstructured":"Izhikevich E (2007) Solving the distal reward problem through linkage of stdp and dopamine signaling. Cerebral Cortex 101093\/cercor\/bhl152","DOI":"10.1186\/1471-2202-8-S2-S15"},{"key":"209_CR15","doi-asserted-by":"crossref","first-page":"237","DOI":"10.1613\/jair.301","volume":"4","author":"LP Kaelbling","year":"1996","unstructured":"Kaelbling LP, Littman ML, Moore AW (1996) Reinforcement learning: a survey. J Artif Intell Res 4: 237\u201385","journal-title":"J Artif Intell Res"},{"key":"209_CR16","unstructured":"Klopf AH (1972) Brain function and adaptive systems\u2014a heterostatic theory. Technical report, Air Force Cambridge Research Laboratories Special Report No. 133, Defense Technical Information Center, Cameron Station, Alexandria, VA 22304"},{"key":"209_CR17","volume-title":"The hedonistic neuron: a theory of memory, learning, and intelligence","author":"AH Klopf","year":"1982","unstructured":"Klopf AH (1982) The hedonistic neuron: a theory of memory, learning, and intelligence. Hemisphere, Washington DC"},{"key":"209_CR18","doi-asserted-by":"crossref","unstructured":"Klopf AH (1986) A drive-reinforcement model of single neuron function. In: Denker JS (ed) Neural networks for computing: AIP Conference Proceedings. American Institute of Physics, New York, vol 151","DOI":"10.1063\/1.36278"},{"issue":"2","key":"209_CR19","doi-asserted-by":"crossref","first-page":"85","DOI":"10.3758\/BF03333113","volume":"16","author":"AH Klopf","year":"1988","unstructured":"Klopf AH (1988) A neuronal model of classical conditioning. Psychobiology 16(2): 85\u201323","journal-title":"Psychobiology"},{"key":"209_CR20","unstructured":"Kolodziejski C, Porr B, W\u00f6rg\u00f6tter F (2006) Fast, flexible and adaptive motor control achieved by pairing neuronal learning with recruitment. In: Proceedings of the fifteenth annual computational neuroscience meeting CNS*2006, Edinburgh"},{"key":"209_CR21","doi-asserted-by":"crossref","unstructured":"Kolodziejski C, Porr B, W\u00f6rg\u00f6tter F (2007) Anticipative adaptive muscle control: Forward modeling with self-induced disturbances and recruitment. In: Proceedings of the fifteenth annual computational neuroscience meeting CNS*2007, Toronto","DOI":"10.1186\/1471-2202-8-S2-P202"},{"key":"209_CR22","doi-asserted-by":"crossref","unstructured":"Kosco B (1986) Differential Hebbian learning. In: Denker JS (eds) Neural networks for computing: AIP Conference proceedings, American Institute of Physics, New York, vol 151","DOI":"10.1063\/1.36225"},{"key":"209_CR23","doi-asserted-by":"publisher","first-page":"197","DOI":"10.1385\/NI:3:3:197","volume":"3","author":"JL Krichmar","year":"2005","unstructured":"Krichmar JL, Seth AK, Nitz DA, Fleischer JG, Edelman GM (2005) Spatial navigation and causal analysis in a brain-based device modeling cortical-hippocampal interactions. Neuroinformatics 3: 197\u201322","journal-title":"Neuroinformatics"},{"key":"209_CR24","doi-asserted-by":"publisher","DOI":"10.1007\/s00422-007-0176-y","author":"T Kulvicius","year":"2007","unstructured":"Kulvicius T, Porr B, W\u00f6rg\u00f6tter F (2007) Chained learning architectures in a simple closed-loop behavioural context. Biol Cybern. doi:10.1007\/s00422-007-0176-y","journal-title":"Biol Cybern"},{"key":"209_CR25","doi-asserted-by":"publisher","first-page":"209","DOI":"10.1126\/science.275.5297.209","volume":"275","author":"JC Magee","year":"1997","unstructured":"Magee JC, Johnston D (1997) A synaptically controlled, associative signal for Hebbian plasticity in hippocampal neurons. Science 275: 209\u201313","journal-title":"Science"},{"issue":"7","key":"209_CR26","doi-asserted-by":"publisher","first-page":"e134","DOI":"10.1371\/journal.pcbi.0030134","volume":"3","author":"P Manoonpong","year":"2007","unstructured":"Manoonpong P, Geng T, Kulvicius T, Porr B, W\u00f6rg\u00f6tter F (2007) Adaptive, fast walking in a biped robot under neuronal control and learning. PLoS Comput Biol 3(7): e134","journal-title":"PLoS Comput Biol"},{"key":"209_CR27","doi-asserted-by":"publisher","first-page":"213","DOI":"10.1126\/science.275.5297.213","volume":"275","author":"H Markram","year":"1997","unstructured":"Markram H, L\u00fcbke J, Frotscher M, Sakmann B (1997) Regulation of synaptic efficacy by coincidence of postsynaptic APs and EPSPs. Science 275: 213\u201315","journal-title":"Science"},{"key":"209_CR28","doi-asserted-by":"publisher","first-page":"1255","DOI":"10.1016\/0024-3205(81)90231-9","volume":"29","author":"JD Miller","year":"1981","unstructured":"Miller JD, Sanghera MK, German DC (1981) Mesencephalic dopaminergic unit activity in the behaviorally conditioned rat. Life Sci 29: 1255\u2013263","journal-title":"Life Sci"},{"key":"209_CR29","doi-asserted-by":"publisher","first-page":"725","DOI":"10.1038\/377725a0","volume":"377","author":"PR Montague","year":"1995","unstructured":"Montague PR, Dayan P, Person C, Sejnowski TJ (1995) Bee foraging in uncertain environments using predictive hebbian learning. Nature 377: 725\u201328","journal-title":"Nature"},{"issue":"5","key":"209_CR30","doi-asserted-by":"crossref","first-page":"1936","DOI":"10.1523\/JNEUROSCI.16-05-01936.1996","volume":"16","author":"PR Montague","year":"1996","unstructured":"Montague PR, Dayan P, Sejnowski TJ (1996) A framework for mesencephalic dopamine systems based on predictive hebbian learning. J Neurosci 16(5): 1936\u2013947","journal-title":"J Neurosci"},{"key":"209_CR31","doi-asserted-by":"publisher","first-page":"1309","DOI":"10.1162\/neco.2006.18.6.1318","volume":"18","author":"JP Pfister","year":"2006","unstructured":"Pfister JP, Toyoizumi T, Barber D, Gerstner W (2006) Optimal spike-timing dependent plasticity for precise action potential firing in supervised learning. Neural Comput 18: 1309\u2013339","journal-title":"Neural Comput"},{"key":"209_CR32","doi-asserted-by":"publisher","first-page":"831","DOI":"10.1162\/08997660360581921","volume":"15","author":"B Porr","year":"2003","unstructured":"Porr B, W\u00f6rg\u00f6tter F (2003) Isotropic sequence order learning. Neural Comput 15: 831\u201364","journal-title":"Neural Comput"},{"key":"209_CR33","doi-asserted-by":"publisher","first-page":"1380","DOI":"10.1162\/neco.2006.18.6.1380","volume":"18","author":"B Porr","year":"2006","unstructured":"Porr B, W\u00f6rg\u00f6tter F (2006) Strongly improved stability and faster convergence of temporal sequence learning by utilising input correlations only. Neural Comput 18: 1380\u2013412","journal-title":"Neural Comput"},{"key":"209_CR34","doi-asserted-by":"crossref","unstructured":"Porr B, W\u00f6rg\u00f6tter F (2007) Learning with \u2018relevance\u2019 Using a third factor to stabilise hebbian learning. Neural Comput (in press)","DOI":"10.1162\/neco.2007.19.10.2694"},{"key":"209_CR35","doi-asserted-by":"publisher","first-page":"865","DOI":"10.1162\/08997660360581930","volume":"15","author":"B Porr","year":"2003","unstructured":"Porr B, von Ferber C, W\u00f6rg\u00f6tter F (2003) ISO-learning approximates a solution to the inverse-controller problem in an unsupervised behavioral paradigm. Neural Comput 15: 865\u201384","journal-title":"Neural Comput"},{"issue":"3","key":"209_CR36","doi-asserted-by":"publisher","first-page":"235","DOI":"10.1023\/A:1008910918445","volume":"7","author":"P Roberts","year":"1999","unstructured":"Roberts P (1999) Computational consequences of temporally asymmetric learning rules: I. differential hebbian learning. J Comput Neurosci 7(3): 235\u20136","journal-title":"J Comput Neurosci"},{"key":"209_CR37","doi-asserted-by":"crossref","unstructured":"Santiago RA, Roberts PD, Lafferriere G (2007) Spike timing dependent plasticity implements reinforcement learning. In: Proceedings of the fifteenth annual computational neuroscience meeting CNS*2007, Toronto","DOI":"10.1186\/1471-2202-8-S2-S16"},{"key":"209_CR38","doi-asserted-by":"publisher","first-page":"595","DOI":"10.1162\/089976604772744929","volume":"16","author":"A Saudargiene","year":"2004","unstructured":"Saudargiene A, Porr B, W\u00f6rg\u00f6tter F (2004) How the shape of pre- and postsynaptic signals can influence STDP: a biophysical model. Neural Comp 16: 595\u201326","journal-title":"Neural Comp"},{"key":"209_CR39","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1152\/jn.1998.80.1.1","volume":"80","author":"W Schultz","year":"1998","unstructured":"Schultz W (1998) Predictive reward signal of dopamine neurons. J Neurophysiol 80: 1\u20137","journal-title":"J Neurophysiol"},{"key":"209_CR40","doi-asserted-by":"publisher","first-page":"1593","DOI":"10.1126\/science.275.5306.1593","volume":"275","author":"W Schultz","year":"1997","unstructured":"Schultz W, Dayan P, Montague PR (1997) A neural substrate of prediction and reward. Science 275: 1593\u2013599","journal-title":"Science"},{"key":"209_CR41","first-page":"123","volume":"22","author":"SP Singh","year":"1996","unstructured":"Singh SP, Sutton RS (1996) Reinforcement learning with replacing eligibility traces. Mach Learn 22: 123\u201358","journal-title":"Mach Learn"},{"issue":"9","key":"209_CR42","doi-asserted-by":"publisher","first-page":"1125","DOI":"10.1016\/j.neunet.2005.08.012","volume":"18","author":"T Str\u00f6sslin","year":"2005","unstructured":"Str\u00f6sslin T, Sheynikhovich D, Chavarriaga R, Gerstner W (2005) Robust self-localisation and navigation based on hippocampal place cells. Neural Netw 18(9): 1125\u2013140","journal-title":"Neural Netw"},{"issue":"4-6","key":"209_CR43","doi-asserted-by":"publisher","first-page":"523","DOI":"10.1016\/S0893-6080(02)00046-1","volume":"15","author":"RE Suri","year":"2002","unstructured":"Suri RE (2002) TD models of reward predictive responses in Dopamine neurons. Neural Netw 15(4-6): 523\u201333","journal-title":"Neural Netw"},{"key":"209_CR44","doi-asserted-by":"publisher","first-page":"350","DOI":"10.1007\/s002210050467","volume":"121","author":"RE Suri","year":"1998","unstructured":"Suri RE, Schultz W (1998) Learning of sequential movements by neural network model with dopamine-like reinforcement signal. Exp Brain Res 121: 350\u201354","journal-title":"Exp Brain Res"},{"issue":"3","key":"209_CR45","doi-asserted-by":"publisher","first-page":"871","DOI":"10.1016\/S0306-4522(98)00697-6","volume":"91","author":"RE Suri","year":"1999","unstructured":"Suri RE, Schultz W (1999) A neural network model with dopamine-like reinforcement signal that learns a spatial delayed response task. Neurosci 91(3): 871\u201390","journal-title":"Neurosci"},{"issue":"4","key":"209_CR46","doi-asserted-by":"publisher","first-page":"841","DOI":"10.1162\/089976601300014376","volume":"13","author":"RE Suri","year":"2001","unstructured":"Suri RE, Schultz W (2001) Temporal difference model reproduces anticipatory neural activity. Neural Comp 13(4): 841\u20132","journal-title":"Neural Comp"},{"issue":"1","key":"209_CR47","doi-asserted-by":"publisher","first-page":"65","DOI":"10.1016\/S0306-4522(00)00554-6","volume":"103","author":"RE Suri","year":"2001","unstructured":"Suri RE, Bargas J, Arbib MA (2001) Modeling functions of striatal dopamine modulation in learning and planning. Neurosci 103(1): 65\u20135","journal-title":"Neurosci"},{"key":"209_CR48","doi-asserted-by":"publisher","first-page":"135","DOI":"10.1037\/0033-295X.88.2.135","volume":"88","author":"R Sutton","year":"1981","unstructured":"Sutton R, Barto A (1981) Towards a modern theory of adaptive networks: Expectation and prediction. Psychol Rev 88: 135\u201370","journal-title":"Psychol Rev"},{"key":"209_CR49","first-page":"9","volume":"3","author":"RS Sutton","year":"1988","unstructured":"Sutton RS (1988) Learning to predict by the methods of temporal differences. Mach Learn 3: 9\u20134","journal-title":"Mach Learn"},{"key":"209_CR50","volume-title":"Learning and computational neuroscience: foundation of adaptive networks","author":"RS Sutton","year":"1990","unstructured":"Sutton RS, Barto AG (1990) Time-derivative models of Pavlovian reinforcement. In: Gabriel M, Moore J(eds) Learning and computational neuroscience: foundation of adaptive networks. MIT Press, Cambridge"},{"key":"209_CR51","volume-title":"Reinforcement learning: an introduction","author":"RS Sutton","year":"1998","unstructured":"Sutton RS, Barto AG (1998) Reinforcement learning: an introduction, 2002nd edn. Bradford Books, MIT Press, Cambridge","edition":"2002"},{"issue":"3","key":"209_CR52","doi-asserted-by":"publisher","first-page":"665","DOI":"10.1113\/jphysiol.2002.033803","volume":"546","author":"M Tsukamoto","year":"2003","unstructured":"Tsukamoto M, Yasui T, Yamada MK, Nishiyama N, Matsuki N, Ikegaya Y (2003) Mossy fibre synaptic NMDA receptors trigger non-Hebbian long-term potentiation at entorhino-CA3 synapses in the rat. J Physiol 546(3): 665\u201375","journal-title":"J Physiol"},{"key":"209_CR53","unstructured":"Watkins CJCH (1989) Learning from delayed rewards. PhD thesis, University of Cambridge, Cambridge"},{"key":"209_CR54","first-page":"279","volume":"8","author":"CJCH Watkins","year":"1992","unstructured":"Watkins CJCH, Dayan P (1992) Technical note: Q-Learning. Mach Learn 8: 279\u201392","journal-title":"Mach Learn"},{"key":"209_CR55","doi-asserted-by":"publisher","first-page":"86","DOI":"10.1016\/S0019-9958(77)90354-0","volume":"34","author":"IH Witten","year":"1977","unstructured":"Witten IH (1977) An adaptive optimal controller for discrete-time Markov environments. Inf Control 34: 86\u201395","journal-title":"Inf Control"},{"key":"209_CR56","doi-asserted-by":"publisher","first-page":"245","DOI":"10.1162\/0899766053011555","volume":"17","author":"F W\u00f6rg\u00f6tter","year":"2005","unstructured":"W\u00f6rg\u00f6tter F, Porr B (2005) Temporal sequence learning for prediction and control - a review of different models and their relation to biological mechanisms. Neural Comput 17: 245\u201319","journal-title":"Neural Comput"}],"container-title":["Biological Cybernetics"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s00422-007-0209-6.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s00422-007-0209-6\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s00422-007-0209-6.pdf","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s00422-007-0209-6.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2022,2,5]],"date-time":"2022-02-05T05:10:49Z","timestamp":1644037849000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s00422-007-0209-6"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2008,1,15]]},"references-count":56,"journal-issue":{"issue":"3","published-print":{"date-parts":[[2008,3]]}},"alternative-id":["209"],"URL":"https:\/\/doi.org\/10.1007\/s00422-007-0209-6","relation":{},"ISSN":["0340-1200","1432-0770"],"issn-type":[{"value":"0340-1200","type":"print"},{"value":"1432-0770","type":"electronic"}],"subject":[],"published":{"date-parts":[[2008,1,15]]},"assertion":[{"value":"19 September 2007","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"19 December 2007","order":2,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"15 January 2008","order":3,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}}],"article-number":"259"}}