{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2023,4,11]],"date-time":"2023-04-11T16:29:40Z","timestamp":1681230580923},"reference-count":54,"publisher":"Springer Science and Business Media LLC","issue":"4","license":[{"start":{"date-parts":[[2009,4,1]],"date-time":"2009-04-01T00:00:00Z","timestamp":1238544000000},"content-version":"tdm","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["Biol Cybern"],"published-print":{"date-parts":[[2009,4]]},"DOI":"10.1007\/s00422-009-0305-x","type":"journal-article","created":{"date-parts":[[2009,4,9]],"date-time":"2009-04-09T07:10:12Z","timestamp":1239261012000},"page":"319-330","source":"Crossref","is-referenced-by-count":3,"title":["Stimulus sampling as an exploration mechanism for fast reinforcement learning"],"prefix":"10.1007","volume":"100","author":[{"given":"Boris B.","family":"Vladimirskiy","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Eleni","family":"Vasilaki","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Robert","family":"Urbanczik","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Walter","family":"Senn","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2009,4,10]]},"reference":[{"key":"305_CR1","doi-asserted-by":"crossref","first-page":"1342","DOI":"10.1152\/jn.00553.2004","volume":"93","author":"N Aggelopoulos","year":"2005","unstructured":"Aggelopoulos N, Franco L, Rolls E (2005) Object perception in natural scenes: encoding by inferior temporal cortex simultaneously recorded neurons. J Neurophysiol 93: 1342\u20131357","journal-title":"J Neurophysiol"},{"key":"305_CR2","doi-asserted-by":"crossref","first-page":"1399","DOI":"10.1016\/S0896-6273(00)80658-3","volume":"21","author":"W Asaad","year":"1998","unstructured":"Asaad W, Rainer G, Miller E (1998) Neural activity in the primate prefrontal cortex during associative learning. Neuron 21: 1399\u20131407","journal-title":"Neuron"},{"key":"305_CR3","unstructured":"Barto A, Jordan M (1987) Gradient following without back-propagation in layered networks. In: Proceedings of the IEEE first annual conference on neural networks, vol 2. San Diego, pp 629\u201336"},{"key":"305_CR4","doi-asserted-by":"crossref","first-page":"129","DOI":"10.1016\/j.neuron.2005.05.020","volume":"47","author":"H Bayer","year":"2005","unstructured":"Bayer H, Glimcher P (2005) Midbrain dopamine neurons encode a quantitative reward prediction error signal. Neuron 47: 129\u2013141","journal-title":"Neuron"},{"key":"305_CR5","doi-asserted-by":"crossref","first-page":"5741","DOI":"10.1523\/JNEUROSCI.22-13-05741.2002","volume":"22","author":"P Bayley","year":"2002","unstructured":"Bayley P, Squire L (2002) Medial temporal lobe amnesia: gradual acquisition of factual information by nondeclarative memory. J Neurosci 22: 5741\u20135748","journal-title":"J Neurosci"},{"key":"305_CR6","doi-asserted-by":"crossref","first-page":"721","DOI":"10.1111\/j.0953-816X.2003.03181.x","volume":"19","author":"P Brasted","year":"2004","unstructured":"Brasted P, Wise S (2004) Comparison of learning-related neuronal activity in the dorsal premotor cortex and striatum. Eur J Neurosci 19: 721\u2013740","journal-title":"Eur J Neurosci"},{"key":"305_CR7","doi-asserted-by":"crossref","first-page":"9811","DOI":"10.1523\/JNEUROSCI.1532-04.2004","volume":"24","author":"C Buckmaster","year":"2004","unstructured":"Buckmaster C, Eichenbaum H, Amaral D, Suzuki W, Rapp P (2004) Entorhinal cortex lesions disrupt the relational organization of memory in monkeys. J Neurosci 24: 9811\u20139825","journal-title":"J Neurosci"},{"key":"305_CR8","doi-asserted-by":"crossref","first-page":"29","DOI":"10.1002\/hipo.450030104","volume":"3","author":"P Cahusac","year":"1993","unstructured":"Cahusac P, Rolls E, Miyashita Y, Niki H (1993) Modification of the responses of hippocampal neurons in the monkey during the learning of a conditional spatial response task. Hippocampus 3: 29\u201342","journal-title":"Hippocampus"},{"key":"305_CR9","first-page":"244","volume-title":"Advances in neural information processing systems, vol 5","author":"G Cauwenberghs","year":"1993","unstructured":"Cauwenberghs G (1993) A fast stochastic error-descent algorithm for supervised learning and optimization. In: Giles C, Hanson S, Cowan J (eds) Advances in neural information processing systems, vol 5. Morgan Kaufmann, San Mateo, pp 244\u2013251"},{"key":"305_CR10","doi-asserted-by":"crossref","first-page":"1101","DOI":"10.1152\/jn.1995.73.3.1101","volume":"73","author":"L Chen","year":"1995","unstructured":"Chen L, Wise S (1995a) Neuronal activity in the supplementary eye field during acquisition of conditional oculomotor associations. J Neurophysiol 73: 1101\u20131121","journal-title":"J Neurophysiol"},{"key":"305_CR11","doi-asserted-by":"crossref","first-page":"1122","DOI":"10.1152\/jn.1995.73.3.1122","volume":"73","author":"L Chen","year":"1995","unstructured":"Chen L, Wise S (1995b) Supplementary eye field contrasted with the frontal eye field during acquisition of conditional oculomotor associations. J Neurophysiol 73: 1122\u20131134","journal-title":"J Neurophysiol"},{"key":"305_CR12","doi-asserted-by":"crossref","first-page":"1137","DOI":"10.1016\/S0306-4522(98)00472-2","volume":"90","author":"D Chialvo","year":"1999","unstructured":"Chialvo D, Bak P (1999) Learning from mistakes. Neuroscience 90: 1137\u20131148","journal-title":"Neuroscience"},{"key":"305_CR13","doi-asserted-by":"crossref","first-page":"199","DOI":"10.1016\/j.conb.2006.03.006","volume":"16","author":"N Daw","year":"2006","unstructured":"Daw N, Doya K (2006) The computational neurobiology of learning and reward. Curr Opin Neurobiol 16: 199\u2013204","journal-title":"Curr Opin Neurobiol"},{"key":"305_CR14","doi-asserted-by":"crossref","first-page":"410","DOI":"10.1038\/nn2077","volume":"11","author":"K Doya","year":"2008","unstructured":"Doya K (2008) Modulators of decision making. Nat Neurosci 11: 410\u2013416","journal-title":"Nat Neurosci"},{"key":"305_CR15","doi-asserted-by":"crossref","first-page":"77","DOI":"10.1007\/978-1-4615-5351-9_8","volume-title":"Central auditory processing and neural modeling","author":"K Doya","year":"1998","unstructured":"Doya K, Sejnowski T (1998) A computational model of birdsong learning by auditory experience and auditory feedback. In: Brugge J, Poon P (eds) Central auditory processing and neural modeling. Plenum Press, New York, pp 77\u201388"},{"key":"305_CR16","doi-asserted-by":"crossref","first-page":"41","DOI":"10.1038\/35036213","volume":"1","author":"H Eichenbaum","year":"1999","unstructured":"Eichenbaum H (1999) Cortical-hippocampal networks for declarative memory. Nat Rev Neurosci 1: 41\u201350","journal-title":"Nat Rev Neurosci"},{"key":"305_CR17","doi-asserted-by":"crossref","first-page":"209","DOI":"10.1016\/S0896-6273(00)80773-4","volume":"23","author":"H Eichenbaum","year":"1999","unstructured":"Eichenbaum H, Dudchenko P, Wood E, Shapiro M, Tanila H (1999) The hippocampus, memory, and place cells: is it spatial memory or a memory space?. Neuron 23: 209\u2013226","journal-title":"Neuron"},{"key":"305_CR18","doi-asserted-by":"crossref","first-page":"048104","DOI":"10.1103\/PhysRevLett.97.048104","volume":"97","author":"I Fiete","year":"2006","unstructured":"Fiete I, Seung H (2006) Gradient learning in spiking neural networks by dynamic perturbation of conductances. Phys Rev Lett 97: 048104","journal-title":"Phys Rev Lett"},{"key":"305_CR19","first-page":"212","volume-title":"Advances in neural information processing systems, vol 5","author":"B Flower","year":"1993","unstructured":"Flower B, Jabri M (1993) Summed weight neuron perturbation: an $$ {\\mathcal{O}}(n) $$ improvement over weight perturbation. In: Giles C, Hanson S, Cowan J (eds) Advances in neural information processing systems, vol 5. Morgan Kaufmann, San Mateo, pp 212\u201319"},{"key":"305_CR20","volume-title":"The organization of behavior","author":"O Hebb","year":"1949","unstructured":"Hebb O (1949) The organization of behavior. Wiley, New York"},{"key":"305_CR21","volume-title":"Introduction to the theory of neural computation","author":"J Hertz","year":"1991","unstructured":"Hertz J, Krogh A, Palmer R (1991) Introduction to the theory of neural computation. Addison-Wesley, Redwood City"},{"key":"305_CR22","doi-asserted-by":"crossref","first-page":"154","DOI":"10.1109\/72.105429","volume":"3","author":"M Jabri","year":"1992","unstructured":"Jabri M, Flower B (1992) Weight perturbation: an optimal architecture and learning technique for analog VLSI feedforward and recurrent multilayered networks. IEEE Trans Neural Netw 3: 154\u2013157","journal-title":"IEEE Trans Neural Netw"},{"key":"305_CR23","doi-asserted-by":"crossref","first-page":"310","DOI":"10.1196\/annals.1390.003","volume":"1104","author":"Y Kobayashi","year":"2007","unstructured":"Kobayashi Y, Okada K (2007) Reward prediction error computation in the pedunculopontine tegmental nucleus neurons. Ann New York Acad Sci 1104: 310\u2013323","journal-title":"Ann New York Acad Sci"},{"key":"305_CR24","doi-asserted-by":"crossref","first-page":"4433","DOI":"10.1073\/pnas.88.10.4433","volume":"88","author":"P Mazzoni","year":"1991","unstructured":"Mazzoni P, Andersen R, Jordan M (1991) A more biologically plausible learning rule for neural networks. Proc Natl Acad Sci USA 88: 4433\u20134437","journal-title":"Proc Natl Acad Sci USA"},{"key":"305_CR25","doi-asserted-by":"crossref","first-page":"423","DOI":"10.1016\/S0166-2236(03)00177-2","volume":"26","author":"S McClure","year":"2003","unstructured":"McClure S, Daw N, Montague P (2003) A computational substrate for incentive salience. Trends Neurosci 26: 423\u2013428","journal-title":"Trends Neurosci"},{"key":"305_CR26","doi-asserted-by":"crossref","first-page":"1855","DOI":"10.1523\/JNEUROSCI.11-06-01855.1991","volume":"11","author":"A Mitz","year":"1991","unstructured":"Mitz A, Godschalk M, Wise S (1991) Learning-dependent neuronal activity in the premotor cortex: activity during the acquisition of conditional motor associations. J Neurosci 11: 1855\u20131872","journal-title":"J Neurosci"},{"key":"305_CR27","doi-asserted-by":"crossref","first-page":"725","DOI":"10.1038\/377725a0","volume":"377","author":"P Montague","year":"1995","unstructured":"Montague P, Dayan P, Person C, Sejnowski T (1995) Bee foraging in uncertain environments using predictive hebbian learning. Nature 377: 725\u2013728","journal-title":"Nature"},{"key":"305_CR28","doi-asserted-by":"crossref","first-page":"1936","DOI":"10.1523\/JNEUROSCI.16-05-01936.1996","volume":"16","author":"P Montague","year":"1996","unstructured":"Montague P, Dayan P, Sejnowski T (1996) A framework for mesencephalic dopamine systems based on predictive Hebbian learning. J Neurosci 16: 1936\u20131947","journal-title":"J Neurosci"},{"key":"305_CR29","doi-asserted-by":"crossref","first-page":"873","DOI":"10.1038\/nature03287","volume":"433","author":"A Pasupathy","year":"2005","unstructured":"Pasupathy A, Miller E (2005) Different time courses of learning-related activity in the prefrontal cortex and striatum. Nature 433: 873\u2013876","journal-title":"Nature"},{"key":"305_CR30","first-page":"64","volume-title":"Classical conditioning II: Current research and theory","author":"R Rescorla","year":"1972","unstructured":"Rescorla R, Wagner A (1972) A theory of Pavlovian conditioning: Variations in the effectiveness of reinforcement and nonreinforcement. In: Black AH, Prokasy WF (eds) Classical conditioning II: Current research and theory. Appleton-Century-Crofts, New York, pp 64\u201399"},{"key":"305_CR31","doi-asserted-by":"crossref","first-page":"4193","DOI":"10.1016\/j.visres.2006.07.026","volume":"46","author":"E Rolls","year":"2006","unstructured":"Rolls E, Franco L, Aggelopoulos N, Jerez J (2006) Information in the first spike, the order of spikes, and the number of spikes provided by neurons in the inferior temporal visual cortex. Vis Res 46: 4193\u20134205","journal-title":"Vis Res"},{"key":"305_CR32","doi-asserted-by":"crossref","first-page":"386","DOI":"10.1037\/h0042519","volume":"65","author":"F Rosenblatt","year":"1958","unstructured":"Rosenblatt F (1958) The perceptron: a probabilistic model for information storage and organization in the brain. Psychol Rev 65: 386\u2013408","journal-title":"Psychol Rev"},{"key":"305_CR33","first-page":"533","volume-title":"Mathematical perspectives on neural networks","author":"D Rumelhart","year":"1996","unstructured":"Rumelhart D, Durbin R, Golden R, Chauvin Y (1996) Backpropagation: the basic theory. In: Smolensky P, Mozer M, Rumelhart D (eds) Mathematical perspectives on neural networks. Lawrence Erlbaum Associates, Hillsdale, pp 533\u2013566"},{"key":"305_CR34","doi-asserted-by":"crossref","first-page":"12860","DOI":"10.1523\/JNEUROSCI.2496-07.2007","volume":"27","author":"T Sch\u00f6nberg","year":"2007","unstructured":"Sch\u00f6nberg T, Daw N, Joel D, O\u2019Doherty J (2007) Reinforcement learning signals in the human striatum distinguish learners from nonlearners during reward-based decision making. J Neurosci 27: 12860\u201312867","journal-title":"J Neurosci"},{"key":"305_CR35","doi-asserted-by":"crossref","first-page":"241","DOI":"10.1016\/S0896-6273(02)00967-4","volume":"36","author":"W Schultz","year":"2002","unstructured":"Schultz W (2002) Getting formal with dopamine and reward. Neuron 36: 241\u2013263","journal-title":"Neuron"},{"key":"305_CR36","doi-asserted-by":"crossref","first-page":"1593","DOI":"10.1126\/science.275.5306.1593","volume":"275","author":"W Schultz","year":"1997","unstructured":"Schultz W, Dayan P, Montague P (1997) A neural substrate of prediction and reward. Science 275: 1593\u20131599","journal-title":"Science"},{"key":"305_CR37","doi-asserted-by":"crossref","first-page":"321","DOI":"10.1016\/S0166-2236(03)00122-X","volume":"26","author":"W Schultz","year":"2003","unstructured":"Schultz W, Tremblay L, Hollerman J (2003) Changes in behavior-related neuronal activity in the striatum during learning. Trends Neurosci 26: 321\u2013328","journal-title":"Trends Neurosci"},{"key":"305_CR38","doi-asserted-by":"crossref","first-page":"061907","DOI":"10.1103\/PhysRevE.71.061907","volume":"71","author":"W Senn","year":"2005","unstructured":"Senn W, Fusi S (2005) Convergence of stochastic learning in perceptrons with binary synapses. Phys Rev E Stat Nonlinear Soft Matter Phys 71: 061907","journal-title":"Phys Rev E Stat Nonlinear Soft Matter Phys"},{"key":"305_CR39","doi-asserted-by":"crossref","first-page":"1063","DOI":"10.1016\/S0896-6273(03)00761-X","volume":"40","author":"H Seung","year":"2003","unstructured":"Seung H (2003) Learning in spiking neural networks by reinforcement of stochastic synaptic transmission. Neuron 40: 1063\u20131073","journal-title":"Neuron"},{"key":"305_CR40","doi-asserted-by":"crossref","first-page":"664","DOI":"10.1038\/nature02581","volume":"429","author":"B Seymour","year":"2004","unstructured":"Seymour B, O\u2019Doherty J, Dayan P, Koltzenburg M, Jones A, Dolan R, Friston K, Frackowiak R (2004) Temporal difference models describe higher-order learning in humans. Nature 429: 664\u2013667","journal-title":"Nature"},{"key":"305_CR41","doi-asserted-by":"crossref","first-page":"238","DOI":"10.1101\/lm.51802","volume":"9","author":"C Stark","year":"2002","unstructured":"Stark C, Bayley P, Squire L (2002) Recognition memory for single items and for associations is similarly impaired following damage to the hippocampal region. Learn Mem 9: 238\u2013242","journal-title":"Learn Mem"},{"key":"305_CR42","doi-asserted-by":"crossref","first-page":"281","DOI":"10.1002\/hipo.10085","volume":"13","author":"C Stark","year":"2003","unstructured":"Stark C, Squire L (2003) Hippocampal damage equally impairs memory for single items and memory for conjunctions. Hippocampus 13: 281\u2013292","journal-title":"Hippocampus"},{"key":"305_CR43","doi-asserted-by":"crossref","first-page":"135","DOI":"10.1037\/0033-295X.88.2.135","volume":"88","author":"R Sutton","year":"1981","unstructured":"Sutton R, Barto A (1981) Toward a modern theory of adaptive networks: expectation and prediction. Psychol Rev 88: 135\u2013170","journal-title":"Psychol Rev"},{"key":"305_CR44","volume-title":"Reinforcement learning: an introduction","author":"R Sutton","year":"1998","unstructured":"Sutton R, Barto A (1998) Reinforcement learning: an introduction. MIT Press, Cambridge"},{"key":"305_CR45","doi-asserted-by":"crossref","first-page":"842","DOI":"10.1002\/hipo.20321","volume":"17","author":"W Suzuki","year":"2007","unstructured":"Suzuki W (2007) Integrating associative learning signals across the brain. Hippocampus 17: 842\u2013850","journal-title":"Hippocampus"},{"key":"305_CR46","doi-asserted-by":"crossref","first-page":"376","DOI":"10.1126\/science.277.5324.376","volume":"277","author":"F Vargha-Khadem","year":"1997","unstructured":"Vargha-Khadem F, Gadian D, Watkins K, Connelly A, Van Paesschen W, Mishkin M (1997) Differential effects of early hippocampal pathology on episodic and semantic memory. Science 277: 376\u2013380","journal-title":"Science"},{"key":"305_CR47","doi-asserted-by":"crossref","first-page":"147","DOI":"10.1007\/s00422-008-0288-z","volume":"100","author":"E Vasilaki","year":"2009","unstructured":"Vasilaki E, Fusi S, Wang X, Senn W (2009) Learning flexible sensori-motor mappings in a complex network. Biol Cybern 100: 147\u2013158","journal-title":"Biol Cybern"},{"key":"305_CR48","doi-asserted-by":"crossref","first-page":"2699","DOI":"10.1162\/089976605774320539","volume":"17","author":"J Werfel","year":"2005","unstructured":"Werfel J, Xie X, Seung H (2005) Learning curves for stochastic gradient descent in linear feedforward networks. Neural Comput 17: 2699\u20132718","journal-title":"Neural Comput"},{"key":"305_CR49","doi-asserted-by":"crossref","first-page":"8181","DOI":"10.1523\/JNEUROSCI.1671-07.2007","volume":"27","author":"J Wickens","year":"2007","unstructured":"Wickens J, Horvitz J, Costa R, Killcross S (2007) Dopaminergic mechanisms in actions and habits. J Neurosci 27: 8181\u20138183","journal-title":"J Neurosci"},{"key":"305_CR50","doi-asserted-by":"crossref","first-page":"1415","DOI":"10.1109\/5.58323","volume":"78","author":"B Widrow","year":"1990","unstructured":"Widrow B, Lehr M (1990) Thirty years of adaptive neural networks: perceptron, madaline, and backpropagation. Proc IEEE 78: 1415\u20131442","journal-title":"Proc IEEE"},{"key":"305_CR51","first-page":"229","volume":"8","author":"R Williams","year":"1992","unstructured":"Williams R (1992) Simple statistical gradient-following algorithms for connectionist reinforcement learning. Mach Learn 8: 229\u2013256","journal-title":"Mach Learn"},{"key":"305_CR52","doi-asserted-by":"crossref","first-page":"562","DOI":"10.1038\/nn1662","volume":"9","author":"Z Williams","year":"2006","unstructured":"Williams Z, Eskandar E (2006) Selective enhancement of associative learning by microstimulation of the anterior caudate. Nat Neurosci 9: 562\u2013568","journal-title":"Nat Neurosci"},{"key":"305_CR53","doi-asserted-by":"crossref","first-page":"1578","DOI":"10.1126\/science.1084324","volume":"300","author":"S Wirth","year":"2003","unstructured":"Wirth S, Yanike M, Frank L, Smith A, Brown W, Suzuki W (2003) Single neurons in the monkey hippocampus and learning of new associations. Science 300: 1578\u20131581","journal-title":"Science"},{"key":"305_CR54","doi-asserted-by":"crossref","first-page":"041909","DOI":"10.1103\/PhysRevE.69.041909","volume":"69","author":"X Xie","year":"2004","unstructured":"Xie X, Seung H (2004) Learning in neural networks by reinforcement of irregular spiking. Phys Rev E Stat Nonlinear Soft Matter Phys 69: 041909","journal-title":"Phys Rev E Stat Nonlinear Soft Matter Phys"}],"container-title":["Biological Cybernetics"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s00422-009-0305-x.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/article\/10.1007\/s00422-009-0305-x\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s00422-009-0305-x","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2019,5,28]],"date-time":"2019-05-28T16:31:11Z","timestamp":1559061071000},"score":1,"resource":{"primary":{"URL":"http:\/\/link.springer.com\/10.1007\/s00422-009-0305-x"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2009,4]]},"references-count":54,"journal-issue":{"issue":"4","published-print":{"date-parts":[[2009,4]]}},"alternative-id":["305"],"URL":"https:\/\/doi.org\/10.1007\/s00422-009-0305-x","relation":{},"ISSN":["0340-1200","1432-0770"],"issn-type":[{"value":"0340-1200","type":"print"},{"value":"1432-0770","type":"electronic"}],"subject":[],"published":{"date-parts":[[2009,4]]}}}