{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,10,17]],"date-time":"2025-10-17T13:43:53Z","timestamp":1760708633200,"version":"3.40.3"},"publisher-location":"Berlin, Heidelberg","reference-count":55,"publisher":"Springer Berlin Heidelberg","isbn-type":[{"type":"print","value":"9783642398742"},{"type":"electronic","value":"9783642398759"}],"license":[{"start":{"date-parts":[[2013,1,1]],"date-time":"2013-01-01T00:00:00Z","timestamp":1356998400000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2013,1,1]],"date-time":"2013-01-01T00:00:00Z","timestamp":1356998400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2013]]},"DOI":"10.1007\/978-3-642-39875-9_12","type":"book-chapter","created":{"date-parts":[[2013,11,13]],"date-time":"2013-11-13T14:29:24Z","timestamp":1384352964000},"page":"271-291","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":15,"title":["Divide and Conquer: Hierarchical Reinforcement Learning and Task Decomposition in Humans"],"prefix":"10.1007","author":[{"given":"Carlos","family":"Diuk","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Anna","family":"Schapiro","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Natalia","family":"C\u00f3rdova","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Jos\u00e9","family":"Ribas-Fernandes","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Yael","family":"Niv","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Matthew","family":"Botvinick","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2013,9,28]]},"reference":[{"issue":"7","key":"12_CR1","doi-asserted-by":"crossref","first-page":"2777","DOI":"10.1523\/JNEUROSCI.18-07-02777.1998","volume":"18","author":"J. W. Aldridge","year":"1998","unstructured":"Aldridge, J. W., & Berridge, K. C. (1998). Coding of serial order by neostriatal neurons: a \u201cnatural action\u201d approach to movement sequence. Journal of Neuroscience, 18(7), 2777\u20132787.","journal-title":"Journal of Neuroscience"},{"issue":"5","key":"12_CR2","doi-asserted-by":"publisher","first-page":"193","DOI":"10.1016\/j.tics.2008.02.004","volume":"12","author":"D. Badre","year":"2008","unstructured":"Badre, D. (2008). Cognitive control, hierarchy, and the rostro-caudal organization of the frontal lobes. Trends in Cognitive Sciences, 12(5), 193\u2013200.","journal-title":"Trends in Cognitive Sciences"},{"volume-title":"Intrinsically motivated learning in natural and artificial systems","year":"2012","key":"12_CR3","unstructured":"Baldassarre, G., & Mirolli, M. (Eds.), (2012). Intrinsically motivated learning in natural and artificial systems. Berlin: Springer."},{"issue":"4","key":"12_CR4","doi-asserted-by":"publisher","first-page":"341","DOI":"10.1023\/A:1025696116075","volume":"13","author":"A. G. Barto","year":"2003","unstructured":"Barto, A. G., & Mahadevan, S. (2003). Recent advances in hierarchical reinforcement learning. Discrete Event Dynamic Systems, 13(4), 341\u2013379.","journal-title":"Discrete Event Dynamic Systems"},{"issue":"2","key":"12_CR5","doi-asserted-by":"publisher","first-page":"395","DOI":"10.1037\/0033-295X.111.2.395","volume":"111","author":"M. Botvinick","year":"2004","unstructured":"Botvinick, M., & Plaut, D. C. (2004). Doing without schema hierarchies: a recurrent connectionist approach to normal and impaired routine sequential action. Psychological Review, 111(2), 395\u2013429.","journal-title":"Psychological Review"},{"issue":"3","key":"12_CR6","doi-asserted-by":"publisher","first-page":"262","DOI":"10.1016\/j.cognition.2008.08.011","volume":"113","author":"M. M. Botvinick","year":"2009","unstructured":"Botvinick, M. M., Niv, Y., Barto, A. C. (2009). Hierarchically organized behavior and its neural foundations: a reinforcement learning perspective. Cognition, 113(3), 262\u2013280.","journal-title":"Cognition"},{"key":"12_CR7","doi-asserted-by":"publisher","first-page":"1","DOI":"10.2307\/1127671","volume":"44","author":"J. Bruner","year":"1975","unstructured":"Bruner, J. (1975). Organization of early skilled action. Child Development, 44, 1\u201311.","journal-title":"Child Development"},{"issue":"12","key":"12_CR8","doi-asserted-by":"publisher","first-page":"539","DOI":"10.1016\/S1364-6613(00)01800-3","volume":"5","author":"C. M. Conway","year":"2001","unstructured":"Conway, C. M., & Christiansen, M. H. (2001). Sequential learning in non-human primates. Trends in Cognitive Sciences, 5(12), 539\u2013546.","journal-title":"Trends in Cognitive Sciences"},{"issue":"4","key":"12_CR9","doi-asserted-by":"publisher","first-page":"297","DOI":"10.1080\/026432900380427","volume":"17","author":"R. Cooper","year":"2000","unstructured":"Cooper, R., & Shallice, T. (2000). Contention scheduling and the control of routine activities. Cognitive Neuropsychology, 17(4), 297\u2013338.","journal-title":"Cognitive Neuropsychology"},{"key":"12_CR10","unstructured":"Daw, N.\u00a0D., Courville, A.\u00a0C., Touretzky, D.\u00a0S. (2003). Timing and partial observability in the dopamine system. In Advances in Neural Information Processing Systems (NIPS). Cambridge: MIT."},{"key":"12_CR11","first-page":"271","volume-title":"Advances in neural information processing systems 5","author":"P. Dayan","year":"1993","unstructured":"Dayan, P., & Hinton, G.\u00a0E. (1993). Feudal reinforcement learning. In Advances in neural information processing systems 5 (pp. 271\u2013278). San Mateo: Morgan Kaufmann."},{"key":"12_CR12","doi-asserted-by":"crossref","first-page":"227","DOI":"10.1613\/jair.639","volume":"13","author":"T. G. Dietterich","year":"2000","unstructured":"Dietterich, T. G. (2000). Hierarchical reinforcement learning with the MAXQ value function decomposition. Journal of Artificial Intelligence Research, 13, 227\u2013303.","journal-title":"Journal of Artificial Intelligence Research"},{"key":"12_CR13","unstructured":"Diuk, C., Cordova, N., Niv, Y., Botvinick, M. (2012a). Discovering hierarchical task structure. Submitted."},{"issue":"13","key":"12_CR14","doi-asserted-by":"publisher","first-page":"5797","DOI":"10.1523\/JNEUROSCI.5445-12.2013","volume":"33","author":"C. Diuk","year":"2012","unstructured":"Diuk, C., Tsai, K., Wallis, J., Niv, Y., Botvinick, M. M. (2012b). Hierarchical learning induces two simultaneous, but separable, prediction errors in human basal ganglia. The Journal of Neuroscience, 33(13), 5797\u20135805.","journal-title":"The Journal of Neuroscience"},{"issue":"2","key":"12_CR15","doi-asserted-by":"publisher","first-page":"249","DOI":"10.1109\/TEVC.2006.890270","volume":"11","author":"S. Elfwing","year":"2007","unstructured":"Elfwing, S., Uchibe, E., Doya, K., Christensen, H. I. (2007). Evolutionary development of hierarchical learning structures. IEEE Transactions on Evolutionary Computation, 11(2), 249\u2013264.","journal-title":"IEEE Transactions on Evolutionary Computation"},{"issue":"6","key":"12_CR16","doi-asserted-by":"publisher","first-page":"477","DOI":"10.1037\/0033-295X.87.6.477","volume":"87","author":"K. W. Fischer","year":"1980","unstructured":"Fischer, K. W. (1980). A theory of cognitive development: the control and construction of hierarchies of skills. Psychological Review, 87(6), 477\u2013537","journal-title":"Psychological Review"},{"key":"12_CR17","volume-title":"The prefrontal cortex: anatomy, physiology, and neuropsychology of the frontal lobe","author":"J. M. Fuster","year":"1997","unstructured":"Fuster, J.\u00a0M. (1997). The prefrontal cortex: anatomy, physiology, and neuropsychology of the frontal lobe, 3rd edn. Philadelphia: Lippincott-Raven.","edition":"3"},{"issue":"8","key":"12_CR18","doi-asserted-by":"publisher","first-page":"1242","DOI":"10.1016\/j.neunet.2006.06.007","volume":"19","author":"M. Haruno","year":"2006","unstructured":"Haruno, M., & Kawato, M. (2006). Heterarchical reinforcement-learning model for integration of multiple cortico-striatal loops: fMRI examination in stimulus-action-reward association learning. Neural networks: the official journal of the international neural network society, 19(8), 1242\u20131254.","journal-title":"Neural networks: the official journal of the international neural network society"},{"key":"12_CR19","unstructured":"Hengst, B. (2002). Discovering hierarchy in reinforcement learning with HEXQ. In Proceedings of the 19th international conference on machine learning, Sydney, Australia."},{"key":"12_CR20","volume-title":"Models of information processing in the basal ganglia","author":"J. Houk","year":"1995","unstructured":"Houk, J., Adams, J., Barto, A. (1995). A model of how the basal ganglia generate and use neural signals that predict reinforcement. In J. Houk, J. Davis, D. Beiser (Eds.), Models of information processing in the basal ganglia. Cambridge: MIT."},{"issue":"3","key":"12_CR21","doi-asserted-by":"publisher","first-page":"368","DOI":"10.1016\/j.conb.2011.04.001","volume":"21","author":"M. Ito","year":"2011","unstructured":"Ito, M., & Doya, K. (2011). Multiple representations and algorithms for reinforcement learning in the cortico-basal ganglia circuit. Current Opinion in Neurobiology, 21(3), 368\u2013373.","journal-title":"Current Opinion in Neurobiology"},{"key":"12_CR22","doi-asserted-by":"publisher","first-page":"535","DOI":"10.1016\/S0893-6080(02)00047-3","volume":"15","author":"D. Joel","year":"2002","unstructured":"Joel, D., Niv, Y., Ruppin, E. (2002). Actor\u2014critic models of the basal ganglia: new anatomical and computational perspectives. Neural Networks, 15, 535\u2013547.","journal-title":"Neural Networks"},{"key":"12_CR23","first-page":"2259","volume":"7","author":"A. Jonsson","year":"2006","unstructured":"Jonsson, A., & Barto, A. (2006). Causal graph based decomposition of factored MDPs. Journal of Machine Learning Research, 7, 2259\u20132301.","journal-title":"Journal of Machine Learning Research"},{"issue":"5648","key":"12_CR24","doi-asserted-by":"crossref","first-page":"1181","DOI":"10.1126\/science.1088545","volume":"302","author":"E. Koechlin","year":"2003","unstructured":"Koechlin, E., Ody, C., Kouneiher, F. (2003). The architecture of cognitive control in the human prefrontal cortex. Science (New York, N.Y.), 302(5648), 1181\u20131185.","journal-title":"Science (New York, N.Y.)"},{"key":"12_CR25","volume-title":"The problem of serial order in behavior","author":"K. S. Lashley","year":"1951","unstructured":"Lashley, K.\u00a0S. (1951). The problem of serial order in behavior. New York: Wiley"},{"key":"12_CR26","unstructured":"Li, L., Walsh, T.\u00a0J., Littman, M.\u00a0L. (2006). Towards a unified theory of state abstraction for MDPs. In Proceedings of the ninth international symposium on artificial intelligence and mathematics (AMAI-06)."},{"key":"12_CR27","unstructured":"McGovern, A., & Barto, A.\u00a0G. (2001). Automatic discovery of subgoals in reinforcement learning using diverse density. Proceedings of the 18th international conference on machine learning."},{"key":"12_CR28","doi-asserted-by":"crossref","unstructured":"Menache, I., Mannor, S., Shimkin, N. (2002). Q-cut-dynamic discovery of sub-goals in reinforcement learning. In European conference on machine learning (ECML 2002) (pp. 295\u2013306).","DOI":"10.1007\/3-540-36755-1_25"},{"key":"12_CR29","doi-asserted-by":"publisher","first-page":"167","DOI":"10.1146\/annurev.neuro.24.1.167","volume":"24","author":"E. K. Miller","year":"2001","unstructured":"Miller, E. K., & Cohen, J. D. (2001). An integrative theory of prefrontal cortex function. Annual Review of Neuroscience, 24, 167\u2013202.","journal-title":"Annual Review of Neuroscience"},{"key":"12_CR30","doi-asserted-by":"publisher","DOI":"10.1037\/10039-000","volume-title":"Plans and the structure of behavior","author":"G. A. Miller","year":"1960","unstructured":"Miller, G.\u00a0A., Galanter, E., Pribram, K.\u00a0H. (1960). Plans and the structure of behavior. New York: Adams-Bannister-Cox"},{"issue":"5","key":"12_CR31","doi-asserted-by":"crossref","first-page":"1936","DOI":"10.1523\/JNEUROSCI.16-05-01936.1996","volume":"16","author":"P. R. Montague","year":"1996","unstructured":"Montague, P. R., Dayan, P., Sejnowski, T. J. (1996). A framework for mesencephalic predictive hebbian learning. Journal of Neuroscience, 16(5), 1936\u20131947.","journal-title":"Journal of Neuroscience"},{"issue":"21","key":"12_CR32","doi-asserted-by":"crossref","first-page":"7931","DOI":"10.1523\/JNEUROSCI.23-21-07931.2003","volume":"23","author":"J. O\u2019Doherty","year":"2003","unstructured":"O\u2019Doherty, J., Critchley, H., Deichmann, R., Dolan, R. J. (2003). Dissociating valence of outcome from behavioral control in human orbital and ventral prefrontal cortices. The Journal of Neuroscience: The Official Journal of the Society for Neuroscience, 23(21), 7931\u20137939.","journal-title":"The Journal of Neuroscience: The Official Journal of the Society for Neuroscience"},{"key":"12_CR33","doi-asserted-by":"publisher","first-page":"245","DOI":"10.1016\/j.socnet.2010.03.006","volume":"32","author":"T. Opsahl","year":"2010","unstructured":"Opsahl, T., Agneessens, F., Skvoretz, J. (2010). Node centrality in weighted networks: generalizing degree and shortest paths. Social Networks, 32, 245\u2013251.","journal-title":"Social Networks"},{"key":"12_CR34","unstructured":"Parr, R., & Russell, S.\u00a0J. (1998). Reinforcement learning with hierarchies of machines. Advances in neural information processing systems."},{"key":"12_CR35","unstructured":"Picket, M., & Barto, A. (2002). Policyblocks: An algorithm for creating useful macro-actions in reinforcement learning. In Proceedings of the 19th International conference on machine learning."},{"issue":"2","key":"12_CR36","doi-asserted-by":"publisher","first-page":"370","DOI":"10.1016\/j.neuron.2011.05.042","volume":"71","author":"J. J. F. Ribas-Fernandes","year":"2011","unstructured":"Ribas-Fernandes, J. J. F., Solway, A., Diuk, C., McGuire, J. T., Barto, A. G., Niv, Y., Botvinick,\u00a0M.\u00a0M. (2011). A neural signature of hierarchical reinforcement learning. Neuron, 71(2), 370\u2013379.","journal-title":"Neuron"},{"key":"12_CR37","volume-title":"Scripts, plans, goals, and understanding: an inquiry into human knowledge structures","author":"R. C. Schank","year":"1977","unstructured":"Schank, R.\u00a0C., & Abelson, R.\u00a0P. (1977). Scripts, plans, goals, and understanding: an inquiry into human knowledge structures. Hillsdale: Lawrence Erlbaum."},{"key":"12_CR38","doi-asserted-by":"publisher","first-page":"486","DOI":"10.1038\/nn.3331","volume":"16","author":"A. Schapiro","year":"2013","unstructured":"Schapiro, A., Rogers, T., Cordova, N., Turk-Browne, N., Botvinick, M. (2013). Neural representations of events arise from temporal community structure. Nature Neuroscience, 16, 486\u2013492.","journal-title":"Nature Neuroscience"},{"key":"12_CR39","doi-asserted-by":"crossref","unstructured":"Schembri, M., Mirolli, M., Baldassarre, G. (2007a). Evolution and learning in an intrinsically motivated reinforcement learning robot. In F. Almeida\u00a0y Costa, L. M. Rocha, E. Costa, I. Harvey, A. Coutinho (Eds.), Advances in artificial life. Proceedings of the 9th European conference on artificial life. LNAI (vol. 4648, pp. 294\u2013333). Berlin: Springer.","DOI":"10.1007\/978-3-540-74913-4_30"},{"key":"12_CR40","first-page":"E1","volume-title":"Proceedings of the 6th international conference on development and learning","author":"M. Schembri","year":"2007","unstructured":"Schembri, M., Mirolli, M., Baldassarre, G. (2007b). Evolving internal reinforcers for an intrinsically motivated reinforcement-learning robot. In Y. Demiris, D. Mareschal, B. Scassellati, J.\u00a0Weng (Eds.), Proceedings of the 6th international conference on development and learning (pp. E1\u2013E6). London: Imperial College."},{"key":"12_CR41","doi-asserted-by":"crossref","unstructured":"Schmidhuber, J. (1991a). A possibility for implementing curiosity and boredom in model-building neural controllers. In Proceedings of the international conference on simulation of adaptive behavior: from animals to animats (pp. 222\u2013227).","DOI":"10.7551\/mitpress\/3115.003.0030"},{"key":"12_CR42","first-page":"1458","volume-title":"Proceedings of the International Conference on Neural Networks, 2","author":"J. Schmidhuber","year":"1991","unstructured":"Schmidhuber, J. (1991b). Curious model-building control systems. Proceedings of the International Conference on Neural Networks, 2, 1458\u20131463."},{"issue":"4","key":"12_CR43","doi-asserted-by":"publisher","first-page":"623","DOI":"10.1037\/0096-3445.135.4.623","volume":"135","author":"D. W. Schneider","year":"2006","unstructured":"Schneider, D. W. & Logan, G. D. (2006). Hierarchical control of cognitive processes: switching tasks in sequences. Journal of Experimental Psychology: General, 135(4), 623\u2013640.","journal-title":"Journal of Experimental Psychology: General"},{"issue":"5","key":"12_CR44","doi-asserted-by":"crossref","first-page":"1876","DOI":"10.1523\/JNEUROSCI.19-05-01876.1999","volume":"19","author":"G. Schoenbaum","year":"1999","unstructured":"Schoenbaum, G., Chiba, A.\u00a0A., Gallagher, M. (1999). Neural encoding in orbitofrontal cortex and basolateral amygdala during olfactory discrimination learning. The Journal of Neuroscience: The Official Journal of the Society for Neuroscience, 19(5), 1876\u201384.","journal-title":"The Journal of Neuroscience: The Official Journal of the Society for Neuroscience"},{"issue":"March 1997","key":"12_CR45","doi-asserted-by":"publisher","first-page":"1593","DOI":"10.1126\/science.275.5306.1593","volume":"275","author":"W. Schultz","year":"1997","unstructured":"Schultz, W., Dayan, P., Montague, P.\u00a0R. (1997). A neural substrate of prediction and reward. Science, 275(March 1997), 1593\u20131599.","journal-title":"Science"},{"issue":"3","key":"12_CR46","doi-asserted-by":"publisher","first-page":"272","DOI":"10.1093\/cercor\/10.3.272","volume":"10","author":"W. Schultz","year":"2000","unstructured":"Schultz, W., Tremblay, L., Hollerman, J.\u00a0R. (2000). Reward processing in primate orbitofrontal cortex and basal ganglia. Cerebral Cortex, 10(3), 272\u201384.","journal-title":"Cerebral Cortex"},{"key":"12_CR47","unstructured":"\u015eim\u015fek, O. (2008). Behavioral building blocks for autonomous agents: description, identification, and learning. PhD thesis, University of Massachussetts, Amherst."},{"key":"12_CR48","unstructured":"\u015eim\u015fek, O., Barto, A.\u00a0G. (2009). Skill Characterization Based on Betweenness. In D. Koller, D.\u00a0Schuurmans, Y. Bengio, L. Bottou (Eds.), Advances in neural information processing systems 21 (pp. 1497\u20131504)."},{"key":"12_CR49","doi-asserted-by":"crossref","unstructured":"\u015eim\u015fek, O., Wolfe, A.\u00a0P., Barto, A.\u00a0G. (2005). Identifying useful subgoals in reinforcement learning by local graph partitioning. In Proceedings of the twenty-second international conference on machine learning.","DOI":"10.1145\/1102351.1102454"},{"key":"12_CR50","unstructured":"Singh, S., Barto, A., & Chentanez, N. (2005). Proceedings of Advances in Neural Information Processing Systems 17."},{"key":"12_CR51","volume-title":"Reinforcement learning: an introduction","author":"R. S. Sutton","year":"1998","unstructured":"Sutton, R.\u00a0S., & Barto, A.\u00a0G. (1998). Reinforcement learning: an introduction. Cambridge: MIT."},{"key":"12_CR52","doi-asserted-by":"publisher","first-page":"181","DOI":"10.1016\/S0004-3702(99)00052-1","volume":"112","author":"R. S. Sutton","year":"1999","unstructured":"Sutton, R.\u00a0S., Precup, D., Singh, S. (1999). Between MDPs and semi-MDPs: a framework for temporal abstraction in reinforcement learning. Artificial Intelligence, 112, 181\u2013211.","journal-title":"Artificial Intelligence"},{"key":"12_CR53","volume-title":"Advances in neural information processing systems (NIPS) 7","author":"S. Thrun","year":"1995","unstructured":"Thrun, S., & Schwartz, A. (1995). Finding structure in reinforcement learning. In G. Tesauro, D. Touretzky, T. Leen (Eds.), Advances in neural information processing systems (NIPS) 7. Cambridge: MIT."},{"key":"12_CR54","first-page":"603","volume-title":"Proceedings of the 11th international joint conference on Artificial intelligence, Volume 1","author":"S. Yamada","year":"1989","unstructured":"Yamada, S., & Tsuji, S. (1989). Selective learning of macro-operators with perfect causality. In Proceedings of the 11th international joint conference on Artificial intelligence, Volume 1 (pp. 603\u2013608), San Francisco: Morgan Kaufmann Publishers Inc."},{"issue":"2","key":"12_CR55","doi-asserted-by":"publisher","first-page":"273","DOI":"10.1037\/0033-2909.133.2.273","volume":"133","author":"J. M. Zacks","year":"2007","unstructured":"Zacks, J. M., Speer, N. K., Swallow, K. M., Braver, T. S., Reynolds, J. R. (2007). Event perception: a mind-brain perspective. Psychological Bulletin, 133(2), 273\u2013293.","journal-title":"Psychological Bulletin"}],"container-title":["Computational and Robotic Models of the Hierarchical Organization of Behavior"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-642-39875-9_12","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,5,19]],"date-time":"2024-05-19T11:57:06Z","timestamp":1716119826000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-3-642-39875-9_12"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2013]]},"ISBN":["9783642398742","9783642398759"],"references-count":55,"URL":"https:\/\/doi.org\/10.1007\/978-3-642-39875-9_12","relation":{},"subject":[],"published":{"date-parts":[[2013]]},"assertion":[{"value":"28 September 2013","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}}]}}