{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,5,12]],"date-time":"2026-05-12T22:58:27Z","timestamp":1778626707104,"version":"3.51.4"},"reference-count":24,"publisher":"Springer Science and Business Media LLC","issue":"1","license":[{"start":{"date-parts":[[2015,1,1]],"date-time":"2015-01-01T00:00:00Z","timestamp":1420070400000},"content-version":"tdm","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["New Gener. Comput."],"published-print":{"date-parts":[[2015,1]]},"DOI":"10.1007\/s00354-015-0102-0","type":"journal-article","created":{"date-parts":[[2015,1,27]],"date-time":"2015-01-27T11:00:00Z","timestamp":1422356400000},"page":"69-114","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":40,"title":["Projective Simulation for Classical Learning Agents: A Comprehensive Investigation"],"prefix":"10.1007","volume":"33","author":[{"given":"Julian","family":"Mautner","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Adi","family":"Makmal","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Daniel","family":"Manzano","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Markus","family":"Tiersch","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Hans J.","family":"Briegel","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2015,1,28]]},"reference":[{"key":"102_CR1","doi-asserted-by":"crossref","unstructured":"Adam, S., Busoniu, L. and Babuska, R., \u201cExperience Replay for Real-Time Reinforcement Learning Control,\u201d in Systems, Man, and Cybernetics, Part C: Applications and Reviews, IEEE Transactions on, 42, pp. 201\u2013212, 2012.","DOI":"10.1109\/TSMCC.2011.2106494"},{"key":"102_CR2","doi-asserted-by":"crossref","unstructured":"Briegel, H. J. and De las Cuevas, G., \u201cProjective simulation for artificial intel-Ligence,\u201d in Sci. Rep. 2, 400, 2012.","DOI":"10.1038\/srep00400"},{"key":"102_CR3","doi-asserted-by":"crossref","unstructured":"Bull, L. and Kovacs, T. (Eds.), Foundations of Learning Classifier Systems, Studies in Fuzziness and Soft Computing, 183, Springer Berlin-Heidelberg, 2005.","DOI":"10.1007\/b100387"},{"key":"102_CR4","doi-asserted-by":"crossref","unstructured":"Butz, M. V., Shirinov, E. and Reif, K. L., \u201cSelf-Organizing Sensorimotor Maps Plus Internal Motivations Yield Animal-Like Behavior,\u201d in Adaptive Behavior, 18, pp. 315\u2013337, 2010.","DOI":"10.1177\/1059712310376842"},{"key":"102_CR5","doi-asserted-by":"crossref","unstructured":"Butz, M. V. and Wilson, S. W., \u201cAn Algorithmic Description of XCS,\u201d in Proc. IWLCS \u201900 Revised Papers from the Third International Workshop on Advances in Learning Classifier Systems, pp. 253\u2013272, Springer-Verlag London, U.K., 2001.","DOI":"10.1007\/3-540-44640-0_15"},{"key":"102_CR6","doi-asserted-by":"crossref","unstructured":"Dietterich, T. G., \u201cHierarchical reinforcement learning with the MAXQ value function decomposition,\u201d in Journal of Artificial Intelligence Research, 13, pp. 227\u2013303, 2000.","DOI":"10.1613\/jair.639"},{"key":"102_CR7","unstructured":"Floreano, D. and Mattiussi, C., Bio-inspired artificial intelligence: theories, methods, and technologies, Intelligent robotics and autonomous agents, MIT Press, Cambridge Massachusetts, 2008."},{"key":"102_CR8","unstructured":"Holland J. H., Adaptation in Natural and Artificial Systems, University of Michigan Press, 1975."},{"key":"102_CR9","doi-asserted-by":"crossref","unstructured":"Lin, L. J., \u201cSelf-improving reactive agents based on reinforcement learning, planning and teaching,\u201d in Machine Learning 8, pp. 292\u2013321, 1992.","DOI":"10.1007\/BF00992699"},{"key":"102_CR10","doi-asserted-by":"crossref","unstructured":"Ormoneit, D. and Sen, S., \u201cKernel-based reinforcement learning,\u201d in Machine Learning, 49, pp. 161178, 2002.","DOI":"10.1023\/A:1017928328829"},{"key":"102_CR11","doi-asserted-by":"crossref","DOI":"10.7551\/mitpress\/6979.001.0001","volume-title":"Understanding intelligence (First ed.)","author":"R. Pfeiffer","year":"1999","unstructured":"Pfeiffer R. and Scheier C. Understanding intelligence (First ed.). MIT Press, Cambridge Massachusetts, (1999)"},{"key":"102_CR12","unstructured":"Poole, D., Mackworth, A. and Goebel R., Computational intelligence: A logical approach, Oxford University Press, 1998."},{"key":"102_CR13","unstructured":"Parr, R. and Russell, S., \u201cReinforcement Learning with Hierarchies of Abstract Machines,\u201d in Advances in Neural Information Processing Systems 10, pp. 1043\u20131049, MIT Press, 1997."},{"key":"102_CR14","unstructured":"Russel, S. J. and Norvig, P., Artificial intelligence - A modern approach (Second ed.), Prentice Hall, New Jersey, 2003."},{"key":"102_CR15","unstructured":"Sutton, R. S., Temporal Credit Assignment in Reinforcement Learning, PhD Thesis, University of Massachusetts at Amherst, 1984."},{"key":"102_CR16","doi-asserted-by":"crossref","unstructured":"Sutton, R. S., \u201cIntegrated architectures for learning, planning, and reacting based on approximating dynamic programming,\u201d in Proc. of the Seventh International Conference on Machine Learning, Morgan Kaufmann, pp. 216\u2013224, 1990.","DOI":"10.1016\/B978-1-55860-141-3.50030-4"},{"key":"102_CR17","unstructured":"Sutton, R. S. and Barto, A. G., Reinforcement learning: An introduction (First edition), MIT Press, Cambridge Massachusetts, 1998."},{"key":"102_CR18","doi-asserted-by":"crossref","unstructured":"Sutton, R. S., Precup, D. and Singh, S., \u201cBetween MDPs and semi-MDPs: A Framework for Temporal Abstraction in Reinforcement Learning,\u201d in Artificial Intelligence, 112, pp. 181\u2013211, 1999.","DOI":"10.1016\/S0004-3702(99)00052-1"},{"key":"102_CR19","unstructured":"Sutton, R. S., Szepesvari, C., Geramifard, A. and Bowling, M., \u201cDyna-style planning with linear function approximation and prioritized sweeping,\u201d in Proc. of the 24th Conference on Uncertainty in Artificial Intelligence, pp. 528\u2013536, 2008."},{"key":"102_CR20","doi-asserted-by":"crossref","unstructured":"Toussaint, M., \u201cA sensorimotor map: Modulating lateral interactions for anticipation and planning,\u201d in Neural Computation 18, pp. 1132\u20131155, 2006.","DOI":"10.1162\/neco.2006.18.5.1132"},{"key":"102_CR21","doi-asserted-by":"crossref","unstructured":"Urbanowicz, R. J. and Moore, J. H., \u201cLearning Classifier Systems: A Complete Introduction, Review, and Roadmap,\u201d in Journal of Artificial Evolution and Applications, 2009, Article ID 736398, 2009. doi: 10.1155\/2009\/736398 .","DOI":"10.1155\/2009\/736398"},{"key":"102_CR22","unstructured":"Watkins, C. J. C. H., Learning from delayed rewards, PhD Thesis, University of Cambridge, England, 1989."},{"key":"102_CR23","doi-asserted-by":"crossref","unstructured":"Watkins, C. J. C. H and Dayan P., \u201cQ-learning\u201d in Machine Learning 8, 279\u2013292, 1992.","DOI":"10.1007\/BF00992698"},{"key":"102_CR24","doi-asserted-by":"crossref","unstructured":"Wilson S. W., \u201cClassifier Fitness Based on Accuracy,\u201d in Evol. Comput. 3(2), pp. 149\u2013175, 1995.","DOI":"10.1162\/evco.1995.3.2.149"}],"container-title":["New Generation Computing"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s00354-015-0102-0.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/article\/10.1007\/s00354-015-0102-0\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s00354-015-0102-0","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,5,17]],"date-time":"2025-05-17T20:59:05Z","timestamp":1747515545000},"score":1,"resource":{"primary":{"URL":"http:\/\/link.springer.com\/10.1007\/s00354-015-0102-0"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2015,1]]},"references-count":24,"journal-issue":{"issue":"1","published-print":{"date-parts":[[2015,1]]}},"alternative-id":["102"],"URL":"https:\/\/doi.org\/10.1007\/s00354-015-0102-0","relation":{},"ISSN":["0288-3635","1882-7055"],"issn-type":[{"value":"0288-3635","type":"print"},{"value":"1882-7055","type":"electronic"}],"subject":[],"published":{"date-parts":[[2015,1]]}}}