{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,5,31]],"date-time":"2025-05-31T05:06:53Z","timestamp":1748668013971},"reference-count":43,"publisher":"Springer Science and Business Media LLC","issue":"4","license":[{"start":{"date-parts":[[2014,4,26]],"date-time":"2014-04-26T00:00:00Z","timestamp":1398470400000},"content-version":"tdm","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Auton Agent Multi-Agent Syst"],"published-print":{"date-parts":[[2015,7]]},"DOI":"10.1007\/s10458-014-9262-4","type":"journal-article","created":{"date-parts":[[2014,4,25]],"date-time":"2014-04-25T17:31:14Z","timestamp":1398447074000},"page":"537-568","update-policy":"http:\/\/dx.doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":8,"title":["Emergence of emotional appraisal signals in reinforcement learning agents"],"prefix":"10.1007","volume":"29","author":[{"given":"Pedro","family":"Sequeira","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Francisco S.","family":"Melo","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Ana","family":"Paiva","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2014,4,26]]},"reference":[{"key":"9262_CR1","unstructured":"Ahn, H., & Picard, R. (2006). Affective cognitive learning and decision making: The role of emotions. Proceedings of the 18th European Meeting on Cybernetics and Systems Research, pp. 1\u20136."},{"key":"9262_CR2","doi-asserted-by":"crossref","unstructured":"Baird, L. (1993). Advantage updating. Technical Report WL-TR-93-1146, Wright Laboratory, Wright-Patterson Air Force Base.","DOI":"10.21236\/ADA280862"},{"issue":"3","key":"9262_CR3","doi-asserted-by":"crossref","first-page":"295","DOI":"10.1093\/cercor\/10.3.295","volume":"10","author":"A Bechara","year":"2000","unstructured":"Bechara, A., Damasio, H., & Damasio, A. (2000). Emotion, decision making and the orbitofrontal cortex. Cerebral Cortex, 10(3), 295\u2013307.","journal-title":"Cerebral Cortex"},{"key":"9262_CR4","unstructured":"Bratman, J., Singh, S., Lewis, R., & Sorg, J. (2012). Strong mitigation: Nesting search for good policies within search for good reward. Proceedings of the 11th International Joint Conference Autonomous Agents and Multiagent Systems."},{"key":"9262_CR5","doi-asserted-by":"crossref","unstructured":"Broekens, D., Kosters, W., & Verbeek, F. (2007). On affect and self-adaptation: Potential benefits of valence-controlled action-selection. Bio-inspired Modeling of Cognitive Tasks: Proceedings of the 2nd International Conference Interplay Between Natural and Artificial Computation, pp. 357\u2013366.","DOI":"10.1007\/978-3-540-73053-8_36"},{"issue":"3","key":"9262_CR6","doi-asserted-by":"crossref","first-page":"321","DOI":"10.1016\/S0149-7634(02)00007-6","volume":"26","author":"R Cardinal","year":"2002","unstructured":"Cardinal, R., Parkinson, J., Hall, J., & Everitt, B. (2002). Emotion and motivation: the role of the amygdala, ventral striatum, and prefrontal cortex. Neuroscience and Biobehavioral Reviews, 26(3), 321\u2013352.","journal-title":"Neuroscience and Biobehavioral Reviews"},{"key":"9262_CR7","volume-title":"Descartes\u2019 error: Emotion, reason, and the human brain","author":"A Damasio","year":"1994","unstructured":"Damasio, A. (1994). Descartes\u2019 error: Emotion, reason, and the human brain. New York: Putnam Publishing."},{"issue":"6","key":"9262_CR8","doi-asserted-by":"crossref","first-page":"883","DOI":"10.1668\/0003-1569(2000)040[0883:AMAAE]2.0.CO;2","volume":"40","author":"M Dawkins","year":"2000","unstructured":"Dawkins, M. (2000). Animal minds and animal emotions. American Zoologist, 40(6), 883\u2013888.","journal-title":"American Zoologist"},{"key":"9262_CR9","volume-title":"Handbook of the affective sciences","author":"P Ellsworth","year":"2003","unstructured":"Ellsworth, P., & Scherer, K. (2003). Appraisal processes in emotion. In R. Davidson, K. Scherer, & H. Goldsmith (Eds.), Handbook of the affective sciences. New York: Oxford University Press."},{"key":"9262_CR10","volume-title":"What develops in emotional development? (Emotions, personality, and psychotherapy)","author":"N Frijda","year":"1998","unstructured":"Frijda, N., & Mesquita, B. (1998). The analysis of emotions: dimensions of variation. In M. Mascolo & S. Griffin (Eds.), What develops in emotional development? (Emotions, personality, and psychotherapy). New York: Springer."},{"issue":"1","key":"9262_CR11","doi-asserted-by":"crossref","first-page":"42","DOI":"10.1177\/105971230200900102","volume":"9","author":"S Gadanho","year":"2001","unstructured":"Gadanho, S., & Hallam, J. (2001). Robot learning driven by emotions. Adaptive Behavior, 9(1), 42\u201364.","journal-title":"Adaptive Behavior"},{"key":"9262_CR12","doi-asserted-by":"crossref","unstructured":"Hester, T., Stone, P. (2012). Intrinsically motivated model learning for a developing curious agent. Procssdings of the IEEE International Conference on Development and Learning and Epigenetic Robotics, pp. 1\u20136. ICDL 2013.","DOI":"10.1109\/DevLrn.2012.6400802"},{"key":"9262_CR13","doi-asserted-by":"crossref","first-page":"237","DOI":"10.1613\/jair.301","volume":"4","author":"L Kaelbling","year":"1996","unstructured":"Kaelbling, L., Littman, M., & Moore, A. (1996). Reinforcement learning: A survey. Journal of Artificial Intelligence Research, 4, 237\u2013285.","journal-title":"Journal of Artificial Intelligence Research"},{"key":"9262_CR14","doi-asserted-by":"crossref","first-page":"99","DOI":"10.1016\/S0004-3702(98)00023-X","volume":"101","author":"L Kaelbling","year":"1998","unstructured":"Kaelbling, L., Littman, M., & Cassandra, A. (1998). Planning and acting in partially observable stochastic domains. Artificial Intelligence, 101, 99\u2013134.","journal-title":"Artificial Intelligence"},{"key":"9262_CR15","volume-title":"Genetic programming: On the programming of computers by means of natural selection","author":"J Koza","year":"1992","unstructured":"Koza, J. (1992). Genetic programming: On the programming of computers by means of natural selection. Cambridge: MIT Press."},{"key":"9262_CR16","volume-title":"Appraisal processes in emotion: Theory, methods, research","author":"R Lazarus","year":"2001","unstructured":"Lazarus, R. (2001). Relational meaning and discrete emotions. In K. Scherer, A. Schorr, & T. Johnstone (Eds.), Appraisal processes in emotion: Theory, methods, research. New York: Oxford University Press."},{"issue":"1","key":"9262_CR17","doi-asserted-by":"crossref","first-page":"155","DOI":"10.1146\/annurev.neuro.23.1.155","volume":"23","author":"J LeDoux","year":"2000","unstructured":"LeDoux, J. (2000). Emotion circuits in the brain. Annual Review of Neuroscience, 23(1), 155\u2013184.","journal-title":"Annual Review of Neuroscience"},{"issue":"1","key":"9262_CR18","doi-asserted-by":"crossref","first-page":"3","DOI":"10.1080\/02699938708408361","volume":"1","author":"H Leventhal","year":"1987","unstructured":"Leventhal, H., & Scherer, K. (1987). The relationship of emotion to cognition: A functional approach to a semantic controversy. Cognition and Emotion, 1(1), 3\u201328.","journal-title":"Cognition and Emotion"},{"key":"9262_CR19","doi-asserted-by":"crossref","unstructured":"Littman, M. (1994). Memoryless policies: Theoretical limitations and practical results. Proceedings of the 3rd International Conference Simulation of Adaptive Behavior-From Animals to Animats 3.","DOI":"10.7551\/mitpress\/3117.003.0041"},{"key":"9262_CR20","unstructured":"Lopes, M., Lang, T., Toussaint, M., & Oudeyer, P.-Y. (2012). Exploration in model-based reinforcement learning by empirically estimating learning progress. Advances in Neural Information Processing Systems 25, pp. 206\u2013214."},{"key":"9262_CR21","unstructured":"Marinier, R. (2008). A computational unification of cognitive control, emotion, and learning. Phd thesis, University of Michigan, Ann Arbor, MI."},{"issue":"1","key":"9262_CR22","doi-asserted-by":"crossref","first-page":"70","DOI":"10.1016\/j.cogsys.2008.03.005","volume":"10","author":"S Marsella","year":"2009","unstructured":"Marsella, S., & Gratch, J. (2009). Ema: A process model of appraisal dynamics. Cognitive Systems Research, 10(1), 70\u201390.","journal-title":"Cognitive Systems Research"},{"key":"9262_CR23","volume-title":"Blueprint for affective computing","author":"S Marsella","year":"2010","unstructured":"Marsella, S., Gratch, J., & Petta, P. (2010). Computational models of emotion. In K. Scherer, T. B\u00e4nziger, & E. Roesch (Eds.), Blueprint for affective computing. New York: Oxford University Press."},{"key":"9262_CR24","first-page":"103","volume":"13","author":"A Moore","year":"1993","unstructured":"Moore, A., & Atkeson, C. (1993). Prioritized sweeping: Reinforcement learning with less data and less real time. Machine Learning, 13, 103\u2013130.","journal-title":"Machine Learning"},{"issue":"2","key":"9262_CR25","doi-asserted-by":"crossref","first-page":"83","DOI":"10.1109\/TAMD.2010.2051436","volume":"2","author":"Scott Niekum","year":"2010","unstructured":"Niekum, Scott, Barto, Andrew G., & Spector, Lee. (2010). Genetic programming for reward function search. IEEE Transactions on Autonomous Mental Development, 2(2), 83\u201390.","journal-title":"IEEE Transactions on Autonomous Mental Development"},{"key":"9262_CR26","volume-title":"Understanding emotions","author":"K Oatley","year":"2006","unstructured":"Oatley, K., & Jenkins, J. (2006). Understanding emotions. Oxford: Wiley-Blackwell."},{"issue":"2","key":"9262_CR27","doi-asserted-by":"crossref","first-page":"175","DOI":"10.1016\/j.neuron.2005.09.025","volume":"48","author":"E Phelps","year":"2005","unstructured":"Phelps, E., & LeDoux, J. (2005). Contributions of the amygdala to emotion processing: From animal models to human behavior. Neuron, 48(2), 175\u2013187.","journal-title":"Neuron"},{"key":"9262_CR28","volume-title":"Appraisal processes in emotion: Theory, methods","author":"I Roseman","year":"2001","unstructured":"Roseman, I., & Smith, C. (2001). Appraisal theory: Overview, assumptions, varieties, controversies, research. In K. Scherer, A. Schorr, & T. Johnstone (Eds.), Appraisal processes in emotion: Theory, methods. Oxford: Oxford University Press."},{"issue":"1","key":"9262_CR29","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1007\/s10458-011-9166-5","volume":"25","author":"T Rumbell","year":"2011","unstructured":"Rumbell, T., Barnden, J., Denham, S., & Wennekers, T. (2011). Emotions in autonomous agents: Comparative analysis of mechanisms and functions. Autonomous Agents and Multiagent Systems, 25(1), 1\u201345.","journal-title":"Autonomous Agents and Multiagent Systems"},{"key":"9262_CR30","unstructured":"Salichs, M., Malfaz, M. (2006). Using emotions on autonomous agents: The role of happiness, sadness and fear. In Proceedings of the Annual Conference on Ambient Intelligence and Simulated Behavior, pp. 157\u2013164."},{"issue":"1","key":"9262_CR31","doi-asserted-by":"crossref","first-page":"56","DOI":"10.1109\/T-AFFC.2011.32","volume":"3","author":"M Salichs","year":"2012","unstructured":"Salichs, M., & Malfaz, M. (2012). A new approach to modeling emotions and their use on a decision-making system for artificial agents. IEEE Transactions on Affective Computing, 3(1), 56\u201368.","journal-title":"IEEE Transactions on Affective Computing"},{"key":"9262_CR32","doi-asserted-by":"crossref","DOI":"10.1093\/oso\/9780195130072.001.0001","volume-title":"Appraisal processes in emotion: Theory, methods","author":"K Scherer","year":"2001","unstructured":"Scherer, K. (2001). Appraisal considered as a process of multilevel sequential checking, research. In K. Scherer, A. Schorr, & T. Johnstone (Eds.), Appraisal processes in emotion: Theory, methods. Oxford: Oxford University Press."},{"key":"9262_CR33","doi-asserted-by":"crossref","unstructured":"Sequeira, P., Melo, F.S., & Paiva, A. (2011a). Emotion-based intrinsic motivation for reinforcement learning agents. Proceedings of the 4th International Conference Affective Computing and Intelligent Interaction, pp. 326\u2013336.","DOI":"10.1007\/978-3-642-24600-5_36"},{"key":"9262_CR34","doi-asserted-by":"crossref","unstructured":"Sequeira, P., Melo, F.S., Prada, R., & Paiva, A. (2011b). Emerging social awareness: Exploring intrinsic motivation in multiagent learning. Proceedings of the 1st IEEE International Joint Conference Development and Learning and Epigenetic Robotics, vol 2, pp. 1\u20136.","DOI":"10.1109\/DEVLRN.2011.6037325"},{"key":"9262_CR35","unstructured":"Sequeira, P., Melo, F.S., & Paiva, A. (2012). Learning by appraising: An emotion-based approach for intrinsic reward design. Technical, Report GAIPS-TR-001-12, INESC-ID."},{"issue":"1","key":"9262_CR36","doi-asserted-by":"crossref","first-page":"14","DOI":"10.1007\/s10458-009-9093-x","volume":"20","author":"M Si","year":"2010","unstructured":"Si, M., Marsella, S., & Pynadath, D. (2010). Modeling appraisal in theory of mind reasoning. Autonomous Agents and Multi-Agent Systems, 20(1), 14\u201331.","journal-title":"Autonomous Agents and Multi-Agent Systems"},{"key":"9262_CR37","doi-asserted-by":"crossref","unstructured":"Singh, S., Jaakkola, T., & Jordan, M. (1994). Learning without state estimation in partially observable Markovian decision processes. Proceedings of the 11th International Conference Machine Learning, pp. 284\u2013292.","DOI":"10.1016\/B978-1-55860-335-6.50042-8"},{"key":"9262_CR38","unstructured":"Singh, S., Lewis, R., & Barto, A. (2009). Where do rewards come from? Proceedings of the Annual Conference Cognitive Science Society, pp. 2601\u20132606."},{"issue":"2","key":"9262_CR39","doi-asserted-by":"crossref","first-page":"70","DOI":"10.1109\/TAMD.2010.2051031","volume":"2","author":"S Singh","year":"2010","unstructured":"Singh, S., Lewis, R., Barto, A., & Sorg, J. (2010). Intrinsically motivated reinforcement learning: An evolutionary perspective. IEEE Transactions on Autonomous Mental Development, 2(2), 70\u201382.","journal-title":"IEEE Transactions on Autonomous Mental Development"},{"key":"9262_CR40","unstructured":"Sorg, J., Singh, S., & Lewis, R. (2010a). Internal rewards mitigate agent boundedness. Proceedings of the 27th International Conference Machine Learning, pp. 1007\u20131014."},{"key":"9262_CR41","first-page":"1","volume":"23","author":"J Sorg","year":"2010","unstructured":"Sorg, J., Singh, S., & Lewis, R. (2010b). Reward design via online gradient ascent. Advances in Neural Information Processing Systems, 23, 1\u20139.","journal-title":"Advances in Neural Information Processing Systems"},{"key":"9262_CR42","volume-title":"Reinforcement learning: An introduction","author":"R Sutton","year":"1998","unstructured":"Sutton, R., & Barto, A. (1998). Reinforcement learning: An introduction. Cambridge: The MIT Press."},{"key":"9262_CR43","unstructured":"Syswerda, G. (1989). Uniform crossover in genetic algorithms. Proceedings of the 3rd International Conference Genetic Algorithms, pp. 2\u20139). CA, USA: San Francisco."}],"container-title":["Autonomous Agents and Multi-Agent Systems"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s10458-014-9262-4.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/article\/10.1007\/s10458-014-9262-4\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s10458-014-9262-4","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,5,26]],"date-time":"2024-05-26T05:46:55Z","timestamp":1716702415000},"score":1,"resource":{"primary":{"URL":"http:\/\/link.springer.com\/10.1007\/s10458-014-9262-4"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2014,4,26]]},"references-count":43,"journal-issue":{"issue":"4","published-print":{"date-parts":[[2015,7]]}},"alternative-id":["9262"],"URL":"https:\/\/doi.org\/10.1007\/s10458-014-9262-4","relation":{},"ISSN":["1387-2532","1573-7454"],"issn-type":[{"value":"1387-2532","type":"print"},{"value":"1573-7454","type":"electronic"}],"subject":[],"published":{"date-parts":[[2014,4,26]]}}}