{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,5,16]],"date-time":"2026-05-16T09:10:25Z","timestamp":1778922625887,"version":"3.51.4"},"reference-count":101,"publisher":"Elsevier BV","issue":"4","license":[{"start":{"date-parts":[[2026,4,1]],"date-time":"2026-04-01T00:00:00Z","timestamp":1775001600000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.elsevier.com\/tdm\/userlicense\/1.0\/"},{"start":{"date-parts":[[2026,4,1]],"date-time":"2026-04-01T00:00:00Z","timestamp":1775001600000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.elsevier.com\/legal\/tdmrep-license"},{"start":{"date-parts":[[2026,2,20]],"date-time":"2026-02-20T00:00:00Z","timestamp":1771545600000},"content-version":"vor","delay-in-days":0,"URL":"http:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"funder":[{"DOI":"10.13039\/100018693","name":"Horizon Europe","doi-asserted-by":"publisher","award":["952026"],"award-info":[{"award-number":["952026"]}],"id":[{"id":"10.13039\/100018693","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100000780","name":"European Commission","doi-asserted-by":"publisher","id":[{"id":"10.13039\/501100000780","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["cell.com","elsevier.com","sciencedirect.com"],"crossmark-restriction":true},"short-container-title":["Patterns"],"published-print":{"date-parts":[[2026,4]]},"DOI":"10.1016\/j.patter.2026.101513","type":"journal-article","created":{"date-parts":[[2026,4,10]],"date-time":"2026-04-10T15:47:11Z","timestamp":1775836031000},"page":"101513","update-policy":"https:\/\/doi.org\/10.1016\/elsevier_cm_policy","source":"Crossref","is-referenced-by-count":0,"title":["Intentional policy graphs: A pipeline for explaining agent behavior through intentions"],"prefix":"10.1016","volume":"7","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-4514-6145","authenticated-orcid":false,"given":"Victor","family":"Gimenez-Abalos","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-9946-9703","authenticated-orcid":false,"given":"Sergio","family":"Alvarez-Napagao","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-1658-9393","authenticated-orcid":false,"given":"Adrian","family":"Tormos","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0009-0000-6446-7871","authenticated-orcid":false,"given":"Sara","family":"Montese","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-0192-3096","authenticated-orcid":false,"given":"Ulises","family":"Cort\u00e9s","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-1732-9446","authenticated-orcid":false,"given":"Javier","family":"V\u00e1zquez-Salceda","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"78","reference":[{"key":"10.1016\/j.patter.2026.101513_bib1","doi-asserted-by":"crossref","first-page":"726","DOI":"10.1109\/TETCI.2021.3100641","article-title":"A Survey on Neural Network Interpretability","volume":"5","author":"Zhang","year":"2021","journal-title":"IEEE Trans. Emerg. Top. Comput. Intell."},{"key":"10.1016\/j.patter.2026.101513_bib2","doi-asserted-by":"crossref","first-page":"31","DOI":"10.1145\/3236386.3241340","article-title":"The mythos of model interpretability: In machine learning, the concept of interpretability is both important and slippery","volume":"16","author":"Lipton","year":"2018","journal-title":"ACM Queue"},{"key":"10.1016\/j.patter.2026.101513_bib3","series-title":"Philosophical Papers","first-page":"214","article-title":"Causal explanation","author":"Lewis","year":"1986"},{"key":"10.1016\/j.patter.2026.101513_bib4","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1016\/j.artint.2018.07.007","article-title":"Explanation in artificial intelligence: Insights from the social sciences","volume":"267","author":"Miller","year":"2019","journal-title":"Artif. Intell."},{"key":"10.1016\/j.patter.2026.101513_bib5","series-title":"Explanation and Understanding","author":"von Wright","year":"2004"},{"key":"10.1016\/j.patter.2026.101513_bib6","doi-asserted-by":"crossref","first-page":"219","DOI":"10.1002\/ejsp.2420230302","article-title":"Attribution in conversational context: Effect of mutual knowledge on explanation-giving","volume":"23","author":"Slugoski","year":"1993","journal-title":"Eur. J. Soc. Psychol."},{"key":"10.1016\/j.patter.2026.101513_bib7","series-title":"Speech Acts","first-page":"41","article-title":"Logic and Conversation","author":"Grice","year":"1975"},{"key":"10.1016\/j.patter.2026.101513_bib8","series-title":"Proceedings of the 36th AAAI Conference on Artificial Intelligence 36","first-page":"9623","article-title":"Do Feature Attribution Methods Correctly Attribute Features?","author":"Zhou","year":"2022"},{"key":"10.1016\/j.patter.2026.101513_bib9","series-title":"Advances in Neural Information Processing Systems 34","first-page":"9391","article-title":"Reliable Post hoc Explanations: Modeling Uncertainty in Explainability","author":"Slack","year":"2021"},{"key":"10.1016\/j.patter.2026.101513_bib10","series-title":"2022 IEEE International Conference on Fuzzy Systems (FUZZ-IEEE)","first-page":"1","article-title":"Focus! Rating XAI Methods and Finding Biases","author":"Arias-Duart","year":"2022"},{"key":"10.1016\/j.patter.2026.101513_bib11","doi-asserted-by":"crossref","first-page":"52138","DOI":"10.1109\/ACCESS.2018.2870052","article-title":"Peeking Inside the Black-Box: A Survey on Explainable Artificial Intelligence (XAI)","volume":"6","author":"Adadi","year":"2018","journal-title":"IEEE Access"},{"key":"10.1016\/j.patter.2026.101513_bib12","first-page":"2","article-title":"How is ChatGPT\u2019s behavior changing over time? Harvard Data","volume":"6","author":"Chen","year":"2024","journal-title":"Science Review"},{"key":"10.1016\/j.patter.2026.101513_bib13","doi-asserted-by":"crossref","first-page":"45","DOI":"10.1007\/s12559-023-10179-8","article-title":"Interpreting Black-Box Models: A Review on Explainable Artificial Intelligence","volume":"16","author":"Hassija","year":"2024","journal-title":"Cognit. Comput."},{"key":"10.1016\/j.patter.2026.101513_bib14","doi-asserted-by":"crossref","DOI":"10.1016\/j.inffus.2024.102301","article-title":"Explainable Artificial Intelligence (XAI) 2.0: A manifesto of open challenges and interdisciplinary research directions","volume":"106","author":"Longo","year":"2024","journal-title":"Inf. Fusion"},{"key":"10.1016\/j.patter.2026.101513_bib15","series-title":"Proceedings of the Third International Workshop on Agent Theories, Architectures, and Languages","first-page":"21","article-title":"Is It an agent, or just a program?: A taxonomy for autonomous agents","author":"Franklin","year":"1997"},{"key":"10.1016\/j.patter.2026.101513_bib16","doi-asserted-by":"crossref","first-page":"243","DOI":"10.2307\/1416950","article-title":"An Experimental Study of Apparent Behavior","volume":"57","author":"Heider","year":"1944","journal-title":"Am. J. Psychol."},{"key":"10.1016\/j.patter.2026.101513_bib17","series-title":"IJCAI 2016 Ethics for AI Workshop","first-page":"6","article-title":"What does the robot think? transparency as a fundamental design requirement for intelligent systems","author":"Wortham","year":"2016"},{"key":"10.1016\/j.patter.2026.101513_bib18","doi-asserted-by":"crossref","first-page":"4","DOI":"10.1007\/s10676-022-09624-3","article-title":"A sociotechnical perspective for the future of AI: narratives, inequalities, and human control","volume":"24","author":"Sartori","year":"2022","journal-title":"Ethics Inf. Technol."},{"key":"10.1016\/j.patter.2026.101513_bib19","series-title":"CD-MAKE 2020: International Cross-Domain Conference for Machine Learning and Knowledge Extraction","first-page":"1","article-title":"Explainable Artificial Intelligence: Concepts, Applications, Research Challenges and Visions","author":"Longo","year":"2020"},{"key":"10.1016\/j.patter.2026.101513_bib20","series-title":"HRI\u201917: Proceedings of the 2017 ACM\/IEEE International Conference on Human-Robot Interaction","first-page":"303","article-title":"Improving Robot Controller Transparency Through Autonomous Policy Explanation","author":"Hayes","year":"2017"},{"key":"10.1016\/j.patter.2026.101513_bib21","series-title":"CD-MAKE 2020: International Cross-Domain Conference for Machine Learning and Knowledge Extraction","first-page":"77","article-title":"Explainable reinforcement learning: A survey","author":"Puiutta","year":"2020"},{"key":"10.1016\/j.patter.2026.101513_bib22","doi-asserted-by":"crossref","first-page":"34","DOI":"10.1007\/s10458-023-09615-8","article-title":"A novel policy-graph approach with natural language and counterfactual abstractions for explaining reinforcement learning agents","volume":"37","author":"Liu","year":"2023","journal-title":"Auton. Agent. Multi. Agent. Syst."},{"key":"10.1016\/j.patter.2026.101513_bib23","series-title":"Proceedings of the First International Workshop on Citizen-Centric Multi-Agent Systems (CMAS\u201923)","first-page":"42","article-title":"Explainable agents adapt to human behaviour","author":"Tormos","year":"2023"},{"key":"10.1016\/j.patter.2026.101513_bib24","doi-asserted-by":"crossref","first-page":"573","DOI":"10.3390\/electronics13030573","article-title":"Explaining the Behaviour of Reinforcement Learning Agents in a Multi-Agent Cooperative Environment Using Policy Graphs","volume":"13","author":"Dom\u00e8nech i Vila","year":"2024","journal-title":"Electronics"},{"key":"10.1016\/j.patter.2026.101513_bib25","series-title":"Proceedings of the 24th International Conference on Autonomous Agents and Multiagent Systems AAMAS \u201925 (International Foundation for Autonomous Agents and Multiagent Systems)","first-page":"904","article-title":"Policy graphs and intention: Answering \u2019why\u2019 and \u2019how\u2019 from a telic perspective","author":"Gimenez-Abalos","year":"2025"},{"key":"10.1016\/j.patter.2026.101513_bib26","series-title":"Explainable, Trustworthy, and Responsible AI and Multi-Agent Systems","first-page":"40","article-title":"Explaining Autonomous Vehicles with Intention-Aware Policy Graphs","author":"Montese","year":"2026"},{"key":"10.1016\/j.patter.2026.101513_bib27","doi-asserted-by":"crossref","first-page":"26","DOI":"10.1016\/j.cogsys.2017.02.002","article-title":"Communicating intent to develop shared situation awareness and engender trust in human-agent teams","volume":"46","author":"Schaefer","year":"2017","journal-title":"Cogn. Syst. Res."},{"key":"10.1016\/j.patter.2026.101513_bib28","series-title":"2019 14th ACM\/IEEE International Conference on Human-Robot Interaction (HRI)","first-page":"751","article-title":"Improving Human-Robot Interaction Through Explainable Reinforcement Learning","author":"Tabrez","year":"2019"},{"key":"10.1016\/j.patter.2026.101513_bib29","first-page":"50","article-title":"European Union regulations on algorithmic decision-making and a \u201cright to explanation\u201d","volume":"38","author":"Goodman","year":"2017","journal-title":"AI Mag."},{"key":"10.1016\/j.patter.2026.101513_bib30","doi-asserted-by":"crossref","DOI":"10.1016\/j.artint.2021.103525","article-title":"Levels of explainable artificial intelligence for human-aligned conversational explanations","volume":"299","author":"Dazeley","year":"2021","journal-title":"Artif. Intell."},{"key":"10.1016\/j.patter.2026.101513_bib31","doi-asserted-by":"crossref","first-page":"285","DOI":"10.1016\/S0004-3702(83)80014-9","article-title":"XPLAIN: a system for creating and explaining expert consulting programs","volume":"21","author":"Swartout","year":"1983","journal-title":"Artif. Intell."},{"key":"10.1016\/j.patter.2026.101513_bib32","doi-asserted-by":"crossref","first-page":"351","DOI":"10.1016\/0025-5564(75)90047-4","article-title":"A model of inexact reasoning in medicine","volume":"23","author":"Shortliffe","year":"1975","journal-title":"Math. Biosci."},{"key":"10.1016\/j.patter.2026.101513_bib33","doi-asserted-by":"crossref","first-page":"15","DOI":"10.1016\/0004-3702(77)90003-0","article-title":"Production rules as a representation for a knowledge-based consultation program","volume":"8","author":"Davis","year":"1977","journal-title":"Artif. Intell."},{"key":"10.1016\/j.patter.2026.101513_bib34","series-title":"Expert systems: the user interface","first-page":"219","article-title":"Explanation: the role of control strategies and deep models","author":"Chandrasekaran","year":"1987"},{"key":"10.1016\/j.patter.2026.101513_bib35","series-title":"IEEE Standard for Transparency of Autonomous Systems. IEEE Std 7001-2021","first-page":"1","year":"2022"},{"key":"10.1016\/j.patter.2026.101513_bib36","first-page":"841","article-title":"Counterfactual explanations without opening the black box: Automated decisions and the GDPR","volume":"31","author":"Wachter","year":"2017","journal-title":"Harv. J. Law Technol."},{"key":"10.1016\/j.patter.2026.101513_bib37","doi-asserted-by":"crossref","first-page":"76","DOI":"10.1093\/idpl\/ipx005","article-title":"Why a right to explanation of automated decision-making does not exist in the general data protection regulation","volume":"7","author":"Wachter","year":"2017","journal-title":"Int. Data Priv. Law"},{"key":"10.1016\/j.patter.2026.101513_bib38","doi-asserted-by":"crossref","first-page":"593","DOI":"10.1007\/s40804-021-00224-0","article-title":"Trustworthy AI and Corporate Governance: The EU\u2019s Ethics Guidelines for Trustworthy Artificial Intelligence from a Company Law Perspective","volume":"22","author":"Hickman","year":"2021","journal-title":"Eur. Bus. Org. Law Rev."},{"key":"10.1016\/j.patter.2026.101513_bib39","series-title":"Theories in Social Psychology","first-page":"93","article-title":"Attribution theories: How people make sense of behavior","author":"Malle","year":"2022"},{"key":"10.1016\/j.patter.2026.101513_bib40","series-title":"Proceedings of the 41st International Conference on Machine Learning 235","first-page":"3996","article-title":"Position: Explain to question not to justify","author":"Biecek","year":"2024"},{"key":"10.1016\/j.patter.2026.101513_bib41","first-page":"1","article-title":"Quantus: An Explainable AI Toolkit for Responsible Evaluation of Neural Network Explanations and Beyond","volume":"24","author":"Hedstr\u00f6m","year":"2023","journal-title":"J. Mach. Learn. Res."},{"key":"10.1016\/j.patter.2026.101513_bib42","doi-asserted-by":"crossref","first-page":"2104","DOI":"10.1109\/TPAMI.2023.3331846","article-title":"Towards human-centered explainable AI: A survey of user studies for model explanations","volume":"46","author":"Rong","year":"2024","journal-title":"IEEE Trans. Pattern Anal. Mach. Intell."},{"key":"10.1016\/j.patter.2026.101513_bib43","series-title":"Proceedings of the 23rd International Conference on Autonomous Agents and Multiagent Systems AAMAS \u201924","first-page":"2706","article-title":"The Cognitive Hourglass: Agent Abstractions in the Large Models Era","author":"Ricci","year":"2024"},{"key":"10.1016\/j.patter.2026.101513_bib44","doi-asserted-by":"crossref","first-page":"206","DOI":"10.1038\/s42256-019-0048-x","article-title":"Stop explaining black box machine learning models for high stakes decisions and use interpretable models instead","volume":"1","author":"Rudin","year":"2019","journal-title":"Nat. Mach. Intell."},{"key":"10.1016\/j.patter.2026.101513_bib45","series-title":"Explainable, Trustworthy, and Responsible AI and Multi-Agent Systems","first-page":"127","article-title":"Ladder of intentions: Unifying agent architectures for explainability and transferability","author":"Gimenez-Abalos","year":"2026"},{"key":"10.1016\/j.patter.2026.101513_bib46","series-title":"Proceedings of the 2020 ACM Designing Interactive Systems Conference (ACM)","first-page":"1195","article-title":"A Survey on Interactive Reinforcement Learning: Design Principles and Open Challenges","author":"Arzate Cruz","year":"2020"},{"key":"10.1016\/j.patter.2026.101513_bib47","doi-asserted-by":"crossref","first-page":"593","DOI":"10.3390\/electronics10050593","article-title":"Evaluating the Quality of Machine Learning Explanations: A Survey on Methods and Metrics","volume":"10","author":"Zhou","year":"2021","journal-title":"Electronics"},{"key":"10.1016\/j.patter.2026.101513_bib48","article-title":"A Survey of Explainable Reinforcement Learning","author":"Milani","year":"2022","journal-title":"arXiv"},{"key":"10.1016\/j.patter.2026.101513_bib49","series-title":"Explainable Agency in Artificial Intelligence: Research and Practice","author":"Aha","year":"2024"},{"key":"10.1016\/j.patter.2026.101513_bib50","article-title":"Explainable Planning","author":"Fox","year":"2017","journal-title":"arXiv"},{"key":"10.1016\/j.patter.2026.101513_bib51","doi-asserted-by":"crossref","first-page":"66","DOI":"10.1016\/j.artint.2018.01.002","article-title":"Autonomous agents modelling other agents: A comprehensive survey and open problems","volume":"258","author":"Albrecht","year":"2018","journal-title":"Artif. Intell."},{"key":"10.1016\/j.patter.2026.101513_bib52","series-title":"Developments in Language Theory 11088","first-page":"521","article-title":"Why Bad Coffee? Explaining Agent Plans with Valuings","author":"Winikoff","year":"2018"},{"key":"10.1016\/j.patter.2026.101513_bib53","series-title":"Explainable, Transparent Autonomous Agents and Multi-Agent Systems 12175","first-page":"3","article-title":"Agent-Based Explanations in AI: Towards an Abstract Framework","author":"Ciatto","year":"2020"},{"key":"10.1016\/j.patter.2026.101513_bib54","first-page":"2493","article-title":"Explainable Reinforcement Learning through a Causal Lens","volume":"34","author":"Madumal","year":"2020","journal-title":"Proc. AAAI Conf. Artif. Intell."},{"key":"10.1016\/j.patter.2026.101513_bib55","series-title":"Explainable and Transparent AI and Multi-Agent Systems","first-page":"18","article-title":"Evaluating a mechanism for explaining bdi agent behaviour","author":"Winikoff","year":"2023"},{"key":"10.1016\/j.patter.2026.101513_bib56","series-title":"Proc. of Adaptive and Learning Agents Workshop (ALA 2023)","first-page":"1","article-title":"Towards Explaining Actions of Learning Agents","author":"Rodrigues","year":"2023"},{"key":"10.1016\/j.patter.2026.101513_bib57","series-title":"Explainable Agency in Artificial Intelligence","first-page":"1","article-title":"From Explainable to Justified Agency","author":"Langley","year":"2024"},{"key":"10.1016\/j.patter.2026.101513_bib58","series-title":"Proceedings of the 23rd International Conference on Autonomous Agents and Multiagent Systems AAMAS \u201924","first-page":"771","article-title":"Causal explanations for sequential decision-making in multi-agent systems","author":"Gyevnar","year":"2024"},{"key":"10.1016\/j.patter.2026.101513_bib59","series-title":"CEUR Workshop Proceedings 2502 (Sun SITE Central Europe","first-page":"40","article-title":"Towards XMAS: explainability through multi-agent systems","author":"Ciatto","year":"2019"},{"key":"10.1016\/j.patter.2026.101513_bib60","series-title":"Proceedings of the Nineteenth International Conference on Principles of Knowledge Representation and Reasoning","first-page":"362","article-title":"Discovering User-Interpretable Capabilities of Black-Box Planning Agents","author":"Verma","year":"2022"},{"key":"10.1016\/j.patter.2026.101513_bib61","series-title":"Adv. Neural Inf. Process. Syst.","first-page":"4768","article-title":"A unified approach to interpreting model predictions","volume":"30","author":"Lundberg","year":"2017"},{"key":"10.1016\/j.patter.2026.101513_bib62","series-title":"Proceedings of the 22nd ACM SIGKDD International Conference on Knowledge Discovery and Data Mining (ACM)","first-page":"1135","article-title":"\u201cWhy Should I Trust You?\u201d: Explaining the Predictions of Any Classifier","author":"Ribeiro","year":"2016"},{"key":"10.1016\/j.patter.2026.101513_bib63","doi-asserted-by":"crossref","first-page":"68","DOI":"10.1145\/3359786","article-title":"Techniques for interpretable machine learning","volume":"63","author":"Du","year":"2019","journal-title":"Commun. ACM"},{"key":"10.1016\/j.patter.2026.101513_bib64","series-title":"Proceedings of the Thirty-First International Joint Conference on Artificial Intelligence, IJCAI-22","first-page":"109","article-title":"Toward Policy Explanations for Multi-Agent Reinforcement Learning","author":"Boggess","year":"2022"},{"key":"10.1016\/j.patter.2026.101513_bib65","series-title":"Advances in Neural Information Processing Systems 32","first-page":"1","article-title":"Learning to predict without looking ahead: World models without forward prediction","author":"Freeman","year":"2019"},{"key":"10.1016\/j.patter.2026.101513_bib66","first-page":"3980","article-title":"Reinforcement learning with non-markovian rewards","volume":"34","author":"Gaon","year":"2020","journal-title":"Proc. AAAI Conf. Artif. Intell."},{"key":"10.1016\/j.patter.2026.101513_bib67","doi-asserted-by":"crossref","first-page":"11397","DOI":"10.1007\/s11063-023-11381-3","article-title":"Smaller World Models for Reinforcement Learning","volume":"55","author":"Robine","year":"2023","journal-title":"Neural Process. Lett."},{"key":"10.1016\/j.patter.2026.101513_bib68","doi-asserted-by":"crossref","first-page":"5873","DOI":"10.1109\/TPAMI.2024.3368158","article-title":"HyperSOR: Context-Aware Graph Hypernetwork for Salient Object Ranking","volume":"46","author":"Qiao","year":"2024","journal-title":"IEEE Trans. Pattern Anal. Mach. Intell."},{"key":"10.1016\/j.patter.2026.101513_bib69","series-title":"Proceedings of the 31st ACM International Conference on Advances in Geographic Information Systems SIGSPATIAL \u201923","first-page":"1","article-title":"Empowering Spatial Knowledge Graph for Mobile Traffic Prediction","author":"Gong","year":"2023"},{"key":"10.1016\/j.patter.2026.101513_bib70","first-page":"2514","article-title":"Generation of Policy-Level Explanations for Reinforcement Learning","volume":"33","author":"Topin","year":"2019","journal-title":"Proc. AAAI Conf. Artif. Intell."},{"key":"10.1016\/j.patter.2026.101513_bib71","series-title":"Advances in Neural Information Processing Systems 36","first-page":"65531","article-title":"When Demonstrations Meet Generative World Models: A Maximum Likelihood Framework for Offline Inverse Reinforcement Learning","author":"Zeng","year":"2023"},{"key":"10.1016\/j.patter.2026.101513_bib72","series-title":"Advances in Neural Information Processing Systems 36","first-page":"54727","article-title":"Autonomous Capability Assessment of Sequential Decision-Making Systems in Stochastic Settings","author":"Verma","year":"2023"},{"key":"10.1016\/j.patter.2026.101513_bib73","series-title":"Advances in Neural Information Processing Systems 36","first-page":"67156","article-title":"State2Explanation: Concept-Based Explanations to Benefit Agent Learning and User Understanding","author":"Das","year":"2023"},{"key":"10.1016\/j.patter.2026.101513_bib74","series-title":"How the Mind Explains Behavior: Folk Explanation, Meaning and Social Interaction","author":"Malle","year":"2004"},{"key":"10.1016\/j.patter.2026.101513_bib75","article-title":"Attributions as Behaviour Explanations: Towards a New Theory","author":"Malle","year":"2007","journal-title":"Cogprints"},{"key":"10.1016\/j.patter.2026.101513_bib76","doi-asserted-by":"crossref","first-page":"288","DOI":"10.1037\/0022-3514.72.2.288","article-title":"Which behaviors do people explain? A basic actor\u2013observer asymmetry","volume":"72","author":"Malle","year":"1997","journal-title":"J. Personality Soc. Psychol."},{"key":"10.1016\/j.patter.2026.101513_bib77","doi-asserted-by":"crossref","first-page":"959","DOI":"10.1016\/j.tics.2022.08.003","article-title":"Planning with Theory of Mind","volume":"26","author":"Ho","year":"2022","journal-title":"Trends Cogn. Sci."},{"key":"10.1016\/j.patter.2026.101513_bib78","article-title":"Why Interpreting Intent Is Key for Trustworthiness in the Age of Opaque Agents","author":"Gimenez-Abalos","year":"2024","journal-title":"Preprints.org"},{"key":"10.1016\/j.patter.2026.101513_bib79","doi-asserted-by":"crossref","first-page":"101","DOI":"10.1006\/jesp.1996.1314","article-title":"The Folk Concept of Intentionality","volume":"33","author":"Malle","year":"1997","journal-title":"J. Exp. Soc. Psychol."},{"key":"10.1016\/j.patter.2026.101513_bib80","doi-asserted-by":"crossref","first-page":"369","DOI":"10.1080\/09515089.2019.1688778","article-title":"Adopting the intentional stance toward natural and artificial agents","volume":"33","author":"Perez-Osorio","year":"2020","journal-title":"Philos. Psychol."},{"key":"10.1016\/j.patter.2026.101513_bib81","doi-asserted-by":"crossref","first-page":"213","DOI":"10.1016\/0004-3702(90)90055-5","article-title":"Intention is choice with commitment","volume":"42","author":"Cohen","year":"1990","journal-title":"Artif. Intell."},{"key":"10.1016\/j.patter.2026.101513_bib82","series-title":"Proceedings of the 2nd International Conference on Principles of Knowledge Representation and Reasoning","first-page":"473","article-title":"Modeling Rational Agents within a BDI-Architecture","author":"Rao","year":"1991"},{"key":"10.1016\/j.patter.2026.101513_bib83","series-title":"Advances in Neural Information Processing Systems 32","first-page":"1","article-title":"On the Utility of Learning about Humans for Human-AI Coordination","author":"Carroll","year":"2019"},{"key":"10.1016\/j.patter.2026.101513_bib84","series-title":"2020 IEEE\/CVF Conference on Computer Vision and Pattern Recognition, CVPR 2020, Seattle, WA, USA, June 13-19, 2020","first-page":"11618","article-title":"nuscenes: A multimodal dataset for autonomous driving","author":"Caesar","year":"2020"},{"key":"10.1016\/j.patter.2026.101513_bib85","series-title":"Frontiers in Artificial Intelligence and Applications","doi-asserted-by":"crossref","first-page":"355","DOI":"10.3233\/FAIA220358","article-title":"Testing Reinforcement Learning Explainability Methods in a Multi-Agent Cooperative Environment","author":"Dom\u00e8nech i Vila","year":"2022"},{"key":"10.1016\/j.patter.2026.101513_bib86","series-title":"36th IEEE Intelligent Vehicles Symposium","first-page":"1928","article-title":"Intention-aware policy graphs for explainable autonomous driving","author":"Montese","year":"2025"},{"key":"10.1016\/j.patter.2026.101513_bib87","series-title":"2017 IEEE International Conference on Computer Vision (ICCV)","first-page":"618","article-title":"Grad-CAM: Visual Explanations from Deep Networks via Gradient-Based Localization","author":"Selvaraju","year":"2017"},{"key":"10.1016\/j.patter.2026.101513_bib88","series-title":"Aristotle on Teleology","first-page":"211","article-title":"Teleology and Humans","author":"Johnson","year":"2005"},{"key":"10.1016\/j.patter.2026.101513_bib89","doi-asserted-by":"crossref","first-page":"497","DOI":"10.1348\/014466604X17452","article-title":"Bridging the intention-behaviour gap: The role of moral norm","volume":"44","author":"Godin","year":"2005","journal-title":"Br. J. Soc. Psychol."},{"key":"10.1016\/j.patter.2026.101513_bib90","doi-asserted-by":"crossref","first-page":"47","DOI":"10.1207\/s15516709cog0401_3","article-title":"The Intentionality of Intention and Action","volume":"4","author":"Searle","year":"1980","journal-title":"Cogn. Sci."},{"key":"10.1016\/j.patter.2026.101513_bib91","unstructured":"Somers, J. (2018). How the artificial-intelligence program alphazero mastered its games. https:\/\/www.newyorker.com\/science\/elements\/how-the-artificial-intelligence-program-alphazero-mastered-its-games."},{"key":"10.1016\/j.patter.2026.101513_bib92","doi-asserted-by":"crossref","first-page":"55","DOI":"10.3390\/philosophies7030055","article-title":"How to Make AlphaGo\u2019s Children Explainable","volume":"7","author":"Park","year":"2022","journal-title":"Philosophies"},{"key":"10.1016\/j.patter.2026.101513_bib93","series-title":"Proceedings of the 7th International Joint Conference on Autonomous Agents and Multiagent Systems - Volume 2 AAMAS \u201908","first-page":"713","article-title":"Goals in agent systems: a unifying framework","author":"van Riemsdijk","year":"2008"},{"key":"10.1016\/j.patter.2026.101513_bib94","series-title":"Wiley Encyclopedia of Operations Research and Management Science","first-page":"25","article-title":"Overweighting of small probabilities","author":"Burns","year":"2010"},{"key":"10.1016\/j.patter.2026.101513_bib95","series-title":"Explainable AI: Interpreting, Explaining and Visualizing Deep Learning 11700","first-page":"193","article-title":"Layer-Wise Relevance Propagation: An Overview","author":"Montavon","year":"2019"},{"key":"10.1016\/j.patter.2026.101513_bib96","unstructured":"Parent, Z., Tormos, A., Gimenez-Abalos, V., Alvarez-Napagao, S., and Montese, S. (2026). HPAI-BSC\/pgeon: Python package that produces explanations for opaque agents using Policy Graphs. Zenodo. https:\/\/doi.org\/10.5281\/zenodo.18167015."},{"key":"10.1016\/j.patter.2026.101513_bib97","author":"Schulman","year":"2017","journal-title":"Proximal Policy Optimization Algorithms Preprint at arXiv"},{"key":"10.1016\/j.patter.2026.101513_bib98","first-page":"12120","article-title":"Predicate Invention for Bilevel Planning","volume":"37","author":"Silver","year":"2023","journal-title":"Proc. AAAI Conf. Artif. Intell."},{"key":"10.1016\/j.patter.2026.101513_bib99","series-title":"Does Zero-Shot Reinforcement Learning Exist? In The Eleventh International Conference on Learning Representations","first-page":"1","author":"Touati","year":"2023"},{"key":"10.1016\/j.patter.2026.101513_bib100","unstructured":"Gimenez-Abalos, V., and Tormos, A. (2026). HPAI-BSC\/intention-aware-pgs: application of Intention-aware Policy Graphs to explain the behaviour of agents in Overcooked-AI. Zenodo. https:\/\/doi.org\/10.5281\/zenodo.18167001."},{"key":"10.1016\/j.patter.2026.101513_bib101","unstructured":"Montese, S., and Tormos, A. (2026). HPAI-BSC\/ipg4v: application of Intention-aware Policy Graphs to explain Autonomous Vehicle (AV) behaviour. Zenodo. https:\/\/doi.org\/10.5281\/zenodo.18167009."}],"container-title":["Patterns"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/api.elsevier.com\/content\/article\/PII:S266638992600022X?httpAccept=text\/xml","content-type":"text\/xml","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/api.elsevier.com\/content\/article\/PII:S266638992600022X?httpAccept=text\/plain","content-type":"text\/plain","content-version":"vor","intended-application":"text-mining"}],"deposited":{"date-parts":[[2026,5,16]],"date-time":"2026-05-16T08:30:39Z","timestamp":1778920239000},"score":1,"resource":{"primary":{"URL":"https:\/\/linkinghub.elsevier.com\/retrieve\/pii\/S266638992600022X"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026,4]]},"references-count":101,"journal-issue":{"issue":"4","published-print":{"date-parts":[[2026,4]]}},"alternative-id":["S266638992600022X"],"URL":"https:\/\/doi.org\/10.1016\/j.patter.2026.101513","relation":{},"ISSN":["2666-3899"],"issn-type":[{"value":"2666-3899","type":"print"}],"subject":[],"published":{"date-parts":[[2026,4]]},"assertion":[{"value":"Elsevier","name":"publisher","label":"This article is maintained by"},{"value":"Intentional policy graphs: A pipeline for explaining agent behavior through intentions","name":"articletitle","label":"Article Title"},{"value":"Patterns","name":"journaltitle","label":"Journal Title"},{"value":"https:\/\/doi.org\/10.1016\/j.patter.2026.101513","name":"articlelink","label":"CrossRef DOI link to publisher maintained version"},{"value":"article","name":"content_type","label":"Content Type"},{"value":"\u00a9 2026 The Author(s). Published by Elsevier Inc.","name":"copyright","label":"Copyright"}],"article-number":"101513"}}