{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,3,27]],"date-time":"2025-03-27T15:03:59Z","timestamp":1743087839598,"version":"3.40.3"},"publisher-location":"Cham","reference-count":56,"publisher":"Springer Nature Switzerland","isbn-type":[{"type":"print","value":"9783031440663"},{"type":"electronic","value":"9783031440670"}],"license":[{"start":{"date-parts":[[2023,1,1]],"date-time":"2023-01-01T00:00:00Z","timestamp":1672531200000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2023,1,1]],"date-time":"2023-01-01T00:00:00Z","timestamp":1672531200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2023]]},"DOI":"10.1007\/978-3-031-44067-0_4","type":"book-chapter","created":{"date-parts":[[2023,10,20]],"date-time":"2023-10-20T06:02:33Z","timestamp":1697781753000},"page":"72-87","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":0,"title":["Contrastive Visual Explanations for\u00a0Reinforcement Learning via\u00a0Counterfactual Rewards"],"prefix":"10.1007","author":[{"given":"Xiaowei","family":"Liu","sequence":"first","affiliation":[]},{"given":"Kevin","family":"McAreavey","sequence":"additional","affiliation":[]},{"given":"Weiru","family":"Liu","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2023,10,21]]},"reference":[{"unstructured":"Amir, D., Amir, O.: Highlights: summarizing agent behavior to people. In: AAMAS 2018, pp. 1168\u20131176 (2018)","key":"4_CR1"},{"doi-asserted-by":"crossref","unstructured":"Anderson, A., et al.: Explaining reinforcement learning to mere mortals: an empirical study. In: IJCAI 2019, pp. 1328\u20131334 (2019)","key":"4_CR2","DOI":"10.24963\/ijcai.2019\/184"},{"doi-asserted-by":"crossref","unstructured":"Annasamy, R., Sycara, K.: Towards better interpretability in deep q-networks. In: AAAI 2019, vol. 33, pp. 4561\u20134569 (2019)","key":"4_CR3","DOI":"10.1609\/aaai.v33i01.33014561"},{"doi-asserted-by":"crossref","unstructured":"Bellman, R.E.: Dynamic Programming. Princeton University Press (2010)","key":"4_CR4","DOI":"10.1515\/9781400835386"},{"doi-asserted-by":"crossref","unstructured":"Chakraborti, T., Kulkarni, A., Sreedharan, S., Smith, D.E., Kambhampati, S.: Explicability? legibility? predictability? transparency? privacy? security? the emerging landscape of interpretable agent behavior. In: ICAPS 2019, vol. 29, pp. 86\u201396 (2019)","key":"4_CR5","DOI":"10.1609\/icaps.v29i1.3463"},{"doi-asserted-by":"crossref","unstructured":"Chakraborti, T., Sreedharan, S., Kambhampati, S.: The emerging landscape of explainable automated planning & decision making. In: IJCAI 2020, pp. 4803\u20134811 (2020). Survey track","key":"4_CR6","DOI":"10.24963\/ijcai.2020\/669"},{"unstructured":"Chevalier-Boisvert, M., Willems, L., Pal, S.: Minimalistic gridworld environment for gymnasium (2018). https:\/\/github.com\/Farama-Foundation\/Minigrid","key":"4_CR7"},{"unstructured":"Christiano, P.F., Leike, J., Brown, T., Martic, M., Legg, S., Amodei, D.: Deep reinforcement learning from human preferences. In: NeurIPS 2017, vol. 30 (2017)","key":"4_CR8"},{"key":"4_CR9","series-title":"Lecture Notes in Computer Science (Lecture Notes in Artificial Intelligence)","doi-asserted-by":"publisher","first-page":"66","DOI":"10.1007\/978-3-030-35288-2_6","volume-title":"AI 2019: Advances in Artificial Intelligence","author":"F Cruz","year":"2019","unstructured":"Cruz, F., Dazeley, R., Vamplew, P.: Memory-based explainable reinforcement learning. In: Liu, J., Bailey, J. (eds.) AI 2019. LNCS (LNAI), vol. 11919, pp. 66\u201377. Springer, Cham (2019). https:\/\/doi.org\/10.1007\/978-3-030-35288-2_6"},{"unstructured":"Gottesman, O., et al.: Interpretable off-policy evaluation in reinforcement learning by highlighting influential transitions. In: ICML 2020, vol. 119, pp. 3658\u20133667 (2020)","key":"4_CR10"},{"unstructured":"Greydanus, S., Koul, A., Dodge, J., Fern, A.: Visualizing and understanding atari agents. In: ICML 2018, pp. 2877\u20132886 (2018)","key":"4_CR11"},{"doi-asserted-by":"crossref","unstructured":"Gunning, D.: Darpa\u2019s explainable artificial intelligence (XAI) program. In: Proceedings of the 24th International Conference on Intelligent User Interfaces, p. ii (2019)","key":"4_CR12","DOI":"10.1145\/3301275.3308446"},{"unstructured":"Gupta, P., et al.: Explain your move: understanding agent actions using specific and relevant feature attribution. In: ICLR 2020 (2020)","key":"4_CR13"},{"doi-asserted-by":"crossref","unstructured":"Hayes, B., Shah, J.A.: Improving robot controller transparency through autonomous policy explanation. In: 2017 12th ACM\/IEEE International Conference on Human-Robot Interaction (HRI), pp. 303\u2013312 (2017)","key":"4_CR14","DOI":"10.1145\/2909824.3020233"},{"key":"4_CR15","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"277","DOI":"10.1007\/978-3-030-31423-1_9","volume-title":"Reasoning Web. Explainable Artificial Intelligence","author":"J Hoffmann","year":"2019","unstructured":"Hoffmann, J., Magazzeni, D.: Explainable AI planning (XAIP): overview and the case of contrastive explanation (extended abstract). In: Kr\u00f6tzsch, M., Stepanova, D. (eds.) Reasoning Web. Explainable Artificial Intelligence. LNCS, vol. 11810, pp. 277\u2013282. Springer, Cham (2019). https:\/\/doi.org\/10.1007\/978-3-030-31423-1_9"},{"doi-asserted-by":"crossref","unstructured":"Huang, S.H., Bhatia, K., Abbeel, P., Dragan, A.D.: Establishing appropriate trust via critical states. In: 2018 IEEE\/RSJ International Conference on Intelligent Robots and Systems (IROS), pp. 3929\u20133936. IEEE (2018)","key":"4_CR16","DOI":"10.1109\/IROS.2018.8593649"},{"key":"4_CR17","doi-asserted-by":"publisher","first-page":"309","DOI":"10.1007\/s10514-018-9771-0","volume":"43","author":"SH Huang","year":"2017","unstructured":"Huang, S.H., Held, D., Abbeel, P., Dragan, A.D.: Enabling robots to communicate their objectives. Auton. Robot. 43, 309\u2013326 (2017)","journal-title":"Auton. Robot."},{"key":"4_CR18","doi-asserted-by":"publisher","DOI":"10.1016\/j.artint.2021.103571","volume":"301","author":"T Huber","year":"2021","unstructured":"Huber, T., Weitz, K., Andr\u00e9, E., Amir, O.: Local and global explanations of agent behavior: integrating strategy summaries with saliency maps. Artif. Intell. 301, 103571 (2021)","journal-title":"Artif. Intell."},{"unstructured":"H\u00fcy\u00fck, A., Jarrett, D., Tekin, C., van der Schaar, M.: Explaining by imitating: understanding decisions by interpretable policy learning. In: ICLR 2021 (2021)","key":"4_CR19"},{"unstructured":"Ibarz, B., Leike, J., Pohlen, T., Irving, G., Legg, S., Amodei, D.: Reward learning from human preferences and demonstrations in atari. In: NeurIPS 2018, vol. 31 (2018)","key":"4_CR20"},{"unstructured":"Juozapaitis, Z., Koul, A., Fern, A., Erwig, M., Doshi-Velez, F.: Explainable reinforcement learning via reward decomposition. arxiv (2019)","key":"4_CR21"},{"key":"4_CR22","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"366","DOI":"10.1007\/978-3-030-61609-0_29","volume-title":"Artificial Neural Networks and Machine Learning \u2013 ICANN 2020","author":"I Karino","year":"2020","unstructured":"Karino, I., Ohmura, Y., Kuniyoshi, Y.: Identifying critical states by the action-based variance of expected return. In: Farka\u0161, I., Masulli, P., Wermter, S. (eds.) ICANN 2020. LNCS, vol. 12396, pp. 366\u2013378. Springer, Cham (2020). https:\/\/doi.org\/10.1007\/978-3-030-61609-0_29"},{"key":"4_CR23","doi-asserted-by":"publisher","first-page":"1238","DOI":"10.1177\/0278364913495721","volume":"32","author":"J Kober","year":"2013","unstructured":"Kober, J., Bagnell, J.A., Peters, J.: Reinforcement learning in robotics: a survey. Int. J. Robot. Res. 32, 1238\u20131274 (2013)","journal-title":"Int. J. Robot. Res."},{"doi-asserted-by":"crossref","unstructured":"Lage, I., Lifschitz, D., Doshi-Velez, F., Amir, O.: Exploring computational user models for agent policy summarization. In: IJCAI 2019, pp. 1401\u20131407 (2019)","key":"4_CR24","DOI":"10.24963\/ijcai.2019\/194"},{"doi-asserted-by":"crossref","unstructured":"Lin, Y.C., Hong, Z.W., Liao, Y.H., Shih, M.L., Liu, M.Y., Sun, M.: Tactics of adversarial attack on deep reinforcement learning agents. In: IJCAI 2017, pp. 3756\u20133762 (2017)","key":"4_CR25","DOI":"10.24963\/ijcai.2017\/525"},{"doi-asserted-by":"crossref","unstructured":"Lipton, P., Knowles, D.: Contrastive Explanations, p. 247\u2013266. Royal Institute of Philosophy Supplements, Cambridge University Press (1991)","key":"4_CR26","DOI":"10.1017\/CBO9780511599705.014"},{"unstructured":"Liu, R., Bai, F., Du, Y., Yang, Y.: Meta-reward-net: implicitly differentiable reward learning for preference-based reinforcement learning. In: NeurIPS 2022, vol. 35, pp. 22270\u201322284 (2022)","key":"4_CR27"},{"doi-asserted-by":"crossref","unstructured":"Lu, W., Magg, S., Zhao, X., Gromniak, M., Wermter, S.: A closer look at reward decomposition for high-level robotic explanations. arXiv abs\/2304.12958 (2023)","key":"4_CR28","DOI":"10.1109\/ICDL55364.2023.10364407"},{"doi-asserted-by":"crossref","unstructured":"Madumal, P., Miller, T., Sonenberg, L., Vetere, F.: Explainable reinforcement learning through a causal lens. In: AAAI 2020, pp. 2493\u20132500 (2020)","key":"4_CR29","DOI":"10.1609\/aaai.v34i03.5631"},{"unstructured":"Marcus, G., Davis, E.: Rebooting AI: Building Artificial Intelligence We Can Trust. Pantheon Books, USA (2019)","key":"4_CR30"},{"key":"4_CR31","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1016\/j.artint.2018.07.007","volume":"267","author":"T Miller","year":"2019","unstructured":"Miller, T.: Explanation in artificial intelligence: insights from the social sciences. Artif. Intell. 267, 1\u201338 (2019)","journal-title":"Artif. Intell."},{"key":"4_CR32","doi-asserted-by":"publisher","DOI":"10.1017\/S0269888921000102","volume":"36","author":"T Miller","year":"2021","unstructured":"Miller, T.: Contrastive explanation: a structural-model approach. Knowl. Eng. Rev. 36, e14 (2021)","journal-title":"Knowl. Eng. Rev."},{"unstructured":"Montavon, G., Samek, W., M\u00fcller, K.R.: Methods for interpreting and understanding deep neural networks. arXiv abs\/1706.07979 (2017)","key":"4_CR33"},{"unstructured":"Mott, A., Zoran, D., Chrzanowski, M., Wierstra, D., Rezende, D.J.: Towards interpretable reinforcement learning using attention augmented agents. In: NeurIPS 2019, pp. 12360\u201312369 (2019)","key":"4_CR34"},{"unstructured":"Narayanan, S., Lage, I., Doshi-Velez, F.: (when) are contrastive explanations of reinforcement learning helpful? arXiv abs\/2211.07719 (2022)","key":"4_CR35"},{"key":"4_CR36","doi-asserted-by":"publisher","DOI":"10.1016\/j.artint.2021.103455","volume":"295","author":"ML Olson","year":"2021","unstructured":"Olson, M.L., Khanna, R., Neal, L., Li, F., Wong, W.K.: Counterfactual state explanations for reinforcement learning agents via generative deep learning. Artif. Intell. 295, 103455 (2021)","journal-title":"Artif. Intell."},{"doi-asserted-by":"crossref","unstructured":"Puiutta, E., Veith, E.M.S.P.: Explainable reinforcement learning: a survey. arXiv abs\/2005.06247 (2020)","key":"4_CR37","DOI":"10.1007\/978-3-030-57321-8_5"},{"doi-asserted-by":"crossref","unstructured":"Puterman, M.L.: Markov decision processes: discrete stochastic dynamic programming. In: Wiley Series in Probability and Statistics (1994)","key":"4_CR38","DOI":"10.1002\/9780470316887"},{"doi-asserted-by":"crossref","unstructured":"Russell, S., Norvig, P.: Artificial Intelligence: A Modern Approach, 4th edn. Pearson (2020)","key":"4_CR39","DOI":"10.1093\/oso\/9780190905033.003.0012"},{"key":"4_CR40","doi-asserted-by":"publisher","first-page":"604","DOI":"10.1038\/s41586-020-03051-4","volume":"588","author":"J Schrittwieser","year":"2019","unstructured":"Schrittwieser, J., et al.: Mastering atari, go, chess and shogi by planning with a learned model. Nature 588, 604\u2013609 (2019)","journal-title":"Nature"},{"key":"4_CR41","doi-asserted-by":"publisher","DOI":"10.1016\/j.artint.2020.103367","volume":"288","author":"P Sequeira","year":"2020","unstructured":"Sequeira, P., Gervasio, M.: Interestingness elements for explainable reinforcement learning: understanding agents\u2019 capabilities and limitations. Artif. Intell. 288, 103367 (2020)","journal-title":"Artif. Intell."},{"unstructured":"Sequeira, P., Hostetler, J., Gervasio, M.T.: Global and local analysis of interestingness for competency-aware deep reinforcement learning. arXiv abs\/2211.06376 (2022)","key":"4_CR42"},{"unstructured":"Shu, T., Xiong, C., Socher, R.: Hierarchical and interpretable skill acquisition in multi-task reinforcement learning. In: ICLR 2018 (2018)","key":"4_CR43"},{"key":"4_CR44","doi-asserted-by":"publisher","first-page":"354","DOI":"10.1038\/nature24270","volume":"550","author":"D Silver","year":"2017","unstructured":"Silver, D., et al.: Mastering the game of go without human knowledge. Nature 550, 354\u2013359 (2017)","journal-title":"Nature"},{"doi-asserted-by":"crossref","unstructured":"Sreedharan, S., Srivastava, S., Kambhampati, S.: TLDR: policy summarization for factored SSP problems using temporal abstractions. In: ICAPS 2020, vol. 30, pp. 272\u2013280 (2020)","key":"4_CR45","DOI":"10.1609\/icaps.v30i1.6671"},{"key":"4_CR46","doi-asserted-by":"publisher","DOI":"10.1016\/j.artint.2021.103570","volume":"301","author":"S Sreedharan","year":"2021","unstructured":"Sreedharan, S., Srivastava, S., Kambhampati, S.: Using state abstractions to compute personalized contrastive explanations for AI agent behavior. Artif. Intell. 301, 103570 (2021)","journal-title":"Artif. Intell."},{"doi-asserted-by":"crossref","unstructured":"Topin, N., Veloso, M.: Generation of policy-level explanations for reinforcement learning. In: AAAI 2019, pp. 2514\u20132521 (2019)","key":"4_CR47","DOI":"10.1609\/aaai.v33i01.33012514"},{"doi-asserted-by":"crossref","unstructured":"Vouros, G.A.: Explainable deep reinforcement learning: state of the art and challenges. ACM Comput. Surv. 55(5) (2022)","key":"4_CR48","DOI":"10.1145\/3527448"},{"unstructured":"Waa, J., Diggelen, J., Bosch, K., Neerincx, M.: Contrastive explanations for reinforcement learning in terms of expected consequences. In: IJCAI 2018 - Explainable Artificial Intelligence (XAI) Workshop (2018)","key":"4_CR49"},{"key":"4_CR50","doi-asserted-by":"publisher","first-page":"279","DOI":"10.1007\/BF00992698","volume":"8","author":"CJ Watkins","year":"1992","unstructured":"Watkins, C.J., Dayan, P.: Q-learning. Mach. Learn. 8, 279\u2013292 (1992)","journal-title":"Mach. Learn."},{"doi-asserted-by":"crossref","unstructured":"Wells, L., Bednarz, T.: Explainable AI and reinforcement learning-a systematic review of current approaches and trends. Front. Artif. Intell. 4 (2021)","key":"4_CR51","DOI":"10.3389\/frai.2021.550030"},{"unstructured":"Yau, H., Russell, C., Hadfield, S.: What did you think would happen? Explaining agent behaviour through intended outcomes. In: NeurIPS 2020, vol. 33, pp. 18375\u201318386 (2020)","key":"4_CR52"},{"doi-asserted-by":"crossref","unstructured":"Yeh, E., Sequeira, P., Hostetler, J., Gervasio, M.T.: Outcome-guided counterfactuals for reinforcement learning agents from a jointly trained generative latent space. arXiv abs\/2207.07710 (2022)","key":"4_CR53","DOI":"10.1007\/978-3-031-44070-0_23"},{"unstructured":"Zahavy, T., Ben-Zrihem, N., Mannor, S.: Graying the black box: understanding DQNs. In: ICML 2016, pp. 1899\u20131908 (2016)","key":"4_CR54"},{"key":"4_CR55","series-title":"Lecture Notes in Computer Science (Lecture Notes in Artificial Intelligence)","doi-asserted-by":"publisher","first-page":"223","DOI":"10.1007\/978-3-030-82017-6_14","volume-title":"Explainable and Transparent AI and Multi-Agent Systems","author":"AE Zelvelder","year":"2021","unstructured":"Zelvelder, A.E., Westberg, M., Fr\u00e4mling, K.: Assessing explainability in reinforcement learning. In: Calvaresi, D., Najjar, A., Winikoff, M., Fr\u00e4mling, K. (eds.) EXTRAAMAS 2021. LNCS (LNAI), vol. 12688, pp. 223\u2013240. Springer, Cham (2021). https:\/\/doi.org\/10.1007\/978-3-030-82017-6_14"},{"doi-asserted-by":"crossref","unstructured":"\u010cyras, K., Rago, A., Albini, E., Baroni, P., Toni, F.: Argumentative XAI: a survey. In: IJCAI 2021, pp. 4392\u20134399 (2021). Survey Track","key":"4_CR56","DOI":"10.24963\/ijcai.2021\/600"}],"container-title":["Communications in Computer and Information Science","Explainable Artificial Intelligence"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-031-44067-0_4","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,2,13]],"date-time":"2024-02-13T06:04:26Z","timestamp":1707804266000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-3-031-44067-0_4"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023]]},"ISBN":["9783031440663","9783031440670"],"references-count":56,"URL":"https:\/\/doi.org\/10.1007\/978-3-031-44067-0_4","relation":{},"ISSN":["1865-0929","1865-0937"],"issn-type":[{"type":"print","value":"1865-0929"},{"type":"electronic","value":"1865-0937"}],"subject":[],"published":{"date-parts":[[2023]]},"assertion":[{"value":"21 October 2023","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"xAI","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"World Conference on Explainable Artificial Intelligence","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Lisbon","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Portugal","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2023","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"26 July 2023","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"28 July 2023","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"1","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"xai2023","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"http:\/\/xaiworldconference.com\/","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Single-blind","order":1,"name":"type","label":"Type","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"EasyChair","order":2,"name":"conference_management_system","label":"Conference Management System","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"220","order":3,"name":"number_of_submissions_sent_for_review","label":"Number of Submissions Sent for Review","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"94","order":4,"name":"number_of_full_papers_accepted","label":"Number of Full Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"0","order":5,"name":"number_of_short_papers_accepted","label":"Number of Short Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"43% - The value is computed by the equation \"Number of Full Papers Accepted \/ Number of Submissions Sent for Review * 100\" and then rounded to a whole number.","order":6,"name":"acceptance_rate_of_full_papers","label":"Acceptance Rate of Full Papers","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"3","order":7,"name":"average_number_of_reviews_per_paper","label":"Average Number of Reviews per Paper","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"3","order":8,"name":"average_number_of_papers_per_reviewer","label":"Average Number of Papers per Reviewer","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"No","order":9,"name":"external_reviewers_involved","label":"External Reviewers Involved","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}}]}}