{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,20]],"date-time":"2026-10-20T16:52:21Z","timestamp":1792515141584,"version":"4.5.0"},"publisher-location":"Cham","reference-count":47,"publisher":"Springer International Publishing","isbn-type":[{"value":"9783030519230","type":"print"},{"value":"9783030519247","type":"electronic"}],"license":[{"start":{"date-parts":[[2020,1,1]],"date-time":"2020-01-01T00:00:00Z","timestamp":1577836800000},"content-version":"tdm","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"},{"start":{"date-parts":[[2020,1,1]],"date-time":"2020-01-01T00:00:00Z","timestamp":1577836800000},"content-version":"vor","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2020]]},"DOI":"10.1007\/978-3-030-51924-7_6","type":"book-chapter","created":{"date-parts":[[2020,7,6]],"date-time":"2020-07-06T23:19:19Z","timestamp":1594077559000},"page":"94-110","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":51,"title":["A Situation Awareness-Based Framework for Design and Evaluation of Explainable AI"],"prefix":"10.1007","author":[{"given":"Lindsay","family":"Sanneman","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Julie A.","family":"Shah","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2020,7,7]]},"reference":[{"key":"6_CR1","unstructured":"Adebayo, J., Gilmer, J., Muelly, M., Goodfellow, I., Hardt, M., Kim, B.: Sanity checks for saliency maps. In: Advances in Neural Information Processing Systems, pp. 9505\u20139515 (2018)"},{"key":"6_CR2","unstructured":"Amir, D., Amir, O.: Highlights: summarizing agent behavior to people. In: Proceedings of the 17th International Conference on Autonomous Agents and MultiAgent Systems, pp. 1168\u20131176. International Foundation for Autonomous Agents and Multiagent Systems (2018)"},{"key":"6_CR3","unstructured":"Anjomshoae, S., Najjar, A., Calvaresi, D., Fr\u00e4mling, K.: Explainable agents and robots: results from a systematic literature review. In: Proceedings of the 18th International Conference on Autonomous Agents and MultiAgent Systems, pp. 1078\u20131088. International Foundation for Autonomous Agents and Multiagent Systems (2019)"},{"key":"6_CR4","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"366","DOI":"10.1007\/978-3-319-10599-4_24","volume-title":"Computer Vision \u2013 ECCV 2014","author":"A Bansal","year":"2014","unstructured":"Bansal, A., Farhadi, A., Parikh, D.: Towards transparent systems: semantic characterization of failure modes. In: Fleet, D., Pajdla, T., Schiele, B., Tuytelaars, T. (eds.) ECCV 2014. LNCS, vol. 8694, pp. 366\u2013381. Springer, Cham (2014). \nhttps:\/\/doi.org\/10.1007\/978-3-319-10599-4_24"},{"issue":"1","key":"6_CR5","doi-asserted-by":"publisher","first-page":"63","DOI":"10.1207\/s15327566ijce0301_5","volume":"3","author":"G Bedny","year":"1999","unstructured":"Bedny, G., Meister, D.: Theory of activity and situation awareness. Int. J. Cogn. Ergon. 3(1), 63\u201372 (1999)","journal-title":"Int. J. Cogn. Ergon."},{"key":"6_CR6","unstructured":"Borgo, R., Cashmore, M., Magazzeni, D.: Towards providing explanations for AI planner decisions. arXiv preprint \narXiv:1810.06338\n\n (2018)"},{"key":"6_CR7","series-title":"Lecture Notes in Computer Science (Lecture Notes in Artificial Intelligence)","doi-asserted-by":"publisher","first-page":"28","DOI":"10.1007\/978-3-642-16178-0_5","volume-title":"Multiagent System Technologies","author":"J Broekens","year":"2010","unstructured":"Broekens, J., Harbers, M., Hindriks, K., van den Bosch, K., Jonker, C., Meyer, J.-J.: Do you get it? User-evaluated explainable BDI agents. In: Dix, J., Witteveen, C. (eds.) MATES 2010. LNCS (LNAI), vol. 6251, pp. 28\u201339. Springer, Heidelberg (2010). \nhttps:\/\/doi.org\/10.1007\/978-3-642-16178-0_5"},{"key":"6_CR8","doi-asserted-by":"crossref","unstructured":"Chakraborti, T., Sreedharan, S., Grover, S., Kambhampati, S.: Plan explanations as model reconciliation. In: 2019 14th ACM\/IEEE International Conference on Human-Robot Interaction (HRI), pp. 258\u2013266. IEEE (2019)","DOI":"10.1109\/HRI.2019.8673193"},{"key":"6_CR9","doi-asserted-by":"crossref","unstructured":"Chen, J.Y., Procci, K., Boyce, M., Wright, J., Garcia, A., Barnes, M.: Situation awareness-based agent transparency. Technical report, Army Research Lab Aberdeen Proving Ground MD Human Research and Engineering (2014)","DOI":"10.21236\/ADA600351"},{"key":"6_CR10","unstructured":"Dannenhauer, D., Floyd, M.W., Molineaux, M., Aha, D.W.: Learning from exploration: towards an explainable goal reasoning agent (2018)"},{"key":"6_CR11","unstructured":"Doshi-Velez, F., Kim, B.: Towards a rigorous science of interpretable machine learning. arXiv preprint \narXiv:1702.08608\n\n (2017)"},{"key":"6_CR12","doi-asserted-by":"crossref","unstructured":"Dragan, A.D., Lee, K.C., Srinivasa, S.S.: Legibility and predictability of robot motion. In: 2013 8th ACM\/IEEE International Conference on Human-Robot Interaction (HRI), pp. 301\u2013308. IEEE (2013)","DOI":"10.1109\/HRI.2013.6483603"},{"key":"6_CR13","doi-asserted-by":"publisher","first-page":"65","DOI":"10.1518\/001872095779049499","volume":"37","author":"M Endsley","year":"1995","unstructured":"Endsley, M.: Measurement of situation awareness in dynamic systems. Hum. Factors 37, 65\u201384 (1995). \nhttps:\/\/doi.org\/10.1518\/001872095779049499","journal-title":"Hum. Factors"},{"key":"6_CR14","unstructured":"Endsley, M.R.: Situation awareness global assessment technique (SAGAT). In: Proceedings of the IEEE 1988 National Aerospace and Electronics Conference, pp. 789\u2013795. IEEE (1988)"},{"issue":"1","key":"6_CR15","doi-asserted-by":"publisher","first-page":"4","DOI":"10.1177\/1555343415572631","volume":"9","author":"MR Endsley","year":"2015","unstructured":"Endsley, M.R.: Situation awareness misconceptions and misunderstandings. J. Cogn. Eng. Decis. Mak. 9(1), 4\u201332 (2015)","journal-title":"J. Cogn. Eng. Decis. Mak."},{"key":"6_CR16","doi-asserted-by":"crossref","unstructured":"Endsley, M.R.: Direct measurement of situation awareness: validity and use of SAGAT. In: Situational Awareness, pp. 129\u2013156. Routledge (2017)","DOI":"10.4324\/9781315087924-9"},{"key":"6_CR17","doi-asserted-by":"crossref","unstructured":"Endsley, M.R.: A systematic review and meta-analysis of direct objective measures of situation awareness: a comparison of SAGAT and spam. Hum. Factors 0018720819875376 (2019)","DOI":"10.1177\/0018720819875376"},{"key":"6_CR18","unstructured":"Endsley, M., Jones, W.: A Model of Inter-and Intrateam Situation Awareness: Implications for Design. New Trends in Cooperative Activities: Understanding System Dynamics in Complex Environments. M. McNeese, E. Salas and M. Endsley. Human Factors and Ergonomics Society, Santa Monica (2001)"},{"key":"6_CR19","series-title":"Lecture Notes in Computer Science (Lecture Notes in Artificial Intelligence)","doi-asserted-by":"publisher","first-page":"124","DOI":"10.1007\/978-3-319-47096-2_9","volume-title":"Case-Based Reasoning Research and Development","author":"MW Floyd","year":"2016","unstructured":"Floyd, M.W., Aha, D.W.: Incorporating transparency during trust-guided behavior adaptation. In: Goel, A., D\u00edaz-Agudo, M.B., Roth-Berghofer, T. (eds.) ICCBR 2016. LNCS (LNAI), vol. 9969, pp. 124\u2013138. Springer, Cham (2016). \nhttps:\/\/doi.org\/10.1007\/978-3-319-47096-2_9"},{"key":"6_CR20","unstructured":"Fox, M., Long, D., Magazzeni, D.: Explainable planning. arXiv preprint \narXiv:1709.10256\n\n (2017)"},{"issue":"2","key":"6_CR21","doi-asserted-by":"publisher","first-page":"44","DOI":"10.1609\/aimag.v40i2.2850","volume":"40","author":"D Gunning","year":"2019","unstructured":"Gunning, D., Aha, D.W.: Darpa\u2019s explainable artificial intelligence program. AI Mag. 40(2), 44\u201358 (2019)","journal-title":"AI Mag."},{"key":"6_CR22","unstructured":"Halpern, J.Y., Pearl, J.: Causes and explanations: a structural-model approach. Part I: causes. Br. J. Philos. Sci. 56(4), 843\u2013887 (2005)"},{"key":"6_CR23","doi-asserted-by":"crossref","unstructured":"Harbers, M., van den Bosch, K., Meyer, J.J.: Design and evaluation of explainable BDI agents. In: 2010 IEEE\/WIC\/ACM International Conference on Web Intelligence and Intelligent Agent Technology, vol. 2, pp. 125\u2013132. IEEE (2010)","DOI":"10.1109\/WI-IAT.2010.115"},{"key":"6_CR24","series-title":"Lecture Notes in Computer Science (Lecture Notes in Artificial Intelligence)","doi-asserted-by":"publisher","first-page":"21","DOI":"10.1007\/978-3-642-35545-5_2","volume-title":"Coordination, Organizations, Institutions, and Norms in Agent System VII","author":"M Harbers","year":"2012","unstructured":"Harbers, M., Bradshaw, J.M., Johnson, M., Feltovich, P., van den Bosch, K., Meyer, J.-J.: Explanation in human-agent teamwork. In: Cranefield, S., van Riemsdijk, M.B., V\u00e1zquez-Salceda, J., Noriega, P. (eds.) COIN -2011. LNCS (LNAI), vol. 7254, pp. 21\u201337. Springer, Heidelberg (2012). \nhttps:\/\/doi.org\/10.1007\/978-3-642-35545-5_2"},{"key":"6_CR25","doi-asserted-by":"crossref","unstructured":"Hayes, B., Shah, J.A.: Improving robot controller transparency through autonomous policy explanation. In: 2017 12th ACM\/IEEE International Conference on Human-Robot Interaction (HRI), pp. 303\u2013312. IEEE (2017)","DOI":"10.1145\/2909824.3020233"},{"issue":"1","key":"6_CR26","doi-asserted-by":"publisher","first-page":"110","DOI":"10.1515\/pjbr-2018-0009","volume":"9","author":"T Hellstr\u00f6m","year":"2018","unstructured":"Hellstr\u00f6m, T., Bensch, S.: Understandable robots-what, why, and how. Paladyn J. Behav. Robot. 9(1), 110\u2013123 (2018)","journal-title":"Paladyn J. Behav. Robot."},{"key":"6_CR27","unstructured":"Hoffman, R.R., Mueller, S.T., Klein, G., Litman, J.: Metrics for explainable AI: challenges and prospects. arXiv preprint \narXiv:1812.04608\n\n (2018)"},{"issue":"1","key":"6_CR28","doi-asserted-by":"publisher","first-page":"43","DOI":"10.5898\/JHRI.3.1.Johnson","volume":"3","author":"M Johnson","year":"2014","unstructured":"Johnson, M., Bradshaw, J.M., Feltovich, P.J., Jonker, C.M., Van Riemsdijk, M.B., Sierhuis, M.: Coactive design: designing support for interdependence in joint activity. J. Hum.-Robot Interact. 3(1), 43\u201369 (2014)","journal-title":"J. Hum.-Robot Interact."},{"key":"6_CR29","unstructured":"Kim, B., Rudin, C., Shah, J.A.: The Bayesian case model: a generative approach for case-based reasoning and prototype classification. In: Advances in Neural Information Processing Systems, pp. 1952\u20131960 (2014)"},{"key":"6_CR30","unstructured":"Kim, B., et al.: Interpretability beyond feature attribution: quantitative testing with concept activation vectors (TCAV). arXiv preprint \narXiv:1711.11279\n\n (2017)"},{"key":"6_CR31","unstructured":"Lipton, Z.C.: The mythos of model interpretability. arXiv preprint \narXiv:1606.03490\n\n (2016)"},{"key":"6_CR32","doi-asserted-by":"crossref","unstructured":"Lomas, M., Chevalier, R., Cross, E.V., Garrett, R.C., Hoare, J., Kopack, M.: Explaining robot actions. In: Proceedings of the Seventh Annual ACM\/IEEE International Conference on Human-Robot Interaction, pp. 187\u2013188 (2012)","DOI":"10.1145\/2157689.2157748"},{"key":"6_CR33","doi-asserted-by":"crossref","unstructured":"Marino, D.L., Wickramasinghe, C.S., Manic, M.: An adversarial approach for explainable AI in intrusion detection systems. In: IECON 2018-44th Annual Conference of the IEEE Industrial Electronics Society, pp. 3237\u20133243. IEEE (2018)","DOI":"10.1109\/IECON.2018.8591457"},{"key":"6_CR34","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1016\/j.artint.2018.07.007","volume":"267","author":"T Miller","year":"2018","unstructured":"Miller, T.: Explanation in artificial intelligence: insights from the social sciences. Artif. Intell. 267, 1\u201338 (2018)","journal-title":"Artif. Intell."},{"key":"6_CR35","unstructured":"Miller, T., Howe, P., Sonenberg, L.: Explainable AI: beware of inmates running the asylum or: how i learnt to stop worrying and love the social and behavioural sciences. arXiv preprint \narXiv:1712.00547\n\n (2017)"},{"key":"6_CR36","series-title":"Lecture Notes in Computer Science (Lecture Notes in Artificial Intelligence)","doi-asserted-by":"publisher","first-page":"204","DOI":"10.1007\/978-3-319-91122-9_18","volume-title":"Engineering Psychology and Cognitive Ergonomics","author":"MA Neerincx","year":"2018","unstructured":"Neerincx, M.A., van der Waa, J., Kaptein, F., van Diggelen, J.: Using perceptual and cognitive explanations for enhanced human-agent team performance. In: Harris, D. (ed.) EPCE 2018. LNCS (LNAI), vol. 10906, pp. 204\u2013214. Springer, Cham (2018). \nhttps:\/\/doi.org\/10.1007\/978-3-319-91122-9_18"},{"issue":"2","key":"6_CR37","doi-asserted-by":"publisher","first-page":"140","DOI":"10.1518\/155534308X284417","volume":"2","author":"R Parasuraman","year":"2008","unstructured":"Parasuraman, R., Sheridan, T.B., Wickens, C.D.: Situation awareness, mental workload, and trust in automation: viable, empirically supported cognitive engineering constructs. J. Cogn. Eng. Decis. Mak. 2(2), 140\u2013160 (2008)","journal-title":"J. Cogn. Eng. Decis. Mak."},{"key":"6_CR38","unstructured":"Preece, A., Harborne, D., Braines, D., Tomsett, R., Chakraborty, S.: Stakeholders in explainable AI. arXiv preprint \narXiv:1810.00184\n\n (2018)"},{"key":"6_CR39","doi-asserted-by":"crossref","unstructured":"Ribeiro, M.T., Singh, S., Guestrin, C.: Why should i trust you?: explaining the predictions of any classifier. In: Proceedings of the 22nd ACM SIGKDD International Conference on Knowledge Discovery and Data Mining, pp. 1135\u20131144. ACM (2016)","DOI":"10.1145\/2939672.2939778"},{"key":"6_CR40","unstructured":"Ribera, M., Lapedriza, A.: Can we do better explanations? A proposal of user-centered explainable AI. In: IUI Workshops (2019)"},{"issue":"4","key":"6_CR41","doi-asserted-by":"publisher","first-page":"297","DOI":"10.1080\/14639220701561775","volume":"9","author":"PM Salmon","year":"2008","unstructured":"Salmon, P.M., et al.: What really is going on? Review of situation awareness models for individuals and teams. Theor. Issues Ergon. Sci. 9(4), 297\u2013323 (2008)","journal-title":"Theor. Issues Ergon. Sci."},{"key":"6_CR42","unstructured":"Sheh, R., Monteath, I.: Introspectively assessing failures through explainable artificial intelligence. In: IROS Workshop on Introspective Methods for Reliable Autonomy (2017)"},{"key":"6_CR43","unstructured":"Sheh, R.K.: Different XAI for different HRI. In: 2017 AAAI Fall Symposium Series (2017)"},{"issue":"1","key":"6_CR44","doi-asserted-by":"publisher","first-page":"137","DOI":"10.1518\/001872095779049444","volume":"37","author":"K Smith","year":"1995","unstructured":"Smith, K., Hancock, P.A.: Situation awareness is adaptive, externally directed consciousness. Hum. Factors 37(1), 137\u2013148 (1995)","journal-title":"Hum. Factors"},{"key":"6_CR45","doi-asserted-by":"crossref","unstructured":"Sreedharan, S., Srivastava, S., Kambhampati, S.: Hierarchical expertise level modeling for user specific contrastive explanations. In: IJCAI, pp. 4829\u20134836 (2018)","DOI":"10.24963\/ijcai.2018\/671"},{"issue":"3","key":"6_CR46","doi-asserted-by":"publisher","first-page":"189","DOI":"10.1016\/S0925-7535(01)00010-8","volume":"39","author":"NA Stanton","year":"2001","unstructured":"Stanton, N.A., Chambers, P.R., Piggott, J.: Situational awareness and safety. Saf. Sci. 39(3), 189\u2013204 (2001)","journal-title":"Saf. Sci."},{"issue":"3","key":"6_CR47","doi-asserted-by":"publisher","first-page":"449","DOI":"10.1518\/001872008X288394","volume":"50","author":"CD Wickens","year":"2008","unstructured":"Wickens, C.D.: Multiple resources and mental workload. Hum. Factors 50(3), 449\u2013455 (2008)","journal-title":"Hum. Factors"}],"container-title":["Lecture Notes in Computer Science","Explainable, Transparent Autonomous Agents and Multi-Agent Systems"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-030-51924-7_6","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2020,7,7]],"date-time":"2020-07-07T00:03:52Z","timestamp":1594080232000},"score":1,"resource":{"primary":{"URL":"http:\/\/link.springer.com\/10.1007\/978-3-030-51924-7_6"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2020]]},"ISBN":["9783030519230","9783030519247"],"references-count":47,"URL":"https:\/\/doi.org\/10.1007\/978-3-030-51924-7_6","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"value":"0302-9743","type":"print"},{"value":"1611-3349","type":"electronic"}],"subject":[],"published":{"date-parts":[[2020]]},"assertion":[{"value":"7 July 2020","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"EXTRAAMAS","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"International Workshop on Explainable, Transparent Autonomous Agents and Multi-Agent Systems","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Auckland","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"New Zealand","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2020","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"9 May 2020","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"13 May 2020","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"extraamas2020","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"https:\/\/extraamas.ehealth.hevs.ch\/index.html","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Single-blind","order":1,"name":"type","label":"Type","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"EasyChair","order":2,"name":"conference_management_system","label":"Conference Management System","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"20","order":3,"name":"number_of_submissions_sent_for_review","label":"Number of Submissions Sent for Review","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"8","order":4,"name":"number_of_full_papers_accepted","label":"Number of Full Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"0","order":5,"name":"number_of_short_papers_accepted","label":"Number of Short Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"40% - The value is computed by the equation \"Number of Full Papers Accepted \/ Number of Submissions Sent for Review * 100\" and then rounded to a whole number.","order":6,"name":"acceptance_rate_of_full_papers","label":"Acceptance Rate of Full Papers","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"3","order":7,"name":"average_number_of_reviews_per_paper","label":"Average Number of Reviews per Paper","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"1","order":8,"name":"average_number_of_papers_per_reviewer","label":"Average Number of Papers per Reviewer","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"No","order":9,"name":"external_reviewers_involved","label":"External Reviewers Involved","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"1 demo paper. The conference was held virtually due to the COVID-19 pandemic.","order":10,"name":"additional_info_on_review_process","label":"Additional Info on Review Process","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"This content has been made available to all.","name":"free","label":"Free to read"}]}}