{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,3,25]],"date-time":"2026-03-25T05:59:15Z","timestamp":1774418355081,"version":"3.50.1"},"publisher-location":"New York, NY, USA","reference-count":71,"publisher":"ACM","license":[{"start":{"date-parts":[[2022,5,19]],"date-time":"2022-05-19T00:00:00Z","timestamp":1652918400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2022,5,19]]},"DOI":"10.1145\/3529320.3529321","type":"proceedings-article","created":{"date-parts":[[2022,5,17]],"date-time":"2022-05-17T16:53:57Z","timestamp":1652806437000},"page":"1-12","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":16,"title":["How Can We Develop Explainable Systems? Insights from a Literature Review and an Interview Study"],"prefix":"10.1145","author":[{"given":"Larissa","family":"Chazette","sequence":"first","affiliation":[{"name":"Software Engineering Group, Leibniz University Hannover, Germany"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Jil","family":"Kl\u00fcnder","sequence":"additional","affiliation":[{"name":"Software Engineering Group, Leibniz University Hannover, Germany"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Merve","family":"Balci","sequence":"additional","affiliation":[{"name":"Leibniz University Hannover, Germany"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Kurt","family":"Schneider","sequence":"additional","affiliation":[{"name":"Software Engineering Group, Leibniz University Hannover, Germany"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2022,5,19]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"publisher","DOI":"10.1109\/RE.2012.6345838"},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.1145\/3334480.3382903"},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1145\/3383313.3412226"},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.inffus.2019.12.012"},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1145\/3236024.3264838"},{"key":"e_1_3_2_1_6_1","volume-title":"Requirements Engineering: Foundation for Software Quality","author":"Brunotte Wasja","unstructured":"Wasja Brunotte, Larissa Chazette, Verena Kl\u00f6s, and Timo Speith. 2022. Quo Vadis, Explainability? \u2013 A Research Roadmap for\u00a0Explainability Engineering. In Requirements Engineering: Foundation for Software Quality, Vincenzo Gervasi and Andreas Vogelsang (Eds.). Springer International Publishing, Cham, 26\u201332."},{"key":"e_1_3_2_1_7_1","volume-title":"CEUR Workshop Proceedings 2327 (1 Jan. 2019). 2019 Joint ACM IUI Workshops, ACMIUI-WS 2019 ; Conference date: 20-03-2019","author":"Brusilovsky Peter","year":"2019","unstructured":"Peter Brusilovsky and Chun Hua Tsai. 2019. Designing explanation interfaces for transparency and beyond. CEUR Workshop Proceedings 2327 (1 Jan. 2019). 2019 Joint ACM IUI Workshops, ACMIUI-WS 2019 ; Conference date: 20-03-2019."},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.1145\/2166966.2166996"},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICHI.2015.26"},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.1145\/3328320.3328402"},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.1109\/RE51729.2021.00025"},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.1007\/s00766-020-00333-1"},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1145\/3290605.3300789"},{"key":"e_1_3_2_1_14_1","volume-title":"Machine Learning and Knowledge Extraction, Andreas Holzinger, Peter Kieseberg, A\u00a0Min Tjoa","author":"Cirqueira Douglas","unstructured":"Douglas Cirqueira, Dietmar Nedbal, Markus Helfert, and Marija Bezbradica. 2020. Scenario-Based Requirements Elicitation for User-Centric Explainable AI. In Machine Learning and Knowledge Extraction, Andreas Holzinger, Peter Kieseberg, A\u00a0Min Tjoa, and Edgar Weippl (Eds.). Springer International Publishing, Cham, 321\u2013341."},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.1145\/3301402"},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.1145\/3411764.3445188"},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-60117-1_33"},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1145\/3172944.3172961"},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","DOI":"10.1007\/s00450-014-0256-x"},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.1111\/j.1471-1842.2009.00848.x"},{"key":"e_1_3_2_1_21_1","volume-title":"Grimaldo and Jasminko Novak","author":"I.","year":"2019","unstructured":"Ana\u00a0I. Grimaldo and Jasminko Novak. 2019. User-Centered Visual Analytics Approach for Interactive and Explainable Energy Demand Analysis in Prosumer Scenarios. In Computer Vision Systems, Dimitrios Tzovaras, Dimitrios Giakoumis, Markus Vincze, and Antonis Argyros (Eds.). Springer International Publishing, Cham, 700\u2013710."},{"key":"e_1_3_2_1_22_1","volume-title":"Proceedings of the IJCAI Workshop on eXplainable Artificial Intelligence (XAI","author":"Hall Mark","year":"2019","unstructured":"Mark Hall, Daniel Harborne, Richard Tomsett, Vedran Galetic, Santiago Quintana-Amate, Alistair Nottle, and Alun Preece. 2019. A systematic method to understand requirements for explainable AI (XAI) systems. In Proceedings of the IJCAI Workshop on eXplainable Artificial Intelligence (XAI 2019), Macau, China, Vol.\u00a011."},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.1145\/2806416.2806504"},{"key":"e_1_3_2_1_24_1","volume-title":"Artificial Intelligence in HCI","author":"Heier Jennifer","unstructured":"Jennifer Heier. 2021. Design Intelligence - Taking Further Steps Towards New Methods and Tools for Designing in the Age of AI. In Artificial Intelligence in HCI, Helmut Degen and Stavroula Ntoa (Eds.). Springer International Publishing, Cham, 202\u2013215."},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.1007\/s42454-020-00021-z"},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.1007\/s13218-020-00636-z"},{"key":"e_1_3_2_1_27_1","first-page":"2019","article-title":"ISO\/IEC 9241-210:2019 Ergonomics of human-system interaction \u2014 Part 210: Human-centred design for interactive systems","volume":"9241","author":"Central Secretary ISO","year":"2019","unstructured":"ISO Central Secretary. 2019. ISO\/IEC 9241-210:2019 Ergonomics of human-system interaction \u2014 Part 210: Human-centred design for interactive systems. Standard ISO 9241-210:2019. International Organization for Standardization. https:\/\/www.iso.org\/standard\/77520.html","journal-title":"Standard ISO"},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"publisher","DOI":"10.1145\/3442188.3445941"},{"key":"e_1_3_2_1_29_1","volume-title":"Systematic literature reviews in software engineering\u2013a systematic literature review. Information and software technology 51, 1","author":"Kitchenham Barbara","year":"2009","unstructured":"Barbara Kitchenham, O\u00a0Pearl Brereton, David Budgen, Mark Turner, John Bailey, and Stephen Linkman. 2009. Systematic literature reviews in software engineering\u2013a systematic literature review. Information and software technology 51, 1 (2009), 7\u201315."},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE-SEIP.2019.00036"},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"publisher","DOI":"10.1145\/3379177.3388898"},{"key":"e_1_3_2_1_32_1","volume-title":"Artificial Intelligence in Medicine: Knowledge Representation and Transparent and Explainable Systems, Mar Marcos, Jose\u00a0M","author":"Kopitar Leon","unstructured":"Leon Kopitar, Leona Cilar, Primoz Kocbek, and Gregor Stiglic. 2019. Local vs. Global Interpretability of Machine Learning Models in Type 2 Diabetes Mellitus Screening. In Artificial Intelligence in Medicine: Knowledge Representation and Transparent and Explainable Systems, Mar Marcos, Jose\u00a0M. Juarez, Richard Lenz, Grzegorz\u00a0J. Nalepa, Slawomir Nowaczyk, Mor Peleg, Jerzy Stefanowski, and Gregor Stiglic (Eds.). Springer International Publishing, Cham, 108\u2013119."},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"publisher","DOI":"10.1109\/VLHCC.2010.15"},{"key":"e_1_3_2_1_34_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.artint.2021.103473"},{"key":"e_1_3_2_1_35_1","doi-asserted-by":"publisher","DOI":"10.1145\/3313831.3376590"},{"key":"e_1_3_2_1_36_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-57321-8_1"},{"key":"e_1_3_2_1_37_1","doi-asserted-by":"publisher","DOI":"10.1007\/s13218-020-00702-6"},{"key":"e_1_3_2_1_38_1","unstructured":"Philipp Mayring. 2019. Qualitative content analysis: Demarcation varieties developments. In Forum: Qualitative Social Research Vol.\u00a020. Freie Universit\u00e4t Berlin 1\u201326."},{"key":"e_1_3_2_1_39_1","volume-title":"Proceedings of the AAAI Conference on Human Computation and Crowdsourcing, Vol.\u00a09. 35\u201347","author":"Melis David\u00a0Alvarez","year":"2021","unstructured":"David\u00a0Alvarez Melis, Harmanpreet Kaur, Hal Daum\u00e9\u00a0III, Hanna Wallach, and Jennifer\u00a0Wortman Vaughan. 2021. From Human Explanation to Model Interpretability: A Framework Based on Weight of Evidence. In Proceedings of the AAAI Conference on Human Computation and Crowdsourcing, Vol.\u00a09. 35\u201347."},{"key":"e_1_3_2_1_40_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICADIWT.2008.4664355"},{"key":"e_1_3_2_1_41_1","doi-asserted-by":"publisher","DOI":"10.1145\/3387166"},{"key":"e_1_3_2_1_42_1","volume-title":"Human-agent Explainability: An Experimental Case Study on the Filtering of Explanations.. In ICAART (1). 378\u2013385.","author":"Mualla Yazan","year":"2020","unstructured":"Yazan Mualla, Igor\u00a0Haman Tchappi, Amro Najjar, Timotheus Kampik, St\u00e9phane Galland, and Christophe Nicolle. 2020. Human-agent Explainability: An Experimental Case Study on the Filtering of Explanations.. In ICAART (1). 378\u2013385."},{"key":"e_1_3_2_1_43_1","doi-asserted-by":"publisher","DOI":"10.1007\/s11280-021-00916-0"},{"key":"e_1_3_2_1_44_1","doi-asserted-by":"publisher","unstructured":"Ingrid Nunes and Dietmar Jannach. 2017. A systematic review and taxonomy of explanations in decision support and recommender systems. User modeling and user-adapted interaction 27 3-5(2017) 393\u2013444. https:\/\/doi.org\/10.1007\/s11257-017-9195-0","DOI":"10.1007\/s11257-017-9195-0"},{"key":"e_1_3_2_1_45_1","doi-asserted-by":"publisher","DOI":"10.1145\/336512.336523"},{"key":"e_1_3_2_1_46_1","doi-asserted-by":"publisher","DOI":"10.1109\/ARSO51874.2021.9542835"},{"key":"e_1_3_2_1_47_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-92325-9_11"},{"key":"e_1_3_2_1_48_1","doi-asserted-by":"publisher","DOI":"10.1145\/3397481.3450676"},{"key":"e_1_3_2_1_49_1","volume-title":"IUI Workshops, Vol.\u00a02327","author":"Ribera Mireia","year":"2019","unstructured":"Mireia Ribera and Agata Lapedriza. 2019. Can we do better explanations? A proposal of user-centered explainable AI.. In IUI Workshops, Vol.\u00a02327. 38."},{"key":"e_1_3_2_1_50_1","volume-title":"Requirements Engineering: Foundation for Software Quality","author":"Schneider Kurt","unstructured":"Kurt Schneider, Melanie Busch, Oliver Karras, Maximilian Schrapel, and Michael Rohs. 2019. Refining Vision Videos. In Requirements Engineering: Foundation for Software Quality, Eric Knauss and Michael Goedicke (Eds.). Springer International Publishing, Cham, 135\u2013150."},{"key":"e_1_3_2_1_51_1","doi-asserted-by":"publisher","DOI":"10.1145\/3411764.3446647"},{"key":"e_1_3_2_1_52_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.ijhcs.2021.102684"},{"key":"e_1_3_2_1_53_1","doi-asserted-by":"publisher","DOI":"10.1145\/3461778.3462012"},{"key":"e_1_3_2_1_54_1","doi-asserted-by":"publisher","DOI":"10.1145\/3490099.3511119"},{"key":"e_1_3_2_1_55_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSSP.2019.00022"},{"key":"e_1_3_2_1_56_1","doi-asserted-by":"publisher","DOI":"10.1186\/s12916-015-0465-6"},{"key":"e_1_3_2_1_57_1","doi-asserted-by":"publisher","DOI":"10.1145\/3301275.3302318"},{"key":"e_1_3_2_1_58_1","doi-asserted-by":"publisher","DOI":"10.1007\/s11257-020-09281-5"},{"key":"e_1_3_2_1_59_1","doi-asserted-by":"publisher","DOI":"10.1145\/3411764.3445101"},{"key":"e_1_3_2_1_60_1","doi-asserted-by":"publisher","DOI":"10.1145\/3290605.3300831"},{"key":"e_1_3_2_1_61_1","doi-asserted-by":"publisher","DOI":"10.18420\/muc2021-mci-ws02-284"},{"key":"e_1_3_2_1_62_1","doi-asserted-by":"publisher","DOI":"10.1145\/3379503.3403554"},{"key":"e_1_3_2_1_63_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-29044-2"},{"key":"e_1_3_2_1_64_1","doi-asserted-by":"publisher","DOI":"10.1145\/3301275.3302317"},{"key":"e_1_3_2_1_65_1","doi-asserted-by":"publisher","DOI":"10.1057\/ejis.2011.51"},{"key":"e_1_3_2_1_66_1","doi-asserted-by":"publisher","DOI":"10.1145\/3313831.3376807"},{"key":"e_1_3_2_1_67_1","doi-asserted-by":"publisher","DOI":"10.1145\/3328485"},{"key":"e_1_3_2_1_68_1","volume-title":"Artificial Intelligence in HCI","author":"Yang Lingxue","unstructured":"Lingxue Yang, Hongrun Wang, and L\u00e9a\u00a0A. Deleris. 2021. What Does It Mean to Explain? A User-Centered Study on AI Explainability. In Artificial Intelligence in HCI, Helmut Degen and Stavroula Ntoa (Eds.). Springer International Publishing, Cham, 107\u2013121."},{"key":"e_1_3_2_1_69_1","doi-asserted-by":"publisher","DOI":"10.1109\/ACCESS.2020.2971712"},{"key":"e_1_3_2_1_70_1","volume-title":"Machine Learning and Knowledge Extraction, Andreas Holzinger, Peter Kieseberg, A\u00a0Min Tjoa","author":"Zhou Jianlong","unstructured":"Jianlong Zhou, Huaiwen Hu, Zhidong Li, Kun Yu, and Fang Chen. 2019. Physiological Indicators for User Trust in Machine Learning with Influence Enhanced Fact-Checking. In Machine Learning and Knowledge Extraction, Andreas Holzinger, Peter Kieseberg, A\u00a0Min Tjoa, and Edgar Weippl (Eds.). Springer International Publishing, Cham, 94\u2013113."},{"key":"e_1_3_2_1_71_1","doi-asserted-by":"publisher","DOI":"10.1145\/3375627.3375856"}],"event":{"name":"ICSSP'22: 16th International Conference on Software and System Processes","location":"Pittsburgh PA USA","acronym":"ICSSP'22"},"container-title":["Proceedings of the International Conference on Software and System Processes and International Conference on Global Software Engineering"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3529320.3529321","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3529320.3529321","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T19:30:38Z","timestamp":1750188638000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3529320.3529321"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2022,5,19]]},"references-count":71,"alternative-id":["10.1145\/3529320.3529321","10.1145\/3529320"],"URL":"https:\/\/doi.org\/10.1145\/3529320.3529321","relation":{},"subject":[],"published":{"date-parts":[[2022,5,19]]},"assertion":[{"value":"2022-05-19","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}