{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,2,27]],"date-time":"2026-02-27T01:47:43Z","timestamp":1772156863318,"version":"3.50.1"},"publisher-location":"New York, NY, USA","reference-count":61,"publisher":"ACM","license":[{"start":{"date-parts":[[2023,4,19]],"date-time":"2023-04-19T00:00:00Z","timestamp":1681862400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2023,4,19]]},"DOI":"10.1145\/3544548.3581373","type":"proceedings-article","created":{"date-parts":[[2023,4,20]],"date-time":"2023-04-20T04:26:08Z","timestamp":1681964768000},"page":"1-16","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":10,"title":["ESCAPE: Countering Systematic Errors from Machine\u2019s Blind Spots via Interactive Visual Analysis"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-5797-5445","authenticated-orcid":false,"given":"Yongsu","family":"Ahn","sequence":"first","affiliation":[{"name":"School of Computing and Information, University of Pittsburgh, United States"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-8497-3015","authenticated-orcid":false,"given":"Yu-Ru","family":"Lin","sequence":"additional","affiliation":[{"name":"School of Computing and Information, University of Pittsburgh, United States"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-7966-2389","authenticated-orcid":false,"given":"Panpan","family":"Xu","sequence":"additional","affiliation":[{"name":"Amazon AWS, United States"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-9613-9434","authenticated-orcid":false,"given":"Zeng","family":"Dai","sequence":"additional","affiliation":[{"name":"Meta, United States"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2023,4,19]]},"reference":[{"key":"e_1_3_3_3_1_1","doi-asserted-by":"publisher","DOI":"10.1109\/TVCG.2019.2934262"},{"key":"e_1_3_3_3_2_1","doi-asserted-by":"publisher","DOI":"10.1109\/TVCG.2014.2346660"},{"key":"e_1_3_3_3_3_1","doi-asserted-by":"publisher","DOI":"10.1145\/2700832"},{"key":"e_1_3_3_3_4_1","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v32i1.11493"},{"key":"e_1_3_3_3_5_1","volume-title":"ICLR workshop on debugging machine learning models.","author":"Barraza Rick","year":"2019","unstructured":"Rick Barraza, Russell Eames, Yan\u00a0Esteve Balducci, Josh Hinds, Scott Hoogerwerf, Eric Horvitz, Ece Kamar, Jacquelyn Krones, Josh Lovejoy, Parham Mohadjer, 2019. Error terrain analysis for machine learning: Tool and visualizations. In ICLR workshop on debugging machine learning models."},{"key":"e_1_3_3_3_6_1","volume-title":"Proceedings of the 29th International Conference on Machine Learning (ICML-12)","author":"Bischof Jonathan","year":"2012","unstructured":"Jonathan Bischof and Edoardo\u00a0M Airoldi. 2012. Summarizing topical content with word frequency and exclusivity. In Proceedings of the 29th International Conference on Machine Learning (ICML-12). 201\u2013208."},{"key":"e_1_3_3_3_7_1","volume-title":"Man is to computer programmer as woman is to homemaker? debiasing word embeddings. Advances in neural information processing systems 29","author":"Bolukbasi Tolga","year":"2016","unstructured":"Tolga Bolukbasi, Kai-Wei Chang, James\u00a0Y Zou, Venkatesh Saligrama, and Adam\u00a0T Kalai. 2016. Man is to computer programmer as woman is to homemaker? debiasing word embeddings. Advances in neural information processing systems 29 (2016)."},{"key":"e_1_3_3_3_8_1","volume-title":"Verification of forecasts expressed in terms of probability. Monthly weather review 78, 1","author":"W Brier","year":"1950","unstructured":"Glenn\u00a0W Brier 1950. Verification of forecasts expressed in terms of probability. Monthly weather review 78, 1 (1950), 1\u20133."},{"key":"e_1_3_3_3_9_1","doi-asserted-by":"publisher","DOI":"10.1145\/3479569"},{"key":"e_1_3_3_3_10_1","volume-title":"FairVis: Visual Analytics for Discovering Intersectional Bias in Machine Learning. arXiv:1904.05419 [cs, stat] (April","author":"Cabrera \u00c1ngel\u00a0Alexander","year":"2019","unstructured":"\u00c1ngel\u00a0Alexander Cabrera, Will Epperson, Fred Hohman, Minsuk Kahng, Jamie Morgenstern, and Duen\u00a0Horng Chau. 2019. FairVis: Visual Analytics for Discovering Intersectional Bias in Machine Learning. arXiv:1904.05419 [cs, stat] (April 2019). http:\/\/arxiv.org\/abs\/1904.05419 arXiv:1904.05419."},{"key":"e_1_3_3_3_11_1","doi-asserted-by":"publisher","DOI":"10.1145\/3542921"},{"key":"e_1_3_3_3_12_1","volume-title":"OoDAnalyzer: Interactive Analysis of Out-of-Distribution Samples. arXiv:2002.03103 [cs] (Feb","author":"Chen Changjian","year":"2020","unstructured":"Changjian Chen, Jun Yuan, Yafeng Lu, Yang Liu, Hang Su, Songtao Yuan, and Shixia Liu. 2020. OoDAnalyzer: Interactive Analysis of Out-of-Distribution Samples. arXiv:2002.03103 [cs] (Feb. 2020). http:\/\/arxiv.org\/abs\/2002.03103 arXiv:2002.03103."},{"key":"e_1_3_3_3_13_1","doi-asserted-by":"publisher","DOI":"10.1038\/s42256-020-00265-z"},{"key":"e_1_3_3_3_14_1","unstructured":"Yeounoh Chung Peter\u00a0J Haas Eli Upfal and Tim Kraska. 2018. Unknown examples & machine learning model generalization. arXiv preprint arXiv:1808.08294(2018)."},{"key":"e_1_3_3_3_15_1","doi-asserted-by":"publisher","DOI":"10.1109\/PacificVis48177.2020.7090"},{"key":"e_1_3_3_3_16_1","volume-title":"There is a blind spot in AI research. Nature 538, 7625","author":"Crawford Kate","year":"2016","unstructured":"Kate Crawford and Ryan Calo. 2016. There is a blind spot in AI research. Nature 538, 7625 (2016), 311\u2013313."},{"key":"e_1_3_3_3_17_1","volume-title":"The Spotlight: A General Method for Discovering Systematic Errors in Deep Learning Models","author":"Eon Greg","year":"2021","unstructured":"Greg d\u2019Eon, Jason d\u2019Eon, James\u00a0R. Wright, and Kevin Leyton-Brown. 2021. The Spotlight: A General Method for Discovering Systematic Errors in Deep Learning Models. http:\/\/arxiv.org\/abs\/2107.00758 arXiv:2107.00758 [cs, stat]."},{"key":"e_1_3_3_3_18_1","doi-asserted-by":"publisher","DOI":"10.1007\/s11263-009-0275-4"},{"key":"e_1_3_3_3_19_1","volume-title":"Domino: Discovering Systematic Errors with Cross-Modal Embeddings","author":"Eyuboglu Sabri","year":"2022","unstructured":"Sabri Eyuboglu, Maya Varma, Khaled Saab, Jean-Benoit Delbrouck, Christopher Lee-Messer, Jared Dunnmon, James Zou, and Christopher R\u00e9. 2022. Domino: Discovering Systematic Errors with Cross-Modal Embeddings. http:\/\/arxiv.org\/abs\/2203.14960 arXiv:2203.14960 [cs]."},{"key":"e_1_3_3_3_20_1","volume-title":"Towards Automatic Concept-based Explanations. arXiv:1902.03129 [cs, stat] (Oct","author":"Ghorbani Amirata","year":"2019","unstructured":"Amirata Ghorbani, James Wexler, James Zou, and Been Kim. 2019. Towards Automatic Concept-based Explanations. arXiv:1902.03129 [cs, stat] (Oct. 2019). http:\/\/arxiv.org\/abs\/1902.03129 arXiv:1902.03129."},{"key":"e_1_3_3_3_21_1","doi-asserted-by":"publisher","DOI":"10.1109\/TVCG.2014.2346260"},{"key":"e_1_3_3_3_22_1","doi-asserted-by":"publisher","DOI":"10.1109\/TVCG.2014.2346260"},{"key":"e_1_3_3_3_23_1","doi-asserted-by":"publisher","DOI":"10.1609\/hcomp.v9i1.18941"},{"key":"e_1_3_3_3_24_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2016.90"},{"key":"e_1_3_3_3_25_1","doi-asserted-by":"publisher","DOI":"10.1145\/3027063.3053103"},{"key":"e_1_3_3_3_26_1","doi-asserted-by":"publisher","DOI":"10.1109\/TVCG.2022.3209466"},{"key":"e_1_3_3_3_27_1","unstructured":"Jinbin Huang Aditi Mishra Bum-Chul Kwon and Chris Bryan. 2022. ConceptExplainer: Understanding the Mental Model of Deep Learning Algorithms via Interactive Concept-based Explanations. http:\/\/arxiv.org\/abs\/2204.01888 arXiv:2204.01888 [cs]."},{"key":"e_1_3_3_3_28_1","doi-asserted-by":"publisher","DOI":"10.1007\/s12650-019-00607-z"},{"key":"e_1_3_3_3_29_1","unstructured":"Kaggle. 2014. Dogs vs. Cats. https:\/\/www.kaggle.com\/c\/dogs-vs-cats"},{"key":"e_1_3_3_3_30_1","doi-asserted-by":"publisher","DOI":"10.1109\/TVCG.2021.3114779"},{"key":"e_1_3_3_3_31_1","volume-title":"Advances in Neural Information Processing Systems 27, Z.\u00a0Ghahramani, M.\u00a0Welling, C.\u00a0Cortes, N.\u00a0D.","author":"Kim Been","year":"1952","unstructured":"Been Kim, Cynthia Rudin, and Julie\u00a0A Shah. 2014. The Bayesian Case Model: A Generative Approach for Case-Based Reasoning and Prototype Classification. In Advances in Neural Information Processing Systems 27, Z.\u00a0Ghahramani, M.\u00a0Welling, C.\u00a0Cortes, N.\u00a0D. Lawrence, and K.\u00a0Q. Weinberger (Eds.). Curran Associates, Inc., 1952\u20131960. http:\/\/papers.nips.cc\/paper\/5313-the-bayesian-case-model-a-generative-approach-for-case-based-reasoning-and-prototype-classification.pdf"},{"key":"e_1_3_3_3_32_1","volume-title":"Interpretability Beyond Feature Attribution: Quantitative Testing with Concept Activation Vectors (TCAV). arXiv:1711.11279 [stat] (June","author":"Kim Been","year":"2018","unstructured":"Been Kim, Martin Wattenberg, Justin Gilmer, Carrie Cai, James Wexler, Fernanda Viegas, and Rory Sayres. 2018. Interpretability Beyond Feature Attribution: Quantitative Testing with Concept Activation Vectors (TCAV). arXiv:1711.11279 [stat] (June 2018). http:\/\/arxiv.org\/abs\/1711.11279 arXiv:1711.11279."},{"key":"e_1_3_3_3_33_1","doi-asserted-by":"publisher","DOI":"10.1109\/VAST.2017.8585720"},{"key":"e_1_3_3_3_34_1","volume-title":"Eurographics Conference on Visualization.","author":"Kwon Bum\u00a0Chul","year":"2022","unstructured":"Bum\u00a0Chul Kwon, Jungsoo Lee, Chaeyeon Chung, Nyoungwoo Lee, Ho-Jin Choi, and Jaegul Choo. 2022. DASH: Visual Analytics for Debiasing Image Classification via User-Driven Synthetic Data Augmentation. In Eurographics Conference on Visualization."},{"key":"e_1_3_3_3_35_1","doi-asserted-by":"publisher","DOI":"10.2312\/evs.20221099"},{"key":"e_1_3_3_3_36_1","doi-asserted-by":"crossref","unstructured":"Himabindu Lakkaraju Ece Kamar Rich Caruana and Eric Horvitz. 2017. Identifying unknown unknowns in the open world: Representations and policies for guided exploration. In Thirty-first aaai conference on artificial intelligence.","DOI":"10.1609\/aaai.v31i1.10821"},{"key":"e_1_3_3_3_37_1","volume-title":"MaskGAN: Towards Diverse and Interactive Facial Image Manipulation. In IEEE Conference on Computer Vision and Pattern Recognition (CVPR).","author":"Lee Cheng-Han","year":"2020","unstructured":"Cheng-Han Lee, Ziwei Liu, Lingyun Wu, and Ping Luo. 2020. MaskGAN: Towards Diverse and Interactive Facial Image Manipulation. In IEEE Conference on Computer Vision and Pattern Recognition (CVPR)."},{"key":"e_1_3_3_3_38_1","doi-asserted-by":"publisher","DOI":"10.1145\/3366423.3380306"},{"key":"e_1_3_3_3_39_1","volume-title":"Analyzing the training processes of deep generative models","author":"Liu Mengchen","year":"2017","unstructured":"Mengchen Liu, Jiaxin Shi, Kelei Cao, Jun Zhu, and Shixia Liu. 2017. Analyzing the training processes of deep generative models. IEEE transactions on visualization and computer graphics 24, 1(2017), 77\u201387."},{"key":"e_1_3_3_3_40_1","volume-title":"Umap: Uniform manifold approximation and projection for dimension reduction. arXiv preprint arXiv:1802.03426(2018).","author":"McInnes Leland","year":"2018","unstructured":"Leland McInnes, John Healy, and James Melville. 2018. Umap: Uniform manifold approximation and projection for dimension reduction. arXiv preprint arXiv:1802.03426(2018)."},{"key":"e_1_3_3_3_41_1","doi-asserted-by":"publisher","DOI":"10.1109\/TVCG.2019.2934267"},{"key":"e_1_3_3_3_42_1","doi-asserted-by":"publisher","DOI":"10.1109\/VIS49827.2021.9623289"},{"key":"e_1_3_3_3_43_1","doi-asserted-by":"publisher","DOI":"10.1109\/TVCG.2017.2744478"},{"key":"e_1_3_3_3_44_1","volume-title":"NeuroCartography: Scalable Automatic Visual Summarization of Concepts in Deep Neural Networks. arXiv:2108.12931 [cs] (Aug","author":"Park Haekyu","year":"2021","unstructured":"Haekyu Park, Nilaksh Das, Rahul Duggal, Austin\u00a0P. Wright, Omar Shaikh, Fred Hohman, and Duen\u00a0Horng Chau. 2021. NeuroCartography: Scalable Automatic Visual Summarization of Concepts in Deep Neural Networks. arXiv:2108.12931 [cs] (Aug. 2021). http:\/\/arxiv.org\/abs\/2108.12931 arXiv:2108.12931."},{"key":"e_1_3_3_3_45_1","unstructured":"Gregory Plumb Nari Johnson \u00c1ngel\u00a0Alexander Cabrera Marco\u00a0Tulio Ribeiro and Ameet Talwalkar. 2022. Evaluating Systemic Error Detection Methods using Synthetic Images. http:\/\/arxiv.org\/abs\/2207.04104 arXiv:2207.04104 [cs]."},{"key":"e_1_3_3_3_46_1","volume-title":"SEAL : Interactive Tool for Systematic Error Analysis and Labeling","author":"Rajani Nazneen","year":"2022","unstructured":"Nazneen Rajani, Weixin Liang, Lingjiao Chen, Meg Mitchell, and James Zou. 2022. SEAL : Interactive Tool for Systematic Error Analysis and Labeling. http:\/\/arxiv.org\/abs\/2210.05839 arXiv:2210.05839 [cs]."},{"key":"e_1_3_3_3_47_1","volume-title":"TopoAct: Visually Exploring the Shape of Activations in Deep Learning. arXiv:1912.06332 [cs] (July","author":"Rathore Archit","year":"2020","unstructured":"Archit Rathore, Nithin Chalapathi, Sourabh Palande, and Bei Wang. 2020. TopoAct: Visually Exploring the Shape of Activations in Deep Learning. arXiv:1912.06332 [cs] (July 2020). http:\/\/arxiv.org\/abs\/1912.06332 arXiv:1912.06332."},{"key":"e_1_3_3_3_48_1","doi-asserted-by":"publisher","DOI":"10.1109\/TVCG.2016.2598828"},{"key":"e_1_3_3_3_49_1","volume-title":"Explaining the Predictions of Any Classifier. arXiv:1602.04938 [cs, stat] (Feb","author":"Ribeiro Marco\u00a0Tulio","year":"2016","unstructured":"Marco\u00a0Tulio Ribeiro, Sameer Singh, and Carlos Guestrin. 2016. \"Why Should I Trust You?\": Explaining the Predictions of Any Classifier. arXiv:1602.04938 [cs, stat] (Feb. 2016). http:\/\/arxiv.org\/abs\/1602.04938 arXiv:1602.04938."},{"key":"e_1_3_3_3_50_1","doi-asserted-by":"publisher","DOI":"10.1145\/3415224"},{"key":"e_1_3_3_3_51_1","doi-asserted-by":"publisher","DOI":"10.1145\/3241379"},{"key":"e_1_3_3_3_52_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-01768-2_27"},{"key":"e_1_3_3_3_53_1","volume-title":"arXiv:1808.09123 [cs, stat] (Dec","author":"Tan Sarah","year":"2018","unstructured":"Sarah Tan, Julius Adebayo, Kori Inkpen, and Ece Kamar. 2018. Investigating Human + Machine Complementarity for Recidivism Predictions. arXiv:1808.09123 [cs, stat] (Dec. 2018). http:\/\/arxiv.org\/abs\/1808.09123 arXiv:1808.09123."},{"key":"e_1_3_3_3_54_1","doi-asserted-by":"crossref","unstructured":"Colin Vandenhof and Edith Law. 2019. Contradict the Machine: A Hybrid Approach to Identifying Unknown Unknowns. (2019) 3.","DOI":"10.1609\/hcomp.v7i1.5274"},{"key":"e_1_3_3_3_55_1","doi-asserted-by":"publisher","DOI":"10.1109\/TVCG.2019.2903943"},{"key":"e_1_3_3_3_56_1","doi-asserted-by":"publisher","DOI":"10.1109\/VAST50239.2020.00006"},{"key":"e_1_3_3_3_57_1","doi-asserted-by":"publisher","DOI":"10.1109\/TVCG.2019.2934619"},{"key":"e_1_3_3_3_58_1","volume-title":"Diagnosing Concept Drift with Visual Analytics. arXiv:2007.14372 [cs, stat] (Sept","author":"Yang Weikai","year":"2020","unstructured":"Weikai Yang, Zhen Li, Mengchen Liu, Yafeng Lu, Kelei Cao, Ross Maciejewski, and Shixia Liu. 2020. Diagnosing Concept Drift with Visual Analytics. arXiv:2007.14372 [cs, stat] (Sept. 2020). http:\/\/arxiv.org\/abs\/2007.14372 arXiv:2007.14372."},{"key":"e_1_3_3_3_59_1","doi-asserted-by":"publisher","DOI":"10.1109\/TVCG.2018.2864499"},{"key":"e_1_3_3_3_60_1","doi-asserted-by":"crossref","unstructured":"Jieyu Zhao Yichao Zhou Zeyu Li Wei Wang and Kai-Wei Chang. 2018. Learning gender-neutral word embeddings. arXiv preprint arXiv:1809.01496(2018).","DOI":"10.18653\/v1\/D18-1521"},{"key":"e_1_3_3_3_61_1","doi-asserted-by":"publisher","DOI":"10.1109\/TVCG.2021.3114837"}],"event":{"name":"CHI '23: CHI Conference on Human Factors in Computing Systems","location":"Hamburg Germany","acronym":"CHI '23","sponsor":["SIGCHI ACM Special Interest Group on Computer-Human Interaction"]},"container-title":["Proceedings of the 2023 CHI Conference on Human Factors in Computing Systems"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3544548.3581373","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3544548.3581373","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T16:47:30Z","timestamp":1750178850000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3544548.3581373"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023,4,19]]},"references-count":61,"alternative-id":["10.1145\/3544548.3581373","10.1145\/3544548"],"URL":"https:\/\/doi.org\/10.1145\/3544548.3581373","relation":{},"subject":[],"published":{"date-parts":[[2023,4,19]]},"assertion":[{"value":"2023-04-19","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}