{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,7]],"date-time":"2026-07-07T22:21:54Z","timestamp":1783462914496,"version":"3.55.0"},"publisher-location":"New York, NY, USA","reference-count":68,"publisher":"ACM","license":[{"start":{"date-parts":[[2020,4,21]],"date-time":"2020-04-21T00:00:00Z","timestamp":1587427200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2020,4,21]]},"DOI":"10.1145\/3313831.3376219","type":"proceedings-article","created":{"date-parts":[[2020,5,27]],"date-time":"2020-05-27T12:35:07Z","timestamp":1590582907000},"page":"1-14","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":374,"title":["Interpreting Interpretability: Understanding Data Scientists' Use of Interpretability Tools for Machine Learning"],"prefix":"10.1145","author":[{"given":"Harmanpreet","family":"Kaur","sequence":"first","affiliation":[{"name":"University of Michigan, Ann Arbor, MI, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Harsha","family":"Nori","sequence":"additional","affiliation":[{"name":"Microsoft Research, Seattle, WA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Samuel","family":"Jenkins","sequence":"additional","affiliation":[{"name":"Microsoft Research, Redmond, WA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Rich","family":"Caruana","sequence":"additional","affiliation":[{"name":"Microsoft Research, Redmond, WA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Hanna","family":"Wallach","sequence":"additional","affiliation":[{"name":"Microsoft Research, New York City, NY, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Jennifer","family":"Wortman Vaughan","sequence":"additional","affiliation":[{"name":"Microsoft Research, New York, NY, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2020,4,23]]},"reference":[{"key":"e_1_3_2_2_1_1","doi-asserted-by":"publisher","DOI":"10.1145\/3173574.3174156"},{"key":"e_1_3_2_2_2_1","volume-title":"Jennifer Wortman Vaughan, and Hanna Wallach","author":"Alvarez-Melis David","year":"2019","unstructured":"David Alvarez-Melis, Hal Daum\u00e9, III, Jennifer Wortman Vaughan, and Hanna Wallach. 2019. Weight of Evidence as a Basis for Human-Oriented Explanations. arXiv preprint arXiv:1910.13503 (2019)."},{"key":"e_1_3_2_2_3_1","doi-asserted-by":"publisher","DOI":"10.1145\/2207676.2207680"},{"key":"e_1_3_2_2_4_1","volume-title":"ProPublica","author":"Angwin Julia","year":"2016","unstructured":"Julia Angwin, Jeff Larson, Surya Mattu, and Kirchner Lauren. 2016. Machine Bias: There's software used across the country to predict future criminals. And it's biased against blacks. ProPublica, May 23 (2016), 2016. http:\/\/www.propublica.org\/article\/machine-bias-risk -assessments-in-criminal-sentencing"},{"key":"e_1_3_2_2_5_1","volume-title":"Communication theory","author":"Barnlund Dean C","unstructured":"Dean C Barnlund. 2017. A transactional model of communication. In Communication theory, Second edition, C. David Mortensen (Ed.). Routledge, 47--57."},{"key":"e_1_3_2_2_6_1","doi-asserted-by":"publisher","DOI":"10.1207\/S15327051HCI16234_05"},{"key":"e_1_3_2_2_7_1","doi-asserted-by":"publisher","DOI":"10.1037\/13620-004"},{"key":"e_1_3_2_2_8_1","doi-asserted-by":"publisher","DOI":"10.1145\/3359206"},{"key":"e_1_3_2_2_9_1","doi-asserted-by":"publisher","DOI":"10.1126\/science.aal4230"},{"key":"e_1_3_2_2_10_1","doi-asserted-by":"publisher","DOI":"10.1145\/2783258.2788613"},{"key":"e_1_3_2_2_11_1","doi-asserted-by":"publisher","DOI":"10.1016\/S0022-5371(83)90189-5"},{"key":"e_1_3_2_2_12_1","doi-asserted-by":"publisher","DOI":"10.1007\/BF00988593"},{"key":"e_1_3_2_2_13_1","doi-asserted-by":"publisher","DOI":"10.1007\/978--3--540--30116--5_58"},{"key":"e_1_3_2_2_14_1","volume-title":"Towards a rigorous science of interpretable machine learning. arXiv preprint arXiv:1702.08608","author":"Doshi-Velez Finale","year":"2017","unstructured":"Finale Doshi-Velez and Been Kim. 2017. Towards a rigorous science of interpretable machine learning. arXiv preprint arXiv:1702.08608 (2017)."},{"key":"e_1_3_2_2_15_1","volume-title":"Accountability of AI under the law: The role of explanation. arXiv preprint arXiv:1711.01134","author":"Doshi-Velez Finale","year":"2017","unstructured":"Finale Doshi-Velez, Mason Kortz, Ryan Budish, Chris Bavitz, Sam Gershman, David O'Brien, Stuart Schieber, James Waldo, David Weinberger, and Alexandra Wood. 2017. Accountability of AI under the law: The role of explanation. arXiv preprint arXiv:1711.01134 (2017)."},{"key":"e_1_3_2_2_16_1","doi-asserted-by":"publisher","DOI":"10.1177\/2053951716665128"},{"key":"e_1_3_2_2_18_1","doi-asserted-by":"publisher","DOI":"10.1145\/604045.604056"},{"key":"e_1_3_2_2_19_1","doi-asserted-by":"publisher","DOI":"10.1109\/dsaa.2018.00018"},{"key":"e_1_3_2_2_20_1","doi-asserted-by":"publisher","unstructured":"Herbert P. Grice. 1975. Logic and Conversation. (1975) 41--58. DOI: http:\/\/dx.doi.org\/10.1163\/9789004368811_003","DOI":"10.1163\/9789004368811_003"},{"key":"e_1_3_2_2_21_1","doi-asserted-by":"publisher","DOI":"10.1016\/S0166--4115(08)62386--9"},{"key":"e_1_3_2_2_22_1","doi-asserted-by":"publisher","DOI":"10.1080\/01621459.1987.10478440"},{"key":"e_1_3_2_2_23_1","doi-asserted-by":"publisher","DOI":"10.1145\/3290605.3300809"},{"key":"e_1_3_2_2_24_1","doi-asserted-by":"publisher","DOI":"10.1207\/S15327566IJCE0401_04"},{"key":"e_1_3_2_2_25_1","doi-asserted-by":"publisher","DOI":"10.2139\/ssrn.2919024"},{"key":"e_1_3_2_2_26_1","doi-asserted-by":"publisher","DOI":"10.1038\/nmeth.2281"},{"key":"e_1_3_2_2_27_1","volume-title":"fast and slow","author":"Kahneman Daniel","unstructured":"Daniel Kahneman. 2011. Thinking, fast and slow. Macmillan."},{"key":"e_1_3_2_2_28_1","volume-title":"Paul Slovic, and Amos Tversky.","author":"Kahneman Daniel","year":"1982","unstructured":"Daniel Kahneman, Stewart Paul Slovic, Paul Slovic, and Amos Tversky. 1982. Judgment under uncertainty: Heuristics and biases. Cambridge university press."},{"key":"e_1_3_2_2_29_1","doi-asserted-by":"publisher","DOI":"10.1145\/2702123.2702520"},{"key":"e_1_3_2_2_30_1","first-page":"I","article-title":"LightGBM: A Highly Efficient Gradient Boosting Decision Tree","volume":"30","author":"Ke Guolin","year":"2017","unstructured":"Guolin Ke, Qi Meng, Thomas Finley, Taifeng Wang, Wei Chen, Weidong Ma, Qiwei Ye, and Tie-Yan Liu. 2017. LightGBM: A Highly Efficient Gradient Boosting Decision Tree. In Advances in Neural Information Processing Systems 30, I. Guyon, U. V. Luxburg, S. Bengio, H. Wallach, R. Fergus, S. Vishwanathan, and R. Garnett (Eds.). Curran Associates, Inc., 3146--3154. http:\/\/papers.nips.cc\/paper\/6907-lightgbm-a-highly-efficient-gradient-boosting-decision-tree.pdf","journal-title":"Advances in Neural Information Processing Systems"},{"key":"e_1_3_2_2_31_1","volume-title":"Advances in Neural Information Processing Systems 29","author":"Kim Been","unstructured":"Been Kim, Rajiv Khanna, and Oluwasanmi O Koyejo. 2016. Examples are not enough, learn to criticize! Criticism for Interpretability. In Advances in Neural Information Processing Systems 29, D. D. Lee, M. Sugiyama, U. V. Luxburg, I. Guyon, and R. Garnett (Eds.). Curran Associates, Inc., 2280--2288. http:\/\/papers.nips.cc\/paper\/6300-examples-are-not-enough-learn-to-criticize-criticism-for-interpretability.pdf"},{"key":"e_1_3_2_2_32_1","volume-title":"Proceedings of the 35th International Conference on Machine Learning (Proceedings of Machine Learning Research), Jennifer Dy and Andreas Krause (Eds.)","volume":"80","author":"Kim Been","year":"2018","unstructured":"Been Kim, Martin Wattenberg, Justin Gilmer, Carrie Cai, James Wexler, Fernanda Viegas, and Rory Sayres. 2018. Interpretability Beyond Feature Attribution: Quantitative Testing with Concept Activation Vectors (TCAV). In Proceedings of the 35th International Conference on Machine Learning (Proceedings of Machine Learning Research), Jennifer Dy and Andreas Krause (Eds.), Vol. 80. PMLR, Stockholmsm\u00e4ssan, Stockholm Sweden, 2668--2677. http:\/\/proceedings.mlr.press\/v80\/kim18d.html"},{"key":"e_1_3_2_2_33_1","doi-asserted-by":"publisher","DOI":"10.1145\/3290605.3300641"},{"key":"e_1_3_2_2_34_1","doi-asserted-by":"publisher","DOI":"10.1145\/2678025.2701399"},{"key":"e_1_3_2_2_35_1","doi-asserted-by":"publisher","DOI":"10.1145\/2207676.2207678"},{"key":"e_1_3_2_2_36_1","volume-title":"Human Evaluation of Models Built for Interpretability. In AAAI Conference on Human Computation and Crowdsourcing (HCOMP).","author":"Lage Isaac","year":"2019","unstructured":"Isaac Lage, Emily Chen, Jeffrey He, Menaka Narayanan, Sam Gershman, Been Kim, and Finale Doshi-Velez. 2019. Human Evaluation of Models Built for Interpretability. In AAAI Conference on Human Computation and Crowdsourcing (HCOMP)."},{"key":"e_1_3_2_2_37_1","doi-asserted-by":"publisher","DOI":"10.1145\/3375627.3375833"},{"key":"e_1_3_2_2_38_1","doi-asserted-by":"publisher","DOI":"10.1016\/0364-0213(91)80017-Y"},{"key":"e_1_3_2_2_39_1","volume-title":"The mythos of model interpretability. arXiv preprint arXiv:1606.03490","author":"Lipton Zachary C","year":"2016","unstructured":"Zachary C Lipton. 2016. The mythos of model interpretability. arXiv preprint arXiv:1606.03490 (2016)."},{"key":"e_1_3_2_2_40_1","doi-asserted-by":"publisher","DOI":"10.1177\/0163443719855301"},{"key":"e_1_3_2_2_41_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.tics.2006.08.004"},{"key":"e_1_3_2_2_42_1","doi-asserted-by":"publisher","DOI":"10.1109\/ACCESS.2017.2696365"},{"key":"e_1_3_2_2_43_1","volume-title":"Consistent individualized feature attribution for tree ensembles. arXiv preprint arXiv:1802.03888","author":"Lundberg Scott M","year":"2018","unstructured":"Scott M Lundberg, Gabriel G Erion, and Su-In Lee. 2018. Consistent individualized feature attribution for tree ensembles. arXiv preprint arXiv:1802.03888 (2018)."},{"key":"e_1_3_2_2_44_1","first-page":"I","article-title":"A Unified Approach to Interpreting Model Predictions","volume":"30","author":"Lundberg Scott M","year":"2017","unstructured":"Scott M Lundberg and Su-In Lee. 2017. A Unified Approach to Interpreting Model Predictions. In Advances in Neural Information Processing Systems 30, I. Guyon, U. V. Luxburg, S. Bengio, H. Wallach, R. Fergus, S. Vishwanathan, and R. Garnett (Eds.). Curran Associates, Inc., 4765--4774. http:\/\/papers.nips.cc\/paper\/7062-a-unified-approach -to-interpreting-model-predictions.pdf","journal-title":"Advances in Neural Information Processing Systems"},{"key":"e_1_3_2_2_45_1","volume-title":"First international workshop on socio-cognitive systems at IJCAI 2018","author":"Madumal Prashan","year":"2018","unstructured":"Prashan Madumal, Tim Miller, Frank Vetere, and Liz Sonenberg. 2018. Towards a Grounded Dialog Model for Explainable Artificial Intelligence. In First international workshop on socio-cognitive systems at IJCAI 2018. https:\/\/arxiv.org\/abs\/1806.08055"},{"key":"e_1_3_2_2_46_1","volume-title":"How the mind explains behavior: Folk explanations, meaning, and social interaction","author":"Malle Bertram F","unstructured":"Bertram F Malle. 2006. How the mind explains behavior: Folk explanations, meaning, and social interaction. Mit Press."},{"key":"e_1_3_2_2_47_1","volume-title":"Explanation in artificial intelligence: Insights from the social sciences. Artificial Intelligence","author":"Miller Tim","year":"2018","unstructured":"Tim Miller. 2018. Explanation in artificial intelligence: Insights from the social sciences. Artificial Intelligence (2018)."},{"key":"e_1_3_2_2_48_1","volume-title":"IJCAI 2017 Workshop on Explainable Artificial Intelligence (XAI).","author":"Miller Tim","year":"2017","unstructured":"Tim Miller, Piers Howe, and Liz Sonenberg. 2017. Explainable AI: Beware of inmates running the asylum or: How I learnt to stop worrying and love the social and behavioural sciences. In IJCAI 2017 Workshop on Explainable Artificial Intelligence (XAI)."},{"key":"e_1_3_2_2_49_1","doi-asserted-by":"publisher","DOI":"10.1145\/3290605.3300356"},{"key":"e_1_3_2_2_50_1","volume-title":"InterpretML: A Unified Framework for Machine Learning Interpretability. arXiv preprint arXiv:1909.09223","author":"Nori Harsha","year":"2019","unstructured":"Harsha Nori, Samuel Jenkins, Paul Koch, and Rich Caruana. 2019. InterpretML: A Unified Framework for Machine Learning Interpretability. arXiv preprint arXiv:1909.09223 (2019)."},{"key":"e_1_3_2_2_51_1","volume-title":"Mental models","author":"Norman Donald A","unstructured":"Donald A Norman. 2014. Some observations on mental models. In Mental models. Psychology Press, 15--22."},{"key":"e_1_3_2_2_52_1","doi-asserted-by":"publisher","DOI":"10.1207\/s15327108ijap0301_1"},{"key":"e_1_3_2_2_53_1","doi-asserted-by":"publisher","DOI":"10.1145\/1357054.1357160"},{"key":"e_1_3_2_2_54_1","volume-title":"Jennifer Wortman Vaughan, and Hanna Wallach","author":"Poursabzi-Sangdeh Forough","year":"2018","unstructured":"Forough Poursabzi-Sangdeh, Daniel G Goldstein, Jake M Hofman, Jennifer Wortman Vaughan, and Hanna Wallach. 2018. Manipulating and measuring model interpretability. arXiv preprint arXiv:1802.07810 (2018)."},{"key":"e_1_3_2_2_55_1","doi-asserted-by":"publisher","DOI":"10.1145\/2939672.2939778"},{"key":"e_1_3_2_2_56_1","doi-asserted-by":"publisher","DOI":"10.1038\/s42256-019-0048-x"},{"key":"e_1_3_2_2_57_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2017.74"},{"key":"e_1_3_2_2_58_1","volume-title":"A value for n-person games. Classics in game theory","author":"Shapley Lloyd S","year":"1997","unstructured":"Lloyd S Shapley. 1997. A value for n-person games. Classics in game theory (1997), 69."},{"key":"e_1_3_2_2_59_1","doi-asserted-by":"publisher","DOI":"10.1002\/ejsp.2420230302"},{"key":"e_1_3_2_2_60_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.ijhcs.2009.03.004"},{"key":"e_1_3_2_2_61_1","volume-title":"Learning global additive explanations for neural nets using model distillation. arXiv preprint arXiv:1801.08640","author":"Tan Sarah","year":"2018","unstructured":"Sarah Tan, Rich Caruana, Giles Hooker, Paul Koch, and Albert Gordo. 2018. Learning global additive explanations for neural nets using model distillation. arXiv preprint arXiv:1801.08640 (2018)."},{"key":"e_1_3_2_2_62_1","volume-title":"Interpretable to whom? A role-based model for analyzing interpretable machine learning systems. arXiv preprint arXiv:1806.07552","author":"Tomsett Richard","year":"2018","unstructured":"Richard Tomsett, Dave Braines, Dan Harborne, Alun Preece, and Supriyo Chakraborty. 2018. Interpretable to whom? A role-based model for analyzing interpretable machine learning systems. arXiv preprint arXiv:1806.07552 (2018)."},{"key":"e_1_3_2_2_63_1","doi-asserted-by":"publisher","DOI":"10.1145\/2110363.2110464"},{"key":"e_1_3_2_2_64_1","doi-asserted-by":"publisher","DOI":"10.1145\/3290605.3300831"},{"key":"e_1_3_2_2_65_1","doi-asserted-by":"publisher","DOI":"10.1145\/3282486"},{"key":"e_1_3_2_2_66_1","doi-asserted-by":"publisher","DOI":"10.1145\/3173574.3173704"},{"key":"e_1_3_2_2_67_1","doi-asserted-by":"publisher","DOI":"10.1145\/3196709.3196730"},{"key":"e_1_3_2_2_68_1","doi-asserted-by":"publisher","DOI":"10.1111\/rssa.12227"},{"key":"e_1_3_2_2_69_1","doi-asserted-by":"publisher","DOI":"10.1109\/CIG.2018.8490433"}],"event":{"name":"CHI '20: CHI Conference on Human Factors in Computing Systems","location":"Honolulu HI USA","acronym":"CHI '20","sponsor":["SIGCHI ACM Special Interest Group on Computer-Human Interaction"]},"container-title":["Proceedings of the 2020 CHI Conference on Human Factors in Computing Systems"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3313831.3376219","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3313831.3376219","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T23:45:03Z","timestamp":1750203903000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3313831.3376219"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2020,4,21]]},"references-count":68,"alternative-id":["10.1145\/3313831.3376219","10.1145\/3313831"],"URL":"https:\/\/doi.org\/10.1145\/3313831.3376219","relation":{},"subject":[],"published":{"date-parts":[[2020,4,21]]},"assertion":[{"value":"2020-04-23","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}