{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,6,24]],"date-time":"2025-06-24T11:31:42Z","timestamp":1750764702541,"version":"3.41.0"},"publisher-location":"New York, NY, USA","reference-count":68,"publisher":"ACM","license":[{"start":{"date-parts":[[2022,6,20]],"date-time":"2022-06-20T00:00:00Z","timestamp":1655683200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"funder":[{"DOI":"10.13039\/100000185","name":"Defense Advanced Research Projects Agency","doi-asserted-by":"publisher","award":["FA8750-20-C-0156, FA8750-20-C-0074, and FA8750-20-C0155"],"award-info":[{"award-number":["FA8750-20-C-0156, FA8750-20-C-0074, and FA8750-20-C0155"]}],"id":[{"id":"10.13039\/100000185","id-type":"DOI","asserted-by":"publisher"}]},{"name":"Google Research Scholar Award"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2022,6,21]]},"DOI":"10.1145\/3531146.3533145","type":"proceedings-article","created":{"date-parts":[[2022,6,20]],"date-time":"2022-06-20T14:27:10Z","timestamp":1655735230000},"page":"799-813","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":1,"title":["Trucks Don\u2019t Mean Trump: Diagnosing Human Error in Image Analysis"],"prefix":"10.1145","author":[{"given":"J.D.","family":"Zamfirescu-Pereira","sequence":"first","affiliation":[{"name":"UC Berkeley, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Jerry","family":"Chen","sequence":"additional","affiliation":[{"name":"Stanford University, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Emily","family":"Wen","sequence":"additional","affiliation":[{"name":"Stanford University, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Allison","family":"Koenecke","sequence":"additional","affiliation":[{"name":"Cornell University, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Nikhil","family":"Garg","sequence":"additional","affiliation":[{"name":"Cornell Tech, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Emma","family":"Pierson","sequence":"additional","affiliation":[{"name":"Cornell University, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2022,6,20]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"publisher","DOI":"10.1145\/3351095.3372871"},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.1609\/icwsm.v13i01.3360"},{"key":"e_1_3_2_1_3_1","unstructured":"Julius Adebayo Justin Gilmer Michael Muelly Ian Goodfellow Moritz Hardt and Been Kim. 2018. Sanity checks for saliency maps. arXiv preprint arXiv:1810.03292(2018)."},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.1073\/pnas.1915841117"},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1145\/3359301"},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"crossref","unstructured":"Ashton Anderson Jon Kleinberg and Sendhil Mullainathan. 2017. Assessing human error against a benchmark of perfection. ACM Transactions on Knowledge Discovery from Data (TKDD) 11 4(2017) 1\u201325.","DOI":"10.1145\/3046947"},{"key":"e_1_3_2_1_7_1","volume-title":"What We Learned From 15 Million Guesses About a Neighborhood\u2019s Politics. The New York Times","author":"Badger Emily","year":"2021","unstructured":"Emily Badger, Josh Katz, and Kevin Quealy. 2021. What We Learned From 15 Million Guesses About a Neighborhood\u2019s Politics. The New York Times (2021)."},{"key":"e_1_3_2_1_8_1","volume-title":"Do you think you can tell how a neighborhood voted just by looking around?The New York Times","author":"Badger Emily","year":"2021","unstructured":"Emily Badger, Josh Katz, Kevin Quealy, and Rumsey Taylor. 2021. Do you think you can tell how a neighborhood voted just by looking around?The New York Times (2021)."},{"volume-title":"The economics of discrimination","author":"Becker S","key":"e_1_3_2_1_9_1","unstructured":"Gary\u00a0S Becker. 2010. The economics of discrimination. University of Chicago press."},{"key":"e_1_3_2_1_10_1","volume-title":"Deep-learning-assisted diagnosis for knee magnetic resonance imaging: development and retrospective validation of MRNet. PLoS medicine 15, 11","author":"Bien Nicholas","year":"2018","unstructured":"Nicholas Bien, Pranav Rajpurkar, Robyn\u00a0L Ball, Jeremy Irvin, Allison Park, Erik Jones, Michael Bereket, Bhavik\u00a0N Patel, Kristen\u00a0W Yeom, Katie Shpanskaya, 2018. Deep-learning-assisted diagnosis for knee magnetic resonance imaging: development and retrospective validation of MRNet. PLoS medicine 15, 11 (2018), e1002699."},{"key":"e_1_3_2_1_11_1","volume-title":"Man is to computer programmer as woman is to homemaker? debiasing word embeddings. Advances in neural information processing systems 29","author":"Bolukbasi Tolga","year":"2016","unstructured":"Tolga Bolukbasi, Kai-Wei Chang, James\u00a0Y Zou, Venkatesh Saligrama, and Adam\u00a0T Kalai. 2016. Man is to computer programmer as woman is to homemaker? debiasing word embeddings. Advances in neural information processing systems 29 (2016), 4349\u20134357."},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.1148\/rg.2015150023"},{"key":"e_1_3_2_1_13_1","volume-title":"Conference on fairness, accountability and transparency. PMLR, 77\u201391","author":"Buolamwini Joy","year":"2018","unstructured":"Joy Buolamwini and Timnit Gebru. 2018. Gender shades: Intersectional accuracy disparities in commercial gender classification. In Conference on fairness, accountability and transparency. PMLR, 77\u201391."},{"key":"e_1_3_2_1_14_1","volume-title":"Semantics derived automatically from language corpora contain human-like biases. Science 356, 6334","author":"Caliskan Aylin","year":"2017","unstructured":"Aylin Caliskan, Joanna\u00a0J Bryson, and Arvind Narayanan. 2017. Semantics derived automatically from language corpora contain human-like biases. Science 356, 6334 (2017), 183\u2013186."},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.7208\/9780226613475-026"},{"key":"e_1_3_2_1_16_1","volume-title":"Ethical Machine Learning in Healthcare. Annual Review of Biomedical Data Science 4","author":"Chen Y","year":"2020","unstructured":"Irene\u00a0Y Chen, Emma Pierson, Sherri Rose, Shalmali Joshi, Kadija Ferryman, and Marzyeh Ghassemi. 2020. Ethical Machine Learning in Healthcare. Annual Review of Biomedical Data Science 4 (2020)."},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.1145\/3173574.3174225"},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1145\/3376898"},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","DOI":"10.1145\/3442188.3445921"},{"key":"e_1_3_2_1_20_1","unstructured":"Sam Corbett-Davies and Sharad Goel. 2018. The measure and mismeasure of fairness: A critical review of fair machine learning. arXiv preprint arXiv:1808.00023(2018)."},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"crossref","unstructured":"Amit Datta Michael\u00a0Carl Tschantz and Anupam Datta. 2014. Automated experiments on ad privacy settings: A tale of opacity choice and discrimination. arXiv preprint arXiv:1408.6491(2014).","DOI":"10.1515\/popets-2015-0007"},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.1037\/h0030868"},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.1126\/science.2648573"},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"publisher","DOI":"10.1145\/3313831.3376638"},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/N19-1304"},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2009.5206848"},{"key":"e_1_3_2_1_27_1","first-page":"564","article-title":"A unified bias-variance decomposition for zero-one and squared loss","volume":"2000","author":"Domingos Pedro","year":"2000","unstructured":"Pedro Domingos. 2000. A unified bias-variance decomposition for zero-one and squared loss. AAAI\/IAAI 2000(2000), 564\u2013569.","journal-title":"AAAI\/IAAI"},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"publisher","DOI":"10.1038\/075450a0"},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.1073\/pnas.1720347115"},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"publisher","DOI":"10.1214\/15-AOAS897"},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"publisher","DOI":"10.1145\/3359152"},{"key":"e_1_3_2_1_32_1","doi-asserted-by":"publisher","DOI":"10.1073\/pnas.2110013119"},{"key":"e_1_3_2_1_33_1","unstructured":"Kaiming He Xiangyu Zhang Shaoqing Ren and Jian Sun. 2015. Deep Residual Learning for Image Recognition. arxiv:1512.03385\u00a0[cs.CV]"},{"key":"e_1_3_2_1_34_1","volume-title":"Proceedings of the 38th International Conference on Machine Learning(Proceedings of Machine Learning Research, Vol.\u00a0139)","author":"Hilgard Sophie","year":"2021","unstructured":"Sophie Hilgard, Nir Rosenfeld, Mahzarin\u00a0R Banaji, Jack Cao, and David Parkes. 2021. Learning Representations by Humans, for Humans. In Proceedings of the 38th International Conference on Machine Learning(Proceedings of Machine Learning Research, Vol.\u00a0139), Marina Meila and Tong Zhang (Eds.). PMLR, 4227\u20134238. https:\/\/proceedings.mlr.press\/v139\/hilgard21a.html"},{"key":"e_1_3_2_1_35_1","doi-asserted-by":"publisher","DOI":"10.1038\/s41591-019-0715-9"},{"key":"e_1_3_2_1_36_1","unstructured":"Jongbin Jung Sam Corbett-Davies Ravi Shroff and Sharad Goel. 2018. Omitted and included variable bias in tests for disparate impact. arXiv preprint arXiv:1809.05651(2018)."},{"volume-title":"fast and slow","author":"Kahneman Daniel","key":"e_1_3_2_1_37_1","unstructured":"Daniel Kahneman. 2011. Thinking, fast and slow. Macmillan."},{"key":"e_1_3_2_1_38_1","unstructured":"Daniel Kahneman Olivier Sibony and Cass\u00a0R Sunstein. 2021. Noise: a flaw in human judgment. Little Brown."},{"volume-title":"Explainable AI: Interpreting, Explaining and Visualizing Deep Learning","author":"Kindermans Pieter-Jan","key":"e_1_3_2_1_39_1","unstructured":"Pieter-Jan Kindermans, Sara Hooker, Julius Adebayo, Maximilian Alber, Kristof\u00a0T Sch\u00fctt, Sven D\u00e4hne, Dumitru Erhan, and Been Kim. 2019. The (un) reliability of saliency methods. In Explainable AI: Interpreting, Explaining and Visualizing Deep Learning. Springer, 267\u2013280."},{"key":"e_1_3_2_1_40_1","volume-title":"Human decisions and machine predictions. The quarterly journal of economics 133, 1","author":"Kleinberg Jon","year":"2018","unstructured":"Jon Kleinberg, Himabindu Lakkaraju, Jure Leskovec, Jens Ludwig, and Sendhil Mullainathan. 2018. Human decisions and machine predictions. The quarterly journal of economics 133, 1 (2018), 237\u2013293."},{"key":"e_1_3_2_1_41_1","doi-asserted-by":"publisher","DOI":"10.1093\/jla\/laz001"},{"key":"e_1_3_2_1_42_1","doi-asserted-by":"publisher","DOI":"10.1073\/pnas.1912790117"},{"key":"e_1_3_2_1_43_1","doi-asserted-by":"publisher","DOI":"10.1073\/pnas.1915768117"},{"key":"e_1_3_2_1_44_1","unstructured":"Himabindu Lakkaraju and Jure Leskovec. 2016. Confusions over Time: An Interpretable Bayesian Model to Characterize Trends in Decision Making. In NIPS. 3261\u20133269."},{"key":"e_1_3_2_1_45_1","doi-asserted-by":"publisher","DOI":"10.1145\/3334480.3375158"},{"key":"e_1_3_2_1_46_1","volume-title":"Opening Up the Black Box: Auditing Google\u2019s Top Stories Algorithm. In The Thirty-Second International Flairs Conference.","author":"Lurie Emma","year":"2019","unstructured":"Emma Lurie and Eni Mustafaraj. 2019. Opening Up the Black Box: Auditing Google\u2019s Top Stories Algorithm. In The Thirty-Second International Flairs Conference."},{"key":"e_1_3_2_1_47_1","unstructured":"David Madras Toniann Pitassi and Richard Zemel. 2017. Predict responsibly: improving fairness and accuracy by learning to defer. arXiv preprint arXiv:1711.06664(2017)."},{"key":"e_1_3_2_1_48_1","doi-asserted-by":"publisher","DOI":"10.1145\/3394486.3403219"},{"key":"e_1_3_2_1_49_1","unstructured":"Reid McIlroy-Young Russell Wang Siddhartha Sen Jon Kleinberg and Ashton Anderson. 2020. Learning personalized models of human behavior in chess. arXiv preprint arXiv:2008.10086(2020)."},{"key":"e_1_3_2_1_50_1","doi-asserted-by":"publisher","DOI":"10.1561\/9781680839173"},{"key":"e_1_3_2_1_51_1","doi-asserted-by":"publisher","DOI":"10.1146\/annurev-statistics-042720-125902"},{"key":"e_1_3_2_1_52_1","volume-title":"International Conference on Machine Learning. PMLR, 7076\u20137087","author":"Mozannar Hussein","year":"2020","unstructured":"Hussein Mozannar and David Sontag. 2020. Consistent estimators for learning to defer to an expert. In International Conference on Machine Learning. PMLR, 7076\u20137087."},{"key":"e_1_3_2_1_53_1","volume-title":"Biased algorithms are easier to fix than biased people. The New York Times","author":"Mullainathan Sendhil","year":"2019","unstructured":"Sendhil Mullainathan. 2019. Biased algorithms are easier to fix than biased people. The New York Times (2019)."},{"key":"e_1_3_2_1_54_1","doi-asserted-by":"publisher","DOI":"10.1093\/qje"},{"key":"e_1_3_2_1_55_1","volume-title":"International Conference on Artificial Intelligence and Statistics. PMLR, 96\u2013105","author":"Pierson Emma","year":"2018","unstructured":"Emma Pierson, Sam Corbett-Davies, and Sharad Goel. 2018. Fast threshold tests for detecting discrimination. In International Conference on Artificial Intelligence and Statistics. PMLR, 96\u2013105."},{"key":"e_1_3_2_1_56_1","doi-asserted-by":"publisher","DOI":"10.1038\/s41591-020-01192-7"},{"key":"e_1_3_2_1_57_1","volume-title":"A large-scale analysis of racial disparities in police stops across the United States. Nature human behaviour 4, 7","author":"Pierson Emma","year":"2020","unstructured":"Emma Pierson, Camelia Simoiu, Jan Overgoor, Sam Corbett-Davies, Daniel Jenson, Amy Shoemaker, Vignesh Ramachandran, Phoebe Barghouty, Cheryl Phillips, Ravi Shroff, 2020. A large-scale analysis of racial disparities in police stops across the United States. Nature human behaviour 4, 7 (2020), 736\u2013745."},{"key":"e_1_3_2_1_58_1","volume-title":"International Conference on Machine Learning. PMLR, 5281\u20135290","author":"Raghu Maithra","year":"2019","unstructured":"Maithra Raghu, Katy Blumer, Rory Sayres, Ziad Obermeyer, Bobby Kleinberg, Sendhil Mullainathan, and Jon Kleinberg. 2019. Direct uncertainty prediction for medical second opinions. In International Conference on Machine Learning. PMLR, 5281\u20135290."},{"key":"e_1_3_2_1_59_1","unstructured":"Christian Sandvig Kevin Hamilton Karrie Karahalios and Cedric Langbort. 2014. Auditing algorithms: Research methods for detecting discrimination on internet platforms. Data and discrimination: converting critical concerns into productive inquiry 22(2014) 4349\u20134357."},{"key":"e_1_3_2_1_60_1","doi-asserted-by":"crossref","unstructured":"Megan\u00a0T Stevenson and Jennifer\u00a0L Doleac. 2021. Algorithmic risk assessment in the hands of humans. Available at SSRN 3489440(2021).","DOI":"10.2139\/ssrn.3513695"},{"key":"e_1_3_2_1_61_1","doi-asserted-by":"publisher","DOI":"10.1145\/2447976.2447990"},{"key":"e_1_3_2_1_62_1","doi-asserted-by":"publisher","DOI":"10.1177\/1477153517693887"},{"key":"e_1_3_2_1_63_1","volume-title":"Judgment under uncertainty: Heuristics and biases. science 185, 4157","author":"Tversky Amos","year":"1974","unstructured":"Amos Tversky and Daniel Kahneman. 1974. Judgment under uncertainty: Heuristics and biases. science 185, 4157 (1974), 1124\u20131131."},{"key":"e_1_3_2_1_64_1","volume-title":"International Conference on Social Computing and Social Media. Springer, 357\u2013371","author":"Valdez Andr\u00e9\u00a0Calero","year":"2018","unstructured":"Andr\u00e9\u00a0Calero Valdez and Martina Ziefle. 2018. Human factors in the age of algorithms. understanding the human-in-the-loop using agent-based modeling. In International Conference on Social Computing and Social Media. Springer, 357\u2013371."},{"key":"e_1_3_2_1_65_1","doi-asserted-by":"crossref","unstructured":"Bryan Wilder Eric Horvitz and Ece Kamar. 2020. Learning to complement humans. arXiv preprint arXiv:2005.00582(2020).","DOI":"10.24963\/ijcai.2020\/212"},{"key":"e_1_3_2_1_66_1","doi-asserted-by":"publisher","DOI":"10.1038\/s41746-021-00427-2"},{"key":"e_1_3_2_1_67_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-10590-1_53"},{"key":"e_1_3_2_1_68_1","unstructured":"Quanshi Zhang and Song-Chun Zhu. 2018. Visual interpretability for deep learning: a survey. arXiv preprint arXiv:1802.00614(2018)."}],"event":{"name":"FAccT '22: 2022 ACM Conference on Fairness, Accountability, and Transparency","sponsor":["ACM Association for Computing Machinery"],"location":"Seoul Republic of Korea","acronym":"FAccT '22"},"container-title":["2022 ACM Conference on Fairness Accountability and Transparency"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3531146.3533145","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3531146.3533145","content-type":"application\/pdf","content-version":"vor","intended-application":"syndication"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3531146.3533145","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T19:02:08Z","timestamp":1750186928000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3531146.3533145"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2022,6,20]]},"references-count":68,"alternative-id":["10.1145\/3531146.3533145","10.1145\/3531146"],"URL":"https:\/\/doi.org\/10.1145\/3531146.3533145","relation":{},"subject":[],"published":{"date-parts":[[2022,6,20]]},"assertion":[{"value":"2022-06-20","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}