{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,1]],"date-time":"2026-04-01T02:16:19Z","timestamp":1775009779598,"version":"3.50.1"},"publisher-location":"New York, NY, USA","reference-count":46,"publisher":"ACM","license":[{"start":{"date-parts":[[2022,6,20]],"date-time":"2022-06-20T00:00:00Z","timestamp":1655683200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"name":"CIFAR","award":["Canada AI Research Chair"],"award-info":[{"award-number":["Canada AI Research Chair"]}]},{"name":"Compute Canada"},{"DOI":"10.13039\/100013020","name":"Nvidia","doi-asserted-by":"publisher","id":[{"id":"10.13039\/100013020","id-type":"DOI","asserted-by":"publisher"}]},{"name":"DARPA","award":["FA8750-19-2-0222, CFDA #12.910"],"award-info":[{"award-number":["FA8750-19-2-0222, CFDA #12.910"]}]},{"name":"NSERC","award":["Discovery Grant Supplement"],"award-info":[{"award-number":["Discovery Grant Supplement"]}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2022,6,21]]},"DOI":"10.1145\/3531146.3533240","type":"proceedings-article","created":{"date-parts":[[2022,6,20]],"date-time":"2022-06-20T14:27:10Z","timestamp":1655735230000},"page":"1962-1981","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":44,"title":["The Spotlight: A General Method for Discovering Systematic Errors in Deep Learning Models"],"prefix":"10.1145","author":[{"given":"Greg","family":"d'Eon","sequence":"first","affiliation":[{"name":"University of British Columbia, Canada"}]},{"given":"Jason","family":"d'Eon","sequence":"additional","affiliation":[{"name":"Dalhousie University\/Vector Institute, Canada"}]},{"given":"James R.","family":"Wright","sequence":"additional","affiliation":[{"name":"University of Alberta, Canada"}]},{"given":"Kevin","family":"Leyton-Brown","sequence":"additional","affiliation":[{"name":"University of British Columbia, Canada"}]}],"member":"320","published-online":{"date-parts":[[2022,6,20]]},"reference":[{"key":"e_1_3_2_1_1_1","volume-title":"FairSight: Visual analytics for fairness in decision making","author":"Ahn Yongsu","year":"2019","unstructured":"Yongsu Ahn and Yu-Ru Lin. 2019. FairSight: Visual analytics for fairness in decision making. IEEE transactions on visualization and computer graphics 26, 1(2019), 1086\u20131095."},{"key":"e_1_3_2_1_2_1","unstructured":"Martin Arjovsky L\u00e9on Bottou Ishaan Gulrajani and David Lopez-Paz. 2019. Invariant risk minimization. arXiv preprint arXiv:1907.02893(2019)."},{"key":"e_1_3_2_1_3_1","volume-title":"Conference on fairness, accountability and transparency. PMLR, 77\u201391","author":"Buolamwini Joy","year":"2018","unstructured":"Joy Buolamwini and Timnit Gebru. 2018. Gender shades: Intersectional accuracy disparities in commercial gender classification. In Conference on fairness, accountability and transparency. PMLR, 77\u201391."},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.1109\/VAST47406.2019.8986948"},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1145\/3359206"},{"key":"e_1_3_2_1_6_1","volume-title":"Klaus\u00a0J Busam, Edi Brogi, Victor\u00a0E Reuter, David\u00a0S Klimstra, and Thomas\u00a0J Fuchs.","author":"Campanella Gabriele","year":"2019","unstructured":"Gabriele Campanella, Matthew\u00a0G Hanna, Luke Geneslaw, Allen Miraflor, Vitor Werneck Krauss\u00a0Silva, Klaus\u00a0J Busam, Edi Brogi, Victor\u00a0E Reuter, David\u00a0S Klimstra, and Thomas\u00a0J Fuchs. 2019. Clinical-grade computational pathology using weakly supervised deep learning on whole slide images. Nature medicine 25, 8 (2019), 1301\u20131309."},{"key":"e_1_3_2_1_7_1","volume-title":"Proceedings of the 32nd International Conference on Neural Information Processing Systems. 3543\u20133554","author":"Chen Y","year":"2018","unstructured":"Irene\u00a0Y Chen, Fredrik\u00a0D Johansson, and David Sontag. 2018. Why is my classifier discriminatory?. In Proceedings of the 32nd International Conference on Neural Information Processing Systems. 3543\u20133554."},{"key":"e_1_3_2_1_8_1","unstructured":"Yeounoh Chung Tim Kraska Neoklis Polyzotis and Steven\u00a0Euijong Whang. 2018. Slice finder: Automated data slicing for model validation. CoRR abs\/1807.06068(2018). arXiv:1807.06068http:\/\/arxiv.org\/abs\/1807.06068"},{"key":"e_1_3_2_1_9_1","unstructured":"Sam Corbett-Davies and Sharad Goel. 2018. The measure and mismeasure of fairness: A critical review of fair machine learning. arXiv preprint arXiv:1808.00023(2018)."},{"key":"e_1_3_2_1_10_1","unstructured":"Elliot Creager J\u00f6rn-Henrik Jacobsen and Richard\u00a0S. Zemel. 2020. Exchanging lessons between algorithmic fairness and domain generalization. CoRR abs\/2010.07249(2020). arXiv:2010.07249https:\/\/arxiv.org\/abs\/2010.07249"},{"key":"e_1_3_2_1_11_1","unstructured":"Alexander D\u2019Amour Katherine Heller Dan Moldovan Ben Adlam Babak Alipanahi Alex Beutel Christina Chen Jonathan Deaton Jacob Eisenstein and Matthew D et.\u00a0al. Hoffman. 2020. Underspecification presents challenges for credibility in modern machine learning. arXiv preprint arXiv:2011.03395(2020)."},{"key":"e_1_3_2_1_12_1","volume-title":"Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition Workshops. 52\u201359","author":"de Vries Terrance","year":"2019","unstructured":"Terrance de Vries, Ishan Misra, Changhan Wang, and Laurens van\u00a0der Maaten. 2019. Does object recognition work for everyone?. In Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition Workshops. 52\u201359."},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1"},{"key":"e_1_3_2_1_14_1","unstructured":"John\u00a0C. Duchi Tatsunori Hashimoto and Hongseok Namkoong. 2020. Distributionally robust losses for latent covariate mixtures. CoRR abs\/2007.13982(2020). arXiv:2007.13982https:\/\/arxiv.org\/abs\/2007.13982"},{"key":"e_1_3_2_1_15_1","unstructured":"Logan Engstrom Andrew Ilyas Hadi Salman Shibani Santurkar and Dimitris Tsipras. 2019. Robustness (Python Library). https:\/\/github.com\/MadryLab\/robustness"},{"key":"e_1_3_2_1_16_1","volume-title":"Domino: Discovering Systematic Errors with Cross-Modal Embeddings. https:\/\/openreview.net\/pdf?id=FPCMqjI0jXN","author":"Eyuboglu Sabri","year":"2022","unstructured":"Sabri Eyuboglu, Maya Varma, Khaled Saab, Jean-Benoit Delbrouck, Christopher Lee-Messer, Jared Dunnmon, James Zou, and Christopher R\u00e9. 2022. Domino: Discovering Systematic Errors with Cross-Modal Embeddings. https:\/\/openreview.net\/pdf?id=FPCMqjI0jXN"},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"crossref","unstructured":"F\u00a0Maxwell Harper and Joseph\u00a0A Konstan. 2015. The MovieLens datasets: History and context. Acm transactions on interactive intelligent systems (tiis) 5 4(2015) 1\u201319.","DOI":"10.1145\/2827872"},{"key":"e_1_3_2_1_18_1","volume-title":"International Conference on Machine Learning. PMLR","author":"Hartford Jason","year":"2018","unstructured":"Jason Hartford, Devon Graham, Kevin Leyton-Brown, and Siamak Ravanbakhsh. 2018. Deep models of interactions across sets. In International Conference on Machine Learning. PMLR, 1909\u20131918."},{"key":"e_1_3_2_1_19_1","volume-title":"International Conference on Machine Learning. PMLR","author":"Hashimoto Tatsunori","year":"2018","unstructured":"Tatsunori Hashimoto, Megha Srivastava, Hongseok Namkoong, and Percy Liang. 2018. Fairness without demographics in repeated loss minimization. In International Conference on Machine Learning. PMLR, 1929\u20131938."},{"key":"e_1_3_2_1_20_1","unstructured":"HuggingFace. 2021. DistilBERT base uncased distilled SQuAD. https:\/\/huggingface.co\/distilbert-base-uncased-distilled-squad. Accessed: 2021-05-28."},{"key":"e_1_3_2_1_21_1","unstructured":"HuggingFace. 2021. DistilBERT base uncased finetuned SST-2. https:\/\/huggingface.co\/distilbert-base-uncased-finetuned-sst-2-english. Accessed: 2021-05-28."},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.1109\/WACV48630.2021.00159"},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.cell.2018.02.010"},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"publisher","DOI":"10.1145\/3306618.3314287"},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.1073\/pnas.1915768117"},{"key":"e_1_3_2_1_26_1","unstructured":"Preethi Lahoti Alex Beutel Jilin Chen Kang Lee Flavien Prost Nithum Thain Xuezhi Wang and Ed\u00a0H. Chi. 2020. Fairness without demographics through adversarially reweighted learning. CoRR abs\/2006.13114(2020). arXiv:2006.13114https:\/\/arxiv.org\/abs\/2006.13114"},{"key":"e_1_3_2_1_27_1","doi-asserted-by":"publisher","DOI":"10.14778\/3476311.3476328"},{"key":"e_1_3_2_1_28_1","volume-title":"The algorithmic audit: working with vendors to validate radiology-AI algorithms\u2014how we do it","author":"Mahajan Vidur","year":"2020","unstructured":"Vidur Mahajan, Vasantha\u00a0Kumar Venugopal, Murali Murugavel, and Harsh Mahajan. 2020. The algorithmic audit: working with vendors to validate radiology-AI algorithms\u2014how we do it. Academic radiology 27, 1 (2020), 132\u2013135."},{"key":"e_1_3_2_1_29_1","volume-title":"Proceedings of the 37th International Conference on Machine Learning(Proceedings of Machine Learning Research, Vol.\u00a0119)","author":"Martinez Natalia","year":"2020","unstructured":"Natalia Martinez, Martin Bertran, and Guillermo Sapiro. 2020. Minimax Pareto fairness: A multi objective perspective. In Proceedings of the 37th International Conference on Machine Learning(Proceedings of Machine Learning Research, Vol.\u00a0119), Hal\u00a0Daum\u00e9 III and Aarti Singh (Eds.). PMLR, 6755\u20136764. https:\/\/proceedings.mlr.press\/v119\/martinez20a.html"},{"key":"e_1_3_2_1_30_1","unstructured":"Tomas Mikolov Kai Chen Greg Corrado and Jeffrey Dean. 2013. Efficient Estimation of Word Representations in Vector Space. (2013). arxiv:1301.3781\u00a0[cs.CL]"},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"publisher","DOI":"10.1145\/3287560.3287596"},{"key":"e_1_3_2_1_32_1","doi-asserted-by":"publisher","DOI":"10.1145\/3368555.3384468"},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"publisher","DOI":"10.23915\/distill.00010"},{"key":"e_1_3_2_1_34_1","doi-asserted-by":"crossref","unstructured":"Yonatan Oren Shiori Sagawa Tatsunori\u00a0B Hashimoto and Percy Liang. 2019. Distributionally robust language modeling. arXiv preprint arXiv:1909.02060(2019).","DOI":"10.18653\/v1\/D19-1432"},{"key":"e_1_3_2_1_35_1","unstructured":"Eliana Pastor Luca de Alfaro and Elena Baralis. 2021. Identifying biased subgroups in ranking and classification. CoRR abs\/2108.07450(2021). arXiv:2108.07450https:\/\/arxiv.org\/abs\/2108.07450"},{"key":"e_1_3_2_1_36_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.asw.2014.05.001"},{"key":"e_1_3_2_1_37_1","unstructured":"PyTorch. 2021. Torchvision models. https:\/\/pytorch.org\/vision\/stable\/models.html. Accessed: 2021-05-28."},{"key":"e_1_3_2_1_38_1","doi-asserted-by":"crossref","unstructured":"Pranav Rajpurkar Jian Zhang Konstantin Lopyrev and Percy Liang. 2016. SQuAD: 100 000+ questions for machine comprehension of text. arXiv preprint arXiv:1606.05250(2016).","DOI":"10.18653\/v1\/D16-1264"},{"key":"e_1_3_2_1_39_1","volume-title":"Visualizing and Measuring the Geometry of BERT. Advances in Neural Information Processing Systems 32","author":"Reif Emily","year":"2019","unstructured":"Emily Reif, Ann Yuan, Martin Wattenberg, Fernanda\u00a0B Viegas, Andy Coenen, Adam Pearce, and Been Kim. 2019. Visualizing and Measuring the Geometry of BERT. Advances in Neural Information Processing Systems 32 (2019). https:\/\/proceedings.neurips.cc\/paper\/2019\/file\/159c1ffe5b61b41b3c4d8f4c2150f6c4-Paper.pdf"},{"key":"e_1_3_2_1_40_1","volume-title":"International Conference on Learning Representations.","author":"Sagawa Shiori","year":"2019","unstructured":"Shiori Sagawa, Pang\u00a0Wei Koh, Tatsunori\u00a0B Hashimoto, and Percy Liang. 2019. Distributionally robust neural networks. In International Conference on Learning Representations."},{"key":"e_1_3_2_1_41_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2019.00681"},{"key":"e_1_3_2_1_42_1","volume-title":"No subclass left behind: Fine-grained robustness in coarse-grained classification problems. Advances in Neural Information Processing Systems 33","author":"Sohoni Nimit","year":"2020","unstructured":"Nimit Sohoni, Jared Dunnmon, Geoffrey Angus, Albert Gu, and Christopher R\u00e9. 2020. No subclass left behind: Fine-grained robustness in coarse-grained classification problems. Advances in Neural Information Processing Systems 33 (2020)."},{"key":"e_1_3_2_1_43_1","unstructured":"Christian Szegedy Wojciech Zaremba Ilya Sutskever Joan Bruna Dumitru Erhan Ian Goodfellow and Rob Fergus. 2014. Intriguing properties of neural networks. arxiv:1312.6199\u00a0[cs.CV]"},{"key":"e_1_3_2_1_44_1","volume-title":"The What-If Tool: Interactive probing of machine learning models","author":"Wexler James","year":"2019","unstructured":"James Wexler, Mahima Pushkarna, Tolga Bolukbasi, Martin Wattenberg, Fernanda Vi\u00e9gas, and Jimbo Wilson. 2019. The What-If Tool: Interactive probing of machine learning models. IEEE transactions on visualization and computer graphics 26, 1(2019), 56\u201365."},{"key":"e_1_3_2_1_45_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/P19-1073"},{"key":"e_1_3_2_1_46_1","volume-title":"Character-level convolutional networks for text classification. Advances in Neural Information Processing Systems 28","author":"Zhang Xiang","year":"2015","unstructured":"Xiang Zhang, Junbo Zhao, and Yann Lecun. 2015. Character-level convolutional networks for text classification. Advances in Neural Information Processing Systems 28 (2015)."}],"event":{"name":"FAccT '22: 2022 ACM Conference on Fairness, Accountability, and Transparency","location":"Seoul Republic of Korea","acronym":"FAccT '22","sponsor":["ACM Association for Computing Machinery"]},"container-title":["2022 ACM Conference on Fairness Accountability and Transparency"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3531146.3533240","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3531146.3533240","content-type":"application\/pdf","content-version":"vor","intended-application":"syndication"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3531146.3533240","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T19:31:30Z","timestamp":1750188690000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3531146.3533240"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2022,6,20]]},"references-count":46,"alternative-id":["10.1145\/3531146.3533240","10.1145\/3531146"],"URL":"https:\/\/doi.org\/10.1145\/3531146.3533240","relation":{},"subject":[],"published":{"date-parts":[[2022,6,20]]},"assertion":[{"value":"2022-06-20","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}