{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,8,7]],"date-time":"2026-08-07T08:18:49Z","timestamp":1786090729958,"version":"3.56.0"},"publisher-location":"New York, NY, USA","reference-count":140,"publisher":"ACM","license":[{"start":{"date-parts":[[2025,4,25]],"date-time":"2025-04-25T00:00:00Z","timestamp":1745539200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"funder":[{"DOI":"10.13039\/501100001602","name":"Science Foundation Ireland","doi-asserted-by":"publisher","award":["13\/RC\/2106_P2"],"award-info":[{"award-number":["13\/RC\/2106_P2"]}],"id":[{"id":"10.13039\/501100001602","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100012642","name":"Mozilla Foundation","doi-asserted-by":"publisher","id":[{"id":"10.13039\/501100012642","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2025,4,26]]},"DOI":"10.1145\/3706598.3713301","type":"proceedings-article","created":{"date-parts":[[2025,4,28]],"date-time":"2025-04-28T14:16:03Z","timestamp":1745849763000},"page":"1-29","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":40,"title":["Towards AI Accountability Infrastructure: Gaps and Opportunities in AI Audit Tooling"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0009-0004-8403-4423","authenticated-orcid":false,"given":"Victor","family":"Ojewale","sequence":"first","affiliation":[{"name":"The Center for Technological Responsibility, Reimagination and Redesign (CNTR), Brown University, Providence, Rhode Island, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-2345-2670","authenticated-orcid":false,"given":"Ryan","family":"Steed","sequence":"additional","affiliation":[{"name":"Heinz College of Information Systems and Public Policy, Carnegie Mellon University, Pittsburgh, Pennsylvania, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-0828-8665","authenticated-orcid":false,"given":"Briana","family":"Vecchione","sequence":"additional","affiliation":[{"name":"Data & Society Research Institute, New York, New York, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-6319-7937","authenticated-orcid":false,"given":"Abeba","family":"Birhane","sequence":"additional","affiliation":[{"name":"School of Computer Science and Statistics, Trinity College Dublin, Dublin, Ireland"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-9510-3015","authenticated-orcid":false,"given":"Inioluwa Deborah","family":"Raji","sequence":"additional","affiliation":[{"name":"EECS, University of California, Berkeley, Berkeley, California, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2025,4,25]]},"reference":[{"key":"e_1_3_3_3_2_2","unstructured":"Claudio Agosti. 2023. Tracking Exposed Manifesto. https:\/\/tracking.exposed\/manifesto"},{"key":"e_1_3_3_3_3_2","doi-asserted-by":"publisher","DOI":"10.1145\/3387940.3392089"},{"key":"e_1_3_3_3_4_2","unstructured":"BABL AI. 2024. AI and Algorithm Auditor Certification. https:\/\/courses.babl.ai\/p\/ai-and-algorithm-auditor-certification?affcode=616760_ujptkhyg"},{"key":"e_1_3_3_3_5_2","unstructured":"AI Action Summit. 2024. Public Interest AI. https:\/\/www.elysee.fr\/en\/sommet-pour-l-action-sur-l-ia\/public-interest-ai"},{"key":"e_1_3_3_3_6_2","unstructured":"Airbnb. 2020. A New Way We\u2019re Fighting Discrimination on Airbnb - Resource Centre. https:\/\/www.airbnb.ca\/resources\/hosting-homes\/a\/a-new-way-were-fighting-discrimination-on-airbnb-201"},{"key":"e_1_3_3_3_7_2","unstructured":"American Civil Liberties Union. 2019. Sandvig v. Barr \u2014 Challenge to CFAA Prohibition on Uncovering Racial Discrimination Online. https:\/\/www.aclu.org\/cases\/sandvig-v-barr-challenge-cfaa-prohibition-uncovering-racial-discrimination-online"},{"key":"e_1_3_3_3_8_2","doi-asserted-by":"publisher","DOI":"10.1145\/2702123.2702509"},{"key":"e_1_3_3_3_9_2","unstructured":"Julia Angwin Jeff Larson Surya Mattu and Lauren Kirchner. 2016. Machine Bias. ProPublica (May 2016). https:\/\/www.propublica.org\/article\/machine-bias-risk-assessments-in-criminal-sentencing"},{"key":"e_1_3_3_3_10_2","doi-asserted-by":"publisher","unstructured":"Jack Bandy. 2021. Problematic Machine Behavior: A Systematic Literature Review of Algorithm Audits. Proceedings of the ACM on Human-Computer Interaction 5 CSCW1 (April 2021) 74:1\u201374:34. 10.1145\/3449148","DOI":"10.1145\/3449148"},{"key":"e_1_3_3_3_11_2","unstructured":"Bissan Barghouti Corinne Bintz Dharma Dailey Micah Epstein Vivian Guetler Bernease Herman Pa\u00a0Ousman Jobe Michael Katell P.M. Krafft Jennifer Lee Shankar Narayan Franziska Putz Daniella Raz Brian Robick Aaron Tam Abiel Woldu and Meg Young. 2020. Algorithmic Equity Toolkit. https:\/\/www.aclu-wa.org\/AEKit"},{"key":"e_1_3_3_3_12_2","doi-asserted-by":"publisher","unstructured":"Rachel K.\u00a0E. Bellamy Kuntal Dey Michael Hind Samuel\u00a0C. Hoffman Stephanie Houde Kalapriya Kannan Pranay Lohia Jacquelyn Martino Sameep Mehta Aleksandra Mojsilovic Seema Nagar Karthikeyan\u00a0Natesan Ramamurthy John Richards Diptikalyan Saha Prasanna Sattigeri Moninder Singh Kush\u00a0R. Varshney and Yunfeng Zhang. 2018. AI Fairness 360: An Extensible Toolkit for Detecting Understanding and Mitigating Unwanted Algorithmic Bias. 10.48550\/arXiv.1810.01943 arxiv:https:\/\/arXiv.org\/abs\/1810.01943\u00a0[cs]","DOI":"10.48550\/arXiv.1810.01943"},{"key":"e_1_3_3_3_13_2","unstructured":"Benefits Tech Advocacy Hub 2023. Benefits Tech Advocacy Hub. https:\/\/btah.org\/"},{"key":"e_1_3_3_3_14_2","volume-title":"International AI Safety Report","author":"Bengio Yoshua","year":"2025","unstructured":"Yoshua Bengio, S\u00f6ren Mindermann, Daniel Privitera, Tamay Besiroglu, Rishi Bommasani, Stephen Casper, Yejin Choi, Philip Fox, Ben Garfinkel, Danielle Goldfarb, Hoda Heidari, Anson Ho, Sayash Kapoor, Leila Khalatbari, Shayne Longpre, Sam Manning, Vasilios Mavroudis, Mantas Mazeika, Julian Michael, Jessica Newman, Kwan\u00a0Yee Ng, Chinasa\u00a0T. Okolo, Deborah Raji, Girish Sastry, Elizabeth Seger, Theodora Skeadas, Tobin South, Emma Strubell, Florian Tram\u00e8r, Lucia Velasco, Nicole Wheeler, Daron Acemoglu, Olubayo Adekanmbi, David Dalrymple, Thomas\u00a0G. Dietterich, Edward\u00a0W. Felten, Pascale Fung, Pierre-Olivier Gourinchas, Fredrik Heintz, Geoffrey Hinton, Nick Jennings, Andreas Krause, Susan Leavy, Percy Liang, Teresa Ludermir, Vidushi Marda, Helen Margetts, John McDermid, Jane Munga, Arvind Narayanan, Alondra Nelson, Clara Neppel, Alice Oh, Gopal Ramchurn, Stuart Russell, Marietje Schaake, Bernhard Sch\u00f6lkopf, Dawn Song, Alvaro Soto, Lee Tiedrich, Ga\u00ebl Varoquaux, Andrew Yao, Ya-Qin Zhang, Fahad Albalawi, Marwan Alserkal, Olubunmi Ajala, Guillaume Avrin, Christian Busch, Andr\u00e9 Carlos Ponce de Leon\u00a0Ferreira de Carvalho, Bronwyn Fox, Amandeep\u00a0Singh Gill, Ahmet\u00a0Halit Hatip, Juha Heikkil\u00e4, Gill Jolly, Ziv Katzir, Hiroaki Kitano, Antonio Kr\u00fcger, Chris Johnson, Saif\u00a0M. Khan, Kyoung\u00a0Mu Lee, Dominic\u00a0Vincent Ligot, Oleksii Molchanovskyi, Andrea Monti, Nusu Mwamanzi, Mona Nemer, Nuria Oliver, Jos\u00e9 Ram\u00f3n\u00a0L\u00f3pez Portillo, Balaraman Ravindran, Raquel\u00a0Pezoa Rivera, Hammam Riza, Crystal Rugege, Ciar\u00e1n Seoighe, Jerry Sheehan, Haroon Sheikh, Denise Wong, and Yi Zeng. 2025. International AI Safety Report. Technical Report. arXiv, AI Action Summit. arxiv:https:\/\/arXiv.org\/abs\/2501.17805\u00a0[cs] http:\/\/arxiv.org\/abs\/2501.17805"},{"key":"e_1_3_3_3_15_2","doi-asserted-by":"publisher","DOI":"10.1145\/3613904.3642398"},{"key":"e_1_3_3_3_16_2","doi-asserted-by":"publisher","DOI":"10.1145\/3544548.3581314"},{"key":"e_1_3_3_3_17_2","unstructured":"Joseph\u00a0R. Biden. 2023. Executive Order on the Safe Secure and Trustworthy Development and Use of Artificial Intelligence. https:\/\/www.whitehouse.gov\/briefing-room\/presidential-actions\/2023\/10\/30\/executive-order-on-the-safe-secure-and-trustworthy-development-and-use-of-artificial-intelligence\/"},{"key":"e_1_3_3_3_18_2","doi-asserted-by":"publisher","DOI":"10.1145\/3551624.3555290"},{"key":"e_1_3_3_3_19_2","doi-asserted-by":"publisher","DOI":"10.1109\/SaTML59370.2024.00037"},{"key":"e_1_3_3_3_20_2","doi-asserted-by":"publisher","DOI":"10.1145\/3531146.3533111"},{"key":"e_1_3_3_3_21_2","doi-asserted-by":"crossref","unstructured":"Mark Bovens. 2007. Analysing and Assessing Accountability: A Conceptual Framework 1. European law journal 13 4 (2007) 447\u2013468.","DOI":"10.1111\/j.1468-0386.2007.00378.x"},{"key":"e_1_3_3_3_22_2","doi-asserted-by":"publisher","DOI":"10.1145\/3290605.3300271"},{"key":"e_1_3_3_3_23_2","series-title":"Proceedings of Machine Learning Research","first-page":"77","volume-title":"Proceedings of the 1st Conference on Fairness, Accountability and Transparency","volume":"81","author":"Buolamwini Joy","year":"2018","unstructured":"Joy Buolamwini and Timnit Gebru. 2018. Gender Shades: Intersectional Accuracy Disparities in Commercial Gender Classification. In Proceedings of the 1st Conference on Fairness, Accountability and Transparency(Proceedings of Machine Learning Research, Vol.\u00a081), Sorelle\u00a0A Friedler and Christo Wilson (Eds.). PMLR, New York, NY, USA, 77\u201391. http:\/\/proceedings.mlr.press\/v81\/buolamwini18a.html"},{"key":"e_1_3_3_3_24_2","unstructured":"Doug Cass Mick Alberts Ken Turetz Pritam Ovhal Alex Buck Tom Pratt Dhanashri Kshirsagar and Harmony Mabrey. 2022. Community Jury - Azure Application Architecture Guide. https:\/\/learn.microsoft.com\/en-us\/azure\/architecture\/guide\/responsible-innovation\/community-jury\/"},{"key":"e_1_3_3_3_25_2","unstructured":"Charlie Pownall. 2021. AI Algorithmic and Automation Incident and Controversy Repository (AIAAIC). https:\/\/www.aiaaic.org\/"},{"key":"e_1_3_3_3_26_2","volume-title":"Constructing Grounded Theory (2nd edition ed.)","author":"Charmaz Kathy","year":"2014","unstructured":"Kathy Charmaz. 2014. Constructing Grounded Theory (2nd edition ed.). Sage, London ; Thousand Oaks, Calif."},{"key":"e_1_3_3_3_27_2","unstructured":"Mike Clark. 2021. Research Cannot Be the Justification for Compromising People\u2019s Privacy. https:\/\/about.fb.com\/news\/2021\/08\/research-cannot-be-the-justification-for-compromising-peoples-privacy\/"},{"key":"e_1_3_3_3_28_2","unstructured":"Yvette\u00a0D. Clarke. 2019. Algorithmic Accountability Act of 2022. https:\/\/www.congress.gov\/117\/bills\/hr6580\/BILLS-117hr6580ih.pdf"},{"key":"e_1_3_3_3_29_2","doi-asserted-by":"publisher","DOI":"10.1145\/3531146.3533213"},{"key":"e_1_3_3_3_30_2","unstructured":"Crunchbase 2023. Crunchbase. https:\/\/www.crunchbase.com"},{"key":"e_1_3_3_3_31_2","unstructured":"Laurie\u00a0A. Cumbo Alicka Ampry-Samuel Helen\u00a0K. Rosenthal Robert\u00a0E. Cornegy Ben Kallos Adrienne\u00a0E. Adams Farah\u00a0N. Louis Margaret\u00a0S. Chin Fernando Cabrera Deborah\u00a0L. Rose Vanessa\u00a0L. Gibson Justin\u00a0L. Brannan Carlina Rivera Mark Levine Diana\u00a0I. Ayala I.\u00a0Daneek Miller Stephen\u00a0T. Levin and Inez\u00a0D. Barron. 2021. A Local Law to Amend the Administrative Code of the City of New York in Relation to Automated Employment Decision Tools. https:\/\/www.nyc.gov\/site\/dca\/about\/automated-employment-decision-tools.page"},{"key":"e_1_3_3_3_32_2","doi-asserted-by":"publisher","DOI":"10.1145\/3617694.3623261"},{"key":"e_1_3_3_3_33_2","doi-asserted-by":"publisher","DOI":"10.1145\/3544548.3581026"},{"key":"e_1_3_3_3_34_2","doi-asserted-by":"publisher","DOI":"10.1145\/3531146.3533113"},{"key":"e_1_3_3_3_35_2","unstructured":"Deven\u00a0R. Desai and Joshua\u00a0A. Kroll. 2017. Trust but Verify: A Guide to Algorithms and the Law. Harvard Journal of Law & Technology (Harvard JOLT) 31 1 (2017) 1\u201364. https:\/\/heinonline.org\/HOL\/P?h=hein.journals\/hjlt31&i=7"},{"key":"e_1_3_3_3_36_2","doi-asserted-by":"publisher","DOI":"10.1145\/3491102.3517441"},{"key":"e_1_3_3_3_37_2","volume-title":"Auditing Algorithms: The Existing Landscape, Role of Regulators and Future Outlook","author":"Forum Digital Regulation Cooperation","year":"2022","unstructured":"Digital Regulation Cooperation Forum. 2022. Auditing Algorithms: The Existing Landscape, Role of Regulators and Future Outlook. Research and Analysis. Ofcom. https:\/\/www.gov.uk\/government\/publications\/findings-from-the-drcf-algorithmic-processing-workstream-spring-2022\/auditing-algorithms-the-existing-landscape-role-of-regulators-and-future-outlook"},{"key":"e_1_3_3_3_38_2","unstructured":"Laura Edelson and Damon McCoy. 2021. We Research Misinformation on Facebook. It Just Disabled Our Accounts. The New York Times (Aug. 2021). https:\/\/www.nytimes.com\/2021\/08\/10\/opinion\/facebook-misinformation.html"},{"key":"e_1_3_3_3_39_2","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-60117-1_33"},{"key":"e_1_3_3_3_40_2","unstructured":"European Parliament and Council of the European Union. 2016. Regulation (EU) 2016\/679 of the European Parliament and of the Council. of 27 April 2016 on the Protection of Natural Persons with Regard to the Processing of Personal Data and on the Free Movement of Such Data and Repealing Directive 95\/46\/EC (General Data Protection Regulation). https:\/\/data.europa.eu\/eli\/reg\/2016\/679\/oj"},{"key":"e_1_3_3_3_41_2","unstructured":"Facets - Know Your Data 2023. Facets - Know Your Data. https:\/\/pair-code.github.io\/facets\/"},{"key":"e_1_3_3_3_42_2","unstructured":"Fast Grants 2024. Fast Grants. https:\/\/www.aisi.gov.uk\/grants"},{"key":"e_1_3_3_3_43_2","doi-asserted-by":"publisher","DOI":"10.1145\/3617694.3623223"},{"key":"e_1_3_3_3_44_2","volume-title":"An Overview of National AI Strategies and Policies","author":"Galindo Laura","year":"2021","unstructured":"Laura Galindo, Karine Perset, and Francesca Sheeka. 2021. An Overview of National AI Strategies and Policies. Technical Report\u00a014. OECD Publishing, Paris. https:\/\/www.oecd.org\/en\/publications\/an-overview-of-national-ai-strategies-and-policies_c05140d9-en.html"},{"key":"e_1_3_3_3_45_2","unstructured":"Melissa Gerchick and Brooke\u00a0Watson Madubuonwu. 2024. Tracking Automated Employment Decision Tool Bias Audits. aclu-national. https:\/\/github.com\/aclu-national\/tracking-ll144-bias-audits"},{"key":"e_1_3_3_3_46_2","doi-asserted-by":"publisher","unstructured":"Marzyeh Ghassemi Luke Oakden-Rayner and Andrew\u00a0L. Beam. 2021. The False Hope of Current Approaches to Explainable Artificial Intelligence in Health Care. The Lancet Digital Health 3 11 (Nov. 2021) e745\u2013e750. 10.1016\/S2589-7500(21)00208-9","DOI":"10.1016\/S2589-7500(21)00208-9"},{"key":"e_1_3_3_3_47_2","unstructured":"Gig Economy Data Hub 2021. Gig Economy Data Hub. https:\/\/www.gigeconomydata.org\/home"},{"key":"e_1_3_3_3_48_2","doi-asserted-by":"publisher","DOI":"10.4324\/9780203793206"},{"key":"e_1_3_3_3_49_2","volume-title":"NTIA Artificial Intelligence Accountability Policy Report","author":"Goodman Ellen","year":"2024","unstructured":"Ellen Goodman. 2024. NTIA Artificial Intelligence Accountability Policy Report. Technical Report. National Telecommunication and Information Administration. https:\/\/www.ntia.gov\/sites\/default\/files\/publications\/ntia_ai_report_final-3-27-24.pdf"},{"key":"e_1_3_3_3_50_2","doi-asserted-by":"publisher","DOI":"10.2139\/ssrn.4227350"},{"key":"e_1_3_3_3_51_2","doi-asserted-by":"publisher","DOI":"10.1145\/3630106.3658959"},{"key":"e_1_3_3_3_52_2","unstructured":"Neel Guha Christie\u00a0M Lawrence Lindsey\u00a0A Gailmard Kit\u00a0T Rodolfa Faiz Surani Rishi Bommasani Inioluwa\u00a0Deborah Raji Mariano-Florentino Cu\u00e9llar Colleen Honigsberg Percy Liang and Daniel\u00a0E Ho. 2023. AI Regulation Has Its Own Alignment Problem: The Technical and Institutional Feasibility of Disclosure Registration Licensing and Auditing. George Washington Law Review 92 6 (2023) 1473\u20131557."},{"key":"e_1_3_3_3_53_2","doi-asserted-by":"publisher","unstructured":"Emma Harvey Emily Sheng Su\u00a0Lin Blodgett Alexandra Chouldechova Jean Garcia-Gathright Alexandra Olteanu and Hanna Wallach. 2024. Gaps Between Research and Practice When Measuring Representational Harms Caused by LLM-Based Systems. 10.48550\/arXiv.2411.15662 arxiv:https:\/\/arXiv.org\/abs\/2411.15662\u00a0[cs]","DOI":"10.48550\/arXiv.2411.15662"},{"key":"e_1_3_3_3_54_2","unstructured":"Merve Hickock. 2023. Ethical AI Frameworks Guidelines Toolkits. https:\/\/www.aiethicist.org\/frameworks-guidelines-toolkits"},{"key":"e_1_3_3_3_55_2","doi-asserted-by":"publisher","DOI":"10.1145\/3290605.3300830"},{"key":"e_1_3_3_3_56_2","volume-title":"Cataloguing LLM Evaluations","author":"Authority Infocomm Media Development","year":"2023","unstructured":"Infocomm Media Development Authority and AI Verify Foundation. 2023. Cataloguing LLM Evaluations. Technical Report. Infocomm Media Development Authority of Singapore. https:\/\/aiverifyfoundation.sg\/downloads\/Cataloguing_LLM_Evaluations.pdf"},{"key":"e_1_3_3_3_57_2","volume-title":"Annex A: Fairness in the AI Lifecycle","author":"Office Information Commissioner\u2019s","year":"2023","unstructured":"Information Commissioner\u2019s Office. 2023. Annex A: Fairness in the AI Lifecycle. Guidance on the AI Auditing Framework Draft Guidance for Consultation. Information Commissioner\u2019s Office. https:\/\/ico.org.uk\/for-organisations\/uk-gdpr-guidance-and-resources\/artificial-intelligence\/guidance-on-ai-and-data-protection\/"},{"key":"e_1_3_3_3_58_2","doi-asserted-by":"publisher","DOI":"10.1145\/3630106.3658910"},{"key":"e_1_3_3_3_59_2","doi-asserted-by":"publisher","DOI":"10.58704\/kcha-1h20"},{"key":"e_1_3_3_3_60_2","doi-asserted-by":"publisher","DOI":"10.1145\/3313831.3376219"},{"key":"e_1_3_3_3_61_2","volume-title":"Risky Analysis: Assessing and Improving AI Governance Tools","author":"Kaye Kate","year":"2023","unstructured":"Kate Kaye and Pam Dixon. 2023. Risky Analysis: Assessing and Improving AI Governance Tools. Technical Report. World Privacy Forum. https:\/\/www.worldprivacyforum.org\/wp-content\/uploads\/2023\/12\/WPF_Risky_Analysis_December_2023_fs.pdf"},{"key":"e_1_3_3_3_62_2","doi-asserted-by":"publisher","DOI":"10.1145\/3544548.3581001"},{"key":"e_1_3_3_3_63_2","volume-title":"Delegated Regulation on Data Access Provided for the Digital Services Act: Response to the Call for Evidence DG CNECT-CNECT F2 by the European Commission","author":"Klinger Ulrike","year":"2023","unstructured":"Ulrike Klinger and Jakob Ohme. 2023. Delegated Regulation on Data Access Provided for the Digital Services Act: Response to the Call for Evidence DG CNECT-CNECT F2 by the European Commission. Technical Report\u00a07. Weizenbaum Institute. https:\/\/www.weizenbaum-library.de\/handle\/id\/380"},{"key":"e_1_3_3_3_64_2","unstructured":"Kulynych Bogdan David Madras Smitha Milli Inioluwa\u00a0Deborah Raji Angela Zhou and Richard Zemel. 2020. Participatory Approaches to Machine Learning. International Conference on Machine Learning Workshop."},{"key":"e_1_3_3_3_65_2","series-title":"ICML\u201920","first-page":"5491","volume-title":"Proceedings of the 37th International Conference on Machine Learning","volume":"119","author":"Kumar I.\u00a0Elizabeth","year":"2020","unstructured":"I.\u00a0Elizabeth Kumar, Suresh Venkatasubramanian, Carlos Scheidegger, and Sorelle\u00a0A. Friedler. 2020. Problems with Shapley-value-based Explanations as Feature Importance Measures. In Proceedings of the 37th International Conference on Machine Learning(ICML\u201920, Vol.\u00a0119). JMLR.org, 5491\u20135500."},{"key":"e_1_3_3_3_66_2","doi-asserted-by":"publisher","unstructured":"Michelle\u00a0S. Lam Ayush Pandit Colin\u00a0H. Kalicki Rachit Gupta Poonam Sahoo and Dana\u00eb Metaxa. 2023. Sociotechnical Audits: Broadening the Algorithm Auditing Lens to Investigate Targeted Advertising. Proceedings of the ACM on Human-Computer Interaction 7 CSCW2 (Oct. 2023) 360:1\u2013360:37. 10.1145\/3610209","DOI":"10.1145\/3610209"},{"key":"e_1_3_3_3_67_2","volume-title":"Standards and Their Stories: How Quantifying, Classifying, and Formalizing Practices Shape Everyday Life","author":"Lampland Martha","year":"2009","unstructured":"Martha Lampland and Susan\u00a0Leigh Star. 2009. Standards and Their Stories: How Quantifying, Classifying, and Formalizing Practices Shape Everyday Life. Cornell University Press."},{"key":"e_1_3_3_3_68_2","doi-asserted-by":"publisher","DOI":"10.1145\/3600211.3604701"},{"key":"e_1_3_3_3_69_2","doi-asserted-by":"publisher","unstructured":"Allen\u00a0S. Lee and Richard\u00a0L. Baskerville. 2003. Generalizing Generalizability in Information Systems Research. Information Systems Research 14 3 (2003) 221\u2013243. 10.1287\/isre.14.3.221.16560","DOI":"10.1287\/isre.14.3.221.16560"},{"key":"e_1_3_3_3_70_2","doi-asserted-by":"publisher","DOI":"10.1145\/3411764.3445261"},{"key":"e_1_3_3_3_71_2","volume-title":"Digital Services Act: Summary Report on the Call for Evidence on the Delegated Regulation on Data Access","author":"Leerssen Paddy","year":"2023","unstructured":"Paddy Leerssen. 2023. Digital Services Act: Summary Report on the Call for Evidence on the Delegated Regulation on Data Access. Technical Report. European Commission. https:\/\/digital-strategy.ec.europa.eu\/en\/library\/digital-services-act-summary-report-call-evidence-delegated-regulation-data-access"},{"key":"e_1_3_3_3_72_2","volume-title":"Federal AI Legislation: An Analysis of Proposals from the 117th Congress Relevant to Generative AI Tools","author":"Lenhart Anna","year":"2023","unstructured":"Anna Lenhart. 2023. Federal AI Legislation: An Analysis of Proposals from the 117th Congress Relevant to Generative AI Tools. Technical Report. Institute for Data, Democracy & Politics, George Washington University. https:\/\/iddp.gwu.edu\/sites\/g\/files\/zaxdzs5791\/files\/2023-06\/federal_ai_legislation_v3.pdf"},{"key":"e_1_3_3_3_73_2","unstructured":"Samuel Levine. 2021. Letter from Acting Director of the Bureau of Consumer Protection Samuel Levine to Facebook. https:\/\/www.ftc.gov\/blog-posts\/2021\/08\/letter-acting-director-bureau-consumer-protection-samuel-levine-facebook"},{"key":"e_1_3_3_3_74_2","doi-asserted-by":"publisher","unstructured":"Tiffany Li. 2022. Algorithmic Destruction. SMU Law Review 75 3 (Jan. 2022) 479. 10.25172\/smulr.75.3.2","DOI":"10.25172\/smulr.75.3.2"},{"key":"e_1_3_3_3_75_2","doi-asserted-by":"publisher","DOI":"10.1145\/3313831.3376590"},{"key":"e_1_3_3_3_76_2","doi-asserted-by":"publisher","unstructured":"Zachary\u00a0C. Lipton. 2018. The Mythos of Model Interpretability: In Machine Learning the Concept of Interpretability Is Both Important and Slippery. Queue 16 3 (June 2018) 31\u201357. 10.1145\/3236386.3241340","DOI":"10.1145\/3236386.3241340"},{"key":"e_1_3_3_3_77_2","doi-asserted-by":"publisher","unstructured":"Shayne Longpre Sayash Kapoor Kevin Klyman Ashwin Ramaswami Rishi Bommasani Borhane Blili-Hamelin Yangsibo Huang Aviya Skowron Zheng-Xin Yong Suhas Kotha Yi Zeng Weiyan Shi Xianjun Yang Reid Southen Alexander Robey Patrick Chao Diyi Yang Ruoxi Jia Daniel Kang Sandy Pentland Arvind Narayanan Percy Liang and Peter Henderson. 2024. A Safe Harbor for AI Evaluation and Red Teaming. 10.48550\/arXiv.2403.04893 arxiv:https:\/\/arXiv.org\/abs\/2403.04893\u00a0[cs]","DOI":"10.48550\/arXiv.2403.04893"},{"key":"e_1_3_3_3_78_2","doi-asserted-by":"publisher","DOI":"10.5555\/3295222.3295230"},{"key":"e_1_3_3_3_79_2","doi-asserted-by":"publisher","DOI":"10.1145\/3313831.3376445"},{"key":"e_1_3_3_3_80_2","volume-title":"Public AI: Making AI Work for Everyone, by Everyone","author":"Marda Nik","year":"2024","unstructured":"Nik Marda, Jasmine Sun, and Mark Surman. 2024. Public AI: Making AI Work for Everyone, by Everyone. Technical Report. Mozilla Foundation."},{"key":"e_1_3_3_3_81_2","unstructured":"Phil Mendelson. 2021. Stop Discrimination by Algorithms Act of 2021. https:\/\/lims.dccouncil.gov\/Legislation\/B24-0558"},{"key":"e_1_3_3_3_82_2","doi-asserted-by":"publisher","unstructured":"Dana\u00eb Metaxa Joon\u00a0Sung Park Ronald\u00a0E. Robertson Karrie Karahalios Christo Wilson Jeff Hancock and Christian Sandvig. 2021. Auditing Algorithms: Understanding Algorithmic Systems from the Outside In. Foundations and Trends\u00ae in Human\u2013Computer Interaction 14 4 (2021) 272\u2013344. 10.1561\/1100000083","DOI":"10.1561\/1100000083"},{"key":"e_1_3_3_3_83_2","doi-asserted-by":"publisher","DOI":"10.1145\/3287560.3287596"},{"key":"e_1_3_3_3_84_2","volume-title":"YouTube Regrets","author":"Foundation Mozilla","year":"2021","unstructured":"Mozilla Foundation. 2021. YouTube Regrets. Technical Report. Mozilla Foundation. https:\/\/assets.mofoprod.net\/network\/documents\/Mozilla_YouTube_Regrets_Report.pdf"},{"key":"e_1_3_3_3_85_2","unstructured":"Mozilla Technology Fund (MTF) 2024. Mozilla Technology Fund (MTF). https:\/\/foundation.mozilla.org\/en\/what-we-fund\/opportunities\/mozilla-technology-fund-mtf\/"},{"key":"e_1_3_3_3_86_2","volume-title":"AI Snake Oil: What Artificial Intelligence Can Do, What It Can\u2019t, and How to Tell the Difference","author":"Narayanan Arvind","year":"2024","unstructured":"Arvind Narayanan and Sayash Kapoor. 2024. AI Snake Oil: What Artificial Intelligence Can Do, What It Can\u2019t, and How to Tell the Difference. Princeton University Press, Princeton."},{"key":"e_1_3_3_3_87_2","doi-asserted-by":"publisher","DOI":"10.6028\/NIST.IR.8331"},{"key":"e_1_3_3_3_88_2","doi-asserted-by":"publisher","DOI":"10.1787\/008232ec-en"},{"key":"e_1_3_3_3_89_2","volume-title":"Blueprint for an AI Bill of Rights","author":"Policy Office of Science and Technology","year":"2022","unstructured":"Office of Science and Technology Policy. 2022. Blueprint for an AI Bill of Rights. Technical Report. Executive Office of the President. https:\/\/www.whitehouse.gov\/ostp\/ai-bill-of-rights\/"},{"key":"e_1_3_3_3_90_2","doi-asserted-by":"publisher","unstructured":"OpenAI. 2023. GPT-4 Technical Report. 10.48550\/arXiv.2303.08774 arxiv:https:\/\/arXiv.org\/abs\/2303.08774\u00a0[cs]","DOI":"10.48550\/arXiv.2303.08774"},{"key":"e_1_3_3_3_91_2","unstructured":"Billy Perrigo. 2023. California Bill Proposes Regulating AI at State Level. Time (Sept. 2023). https:\/\/time.com\/6313588\/california-ai-regulation-bill\/"},{"key":"e_1_3_3_3_92_2","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2023.acl-demo.29"},{"key":"e_1_3_3_3_93_2","doi-asserted-by":"publisher","DOI":"10.1093\/acprof:oso\/9780198296034.001.0001"},{"key":"e_1_3_3_3_94_2","volume-title":"Community Engagement Toolkit for Planning","author":"Government Queensland","year":"2017","unstructured":"Queensland Government. 2017. Community Engagement Toolkit for Planning. Technical Report. https:\/\/dilgpprd.blob.core.windows.net\/general\/community-engagement-toolkit.pdf"},{"key":"e_1_3_3_3_95_2","doi-asserted-by":"publisher","unstructured":"Inioluwa\u00a0Deborah Raji and Joy Buolamwini. 2022. Actionable Auditing Revisited: Investigating the Impact of Publicly Naming Biased Performance Results of Commercial AI Products. Commun. ACM 66 1 (Dec. 2022) 101\u2013108. 10.1145\/3571151","DOI":"10.1145\/3571151"},{"key":"e_1_3_3_3_96_2","doi-asserted-by":"publisher","DOI":"10.1145\/3531146.3533158"},{"key":"e_1_3_3_3_97_2","doi-asserted-by":"publisher","DOI":"10.1145\/3351095.3372873"},{"key":"e_1_3_3_3_98_2","doi-asserted-by":"publisher","DOI":"10.1145\/3514094.3534181"},{"key":"e_1_3_3_3_99_2","doi-asserted-by":"publisher","unstructured":"Bogdana Rakova Jingying Yang Henriette Cramer and Rumman Chowdhury. 2021. Where Responsible AI Meets Reality: Practitioner Perspectives on Enablers for Shifting Organizational Practices. Proceedings of the ACM on Human-Computer Interaction 5 CSCW1 (April 2021) 7:1\u20137:23. 10.1145\/3449081","DOI":"10.1145\/3449081"},{"key":"e_1_3_3_3_100_2","unstructured":"Tate Ryan-Mosley. 2023. Why Everyone Is Mad about New York\u2019s AI Hiring Law. MIT Technology Review (July 2023). https:\/\/www.technologyreview.com\/2023\/07\/10\/1076013\/new-york-ai-hiring-law\/"},{"key":"e_1_3_3_3_101_2","unstructured":"Sheryl Sandberg. 2019. Doing More to Protect Against Discrimination in Housing Employment and Credit Advertising. https:\/\/about.fb.com\/news\/2019\/03\/protecting-against-discrimination-in-ads\/"},{"key":"e_1_3_3_3_102_2","first-page":"23","volume-title":"Data and Discrimination: Converting Critical Concerns into Productive: A Preconference at the 64th Annual Meeting of the International Communication Association","author":"Sandvig Christian","year":"2014","unstructured":"Christian Sandvig, Kevin Hamilton, Karrie Karahalios, and Cedric Langbort. 2014. Auditing Algorithms: Research Methods for Detecting Discrimination on Internet Platforms. In Data and Discrimination: Converting Critical Concerns into Productive: A Preconference at the 64th Annual Meeting of the International Communication Association. Seattle, WA, 23."},{"key":"e_1_3_3_3_103_2","unstructured":"Sandvig v. Bar 2020. Sandvig v. Bar. https:\/\/casetext.com\/case\/sandvig-v-barr"},{"key":"e_1_3_3_3_104_2","doi-asserted-by":"publisher","DOI":"10.1145\/3287560.3287598"},{"key":"e_1_3_3_3_105_2","unstructured":"Selenium 2023. Selenium. https:\/\/www.selenium.dev\/"},{"key":"e_1_3_3_3_106_2","unstructured":"Nathan Sheard. 2021. Banning Government Use of Face Recognition Technology: 2020 Year in Review. https:\/\/www.eff.org\/deeplinks\/2020\/12\/banning-government-use-face-recognition-technology-2020-year-review"},{"key":"e_1_3_3_3_107_2","doi-asserted-by":"publisher","DOI":"10.1145\/3600211.3604673"},{"key":"e_1_3_3_3_108_2","doi-asserted-by":"publisher","unstructured":"Hong Shen \u00c1ngel\u00a0Alexander Cabrera Adam Perer and Jason Hong. 2022. \"Public(s)-in-the-Loop\": Facilitating Deliberation of Algorithmic Decisions in Contentious Public Policy Domains. 10.48550\/arXiv.2204.10814 arxiv:https:\/\/arXiv.org\/abs\/2204.10814\u00a0[cs]","DOI":"10.48550\/arXiv.2204.10814"},{"key":"e_1_3_3_3_109_2","doi-asserted-by":"crossref","unstructured":"Hong Shen Alicia DeVos Motahhare Eslami and Kenneth Holstein. 2021. Everyday Algorithm Auditing: Understanding the Power of Everyday Users in Surfacing Harmful Algorithmic Behaviors. Proceedings of the ACM on Human-Computer Interaction 5 CSCW2 (2021) 1\u201329.","DOI":"10.1145\/3479577"},{"key":"e_1_3_3_3_110_2","unstructured":"Galen Sherwin and Esha Bhandari. 2019. Facebook Settles Civil Rights Cases by Making Sweeping Changes to Its Online Ad Platform | ACLU of Northern CA. https:\/\/www.aclunc.org\/blog\/facebook-settles-civil-rights-cases-making-sweeping-changes-its-online-ad-platform"},{"key":"e_1_3_3_3_111_2","doi-asserted-by":"publisher","DOI":"10.1145\/3551624.3555285"},{"key":"e_1_3_3_3_112_2","doi-asserted-by":"publisher","unstructured":"Mona Sloane Emanuel Moss and Rumman Chowdhury. 2022. A Silicon Valley Love Triangle: Hiring Algorithms Pseudo-Science and the Quest for Auditability. Patterns (New York N.Y.) 3 2 (Feb. 2022) 100425. 10.1016\/j.patter.2021.100425","DOI":"10.1016\/j.patter.2021.100425"},{"key":"e_1_3_3_3_113_2","doi-asserted-by":"publisher","DOI":"10.1145\/3313831.3376624"},{"key":"e_1_3_3_3_114_2","unstructured":"Chandler\u00a0Nicholle Spinks. 2020. Contemporary Housing Discrimination: Facebook Targeted Advertising and the Fair Housing Act. Houston Law Review 57 4 (May 2020) 925\u2013952. https:\/\/houstonlawreview.org\/article\/12762-contemporary-housing-discrimination-facebook-targeted-advertising-and-the-fair-housing-act"},{"key":"e_1_3_3_3_115_2","doi-asserted-by":"publisher","unstructured":"Susan\u00a0Leigh Star. 1999. The Ethnography of Infrastructure. American Behavioral Scientist 43 3 (Nov. 1999) 377\u2013391. 10.1177\/00027649921955326","DOI":"10.1177\/00027649921955326"},{"key":"e_1_3_3_3_116_2","doi-asserted-by":"crossref","unstructured":"Luke Stark and Jevan Hutson. 2021. Physiognomic Artificial Intelligence. Fordham Intell. Prop. Media & Ent. LJ 32 (2021) 922.","DOI":"10.2139\/ssrn.3927300"},{"key":"e_1_3_3_3_117_2","unstructured":"Stop LAPD Spying Coalition and Free Radicals. 2020. The Algorithmic Ecology: An Abolitionist Tool for Organizing Against Algorithms. https:\/\/freerads.org\/2020\/03\/02\/the-algorithmic-ecology-an-abolitionist-tool-for-organizing-against-algorithms\/"},{"key":"e_1_3_3_3_118_2","doi-asserted-by":"publisher","DOI":"10.6028\/NIST.AI.100-1"},{"key":"e_1_3_3_3_119_2","doi-asserted-by":"publisher","DOI":"10.1145\/3411764.3445768"},{"key":"e_1_3_3_3_120_2","doi-asserted-by":"publisher","DOI":"10.1145\/3630106.3658970"},{"key":"e_1_3_3_3_121_2","unstructured":"The Markup. 2022. Citizen Browser. https:\/\/themarkup.org\/series\/citizen-browser"},{"key":"e_1_3_3_3_122_2","doi-asserted-by":"publisher","DOI":"10.1145\/3600211.3604700"},{"key":"e_1_3_3_3_123_2","unstructured":"Twitter. 2021. Twitter Algorithmic Bias - Bug Bounty Program. https:\/\/hackerone.com\/twitter-algorithmic-bias"},{"key":"e_1_3_3_3_124_2","doi-asserted-by":"publisher","unstructured":"Aleksandra Urman Ivan Smirnov and Jana Lasser. 2024. The Right to Audit and Power Asymmetries in Algorithm Auditing. EPJ Data Science 13 1 (Dec. 2024) 1\u201315. 10.1140\/epjds\/s13688-024-00454-5","DOI":"10.1140\/epjds\/s13688-024-00454-5"},{"key":"e_1_3_3_3_125_2","doi-asserted-by":"publisher","DOI":"10.1145\/3465416.3483294"},{"key":"e_1_3_3_3_126_2","doi-asserted-by":"publisher","unstructured":"Peter-Paul Verbeek. 2006. Materializing Morality: Design Ethics and Technological Mediation. Science Technology & Human Values 31 3 (May 2006) 361\u2013380. 10.1177\/0162243905285847","DOI":"10.1177\/0162243905285847"},{"key":"e_1_3_3_3_127_2","unstructured":"Sandra Wachter Brent Mittelstadt and Chris Russell. 2021. Bias Preservation in Machine Learning: The Legality of Fairness Metrics Under EU Non-Discrimination Law. West Virginia Law Review 123 3 (April 2021) 735. https:\/\/researchrepository.wvu.edu\/wvlr\/vol123\/iss3\/4"},{"key":"e_1_3_3_3_128_2","unstructured":"Annie Waldman. 2024. How UnitedHealth\u2019s Playbook for Limiting Mental Health Coverage Puts Countless Americans\u2019 Treatment at Risk. ProPublica (Nov. 2024). https:\/\/www.propublica.org\/article\/unitedhealth-mental-health-care-denied-illegal-algorithm"},{"key":"e_1_3_3_3_129_2","doi-asserted-by":"publisher","DOI":"10.1145\/3593013.3594030"},{"key":"e_1_3_3_3_130_2","doi-asserted-by":"publisher","DOI":"10.1145\/3290605.3300831"},{"key":"e_1_3_3_3_131_2","unstructured":"Tris Warkentin and Josh Woodward. 2022. Join Us in the AI Test Kitchen. https:\/\/blog.google\/technology\/ai\/join-us-in-the-ai-test-kitchen\/"},{"key":"e_1_3_3_3_132_2","doi-asserted-by":"publisher","DOI":"10.1145\/3461702.3462580"},{"key":"e_1_3_3_3_133_2","unstructured":"Hilde Weerts Miroslav Dud\u00edk Richard Edgar Adrin Jalali Roman Lutz and Michael Madaio. 2024. Fairlearn: Assessing and Improving Fairness of AI Systems. J. Mach. Learn. Res. 24 1 (March 2024) 257:12058\u2013257:12065."},{"key":"e_1_3_3_3_134_2","doi-asserted-by":"publisher","unstructured":"David\u00a0Gray Widder Meredith Whittaker and Sarah\u00a0Myers West. 2024. Why \u2018Open\u2019 AI Systems Are Actually Closed and Why This Matters. Nature 635 8040 (Nov. 2024) 827\u2013833. 10.1038\/s41586-024-08141-1","DOI":"10.1038\/s41586-024-08141-1"},{"key":"e_1_3_3_3_135_2","doi-asserted-by":"publisher","DOI":"10.1145\/3593013.3594012"},{"key":"e_1_3_3_3_136_2","unstructured":"Langdon Winner. 1980. Do Artifacts Have Politics? Daedalus 109 1 (1980) 121\u2013136. jstor:20024652https:\/\/www.jstor.org\/stable\/20024652"},{"key":"e_1_3_3_3_137_2","volume-title":"Transforming Qualitative Data: Description, Analysis, and Interpretation","author":"Wolcott Harry\u00a0F.","year":"1994","unstructured":"Harry\u00a0F. Wolcott. 1994. Transforming Qualitative Data: Description, Analysis, and Interpretation. SAGE."},{"key":"e_1_3_3_3_138_2","doi-asserted-by":"publisher","unstructured":"Richmond\u00a0Y. Wong Michael\u00a0A. Madaio and Nick Merrill. 2023. Seeing Like a Toolkit: How Toolkits Envision the Work of AI Ethics. Proceedings of the ACM on Human-Computer Interaction 7 CSCW1 (April 2023) 145:1\u2013145:27. 10.1145\/3579621","DOI":"10.1145\/3579621"},{"key":"e_1_3_3_3_139_2","doi-asserted-by":"publisher","DOI":"10.1145\/3173574.3174230"},{"key":"e_1_3_3_3_140_2","doi-asserted-by":"publisher","DOI":"10.1145\/3630106.3658998"},{"key":"e_1_3_3_3_141_2","doi-asserted-by":"publisher","unstructured":"Alice Xiang and Inioluwa\u00a0Deborah Raji. 2019. On the Legal Compatibility of Fairness Definitions. 10.48550\/arXiv.1912.00761 arxiv:https:\/\/arXiv.org\/abs\/1912.00761\u00a0[cs stat]","DOI":"10.48550\/arXiv.1912.00761"}],"event":{"name":"CHI 2025: CHI Conference on Human Factors in Computing Systems","location":"Yokohama Japan","acronym":"CHI '25","sponsor":["SIGCHI ACM Special Interest Group on Computer-Human Interaction"]},"container-title":["Proceedings of the 2025 CHI Conference on Human Factors in Computing Systems"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3706598.3713301","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3706598.3713301","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,7,4]],"date-time":"2025-07-04T05:38:21Z","timestamp":1751607501000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3706598.3713301"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,4,25]]},"references-count":140,"alternative-id":["10.1145\/3706598.3713301","10.1145\/3706598"],"URL":"https:\/\/doi.org\/10.1145\/3706598.3713301","relation":{},"subject":[],"published":{"date-parts":[[2025,4,25]]},"assertion":[{"value":"2025-04-25","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}