{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,6,29]],"date-time":"2026-06-29T18:46:37Z","timestamp":1782758797897,"version":"3.54.5"},"publisher-location":"New York, NY, USA","reference-count":70,"publisher":"ACM","license":[{"start":{"date-parts":[[2026,6,25]],"date-time":"2026-06-25T00:00:00Z","timestamp":1782345600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/legalcode"}],"funder":[{"DOI":"10.13039\/501100000038","name":"Natural Sciences and Engineering Research Council of Canada","doi-asserted-by":"publisher","award":["599255"],"award-info":[{"award-number":["599255"]}],"id":[{"id":"10.13039\/501100000038","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100000038","name":"PricewaterhouseCoopers LLP","doi-asserted-by":"publisher","award":["1012955"],"award-info":[{"award-number":["1012955"]}],"id":[{"id":"10.13039\/501100000038","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2026,6,25]]},"DOI":"10.1145\/3805689.3806759","type":"proceedings-article","created":{"date-parts":[[2026,6,23]],"date-time":"2026-06-23T16:20:39Z","timestamp":1782231639000},"page":"6006-6066","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":0,"title":["Toward Third-Party Assurance of AI Systems: Design Requirements, Prototype, and Early Testing"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0009-0000-3325-243X","authenticated-orcid":false,"given":"Rachel M.","family":"Kim","sequence":"first","affiliation":[{"name":"Software and Societal Systems, Carnegie Mellon University, Pittsburgh, Pennsylvania, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0001-5702-5693","authenticated-orcid":false,"given":"Blaine","family":"Kuehnert","sequence":"additional","affiliation":[{"name":"Carnegie Mellon University, Pittsburgh, PENNSYLVANIA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0009-0352-2596","authenticated-orcid":false,"given":"Alice","family":"Lai","sequence":"additional","affiliation":[{"name":"Carnegie Mellon University, Pittsburgh, PENNSYLVANIA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-6730-922X","authenticated-orcid":false,"given":"Kenneth","family":"Holstein","sequence":"additional","affiliation":[{"name":"Carnegie Mellon University, Pittsburgh, PENNSYLVANIA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-3710-4076","authenticated-orcid":false,"given":"Hoda","family":"Heidari","sequence":"additional","affiliation":[{"name":"Carnegie Mellon University, Pittsburgh, PENNSYLVANIA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-0235-1843","authenticated-orcid":false,"given":"Rayid","family":"Ghani","sequence":"additional","affiliation":[{"name":"Carnegie Mellon University, Pittsburgh, PENNSYLVANIA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2026,6,25]]},"reference":[{"key":"e_1_3_2_1_1_1","first-page":"2023","volume":"42001","year":"2023","unstructured":"2023. ISO\/IEC 42001:2023. https:\/\/www.iso.org\/standard\/42001","journal-title":"ISO\/IEC"},{"key":"e_1_3_2_1_2_1","first-page":"621","article-title":"An auditing imperative for automated hiring systems","volume":"34","author":"Ajunwa Ifeoma","year":"2020","unstructured":"Ifeoma Ajunwa. 2020. An auditing imperative for automated hiring systems. Harv. JL & Tech. 34 (2020), 621.","journal-title":"Harv. JL & Tech."},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1145\/3453444"},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.1145\/3449148"},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1145\/3593434.3593478"},{"key":"e_1_3_2_1_6_1","volume-title":"Bender and Batya Friedman","author":"Emily","year":"2018","unstructured":"Emily M. Bender and Batya Friedman. 2018. Data Statements for Natural Language Processing: Toward Mitigating System Bias and"},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"publisher","DOI":"10.1145\/3613904.3642398"},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.1109\/SaTML59370.2024.00037"},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1145\/3617694.3623259"},{"key":"e_1_3_2_1_10_1","volume-title":"Using thematic analysis in psychology. Qualitative research in psychology 3, 2","author":"Braun Virginia","year":"2006","unstructured":"Virginia Braun and Victoria Clarke. 2006. Using thematic analysis in psychology. Qualitative research in psychology 3, 2 (2006), 77\u2013101."},{"key":"e_1_3_2_1_11_1","unstructured":"John Brooke et al. 1996. SUS-A quick and dirty usability scale. Usability evaluation in industry 189 194 (1996) 4\u20137."},{"key":"e_1_3_2_1_12_1","volume-title":"The system cannot fail-Reflections on The Audit Society: Rituals of Verification. Policy & Practice-A Development Education Review 11","author":"Bruen Carlos","year":"2010","unstructured":"Carlos Bruen. 2010. The system cannot fail-Reflections on The Audit Society: Rituals of Verification. Policy & Practice-A Development Education Review 11 (2010)."},{"key":"e_1_3_2_1_13_1","volume-title":"What is this thing called efficacy. Philosophy of the social sciences: Philosophical theory and scientific practice","author":"Cartwright Nancy","year":"2009","unstructured":"Nancy Cartwright. 2009. What is this thing called efficacy. Philosophy of the social sciences: Philosophical theory and scientific practice (2009), 185\u2013206."},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.1145\/3630106.3659037"},{"key":"e_1_3_2_1_15_1","unstructured":"James Chalmers. 2021. Trust in non-financial reporting: The role of Assurance. https:\/\/www.pwc.com\/gx\/en\/services\/audit-assurance\/corporate-reporting\/trust-in-non-financial-reporting.html"},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.1145\/3531146.3533213"},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.1109\/SaTML54575.2023.00050"},{"key":"e_1_3_2_1_18_1","unstructured":"Rikke Friis Dam. 2025. The 5 Stages in the Design Thinking Process. https:\/\/www.interaction-design.org\/literature\/article\/5-stages-in-the-design-thinking-process Interaction Design Foundation."},{"key":"e_1_3_2_1_19_1","unstructured":"Dan Faggella. 2018. Enterprise adoption of artificial intelligence - when it does and doesn't make sense. https:\/\/emerj.com\/enterprise-adoption-of- artificial-intelligence\/"},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.jenvman.2016.09.047"},{"key":"e_1_3_2_1_21_1","volume-title":"Shruti Nair, Shambhavi Bhusan, Sanjana Dulam, Mouni Pendharkar, et al.","author":"Flanagan William","year":"2025","unstructured":"William Flanagan, Mukunda Das, Rajitha Ramanayake, Swanuja Maslekar, Meghana Mangipudi, Joong Ho Choi, Shruti Nair, Shambhavi Bhusan, Sanjana Dulam, Mouni Pendharkar, et al. 2025. A Methodology for Assessing the Risk of Metric Failure in LLMs Within the Financial Domain. arXiv preprint arXiv:2510.13524 (2025)."},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-69796-8_1"},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.1145\/3458723"},{"key":"e_1_3_2_1_24_1","unstructured":"Aditi Godbole. 2025. AI for Business: When to use it and when to skip it. https:\/\/www.forbes.com\/councils\/forbestechcouncil\/2025\/03\/10\/ai-for-business-when-to-use-it-and-when-to-skip-it\/?utm_source=chatgpt.com"},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.1145\/1357054.1357074"},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.1145\/3630106.3658959"},{"key":"e_1_3_2_1_27_1","volume-title":"Comment: Snowball versus respondent-driven sampling. Sociological methodology 41, 1","author":"Heckathorn Douglas D","year":"2011","unstructured":"Douglas D Heckathorn. 2011. Comment: Snowball versus respondent-driven sampling. Sociological methodology 41, 1 (2011), 355\u2013366."},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"publisher","DOI":"10.1145\/3555760"},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.1145\/3613904.3642849"},{"key":"e_1_3_2_1_30_1","volume-title":"Proceedings of the AAAI\/ACM Conference on AI, Ethics, and Society","volume":"7","author":"Kawakami Anna","year":"2024","unstructured":"Anna Kawakami, Daricia Wilkinson, and Alexandra Chouldechova. 2024. Do responsible AI artifacts advance stakeholder goals? four key barriers perceived by legal and civil stakeholders. In Proceedings of the AAAI\/ACM Conference on AI, Ethics, and Society, Vol. 7. 670\u2013682."},{"key":"e_1_3_2_1_31_1","unstructured":"Kate Kaye and Pam Dixon. 2023. Risky Analysis: Assessing and Improving AI Governance Tools: An international review of AI Governance Tools and suggestions for pathways forward. https:\/\/www.worldprivacyforum.org\/wp-content\/uploads\/2023\/12\/WPF_Risky_Analysis_December_2023_fs.pdf [Accessed 10-01-2025]."},{"key":"e_1_3_2_1_32_1","volume-title":"The rise of checkbox AI ethics: a review. AI and Ethics","author":"Kijewski Sara","year":"2024","unstructured":"Sara Kijewski, Elettra Ronchi, and Effy Vayena. 2024. The rise of checkbox AI ethics: a review. AI and Ethics (2024), 1\u201310."},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"publisher","DOI":"10.1145\/3715275.3732191"},{"key":"e_1_3_2_1_34_1","unstructured":"The Algorithmic Bias Lab. [n. d.]. AI and Algorithm Auditor Certification Program Essential Tools for AI and Algorithm Auditing. https:\/\/courses.babl.ai\/p\/ai-and-algorithm-auditor-certification"},{"key":"e_1_3_2_1_35_1","doi-asserted-by":"publisher","DOI":"10.1145\/3630106.3658957"},{"key":"e_1_3_2_1_36_1","doi-asserted-by":"publisher","DOI":"10.1145\/3173574.3173610"},{"key":"e_1_3_2_1_37_1","doi-asserted-by":"publisher","DOI":"10.1145\/3411764.3445261"},{"key":"e_1_3_2_1_38_1","unstructured":"Susan S. Lewis. 2023. Audit vs. Assurance \u2014 What Is the Difference? https:\/\/www.lewis.cpa\/blog\/audit-vs-assurance-what-is-the-difference"},{"key":"e_1_3_2_1_39_1","doi-asserted-by":"publisher","DOI":"10.1109\/MIC.2025.3621976"},{"key":"e_1_3_2_1_40_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.clsr.2018.05.017"},{"key":"e_1_3_2_1_41_1","doi-asserted-by":"publisher","DOI":"10.1145\/3287560.3287596"},{"key":"e_1_3_2_1_42_1","doi-asserted-by":"publisher","DOI":"10.1007\/s44206-023-00074-y"},{"key":"e_1_3_2_1_43_1","doi-asserted-by":"publisher","DOI":"10.1007\/s11023-021-09557-8"},{"key":"e_1_3_2_1_44_1","doi-asserted-by":"publisher","DOI":"10.1007\/s43681-022-00171-7"},{"key":"e_1_3_2_1_45_1","doi-asserted-by":"publisher","DOI":"10.1007\/s11948-019-00165-5"},{"key":"e_1_3_2_1_46_1","doi-asserted-by":"crossref","unstructured":"Jakob Nielsen. 1994. Usability engineering. Morgan Kaufmann.","DOI":"10.1016\/B978-0-08-052029-2.50009-7"},{"key":"e_1_3_2_1_47_1","doi-asserted-by":"crossref","unstructured":"NIST. 2023. Artificial Intelligence Risk Management Framework (AI RMF 1.0).","DOI":"10.6028\/NIST.AI.100-1.jpn"},{"key":"e_1_3_2_1_48_1","unstructured":"Treasury Board of Canada Secretariat. 2024. Algorithmic Impact Assessment Tool. https:\/\/www.canada.ca\/en\/government\/system\/digital-government\/digital-government-innovations\/responsible-use-ai\/algorithmic-impact-assessment.html [Accessed 08-01-2025]."},{"key":"e_1_3_2_1_49_1","unstructured":"Board of Governors of the Federal Reserve System. 2011. SR 11-7: Guidance on Model Risk Management. https:\/\/www.federalreserve.gov\/supervisionreg\/srletters\/sr1107.htm"},{"key":"e_1_3_2_1_50_1","doi-asserted-by":"publisher","DOI":"10.1145\/3706598.3713301"},{"key":"e_1_3_2_1_51_1","doi-asserted-by":"publisher","DOI":"10.1145\/1294211.1294256"},{"key":"e_1_3_2_1_52_1","unstructured":"Kim Orr. 2024. Engineering Design Process Flow Chart. https:\/\/www.jpl.nasa.gov\/edu\/resources\/image\/engineering-design-process-flow-chart\/"},{"key":"e_1_3_2_1_53_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10462-024-10740-3"},{"key":"e_1_3_2_1_54_1","unstructured":"Michael Power. 1997. The audit society: Rituals of verification. OUP Oxford."},{"key":"e_1_3_2_1_55_1","doi-asserted-by":"publisher","DOI":"10.1145\/3531146.3533158"},{"key":"e_1_3_2_1_56_1","doi-asserted-by":"publisher","DOI":"10.1145\/3351095.3372873"},{"key":"e_1_3_2_1_57_1","doi-asserted-by":"publisher","DOI":"10.1145\/3514094.3534181"},{"key":"e_1_3_2_1_58_1","volume-title":"Algorithmic impact assessments: a practical Framework for Public Agency. AI Now 9","author":"Reisman Dillon","year":"2018","unstructured":"Dillon Reisman, Jason Schultz, Kate Crawford, and Meredith Whittaker. 2018. Algorithmic impact assessments: a practical Framework for Public Agency. AI Now 9 (2018)."},{"key":"e_1_3_2_1_59_1","doi-asserted-by":"publisher","DOI":"10.1145\/3544548.3581407"},{"key":"e_1_3_2_1_60_1","unstructured":"Julia Rittenberg and Rob Watts. 2024. Everything you need to know to set up a responsibility assignment matrix (RAM). https:\/\/www.forbes.com\/advisor\/business\/software\/what-is-responsibility-assignment-matrix\/"},{"key":"e_1_3_2_1_61_1","unstructured":"D Robbins O Eris A Kapusta L Booker and P Ward. 2024. AI assurance: A repeatable process for assuring AI-enabled systems. The MITRE Corporation Tech. Rep (2024) 2024\u201306."},{"key":"e_1_3_2_1_62_1","unstructured":"Stuart Russell Karine Perset and Marko Grobelnik. 2023. Updates to the OECD's definition of an AI system explained. https:\/\/oecd.ai\/en\/wonk\/ai-system-definition-update"},{"key":"e_1_3_2_1_63_1","unstructured":"Kristina Russo. 2023. What is a financial statement audit? https:\/\/www.netsuite.com\/portal\/resource\/articles\/financial-management\/financial-statement-audit.shtml"},{"key":"e_1_3_2_1_64_1","volume-title":"Principles to practices for responsible AI: closing the gap. arXiv preprint arXiv:2006.04707","author":"Schiff Daniel","year":"2020","unstructured":"Daniel Schiff, Bogdana Rakova, Aladdin Ayesh, Anat Fanti, and Michael Lennon. 2020. Principles to practices for responsible AI: closing the gap. arXiv preprint arXiv:2006.04707 (2020)."},{"key":"e_1_3_2_1_65_1","doi-asserted-by":"publisher","DOI":"10.1177\/20539517241299732"},{"key":"e_1_3_2_1_66_1","unstructured":"The European Parliament and the Council of the European Union. 2024. High-level summary of the AI Act. https:\/\/artificialintelligenceact.eu\/high-level-summary\/"},{"key":"e_1_3_2_1_67_1","volume-title":"Innovation and Technology","author":"UK Department for Science","year":"2024","unstructured":"UK Department for Science, Innovation and Technology. 2024. Introduction to AI Assurance. https:\/\/www.gov.uk\/government\/publications\/introduction-to-ai-assurance\/introduction-to-ai-assurance#ai-assurance-in-practice"},{"key":"e_1_3_2_1_68_1","unstructured":"ForHumanity University. 2024. Welcome to ForHumanity AI Education and Training Center. https:\/\/forhumanity.center\/forhumanity-university\/"},{"key":"e_1_3_2_1_69_1","unstructured":"U.S. Department of Transportation. 2024. An Overview of AI Assurance for Transportation. https:\/\/www.transportation.gov\/sites\/dot.gov\/files\/2024-04\/HASS_COE_AI_Assurance_Whitepaper_Apr2024.pdf"},{"key":"e_1_3_2_1_70_1","doi-asserted-by":"publisher","DOI":"10.1145\/3544548.3580900"}],"event":{"name":"FAccT '26: The 2026 ACM Conference on Fairness, Accountability, and Transparency","location":"Montreal QC Canada","acronym":"FAccT '26","sponsor":["ACM\/SIG"]},"container-title":["Proceedings of the 2026 ACM Conference on Fairness, Accountability, and Transparency"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3805689.3806759","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,6,29]],"date-time":"2026-06-29T18:19:54Z","timestamp":1782757194000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3805689.3806759"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026,6,25]]},"references-count":70,"alternative-id":["10.1145\/3805689.3806759","10.1145\/3805689"],"URL":"https:\/\/doi.org\/10.1145\/3805689.3806759","relation":{},"subject":[],"published":{"date-parts":[[2026,6,25]]},"assertion":[{"value":"2026-06-25","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}