{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,6,30]],"date-time":"2026-06-30T06:59:02Z","timestamp":1782802742615,"version":"3.54.5"},"publisher-location":"New York, NY, USA","reference-count":244,"publisher":"ACM","license":[{"start":{"date-parts":[[2021,3,1]],"date-time":"2021-03-01T00:00:00Z","timestamp":1614556800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2021,3,3]]},"DOI":"10.1145\/3442188.3445937","type":"proceedings-article","created":{"date-parts":[[2021,2,25]],"date-time":"2021-02-25T01:45:48Z","timestamp":1614217548000},"page":"758-771","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":68,"title":["Outlining Traceability"],"prefix":"10.1145","author":[{"given":"Joshua A.","family":"Kroll","sequence":"first","affiliation":[{"name":"Naval Postgraduate School, Monterey, CA"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2021,3]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"publisher","DOI":"10.1145\/3173574.3174156"},{"key":"e_1_3_2_1_2_1","volume-title":"USENIX Security Symposium","volume":"17","author":"Adida Ben","year":"2008","unstructured":"Ben Adida. 2008. Helios: Web-based Open-Audit Voting.. In USENIX Security Symposium, Vol. 17. 335--348."},{"key":"e_1_3_2_1_3_1","volume-title":"The Auditing Imperative for Automated Hiring. Harvard Journal of Law & Technology 34","author":"Ajunwa Ifeoma","year":"2021","unstructured":"Ifeoma Ajunwa. 2021. The Auditing Imperative for Automated Hiring. Harvard Journal of Law & Technology 34 (2021)."},{"key":"e_1_3_2_1_4_1","volume-title":"Fairness as a program property. arXiv preprint arXiv:1610.06067","author":"Albarghouthi Aws","year":"2016","unstructured":"Aws Albarghouthi, Loris D'Antoni, Samuel Drews, and Aditya Nori. 2016. Fairness as a program property. arXiv preprint arXiv:1610.06067 (2016)."},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1145\/3287560.3287588"},{"key":"e_1_3_2_1_6_1","volume-title":"We Need Bug Bounties for Bad Algorithms. https:\/\/www.vice.com\/en\/article\/8xkyj3\/we-need-bug-bounties-for-bad-algorithms. Vice News (3","author":"Bar On Amit Elazari","year":"2018","unstructured":"Amit Elazari Bar On. 2018. We Need Bug Bounties for Bad Algorithms. https:\/\/www.vice.com\/en\/article\/8xkyj3\/we-need-bug-bounties-for-bad-algorithms. Vice News (3 May 2018)."},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-19718-5_1"},{"key":"e_1_3_2_1_8_1","volume-title":"Proc. 14th ICNP.","author":"Argyraki K","year":"2007","unstructured":"K Argyraki, P Maniatis, O Irzak, and S Shenker. 2007. An accountability interface for the Internet. In Proc. 14th ICNP."},{"key":"e_1_3_2_1_9_1","volume-title":"Double loop learning in organizations. Harvard business review 55, 5","author":"Argyris Chris","year":"1977","unstructured":"Chris Argyris. 1977. Double loop learning in organizations. Harvard business review 55, 5 (1977), 115--125."},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.1147\/JRD.2019.2942288"},{"key":"e_1_3_2_1_11_1","volume-title":"Proc. NDSS","author":"Backes Michael","year":"2009","unstructured":"Michael Backes, Peter Druschel, Andreas Haeberlen, and Dominique Unruh. 2009. CSAR: A Practical and Provable Technique to Make Randomized Systems Accountable. Proc. NDSS (2009)."},{"key":"e_1_3_2_1_12_1","volume-title":"Analysis, design and evaluation of man-machine systems","author":"Bainbridge Lisanne","unstructured":"Lisanne Bainbridge. 1983. Ironies of automation. In Analysis, design and evaluation of man-machine systems. Elsevier, 129--135."},{"key":"e_1_3_2_1_13_1","volume-title":"USENIX Security Symposium. 63--78","author":"Balasch Josep","year":"2010","unstructured":"Josep Balasch, Alfredo Rial, Carmela Troncoso, Bart Preneel, Ingrid Verbauwhede, and Christophe Geuens. 2010. PrETP: Privacy-Preserving Electronic Toll Pricing.. In USENIX Security Symposium. 63--78."},{"key":"e_1_3_2_1_14_1","first-page":"247","article-title":"Privacy on the Books and on the Ground","volume":"63","author":"Bamberger Kenneth A","year":"2010","unstructured":"Kenneth A Bamberger and Deirdre K Mulligan. 2010. Privacy on the Books and on the Ground. Stan. L. Rev. 63 (2010), 247.","journal-title":"Stan. L. Rev."},{"key":"e_1_3_2_1_15_1","volume-title":"Privacy on the ground: driving corporate behavior in the United States and Europe","author":"Bamberger Kenneth A","unstructured":"Kenneth A Bamberger and Deirdre K Mulligan. 2015. Privacy on the ground: driving corporate behavior in the United States and Europe. MIT Press."},{"key":"e_1_3_2_1_16_1","volume-title":"Reporting and preventing medical mishaps: lessons from non-medical near miss reporting systems. Bmj 320, 7237","author":"Barach Paul","year":"2000","unstructured":"Paul Barach and Stephen D Small. 2000. Reporting and preventing medical mishaps: lessons from non-medical near miss reporting systems. Bmj 320, 7237 (2000), 759--763."},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.1109\/CSF.2007.26"},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1145\/2987443.2987469"},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.1975.6312870"},{"key":"e_1_3_2_1_20_1","volume-title":"DevOps: A software architect's perspective","author":"Bass Len","unstructured":"Len Bass, Ingo Weber, and Liming Zhu. 2015. DevOps: A software architect's perspective. Addison-Wesley Professional."},{"key":"e_1_3_2_1_21_1","volume-title":"Test-driven development: by example","author":"Beck Kent","unstructured":"Kent Beck. 2003. Test-driven development: by example. Addison-Wesley Prof."},{"key":"e_1_3_2_1_22_1","volume-title":"Alistair Cockburn, Ward Cunningham, et al. [n.d.]. Manifesto for agile software development","author":"Beck Kent","year":"2001","unstructured":"Kent Beck, Mike Beedle, Arie Van Bennekum, Alistair Cockburn, Ward Cunningham, et al. [n.d.]. Manifesto for agile software development, 2001. ([n. d.])."},{"key":"e_1_3_2_1_23_1","first-page":"96","article-title":"On the brittleness of software and the infeasibility of security metrics","volume":"4","author":"Bellovin Steven M","year":"2006","unstructured":"Steven M Bellovin. 2006. On the brittleness of software and the infeasibility of security metrics. IEEE Annals of the History of Computing 4, 04 (2006), 96--96.","journal-title":"IEEE Annals of the History of Computing"},{"key":"e_1_3_2_1_25_1","volume-title":"SNARKs for C: Verifying program executions succinctly and in zero knowledge. CRYPTO","author":"Ben-Sasson Eli","year":"2013","unstructured":"Eli Ben-Sasson, Alessandro Chiesa, Daniel Genkin, Eran Tromer, and Madars Virza. 2013. SNARKs for C: Verifying program executions succinctly and in zero knowledge. CRYPTO (2013)."},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.1109\/SP.2015.25"},{"key":"e_1_3_2_1_27_1","unstructured":"Eli Ben-Sasson Alessandro Chiesa Eran Tromer and Madars Virza. 2014. Succinct non-interactive zero knowledge for a von Neumann architecture. In USENIX Security. 781--796."},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"publisher","DOI":"10.1162\/tacl_a_00041"},{"key":"e_1_3_2_1_29_1","volume-title":"H Chi","author":"Beutel Alex","year":"2019","unstructured":"Alex Beutel, Jilin Chen, Tulsee Doshi, Hai Qian, Allison Woodruff, Christine Luu, Pierre Kreitmann, Jonathan Bischof, and Ed H Chi. 2019. Putting Fairness Principles into Practice: Challenges, Metrics, and Improvements. arXiv preprint arXiv:1901.04562 (2019)."},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"publisher","DOI":"10.1145\/3173574.3173951"},{"key":"e_1_3_2_1_31_1","volume-title":"Proceedings of the 18th National Information Systems Security Conference. 136--145","author":"Bishop Matt","year":"1995","unstructured":"Matt Bishop. 1995. A standard audit trail format. In Proceedings of the 18th National Information Systems Security Conference. 136--145."},{"key":"e_1_3_2_1_32_1","volume-title":"Proceedings of the Twenty-Third international joint conference on Artificial Intelligence. AAAI Press, 41--47","author":"Blocki Jeremiah","year":"2013","unstructured":"Jeremiah Blocki, Nicolas Christin, Anupam Datta, Ariel D Procaccia, and Arunesh Sinha. 2013. Audit games. In Proceedings of the Twenty-Third international joint conference on Artificial Intelligence. AAAI Press, 41--47."},{"key":"e_1_3_2_1_34_1","unstructured":"Gabriela Bodea Kristina Karanikolova Deirdre K. Mulligan and Jael Makagon. 2018. Automated decision-making on the basis of personal data that has been transferred from the EU to companies certified under the EU-U.S. Privacy Shield. Fact-finding and assessment of safeguards provided by U.S. law. Independent Report promulgated by the European Commission Directorate-General for Justice and Consumers. https:\/\/ec.europa.eu\/info\/sites\/info\/files\/independent_study_on_automated_decision-making.pdf"},{"key":"e_1_3_2_1_35_1","doi-asserted-by":"publisher","DOI":"10.1109\/2.976920"},{"key":"e_1_3_2_1_36_1","doi-asserted-by":"publisher","DOI":"10.1109\/MS.1984.233702"},{"key":"e_1_3_2_1_37_1","doi-asserted-by":"publisher","DOI":"10.1145\/2723872.2723882"},{"key":"e_1_3_2_1_38_1","unstructured":"Francisco Bonchi Carlos Castillo and Sara Hajian. 2016. Algorithmic bias: from discrimination discovery to fairness-aware data mining. Knowledge Discovery and Data Mining Tutorials Track."},{"key":"e_1_3_2_1_39_1","doi-asserted-by":"publisher","DOI":"10.1145\/2517349.2522733"},{"key":"e_1_3_2_1_40_1","doi-asserted-by":"publisher","DOI":"10.1109\/POLICY.2006.18"},{"key":"e_1_3_2_1_41_1","volume-title":"A distributed requirements management framework for legal compliance and accountability. computers & security 28, 1-2","author":"Breaux Travis D","year":"2009","unstructured":"Travis D Breaux, Annie I Ant\u00f3n, and Eugene H Spafford. 2009. A distributed requirements management framework for legal compliance and accountability. computers & security 28, 1-2 (2009), 8--17."},{"key":"e_1_3_2_1_42_1","doi-asserted-by":"publisher","DOI":"10.1109\/RE.2006.68"},{"key":"e_1_3_2_1_43_1","volume-title":"The mythical man-month: essays on software engineering","author":"Brooks Frederick P","unstructured":"Frederick P Brooks Jr. 1995. The mythical man-month: essays on software engineering. Addison-Wesley."},{"key":"e_1_3_2_1_44_1","volume-title":"Kroll","author":"Bruening Paula","year":"2019","unstructured":"Paula Bruening and Joshua A. Kroll. 2019. Considering Transparency: AI, Data Science, and the GDPR. Computers, Privacy, and Data Protection (2019)."},{"key":"e_1_3_2_1_45_1","unstructured":"Miles Brundage Shahar Avin Jasmine Wang Haydn Belfield Gretchen Krueger Gillian Hadfield Heidy Khlaaf Jingying Yang Helen Toner Ruth Fong et al. 2020. Toward trustworthy AI development: mechanisms for supporting verifiable claims. OpenAI Technical Report arXiv preprint arXiv:2004.07213 (2020)."},{"key":"e_1_3_2_1_46_1","doi-asserted-by":"publisher","DOI":"10.1007\/3-540-44503-X_20"},{"key":"e_1_3_2_1_47_1","volume-title":"Conference on Fairness, Accountability and Transparency. 77--91","author":"Buolamwini Joy","year":"2018","unstructured":"Joy Buolamwini and Timnit Gebru. 2018. Gender shades: Intersectional accuracy disparities in commercial gender classification. In Conference on Fairness, Accountability and Transparency. 77--91."},{"key":"e_1_3_2_1_48_1","volume-title":"How the machine 'thinks': Understanding opacity in machine learning algorithms","author":"Burrell Jenna","year":"2016","unstructured":"Jenna Burrell. 2016. How the machine 'thinks': Understanding opacity in machine learning algorithms. Big Data & Society 3, 1 (2016)."},{"key":"e_1_3_2_1_49_1","volume-title":"Security and Cryptography for Networks","author":"Camenisch Jan","unstructured":"Jan Camenisch, Susan Hohenberger, and Anna Lysyanskaya. 2006. Balancing accountability and privacy using e-cash. In Security and Cryptography for Networks. Springer, 141--155."},{"key":"e_1_3_2_1_50_1","volume-title":"Accounting for rationality: Double-entry bookkeeping and the rhetoric of economic rationality. American journal of sociology 97, 1","author":"Carruthers Bruce G","year":"1991","unstructured":"Bruce G Carruthers and Wendy Nelson Espeland. 1991. Accounting for rationality: Double-entry bookkeeping and the rhetoric of economic rationality. American journal of sociology 97, 1 (1991), 31--69."},{"key":"e_1_3_2_1_51_1","doi-asserted-by":"publisher","DOI":"10.1002\/asi.23563"},{"key":"e_1_3_2_1_52_1","volume-title":"Privacy by design in law, policy and practice. A white paper for regulators, decision-makers and policy-makers","author":"Cavoukian Ann","year":"2011","unstructured":"Ann Cavoukian. 2011. Privacy by design in law, policy and practice. A white paper for regulators, decision-makers and policy-makers (2011)."},{"key":"e_1_3_2_1_53_1","unstructured":"Ann Cavoukian et al. 2009. Privacy by design: The 7 foundational principles. Information and Privacy Commissioner of Ontario Canada 5 (2009)."},{"key":"e_1_3_2_1_54_1","doi-asserted-by":"publisher","DOI":"10.1145\/3173574.3174225"},{"key":"e_1_3_2_1_55_1","doi-asserted-by":"publisher","DOI":"10.1145\/3055589.3055594"},{"key":"e_1_3_2_1_56_1","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.1982.236161"},{"key":"e_1_3_2_1_57_1","volume-title":"Fair prediction with disparate impact: A study of bias in recidivism prediction instruments. Big data 5, 2","author":"Chouldechova Alexandra","year":"2017","unstructured":"Alexandra Chouldechova. 2017. Fair prediction with disparate impact: A study of bias in recidivism prediction instruments. Big data 5, 2 (2017), 153--163."},{"key":"e_1_3_2_1_58_1","volume-title":"Conference on Fairness, Accountability and Transparency. 134--148","author":"Chouldechova Alexandra","year":"2018","unstructured":"Alexandra Chouldechova, Diana Benavides-Prado, Oleksandr Fialko, and Rhema Vaithianathan. 2018. A case study of algorithm-assisted decision making in child maltreatment hotline screening decisions. In Conference on Fairness, Accountability and Transparency. 134--148."},{"key":"e_1_3_2_1_59_1","doi-asserted-by":"publisher","DOI":"10.4324\/9781315144788"},{"key":"e_1_3_2_1_60_1","first-page":"1249","article-title":"Technological due process","volume":"85","author":"Citron Danielle","year":"2007","unstructured":"Danielle Citron. 2007. Technological due process. Washington University Law Review 85 (2007), 1249--1313.","journal-title":"Washington University Law Review"},{"key":"e_1_3_2_1_61_1","volume-title":"Open code governance","author":"Citron Danielle Keats","unstructured":"Danielle Keats Citron. 2008. Open code governance. In University of Chicago Legal Forum. 355--387."},{"key":"e_1_3_2_1_62_1","volume-title":"The Automated Administrative State: A Crisis of Legitimacy. Emory Law Journal","author":"Citron Danielle K","year":"2020","unstructured":"Danielle K Citron and Ryan Calo. 2020. The Automated Administrative State: A Crisis of Legitimacy. Emory Law Journal (2020)."},{"key":"e_1_3_2_1_63_1","volume-title":"Enigma","author":"Clark Robert","year":"2018","unstructured":"Robert Clark. 2018. Compliance != Security (Except When It Might Be). In Enigma 2018."},{"key":"e_1_3_2_1_64_1","volume-title":"SAC, COSO and SAS. IS Audit and Control Journal","author":"Colbert Janet L","year":"1996","unstructured":"Janet L Colbert and Paul Bowen. 1996. A Comparison of Internal Control: COBIT, SAC, COSO and SAS. IS Audit and Control Journal (1996), 26--35."},{"key":"e_1_3_2_1_65_1","unstructured":"Computing Community Consortium. 2015. Privacy by Design---Engineering Privacy: Workshop 3 Report. https:\/\/cra.org\/ccc\/wp-content\/uploads\/sites\/2\/2015\/12\/PbD3-Workshop-Report-v2.pdf."},{"key":"e_1_3_2_1_66_1","unstructured":"Computing Community Consortium. 2015. Privacy by Design---Privacy Enabling Design: Workshop 2 Report. https:\/\/cra.org\/ccc\/wp-content\/uploads\/sites\/2\/2015\/05\/PbD2-Report-v5.pdf."},{"key":"e_1_3_2_1_67_1","unstructured":"Computing Community Consortium. 2015. Privacy by Design-State of Research and Practice: Workshop 1 Report. https:\/\/cra.org\/ccc\/wp-content\/uploads\/sites\/2\/2015\/02\/PbD-Workshop-1-Report-.pdf."},{"key":"e_1_3_2_1_68_1","unstructured":"Consultative Committee for Space Data Systems. 2011. Audit and certification of Trustworthy Digital Repositories Recommended Practice."},{"key":"e_1_3_2_1_69_1","unstructured":"Consumer Financial Protection Bureau. 2014. Using Publicly Available Information to Proxy for Unidentified Race and Ethnicity: A Methodology and Assessment. https:\/\/www.consumerfinance.gov\/data-research\/research-reports\/using-publicly-available-information-to-proxy-for-unidentified-race-and-ethnicity\/."},{"key":"e_1_3_2_1_70_1","volume-title":"How complex systems fail. Cognitive Technologies Laboratory","author":"Cook Richard I","year":"1998","unstructured":"Richard I Cook. 1998. How complex systems fail. Cognitive Technologies Laboratory, University of Chicago. Chicago IL (1998)."},{"key":"e_1_3_2_1_71_1","volume-title":"USENIX Security Symposium. 317--334","author":"Crosby Scott A","year":"2009","unstructured":"Scott A Crosby and Dan S Wallach. 2009. Efficient Data Structures For Tamper-Evident Logging.. In USENIX Security Symposium. 317--334."},{"key":"e_1_3_2_1_72_1","volume-title":"Distributed Computing and Internet Technology","author":"Datta Anupam","unstructured":"Anupam Datta. 2014. Privacy through Accountability: A Computer Science Perspective. In Distributed Computing and Internet Technology. Springer, 43--49."},{"key":"e_1_3_2_1_73_1","volume-title":"SCRAM: A Platform for Securely Measuring Cyber Risk. Harvard Data Science Review","author":"de Castro Leo","year":"2020","unstructured":"Leo de Castro, Andrew W Lo, Taylor Reynolds, Fransisca Susan, Vinod Vaikuntanathan, Daniel Weitzner, and Nicolas Zhang. 2020. SCRAM: A Platform for Securely Measuring Cyber Risk. Harvard Data Science Review (2020)."},{"key":"e_1_3_2_1_74_1","unstructured":"Defense Innovation Board. 2019. AI Principles: Recommendations on the Ethical Use of Artificial Intelligence by the Department of Defense: Supporting Document. https:\/\/innovation.defense.gov\/ai\/."},{"key":"e_1_3_2_1_75_1","article-title":"Trust but Verify: A Guide to Algorithms and the Law","volume":"31","author":"Desai Deven","year":"2018","unstructured":"Deven Desai and Joshua A. Kroll. 2018. Trust but Verify: A Guide to Algorithms and the Law. Harvard J. of Law and Tech. 31, 1 (2018).","journal-title":"Harvard J. of Law and Tech."},{"key":"e_1_3_2_1_76_1","unstructured":"Division of Banking Supervision and Regulation. 2011. SR 11-7: Guidance on Model Risk Management."},{"key":"e_1_3_2_1_77_1","volume-title":"Towards a rigorous science of interpretable machine learning. arXiv preprint arXiv:1702.08608","author":"Doshi-Velez Finale","year":"2017","unstructured":"Finale Doshi-Velez and Been Kim. 2017. Towards a rigorous science of interpretable machine learning. arXiv preprint arXiv:1702.08608 (2017)."},{"key":"e_1_3_2_1_78_1","volume-title":"Accountability of AI Under the Law: The Role of Explanation. arXiv preprint arXiv:1711.01134","author":"Doshi-Velez Finale","year":"2017","unstructured":"Finale Doshi-Velez, Mason Kortz, Ryan Budish, Chris Bavitz, Sam Gershman, David O'Brien, Stuart Schieber, James Waldo, David Weinberger, and Alexandra Wood. 2017. Accountability of AI Under the Law: The Role of Explanation. arXiv preprint arXiv:1711.01134 (2017)."},{"key":"e_1_3_2_1_79_1","volume-title":"Theory of cryptography conference","author":"Dwork Cynthia","unstructured":"Cynthia Dwork, Frank McSherry, Kobbi Nissim, and Adam Smith. 2006. Calibrating noise to sensitivity in private data analysis. In Theory of cryptography conference. Springer, 265--284."},{"key":"e_1_3_2_1_80_1","first-page":"35","article-title":"It's not privacy, and it's not fair","volume":"66","author":"Dwork Cynthia","year":"2013","unstructured":"Cynthia Dwork and Deirdre K Mulligan. 2013. It's not privacy, and it's not fair. Stanford Law Review Online 66 (2013), 35.","journal-title":"Stanford Law Review Online"},{"key":"e_1_3_2_1_81_1","volume-title":"More Accountability for Big Data Algorithms. Nature 537","year":"2016","unstructured":"Editorial. 2016. More Accountability for Big Data Algorithms. Nature 537 (2016). Issue 7621."},{"key":"e_1_3_2_1_82_1","volume-title":"Enslaving the Algorithm: From a \"Right to an Explanation\" to a \"Right to Better Decisions\"? IEEE Security and Privacy","author":"Edwards Lilian","year":"2017","unstructured":"Lilian Edwards and Michael Veale. 2017. Enslaving the Algorithm: From a \"Right to an Explanation\" to a \"Right to Better Decisions\"? IEEE Security and Privacy (2017)."},{"key":"e_1_3_2_1_83_1","first-page":"18","article-title":"Slave to the Algorithm? Why a \"Right to Explanation\" is Probably Not the Remedy You are Looking for","volume":"16","author":"Edwards Lilian","year":"2017","unstructured":"Lilian Edwards and Michael Veale. 2017. Slave to the Algorithm? Why a \"Right to Explanation\" is Probably Not the Remedy You are Looking for. Duke Technology Law Journal 16, 1 (2017), 18--84.","journal-title":"Duke Technology Law Journal"},{"key":"e_1_3_2_1_84_1","doi-asserted-by":"publisher","DOI":"10.2139\/ssrn.2720477"},{"key":"e_1_3_2_1_85_1","doi-asserted-by":"publisher","DOI":"10.17351\/ests2019.260"},{"key":"e_1_3_2_1_86_1","volume-title":"Ceremony Design and Analysis. IACR eprint archive 399","author":"Ellison Carl","year":"2007","unstructured":"Carl Ellison. 2007. Ceremony Design and Analysis. IACR eprint archive 399 (2007). http:\/\/eprint.iacr.org\/2007\/399.pdf."},{"key":"e_1_3_2_1_88_1","doi-asserted-by":"publisher","DOI":"10.1146\/annurev.lawsocsci.2.081805.105908"},{"key":"e_1_3_2_1_89_1","unstructured":"European Commission Independent High-Level Expert Group on Artificial Intelligence. 2020. Ethics Guidelines for Trustworthy AI. https:\/\/ec.europa.eu\/digital-single-market\/en\/news\/ethics-guidelines-trustworthy-ai."},{"key":"e_1_3_2_1_90_1","volume-title":"Software development failures","author":"Ewusi-Mensah Kweku","unstructured":"Kweku Ewusi-Mensah. 2003. Software development failures. Mit Press."},{"key":"e_1_3_2_1_91_1","unstructured":"Executive Office of the President of the United States. 2019. Executive Order 13859: Maintaining American Leadership in Artificial Intelligence. https:\/\/www.govinfo.gov\/app\/details\/DCPD-201900073."},{"key":"e_1_3_2_1_92_1","unstructured":"Executive Office of the President of the United States. 2020. Executive Order 13960: Promoting the Use of Trustworthy Artificial Intelligence in the Federal Government. https:\/\/www.govinfo.gov\/app\/details\/DCPD-202000870."},{"key":"e_1_3_2_1_93_1","volume-title":"Technical Report YALEU\/DCS\/TR-1452","author":"Feigenbaum Joan","unstructured":"Joan Feigenbaum, Aaron D Jaggard, Rebecca N Wright, and Hongda Xiao. 2012. Systematizing \"Accountability\" in Computer Science (Version of Feb. 17, 2012). Technical Report YALEU\/DCS\/TR-1452. Yale University, New Haven, CT."},{"key":"e_1_3_2_1_94_1","volume-title":"Principled artificial intelligence: Mapping consensus in ethical and rights-based approaches to principles for AI","author":"Fjeld Jessica","year":"2020","unstructured":"Jessica Fjeld, Nele Achten, Hannah Hilligoss, Adam Nagy, and Madhulika Srikumar. 2020. Principled artificial intelligence: Mapping consensus in ethical and rights-based approaches to principles for AI. Berkman Klein Center Research Publication 2020-1 (2020)."},{"key":"e_1_3_2_1_95_1","doi-asserted-by":"publisher","DOI":"10.1145\/1054972.1055076"},{"key":"e_1_3_2_1_96_1","doi-asserted-by":"publisher","DOI":"10.1038\/s42256-019-0055-y"},{"key":"e_1_3_2_1_97_1","volume-title":"Value-sensitive design. interactions 3, 6","author":"Friedman Batya","year":"1996","unstructured":"Batya Friedman. 1996. Value-sensitive design. interactions 3, 6 (1996), 16--23."},{"key":"e_1_3_2_1_98_1","volume-title":"The Handbook of Information and Computer Ethics, Kenneth Einar Himma and Herman T","author":"Friedman Batya","unstructured":"Batya Friedman, Peter H. Kahn, and Alan Borning. 2008. Value Sensitive Design and Information Systems. In The Handbook of Information and Computer Ethics, Kenneth Einar Himma and Herman T. Tavani (Eds.). John Wiley & Sons, Inc., Chapter 4, 69--101."},{"key":"e_1_3_2_1_99_1","doi-asserted-by":"publisher","DOI":"10.1145\/632716.632766"},{"key":"e_1_3_2_1_100_1","volume-title":"The Checklist Manifesto: How to Get Things Right","author":"Gawande Atul","unstructured":"Atul Gawande. 2009. The Checklist Manifesto: How to Get Things Right. Metropolitan Books."},{"key":"e_1_3_2_1_101_1","volume-title":"Hanna Wallach, Hal Daum\u00e9 III, and Kate Crawford.","author":"Gebru Timnit","year":"2018","unstructured":"Timnit Gebru, Jamie Morgenstern, Briana Vecchione, Jennifer Wortman Vaughan, Hanna Wallach, Hal Daum\u00e9 III, and Kate Crawford. 2018. Datasheets for datasets. arXiv preprint arXiv:1803.09010 (2018)."},{"key":"e_1_3_2_1_102_1","volume-title":"Proceedings of the ACM on Human-Computer Interaction 1, CSCW (dec","author":"Stuart Geiger R.","year":"2017","unstructured":"R. Stuart Geiger and Aaron Halfaker. 2017. Operationalizing Conflict and Cooperation between Automated Software Agents in Wikipedia: A Replication and Expansion of \"Even Good Bots Fight\". Proceedings of the ACM on Human-Computer Interaction 1, CSCW (dec 2017), 1--33."},{"key":"e_1_3_2_1_103_1","volume-title":"Fair information practices: A basic history. SSRN, 2415020","author":"Gellman Robert","year":"2017","unstructured":"Robert Gellman. 2017. Fair information practices: A basic history. SSRN, 2415020 (2017)."},{"key":"e_1_3_2_1_104_1","doi-asserted-by":"publisher","DOI":"10.1145\/62959.62965"},{"key":"e_1_3_2_1_105_1","doi-asserted-by":"publisher","DOI":"10.1177\/0306312706056047"},{"key":"e_1_3_2_1_106_1","volume-title":"The politics of 'platforms'. New media & society 12, 3","author":"Gillespie Tarleton","year":"2010","unstructured":"Tarleton Gillespie. 2010. The politics of 'platforms'. New media & society 12, 3 (2010), 347--364."},{"key":"e_1_3_2_1_107_1","doi-asserted-by":"publisher","DOI":"10.1109\/RE.2013.6636714"},{"key":"e_1_3_2_1_108_1","volume-title":"Government Auditing Standards","author":"Office Government Accountability","year":"2018","unstructured":"Government Accountability Office. 2018. Government Auditing Standards, 2018 Revision."},{"key":"e_1_3_2_1_109_1","volume-title":"A Survey Of Methods For Explaining Black Box Models. arXiv preprint arXiv:1802.01933","author":"Guidotti Riccardo","year":"2018","unstructured":"Riccardo Guidotti, Anna Monreale, Franco Turini, Dino Pedreschi, and Fosca Giannotti. 2018. A Survey Of Methods For Explaining Black Box Models. arXiv preprint arXiv:1802.01933 (2018)."},{"key":"e_1_3_2_1_110_1","volume-title":"Conference on Computers, Privacy, and Data Protection","author":"G\u00fcrses Seda","year":"2011","unstructured":"Seda G\u00fcrses, Carmela Troncoso, and Claudia Diaz. 2011. Engineering privacy by design. Conference on Computers, Privacy, and Data Protection (2011)."},{"key":"e_1_3_2_1_111_1","volume-title":"Abdelaziz Mounji, and Isabelle Mathieu.","author":"Habra Naji","year":"1992","unstructured":"Naji Habra, Baudouin Le Charlier, Abdelaziz Mounji, and Isabelle Mathieu. 1992. ASAX: Software architecture and rule-based language for universal audit trail analysis. In Computer Security---ESORICS 92. Springer, 435--450."},{"key":"e_1_3_2_1_112_1","doi-asserted-by":"publisher","DOI":"10.1145\/1773912.1773926"},{"key":"e_1_3_2_1_113_1","unstructured":"Andreas Haeberlen Paarijaat Aditya Rodrigo Rodrigues and Peter Druschel. 2010. Accountable Virtual Machines.. In OSDI. 119--134."},{"key":"e_1_3_2_1_114_1","doi-asserted-by":"publisher","DOI":"10.1145\/1323293.1294279"},{"key":"e_1_3_2_1_115_1","unstructured":"Joseph Lorenzo Hall. 2010. Election Auditing Bibliography. https:\/\/josephhall.org\/papers\/auditing_biblio.pdf."},{"key":"e_1_3_2_1_116_1","volume-title":"Causes and explanations: A structural-model approach. Part I: Causes. The British journal for the philosophy of science 56, 4","author":"Halpern Joseph Y","year":"2005","unstructured":"Joseph Y Halpern and Judea Pearl. 2005. Causes and explanations: A structural-model approach. Part I: Causes. The British journal for the philosophy of science 56, 4 (2005), 843--887."},{"key":"e_1_3_2_1_117_1","doi-asserted-by":"publisher","DOI":"10.1093\/bjps\/axi148"},{"key":"e_1_3_2_1_118_1","doi-asserted-by":"publisher","DOI":"10.1145\/2488388.2488435"},{"key":"e_1_3_2_1_119_1","volume-title":"Proceedings of the 2014 conference on internet measurement conference. ACM, 305--318","author":"Hannak Aniko","unstructured":"Aniko Hannak, Gary Soeller, David Lazer, Alan Mislove, and Christo Wilson. [n.d.]. Measuring price discrimination and steering on e-commerce web sites. In Proceedings of the 2014 conference on internet measurement conference. ACM, 305--318."},{"key":"e_1_3_2_1_120_1","doi-asserted-by":"publisher","DOI":"10.1023\/A:1000676723815"},{"key":"e_1_3_2_1_121_1","doi-asserted-by":"publisher","DOI":"10.1109\/32.241771"},{"key":"e_1_3_2_1_122_1","doi-asserted-by":"publisher","DOI":"10.1007\/s00778-017-0486-1"},{"key":"e_1_3_2_1_123_1","volume-title":"The dataset nutrition label: A framework to drive higher data quality standards. arXiv preprint arXiv:1805.03677","author":"Holland Sarah","year":"2018","unstructured":"Sarah Holland, Ahmed Hosny, Sarah Newman, Joshua Joseph, and Kasia Chmielinski. 2018. The dataset nutrition label: A framework to drive higher data quality standards. arXiv preprint arXiv:1805.03677 (2018)."},{"key":"e_1_3_2_1_124_1","volume-title":"Aviation safety through the rule of law: ICAO's mechanisms and practices","author":"Huang Jiefang","unstructured":"Jiefang Huang. 2009. Aviation safety through the rule of law: ICAO's mechanisms and practices. Kluwer Law International BV."},{"key":"e_1_3_2_1_125_1","doi-asserted-by":"publisher","DOI":"10.1145\/2627392"},{"key":"e_1_3_2_1_126_1","unstructured":"ISO. 2012. Certification of Trustworthy Digital Repositories."},{"key":"e_1_3_2_1_127_1","unstructured":"J.G. Jabbra and O.P. Dwivedi (Eds.). 1989. Public Service Accountability: A Comparative Perspective. Kumarian Press."},{"key":"e_1_3_2_1_128_1","doi-asserted-by":"publisher","DOI":"10.1145\/2531602.2531674"},{"key":"e_1_3_2_1_129_1","volume-title":"Measurement and Fairness. In ACM Conference on Fairness, Accountability, and Transparency.","author":"Abigail","unstructured":"Abigail Z.Jacobs and Hanna Wallach. 2021. Measurement and Fairness. In ACM Conference on Fairness, Accountability, and Transparency."},{"key":"e_1_3_2_1_130_1","doi-asserted-by":"publisher","DOI":"10.1162\/DESI_a_00354"},{"key":"e_1_3_2_1_131_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-04444-1_10"},{"key":"e_1_3_2_1_133_1","volume-title":"Don't ask if artificial intelligence is good or fair, ask how it shifts power. Nature 583, 7815","author":"Kalluri Pratyusha","year":"2020","unstructured":"Pratyusha Kalluri. 2020. Don't ask if artificial intelligence is good or fair, ask how it shifts power. Nature 583, 7815 (2020), 169--169."},{"key":"e_1_3_2_1_134_1","volume-title":"An Algorithmic Equity Toolkit for Technology Audits by Community Advocates and Activists. arXiv preprint arXiv:1912.02943","author":"Katell Michael","year":"2019","unstructured":"Michael Katell, Meg Young, Bernease Herman, Dharma Dailey, Aaron Tam, Vivian Guetler, Corinne Binz, Daniella Raz, and PM Krafft. 2019. An Algorithmic Equity Toolkit for Technology Audits by Community Advocates and Activists. arXiv preprint arXiv:1912.02943 (2019)."},{"key":"e_1_3_2_1_135_1","volume-title":"Preventing Fairness Gerrymandering: Auditing and Learning for Subgroup Fairness. arXiv preprint arXiv:1711.05144","author":"Kearns Michael","year":"2017","unstructured":"Michael Kearns, Seth Neel, Aaron Roth, and Zhiwei Steven Wu. 2017. Preventing Fairness Gerrymandering: Auditing and Learning for Subgroup Fairness. arXiv preprint arXiv:1711.05144 (2017)."},{"key":"e_1_3_2_1_136_1","doi-asserted-by":"publisher","DOI":"10.1145\/1065167.1065183"},{"key":"e_1_3_2_1_137_1","article-title":"Auditing Algorithms for Discrimination","volume":"166","author":"Kim Pauline","year":"2017","unstructured":"Pauline Kim. 2017. Auditing Algorithms for Discrimination. University of Pennsylvania Law Review Online 166, 189 (2017).","journal-title":"University of Pennsylvania Law Review Online"},{"key":"e_1_3_2_1_138_1","volume-title":"Thousands of Criminal Cases in New York Relied on Disputed DNA Testing Techniques. ProPublica (4","author":"Kirchner Lauren","year":"2017","unstructured":"Lauren Kirchner. 2017. Thousands of Criminal Cases in New York Relied on Disputed DNA Testing Techniques. ProPublica (4 Sept. 2017)."},{"key":"e_1_3_2_1_139_1","doi-asserted-by":"publisher","DOI":"10.1145\/1965724.1965735"},{"key":"e_1_3_2_1_141_1","doi-asserted-by":"publisher","DOI":"10.1098\/rsta.2018.0084"},{"key":"e_1_3_2_1_142_1","volume-title":"The Oxford Handbook of the Ethics of Artificial Intelligence","author":"Kroll Joshua A.","unstructured":"Joshua A. Kroll. 2020. Accountability in Computer Systems. In The Oxford Handbook of the Ethics of Artificial Intelligence, Markus Dubber, Frank Pasquale, and Sunit Das (Eds.). Oxford University Press, Oxford, UK, 181--196."},{"key":"e_1_3_2_1_143_1","unstructured":"Joshua A. Kroll Joanna Huey Solon Barocas Edward W. Felten Joel R. Reidenberg David G. Robinson and Harlan Yu. 2017. Accountable Algorithms. University of Pennsylvania Law Review (to appear) 165 (2017) 633--705. Issue 3."},{"key":"e_1_3_2_1_144_1","volume-title":"Automated Verification of Accountability in Security Protocols. In 2019 IEEE 32nd Computer Security Foundations Symposium (CSF). IEEE, 397--39716","author":"K\u00fcnnemann Robert","year":"2019","unstructured":"Robert K\u00fcnnemann, Ilkan Esiyok, and Michael Backes. 2019. Automated Verification of Accountability in Security Protocols. In 2019 IEEE 32nd Computer Security Foundations Symposium (CSF). IEEE, 397--39716."},{"key":"e_1_3_2_1_145_1","doi-asserted-by":"publisher","DOI":"10.1145\/1866307.1866366"},{"key":"e_1_3_2_1_146_1","first-page":"275","article-title":"The Algorithm says you did it: The use of Black Box Algorithms to analyze complex DNA evidence","volume":"31","author":"Kwong Katherine","year":"2017","unstructured":"Katherine Kwong. 2017. The Algorithm says you did it: The use of Black Box Algorithms to analyze complex DNA evidence. Harv. JL & Tech. 31 (2017), 275.","journal-title":"Harv. JL & Tech."},{"key":"e_1_3_2_1_147_1","unstructured":"Chris Lamb Holger Levson Mattia Rizzolo and Vagrant Cascadian. 2020. Reproducible Builds. https:\/\/reproducible-builds.org."},{"key":"e_1_3_2_1_148_1","doi-asserted-by":"crossref","unstructured":"Ben Laurie Adam Langley and Emilia Kasper. 2013. Certificate Transparency. Technical Report RFC 6962. Internet Engineering Task Force.","DOI":"10.17487\/rfc6962"},{"key":"e_1_3_2_1_149_1","volume-title":"Proceedings of the 27th international conference on Human factors in computing systems - CHI 09","author":"Le Dantec Christopher A.","unstructured":"Christopher A. Le Dantec, Erika Shehan Poole, and Susan P. Wyche. 2009. Values as lived experience: Evolving value sensitive design in support of value discovery. In Proceedings of the 27th international conference on Human factors in computing systems - CHI 09. ACM Press, New York, New York, USA, 1141."},{"key":"e_1_3_2_1_150_1","doi-asserted-by":"publisher","DOI":"10.1145\/2998181.2998230"},{"key":"e_1_3_2_1_151_1","doi-asserted-by":"publisher","DOI":"10.1145\/3334480.3375158"},{"key":"e_1_3_2_1_152_1","doi-asserted-by":"publisher","DOI":"10.1145\/3359284"},{"key":"e_1_3_2_1_153_1","volume-title":"Proceedings of the ACM on Human-Computer Interaction 3, CSCW","author":"Noothigattu Ritesh","year":"2019","unstructured":"Min Kyung Lee, Daniel Kusbit, Anson Kahng, Ji Tae Kim, Xinran Yuan, Allissa Chan, Daniel See, Ritesh Noothigattu, Siheon Lee, Alexandros Psomas, et al. 2019. WeBuildAI: Participatory framework for algorithmic governance. Proceedings of the ACM on Human-Computer Interaction 3, CSCW (2019), 1--35."},{"key":"e_1_3_2_1_154_1","unstructured":"Leveraging Data as a Strategic Asset Cross-Agency Priority Team. 2020. Federal Data Strategy: 2020 Action Plan. https:\/\/strategy.data.gov\/action-plan\/."},{"key":"e_1_3_2_1_155_1","volume-title":"Proceedings of a workshop on human error and system development. Glasgow Accident Analysis Group, 132--146","author":"Leveson Nancy","year":"1997","unstructured":"Nancy Leveson, L Denise Pinnel, Sean David Sandys, Shuichi Koga, and Jon Damon Reese. 1997. Analyzing software specifications for mode confusion potential. In Proceedings of a workshop on human error and system development. Glasgow Accident Analysis Group, 132--146."},{"key":"e_1_3_2_1_156_1","volume-title":"Engineering a safer world: Systems thinking applied to safety","author":"Leveson Nancy G","unstructured":"Nancy G Leveson. 2016. Engineering a safer world: Systems thinking applied to safety. The MIT Press."},{"key":"e_1_3_2_1_157_1","volume-title":"Does mitigating ML's disparate impact require disparate treatment? arXiv preprint arXiv:1711.07076","author":"Lipton Zachary C","year":"2017","unstructured":"Zachary C Lipton, Alexandra Chouldechova, and Julian McAuley. 2017. Does mitigating ML's disparate impact require disparate treatment? arXiv preprint arXiv:1711.07076 (2017)."},{"key":"e_1_3_2_1_158_1","volume-title":"Version Control with Git: Powerful tools and techniques for collaborative software development","author":"Loeliger Jon","unstructured":"Jon Loeliger and Matthew McCullough. 2012. Version Control with Git: Powerful tools and techniques for collaborative software development. O'Reilly Media."},{"key":"e_1_3_2_1_159_1","doi-asserted-by":"publisher","DOI":"10.1145\/2339530.2339556"},{"key":"e_1_3_2_1_160_1","volume-title":"Building Trust in Information","author":"Lud\u00e4scher Bertram","unstructured":"Bertram Lud\u00e4scher. 2016. A brief tour through provenance in scientific workflows and databases. In Building Trust in Information. Springer, 103--126."},{"key":"e_1_3_2_1_161_1","doi-asserted-by":"publisher","DOI":"10.1111\/j.1740-9713.2016.00960.x"},{"key":"e_1_3_2_1_162_1","volume-title":"In Proceedings of the 11th National Computer Security Conference.","author":"Lunt Teresa F","year":"1988","unstructured":"Teresa F Lunt. 1988. Automated audit trail analysis and intrusion detection: A survey. In In Proceedings of the 11th National Computer Security Conference."},{"key":"e_1_3_2_1_163_1","doi-asserted-by":"publisher","DOI":"10.1145\/2815400.2815415"},{"key":"e_1_3_2_1_164_1","volume-title":"Andrew Smart, and William S Isaac","author":"Jr Donald Martin","year":"2020","unstructured":"Donald Martin Jr, Vinodkumar Prabhakaran, Jill Kuhlberg, Andrew Smart, and William S Isaac. 2020. Extending the machine learning abstraction boundary: A Complex systems approach to incorporate societal context. arXiv preprint arXiv:2006.09663 (2020)."},{"key":"e_1_3_2_1_165_1","volume-title":"Questioning extreme programming","author":"McBreen Pete","unstructured":"Pete McBreen and Kent Foreword By-Beck. 2002. Questioning extreme programming. Addison-Wesley Longman Publishing Co., Inc."},{"key":"e_1_3_2_1_166_1","doi-asserted-by":"publisher","DOI":"10.2218\/ijdc.v10i1.370"},{"key":"e_1_3_2_1_167_1","volume-title":"A Digital Signature Based on a Conventional Encryption Function. CRYPTO","author":"Merkle Ralph C.","year":"1987","unstructured":"Ralph C. Merkle. 1987. A Digital Signature Based on a Conventional Encryption Function. CRYPTO (1987)."},{"key":"e_1_3_2_1_168_1","doi-asserted-by":"publisher","DOI":"10.1353\/sor.2019.0022"},{"key":"e_1_3_2_1_169_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.artint.2018.07.007"},{"key":"e_1_3_2_1_170_1","doi-asserted-by":"publisher","DOI":"10.1093\/oso\/9780195124415.001.0001"},{"key":"e_1_3_2_1_171_1","doi-asserted-by":"publisher","DOI":"10.1145\/3287560.3287596"},{"key":"e_1_3_2_1_172_1","volume-title":"AI Ethics-Too principled to fail. arXiv preprint arXiv:1906.06668","author":"Mittelstadt Brent","year":"2019","unstructured":"Brent Mittelstadt. 2019. AI Ethics-Too principled to fail. arXiv preprint arXiv:1906.06668 (2019)."},{"key":"e_1_3_2_1_173_1","volume-title":"Principles alone cannot guarantee ethical AI. Nature Machine Intelligence","author":"Mittelstadt Brent","year":"2019","unstructured":"Brent Mittelstadt. 2019. Principles alone cannot guarantee ethical AI. Nature Machine Intelligence (2019), 1--7."},{"key":"e_1_3_2_1_174_1","unstructured":"Christoph Molnar. 2018. Interpretable Machine Learning: A Guide for Making Black-box Models Explainable. Online: https:\/\/christophm.github.io\/interpretable-ml-book\/."},{"key":"e_1_3_2_1_175_1","volume-title":"What is computer ethics? Metaphilosophy 16, 4","author":"Moor James H","year":"1985","unstructured":"James H Moor. 1985. What is computer ethics? Metaphilosophy 16, 4 (1985)."},{"key":"e_1_3_2_1_176_1","doi-asserted-by":"publisher","DOI":"10.1145\/1330311.1330323"},{"key":"e_1_3_2_1_177_1","volume-title":"Accountability': An ever-expanding concept? Public administration 78, 3","author":"Mulgan Richard","year":"2000","unstructured":"Richard Mulgan. 2000. 'Accountability': An ever-expanding concept? Public administration 78, 3 (2000), 555--573."},{"key":"e_1_3_2_1_178_1","volume-title":"Holding power to account: accountability in modern democracies","author":"Mulgan Richard G","unstructured":"Richard G Mulgan. 2003. Holding power to account: accountability in modern democracies. Palgrave Macmillan."},{"key":"e_1_3_2_1_179_1","article-title":"Saving Governance-by-Design","volume":"106","author":"Mulligan Deirdre K","year":"2018","unstructured":"Deirdre K Mulligan and Kenneth A Bamberger. 2018. Saving Governance-by-Design. California Law Review 106, 101 (2018).","journal-title":"California Law Review"},{"key":"e_1_3_2_1_180_1","volume-title":"Shaping Our Tools: Contestability as a Means to Promote Responsible Algorithmic Decision Making in the Professions. Preprint available at SSRN 3311894","author":"Mulligan Deirdre K","year":"2019","unstructured":"Deirdre K Mulligan, Daniel Kluttz, and Nitin Kohli. 2019. Shaping Our Tools: Contestability as a Means to Promote Responsible Algorithmic Decision Making in the Professions. Preprint available at SSRN 3311894 (2019)."},{"key":"e_1_3_2_1_181_1","doi-asserted-by":"publisher","DOI":"10.1098\/rsta.2016.0118"},{"key":"e_1_3_2_1_182_1","doi-asserted-by":"publisher","DOI":"10.1145\/3359221"},{"key":"e_1_3_2_1_183_1","volume-title":"The Oxford Handbook of Ethics of AI","author":"Mulligan Deirdre K","unstructured":"Deirdre K Mulligan and Helen Nissenbaum. 2020. The Concept of Handoff as a Model for Ethical Analysis and Design. In The Oxford Handbook of Ethics of AI, Markus Dubber, Frank Pasquale, and Sunit Das (Eds.). Oxford University Press, Oxford, UK."},{"key":"e_1_3_2_1_184_1","unstructured":"Multistakeholder Process on Software Component Transparency. 2019. Framing Software Component Transparency: Establishing a Common Software Bill of Material (SBOM). https:\/\/www.ntia.gov\/files\/ntia\/publications\/framingsbom_20191112.pdf."},{"key":"e_1_3_2_1_185_1","volume-title":"USENIX Annual Technical Conference, General Track. 43--56","author":"Muniswamy-Reddy Kiran-Kumar","year":"2006","unstructured":"Kiran-Kumar Muniswamy-Reddy, David A Holland, Uri Braun, and Margo I Seltzer. 2006. Provenance-Aware Storage Systems.. In USENIX Annual Technical Conference, General Track. 43--56."},{"key":"e_1_3_2_1_186_1","first-page":"15","article-title":"Provenance for the Cloud","volume":"10","author":"Muniswamy-Reddy Kiran-Kumar","year":"2010","unstructured":"Kiran-Kumar Muniswamy-Reddy, Peter Macko, and Margo I Seltzer. 2010. Provenance for the Cloud.. In FAST, Vol. 10. 15--14.","journal-title":"FAST"},{"key":"e_1_3_2_1_187_1","doi-asserted-by":"publisher","DOI":"10.5555\/539883"},{"key":"e_1_3_2_1_188_1","volume-title":"Privacy-Preserving Data Mining","author":"Nabar Shubha U","unstructured":"Shubha U Nabar, Krishnaram Kenthapadi, Nina Mishra, and Rajeev Motwani. 2008. A survey of query auditing techniques for data privacy. In Privacy-Preserving Data Mining. Springer, 415--431."},{"key":"e_1_3_2_1_189_1","doi-asserted-by":"publisher","DOI":"10.5555\/1182635.1164142"},{"key":"e_1_3_2_1_190_1","unstructured":"National Institute of Standards and Technology. 2018. Framework for Improving Critical Infrastructure Cybersecurity. https:\/\/www.nist.gov\/cyberframework."},{"key":"e_1_3_2_1_191_1","unstructured":"National Institute of Standards and Technology. 2020. NIST Privacy Framework: A Tool for Improving Privacy through Enterprise Risk Management. https:\/\/www.nist.gov\/privacy-framework."},{"key":"e_1_3_2_1_192_1","unstructured":"National New Generation Artificial Intelligence Governance Expert Committee (Ministry of Science and Technology convening). 2019. Governance Principles for a New Generation of Artificial Intelligence: Develop Responsible Artificial Intelligence."},{"key":"e_1_3_2_1_193_1","unstructured":"National Security Commission on Artificial Intelligence. 2020. Second Quarter Recommendations. https:\/\/www.nscai.gov."},{"key":"e_1_3_2_1_194_1","volume-title":"26th {USENIX} Security Symposium ({ USENIX} Security 17). 1271--1287.","author":"Nikitin Kirill","unstructured":"Kirill Nikitin, Eleftherios Kokoris-Kogias, Philipp Jovanovic, Nicolas Gailly, Linus Gasser, Ismail Khoffi, Justin Cappos, and Bryan Ford. 2017. {CHAINIAC}: Proactive software-update transparency via collectively signed skipchains and verified builds. In 26th {USENIX} Security Symposium ({ USENIX} Security 17). 1271--1287."},{"key":"e_1_3_2_1_195_1","volume-title":"Accountability in a computerized society. Science and engineering ethics 2, 1","author":"Nissenbaum Helen","year":"1996","unstructured":"Helen Nissenbaum. 1996. Accountability in a computerized society. Science and engineering ethics 2, 1 (1996), 25--42."},{"key":"e_1_3_2_1_196_1","doi-asserted-by":"publisher","DOI":"10.1109\/2.910905"},{"key":"e_1_3_2_1_197_1","volume-title":"Encyclopedia of science, technology, and ethics","author":"Nissenbaum Helen","unstructured":"Helen Nissenbaum. 2005. Values in technical design. In Encyclopedia of science, technology, and ethics. Macmillan New York, NY, 66--70."},{"key":"e_1_3_2_1_198_1","unstructured":"Office of the Secretary of Defense. 2020. Artificial Intelligence Ethical Principles for the Department of Defense. OSD Memorandum."},{"key":"e_1_3_2_1_199_1","unstructured":"Organization for Economic Cooperation and Development. 2019. Recommendation of the Council on Artificial Intelligence. OECD\/LEGAL\/0449 https:\/\/legalinstruments.oecd.org\/en\/instruments\/OECD-LEGAL-0449."},{"key":"e_1_3_2_1_200_1","unstructured":"Partnership on AI. 2019. Annotation and Benchmarking on Understanding and Transparency of Machine Learning Lifecycles (ABOUT ML). https:\/\/partnershiponai.com\/about-ml."},{"key":"e_1_3_2_1_201_1","doi-asserted-by":"publisher","DOI":"10.1145\/3287560.3287567"},{"key":"e_1_3_2_1_202_1","doi-asserted-by":"publisher","DOI":"10.1145\/3274405"},{"key":"e_1_3_2_1_203_1","volume-title":"Cloud computing","author":"Pearson Siani","unstructured":"Siani Pearson and Andrew Charlesworth. 2009. Accountability as a way forward for privacy protection in the cloud. In Cloud computing. Springer, 131--144."},{"key":"e_1_3_2_1_204_1","volume-title":"A systematic review of provenance systems. Knowledge and Information Systems","author":"P\u00e9rez Beatriz","year":"2018","unstructured":"Beatriz P\u00e9rez, Julio Rubio, and Carlos S\u00e1enz-Ad\u00e1n. 2018. A systematic review of provenance systems. Knowledge and Information Systems (2018), 1--49."},{"key":"e_1_3_2_1_205_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-02152-7_29"},{"key":"e_1_3_2_1_206_1","volume-title":"Quantification and the accounting ideal in science. Social studies of science 22, 4","author":"Porter Theodore M","year":"1992","unstructured":"Theodore M Porter. 1992. Quantification and the accounting ideal in science. Social studies of science 22, 4 (1992), 633--651."},{"key":"e_1_3_2_1_207_1","doi-asserted-by":"publisher","DOI":"10.1145\/3173574.3173677"},{"key":"e_1_3_2_1_208_1","doi-asserted-by":"publisher","DOI":"10.1145\/3351095.3372828"},{"key":"e_1_3_2_1_209_1","doi-asserted-by":"publisher","DOI":"10.1145\/3351095.3372873"},{"key":"e_1_3_2_1_210_1","volume-title":"Hostages of each other: The transformation of nuclear safety since Three Mile Island","author":"Rees Joseph V","unstructured":"Joseph V Rees. 2009. Hostages of each other: The transformation of nuclear safety since Three Mile Island. University of Chicago Press."},{"key":"e_1_3_2_1_211_1","volume-title":"Algorithmic Impact Assessements: A Practical Framework for Public Agency Accountability","author":"Reisman Dillon","year":"2018","unstructured":"Dillon Reisman, Jason Schultz, Kate Crawford, and Meredith Whittaker. 2018. Algorithmic Impact Assessements: A Practical Framework for Public Agency Accountability. AI Now Institute Report https:\/\/ainowinstitute.org\/aiareport2018.pdf."},{"key":"e_1_3_2_1_212_1","doi-asserted-by":"publisher","DOI":"10.1515\/popets-2018-0021"},{"key":"e_1_3_2_1_213_1","doi-asserted-by":"crossref","unstructured":"Henry Gordon Rice. 1953. Classes of recursively enumerable sets and their decision problems. Trans. Amer. Math. Soc. (1953) 358--366.","DOI":"10.1090\/S0002-9947-1953-0053041-6"},{"key":"e_1_3_2_1_214_1","volume-title":"Proceedings of the 17th ACM conference on Computer supported cooperative work & social computing - CSCW '14. ACM Press","author":"Daniela","unstructured":"Daniela K. Rosner and Morgan Ames. 2014. Designing for repair?: infrastructures and materialities of breakdown. In Proceedings of the 17th ACM conference on Computer supported cooperative work & social computing - CSCW '14. ACM Press, Baltimore, 319--331."},{"key":"e_1_3_2_1_215_1","unstructured":"Jeff Rothenberg. 1999. Avoiding Technological Quicksand: Finding a Viable Technical Foundation for Digital Preservation. A Report to the Council on Library and Information Resources. ERIC."},{"key":"e_1_3_2_1_216_1","first-page":"1333","article-title":"Privacy by design: A counterfactual analysis of Google and Facebook privacy incidents","volume":"28","author":"Rubinstein Ira S","year":"2013","unstructured":"Ira S Rubinstein and Nathaniel Good. 2013. Privacy by design: A counterfactual analysis of Google and Facebook privacy incidents. Berkeley Tech. LJ 28 (2013), 1333.","journal-title":"Berkeley Tech. LJ"},{"key":"e_1_3_2_1_217_1","volume-title":"Handbook of human factors and ergonomics","author":"Salvendy Gavriel","unstructured":"Gavriel Salvendy. 2012. Handbook of human factors and ergonomics. John Wiley & Sons."},{"key":"e_1_3_2_1_218_1","volume-title":"Seeing the Sort: The Aesthetic and Industrial Defense of \"The Algorithm\". Journal of the New Media Caucus","author":"Sandvig Christian","year":"2015","unstructured":"Christian Sandvig. 2015. Seeing the Sort: The Aesthetic and Industrial Defense of \"The Algorithm\". Journal of the New Media Caucus (2015). ISSN: 1942-017X, [Online] http:\/\/median.newmediacaucus.org\/art-infrastructures-information\/seeing-the-sort-the-aesthetic-and-industrial-defense-of-the-algorithm\/."},{"key":"e_1_3_2_1_219_1","volume-title":"Auditing algorithms: Research methods for detecting discrimination on internet platforms. Data and Discrimination: Converting Critical Concerns into Productive Inquiry","author":"Sandvig Christian","year":"2014","unstructured":"Christian Sandvig, Kevin Hamilton, Karrie Karahalios, and Cedric Langbort. 2014. Auditing algorithms: Research methods for detecting discrimination on internet platforms. Data and Discrimination: Converting Critical Concerns into Productive Inquiry (2014)."},{"key":"e_1_3_2_1_220_1","doi-asserted-by":"publisher","DOI":"10.1109\/SP.2014.36"},{"key":"e_1_3_2_1_221_1","article-title":"Disparate Impact in Big Data Policing","volume":"52","author":"Selbst Andrew","year":"2017","unstructured":"Andrew Selbst. 2017. Disparate Impact in Big Data Policing. Georgia Law Review 52, 109 (2017).","journal-title":"Georgia Law Review"},{"key":"e_1_3_2_1_222_1","volume-title":"The intuitive appeal of explainable machines. Fordham L. Rev. 87","author":"Selbst Andrew D","year":"2018","unstructured":"Andrew D Selbst and Solon Barocas. 2018. The intuitive appeal of explainable machines. Fordham L. Rev. 87 (2018)."},{"key":"e_1_3_2_1_223_1","doi-asserted-by":"publisher","DOI":"10.1145\/3287560.3287598"},{"key":"e_1_3_2_1_224_1","doi-asserted-by":"publisher","DOI":"10.1093\/idpl\/ipx022"},{"key":"e_1_3_2_1_225_1","volume-title":"Developing bug-free machine learning systems with formal mathematics. arXiv:1706.08605","author":"Selsam Daniel","year":"2017","unstructured":"Daniel Selsam, Percy Liang, and David L Dill. 2017. Developing bug-free machine learning systems with formal mathematics. arXiv:1706.08605 (2017)."},{"key":"e_1_3_2_1_226_1","volume-title":"Conference on Fairness, Accountability, and Transparency. 99--109","author":"Sendak Mark","year":"2020","unstructured":"Mark Sendak, Madeleine Clare Elish, Michael Gao, Joseph Futoma, William Ratliff, Marshall Nichols, Armando Bedoya, Suresh Balu, and Cara O'Brien. 2020. \"The human body is a black box\" supporting clinical decision-making with deep learning. In Conference on Fairness, Accountability, and Transparency. 99--109."},{"key":"e_1_3_2_1_227_1","doi-asserted-by":"publisher","DOI":"10.1561\/1100000073"},{"key":"e_1_3_2_1_228_1","volume-title":"Proceedings of the 17th ACM Conference on Computer Supported Cooperative Work & Social Computing (CSCW '14)","author":"Shilton Katie","unstructured":"Katie Shilton, Jes A. Koepfler, and Kenneth R. Fleischmann. 2014. How to see values in social computing: Methods for Studying Values Dimensions. In Proceedings of the 17th ACM Conference on Computer Supported Cooperative Work & Social Computing (CSCW '14). 426--435."},{"key":"e_1_3_2_1_229_1","first-page":"258","article-title":"The Urban Institute audit studies: Their methods and findings. Clear and Convincing Evidence: Measurement of Discrimination in America","volume":"187","author":"Siegelman Peter","year":"1993","unstructured":"Peter Siegelman and J Heckman. 1993. The Urban Institute audit studies: Their methods and findings. Clear and Convincing Evidence: Measurement of Discrimination in America, Washington 187 (1993), 258.","journal-title":"Washington"},{"key":"e_1_3_2_1_230_1","doi-asserted-by":"publisher","DOI":"10.1145\/192844.193021"},{"key":"e_1_3_2_1_231_1","volume-title":"Steps toward an ecology of infrastructure: Design and access for large information spaces. Information systems research 7, 1","author":"Star Susan Leigh","year":"1996","unstructured":"Susan Leigh Star and Karen Ruhleder. 1996. Steps toward an ecology of infrastructure: Design and access for large information spaces. Information systems research 7, 1 (1996), 111--134."},{"key":"e_1_3_2_1_232_1","doi-asserted-by":"publisher","DOI":"10.1145\/2858036.2858420"},{"key":"e_1_3_2_1_233_1","volume-title":"Implementing reproducible research","author":"Stodden Victoria","unstructured":"Victoria Stodden, Friedrich Leisch, and Roger D Peng. 2014. Implementing reproducible research. CRC Press."},{"key":"e_1_3_2_1_234_1","doi-asserted-by":"publisher","DOI":"10.5334\/jors.ay"},{"key":"e_1_3_2_1_235_1","doi-asserted-by":"publisher","DOI":"10.4018\/jdm.2005100104"},{"key":"e_1_3_2_1_236_1","unstructured":"United States Office of the Director of National Intelligence. 2020. Artificial Intelligence Ethics Framework for the Intelligence Community."},{"key":"e_1_3_2_1_237_1","volume-title":"The Challenger launch decision: Risky technology, culture, and deviance at NASA","author":"Vaughan Diane","unstructured":"Diane Vaughan. 1996. The Challenger launch decision: Risky technology, culture, and deviance at NASA. University of Chicago press."},{"key":"e_1_3_2_1_238_1","doi-asserted-by":"publisher","DOI":"10.31235\/osf.io\/6cdhe"},{"key":"e_1_3_2_1_239_1","doi-asserted-by":"publisher","DOI":"10.1109\/SP.2013.48"},{"key":"e_1_3_2_1_240_1","volume-title":"Design Implications. In Proceedings of the 19th ACM Conference on Computer Supported Cooperative Work and Social Computing Companion - CSCW '16 Companion. ACM Press","author":"Wagenknecht Susann","year":"2016","unstructured":"Susann Wagenknecht, Min Lee, Caitlin Lustig, Jacki O'Neill, and Himanshu Zade. 2016. Algorithms at Work: Empirical Diversity, Analytic Vocabularies, Design Implications. In Proceedings of the 19th ACM Conference on Computer Supported Cooperative Work and Social Computing Companion - CSCW '16 Companion. ACM Press, New York, New York, USA, 536--543."},{"key":"e_1_3_2_1_241_1","unstructured":"Patrick Warden. 2018. The Machine Learning Reproducibility Crisis. https:\/\/petewarden.com\/2018\/03\/19\/the-machine-learning-reproducibility-crisis\/."},{"key":"e_1_3_2_1_242_1","first-page":"5","article-title":"Building an Encrypted and Searchable Audit Log","volume":"4","author":"Waters Brent R","year":"2004","unstructured":"Brent R Waters, Dirk Balfanz, Glenn Durfee, and Diana K Smetters. 2004. Building an Encrypted and Searchable Audit Log. In NDSS, Vol. 4. 5--6.","journal-title":"NDSS"},{"key":"e_1_3_2_1_244_1","doi-asserted-by":"publisher","DOI":"10.1145\/3351095.3372833"},{"key":"e_1_3_2_1_245_1","doi-asserted-by":"publisher","DOI":"10.1145\/3272973.3273005"},{"key":"e_1_3_2_1_246_1","doi-asserted-by":"publisher","DOI":"10.1145\/3290605.3300492"},{"key":"e_1_3_2_1_247_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10676-019-09497-z"},{"key":"e_1_3_2_1_248_1","doi-asserted-by":"publisher","DOI":"10.1145\/3287560.3287577"},{"key":"e_1_3_2_1_249_1","doi-asserted-by":"publisher","DOI":"10.1145\/3274463"},{"key":"e_1_3_2_1_250_1","volume-title":"A not quite random walk: Experimenting with the ethnomethods of the algorithm","author":"Ziewitz Malte","year":"2017","unstructured":"Malte Ziewitz. 2017. A not quite random walk: Experimenting with the ethnomethods of the algorithm. Big Data & Society 4, 2 (2017)."}],"event":{"name":"FAccT '21: 2021 ACM Conference on Fairness, Accountability, and Transparency","location":"Virtual Event Canada","acronym":"FAccT '21","sponsor":["ACM Association for Computing Machinery"]},"container-title":["Proceedings of the 2021 ACM Conference on Fairness, Accountability, and Transparency"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3442188.3445937","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3442188.3445937","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T20:48:58Z","timestamp":1750193338000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3442188.3445937"}},"subtitle":["A Principle for Operationalizing Accountability in Computing Systems"],"short-title":[],"issued":{"date-parts":[[2021,3]]},"references-count":244,"alternative-id":["10.1145\/3442188.3445937","10.1145\/3442188"],"URL":"https:\/\/doi.org\/10.1145\/3442188.3445937","relation":{},"subject":[],"published":{"date-parts":[[2021,3]]},"assertion":[{"value":"2021-03-01","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}