{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,15]],"date-time":"2026-04-15T22:48:17Z","timestamp":1776293297505,"version":"3.50.1"},"publisher-location":"New York, NY, USA","reference-count":156,"publisher":"ACM","license":[{"start":{"date-parts":[[2021,3,1]],"date-time":"2021-03-01T00:00:00Z","timestamp":1614556800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2021,3,3]]},"DOI":"10.1145\/3442188.3445918","type":"proceedings-article","created":{"date-parts":[[2021,3,2]],"date-time":"2021-03-02T20:26:24Z","timestamp":1614716784000},"page":"560-575","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":197,"title":["Towards Accountability for Machine Learning Datasets: Practices from Software Engineering and Infrastructure"],"prefix":"10.1145","author":[{"given":"Ben","family":"Hutchinson","sequence":"first","affiliation":[]},{"given":"Andrew","family":"Smart","sequence":"additional","affiliation":[]},{"given":"Alex","family":"Hanna","sequence":"additional","affiliation":[]},{"given":"Remi","family":"Denton","sequence":"additional","affiliation":[]},{"given":"Christina","family":"Greer","sequence":"additional","affiliation":[]},{"given":"Oddur","family":"Kjartansson","sequence":"additional","affiliation":[]},{"given":"Parker","family":"Barnes","sequence":"additional","affiliation":[]},{"given":"Margaret","family":"Mitchell","sequence":"additional","affiliation":[]}],"member":"320","published-online":{"date-parts":[[2021,3]]},"reference":[{"key":"e_1_3_2_1_1_1","volume-title":"Recoding gender: Women's changing participation in computing","author":"Abbate Janet","unstructured":"Janet Abbate. 2012. Recoding gender: Women's changing participation in computing. MIT Press."},{"key":"e_1_3_2_1_2_1","volume-title":"Software engineering body of knowledge","author":"Abran Alain","year":"2004","unstructured":"Alain Abran, James W Moore, Pierre Bourque, Robert Dupuis, and L Tripp. 2004. Software engineering body of knowledge. IEEE Computer Society, Angela Burgess (2004)."},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1061\/9780784412848.081"},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE-SEIP.2019.00042"},{"key":"e_1_3_2_1_5_1","volume-title":"Fair Active Learning. arXiv preprint arXiv:2001.01796","author":"Anahideh Hadis","year":"2020","unstructured":"Hadis Anahideh and Abolfazl Asudeh. 2020. Fair Active Learning. arXiv preprint arXiv:2001.01796 (2020)."},{"key":"e_1_3_2_1_6_1","volume-title":"Infrastructure and the principle of the hiding hand. The Governance of Infrastructure","author":"Anheier Helmut K","year":"2017","unstructured":"Helmut K Anheier. 2017. Infrastructure and the principle of the hiding hand. The Governance of Infrastructure (2017), 63."},{"key":"e_1_3_2_1_7_1","volume-title":"Theoretical Foundations of Artificial General Intelligence","author":"Arel Itamar","unstructured":"Itamar Arel. 2012. Deep reinforcement learning as foundation for artificial general intelligence. In Theoretical Foundations of Artificial General Intelligence. Springer, 89--102."},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.1147\/JRD.2019.2942288"},{"key":"e_1_3_2_1_9_1","unstructured":"Rob Ashmore Radu Calinescu and Colin Paterson. 2019. Assuring the machine learning lifecycle: Desiderata methods and challenges. arXiv preprint arXiv:1905.04223."},{"key":"e_1_3_2_1_10_1","unstructured":"Camera & Imaging Products Association et al. 2010. Exchangeable image file format for digital still cameras: Exif Version 2.3. CIPA DC-008 Translation-2010 (2010)."},{"key":"e_1_3_2_1_11_1","volume-title":"Quantifying Gender Bias in Different Corpora. In Companion Proceedings of the Web Conference","author":"Babaeianjelodar Marzieh","year":"2020","unstructured":"Marzieh Babaeianjelodar, Stephen Lorenz, Josh Gordon, Jeanna Matthews, and Evan Freitag. 2020. Quantifying Gender Bias in Different Corpora. In Companion Proceedings of the Web Conference 2020. 752--759."},{"key":"e_1_3_2_1_12_1","volume-title":"Towards Traceability in Data Ecosystems using a Bill of Materials Model. arXiv preprint arXiv:1904.04253","author":"Barclay Iain","year":"2019","unstructured":"Iain Barclay, Alun Preece, Ian Taylor, and Dinesh Verma. 2019. Towards Traceability in Data Ecosystems using a Bill of Materials Model. arXiv preprint arXiv:1904.04253 (2019)."},{"key":"e_1_3_2_1_13_1","first-page":"671","article-title":"Big data's disparate impact","volume":"104","author":"Barocas Solon","year":"2016","unstructured":"Solon Barocas and Andrew D Selbst. 2016. Big data's disparate impact. Calif. L. Rev. 104 (2016), 671.","journal-title":"Calif. L. Rev."},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.1145\/3097983.3098021"},{"key":"e_1_3_2_1_15_1","volume-title":"Information Systems Development","author":"Bednar Peter M","unstructured":"Peter M Bednar and Christine Welch. 2009. Contextual inquiry and requirements shaping. In Information Systems Development. Springer, 225--236."},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.1162\/tacl_a_00041"},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.1145\/336512.336534"},{"key":"e_1_3_2_1_18_1","volume-title":"Ethical and Socially-Aware Data Labels. In Annual International Symposium on Information Management and Big Data. Springer, 320--327","author":"Beretta Elena","year":"2018","unstructured":"Elena Beretta, Antonio Vetr\u00f2, Bruno Lepri, and Juan Carlos De Martin. 2018. Ethical and Socially-Aware Data Labels. In Annual International Symposium on Information Management and Big Data. Springer, 320--327."},{"key":"e_1_3_2_1_19_1","volume-title":"A resource-based perspective on information technology capability and firm performance: an empirical investigation. MIS quarterly","author":"Bharadwaj Anandhi S","year":"2000","unstructured":"Anandhi S Bharadwaj. 2000. A resource-based perspective on information technology capability and firm performance: an empirical investigation. MIS quarterly (2000), 169--196."},{"key":"e_1_3_2_1_20_1","volume-title":"Representativeness in corpus design. Literary and linguistic computing 8, 4","author":"Biber Douglas","year":"1993","unstructured":"Douglas Biber. 1993. Representativeness in corpus design. Literary and linguistic computing 8, 4 (1993), 243--257."},{"key":"e_1_3_2_1_21_1","unstructured":"Matthias Boehm Iulian Antonov Sebastian Baunsgaard Mark Dokter Robert Ginth \u00f6r Kevin Innerebner Florijan Klezin Stefanie Lindstaedt Arnab Phani Benjamin Rath Berthold Reinwald Shafaq Siddiqi and Sebastian Benjamin Wrede. 2019. SystemDS: A Declarative Machine Learning System for the End-to-End Data Science Lifecycle. arXiv preprint arXiv:1909.02976."},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.5555\/380681"},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.1109\/BigData.2017.8258038"},{"key":"e_1_3_2_1_24_1","volume-title":"The relationship between science and technology. Research policy 23, 5","author":"Brooks Harvey","year":"1994","unstructured":"Harvey Brooks. 1994. The relationship between science and technology. Research policy 23, 5 (1994), 477--486."},{"key":"e_1_3_2_1_25_1","volume-title":"Wicked problems in design thinking. Design issues 8, 2","author":"Buchanan Richard","year":"1992","unstructured":"Richard Buchanan. 1992. Wicked problems in design thinking. Design issues 8, 2 (1992), 5--21."},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.1061\/(ASCE)EI.1943-5541.0000205"},{"key":"e_1_3_2_1_27_1","volume-title":"Conference on fairness, accountability and transparency. 77--91","author":"Buolamwini Joy","year":"2018","unstructured":"Joy Buolamwini and Timnit Gebru. 2018. Gender shades: Intersectional accuracy disparities in commercial gender classification. In Conference on fairness, accountability and transparency. 77--91."},{"key":"e_1_3_2_1_28_1","volume-title":"Women also Snowboard: Overcoming Bias in Captioning Models. (03","author":"Burns Kaylee","year":"2018","unstructured":"Kaylee Burns, Lisa Hendricks, Trevor Darrell, and Anna Rohrbach. 2018. Women also Snowboard: Overcoming Bias in Captioning Models. (03 2018)."},{"key":"e_1_3_2_1_29_1","first-page":"18","article-title":"A dozen ways to get lost in translation: Inherent challenges in large scale data sets","volume":"8","author":"Busch Lawrence","year":"2014","unstructured":"Lawrence Busch. 2014. A dozen ways to get lost in translation: Inherent challenges in large scale data sets. International Journal of Communication 8 (2014), 18.","journal-title":"International Journal of Communication"},{"key":"e_1_3_2_1_30_1","volume-title":"Influence of personality types in software tasks choices. Computers in Human behavior 52","author":"Capretz Luiz Fernando","year":"2015","unstructured":"Luiz Fernando Capretz, Daniel Varona, and Arif Raza. 2015. Influence of personality types in software tasks choices. Computers in Human behavior 52 (2015), 373--378."},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"crossref","unstructured":"Supriyo Chakraborty Richard Tomsett Ramya Raghavendra Daniel Harborne Moustafa Alzantot Federico Cerutti Mani Srivastava Alun Preece Simon Julier Raghuveer M Rao et al. 2017. Interpretability of deep learning models: a survey of results. In 2017 IEEE Smart World Ubiquitous Intelligence & Computing Advanced & Trusted Computed Scalable Computing & Communications Cloud & Big Data Computing Internet of People and Smart City Innovation (Smart-World\/SCALCOM\/UIC\/ATC\/CBDCom\/IOP\/SCI). IEEE 1--6.","DOI":"10.1109\/UIC-ATC.2017.8397411"},{"key":"e_1_3_2_1_32_1","volume-title":"The scored society: Due process for automated predictions. Washington Law Review 89 (03","author":"Citron Danielle","year":"2014","unstructured":"Danielle Citron and Frank Pasquale. 2014. The scored society: Due process for automated predictions. Washington Law Review 89 (03 2014), 1--33."},{"key":"e_1_3_2_1_33_1","unstructured":"CMMI Institute. 2018. Patient Demographic Data Quality (PDDQ) Framework."},{"key":"e_1_3_2_1_34_1","volume-title":"CA: IEEE Computer Society 169","author":"IEEE Standards Coordinating Committee et al.","year":"1990","unstructured":"IEEE Standards Coordinating Committee et al. 1990. IEEE Standard Glossary of Software Engineering Terminology (IEEE Std 610.12-1990). Los Alamitos. CA: IEEE Computer Society 169 (1990)."},{"key":"e_1_3_2_1_35_1","doi-asserted-by":"publisher","DOI":"10.1098\/rsta.2016.0153"},{"key":"e_1_3_2_1_36_1","first-page":"10","article-title":"Critiquing Big Data: Politics, ethics, epistemology (special section introduction)","volume":"8","author":"Crawford Kate","year":"2014","unstructured":"Kate Crawford, Mary L Gray, and Kate Miltner. 2014. Critiquing Big Data: Politics, ethics, epistemology (special section introduction). International Journal of Communication 8 (2014), 10.","journal-title":"International Journal of Communication"},{"key":"e_1_3_2_1_37_1","volume-title":"Excavating AI: The politics of images in machine learning training sets. Excavating AI","author":"Crawford Kate","year":"2019","unstructured":"Kate Crawford and Trevor Paglen. 2019. Excavating AI: The politics of images in machine learning training sets. Excavating AI (2019)."},{"key":"e_1_3_2_1_38_1","volume-title":"Racial bias in hate speech and abusive language detection datasets. arXiv preprint arXiv:1905.12516","author":"Davidson Thomas","year":"2019","unstructured":"Thomas Davidson, Debasmita Bhattacharya, and Ingmar Weber. 2019. Racial bias in hate speech and abusive language detection datasets. arXiv preprint arXiv:1905.12516 (2019)."},{"key":"e_1_3_2_1_39_1","volume-title":"The limitations of standardized science tests as benchmarks for artificial intelligence research: Position paper. arXiv preprint arXiv:1411.1629","author":"Davis Ernest","year":"2014","unstructured":"Ernest Davis. 2014. The limitations of standardized science tests as benchmarks for artificial intelligence research: Position paper. arXiv preprint arXiv:1411.1629 (2014)."},{"key":"e_1_3_2_1_40_1","volume-title":"Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition Workshops. 52--59","author":"de Vries Terrance","unstructured":"Terrance de Vries, Ishan Misra, Changhan Wang, and Laurens van der Maaten. 2019. Does object recognition work for everyone?. In Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition Workshops. 52--59."},{"key":"e_1_3_2_1_41_1","volume-title":"Hilary Nicole, and Morgan Klaus Scheuerman.","author":"Denton Remi","year":"2020","unstructured":"Remi Denton, Alex Hanna, Razvan Amironesei, Andrew Smart, Hilary Nicole, and Morgan Klaus Scheuerman. 2020. Bringing the People Back In: Contesting Benchmark Machine Learning Datasets. arXiv preprint arXiv:2007.07399 (2020)."},{"key":"e_1_3_2_1_42_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.infsof.2004.08.008"},{"key":"e_1_3_2_1_43_1","volume-title":"A survey of data quality measurement and monitoring tools. arXiv preprint arXiv:1907.08138","author":"Ehrlinger Lisa","year":"2019","unstructured":"Lisa Ehrlinger, Elisa Rusz, and Wolfram W\u00f6\u00df. 2019. A survey of data quality measurement and monitoring tools. arXiv preprint arXiv:1907.08138 (2019)."},{"key":"e_1_3_2_1_44_1","volume-title":"Automating inequality: How high-tech tools profile, police, and punish the poor","author":"Eubanks Virginia","unstructured":"Virginia Eubanks. 2018. Automating inequality: How high-tech tools profile, police, and punish the poor. St. Martin's Press."},{"key":"e_1_3_2_1_45_1","doi-asserted-by":"publisher","DOI":"10.1353\/sor.2016.0063"},{"key":"e_1_3_2_1_46_1","unstructured":"Food Drug Administration et al. 2016. Data Integrity and Compliance With CGMP Guidance for Industry. Draft Guidance (2016)."},{"key":"e_1_3_2_1_47_1","unstructured":"US Food Drug Administration et al. 2018. Data Integrity and Compliance with Drug CGMP Questions and Answers Guidance for Industry."},{"key":"e_1_3_2_1_48_1","doi-asserted-by":"publisher","DOI":"10.1093\/acprof:oso\/9780199895656.001.0001"},{"key":"e_1_3_2_1_49_1","volume-title":"Information cultures in the digital age","author":"Furner Jonathan","unstructured":"Jonathan Furner. 2016. \"Data\": The data. In Information cultures in the digital age. Springer, 287--306."},{"key":"e_1_3_2_1_50_1","doi-asserted-by":"publisher","DOI":"10.1073\/pnas.1720347115"},{"key":"e_1_3_2_1_51_1","volume-title":"Hanna Wallach, Hal Daum\u00e9 III, and Kate Crawford.","author":"Gebru Timnit","year":"2018","unstructured":"Timnit Gebru, Jamie Morgenstern, Briana Vecchione, Jennifer Wortman Vaughan, Hanna Wallach, Hal Daum\u00e9 III, and Kate Crawford. 2018. Datasheets for datasets. arXiv preprint arXiv:1803.09010 (2018)."},{"key":"e_1_3_2_1_52_1","doi-asserted-by":"publisher","DOI":"10.1145\/3351095.3372862"},{"key":"e_1_3_2_1_53_1","unstructured":"Dave Gershgorn. 2018. If AI is going to be the world's doctor it needs better textbooks. https:\/\/qz.com\/1367177\/if-ai-is-going-to-be-theworlds-doctor-it-needs-better-textbooks. Accessed: 2020-09-19."},{"key":"e_1_3_2_1_54_1","doi-asserted-by":"publisher","DOI":"10.1109\/NYSDS.2016.7747817"},{"key":"e_1_3_2_1_55_1","first-page":"56","article-title":"A glossary of requirements engineering terminology. Standard Glossary of the Certified Professional for Requirements Engineering (CPRE) Studies and Exam","volume":"1","author":"Glinz Martin","year":"2011","unstructured":"Martin Glinz. 2011. A glossary of requirements engineering terminology. Standard Glossary of the Certified Professional for Requirements Engineering (CPRE) Studies and Exam, Version 1 (2011), 56.","journal-title":"Version"},{"key":"e_1_3_2_1_56_1","doi-asserted-by":"publisher","DOI":"10.1179\/030801804225012572"},{"key":"e_1_3_2_1_57_1","unstructured":"Steven L Goldman. 2010. Beyond satisficing: Design trade offs and the rationality of engineering. In 2010 Forum on philosophy engineering & technology."},{"key":"e_1_3_2_1_58_1","volume-title":"Data science as political action: Grounding data science in a politics of justice. Available at SSRN 3658431","author":"Green Ben","year":"2020","unstructured":"Ben Green. 2020. Data science as political action: Grounding data science in a politics of justice. Available at SSRN 3658431 (2020)."},{"key":"e_1_3_2_1_59_1","unstructured":"Tristan Greene. 2020. 2010-2019: The rise of deep learning. https:\/\/thenextweb.com\/artificial-intelligence\/2020\/01\/02\/2010- 2019-the-rise-of-deep-learning\/. Accessed: 2020-09-26."},{"key":"e_1_3_2_1_60_1","doi-asserted-by":"crossref","unstructured":"Ian Hacking Jan Hacking et al. 1983. Representing and intervening: Introductory topics in the philosophy of natural science. Cambridge University Press.","DOI":"10.1017\/CBO9780511814563"},{"key":"e_1_3_2_1_61_1","doi-asserted-by":"publisher","DOI":"10.1109\/MIS.2009.36"},{"key":"e_1_3_2_1_62_1","unstructured":"Brendan Hall and Kevin Driscoll. 2014. Distributed System Design Checklist. (2014)."},{"key":"e_1_3_2_1_63_1","doi-asserted-by":"crossref","unstructured":"Eldon C Hall. 1996. Journey to the moon: the history of the Apollo guidance computer. Aiaa.","DOI":"10.2514\/4.868023"},{"key":"e_1_3_2_1_64_1","volume-title":"Situated knowledges: The science question in feminism and the privilege of partial perspective. Feminist studies 14, 3","author":"Haraway Donna","year":"1988","unstructured":"Donna Haraway. 1988. Situated knowledges: The science question in feminism and the privilege of partial perspective. Feminist studies 14, 3 (1988), 575--599."},{"key":"e_1_3_2_1_65_1","doi-asserted-by":"publisher","DOI":"10.1023\/B:SYNT.0000016424.47883.b9"},{"key":"e_1_3_2_1_66_1","first-page":"159","article-title":"NLP's Clever Hans Moment has Arrived","volume":"21","author":"Heinzerling Benjamin","year":"2020","unstructured":"Benjamin Heinzerling. 2020. NLP's Clever Hans Moment has Arrived. Journal of Cognitive Science 21, 1 (2020), 159--167.","journal-title":"Journal of Cognitive Science"},{"key":"e_1_3_2_1_67_1","doi-asserted-by":"publisher","DOI":"10.1145\/3064663.3064703"},{"key":"e_1_3_2_1_68_1","volume-title":"New Media, and Technology 9","author":"Hoffmann Anna Lauren","year":"2016","unstructured":"Anna Lauren Hoffmann and Raina Bloom. 2016. Digitizing Books, Obscuring Women's Work: Google Books, Librarians, and Ideologies of Access. Ada: A Journal of Gender, New Media, and Technology 9 (2016)."},{"key":"e_1_3_2_1_69_1","volume-title":"The dataset nutrition label: A framework to drive higher data quality standards. arXiv preprint arXiv:1805.03677","author":"Holland Sarah","year":"2018","unstructured":"Sarah Holland, Ahmed Hosny, Sarah Newman, Joshua Joseph, and Kasia Chmielinski. 2018. The dataset nutrition label: A framework to drive higher data quality standards. arXiv preprint arXiv:1805.03677 (2018)."},{"key":"e_1_3_2_1_70_1","doi-asserted-by":"publisher","DOI":"10.1145\/3290605.3300830"},{"key":"e_1_3_2_1_71_1","volume-title":"Social Biases in NLP Models as Barriers for Persons with Disabilities. ACL","author":"Hutchinson Ben","year":"2020","unstructured":"Ben Hutchinson, Vinodkumar Prabhakaran, Remi Denton, Kellie Webster, Yu Zhong, and Stephen Denuyl. 2020. Social Biases in NLP Models as Barriers for Persons with Disabilities. ACL (2020)."},{"key":"e_1_3_2_1_72_1","doi-asserted-by":"publisher","DOI":"10.1145\/2470654.2470742"},{"key":"e_1_3_2_1_73_1","volume-title":"Measurement and fairness. arXiv preprint arXiv:1912.05511","author":"Jacobs Abigail Z","year":"2019","unstructured":"Abigail Z Jacobs and Hanna Wallach. 2019. Measurement and fairness. arXiv preprint arXiv:1912.05511 (2019)."},{"key":"e_1_3_2_1_74_1","doi-asserted-by":"publisher","DOI":"10.1145\/3351095.3372829"},{"key":"e_1_3_2_1_75_1","volume-title":"Accountability Challenges in the Governance of Infrastructure. The Governance of Infrastructure","author":"Jordana Jacint","year":"2017","unstructured":"Jacint Jordana. 2017. Accountability Challenges in the Governance of Infrastructure. The Governance of Infrastructure (2017), 43."},{"key":"e_1_3_2_1_76_1","volume-title":"Earlier Isn't Always Better: Sub-aspect Analysis on Corpus and System Biases in Summarization. arXiv preprint arXiv:1908.11723","author":"Jung Taehee","year":"2019","unstructured":"Taehee Jung, Dongyeop Kang, Lucas Mentch, and Eduard Hovy. 2019. Earlier Isn't Always Better: Sub-aspect Analysis on Corpus and System Biases in Summarization. arXiv preprint arXiv:1908.11723 (2019)."},{"key":"e_1_3_2_1_77_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/N18-1023"},{"key":"e_1_3_2_1_78_1","volume-title":"Introduction to the special issue on the web as corpus. Computational linguistics 29, 3","author":"Kilgarriff Adam","year":"2003","unstructured":"Adam Kilgarriff and Gregory Grefenstette. 2003. Introduction to the special issue on the web as corpus. Computational linguistics 29, 3 (2003), 333--347."},{"key":"e_1_3_2_1_79_1","doi-asserted-by":"crossref","unstructured":"Gary King. 2007. An introduction to the dataverse network as an infrastructure for data sharing.","DOI":"10.1177\/0049124107306660"},{"key":"e_1_3_2_1_80_1","volume-title":"Performing a project premortem. Harvard business review 85, 9","author":"Klein Gary","year":"2007","unstructured":"Gary Klein. 2007. Performing a project premortem. Harvard business review 85, 9 (2007), 18--19."},{"key":"e_1_3_2_1_81_1","unstructured":"Billy Vaughn Koen. 2003. Discussion of The Method: Conducting the Engineer's Approach to Problem Solving."},{"key":"e_1_3_2_1_82_1","volume-title":"1st Conference on Fairness, Accountability, and Transparancy","volume":"7","author":"Kohli Nitin","year":"2018","unstructured":"Nitin Kohli, Renata Barreto, and Joshua A Kroll. 2018. Translation tutorial: a shared lexicon for research and practice in human-centered software systems. In 1st Conference on Fairness, Accountability, and Transparancy. New York, NY, USA, Vol. 7."},{"key":"e_1_3_2_1_83_1","unstructured":"Tobias Krafft Marc Hauer Lajla Fetic Andreas Kaminski Michael Puntschuh Philipp Otto Christoph Hubig Torsten Fleischer Paul Gr\u00fcnke Rafaela Hillerbrand Carla Hustedt and Sebastian Hallensleben. 2020. From Principles to Practice - An interdisciplinary framework to operationalise AI ethics."},{"key":"e_1_3_2_1_84_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2015.7299194"},{"key":"e_1_3_2_1_85_1","volume-title":"Engineering problems in machine learning systems. Machine Learning","author":"Kuwajima Hiroshi","year":"2020","unstructured":"Hiroshi Kuwajima, Hirotoshi Yasuoka, and Toshihiro Nakae. 2020. Engineering problems in machine learning systems. Machine Learning (2020), 1--24."},{"key":"e_1_3_2_1_86_1","doi-asserted-by":"publisher","DOI":"10.1002\/esp.4775"},{"key":"e_1_3_2_1_87_1","volume-title":"The politics and poetics of infrastructure. Annual review of anthropology 42","author":"Larkin Brian","year":"2013","unstructured":"Brian Larkin. 2013. The politics and poetics of infrastructure. Annual review of anthropology 42 (2013), 327--343."},{"key":"e_1_3_2_1_88_1","volume-title":"Science in action: How to follow scientists and engineers through society","author":"Latour Bruno","unstructured":"Bruno Latour. 1987. Science in action: How to follow scientists and engineers through society. Harvard university press."},{"key":"e_1_3_2_1_89_1","volume-title":"Laboratory life: The construction of scientific facts","author":"Latour Bruno","unstructured":"Bruno Latour and Steve Woolgar. 2013. Laboratory life: The construction of scientific facts. Princeton University Press."},{"key":"e_1_3_2_1_90_1","unstructured":"Alexander Lavin. 2020. Machine Learning Is No Place To \"Move Fast And Break Things\". https:\/\/www.forbes.com\/sites\/alexanderlavin\/2020\/02\/17\/machine-learning-is-no-place-to-move-fast-and-break-things\/#2bfee96d15f2. Accessed: 2020-09-19."},{"key":"e_1_3_2_1_91_1","unstructured":"Sabina Leonelli. 2020. Scientific Research and Big Data. https:\/\/plato.stanford.edu\/entries\/science-big-data\/. Accessed: 2020-10-04."},{"key":"e_1_3_2_1_92_1","volume-title":"LogiQA: A Challenge Dataset for Machine Reading Comprehension with Logical Reasoning. arXiv preprint arXiv:2007.08124","author":"Liu Jian","year":"2020","unstructured":"Jian Liu, Leyang Cui, Hanmeng Liu, Dandan Huang, Yile Wang, and Yue Zhang. 2020. LogiQA: A Challenge Dataset for Machine Reading Comprehension with Logical Reasoning. arXiv preprint arXiv:2007.08124 (2020)."},{"key":"e_1_3_2_1_93_1","volume-title":"Alice Bruynseels, Thushika Mahendiran, Gabriella Moraes, Mohith Shamdas, Christoph Kern, et al.","author":"Liu Xiaoxuan","year":"2019","unstructured":"Xiaoxuan Liu, Livia Faes, Aditya U Kale, Siegfried K Wagner, Dun Jack Fu, Alice Bruynseels, Thushika Mahendiran, Gabriella Moraes, Mohith Shamdas, Christoph Kern, et al. 2019. A comparison of deep learning performance against health-care professionals in detecting diseases from medical imaging: a systematic review and meta-analysis. The lancet digital health 1, 6 (2019), e271-e297."},{"key":"e_1_3_2_1_94_1","volume-title":"Cultures of proving. Social studies of science 29, 6","author":"Livingston Eric","year":"1999","unstructured":"Eric Livingston. 1999. Cultures of proving. Social studies of science 29, 6 (1999), 867--888."},{"key":"e_1_3_2_1_95_1","doi-asserted-by":"crossref","unstructured":"David Loshin. 2010. Master data management. Morgan Kaufmann.","DOI":"10.1016\/B978-0-12-374225-4.00006-0"},{"key":"e_1_3_2_1_96_1","volume-title":"The practitioner's guide to data quality improvement","author":"Loshin David","unstructured":"David Loshin. 2010. The practitioner's guide to data quality improvement. Elsevier."},{"key":"e_1_3_2_1_97_1","volume-title":"Research dilemmas: Paradigms, methods and methodology. Issues in educational research 16, 2","author":"Mackenzie Noella","year":"2006","unstructured":"Noella Mackenzie and Sally Knipe. 2006. Research dilemmas: Paradigms, methods and methodology. Issues in educational research 16, 2 (2006), 193--205."},{"key":"e_1_3_2_1_98_1","doi-asserted-by":"publisher","DOI":"10.1145\/3287560.3287596"},{"key":"e_1_3_2_1_99_1","doi-asserted-by":"publisher","DOI":"10.1177\/2053951716679679"},{"key":"e_1_3_2_1_100_1","volume-title":"Playing Atari with deep reinforcement learning. arXiv preprint arXiv:1312.5602","author":"Mnih Volodymyr","year":"2013","unstructured":"Volodymyr Mnih, Koray Kavukcuoglu, David Silver, Alex Graves, Ioannis Antonoglou, Daan Wierstra, and Martin Riedmiller. 2013. Playing Atari with deep reinforcement learning. arXiv preprint arXiv:1312.5602 (2013)."},{"key":"e_1_3_2_1_101_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D19-1625"},{"key":"e_1_3_2_1_102_1","volume-title":"Shaping Our Tools: Contestability as a Means to Promote Responsible Algorithmic Decision Making in the Professions. Available at SSRN 3311894","author":"Mulligan Deirdre K","year":"2019","unstructured":"Deirdre K Mulligan, Daniel Kluttz, and Nitin Kohli. 2019. Shaping Our Tools: Contestability as a Means to Promote Responsible Algorithmic Decision Making in the Professions. Available at SSRN 3311894 (2019)."},{"key":"e_1_3_2_1_103_1","volume-title":"The view from nowhere","author":"Nagel Thomas","unstructured":"Thomas Nagel. 1989. The view from nowhere. Oxford University Press."},{"key":"e_1_3_2_1_104_1","volume-title":"Critique and contribute: A practice-based framework for improving critical data studies and data science. Big data 5, 2","author":"Neff Gina","year":"2017","unstructured":"Gina Neff, Anissa Tanweer, Brittany Fiore-Gartland, and Laura Osburn. 2017. Critique and contribute: A practice-based framework for improving critical data studies and data science. Big data 5, 2 (2017), 85--97."},{"key":"e_1_3_2_1_105_1","volume-title":"Accountability ina computerized society. Science and engineering ethics 2, 1","author":"Nissenbaum Helen","year":"1996","unstructured":"Helen Nissenbaum. 1996. Accountability ina computerized society. Science and engineering ethics 2, 1 (1996), 25--42."},{"key":"e_1_3_2_1_106_1","volume-title":"The art of unit testing","author":"Osherove Roy","unstructured":"Roy Osherove. 2015. The art of unit testing. MITP-Verlags GmbH & Co. KG."},{"key":"e_1_3_2_1_107_1","doi-asserted-by":"publisher","DOI":"10.5334\/dsj-2017-008"},{"key":"e_1_3_2_1_108_1","doi-asserted-by":"publisher","DOI":"10.1145\/3287560.3287567"},{"key":"e_1_3_2_1_109_1","unstructured":"Ron Patton. 2006. Software testing. Pearson Education India."},{"key":"e_1_3_2_1_110_1","unstructured":"Tomas Petricek. 2019. Cultures of programming. (2019). unpublished."},{"key":"e_1_3_2_1_111_1","doi-asserted-by":"publisher","DOI":"10.7717\/peerj.175"},{"key":"e_1_3_2_1_112_1","doi-asserted-by":"publisher","DOI":"10.1145\/3299887.3299891"},{"key":"e_1_3_2_1_113_1","volume-title":"Large image datasets: A pyrrhic win for computer vision? arXiv preprint arXiv:2006.16923","author":"Prabhu Vinay Uday","year":"2020","unstructured":"Vinay Uday Prabhu and Abeba Birhane. 2020. Large image datasets: A pyrrhic win for computer vision? arXiv preprint arXiv:2006.16923 (2020)."},{"key":"e_1_3_2_1_114_1","doi-asserted-by":"publisher","DOI":"10.1145\/3351095.3372873"},{"key":"e_1_3_2_1_115_1","volume-title":"ABOUT ML: Annotation and Benchmarking on Understanding and Transparency of Machine Learning Lifecycles. arXiv preprint arXiv:1912.06166","author":"Raji Inioluwa Deborah","year":"2019","unstructured":"Inioluwa Deborah Raji and Jingying Yang. 2019. ABOUT ML: Annotation and Benchmarking on Understanding and Transparency of Machine Learning Lifecycles. arXiv preprint arXiv:1912.06166 (2019)."},{"key":"e_1_3_2_1_116_1","unstructured":"Ari Ramkilowan. 2018. The rise and rise of AI in Africa. https:\/\/medium.com\/datadriveninvestor\/the-rise-and-rise-of-ai-in-africa-a6cf6bf89217. Accessed: 2020-09-26."},{"key":"e_1_3_2_1_117_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D13-1020"},{"key":"e_1_3_2_1_118_1","first-page":"15","article-title":"Dirty data, bad predictions: How civil rights violations impact police data, predictive policing systems, and justice","volume":"94","author":"Richardson Rashida","year":"2019","unstructured":"Rashida Richardson, Jason M Schultz, and Kate Crawford. 2019. Dirty data, bad predictions: How civil rights violations impact police data, predictive policing systems, and justice. NYUL Rev. Online 94 (2019), 15.","journal-title":"NYUL Rev. Online"},{"key":"e_1_3_2_1_119_1","volume-title":"Dilemmas in a general theory of planning. Policy sciences 4, 2","author":"Rittel Horst WJ","year":"1973","unstructured":"Horst WJ Rittel and Melvin M Webber. 1973. Dilemmas in a general theory of planning. Policy sciences 4, 2 (1973), 155--169."},{"key":"e_1_3_2_1_120_1","volume-title":"A survey on data collection for machine learning: a big data-AI integration perspective","author":"Roh Yuji","year":"2019","unstructured":"Yuji Roh, Geon Heo, and Steven Euijong Whang. 2019. A survey on data collection for machine learning: a big data-AI integration perspective. IEEE Transactions on Knowledge and Data Engineering (2019)."},{"key":"e_1_3_2_1_121_1","volume-title":"Cynthia Mathis Beath, and Dale L Goodhue","author":"Ross Jeanne W","year":"1996","unstructured":"Jeanne W Ross, Cynthia Mathis Beath, and Dale L Goodhue. 1996. Develop long-term competitiveness through IT assets. Sloan management review 38, 1 (1996), 31--42."},{"key":"e_1_3_2_1_122_1","volume-title":"Proceedings of the Aristotelian society","volume":"46","author":"Ryle Gilbert","year":"1945","unstructured":"Gilbert Ryle. 1945. Knowing how and knowing that: The presidential address. In Proceedings of the Aristotelian society, Vol. 46. JSTOR, 1--16."},{"key":"e_1_3_2_1_123_1","volume-title":"The algorithm at work? Explanation and repair in the enactment of similarity in art data. Information","author":"Sachs SE","year":"2019","unstructured":"SE Sachs. 2019. The algorithm at work? Explanation and repair in the enactment of similarity in art data. Information, Communication & Society (2019), 1--17."},{"key":"e_1_3_2_1_124_1","doi-asserted-by":"publisher","DOI":"10.1145\/3411764.3445518"},{"key":"e_1_3_2_1_125_1","doi-asserted-by":"publisher","DOI":"10.1145\/3392866"},{"key":"e_1_3_2_1_126_1","unstructured":"David Sculley Gary Holt Daniel Golovin Eugene Davydov Todd Phillips Dietmar Ebner Vinay Chaudhary Michael Young Jean-Francois Crespo and Dan Dennison. 2015. Hidden technical debt in machine learning systems. In Advances in neural information processing systems. 2503--2511."},{"key":"e_1_3_2_1_127_1","unstructured":"David Sculley Jasper Snoek Alex Wiltschko and Ali Rahimi. 2018. Winner's curse? On pace progress and empirical rigor. (2018)."},{"key":"e_1_3_2_1_128_1","doi-asserted-by":"publisher","DOI":"10.1145\/3287560.3287598"},{"key":"e_1_3_2_1_129_1","unstructured":"Burr Settles. 2009. Active learning literature survey. Technical Report. University of Wisconsin-Madison Department of Computer Sciences."},{"key":"e_1_3_2_1_130_1","volume-title":"No classification without representation: Assessing geodiversity issues in open data sets for the developing world. arXiv preprint arXiv:1711.08536","author":"Shankar Shreya","year":"2017","unstructured":"Shreya Shankar, Yoni Halpern, Eric Breck, James Atwood, Jimbo Wilson, and D Sculley. 2017. No classification without representation: Assessing geodiversity issues in open data sets for the developing world. arXiv preprint arXiv:1711.08536 (2017)."},{"key":"e_1_3_2_1_131_1","doi-asserted-by":"publisher","DOI":"10.1086\/659003"},{"key":"e_1_3_2_1_132_1","volume-title":"Responsibility from the Margins","author":"Shoemaker David","unstructured":"David Shoemaker. 2015. Responsibility from the Margins. Oxford University Press, USA."},{"key":"e_1_3_2_1_133_1","volume-title":"The structure of ill structured problems. Artificial intelligence 4, 3-4","author":"Simon Herbert A","year":"1973","unstructured":"Herbert A Simon. 1973. The structure of ill structured problems. Artificial intelligence 4, 3-4 (1973), 181--201."},{"key":"e_1_3_2_1_134_1","doi-asserted-by":"publisher","DOI":"10.1145\/3375627.3375866"},{"key":"e_1_3_2_1_135_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.acl-main.446"},{"key":"e_1_3_2_1_136_1","doi-asserted-by":"publisher","DOI":"10.5334\/jors.ay"},{"key":"e_1_3_2_1_137_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2017.97"},{"key":"e_1_3_2_1_138_1","unstructured":"Herb Sutter and Andrei Alexandrescu. 2004. C++ coding standards: 101 rules guidelines and best practices. Pearson Education."},{"key":"e_1_3_2_1_139_1","volume-title":"The automation charade. Logic Magazine","author":"Taylor Astra","year":"2018","unstructured":"Astra Taylor. 2018. The automation charade. Logic Magazine (2018)."},{"key":"e_1_3_2_1_140_1","first-page":"1","article-title":"Making Better Use of the Crowd: How Crowdsourcing Can Advance Machine Learning Research","volume":"18","author":"Vaughan Jennifer Wortman","year":"2018","unstructured":"Jennifer Wortman Vaughan. 2018. Making Better Use of the Crowd: How Crowdsourcing Can Advance Machine Learning Research. Journal of Machine Learning Research 18, 193 (2018), 1--46. http:\/\/jmlr.org\/papers\/v18\/17-234.html","journal-title":"Journal of Machine Learning Research"},{"key":"e_1_3_2_1_141_1","doi-asserted-by":"publisher","DOI":"10.1109\/REW.2019.00050"},{"key":"e_1_3_2_1_142_1","volume-title":"Artificial intelligence and the value of transparency. AI & SOCIETY","author":"Walmsley Joel","year":"2020","unstructured":"Joel Walmsley. 2020. Artificial intelligence and the value of transparency. AI & SOCIETY (2020), 1--11."},{"key":"e_1_3_2_1_143_1","volume-title":"Infrastructure governance as political choice. The governance of infrastructure","author":"Wegrich Kai","year":"2017","unstructured":"Kai Wegrich and Gerhard Hammerschmid. 2017. Infrastructure governance as political choice. The governance of infrastructure (2017), 21--42."},{"key":"e_1_3_2_1_144_1","volume-title":"Simulation and similarity: Using models to understand the world","author":"Weisberg Michael","unstructured":"Michael Weisberg. 2012. Simulation and similarity: Using models to understand the world. Oxford University Press."},{"key":"e_1_3_2_1_145_1","volume-title":"Metrology for AI: From Benchmarks to Instruments. arXiv preprint arXiv:1911.01875","author":"Welty Chris","year":"2019","unstructured":"Chris Welty, Praveen Paritosh, and Lora Aroyo. 2019. Metrology for AI: From Benchmarks to Instruments. arXiv preprint arXiv:1911.01875 (2019)."},{"key":"e_1_3_2_1_146_1","doi-asserted-by":"publisher","DOI":"10.1145\/3351095.3372833"},{"key":"e_1_3_2_1_147_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1"},{"key":"e_1_3_2_1_148_1","volume-title":"Do artifacts have politics? Daedalus","author":"Winner Langdon","year":"1980","unstructured":"Langdon Winner. 1980. Do artifacts have politics? Daedalus (1980), 121--136."},{"key":"e_1_3_2_1_149_1","volume-title":"Corpus creation. Handbook of Natural Language Processing (2n Revised edition)","author":"Xiao Richard","year":"2010","unstructured":"Richard Xiao. 2010. Corpus creation. Handbook of Natural Language Processing (2n Revised edition) (2010), 147--165."},{"key":"e_1_3_2_1_150_1","volume-title":"RecipeQA: A challenge dataset for multimodal comprehension of cooking recipes. arXiv preprint arXiv:1809.00812","author":"Yagcioglu Semih","year":"2018","unstructured":"Semih Yagcioglu, Aykut Erdem, Erkut Erdem, and Nazli Ikizler-Cinbis. 2018. RecipeQA: A challenge dataset for multimodal comprehension of cooking recipes. arXiv preprint arXiv:1809.00812 (2018)."},{"key":"e_1_3_2_1_151_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1"},{"issue":"4","key":"e_1_3_2_1_152_1","first-page":"39","article-title":"Accelerating the Machine Learning Lifecycle with MLflow","volume":"41","author":"Zaharia Matei","year":"2018","unstructured":"Matei Zaharia, Andrew Chen, Aaron Davidson, Ali Ghodsi, Sue Ann Hong, Andy Konwinski, Siddharth Murching, Tomas Nykodym, Paul Ogilvie, Mani Parkhe, Fen Xie, and Corey Zumaret. 2018. Accelerating the Machine Learning Lifecycle with MLflow. In IEEE Data Eng. Bull. 41, no. 4. 39--45.","journal-title":"IEEE Data Eng. Bull."},{"key":"e_1_3_2_1_153_1","volume-title":"Swag: A large-scale adversarial dataset for grounded commonsense inference. arXiv preprint arXiv:1808.05326","author":"Zellers Rowan","year":"2018","unstructured":"Rowan Zellers, Yonatan Bisk, Roy Schwartz, and Yejin Choi. 2018. Swag: A large-scale adversarial dataset for grounded commonsense inference. arXiv preprint arXiv:1808.05326 (2018)."},{"key":"e_1_3_2_1_154_1","volume-title":"Men also like shopping: Reducing gender bias amplification using corpus-level constraints. arXiv preprint arXiv:1707.09457","author":"Zhao Jieyu","year":"2017","unstructured":"Jieyu Zhao, Tianlu Wang, Mark Yatskar, Vicente Ordonez, and Kai-Wei Chang. R@2017. Men also like shopping: Reducing gender bias amplification using corpus-level constraints. arXiv preprint arXiv:1707.09457 (2017)."},{"key":"e_1_3_2_1_155_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.jss.2014.09.042"},{"key":"e_1_3_2_1_156_1","volume-title":"Alyssa Goodman, Rachelle Hollander, Barbara A Koenig, Jacob Metcalf, et al.","author":"Zook Matthew","year":"2017","unstructured":"Matthew Zook, Solon Barocas, Danah Boyd, Kate Crawford, Emily Keller, Seeta Pe\u00f1a Gangadharan, Alyssa Goodman, Rachelle Hollander, Barbara A Koenig, Jacob Metcalf, et al. 2017. Ten simple rules for responsible big data research."}],"event":{"name":"FAccT '21: 2021 ACM Conference on Fairness, Accountability, and Transparency","location":"Virtual Event Canada","acronym":"FAccT '21","sponsor":["ACM Association for Computing Machinery"]},"container-title":["Proceedings of the 2021 ACM Conference on Fairness, Accountability, and Transparency"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3442188.3445918","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3442188.3445918","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,8,18]],"date-time":"2025-08-18T17:33:22Z","timestamp":1755538402000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3442188.3445918"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2021,3]]},"references-count":156,"alternative-id":["10.1145\/3442188.3445918","10.1145\/3442188"],"URL":"https:\/\/doi.org\/10.1145\/3442188.3445918","relation":{},"subject":[],"published":{"date-parts":[[2021,3]]},"assertion":[{"value":"2021-03-01","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}