{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,13]],"date-time":"2026-04-13T21:41:00Z","timestamp":1776116460693,"version":"3.50.1"},"publisher-location":"New York, NY, USA","reference-count":127,"publisher":"ACM","license":[{"start":{"date-parts":[[2022,6,20]],"date-time":"2022-06-20T00:00:00Z","timestamp":1655683200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"funder":[{"DOI":"10.13039\/100000001","name":"National Science Foundation","doi-asserted-by":"publisher","award":["1752814"],"award-info":[{"award-number":["1752814"]}],"id":[{"id":"10.13039\/100000001","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2022,6,21]]},"DOI":"10.1145\/3531146.3533228","type":"proceedings-article","created":{"date-parts":[[2022,6,20]],"date-time":"2022-06-20T14:27:10Z","timestamp":1655735230000},"page":"1733-1746","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":3,"title":["Adversarial Scrutiny of Evidentiary Statistical Software"],"prefix":"10.1145","author":[{"given":"Rediet","family":"Abebe","sequence":"first","affiliation":[{"name":"Department of Electrical Engineering and Computer Sciences, University of California, Berkeley, USA"}]},{"given":"Moritz","family":"Hardt","sequence":"additional","affiliation":[{"name":"Max Planck Institute for Intelligent Systems, Germany"}]},{"given":"Angela","family":"Jin","sequence":"additional","affiliation":[{"name":"Department of Electrical Engineering and Computer Sciences, University of California, Berkeley, USA"}]},{"given":"John","family":"Miller","sequence":"additional","affiliation":[{"name":"Department of Electrical Engineering and Computer Sciences, University of California, Berkeley, USA"}]},{"given":"Ludwig","family":"Schmidt","sequence":"additional","affiliation":[{"name":"Paul G. Allen School of Computer Science, University of Washington, USA"}]},{"given":"Rebecca","family":"Wexler","sequence":"additional","affiliation":[{"name":"School of Law, University of California, Berkeley, USA"}]}],"member":"320","published-online":{"date-parts":[[2022,6,20]]},"reference":[{"key":"e_1_3_2_1_1_1","unstructured":"2020. Survey Shows STRmix Has Been Used in 220 000 Cases Worldwide. https:\/\/www.strmix.com\/news\/survey-shows-strmix-has-been-used-in-220000-cases-worldwide\/. Accessed: 2022-01-21."},{"key":"e_1_3_2_1_2_1","volume-title":"117th Congress (2021-2022): Justice in Forensic Algorithms Act of 2021. Congress.gov, Library of Congress. https:\/\/www.congress.gov\/bill\/117th-congress\/house-bill\/2438\/titles.","unstructured":"2021. H.R.2438 - 117th Congress (2021-2022): Justice in Forensic Algorithms Act of 2021. Congress.gov, Library of Congress. https:\/\/www.congress.gov\/bill\/117th-congress\/house-bill\/2438\/titles."},{"key":"e_1_3_2_1_3_1","unstructured":"Fed. R. Evid. 401."},{"key":"e_1_3_2_1_4_1","unstructured":"Fed. R. Evid. 402."},{"key":"e_1_3_2_1_5_1","unstructured":"Fed. R. Evid. 403."},{"key":"e_1_3_2_1_6_1","unstructured":"Fed. R. Evid. 702."},{"key":"e_1_3_2_1_7_1","unstructured":"Fed. R. Evid. 902."},{"key":"e_1_3_2_1_8_1","unstructured":"U.S. Const. Amend. VI."},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1145\/3351095.3372871"},{"key":"e_1_3_2_1_10_1","unstructured":"[10] Ake v. Oklahoma 470 U.S. 68.(1985)."},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.fsigen.2017.10.006"},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.1086\/508269"},{"key":"e_1_3_2_1_13_1","volume-title":"ProPublica","author":"Angwin Julia","year":"2016","unstructured":"Julia Angwin, Jeff Larson, Surya Mattu, and Lauren Kirchner. 2016. Machine bias. ProPublica, May 23, 2016 (2016), 139\u2013159."},{"key":"e_1_3_2_1_14_1","first-page":"1","article-title":"Seeking the Source: Criminal Defendants\u2019 Constitutional Right to Source Code","volume":"17","author":"Bellovin M","year":"2021","unstructured":"Steven\u00a0M Bellovin, Matt Blaze, Susan Landau, and Brian Owsley. 2021. Seeking the Source: Criminal Defendants\u2019 Constitutional Right to Source Code. Ohio State Technology Law Journal 17 (2021), 1.","journal-title":"Ohio State Technology Law Journal"},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.1515\/9781400831050"},{"key":"e_1_3_2_1_16_1","first-page":"1","article-title":"Defending Daubert: it\u2019s time to amend federal rule of evidence 702","volume":"57","author":"Bernstein E","year":"2015","unstructured":"David\u00a0E Bernstein and Eric\u00a0G Lasker. 2015. Defending Daubert: it\u2019s time to amend federal rule of evidence 702. William & Mary Law Review 57 (2015), 1.","journal-title":"William & Mary Law Review"},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.1145\/3243734.3264418"},{"key":"e_1_3_2_1_18_1","unstructured":"[18] Bullcoming v. New Mexico 564 U.S. 647.(2011)."},{"key":"e_1_3_2_1_19_1","volume-title":"Conference on fairness, accountability and transparency. PMLR, 77\u201391","author":"Buolamwini Joy","year":"2018","unstructured":"Joy Buolamwini and Timnit Gebru. 2018. Gender shades: Intersectional accuracy disparities in commercial gender classification. In Conference on fairness, accountability and transparency. PMLR, 77\u201391."},{"key":"e_1_3_2_1_20_1","volume-title":"DNA Mixture Interpretation: A NIST Scientific Foundation Review. National Institutes of Standards and Technology","author":"Butler M","year":"2021","unstructured":"John\u00a0M Butler, Hari Iyer, Rich Press, Melissa Taylor, Peter\u00a0M Vallone, and Sheila Willis. 2021. DNA Mixture Interpretation: A NIST Scientific Foundation Review. National Institutes of Standards and Technology (2021)."},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.1109\/MC.2020.3038630"},{"key":"e_1_3_2_1_22_1","unstructured":"Cellebrite Inc.[n. d.]. Cellebrite UFED. https:\/\/cellebrite.com\/en\/ufed. Accessed: 2022-05-31."},{"key":"e_1_3_2_1_23_1","first-page":"2081","article-title":"A practical solution to the reference class problem","volume":"109","author":"Cheng K.","year":"2009","unstructured":"Edward\u00a0K. Cheng. 2009. A practical solution to the reference class problem. Columbia Law Review 109(2009), 2081.","journal-title":"Columbia Law Review"},{"key":"e_1_3_2_1_24_1","unstructured":"Edward\u00a0K Cheng David\u00a0L Faigman Jennifer\u00a0L Mnookin Erin\u00a0E Murphy Joseph Sanders and Christopher Slobogin. 2021. Modern scientific evidence: the law and science of expert testimony. Thomson West."},{"key":"e_1_3_2_1_25_1","first-page":"1077","article-title":"Beyond the witness: Bringing a process perspective to modern evidence law","volume":"97","author":"Cheng K","year":"2018","unstructured":"Edward\u00a0K Cheng and G\u00a0Alexander Nunn. 2018. Beyond the witness: Bringing a process perspective to modern evidence law. Texas Law Review 97(2018), 1077.","journal-title":"Texas Law Review"},{"key":"e_1_3_2_1_26_1","volume-title":"Does Frye or Daubert matter? A study of scientific admissibility standards. Virginia Law Review","author":"Cheng K","year":"2005","unstructured":"Edward\u00a0K Cheng and Albert\u00a0H Yoon. 2005. Does Frye or Daubert matter? A study of scientific admissibility standards. Virginia Law Review (2005), 471\u2013513."},{"key":"e_1_3_2_1_27_1","volume-title":"Source","author":"Chessman Christian","year":"2017","unstructured":"Christian Chessman. 2017. A\u201d Source\u201d of Error: Computer Code, Criminal Defendants, and the Constitution. California Law Review(2017), 179\u2013228."},{"key":"e_1_3_2_1_28_1","volume-title":"Fair prediction with disparate impact: A study of bias in recidivism prediction instruments. Big data 5, 2","author":"Chouldechova Alexandra","year":"2017","unstructured":"Alexandra Chouldechova. 2017. Fair prediction with disparate impact: A study of bias in recidivism prediction instruments. Big data 5, 2 (2017), 153\u2013163."},{"key":"e_1_3_2_1_29_1","unstructured":"CMI Inc.[n. d.]. Intoxilyzer 8000. https:\/\/www.alcoholtest.com\/intoxilyzer-8000\/. Accessed: 2022-05-31."},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.fsigen.2018.11.009"},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"crossref","unstructured":"A\u00a0Feder Cooper Benjamin Laufer Emanuel Moss and Helen Nissenbaum. 2022. Accountability in an Algorithmic Society: Relationality Responsibility and Robustness in Machine Learning. arXiv preprint arXiv:2202.05338(2022).","DOI":"10.1145\/3531146.3533150"},{"key":"e_1_3_2_1_32_1","volume-title":"Hyperparameter Optimization Is Deceiving Us, and How to Stop It. Advances in Neural Information Processing Systems 34","author":"Cooper A\u00a0Feder","year":"2021","unstructured":"A\u00a0Feder Cooper, Yucheng Lu, Jessica Forde, and Christopher\u00a0M De\u00a0Sa. 2021. Hyperparameter Optimization Is Deceiving Us, and How to Stop It. Advances in Neural Information Processing Systems 34 (2021)."},{"key":"e_1_3_2_1_33_1","unstructured":"Sam Corbett-Davies and Sharad Goel. 2018. The measure and mismeasure of fairness: A critical review of fair machine learning. arXiv preprint arXiv:1808.00023(2018)."},{"key":"e_1_3_2_1_34_1","volume-title":"Reference Manual on Scientific Evidence","author":"National\u00a0Research Council","unstructured":"National\u00a0Research Council. 2011. Reference Manual on Scientific Evidence: Third Edition. The National Academies Press. https:\/\/nap.nationalacademies.org\/catalog\/13163\/reference-manual-on-scientific-evidence-third-edition"},{"key":"e_1_3_2_1_35_1","unstructured":"Cybergenetics. [n. d.]. TrueAllele. https:\/\/www.cybgen.com. Accessed: 2022-05-31."},{"key":"e_1_3_2_1_36_1","volume-title":"Merrell Dow Pharmaceuticals","author":"Daubert","year":"1993","unstructured":"[36] Daubert v. Merrell Dow Pharmaceuticals, Inc., 509 U.S. 579.(1993)."},{"key":"e_1_3_2_1_37_1","doi-asserted-by":"publisher","DOI":"10.1145\/3512898"},{"key":"e_1_3_2_1_38_1","unstructured":"Dr\u00e4ger. [n. d.]. Dr\u00e4ger Alcotest 9510. https:\/\/www.draeger.com\/en-us_us\/Products\/Alcotest-9510. Accessed: 2022-05-31."},{"key":"e_1_3_2_1_39_1","volume-title":"The accuracy, fairness, and limits of predicting recidivism. Science Advances 4, 1","author":"Dressel Julia","year":"2018","unstructured":"Julia Dressel and Hany Farid. 2018. The accuracy, fairness, and limits of predicting recidivism. Science Advances 4, 1 (2018), eaao5580."},{"key":"e_1_3_2_1_40_1","doi-asserted-by":"publisher","DOI":"10.1145\/3406325.3451064"},{"key":"e_1_3_2_1_41_1","doi-asserted-by":"publisher","DOI":"10.1177\/0093854818811379"},{"key":"e_1_3_2_1_42_1","first-page":"1001","article-title":"Legal Tech, Civil Procedure, and the Future of Adversarialism","volume":"169","author":"Engstrom David\u00a0Freeman","year":"2020","unstructured":"David\u00a0Freeman Engstrom and Jonah\u00a0B Gelbach. 2020. Legal Tech, Civil Procedure, and the Future of Adversarialism. University of Pennsylvania Law Review 169 (2020), 1001.","journal-title":"University of Pennsylvania Law Review"},{"key":"e_1_3_2_1_43_1","first-page":"417","article-title":"Group to Individual (G2i) Inference in Scientific Expert Testimony","volume":"81","author":"Faigman L.","year":"2014","unstructured":"David\u00a0L. Faigman, John Monahan, and Christopher Slobogin. 2014. Group to Individual (G2i) Inference in Scientific Expert Testimony. The University of Chicago Law Review 81, 2 (2014), 417\u2013480.","journal-title":"The University of Chicago Law Review"},{"key":"e_1_3_2_1_44_1","unstructured":"[44] Frye v. U.S. 315 F.2d 491.(1963)."},{"key":"e_1_3_2_1_45_1","first-page":"1147","article-title":"Constitutional regulation of forensic evidence","volume":"73","author":"Garrett L","year":"2016","unstructured":"Brandon\u00a0L Garrett. 2016. Constitutional regulation of forensic evidence. Washington & Lee Law Review 73 (2016), 1147.","journal-title":"Washington & Lee Law Review"},{"key":"e_1_3_2_1_46_1","volume-title":"Autopsy of a Crime Lab","author":"Garrett L","unstructured":"Brandon\u00a0L Garrett. 2021. Autopsy of a Crime Lab. University of California Press, 189."},{"key":"e_1_3_2_1_47_1","volume-title":"Autopsy of a Crime Lab","author":"Garrett L","unstructured":"Brandon\u00a0L Garrett. 2021. Autopsy of a Crime Lab. University of California Press, 192."},{"key":"e_1_3_2_1_48_1","first-page":"1559","article-title":"The Myth of the Reliability Test","volume":"86","author":"Garrett L","year":"2017","unstructured":"Brandon\u00a0L Garrett and M\u00a0Chris Fabricant. 2017. The Myth of the Reliability Test. Fordham Law Review 86(2017), 1559.","journal-title":"Fordham Law Review"},{"key":"e_1_3_2_1_49_1","volume-title":"Invalid forensic science testimony and wrongful convictions. Virginia Law Review","author":"Garrett L","year":"2009","unstructured":"Brandon\u00a0L Garrett and Peter\u00a0J Neufeld. 2009. Invalid forensic science testimony and wrongful convictions. Virginia Law Review (2009), 1\u201397."},{"key":"e_1_3_2_1_50_1","first-page":"789","article-title":"Commentary on the Need for a Research Culture in the Forensic Sciences","volume":"58","author":"Gertner Nancy","year":"2010","unstructured":"Nancy Gertner. 2010. Commentary on the Need for a Research Culture in the Forensic Sciences. UCLA Law Review 58(2010), 789.","journal-title":"UCLA Law Review"},{"key":"e_1_3_2_1_51_1","first-page":"791","article-title":"Criminal discovery, scientific evidence, and DNA","volume":"44","author":"Giannelli C","year":"1991","unstructured":"Paul\u00a0C Giannelli. 1991. Criminal discovery, scientific evidence, and DNA. Vanderbilt Law Review 44(1991), 791.","journal-title":"Vanderbilt Law Review"},{"key":"e_1_3_2_1_52_1","first-page":"1305","article-title":"Ake v. Oklahoma: The right to expert assistance in a post-Daubert, post-DNA world","volume":"89","author":"Giannelli C","year":"2003","unstructured":"Paul\u00a0C Giannelli. 2003. Ake v. Oklahoma: The right to expert assistance in a post-Daubert, post-DNA world. Cornell Law Review 89(2003), 1305.","journal-title":"Cornell Law Review"},{"key":"e_1_3_2_1_53_1","unstructured":"Grayshift. [n. d.]. GrayKey Cell Phone Forensics Tool. https:\/\/www.grayshift.com\/graykey\/. Accessed: 2022-05-31."},{"key":"e_1_3_2_1_54_1","volume-title":"Public Policy, and Law 8, 4","author":"Groscup L","year":"2002","unstructured":"Jennifer\u00a0L Groscup, Steven\u00a0D Penrod, Christina\u00a0A Studebaker, Matthew\u00a0T Huss, and Kevin\u00a0M O\u2019Neil. 2002. The effects of Daubert on the admissibility of expert testimony in state and federal criminal cases.Psychology, Public Policy, and Law 8, 4 (2002), 339."},{"key":"e_1_3_2_1_55_1","unstructured":"Moritz Hardt and Benjamin Recht. 2021. Patterns predictions and actions: A story about machine learning. https:\/\/mlstory.org. arxiv:2102.05242\u00a0[cs.LG]"},{"key":"e_1_3_2_1_56_1","volume-title":"Proceedings of the 35th International Conference on Machine Learning(Proceedings of Machine Learning Research, Vol.\u00a080)","author":"Hebert-Johnson Ursula","year":"2018","unstructured":"Ursula Hebert-Johnson, Michael Kim, Omer Reingold, and Guy Rothblum. 2018. Multicalibration: Calibration for the (Computationally-Identifiable) Masses. In Proceedings of the 35th International Conference on Machine Learning(Proceedings of Machine Learning Research, Vol.\u00a080), Jennifer Dy and Andreas Krause (Eds.). PMLR, 1939\u20131948."},{"key":"e_1_3_2_1_57_1","first-page":"759","article-title":"The disappointing history of science in the courtroom: Frye, Daubert, and the ongoing crisis of junk science in criminal trials","volume":"71","author":"Hilbert Jim","year":"2018","unstructured":"Jim Hilbert. 2018. The disappointing history of science in the courtroom: Frye, Daubert, and the ongoing crisis of junk science in criminal trials. Oklahoma Law Review 71(2018), 759.","journal-title":"Oklahoma Law Review"},{"key":"e_1_3_2_1_58_1","volume-title":"Forensic Technology: Algorithms Strengthen Forensic Analysis, but Several Factors Can Affect Outcomes. GAO Report No. GAO-21-435SP. https:\/\/www.gao.gov\/assets\/gao-21-435sp.pdf","author":"Howard L","year":"2021","unstructured":"Karen\u00a0L Howard. 2021. Forensic Technology: Algorithms Strengthen Forensic Analysis, but Several Factors Can Affect Outcomes. GAO Report No. GAO-21-435SP. https:\/\/www.gao.gov\/assets\/gao-21-435sp.pdf"},{"key":"e_1_3_2_1_59_1","first-page":"1043","article-title":"Racial equity in algorithmic criminal justice","volume":"68","author":"Huq Z","year":"2018","unstructured":"Aziz\u00a0Z Huq. 2018. Racial equity in algorithmic criminal justice. Duke LJ 68(2018), 1043.","journal-title":"Duke LJ"},{"key":"e_1_3_2_1_60_1","first-page":"97","article-title":"Computer source code: a source of the growing controversy over the reliability of automated forensic techniques","volume":"66","author":"Imwinkelried J","year":"2016","unstructured":"Edward\u00a0J Imwinkelried. 2016. Computer source code: a source of the growing controversy over the reliability of automated forensic techniques. DePaul Law Review 66(2016), 97.","journal-title":"DePaul Law Review"},{"key":"e_1_3_2_1_61_1","unstructured":"Institute of Environmental Science and Research Ltd.[n. d.]. STRMix. https:\/\/www.strmix.com\/strmix\/. Accessed: 2022-05-31."},{"key":"e_1_3_2_1_62_1","doi-asserted-by":"crossref","unstructured":"Sonia Katyal and Charles Graves. 2021. From Trade Secrecy to Seclusion. Georgetown Law Journal Forthcoming(2021).","DOI":"10.2139\/ssrn.3760123"},{"key":"e_1_3_2_1_63_1","volume-title":"The new Wigmore: a treatise on evidence: expert evidence","author":"Kaye H","unstructured":"David\u00a0H Kaye. 2011. The new Wigmore: a treatise on evidence: expert evidence. Wolters Kluwer Law & Business."},{"key":"e_1_3_2_1_64_1","first-page":"1639","article-title":"How Daubert and its progeny have failed criminalistics evidence and a few things the judiciary could do about it","volume":"86","author":"Kaye H","year":"2017","unstructured":"David\u00a0H Kaye. 2017. How Daubert and its progeny have failed criminalistics evidence and a few things the judiciary could do about it. Fordham Law Review 86(2017), 1639.","journal-title":"Fordham Law Review"},{"key":"e_1_3_2_1_65_1","unstructured":"Danielle\u00a0Leah Kehl and Samuel\u00a0Ari Kessler. 2017. Algorithms in the criminal justice system: Assessing the use of risk assessments in sentencing. (2017)."},{"key":"e_1_3_2_1_66_1","unstructured":"Lauren Kirchner. 2017. Thousands of Criminal Cases in New York Relied on Disputed DNA Testing Techniques. https:\/\/www.propublica.org\/article\/thousands-of-criminal-cases-in-new-york-relied-on-disputed-dna-testing-techniques. Accessed: 2022-04-17."},{"key":"e_1_3_2_1_67_1","volume-title":"Powerful DNA Software Used in Hundreds of Criminal Cases Faces New Scrutiny. The Markup (Mar","author":"Kirchner Lauren","year":"2021","unstructured":"Lauren Kirchner. 2021. Powerful DNA Software Used in Hundreds of Criminal Cases Faces New Scrutiny. The Markup (Mar 2021). https:\/\/themarkup.org\/news\/2021\/03\/09\/powerful-dna-software-used-in-hundreds-of-criminal-cases-faces-new-scrutiny"},{"key":"e_1_3_2_1_68_1","unstructured":"Jon Kleinberg Sendhil Mullainathan and Manish Raghavan. 2016. Inherent trade-offs in the fair determination of risk scores. arXiv preprint arXiv:1609.05807(2016)."},{"key":"e_1_3_2_1_69_1","doi-asserted-by":"publisher","DOI":"10.1073\/pnas.1915768117"},{"key":"e_1_3_2_1_70_1","first-page":"1725","article-title":"Danger ahead: Risk assessment and the future of bail reform","volume":"93","author":"Koepke John\u00a0Logan","year":"2018","unstructured":"John\u00a0Logan Koepke and David\u00a0G Robinson. 2018. Danger ahead: Risk assessment and the future of bail reform. Washington Law Review. 93 (2018), 1725.","journal-title":"Washington Law Review."},{"key":"e_1_3_2_1_71_1","volume-title":"International Conference on Machine Learning (ICML).","author":"Koh Pang\u00a0Wei","year":"2021","unstructured":"Pang\u00a0Wei Koh, Shiori Sagawa, Henrik Marklund, Sang\u00a0Michael Xie, Marvin Zhang, Akshay Balsubramani, Weihua Hu, Michihiro Yasunaga, Richard\u00a0Lanas Phillips, Irena Gao, Tony Lee, Etienne David, Ian Stavness, Wei Guo, Berton Earnshaw, Imran Haque, Sara\u00a0M. Beery, Jure Leskovec, Anshul Kundaje, Emma Pierson, Sergey Levine, Chelsea Finn, and Percy Liang. 2021. WILDS: A Benchmark of in-the-Wild Distribution Shifts. In International Conference on Machine Learning (ICML)."},{"key":"e_1_3_2_1_72_1","first-page":"633","article-title":"Accountable algorithms","volume":"165","author":"Kroll A","year":"2017","unstructured":"Joshua\u00a0A Kroll, Joanna Huey, Solon Barocas, Edward\u00a0W Felten, Joel\u00a0R Reidenberg, David\u00a0G Robinson, and Harlan Yu. 2017. Accountable algorithms. University of Pennsylvania Law Review 165 (2017), 633.","journal-title":"University of Pennsylvania Law Review"},{"key":"e_1_3_2_1_73_1","first-page":"275","article-title":"The Algorithm says you did it: The use of Black Box Algorithms to analyze complex DNA evidence","volume":"31","author":"Kwong Katherine","year":"2017","unstructured":"Katherine Kwong. 2017. The Algorithm says you did it: The use of Black Box Algorithms to analyze complex DNA evidence. Harv. JL & Tech. 31(2017), 275.","journal-title":"Harv. JL & Tech."},{"key":"e_1_3_2_1_74_1","volume-title":"Opening the black box: Defendants","author":"Lacambra J","year":"2018","unstructured":"Stephanie\u00a0J Lacambra, Jeanna Matthews, and K Walsh. 2018. Opening the black box: Defendants\u2019 rights to confront forensic software. The Champion 5(2018)."},{"key":"e_1_3_2_1_75_1","doi-asserted-by":"publisher","DOI":"10.1109\/5.726791"},{"key":"e_1_3_2_1_76_1","volume-title":"Thirty-fifth Conference on Neural Information Processing Systems Datasets and Benchmarks Track (Round 2).","author":"Liao Thomas","year":"2021","unstructured":"Thomas Liao, Rohan Taori, Inioluwa\u00a0Deborah Raji, and Ludwig Schmidt. 2021. Are We Learning Yet? A Meta Review of Evaluation Failures Across Machine Learning. In Thirty-fifth Conference on Neural Information Processing Systems Datasets and Benchmarks Track (Round 2)."},{"key":"e_1_3_2_1_77_1","unstructured":"Lifeloc Technologies. [n. d.]. LX9 Breathalyzer. https:\/\/lifeloc.com\/lx9. Accessed: 2022-05-31."},{"key":"e_1_3_2_1_78_1","unstructured":"Magnet Forensics. [n. d.]. Magnet AXIOM. https:\/\/www.magnetforensics.com\/products\/magnet-axiom\/. Accessed: 2022-05-31."},{"key":"e_1_3_2_1_79_1","volume-title":"Facing Facts: A case in Florida demonstrates the problems with using facial recognition to identify suspects in low-stakes crimes.Slate (Jan. 25","author":"Mak Aaron","year":"2019","unstructured":"Aaron Mak. 2019 [Online]. Facing Facts: A case in Florida demonstrates the problems with using facial recognition to identify suspects in low-stakes crimes.Slate (Jan. 25, 2019 [Online]). https:\/\/slate.com\/technology\/2019\/01\/facial-recognition-arrest-transparency-willie-allen-lynch.html"},{"key":"e_1_3_2_1_80_1","doi-asserted-by":"publisher","DOI":"10.1145\/3306618.3314279"},{"key":"e_1_3_2_1_81_1","doi-asserted-by":"publisher","DOI":"10.1145\/3375627.3375807"},{"key":"e_1_3_2_1_82_1","first-page":"2218","article-title":"Bias in, bias out","volume":"128","author":"Mayson G","year":"2018","unstructured":"Sandra\u00a0G Mayson. 2018. Bias in, bias out. Yale Law Journal 128(2018), 2218.","journal-title":"Yale Law Journal"},{"key":"e_1_3_2_1_83_1","unstructured":"[83] Melendez-Diaz v. Massachusetts 557 U.S. 305.(2009)."},{"key":"e_1_3_2_1_84_1","first-page":"571","article-title":"Repeat Play Evidence: Jack Weinstein, Pedagogical Devices, Technology, and Evidence","volume":"64","author":"Mnookin L","year":"2014","unstructured":"Jennifer\u00a0L Mnookin. 2014. Repeat Play Evidence: Jack Weinstein, Pedagogical Devices, Technology, and Evidence. DePaul Law Review 64(2014), 571.","journal-title":"DePaul Law Review"},{"key":"e_1_3_2_1_85_1","doi-asserted-by":"publisher","DOI":"10.1162\/daed_a_00523"},{"key":"e_1_3_2_1_86_1","first-page":"725","article-title":"The need for a research culture in the forensic sciences","volume":"58","author":"Mnookin L","year":"2010","unstructured":"Jennifer\u00a0L Mnookin, Simon\u00a0A Cole, Itiel\u00a0E Dror, and Barry\u00a0AJ Fisher. 2010. The need for a research culture in the forensic sciences. UCLA Law Review 58(2010), 725.","journal-title":"UCLA Law Review"},{"key":"e_1_3_2_1_87_1","doi-asserted-by":"publisher","DOI":"10.2307\/1341633"},{"key":"e_1_3_2_1_88_1","volume-title":"An intelligence in our image: The risks of bias and errors in artificial intelligence","author":"Osoba A","unstructured":"Osonde\u00a0A Osoba and William Welser\u00a0IV. 2017. An intelligence in our image: The risks of bias and errors in artificial intelligence. RAND Corporation."},{"key":"e_1_3_2_1_89_1","volume-title":"The black box society","author":"Pasquale Frank","unstructured":"Frank Pasquale. 2015. The black box society. In The Black Box Society. Harvard University Press."},{"key":"e_1_3_2_1_90_1","doi-asserted-by":"publisher","DOI":"10.1145\/3287560.3287567"},{"key":"e_1_3_2_1_91_1","unstructured":"[91] PDQuery.[n. d.]. https:\/\/pdquery.orgAccessed: 2022-05-09."},{"key":"e_1_3_2_1_92_1","doi-asserted-by":"crossref","unstructured":"[92] People v. Lopez 55 Cal.4th 569.(2012).","DOI":"10.1590\/S0011-52582012000300001"},{"key":"e_1_3_2_1_93_1","first-page":"50","article-title":"When DNA is not a gold standard: failing to interpret mixture evidence","volume":"42","author":"Perlin W","year":"2018","unstructured":"Mark\u00a0W Perlin. 2018. When DNA is not a gold standard: failing to interpret mixture evidence. The Champion 42(2018), 50\u20136.","journal-title":"The Champion"},{"key":"e_1_3_2_1_94_1","first-page":"245","article-title":"Deepfakes\u201d in the Courtroom","volume":"29","author":"Pfefferkorn Riana","year":"2019","unstructured":"Riana Pfefferkorn. 2019. \u201cDeepfakes\u201d in the Courtroom. Boston University Public Interest Law Journal 29 (2019), 245.","journal-title":"Boston University Public Interest Law Journal"},{"key":"e_1_3_2_1_95_1","volume-title":"Forensic Science in Criminal Courts: Ensuring Scientific Validation of Feature-Comparison Methods","author":"President\u2019s Council of Advisors on Science and Technology. 2016.","year":"2016","unstructured":"President\u2019s Council of Advisors on Science and Technology. 2016. Forensic Science in Criminal Courts: Ensuring Scientific Validation of Feature-Comparison Methods. President\u2019s Council of Advisors on Science and Technology (2016)."},{"key":"e_1_3_2_1_96_1","volume-title":"Addendum to the PCAST Report on Forensic Science in Criminal Courts","author":"President\u2019s Council of Advisors on Science and Technology. 2017.","unstructured":"President\u2019s Council of Advisors on Science and Technology. 2017. Addendum to the PCAST Report on Forensic Science in Criminal Courts. President\u2019s Council of Advisors on Science and Technology."},{"key":"e_1_3_2_1_97_1","unstructured":"Innocence Project. [n. d.]. Overturning Wrongful Convictions Involving Misapplied Forensics. https:\/\/innocenceproject.org\/overturning-wrongful-convictions-involving-flawed-forensics\/. Accessed: 2022-05-09."},{"key":"e_1_3_2_1_98_1","volume-title":"Dataset shift in machine learning","author":"Qui\u00f1onero-Candela Joaquin","unstructured":"Joaquin Qui\u00f1onero-Candela, Masashi Sugiyama, Neil\u00a0D Lawrence, and Anton Schwaighofer. 2009. Dataset shift in machine learning. MIT Press."},{"key":"e_1_3_2_1_99_1","doi-asserted-by":"publisher","DOI":"10.1145\/3306618.3314244"},{"key":"e_1_3_2_1_100_1","first-page":"659","article-title":"Innovating criminal justice","volume":"112","author":"Ram Natalie","year":"2017","unstructured":"Natalie Ram. 2017. Innovating criminal justice. Northwestern University Law Review 112 (2017), 659.","journal-title":"Northwestern University Law Review"},{"key":"e_1_3_2_1_101_1","volume-title":"International Conference on Machine Learning. PMLR, 5389\u20135400","author":"Recht Benjamin","year":"2019","unstructured":"Benjamin Recht, Rebecca Roelofs, Ludwig Schmidt, and Vaishaal Shankar. 2019. Do imagenet classifiers generalize to imagenet?. In International Conference on Machine Learning. PMLR, 5389\u20135400."},{"key":"e_1_3_2_1_102_1","unstructured":"William Robinson. 2021. RFC Response: NIST Internal Report 8351-DRAFT DNA Mixture Interpretation: A NIST Scientific Foundation Review. https:\/\/ieeeusa.org\/assets\/public-policy\/policy-log\/2021\/111821.pdf"},{"key":"e_1_3_2_1_103_1","first-page":"1972","article-title":"Machine testimony","volume":"126","author":"Roth Andrea","year":"2016","unstructured":"Andrea Roth. 2016. Machine testimony. Yale Law Journal 126(2016), 1972.","journal-title":"Yale Law Journal"},{"key":"e_1_3_2_1_104_1","volume-title":"d.]. Safety Dynamics - Gunshot Detection - Gunshot Location System - Products","author":"Safety Dynamics Inc.","unstructured":"Safety Dynamics Inc.[n. d.]. Safety Dynamics - Gunshot Detection - Gunshot Location System - Products. http:\/\/www.safetydynamics.net\/prods.html. Accessed: 2022-05-31."},{"key":"e_1_3_2_1_105_1","volume-title":"Federal Rules of Evidence Manual. Vol.\u00a05","author":"Saltzburg A","unstructured":"Stephen\u00a0A Saltzburg, Michael\u00a0M Martin, and Daniel\u00a0J Capra. 2006. Federal Rules of Evidence Manual. Vol.\u00a05. LEXIS Law Publishing."},{"key":"e_1_3_2_1_106_1","unstructured":"ShotSpotter Inc.[n. d.]. ShotSpotter. https:\/\/www.shotspotter.com\/law-enforcement\/gunshot-detection-technology\/. Accessed: 2022-05-31."},{"key":"e_1_3_2_1_107_1","article-title":"Trade Secrecy and Innovation in Forensic Technology","volume":"73","author":"Siems Eli","year":"2022","unstructured":"Eli Siems, Katherine\u00a0J Strandburg, and Nicholas Vincent. 2022. Trade Secrecy and Innovation in Forensic Technology. Hastings Law Journal 73, 3 (2022).","journal-title":"Hastings Law Journal"},{"key":"e_1_3_2_1_108_1","volume-title":"Brief of Amicus Curiae Upturn","author":"Singer William","year":"2020","unstructured":"William Singer and Kendra\u00a0K Albert. 2020. Brief of Amicus Curiae Upturn, Inc. In Support of Movant-Appellant. https:\/\/clinic.cyber.harvard.edu\/files\/2020\/10\/Upturn-Amicus-Brief-for-Filing-10-15-20-115-pm.pdf. Accessed: 2022-01-21."},{"key":"e_1_3_2_1_109_1","unstructured":"Maneka Sinha. 2021. Radically Reimagining Forensic Evidence. Alabama Law Review Forthcoming U of Maryland Legal Studies Research Paper2021-10(2021)."},{"key":"e_1_3_2_1_110_1","unstructured":"Mona Sloane. 2021. The Algorithmic Auditing Trap. OneZero (blog) March 17(2021)."},{"key":"e_1_3_2_1_111_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.patter.2021.100425"},{"key":"e_1_3_2_1_112_1","doi-asserted-by":"publisher","DOI":"10.1017\/9781108988025"},{"key":"e_1_3_2_1_113_1","first-page":"925","article-title":"Defending Defending: The Case for Unmitigated Zeal on Behalf of People Who Do Terrible Things","volume":"28","author":"Smith Abbe","year":"1999","unstructured":"Abbe Smith. 1999. Defending Defending: The Case for Unmitigated Zeal on Behalf of People Who Do Terrible Things. Hofstra Law Review 28(1999), 925.","journal-title":"Hofstra Law Review"},{"key":"e_1_3_2_1_114_1","unstructured":"[114] State v. Pickett 246 A. 3d 279.(2021)."},{"key":"e_1_3_2_1_115_1","unstructured":"The Legal Aid Society. [n. d.]. Digital Forensics Unit - The Legal Aid Society. https:\/\/legalaidnyc.org\/programs-projects-units\/digital-forensics-unit\/. Accessed: 2022-05-09."},{"key":"e_1_3_2_1_116_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2011.5995347"},{"key":"e_1_3_2_1_117_1","unstructured":"Ultra Electronics Forensic Technology. [n. d.]. Correlation Engine. https:\/\/www.ultra-forensictechnology.com\/en\/our-products\/ballistic-identification\/correlation-server. Accessed: 2022-05-31."},{"key":"e_1_3_2_1_118_1","unstructured":"Ultra Electronics Forensic Technology. [n. d.]. IBIS Matchpoint. https:\/\/www.ultra-forensictechnology.com\/en\/our-products\/ballistic-identification\/matchpoint. Accessed: 2022-05-31."},{"key":"e_1_3_2_1_119_1","unstructured":"[119] U.S. v. Lizarraga-Tirado 789 F.3d 1107.(2015)."},{"key":"e_1_3_2_1_120_1","unstructured":"[120] U.S. v. Vayner 769 F.3d 125.(2014)."},{"key":"e_1_3_2_1_121_1","doi-asserted-by":"crossref","unstructured":"Abraham Wald. 1945. Statistical Decision Functions Which Minimize the Maximum Risk. Annals of Mathematics(1945).","DOI":"10.2307\/1969022"},{"key":"e_1_3_2_1_122_1","first-page":"717","article-title":"Innocence project: DNA exonerations, 1989-2014: review of data and findings from the first 25 years","volume":"79","author":"West Emily","year":"2015","unstructured":"Emily West and Vanessa Meterko. 2015. Innocence project: DNA exonerations, 1989-2014: review of data and findings from the first 25 years. Albany Law Review 79(2015), 717.","journal-title":"Albany Law Review"},{"key":"e_1_3_2_1_123_1","first-page":"1343","article-title":"Life, liberty, and trade secrets: Intellectual property in the criminal justice system","volume":"70","author":"Wexler Rebecca","year":"2018","unstructured":"Rebecca Wexler. 2018. Life, liberty, and trade secrets: Intellectual property in the criminal justice system. Stanford Law Review 70(2018), 1343.","journal-title":"Stanford Law Review"},{"key":"e_1_3_2_1_124_1","first-page":"2721","article-title":"Privacy as Privilege: The Stored Communications Act and Internet Evidence","volume":"134","author":"Wexler Rebecca","year":"2020","unstructured":"Rebecca Wexler. 2020. Privacy as Privilege: The Stored Communications Act and Internet Evidence. Harvard Law Review 134(2020), 2721.","journal-title":"Harvard Law Review"},{"key":"e_1_3_2_1_125_1","article-title":"Privacy Asymmetries: Access to Data in Criminal Investigations","volume":"68","author":"Wexler Rebecca","year":"2021","unstructured":"Rebecca Wexler. 2021. Privacy Asymmetries: Access to Data in Criminal Investigations. UCLA Law Review 68, 1 (2021).","journal-title":"UCLA Law Review"},{"key":"e_1_3_2_1_126_1","doi-asserted-by":"crossref","unstructured":"[126] Williams v. Illinois 567 U.S. 50.(2012).","DOI":"10.1097\/AIA.0b013e31821a00d0"},{"key":"e_1_3_2_1_127_1","volume-title":"Variable generalization performance of a deep learning model to detect pneumonia in chest radiographs: a cross-sectional study. PLoS medicine 15, 11","author":"Zech R","year":"2018","unstructured":"John\u00a0R Zech, Marcus\u00a0A Badgeley, Manway Liu, Anthony\u00a0B Costa, Joseph\u00a0J Titano, and Eric\u00a0Karl Oermann. 2018. Variable generalization performance of a deep learning model to detect pneumonia in chest radiographs: a cross-sectional study. PLoS medicine 15, 11 (2018)."}],"event":{"name":"FAccT '22: 2022 ACM Conference on Fairness, Accountability, and Transparency","location":"Seoul Republic of Korea","acronym":"FAccT '22","sponsor":["ACM Association for Computing Machinery"]},"container-title":["2022 ACM Conference on Fairness Accountability and Transparency"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3531146.3533228","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3531146.3533228","content-type":"application\/pdf","content-version":"vor","intended-application":"syndication"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3531146.3533228","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T19:31:30Z","timestamp":1750188690000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3531146.3533228"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2022,6,20]]},"references-count":127,"alternative-id":["10.1145\/3531146.3533228","10.1145\/3531146"],"URL":"https:\/\/doi.org\/10.1145\/3531146.3533228","relation":{},"subject":[],"published":{"date-parts":[[2022,6,20]]},"assertion":[{"value":"2022-06-20","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}