{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,14]],"date-time":"2026-04-14T00:40:35Z","timestamp":1776127235234,"version":"3.50.1"},"publisher-location":"New York, NY, USA","reference-count":62,"publisher":"ACM","license":[{"start":{"date-parts":[[2022,6,20]],"date-time":"2022-06-20T00:00:00Z","timestamp":1655683200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by-sa\/4.0\/"}],"funder":[{"DOI":"10.13039\/100007887","name":"Harvard School of Engineering and Applied Sciences","doi-asserted-by":"publisher","id":[{"id":"10.13039\/100007887","id-type":"DOI","asserted-by":"publisher"}]},{"name":"Amazon Graduate Research Fellowship"},{"DOI":"10.13039\/100006112","name":"Microsoft Research","doi-asserted-by":"publisher","id":[{"id":"10.13039\/100006112","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2022,6,21]]},"DOI":"10.1145\/3531146.3533149","type":"proceedings-article","created":{"date-parts":[[2022,6,20]],"date-time":"2022-06-20T14:27:10Z","timestamp":1655735230000},"page":"850-863","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":56,"title":["Model Multiplicity: Opportunities, Concerns, and Solutions"],"prefix":"10.1145","author":[{"given":"Emily","family":"Black","sequence":"first","affiliation":[{"name":"Carngie Mellon University, USA"}]},{"given":"Manish","family":"Raghavan","sequence":"additional","affiliation":[{"name":"Harvard SEAS, USA"}]},{"given":"Solon","family":"Barocas","sequence":"additional","affiliation":[{"name":"Microsoft Research, USA"}]}],"member":"320","published-online":{"date-parts":[[2022,6,20]]},"reference":[{"key":"e_1_3_2_1_1_1","unstructured":"1970. 116 Cong. Reg. 36572."},{"key":"e_1_3_2_1_2_1","unstructured":"2011. Comment for 1002.6 - Rules Concerning Evaluation of Applications. https:\/\/www.consumerfinance.gov\/rules-policy\/regulations\/1002\/interp-6\/."},{"key":"e_1_3_2_1_3_1","volume-title":"International Conference on Machine Learning. 60\u201369","author":"Agarwal Alekh","year":"2018","unstructured":"Alekh Agarwal, Alina Beygelzimer, Miroslav Dudik, John Langford, and Hanna Wallach. 2018. A Reductions Approach to Fair Classification. In International Conference on Machine Learning. 60\u201369."},{"key":"e_1_3_2_1_4_1","volume-title":"International Conference on Machine Learning. PMLR, 314\u2013323","author":"Anders Christopher","year":"2020","unstructured":"Christopher Anders, Plamen Pasliev, Ann-Kathrin Dombrowski, Klaus-Robert M\u00fcller, and Pan Kessel. 2020. Fairwashing explanations with off-manifold detergent. In International Conference on Machine Learning. PMLR, 314\u2013323."},{"key":"e_1_3_2_1_5_1","first-page":"671","article-title":"Big data\u2019s disparate impact","volume":"104","author":"Barocas Solon","year":"2016","unstructured":"Solon Barocas and Andrew\u00a0D Selbst. 2016. Big data\u2019s disparate impact. California Law Review 104 (2016), 671\u2013732.","journal-title":"California Law Review"},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"publisher","DOI":"10.1145\/3351095.3372830"},{"key":"e_1_3_2_1_7_1","unstructured":"Dimitris Bertsimas Arthur Delarue Patrick Jaillet and Sebastien Martin. 2019. The price of interpretability. arXiv preprint arXiv:1907.03419(2019)."},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"crossref","unstructured":"Abeba Birhane Pratyusha Kalluri Dallas Card William Agnew Ravit Dotan and Michelle Bao. 2021. The values encoded in machine learning research. arXiv preprint arXiv:2106.15590(2021).","DOI":"10.1145\/3531146.3533083"},{"key":"e_1_3_2_1_9_1","unstructured":"Emily Black Solon Barocas Alexandra Chouldechova Logan Koepke Kristian Lum Michael Madaio and Sarah Riley. 2022. Reducing Racial Disparity Through Procedural Interventions: Conceptions and Outcomes."},{"key":"e_1_3_2_1_10_1","volume-title":"Leave-one-out Unfairness. In Proceedings of the 2021 ACM Conference on Fairness, Accountability, and Transparency. 285\u2013295","author":"Black Emily","year":"2021","unstructured":"Emily Black and Matt Fredrikson. 2021. Leave-one-out Unfairness. In Proceedings of the 2021 ACM Conference on Fairness, Accountability, and Transparency. 285\u2013295."},{"key":"e_1_3_2_1_11_1","volume-title":"Selective Ensembles for Consistent Predictions. In International Conference on Learning Representations.","author":"Black Emily","year":"2022","unstructured":"Emily Black, Klas Leino, and Matt Fredrikson. 2022. Selective Ensembles for Consistent Predictions. In International Conference on Learning Representations."},{"key":"e_1_3_2_1_12_1","volume-title":"Consistent Counterfactuals for Deep Models. In International Conference on Learning Representations.","author":"Black Emily","year":"2022","unstructured":"Emily Black, Zifan Wang, Matt Fredrikson, and Anupam Datta. 2022. Consistent Counterfactuals for Deep Models. In International Conference on Learning Representations."},{"key":"e_1_3_2_1_13_1","volume-title":"Statistical modeling: The two cultures (with comments and a rejoinder by the author). Statistical science 16, 3","author":"Breiman Leo","year":"2001","unstructured":"Leo Breiman. 2001. Statistical modeling: The two cultures (with comments and a rejoinder by the author). Statistical science 16, 3 (2001), 199\u2013231."},{"key":"e_1_3_2_1_14_1","first-page":"461","article-title":"Beyond accountability: Arbitrariness and legitimacy in the administrative state","volume":"78","author":"Bressman Lisa\u00a0Schultz","year":"2003","unstructured":"Lisa\u00a0Schultz Bressman. 2003. Beyond accountability: Arbitrariness and legitimacy in the administrative state. NYUL Rev. 78(2003), 461.","journal-title":"NYUL Rev."},{"key":"e_1_3_2_1_15_1","unstructured":"Chaofan Chen Kangcheng Lin Cynthia Rudin Yaron Shaposhnik Sijia Wang and Tong Wang. 2018. An interpretable model with globally consistent explanations for credit risk. arXiv preprint arXiv:1811.12615(2018)."},{"key":"e_1_3_2_1_16_1","unstructured":"Irene\u00a0Y. Chen Fredrik\u00a0D. Johansson and David\u00a0A. Sontag. 2018. Why Is My Classifier Discriminatory?. In Advances in Neural Information Processing Systems. 3543\u20133554."},{"key":"e_1_3_2_1_17_1","first-page":"1249","article-title":"Technological due process","volume":"85","author":"Citron Danielle\u00a0Keats","year":"2007","unstructured":"Danielle\u00a0Keats Citron. 2007. Technological due process. Wash. L Rev. 85(2007), 1249.","journal-title":"Wash. L Rev."},{"key":"e_1_3_2_1_18_1","first-page":"1","article-title":"The scored society: Due process for automated predictions","volume":"89","author":"Citron Danielle\u00a0Keats","year":"2014","unstructured":"Danielle\u00a0Keats Citron and Frank Pasquale. 2014. The scored society: Due process for automated predictions. Wash. L. Rev. 89(2014), 1.","journal-title":"Wash. L. Rev."},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","DOI":"10.1145\/3461702.3462519"},{"key":"e_1_3_2_1_20_1","volume-title":"Proceedings of the 38th International Conference on Machine Learning(Proceedings of Machine Learning Research, Vol.\u00a0139)","author":"Coston Amanda","year":"2021","unstructured":"Amanda Coston, Ashesh Rambachan, and Alexandra Chouldechova. 2021. Characterizing Fairness Over the Set of Good Models Under Selective Labels. In Proceedings of the 38th International Conference on Machine Learning(Proceedings of Machine Learning Research, Vol.\u00a0139). PMLR, 2144\u20132155."},{"key":"e_1_3_2_1_21_1","first-page":"93","article-title":"Big data and due process: Toward a framework to redress predictive privacy harms","volume":"55","author":"Crawford Kate","year":"2014","unstructured":"Kate Crawford and Jason Schultz. 2014. Big data and due process: Toward a framework to redress predictive privacy harms. BCL Rev. 55(2014), 93.","journal-title":"BCL Rev."},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.1145\/3442188.3445942"},{"key":"e_1_3_2_1_23_1","unstructured":"Alexander D\u2019Amour Katherine Heller Dan Moldovan Ben Adlam Babak Alipanahi Alex Beutel Christina Chen Jonathan Deaton Jacob Eisenstein Matthew\u00a0D Hoffman 2020. Underspecification presents challenges for credibility in modern machine learning. arXiv preprint arXiv:2011.03395(2020)."},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"crossref","unstructured":"Anupam Datta Matt Fredrikson Gihyuk Ko Piotr Mardziel and Shayak Sen. 2017. Proxy Discrimination in Data-Driven Systems. arXiv preprint arXiv:1707.08120(2017).","DOI":"10.1145\/3133956.3134097"},{"key":"e_1_3_2_1_25_1","volume-title":"Proceedings of 17th International Conference on Machine Learning. 231\u2013238","author":"Domingos Pedro","year":"2000","unstructured":"Pedro Domingos. 2000. A unified bias-variance decomposition. In Proceedings of 17th International Conference on Machine Learning. 231\u2013238."},{"key":"e_1_3_2_1_26_1","unstructured":"Jiayun Dong and Cynthia Rudin. 2019. Variable importance clouds: A way to explore variable importance for the set of good models. arXiv preprint arXiv:1901.03209(2019)."},{"key":"e_1_3_2_1_27_1","doi-asserted-by":"publisher","DOI":"10.1080\/10618600.2017.1384734"},{"key":"e_1_3_2_1_28_1","volume-title":"International Conference on Machine Learning. PMLR, 2803\u20132813","author":"Dutta Sanghamitra","year":"2020","unstructured":"Sanghamitra Dutta, Dennis Wei, Hazar Yueksel, Pin-Yu Chen, Sijia Liu, and Kush Varshney. 2020. Is there a trade-off between fairness and accuracy? a perspective using mismatched hypothesis testing. In International Conference on Machine Learning. PMLR, 2803\u20132813."},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.1145\/2090236.2090255"},{"key":"e_1_3_2_1_30_1","unstructured":"Equal Credit Opportunities Act Public Law 93-495 1974. Codified at 15 U.S.C. \u00a7\u00a01691 et seq."},{"key":"e_1_3_2_1_31_1","unstructured":"Fair Credit Reporting Act Public Law 91-508 1970. Codified at 15 U.S.C. \u00a7\u00a01681 et seq."},{"key":"e_1_3_2_1_32_1","first-page":"1","article-title":"All Models are Wrong, but Many are Useful: Learning a Variable\u2019s Importance by Studying an Entire Class of Prediction Models Simultaneously.J","volume":"20","author":"Fisher Aaron","year":"2019","unstructured":"Aaron Fisher, Cynthia Rudin, and Francesca Dominici. 2019. All Models are Wrong, but Many are Useful: Learning a Variable\u2019s Importance by Studying an Entire Class of Prediction Models Simultaneously.J. Mach. Learn. Res. 20, 177 (2019), 1\u201381.","journal-title":"Mach. Learn. Res."},{"key":"e_1_3_2_1_33_1","volume-title":"Neural networks and the bias\/variance dilemma. Neural computation 4, 1","author":"Geman Stuart","year":"1992","unstructured":"Stuart Geman, Elie Bienenstock, and Ren\u00e9 Doursat. 1992. Neural networks and the bias\/variance dilemma. Neural computation 4, 1 (1992), 1\u201358."},{"key":"e_1_3_2_1_34_1","unstructured":"Amir-Hossein Karimi Gilles Barthe Bernhard Sch\u00f6lkopf and Isabel Valera. 2020. A survey of algorithmic recourse: definitions formulations solutions and prospects. arXiv preprint arXiv:2010.04050(2020)."},{"key":"e_1_3_2_1_35_1","volume-title":"Race-aware algorithms: Fairness, nondiscrimination and affirmative action.California Law Review 110","author":"Kim T.","year":"2022","unstructured":"Pauline\u00a0T. Kim. 2022. Race-aware algorithms: Fairness, nondiscrimination and affirmative action.California Law Review 110 (2022)."},{"key":"e_1_3_2_1_36_1","doi-asserted-by":"publisher","DOI":"10.1073\/pnas.2018340118"},{"key":"e_1_3_2_1_37_1","unstructured":"Ron Kohavi David\u00a0H Wolpert 1996. Bias plus variance decomposition for zero-one loss functions. In ICML Vol.\u00a096. 275\u201383."},{"key":"e_1_3_2_1_38_1","unstructured":"Loren Larsen. 2019. Resumes Robots and Racism: The Truth about AI in Hiring. HireVue."},{"key":"e_1_3_2_1_39_1","first-page":"653","article-title":"Playing with the data: what legal scholars should learn about machine learning","volume":"51","author":"Lehr David","year":"2017","unstructured":"David Lehr and Paul Ohm. 2017. Playing with the data: what legal scholars should learn about machine learning. UCDL Rev. 51(2017), 653.","journal-title":"UCDL Rev."},{"key":"e_1_3_2_1_40_1","doi-asserted-by":"publisher","DOI":"10.5555\/3524938.3525566"},{"key":"e_1_3_2_1_41_1","volume-title":"Individual differences among deep neural network models. Nature communications 11, 1","author":"Mehrer Johannes","year":"2020","unstructured":"Johannes Mehrer, Courtney\u00a0J Spoerer, Nikolaus Kriegeskorte, and Tim\u00a0C Kietzmann. 2020. Individual differences among deep neural network models. Nature communications 11, 1 (2020), 1\u201312."},{"key":"e_1_3_2_1_42_1","volume-title":"Conference on Fairness, Accountability and Transparency. PMLR, 107\u2013118","author":"Menon Aditya\u00a0Krishna","year":"2018","unstructured":"Aditya\u00a0Krishna Menon and Robert\u00a0C Williamson. 2018. The cost of fairness in binary classification. In Conference on Fairness, Accountability and Transparency. PMLR, 107\u2013118."},{"key":"e_1_3_2_1_43_1","volume-title":"Dissecting racial bias in an algorithm used to manage the health of populations. Science 366, 6464","author":"Obermeyer Ziad","year":"2019","unstructured":"Ziad Obermeyer, Brian Powers, Christine Vogeli, and Sendhil Mullainathan. 2019. Dissecting racial bias in an algorithm used to manage the health of populations. Science 366, 6464 (2019), 447\u2013453."},{"key":"e_1_3_2_1_44_1","doi-asserted-by":"publisher","DOI":"10.1145\/3287560.3287567"},{"key":"e_1_3_2_1_45_1","volume-title":"Proceedings of the 36th Conference on Uncertainty in Artificial Intelligence (UAI)(Proceedings of Machine Learning Research).","author":"Pawelczyk Martin","year":"2020","unstructured":"Martin Pawelczyk, Klaus Broelemann, and Gjergji. Kasneci. 2020. On Counterfactual Explanations under Predictive Multiplicity. In Proceedings of the 36th Conference on Uncertainty in Artificial Intelligence (UAI)(Proceedings of Machine Learning Research)."},{"key":"e_1_3_2_1_46_1","first-page":"1035","article-title":"May the Odds Be Ever in Your Favor","volume":"66","author":"Perry Ronen","year":"2014","unstructured":"Ronen Perry and Tal\u00a0Z Zarsky. 2014. May the Odds Be Ever in Your Favor: Lotteries in Law. Ala. L. Rev. 66(2014), 1035.","journal-title":"Lotteries in Law. Ala. L. Rev."},{"key":"e_1_3_2_1_47_1","volume-title":"Challenges for mitigating bias in algorithmic hiring. Brookings. Retrieved February 25","author":"Raghavan Manish","year":"2019","unstructured":"Manish Raghavan and Solon Barocas. 2019. Challenges for mitigating bias in algorithmic hiring. Brookings. Retrieved February 25 (2019), 2020."},{"key":"e_1_3_2_1_48_1","doi-asserted-by":"publisher","DOI":"10.1145\/3351095.3372828"},{"key":"e_1_3_2_1_49_1","unstructured":"Xavier Renard Thibault Laugel and Marcin Detyniecki. 2021. Understanding Prediction Discrepancies in Machine Learning Classifiers. arXiv preprint arXiv:2104.05467(2021)."},{"key":"e_1_3_2_1_50_1","volume-title":"Machine learning, automated suspicion algorithms, and the fourth amendment","author":"Rich L","unstructured":"Michael\u00a0L Rich. 2016. Machine learning, automated suspicion algorithms, and the fourth amendment. University of Pennsylvania Law Review(2016), 871\u2013929."},{"key":"e_1_3_2_1_51_1","doi-asserted-by":"publisher","DOI":"10.1038\/s42256-021-00396-x"},{"key":"e_1_3_2_1_52_1","doi-asserted-by":"publisher","DOI":"10.1145\/3351095.3372863"},{"key":"e_1_3_2_1_53_1","doi-asserted-by":"publisher","DOI":"10.1038\/s42256-019-0048-x"},{"key":"e_1_3_2_1_54_1","first-page":"1085","article-title":"The intuitive appeal of explainable machines","volume":"87","author":"Selbst D","year":"2018","unstructured":"Andrew\u00a0D Selbst and Solon Barocas. 2018. The intuitive appeal of explainable machines. Fordham L. Rev. 87(2018), 1085.","journal-title":"Fordham L. Rev."},{"key":"e_1_3_2_1_55_1","unstructured":"Lesia Semenova Cynthia Rudin and Ronald Parr. 2019. A study in Rashomon curves and volumes: A new perspective on generalization and model simplicity in machine learning. arXiv preprint arXiv:1908.01755(2019)."},{"key":"e_1_3_2_1_56_1","volume-title":"Understanding machine learning: From theory to algorithms","author":"Shalev-Shwartz Shai","unstructured":"Shai Shalev-Shwartz and Shai Ben-David. 2014. Understanding machine learning: From theory to algorithms. Cambridge university press."},{"key":"e_1_3_2_1_57_1","volume-title":"International Conference on Learning Representations.","author":"Tsipras Dimitris","year":"2019","unstructured":"Dimitris Tsipras, Shibani Santurkar, Logan Engstrom, Alexander Turner, and Aleksander Madry. 2019. Robustness may be at odds with accuracy. In International Conference on Learning Representations."},{"key":"e_1_3_2_1_58_1","volume-title":"Actionable Recourse in Linear Classification. In Conference on Fairness, Accountability, and Transparency. 10\u201319","author":"Ustun Berk","year":"2019","unstructured":"Berk Ustun, Alexander Spangher, and Yang Liu. 2019. Actionable Recourse in Linear Classification. In Conference on Fairness, Accountability, and Transparency. 10\u201319."},{"key":"e_1_3_2_1_59_1","first-page":"841","article-title":"Counterfactual explanations without opening the black box: Automated decisions and the GDPR","volume":"31","author":"Wachter Sandra","year":"2018","unstructured":"Sandra Wachter, Brent Mittelstadt, and Chris Russell. 2018. Counterfactual explanations without opening the black box: Automated decisions and the GDPR. Harvard Journal of Law & Technology 31, 2 (2018), 841\u2013887.","journal-title":"Harvard Journal of Law & Technology"},{"key":"e_1_3_2_1_60_1","volume-title":"International Conference on Machine Learning. PMLR, 6505\u20136514","author":"Wang Tong","year":"2019","unstructured":"Tong Wang. 2019. Gaining free or low-cost interpretability with interpretable partial substitute. In International Conference on Machine Learning. PMLR, 6505\u20136514."},{"key":"e_1_3_2_1_61_1","volume-title":"swetasudha panda, and Jean-Baptiste Tristan","author":"Wick Michael","year":"2019","unstructured":"Michael Wick, swetasudha panda, and Jean-Baptiste Tristan. 2019. Unlocking Fairness: a Trade-off Revisited. In Advances in Neural Information Processing Systems, Vol.\u00a032."},{"key":"e_1_3_2_1_62_1","volume-title":"International Conference on Machine Learning. PMLR, 7472\u20137482","author":"Zhang Hongyang","year":"2019","unstructured":"Hongyang Zhang, Yaodong Yu, Jiantao Jiao, Eric Xing, Laurent El\u00a0Ghaoui, and Michael Jordan. 2019. Theoretically principled trade-off between robustness and accuracy. In International Conference on Machine Learning. PMLR, 7472\u20137482."}],"event":{"name":"FAccT '22: 2022 ACM Conference on Fairness, Accountability, and Transparency","location":"Seoul Republic of Korea","acronym":"FAccT '22","sponsor":["ACM Association for Computing Machinery"]},"container-title":["2022 ACM Conference on Fairness Accountability and Transparency"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3531146.3533149","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3531146.3533149","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T19:02:08Z","timestamp":1750186928000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3531146.3533149"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2022,6,20]]},"references-count":62,"alternative-id":["10.1145\/3531146.3533149","10.1145\/3531146"],"URL":"https:\/\/doi.org\/10.1145\/3531146.3533149","relation":{},"subject":[],"published":{"date-parts":[[2022,6,20]]},"assertion":[{"value":"2022-06-20","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}