{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,3,18]],"date-time":"2026-03-18T05:10:01Z","timestamp":1773810601445,"version":"3.50.1"},"publisher-location":"New York, NY, USA","reference-count":49,"publisher":"ACM","license":[{"start":{"date-parts":[[2020,1,27]],"date-time":"2020-01-27T00:00:00Z","timestamp":1580083200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"DOI":"10.13039\/100000001","name":"National Science Foundation","doi-asserted-by":"publisher","award":["1704845"],"award-info":[{"award-number":["1704845"]}],"id":[{"id":"10.13039\/100000001","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2020,1,27]]},"DOI":"10.1145\/3351095.3372845","type":"proceedings-article","created":{"date-parts":[[2020,1,22]],"date-time":"2020-01-22T19:58:18Z","timestamp":1579723098000},"page":"111-121","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":47,"title":["FlipTest"],"prefix":"10.1145","author":[{"given":"Emily","family":"Black","sequence":"first","affiliation":[{"name":"Carnegie Mellon University"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Samuel","family":"Yeom","sequence":"additional","affiliation":[{"name":"Carnegie Mellon University"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Matt","family":"Fredrikson","sequence":"additional","affiliation":[{"name":"Carnegie Mellon University"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2020,1,27]]},"reference":[{"key":"e_1_3_2_2_1_1","volume-title":"TensorFlow: Large-scale machine learning on heterogeneous systems. https:\/\/www.tensorflow.org\/","author":"Abadi Mart\u00edn","year":"2015","unstructured":"Mart\u00edn Abadi, Ashish Agarwal, Paul Barham, Eugene Brevdo, Zhifeng Chen, Craig Citro, Greg S. Corrado, Andy Davis, Jeffrey Dean, Matthieu Devin, Sanjay Ghemawat, Ian Goodfellow, Andrew Harp, Geoffrey Irving, Michael Isard, Yangqing Jia, Rafal Jozefowicz, Lukasz Kaiser, Manjunath Kudlur, Josh Levenberg, Dandelion Man\u00e9, Rajat Monga, Sherry Moore, Derek Murray, Chris Olah, Mike Schuster, Jonathon Shlens, Benoit Steiner, Ilya Sutskever, Kunal Talwar, Paul Tucker, Vincent Vanhoucke, Vijay Vasudevan, Fernanda Vi\u00e9gas, Oriol Vinyals, Pete Warden, Martin Wattenberg, Martin Wicke, Yuan Yu, and Xiaoqiang Zheng. TensorFlow: Large-scale machine learning on heterogeneous systems. https:\/\/www.tensorflow.org\/, 2015."},{"key":"e_1_3_2_2_2_1","volume-title":"Automated test generation to detect individual discrimination in AI models. arXiv preprint arXiv:1809.03260","author":"Agarwal Aniya","year":"2018","unstructured":"Aniya Agarwal, Pranay Lohia, Seema Nagar, Kuntal Dey, and Diptikalyan Saha. Automated test generation to detect individual discrimination in AI models. arXiv preprint arXiv:1809.03260, 2018."},{"key":"e_1_3_2_2_3_1","first-page":"1964","volume-title":"Advances in Neural Information Processing Systems","author":"Altschuler Jason","year":"2017","unstructured":"Jason Altschuler, Jonathan Weed, and Philippe Rigollet. Near-linear time approximation algorithms for optimal transport via Sinkhorn iteration. In Advances in Neural Information Processing Systems, pages 1964--1974, 2017."},{"key":"e_1_3_2_2_4_1","volume-title":"ProPublica","author":"Angwin Julia","year":"2016","unstructured":"Julia Angwin, Jeff Larson, Surya Mattu, and Lauren Kirchner. Machine bias: There's software used across the country to predict future criminals. and it's biased against blacks. ProPublica, 2016."},{"key":"e_1_3_2_2_5_1","volume-title":"arXiv preprint arXiv:1701.07875","author":"Arjovsky Martin","year":"2017","unstructured":"Martin Arjovsky, Soumith Chintala, and L\u00e9on Bottou. Wasserstein GAN. arXiv preprint arXiv:1701.07875, 2017."},{"key":"e_1_3_2_2_6_1","volume-title":"Big data's disparate impact. California Law Review, 104:671--732","author":"Barocas Solon","year":"2016","unstructured":"Solon Barocas and Andrew D Selbst. Big data's disparate impact. California Law Review, 104:671--732, 2016."},{"key":"e_1_3_2_2_7_1","unstructured":"Rachel K. E. Bellamy Kuntal Dey Michael Hind Samuel C. Hoffman Stephanie Houde Kalapriya Kannan Pranay Lohia Jacquelyn Martino Sameep Mehta Aleksandra Mojsilovic Seema Nagar Karthikeyan Natesan Ramamurthy John T. Richards Diptikalyan Saha Prasanna Sattigeri Moninder Singh Kush R. Varshney and Yunfeng Zhang. AI fairness 360: An extensible toolkit for detecting understanding and mitigating unwanted algorithmic bias. CoRR abs\/1810.01943 2018. URL http:\/\/arxiv.org\/abs\/1810.01943."},{"key":"e_1_3_2_2_8_1","volume-title":"https:\/\/keras.io","author":"Keras Fran\u00e7ois Chollet","year":"2015","unstructured":"Fran\u00e7ois Chollet et al. Keras. https:\/\/keras.io, 2015."},{"key":"e_1_3_2_2_9_1","doi-asserted-by":"publisher","DOI":"10.1089\/big.2016.0047"},{"key":"e_1_3_2_2_10_1","volume-title":"Strategic Subject List. https:\/\/data.cityofchicago.org\/Public-Safety\/Strategic-Subject-List\/4aki-r3np","author":"Chicago City","year":"2017","unstructured":"City of Chicago. Strategic Subject List. https:\/\/data.cityofchicago.org\/Public-Safety\/Strategic-Subject-List\/4aki-r3np, 2017."},{"key":"e_1_3_2_2_11_1","volume-title":"Amazon scraps secret AI recruiting tool that showed bias against women","author":"Dastin Jeffrey","year":"2018","unstructured":"Jeffrey Dastin. Amazon scraps secret AI recruiting tool that showed bias against women. Reuters, 2018."},{"key":"e_1_3_2_2_12_1","doi-asserted-by":"publisher","DOI":"10.1109\/SP.2016.42"},{"key":"e_1_3_2_2_13_1","volume-title":"Proxy discrimination in data-driven systems. arXiv preprint arXiv:1707.08120","author":"Datta Anupam","year":"2017","unstructured":"Anupam Datta, Matt Fredrikson, Gihyuk Ko, Piotr Mardziel, and Shayak Sen. Proxy discrimination in data-driven systems. arXiv preprint arXiv:1707.08120, 2017."},{"key":"e_1_3_2_2_14_1","volume-title":"Obtaining fairness using optimal transport theory. arXiv preprint arXiv:1806.03195","author":"del Barrio Eustasio","year":"2018","unstructured":"Eustasio del Barrio, Fabrice Gamboa, Paula Gordaliza, and Jean-Michel Loubes. Obtaining fairness using optimal transport theory. arXiv preprint arXiv:1806.03195, 2018."},{"key":"e_1_3_2_2_15_1","volume-title":"Fairness under composition. arXiv preprint arXiv:1806.06122","author":"Dwork Cynthia","year":"2018","unstructured":"Cynthia Dwork and Christina Ilvento. Fairness under composition. arXiv preprint arXiv:1806.06122, 2018."},{"key":"e_1_3_2_2_16_1","doi-asserted-by":"publisher","DOI":"10.1145\/2090236.2090255"},{"key":"e_1_3_2_2_17_1","unstructured":"Equivant. Practitioner's guide to COMPAS core. http:\/\/www.equivant.com\/wp-content\/uploads\/Practitioners-Guide-to-COMPAS-Core-040419.pdf 2019."},{"key":"e_1_3_2_2_18_1","doi-asserted-by":"publisher","DOI":"10.1145\/2783258.2783311"},{"key":"e_1_3_2_2_19_1","first-page":"498","volume-title":"Foundations of Software Engineering","author":"Galhotra Sainyam","year":"2017","unstructured":"Sainyam Galhotra, Yuriy Brun, and Alexandra Meliou. Fairness testing: testing software for discrimination. In Foundations of Software Engineering, pages 498--510, 2017."},{"key":"e_1_3_2_2_20_1","first-page":"2672","volume-title":"Advances in Neural Information Processing Systems","author":"Goodfellow Ian","year":"2014","unstructured":"Ian Goodfellow, Jean Pouget-Abadie, Mehdi Mirza, Bing Xu, David Warde-Farley, Sherjil Ozair, Aaron Courville, and Yoshua Bengio. Generative adversarial nets. In Advances in Neural Information Processing Systems, pages 2672--2680, 2014."},{"key":"e_1_3_2_2_21_1","volume-title":"Gurobi optimizer reference manual. https:\/\/www.gurobi.com\/documentation\/8.1\/refman.pdf","author":"Optimization Gurobi","year":"2019","unstructured":"Gurobi Optimization, LLC. Gurobi optimizer reference manual. https:\/\/www.gurobi.com\/documentation\/8.1\/refman.pdf, 2019."},{"key":"e_1_3_2_2_22_1","first-page":"3315","volume-title":"Advances in Neural Information Processing Systems","author":"Hardt Moritz","year":"2016","unstructured":"Moritz Hardt, Eric Price, and Nati Srebro. Equality of opportunity in supervised learning. In Advances in Neural Information Processing Systems, pages 3315--3323, 2016."},{"key":"e_1_3_2_2_23_1","first-page":"1944","volume-title":"International Conference on Machine Learning","author":"H\u00e9bert-Johnson \u00farsula","year":"2018","unstructured":"\u00farsula H\u00e9bert-Johnson, Michael Kim, Omer Reingold, and Guy Rothblum. Multicalibration: Calibration for the (computationally-identifiable) masses. In International Conference on Machine Learning, pages 1944--1953, 2018."},{"key":"e_1_3_2_2_24_1","first-page":"325","volume-title":"Advances in Neural Information Processing Systems","author":"Joseph Matthew","year":"2016","unstructured":"Matthew Joseph, Michael Kearns, Jamie H Morgenstern, and Aaron Roth. Fairness in learning: Classic and contextual bandits. In Advances in Neural Information Processing Systems, pages 325--333, 2016."},{"key":"e_1_3_2_2_25_1","first-page":"2564","volume-title":"International Conference on Machine Learning","author":"Kearns Michael","year":"2018","unstructured":"Michael Kearns, Seth Neel, Aaron Roth, and Zhiwei Steven Wu. Preventing fairness gerrymandering: Auditing and learning for subgroup fairness. In International Conference on Machine Learning, pages 2564--2572, 2018."},{"key":"e_1_3_2_2_26_1","doi-asserted-by":"publisher","DOI":"10.1145\/3287560.3287592"},{"key":"e_1_3_2_2_27_1","first-page":"1","volume-title":"Innovations in Theoretical Computer Science","author":"Kleinberg Jon","year":"2017","unstructured":"Jon Kleinberg, Sendhil Mullainathan, and Manish Raghavan. Inherent trade-offs in the fair determination of risk scores. In Innovations in Theoretical Computer Science, pages 43:1--43:23, 2017."},{"key":"e_1_3_2_2_28_1","doi-asserted-by":"publisher","DOI":"10.1002\/nav.3800020109"},{"key":"e_1_3_2_2_29_1","first-page":"4066","volume-title":"Advances in Neural Information Processing Systems","author":"Kusner Matt J","year":"2017","unstructured":"Matt J Kusner, Joshua Loftus, Chris Russell, and Ricardo Silva. Counterfactual fairness. In Advances in Neural Information Processing Systems, pages 4066--4076, 2017."},{"key":"e_1_3_2_2_30_1","volume-title":"Adversarial computation of optimal transport maps. arXiv preprint arXiv:1906.09691","author":"Leygonie Jacob","year":"2019","unstructured":"Jacob Leygonie, Jennifer She, Amjad Almahairi, Sai Rajeswar, and Aaron Courville. Adversarial computation of optimal transport maps. arXiv preprint arXiv:1906.09691, 2019."},{"key":"e_1_3_2_2_31_1","first-page":"8125","volume-title":"Advances in Neural Information Processing Systems","author":"Lipton Zachary","year":"2018","unstructured":"Zachary Lipton, Julian McAuley, and Alexandra Chouldechova. Does mitigating ML's impact disparity require treatment disparity? In Advances in Neural Information Processing Systems, pages 8125--8135, 2018."},{"key":"e_1_3_2_2_32_1","volume-title":"https:\/\/www.legislation.gov.uk\/ukpga\/2010\/15\/contents","author":"Parliament of the United Kingdom. Equality Act 2010.","year":"2010","unstructured":"Parliament of the United Kingdom. Equality Act 2010. https:\/\/www.legislation.gov.uk\/ukpga\/2010\/15\/contents, 2010."},{"key":"e_1_3_2_2_33_1","first-page":"4197","volume-title":"Advances in Neural Information Processing Systems","author":"Perrot Micha\u00ebl","year":"2016","unstructured":"Micha\u00ebl Perrot, Nicolas Courty, R\u00e9mi Flamary, and Amaury Habrard. Mapping estimation for discrete optimal transport. In Advances in Neural Information Processing Systems, pages 4197--4205, 2016."},{"key":"e_1_3_2_2_34_1","doi-asserted-by":"publisher","DOI":"10.1561\/9781680835519"},{"key":"e_1_3_2_2_35_1","volume-title":"Approximating optimal transport with linear programs. arXiv preprint arXiv:1810.05957","author":"Quanrud Kent","year":"2018","unstructured":"Kent Quanrud. Approximating optimal transport with linear programs. arXiv preprint arXiv:1810.05957, 2018."},{"key":"e_1_3_2_2_36_1","volume-title":"R\u00e9mi Flamary, Nicolas Courty, Antoine Rolet, and Mathieu Blondel. Large-scale optimal transport and mapping estimation. arXiv preprint arXiv:1711.02283","author":"Seguy Vivien","year":"2017","unstructured":"Vivien Seguy, Bharath Bhushan Damodaran, R\u00e9mi Flamary, Nicolas Courty, Antoine Rolet, and Mathieu Blondel. Large-scale optimal transport and mapping estimation. arXiv preprint arXiv:1711.02283, 2017."},{"key":"e_1_3_2_2_37_1","volume-title":"Griggs v","author":"Supreme Court of the United States.","year":"1971","unstructured":"Supreme Court of the United States. Griggs v. Duke Power Co. 401 U.S. 424, 1971."},{"key":"e_1_3_2_2_38_1","doi-asserted-by":"publisher","DOI":"10.1109\/EuroSP.2017.29"},{"key":"e_1_3_2_2_39_1","doi-asserted-by":"publisher","DOI":"10.1145\/3287560.3287566"},{"key":"e_1_3_2_2_40_1","unstructured":"Rhema Vaithianathan Emily Putnam-Hornstein Nan Jiang Parma Nand and Tim Maloney. Developing predictive models to support child maltreatment hotline screening decisions: Allegheny County methodology and implementation. https:\/\/www.alleghenycountyanalytics.us\/wp-content\/uploads\/2018\/02\/DevelopingPredictiveRiskModels-package_011618.pdf 2017."},{"key":"e_1_3_2_2_41_1","volume-title":"Optimal transport: Old and new","author":"Villani C\u00e9dric","year":"2008","unstructured":"C\u00e9dric Villani. Optimal transport: Old and new, volume 338. Springer Science & Business Media, 2008."},{"issue":"2","key":"e_1_3_2_2_42_1","first-page":"841","article-title":"Counterfactual explanations without opening the black box: Automated decisions and the gdpr","volume":"31","author":"Wachter Sandra","year":"2018","unstructured":"Sandra Wachter, Brent Mittelstadt, and Chris Russell. Counterfactual explanations without opening the black box: Automated decisions and the gdpr. Harvard Journal of Law & Technology, 31(2):841--887, 2018.","journal-title":"Harvard Journal of Law & Technology"},{"key":"e_1_3_2_2_43_1","volume-title":"LSAC national longitudinal bar passage study","author":"Wightman Linda F","year":"1998","unstructured":"Linda F Wightman and Henry Ramsey. LSAC national longitudinal bar passage study. Law School Admission Council, 1998."},{"key":"e_1_3_2_2_44_1","first-page":"1920","volume-title":"Conference on Learning Theory","author":"Woodworth Blake","year":"2017","unstructured":"Blake Woodworth, Suriya Gunasekar, Mesrob I Ohannessian, and Nathan Srebro. Learning non-discriminatory predictors. In Conference on Learning Theory, pages 1920--1953, 2017."},{"key":"e_1_3_2_2_45_1","volume-title":"Yang and Caroline Uhler. Scalable unbalanced optimal transport using generative adversarial networks. arXiv preprint arXiv:1810.11447","author":"Karren","year":"2018","unstructured":"Karren D. Yang and Caroline Uhler. Scalable unbalanced optimal transport using generative adversarial networks. arXiv preprint arXiv:1810.11447, 2018."},{"key":"e_1_3_2_2_46_1","first-page":"4568","volume-title":"Advances in Neural Information Processing Systems","author":"Yeom Samuel","year":"2018","unstructured":"Samuel Yeom, Anupam Datta, and Matt Fredrikson. Hunting for discriminatory proxies in linear regression models. In Advances in Neural Information Processing Systems, pages 4568--4578, 2018."},{"key":"e_1_3_2_2_47_1","first-page":"962","volume-title":"Artificial Intelligence and Statistics","author":"Zafar Muhammad Bilal","year":"2017","unstructured":"Muhammad Bilal Zafar, Isabel Valera, Manuel Gomez Rogriguez, and Krishna P Gummadi. Fairness constraints: Mechanisms for fair classification. In Artificial Intelligence and Statistics, pages 962--970, 2017."},{"key":"e_1_3_2_2_48_1","first-page":"325","volume-title":"International Conference on Machine Learning","author":"Zemel Rich","year":"2013","unstructured":"Rich Zemel, Yu Wu, Kevin Swersky, Toni Pitassi, and Cynthia Dwork. Learning fair representations. In International Conference on Machine Learning, pages 325--333, 2013."},{"key":"e_1_3_2_2_49_1","volume-title":"Identifying significant predictive bias in classifiers. arXiv preprint arXiv:1611.08292","author":"Zhang Zhe","year":"2016","unstructured":"Zhe Zhang and Daniel B Neill. Identifying significant predictive bias in classifiers. arXiv preprint arXiv:1611.08292, 2016."}],"event":{"name":"FAT* '20: Conference on Fairness, Accountability, and Transparency","location":"Barcelona Spain","acronym":"FAT* '20","sponsor":["ACM Association for Computing Machinery"]},"container-title":["Proceedings of the 2020 Conference on Fairness, Accountability, and Transparency"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3351095.3372845","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3351095.3372845","content-type":"application\/pdf","content-version":"vor","intended-application":"syndication"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3351095.3372845","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T22:41:22Z","timestamp":1750200082000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3351095.3372845"}},"subtitle":["fairness testing via optimal transport"],"short-title":[],"issued":{"date-parts":[[2020,1,27]]},"references-count":49,"alternative-id":["10.1145\/3351095.3372845","10.1145\/3351095"],"URL":"https:\/\/doi.org\/10.1145\/3351095.3372845","relation":{},"subject":[],"published":{"date-parts":[[2020,1,27]]},"assertion":[{"value":"2020-01-27","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}