{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,2,11]],"date-time":"2026-02-11T15:38:26Z","timestamp":1770824306462,"version":"3.50.1"},"publisher-location":"New York, NY, USA","reference-count":53,"publisher":"ACM","license":[{"start":{"date-parts":[[2022,6,20]],"date-time":"2022-06-20T00:00:00Z","timestamp":1655683200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by-nc-nd\/4.0\/"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2022,6,21]]},"DOI":"10.1145\/3531146.3533105","type":"proceedings-article","created":{"date-parts":[[2022,6,20]],"date-time":"2022-06-20T14:27:10Z","timestamp":1655735230000},"page":"379-389","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":14,"title":["De-biasing \u201cbias\u201d measurement"],"prefix":"10.1145","author":[{"given":"Kristian","family":"Lum","sequence":"first","affiliation":[{"name":"Twitter, USA"}]},{"given":"Yunfeng","family":"Zhang","sequence":"additional","affiliation":[{"name":"Twitter, USA"}]},{"given":"Amanda","family":"Bower","sequence":"additional","affiliation":[{"name":"Twitter, USA"}]}],"member":"320","published-online":{"date-parts":[[2022,6,20]]},"reference":[{"key":"e_1_3_2_1_1_1","volume-title":"Fairness and robustness in invariant learning: A case study in toxicity classification. Algorithmic Fairness through the Lens of Causality and Interpretability Workshop at NeurIPS","author":"Adragna Robert","year":"2020","unstructured":"Robert Adragna, Elliot Creager, David Madras, and Richard Zemel. 2020. Fairness and robustness in invariant learning: A case study in toxicity classification. Algorithmic Fairness through the Lens of Causality and Interpretability Workshop at NeurIPS (2020)."},{"key":"e_1_3_2_1_2_1","volume-title":"International Conference on Machine Learning. PMLR, 60\u201369","author":"Agarwal Alekh","year":"2018","unstructured":"Alekh Agarwal, Alina Beygelzimer, Miroslav Dud\u00edk, John Langford, and Hanna Wallach. 2018. A reductions approach to fair classification. In International Conference on Machine Learning. PMLR, 60\u201369."},{"key":"e_1_3_2_1_3_1","volume-title":"Fairsight: Visual analytics for fairness in decision making","author":"Ahn Yongsu","year":"2019","unstructured":"Yongsu Ahn and Yu-Ru Lin. 2019. Fairsight: Visual analytics for fairness in decision making. IEEE transactions on visualization and computer graphics 26, 1(2019), 1086\u20131095."},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.1145\/3236024.3264590"},{"key":"e_1_3_2_1_5_1","volume-title":"Machine Bias. ProPublica","author":"Angwin Julia","year":"2016","unstructured":"Julia Angwin, Jeff Larson, Surya Mattu, and Lauren Kirchner. 2016. Machine Bias. ProPublica (2016)."},{"key":"e_1_3_2_1_6_1","unstructured":"Chlo\u00e9 Bakalar Renata Barreto Stevie Bergman Miranda Bogen Bobbie Chern Sam Corbett-Davies Melissa Hall Isabel Kloumann Michelle Lam Joaquin\u00a0Qui\u00f1onero Candela 2021. Fairness On The Ground: Applying Algorithmic Fairness Approaches to Production Systems. arXiv preprint arXiv:2103.06172(2021)."},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"publisher","DOI":"10.1145\/3449152"},{"key":"e_1_3_2_1_8_1","volume-title":"Accountability and Transparency in Machine Learning Workshop at KDD","author":"Bechavod Yahav","year":"2017","unstructured":"Yahav Bechavod and Katrina Ligett. 2017. Penalizing unfairness in binary classification. Fairness, Accountability and Transparency in Machine Learning Workshop at KDD (2017)."},{"key":"e_1_3_2_1_9_1","unstructured":"Rachel\u00a0KE Bellamy Kuntal Dey Michael Hind Samuel\u00a0C Hoffman Stephanie Houde Kalapriya Kannan Pranay Lohia Jacquelyn Martino Sameep Mehta Aleksandra Mojsilovic 2018. AI Fairness 360: An extensible toolkit for detecting understanding and mitigating unwanted algorithmic bias. arXiv preprint arXiv:1810.01943(2018)."},{"key":"e_1_3_2_1_10_1","volume-title":"Fairlearn: A toolkit for assessing and improving fairness in AI. Microsoft, Tech. Rep. MSR-TR-2020-32(2020).","author":"Bird Sarah","year":"2020","unstructured":"Sarah Bird, Miro Dud\u00edk, Richard Edgar, Brandon Horn, Roman Lutz, Vanessa Milan, Mehrnoosh Sameki, Hanna Wallach, and Kathleen Walker. 2020. Fairlearn: A toolkit for assessing and improving fairness in AI. Microsoft, Tech. Rep. MSR-TR-2020-32(2020)."},{"key":"e_1_3_2_1_11_1","volume-title":"Individually Fair Ranking. International Conference on Learning Representations","author":"Bower Amanda","year":"2021","unstructured":"Amanda Bower, Hamid Eftekhari, Mikhail Yurochkin, and Yuekai Sun. 2021. Individually Fair Ranking. International Conference on Learning Representations (2021)."},{"key":"e_1_3_2_1_12_1","volume-title":"Conference on fairness, accountability and transparency. PMLR, 77\u201391","author":"Buolamwini Joy","year":"2018","unstructured":"Joy Buolamwini and Timnit Gebru. 2018. Gender shades: Intersectional accuracy disparities in commercial gender classification. In Conference on fairness, accountability and transparency. PMLR, 77\u201391."},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1109\/VAST47406.2019.8986948"},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.2307\/3001666"},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.1145\/3394486.3403199"},{"key":"e_1_3_2_1_16_1","volume-title":"International Conference on Machine Learning. PMLR, 2803\u20132813","author":"Dutta Sanghamitra","year":"2020","unstructured":"Sanghamitra Dutta, Dennis Wei, Hazar Yueksel, Pin-Yu Chen, Sijia Liu, and Kush Varshney. 2020. Is there a trade-off between fairness and accuracy? a perspective using mismatched hypothesis testing. In International Conference on Machine Learning. PMLR, 2803\u20132813."},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.1145\/2090236.2090255"},{"key":"e_1_3_2_1_18_1","volume-title":"Conference on fairness, accountability and transparency. PMLR, 119\u2013133","author":"Dwork Cynthia","year":"2018","unstructured":"Cynthia Dwork, Nicole Immorlica, Adam\u00a0Tauman Kalai, and Max Leiserson. 2018. Decoupled classifiers for group-fair and efficient machine learning. In Conference on fairness, accountability and transparency. PMLR, 119\u2013133."},{"key":"e_1_3_2_1_19_1","volume-title":"An introduction to the bootstrap","author":"Efron Bradley","unstructured":"Bradley Efron and Robert\u00a0J Tibshirani. 1994. An introduction to the bootstrap. CRC press."},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"crossref","unstructured":"Sorelle\u00a0A Friedler Carlos Scheidegger Suresh Venkatasubramanian Sonam Choudhary Evan\u00a0P Hamilton and Derek Roth. 2019. A comparative study of fairness-enhancing interventions in machine learning. In Fairness Accountability and Transparency (FAT*). 329\u2013338.","DOI":"10.1145\/3287560.3287589"},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.1145\/3292500.3330691"},{"key":"e_1_3_2_1_22_1","volume-title":"Proceedings of 2nd Workshop on Diversity in Artificial Intelligence (AIDBEI)(Proceedings of Machine Learning Research, Vol.\u00a0142)","author":"Ghosh Avijit","year":"2021","unstructured":"Avijit Ghosh, Lea Genuit, and Mary Reagan. 2021. Characterizing Intersectional Group Fairness with Worst-Case Comparisons. In Proceedings of 2nd Workshop on Diversity in Artificial Intelligence (AIDBEI)(Proceedings of Machine Learning Research, Vol.\u00a0142), Deepti Lamba and William\u00a0H. Hsu (Eds.). PMLR, 22\u201334. https:\/\/proceedings.mlr.press\/v142\/ghosh21a.html"},{"key":"e_1_3_2_1_23_1","unstructured":"Google. [n.d.]. TensorFlow Fairness Indicators. https:\/\/www.tensorflow.org\/tfx\/guide\/fairness_indicators [Online; accessed 1-20-22]."},{"key":"e_1_3_2_1_24_1","volume-title":"On the Problem of Underranking in Group-Fair Ranking. In International Conference on Machine Learning. PMLR, 3777\u20133787","author":"Gorantla Sruthi","year":"2021","unstructured":"Sruthi Gorantla, Amit Deshpande, and Anand Louis. 2021. On the Problem of Underranking in Group-Fair Ranking. In International Conference on Machine Learning. PMLR, 3777\u20133787."},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.1080\/01621459.1977.10480998"},{"key":"e_1_3_2_1_26_1","volume-title":"International Conference on Machine Learning. PMLR","author":"Hashimoto Tatsunori","year":"2018","unstructured":"Tatsunori Hashimoto, Megha Srivastava, Hongseok Namkoong, and Percy Liang. 2018. Fairness without demographics in repeated loss minimization. In International Conference on Machine Learning. PMLR, 1929\u20131938."},{"key":"e_1_3_2_1_27_1","volume-title":"Statistical methods for meta-analysis","author":"Hedges V","unstructured":"Larry\u00a0V Hedges and Ingram Olkin. 1985. Statistical methods for meta-analysis. Academic press."},{"key":"e_1_3_2_1_28_1","unstructured":"Shlomi Hod. 2018\u2013. Responsibly: Toolkit for Auditing and Mitigating Bias and Fairness of Machine Learning Systems. http:\/\/docs.responsibly.ai\/ [Online; accessed 1-20-22]."},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.1145\/3287560.3287597"},{"key":"e_1_3_2_1_30_1","volume-title":"Proceedings of the 12th Language Resources and Evaluation Conference. European Language Resources Association","author":"Huang Xiaolei","year":"2020","unstructured":"Xiaolei Huang, Linzi Xing, Franck Dernoncourt, and Michael\u00a0J. Paul. 2020. Multilingual Twitter Corpus and Baselines for Evaluating Demographic Bias in Hate Speech Recognition. In Proceedings of the 12th Language Resources and Evaluation Conference. European Language Resources Association, Marseille, France, 1440\u20131448. https:\/\/aclanthology.org\/2020.lrec-1.180"},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"publisher","DOI":"10.1214\/18-AOAS1201"},{"key":"e_1_3_2_1_32_1","volume-title":"International Conference on Machine Learning. PMLR, 2564\u20132572","author":"Kearns Michael","year":"2018","unstructured":"Michael Kearns, Seth Neel, Aaron Roth, and Zhiwei\u00a0Steven Wu. 2018. Preventing fairness gerrymandering: Auditing and learning for subgroup fairness. In International Conference on Machine Learning. PMLR, 2564\u20132572."},{"key":"e_1_3_2_1_33_1","volume-title":"International conference on machine learning. PMLR, 2737\u20132746","author":"Komiyama Junpei","year":"2018","unstructured":"Junpei Komiyama, Akiko Takeda, Junya Honda, and Hajime Shimao. 2018. Nonconvex optimization for regression with fairness constraints. In International conference on machine learning. PMLR, 2737\u20132746."},{"key":"e_1_3_2_1_34_1","volume-title":"A comparison of heterogeneity variance estimators in simulated random-effects meta-analyses. Research synthesis methods 10, 1","author":"Langan Dean","year":"2019","unstructured":"Dean Langan, Julian\u00a0PT Higgins, Dan Jackson, Jack Bowden, Areti\u00a0Angeliki Veroniki, Evangelos Kontopantelis, Wolfgang Viechtbauer, and Mark Simmonds. 2019. A comparison of heterogeneity variance estimators in simulated random-effects meta-analyses. Research synthesis methods 10, 1 (2019), 83\u201398."},{"key":"e_1_3_2_1_35_1","doi-asserted-by":"crossref","unstructured":"Tomo Lazovich Luca Belli Aaron Gonzales Amanda Bower Uthaipon Tantipongpipat Kristian Lum Ferenc Huszar and Rumman Chowdhury. 2021. Measuring Disparate Outcomes of Content Recommendation Algorithms with Distributional Inequality Metrics. (2021).","DOI":"10.1016\/j.patter.2022.100568"},{"key":"e_1_3_2_1_36_1","volume-title":"International Conference on Machine Learning. PMLR, 3150\u20133158","author":"Liu T","year":"2018","unstructured":"Lydia\u00a0T Liu, Sarah Dean, Esther Rolf, Max Simchowitz, and Moritz Hardt. 2018. Delayed impact of fair machine learning. In International Conference on Machine Learning. PMLR, 3150\u20133158."},{"key":"e_1_3_2_1_37_1","volume-title":"To predict and serve?Significance 13, 5","author":"Lum Kristian","year":"2016","unstructured":"Kristian Lum and William Isaac. 2016. To predict and serve?Significance 13, 5 (2016), 14\u201319."},{"key":"e_1_3_2_1_38_1","volume-title":"International Conference on Machine Learning. PMLR, 3384\u20133393","author":"Madras David","year":"2018","unstructured":"David Madras, Elliot Creager, Toniann Pitassi, and Richard Zemel. 2018. Learning adversarially fair and transferable representations. In International Conference on Machine Learning. PMLR, 3384\u20133393."},{"key":"e_1_3_2_1_39_1","volume-title":"International Conference on Machine Learning. PMLR, 4382\u20134391","author":"Mary J\u00e9r\u00e9mie","year":"2019","unstructured":"J\u00e9r\u00e9mie Mary, Cl\u00e9ment Calauzenes, and Noureddine El\u00a0Karoui. 2019. Fairness-aware learning for continuous attributes and treatments. In International Conference on Machine Learning. PMLR, 4382\u20134391."},{"key":"e_1_3_2_1_40_1","unstructured":"Kevin\u00a0S McCurley. 2008. Income inequality in the attention economy. http:\/\/static.googleusercontent.com\/media\/research.google.com\/en\/\/pubs\/archive\/33367.pdf. (2008)."},{"key":"e_1_3_2_1_41_1","unstructured":"Guillaume Saint-Jacques Amir Sepehri Nicole Li and Igor Perisic. 2020. Fairness through Experimentation: Inequality in A\/B testing as an approach to responsible design. arXiv preprint arXiv:2002.05819(2020)."},{"key":"e_1_3_2_1_42_1","volume-title":"Aequitas: A bias and fairness audit toolkit. arXiv preprint arXiv:1811.05577(2018).","author":"Saleiro Pedro","year":"2018","unstructured":"Pedro Saleiro, Benedict Kuester, Loren Hinkson, Jesse London, Abby Stevens, Ari Anisfeld, Kit\u00a0T Rodolfa, and Rayid Ghani. 2018. Aequitas: A bias and fairness audit toolkit. arXiv preprint arXiv:1811.05577(2018)."},{"key":"e_1_3_2_1_43_1","unstructured":"Kacper Sokol Raul Santos-Rodriguez and Peter Flach. 2019. FAT Forensics: A Python toolbox for algorithmic fairness accountability and transparency. Journal of Open Source Software(2019)."},{"key":"e_1_3_2_1_44_1","doi-asserted-by":"publisher","DOI":"10.1145\/3219819.3220046"},{"key":"e_1_3_2_1_45_1","doi-asserted-by":"publisher","DOI":"10.1109\/EuroSP.2017.29"},{"key":"e_1_3_2_1_46_1","doi-asserted-by":"publisher","DOI":"10.1145\/3340531.3412705"},{"key":"e_1_3_2_1_47_1","volume-title":"Methods to estimate the between-study variance and its uncertainty in meta-analysis. Research synthesis methods 7, 1","author":"Veroniki Areti\u00a0Angeliki","year":"2016","unstructured":"Areti\u00a0Angeliki Veroniki, Dan Jackson, Wolfgang Viechtbauer, Ralf Bender, Jack Bowden, Guido Knapp, Oliver Kuss, Julian\u00a0PT Higgins, Dean Langan, and Georgia Salanti. 2016. Methods to estimate the between-study variance and its uncertainty in meta-analysis. Research synthesis methods 7, 1 (2016), 55\u201379."},{"key":"e_1_3_2_1_48_1","doi-asserted-by":"publisher","DOI":"10.3102\/10769986030003261"},{"key":"e_1_3_2_1_49_1","doi-asserted-by":"publisher","DOI":"10.1145\/3313831.3376447"},{"key":"e_1_3_2_1_50_1","volume-title":"International Conference for Learning Representations","author":"Yurochkin Mikhail","year":"2019","unstructured":"Mikhail Yurochkin, Amanda Bower, and Yuekai Sun. 2019. Training individually fair ML models with sensitive subspace robustness. International Conference for Learning Representations (2019)."},{"key":"e_1_3_2_1_51_1","doi-asserted-by":"publisher","DOI":"10.1145\/3132847.3132938"},{"key":"e_1_3_2_1_52_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.ipm.2021.102707"},{"key":"e_1_3_2_1_53_1","doi-asserted-by":"publisher","DOI":"10.1145\/3278721.3278779"}],"event":{"name":"FAccT '22: 2022 ACM Conference on Fairness, Accountability, and Transparency","location":"Seoul Republic of Korea","acronym":"FAccT '22","sponsor":["ACM Association for Computing Machinery"]},"container-title":["2022 ACM Conference on Fairness Accountability and Transparency"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3531146.3533105","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3531146.3533105","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T19:02:08Z","timestamp":1750186928000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3531146.3533105"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2022,6,20]]},"references-count":53,"alternative-id":["10.1145\/3531146.3533105","10.1145\/3531146"],"URL":"https:\/\/doi.org\/10.1145\/3531146.3533105","relation":{},"subject":[],"published":{"date-parts":[[2022,6,20]]},"assertion":[{"value":"2022-06-20","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}