{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,6,24]],"date-time":"2026-06-24T05:55:11Z","timestamp":1782280511909,"version":"3.54.5"},"publisher-location":"New York, NY, USA","reference-count":70,"publisher":"ACM","license":[{"start":{"date-parts":[[2023,4,19]],"date-time":"2023-04-19T00:00:00Z","timestamp":1681862400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2023,4,19]]},"DOI":"10.1145\/3544548.3581227","type":"proceedings-article","created":{"date-parts":[[2023,4,20]],"date-time":"2023-04-20T04:27:55Z","timestamp":1681964875000},"page":"1-20","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":14,"title":["Fairness Evaluation in Text Classification: Machine Learning Practitioner Perspectives of Individual and Group Fairness"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-0686-7911","authenticated-orcid":false,"given":"Zahra","family":"Ashktorab","sequence":"first","affiliation":[{"name":"IBM Research, IBM Research, United States"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-5218-3185","authenticated-orcid":false,"given":"Benjamin","family":"Hoover","sequence":"additional","affiliation":[{"name":"IBM Research AI, United States"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-0641-2458","authenticated-orcid":false,"given":"Mayank","family":"Agarwal","sequence":"additional","affiliation":[{"name":"IBM Research, United States"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-1508-2091","authenticated-orcid":false,"given":"Casey","family":"Dugan","sequence":"additional","affiliation":[{"name":"IBM Research, United States"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-4699-5026","authenticated-orcid":false,"given":"Werner","family":"Geyer","sequence":"additional","affiliation":[{"name":"IBM Research, United States"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-4332-3905","authenticated-orcid":false,"given":"Hao Bang","family":"Yang","sequence":"additional","affiliation":[{"name":"Massachusetts Institute of Technology, United States"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-0153-6811","authenticated-orcid":false,"given":"Mikhail","family":"Yurochkin","sequence":"additional","affiliation":[{"name":"IBM Research, United States"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2023,4,19]]},"reference":[{"key":"e_1_3_3_3_1_1","unstructured":"Alekh Agarwal Alina Beygelzimer Miroslav Dud\u00edk John Langford and Hanna Wallach. 2018. A reductions approach to fair classification. In ICML."},{"key":"e_1_3_3_3_2_1","volume-title":"Ethics of Data and Analytics","author":"Angwin Julia","unstructured":"Julia Angwin, Jeff Larson, Surya Mattu, and Lauren Kirchner. 2016. Machine bias. In Ethics of Data and Analytics. Auerbach Publications, 254\u2013264."},{"key":"e_1_3_3_3_3_1","doi-asserted-by":"publisher","DOI":"10.1145\/3442188.3445875"},{"key":"e_1_3_3_3_4_1","unstructured":"Solon Barocas Moritz Hardt and Arvind Narayanan. 2019. Fairness and Machine Learning. fairmlbook.org."},{"key":"e_1_3_3_3_5_1","doi-asserted-by":"publisher","DOI":"10.1147\/JRD.2019.2942287"},{"key":"e_1_3_3_3_6_1","doi-asserted-by":"publisher","DOI":"10.1177\/0049124118782533"},{"key":"e_1_3_3_3_7_1","doi-asserted-by":"publisher","DOI":"10.1257\/0002828042002561"},{"key":"e_1_3_3_3_8_1","volume-title":"Fairlearn: A toolkit for assessing and improving fairness in AI. Microsoft, Tech. Rep. MSR-TR-2020-32(2020).","author":"Bird Sarah","year":"2020","unstructured":"Sarah Bird, Miro Dud\u00edk, Richard Edgar, Brandon Horn, Roman Lutz, Vanessa Milan, Mehrnoosh Sameki, Hanna Wallach, and Kathleen Walker. 2020. Fairlearn: A toolkit for assessing and improving fairness in AI. Microsoft, Tech. Rep. MSR-TR-2020-32(2020)."},{"key":"e_1_3_3_3_9_1","doi-asserted-by":"publisher","DOI":"10.1145\/3308560.3317593"},{"key":"e_1_3_3_3_10_1","volume-title":"Optimized pre-processing for discrimination prevention. Advances in neural information processing systems 30","author":"Calmon Flavio","year":"2017","unstructured":"Flavio Calmon, Dennis Wei, Bhanukiran Vinzamuri, Karthikeyan Natesan\u00a0Ramamurthy, and Kush\u00a0R Varshney. 2017. Optimized pre-processing for discrimination prevention. Advances in neural information processing systems 30 (2017)."},{"key":"e_1_3_3_3_11_1","unstructured":"Richard\u00a0J Chen Tiffany\u00a0Y Chen Jana Lipkova Judy\u00a0J Wang Drew\u00a0FK Williamson Ming\u00a0Y Lu Sharifa Sahai and Faisal Mahmood. 2021. Algorithm fairness in ai for medicine and healthcare. arXiv preprint arXiv:2110.00603(2021)."},{"key":"e_1_3_3_3_12_1","volume-title":"Fair prediction with disparate impact: A study of bias in recidivism prediction instruments. Big data 5, 2","author":"Chouldechova Alexandra","year":"2017","unstructured":"Alexandra Chouldechova. 2017. Fair prediction with disparate impact: A study of bias in recidivism prediction instruments. Big data 5, 2 (2017), 153\u2013163."},{"key":"e_1_3_3_3_13_1","doi-asserted-by":"publisher","DOI":"10.1145\/3376898"},{"key":"e_1_3_3_3_14_1","unstructured":"Andrew Cotter Heinrich Jiang and Karthik Sridharan. 2019. Two-player games for efficient non-convex constrained optimization. In Algorithmic Learning Theory. PMLR 300\u2013332."},{"key":"e_1_3_3_3_15_1","unstructured":"Kate Crawford. 2017. The trouble with bias. keynote at neurips. (2017)."},{"key":"e_1_3_3_3_16_1","doi-asserted-by":"publisher","DOI":"10.1145\/3287560.3287572"},{"key":"e_1_3_3_3_17_1","doi-asserted-by":"publisher","DOI":"10.1145\/3531146.3533113"},{"key":"e_1_3_3_3_18_1","first-page":"246","article-title":". On Measures of Biases and Harms in NLP","volume":"2022","author":"Dev Sunipa","year":"2022","unstructured":"Sunipa Dev, Emily Sheng, Jieyu Zhao, Aubrie Amstutz, Jiao Sun, Yu Hou, Mattie Sanseverino, Jiin Kim, Akihiro Nishi, Nanyun Peng, 2022. On Measures of Biases and Harms in NLP. In Findings of the Association for Computational Linguistics: AACL-IJCNLP 2022. 246\u2013267.","journal-title":"Findings of the Association for Computational Linguistics: AACL-IJCNLP"},{"key":"e_1_3_3_3_19_1","volume-title":"Bert: Pre-training of deep bidirectional transformers for language understanding. arXiv preprint arXiv:1810.04805(2018).","author":"Devlin Jacob","year":"2018","unstructured":"Jacob Devlin, Ming-Wei Chang, Kenton Lee, and Kristina Toutanova. 2018. Bert: Pre-training of deep bidirectional transformers for language understanding. arXiv preprint arXiv:1810.04805(2018)."},{"key":"e_1_3_3_3_20_1","doi-asserted-by":"publisher","DOI":"10.1145\/3278721.3278729"},{"key":"e_1_3_3_3_21_1","doi-asserted-by":"crossref","unstructured":"Cynthia Dwork Moritz Hardt Toniann Pitassi Omer Reingold and Richard Zemel. 2012. Fairness through awareness. In ITCS.","DOI":"10.1145\/2090236.2090255"},{"key":"e_1_3_3_3_22_1","doi-asserted-by":"publisher","DOI":"10.1145\/3461702.3462621"},{"key":"e_1_3_3_3_23_1","doi-asserted-by":"publisher","DOI":"10.1145\/3287560.3287589"},{"key":"e_1_3_3_3_24_1","doi-asserted-by":"publisher","DOI":"10.1145\/3306618.3317950"},{"key":"e_1_3_3_3_25_1","doi-asserted-by":"crossref","unstructured":"Andrew Gaut Tony Sun Shirlyn Tang Yuxin Huang Jing Qian Mai ElSherief Jieyu Zhao Diba Mirza Elizabeth Belding Kai-Wei Chang 2019. Towards understanding gender bias in relation extraction. arXiv preprint arXiv:1911.03642(2019).","DOI":"10.18653\/v1\/2020.acl-main.265"},{"key":"e_1_3_3_3_26_1","volume-title":"Equality of opportunity in supervised learning. Advances in neural information processing systems 29","author":"Hardt Moritz","year":"2016","unstructured":"Moritz Hardt, Eric Price, and Nati Srebro. 2016. Equality of opportunity in supervised learning. Advances in neural information processing systems 29 (2016)."},{"key":"e_1_3_3_3_27_1","volume-title":"Dimensionalizing cultures: The Hofstede model in context. Online readings in psychology and culture 2, 1","author":"Hofstede Geert","year":"2011","unstructured":"Geert Hofstede. 2011. Dimensionalizing cultures: The Hofstede model in context. Online readings in psychology and culture 2, 1 (2011), 2307\u20130919."},{"key":"e_1_3_3_3_28_1","doi-asserted-by":"publisher","DOI":"10.1145\/3290605.3300830"},{"key":"e_1_3_3_3_29_1","unstructured":"Christina Ilvento. 2019. Metric learning for individual fairness. arXiv preprint arXiv:1906.00250(2019)."},{"key":"e_1_3_3_3_30_1","doi-asserted-by":"publisher","DOI":"10.1145\/3267851.3267916"},{"key":"e_1_3_3_3_31_1","doi-asserted-by":"publisher","DOI":"10.1371\/journal.pone.0256762"},{"key":"e_1_3_3_3_32_1","unstructured":"Jean-Marie John-Mathews Dominique Cardon and Christine Balagu\u00e9. 2022. From reality to world. A critical perspective on AI fairness. Journal of Business Ethics(2022) 1\u201315."},{"key":"e_1_3_3_3_33_1","unstructured":"Jon Kleinberg Sendhil Mullainathan and Manish Raghavan. 2016. Inherent trade-offs in the fair determination of risk scores. arXiv preprint arXiv:1609.05807(2016)."},{"key":"e_1_3_3_3_34_1","volume-title":"International Conference on Machine Learning. PMLR, 5637\u20135664","author":"Koh Pang\u00a0Wei","year":"2021","unstructured":"Pang\u00a0Wei Koh, Shiori Sagawa, Henrik Marklund, Sang\u00a0Michael Xie, Marvin Zhang, Akshay Balsubramani, Weihua Hu, Michihiro Yasunaga, Richard\u00a0Lanas Phillips, Irena Gao, 2021. Wilds: A benchmark of in-the-wild distribution shifts. In International Conference on Machine Learning. PMLR, 5637\u20135664."},{"key":"e_1_3_3_3_35_1","doi-asserted-by":"publisher","DOI":"10.1109\/TTS.2020.2974996"},{"key":"e_1_3_3_3_36_1","first-page":"22","article-title":"Spam detection using artificial neural networks (perceptron learning rule)","volume":"1","author":"Kufandirimbwa Owen","year":"2012","unstructured":"Owen Kufandirimbwa and Richard Gotora. 2012. Spam detection using artificial neural networks (perceptron learning rule). Online Journal of Physical and Environmental Science Research 1, 2(2012), 22\u201329.","journal-title":"Online Journal of Physical and Environmental Science Research"},{"key":"e_1_3_3_3_37_1","unstructured":"Deepak Kumar Patrick\u00a0Gage Kelley Sunny Consolvo Joshua Mason Elie Bursztein Zakir Durumeric Kurt Thomas and Michael Bailey. 2021. Designing Toxic Content Classification for a Diversity of Perspectives.. In SOUPS@ USENIX Security Symposium. 299\u2013318."},{"key":"e_1_3_3_3_38_1","doi-asserted-by":"crossref","unstructured":"Tao Li Tushar Khot Daniel Khashabi Ashish Sabharwal and Vivek Srikumar. 2020. UNQOVERing stereotyping biases via underspecified questions. arXiv preprint arXiv:2010.02428(2020).","DOI":"10.18653\/v1\/2020.findings-emnlp.311"},{"key":"e_1_3_3_3_39_1","doi-asserted-by":"publisher","DOI":"10.1145\/3512899"},{"key":"e_1_3_3_3_40_1","doi-asserted-by":"publisher","DOI":"10.1145\/3313831.3376445"},{"key":"e_1_3_3_3_41_1","unstructured":"Subha Maity Songkai Xue Mikhail Yurochkin and Yuekai Sun. 2021. Statistical inference for individual fairness. arXiv preprint arXiv:2103.16714(2021)."},{"key":"e_1_3_3_3_42_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-78818-6_5"},{"key":"e_1_3_3_3_43_1","doi-asserted-by":"publisher","DOI":"10.1145\/3313831.3376257"},{"key":"e_1_3_3_3_44_1","volume-title":"Algorithmic misogynoir in content moderation practice","author":"Marshall Brandeis","year":"2021","unstructured":"Brandeis Marshall. 2021. Algorithmic misogynoir in content moderation practice. Heinrich-B\u00f6ll-Stiftung European Union(2021)."},{"key":"e_1_3_3_3_45_1","unstructured":"Debarghya Mukherjee Mikhail Yurochkin Moulinath Banerjee and Yuekai Sun. 2020. Two simple ways to learn individual fairness metrics from data. In ICML."},{"key":"e_1_3_3_3_46_1","volume-title":"Towards Responsible AI: A Design Space Exploration of Human-Centered Artificial Intelligence User Interfaces to Investigate Fairness. International Journal of Human\u2013Computer Interaction","author":"Nakao Yuri","year":"2022","unstructured":"Yuri Nakao, Lorenzo Strappelli, Simone Stumpf, Aisha Naseer, Daniele Regoli, and Giulia\u00a0Del Gamba. 2022. Towards Responsible AI: A Design Space Exploration of Human-Centered Artificial Intelligence User Interfaces to Investigate Fairness. International Journal of Human\u2013Computer Interaction (2022), 1\u201327."},{"key":"e_1_3_3_3_47_1","volume-title":"Towards Involving End-users in Interactive Human-in-the-loop AI Fairness. ACM Transactions on Interactive Intelligent Systems (TiiS)","author":"Nakao Yuri","year":"2022","unstructured":"Yuri Nakao, Simone Stumpf, Subeida Ahmed, Aisha Naseer, and Lorenzo Strappelli. 2022. Towards Involving End-users in Interactive Human-in-the-loop AI Fairness. ACM Transactions on Interactive Intelligent Systems (TiiS) (2022)."},{"key":"e_1_3_3_3_48_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D18-1302"},{"key":"e_1_3_3_3_49_1","first-page":"25944","article-title":"Post-processing for individual fairness","volume":"34","author":"Petersen Felix","year":"2021","unstructured":"Felix Petersen, Debarghya Mukherjee, Yuekai Sun, and Mikhail Yurochkin. 2021. Post-processing for individual fairness. Advances in Neural Information Processing Systems 34 (2021), 25944\u201325955.","journal-title":"Advances in Neural Information Processing Systems"},{"key":"e_1_3_3_3_50_1","unstructured":"David\u00a0MW Powers. 2020. Evaluation: from precision recall and F-measure to ROC informedness markedness and correlation. arXiv preprint arXiv:2010.16061(2020)."},{"key":"e_1_3_3_3_51_1","doi-asserted-by":"crossref","unstructured":"Flavien Prost Nithum Thain and Tolga Bolukbasi. 2019. Debiasing embeddings for reduced gender bias in text classification. arXiv preprint arXiv:1908.02810(2019).","DOI":"10.18653\/v1\/W19-3810"},{"key":"e_1_3_3_3_52_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2021.findings-acl.294"},{"key":"e_1_3_3_3_53_1","unstructured":"Tim R\u00e4z. 2022. Gerrymandering Individual Fairness. arXiv preprint arXiv:2204.11615(2022)."},{"key":"e_1_3_3_3_54_1","doi-asserted-by":"publisher","DOI":"10.1145\/3411764.3445604"},{"key":"e_1_3_3_3_55_1","volume-title":"Aequitas: A bias and fairness audit toolkit. arXiv preprint arXiv:1811.05577(2018).","author":"Saleiro Pedro","year":"2018","unstructured":"Pedro Saleiro, Benedict Kuester, Loren Hinkson, Jesse London, Abby Stevens, Ari Anisfeld, Kit\u00a0T Rodolfa, and Rayid Ghani. 2018. Aequitas: A bias and fairness audit toolkit. arXiv preprint arXiv:1811.05577(2018)."},{"key":"e_1_3_3_3_56_1","doi-asserted-by":"publisher","DOI":"10.1145\/3306618.3314248"},{"key":"e_1_3_3_3_57_1","doi-asserted-by":"crossref","unstructured":"Emily Sheng Kai-Wei Chang Premkumar Natarajan and Nanyun Peng. 2019. The woman worked as a babysitter: On biases in language generation. arXiv preprint arXiv:1909.01326(2019).","DOI":"10.18653\/v1\/D19-1339"},{"key":"e_1_3_3_3_58_1","volume-title":"Your Fairness May Vary: Pretrained Language Model Fairness in Toxic Text Classification. In Annual Meeting of the Association for Computational Linguistics.","author":"Soares Ioana\u00a0Baldini","year":"2022","unstructured":"Ioana\u00a0Baldini Soares, Dennis Wei, Karthikeyan\u00a0Natesan Ramamurthy, Moninder Singh, and Mikhail Yurochkin. 2022. Your Fairness May Vary: Pretrained Language Model Fairness in Toxic Text Classification. In Annual Meeting of the Association for Computational Linguistics."},{"key":"e_1_3_3_3_59_1","doi-asserted-by":"publisher","DOI":"10.1145\/3292500.3330664"},{"key":"e_1_3_3_3_60_1","doi-asserted-by":"crossref","unstructured":"Gabriel Stanovsky Noah\u00a0A Smith and Luke Zettlemoyer. 2019. Evaluating gender bias in machine translation. arXiv preprint arXiv:1906.00591(2019).","DOI":"10.18653\/v1\/P19-1164"},{"key":"e_1_3_3_3_61_1","doi-asserted-by":"publisher","DOI":"10.1145\/3173574.3174014"},{"key":"e_1_3_3_3_62_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.cose.2022.102786"},{"key":"e_1_3_3_3_63_1","doi-asserted-by":"crossref","unstructured":"Thomas Wolf Lysandre Debut Victor Sanh Julien Chaumond Clement Delangue Anthony Moi Pierric Cistac Tim Rault R\u00e9mi Louf Morgan Funtowicz 2019. Huggingface\u2019s transformers: State-of-the-art natural language processing. arXiv preprint arXiv:1910.03771(2019).","DOI":"10.18653\/v1\/2020.emnlp-demos.6"},{"key":"e_1_3_3_3_64_1","doi-asserted-by":"publisher","DOI":"10.1145\/3173574.3174230"},{"key":"e_1_3_3_3_65_1","unstructured":"Songkai Xue Mikhail Yurochkin and Yuekai Sun. 2020. Auditing ml models for individual bias and unfairness. In AI&STAT."},{"key":"e_1_3_3_3_66_1","volume-title":"Beyond parity: Fairness objectives for collaborative filtering. Advances in neural information processing systems 30","author":"Yao Sirui","year":"2017","unstructured":"Sirui Yao and Bert Huang. 2017. Beyond parity: Fairness objectives for collaborative filtering. Advances in neural information processing systems 30 (2017)."},{"key":"e_1_3_3_3_67_1","volume-title":"International Conference on Learning Representations.","author":"Yurochkin Mikhail","year":"2020","unstructured":"Mikhail Yurochkin, Amanda Bower, and Yuekai Sun. 2020. Training individually fair ML models with sensitive subspace robustness. In International Conference on Learning Representations."},{"key":"e_1_3_3_3_68_1","volume-title":"International Conference on Learning Representations. https:\/\/github.com\/IBM\/inFairness","author":"Yurochkin Mikhail","year":"2021","unstructured":"Mikhail Yurochkin and Yuekai Sun. 2021. SenSeI: Sensitive set invariance for enforcing individual fairness. In International Conference on Learning Representations. https:\/\/github.com\/IBM\/inFairness"},{"key":"e_1_3_3_3_69_1","volume-title":"Individual and contextual factors associated with school staff responses to homophobic bullying. Sexuality research and social policy 16, 4","author":"Zotti Davide","year":"2019","unstructured":"Davide Zotti, Andrea Carnaghi, Valentina Piccoli, and Mauro Bianchi. 2019. Individual and contextual factors associated with school staff responses to homophobic bullying. Sexuality research and social policy 16, 4 (2019), 543\u2013558."},{"key":"e_1_3_3_3_70_1","doi-asserted-by":"crossref","unstructured":"James Zou and Londa Schiebinger. 2018. AI can be sexist and racist\u2014it\u2019s time to make it fair.","DOI":"10.1038\/d41586-018-05707-8"}],"event":{"name":"CHI '23: CHI Conference on Human Factors in Computing Systems","location":"Hamburg Germany","acronym":"CHI '23","sponsor":["SIGCHI ACM Special Interest Group on Computer-Human Interaction"]},"container-title":["Proceedings of the 2023 CHI Conference on Human Factors in Computing Systems"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3544548.3581227","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3544548.3581227","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T16:37:29Z","timestamp":1750178249000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3544548.3581227"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023,4,19]]},"references-count":70,"alternative-id":["10.1145\/3544548.3581227","10.1145\/3544548"],"URL":"https:\/\/doi.org\/10.1145\/3544548.3581227","relation":{},"subject":[],"published":{"date-parts":[[2023,4,19]]},"assertion":[{"value":"2023-04-19","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}