{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,6,30]],"date-time":"2026-06-30T06:25:06Z","timestamp":1782800706581,"version":"3.54.5"},"publisher-location":"New York, NY, USA","reference-count":102,"publisher":"ACM","license":[{"start":{"date-parts":[[2026,6,23]],"date-time":"2026-06-23T00:00:00Z","timestamp":1782172800000},"content-version":"vor","delay-in-days":365,"URL":"http:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"DOI":"10.13039\/100000006","name":"Office of Naval Research","doi-asserted-by":"publisher","award":["N00014-23-1-2436"],"award-info":[{"award-number":["N00014-23-1-2436"]}],"id":[{"id":"10.13039\/100000006","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2025,6,23]]},"DOI":"10.1145\/3715275.3732067","type":"proceedings-article","created":{"date-parts":[[2025,6,23]],"date-time":"2025-06-23T17:03:13Z","timestamp":1750698193000},"page":"1028-1047","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":2,"title":["Towards Effective Discrimination Testing for Generative AI"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0009-0001-6840-4015","authenticated-orcid":false,"given":"Thomas","family":"Zollo","sequence":"first","affiliation":[{"name":"Columbia University, New York, New York, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0004-2271-3551","authenticated-orcid":false,"given":"Nikita","family":"Rajaneesh","sequence":"additional","affiliation":[{"name":"Columbia University, New York, New York, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-9353-7509","authenticated-orcid":false,"given":"Richard","family":"Zemel","sequence":"additional","affiliation":[{"name":"Columbia University, New York, New York, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-8084-5158","authenticated-orcid":false,"given":"Talia","family":"Gillis","sequence":"additional","affiliation":[{"name":"Columbia University, New York, New York, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-4301-3739","authenticated-orcid":false,"given":"Emily","family":"Black","sequence":"additional","affiliation":[{"name":"New York University, New York, New York, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2025,6,23]]},"reference":[{"key":"e_1_3_3_2_2_2","unstructured":"Usman Anwar Abulhair Saparov Javier Rando Daniel Paleka Miles Turpin Peter Hase Ekdeep\u00a0Singh Lubana Erik Jenner Stephen Casper Oliver Sourbut Benjamin\u00a0L. Edelman Zhaowei Zhang Mario G\u00fcnther Anton Korinek Jose Hernandez-Orallo Lewis Hammond Eric Bigelow Alexander Pan Lauro Langosco Tomasz Korbak Heidi Zhang Ruiqi Zhong Se\u00e1n\u00a0\u00d3 h\u00c9igeartaigh Gabriel Recchia Giulio Corsi Alan Chan Markus Anderljung Lilian Edwards Aleksandar Petrov Christian\u00a0Schroeder de Witt Sumeet\u00a0Ramesh Motwan Yoshua Bengio Danqi Chen Philip H.\u00a0S. Torr Samuel Albanie Tegan Maharaj Jakob Foerster Florian Tramer He He Atoosa Kasirzadeh Yejin Choi and David Krueger. 2024. Foundational Challenges in Assuring Alignment and Safety of Large Language Models. Transactions on Machine Learning Research (2024)."},{"key":"e_1_3_3_2_3_2","doi-asserted-by":"crossref","unstructured":"Lisa\u00a0P. Argyle E. Busby Nancy Fulda Joshua\u00a0R Gubler Christopher Rytting and David Wingate. 2022. Out of One Many: Using Language Models to Simulate Human Samples. Political Analysis 31 (2022) 337 \u2013 351.","DOI":"10.1017\/pan.2023.2"},{"key":"e_1_3_3_2_4_2","unstructured":"Yuntao Bai Andy Jones Kamal Ndousse Amanda Askell Anna Chen Nova DasSarma Dawn Drain Stanislav Fort Deep Ganguli Tom Henighan Nicholas Joseph Saurav Kadavath Jackson Kernion Tom Conerly Sheer El-Showk Nelson Elhage Zac Hatfield-Dodds Danny Hernandez Tristan Hume Scott Johnston Shauna Kravec Liane Lovitt Neel Nanda Catherine Olsson Dario Amodei Tom Brown Jack Clark Sam McCandlish Chris Olah Ben Mann and Jared Kaplan. 2022. Training a Helpful and Harmless Assistant with Reinforcement Learning from Human Feedback. arXiv:https:\/\/arXiv.org\/abs\/2204.05862 (2022)."},{"key":"e_1_3_3_2_5_2","unstructured":"Yanhong Bai Jiabao Zhao Jinxin Shi Zhentao Xie Xingjiao Wu and Liang He. 2024. FairMonitor: A Dual-framework for Detecting Stereotypes and Biases in Large Language Models. arXiv:https:\/\/arXiv.org\/abs\/2405.03098 (2024)."},{"key":"e_1_3_3_2_6_2","unstructured":"Ainhize Barrainkua Paula Gordaliza Jose\u00a0A. Lozano and Novi Quadrianto. 2023. Preserving the Fairness Guarantees of Classifiers in Changing Environments: a Survey. ACM Comput. Surv. (2023)."},{"key":"e_1_3_3_2_7_2","doi-asserted-by":"crossref","unstructured":"Asma Ben\u00a0Abacha and Dina Demner-Fushman. 2019. A question-entailment approach to question answering. BMC Bioinformatics:1471-2105 (2019).","DOI":"10.1186\/s12859-019-3119-4"},{"key":"e_1_3_3_2_8_2","doi-asserted-by":"publisher","DOI":"10.1145\/3442188.3445922"},{"key":"e_1_3_3_2_9_2","doi-asserted-by":"publisher","DOI":"10.3386\/w9873"},{"key":"e_1_3_3_2_10_2","doi-asserted-by":"publisher","DOI":"10.1145\/3593013.3594095"},{"key":"e_1_3_3_2_11_2","doi-asserted-by":"publisher","DOI":"10.1145\/3442188.3445894"},{"key":"e_1_3_3_2_12_2","doi-asserted-by":"publisher","DOI":"10.1145\/3630106.3658928"},{"key":"e_1_3_3_2_13_2","unstructured":"Bloomberg. 2024. OpenAI Faces Scrutiny Over Racial Discrimination in Hiring Practices. Bloomberg (2024)."},{"key":"e_1_3_3_2_14_2","doi-asserted-by":"crossref","unstructured":"Samuel\u00a0R. Bowman and George\u00a0E. Dahl. 2021. What Will it Take to Fix Benchmarking in Natural Language Understanding? arxiv:https:\/\/arXiv.org\/abs\/2104.02145\u00a0[cs.CL] https:\/\/arxiv.org\/abs\/2104.02145","DOI":"10.18653\/v1\/2021.naacl-main.385"},{"key":"e_1_3_3_2_15_2","unstructured":"Miles Brundage Shahar Avin Jasmine Wang Haydn Belfield Gretchen Krueger Gillian Hadfield Heidy Khlaaf Jingying Yang Helen Toner Ruth Fong Tegan Maharaj Pang\u00a0Wei Koh Sara Hooker Jade Leung Andrew Trask Emma Bluemke Jonathan Lebensold Cullen O\u2019Keefe Mark Koren Th\u00e9o Ryffel JB Rubinovitz Tamay Besiroglu Federica Carugati Jack Clark Peter Eckersley Sarah de Haas Maritza Johnson Ben Laurie Alex Ingerman Igor Krawczuk Amanda Askell Rosario Cammarota Andrew Lohn David Krueger Charlotte Stix Peter Henderson Logan Graham Carina Prunkl Bianca Martin Elizabeth Seger Noa Zilberman Se\u00e1n\u00a0\u00d3 h\u00c9igeartaigh Frens Kroeger Girish Sastry Rebecca Kagan Adrian Weller Brian Tse Elizabeth Barnes Allan Dafoe Paul Scharre Ariel Herbert-Voss Martijn Rasser Shagun Sodhani Carrick Flynn Thomas\u00a0Krendl Gilbert Lisa Dyer Saif Khan Yoshua Bengio and Markus Anderljung. 2020. Toward Trustworthy AI Development: Mechanisms for Supporting Verifiable Claims."},{"key":"e_1_3_3_2_16_2","unstructured":"Canada. 2023. Voluntary Code of Conduct on the Responsible Development and Management of Advanced Generative AI Systems. Policy Document."},{"key":"e_1_3_3_2_17_2","unstructured":"Canada. 2024. Artificial Intelligence and Data Act (AIDA). Proposed Legislation."},{"key":"e_1_3_3_2_18_2","doi-asserted-by":"publisher","DOI":"10.5555\/3620237.3620531"},{"key":"e_1_3_3_2_19_2","volume-title":"USENIX Security Symposium","author":"Carlini Nicholas","year":"2020","unstructured":"Nicholas Carlini, Florian Tram\u00e8r, Eric Wallace, Matthew Jagielski, Ariel Herbert-Voss, Katherine Lee, Adam Roberts, Tom\u00a0B. Brown, Dawn\u00a0Xiaodong Song, \u00dalfar Erlingsson, Alina Oprea, and Colin Raffel. 2020. Extracting Training Data from Large Language Models. In USENIX Security Symposium."},{"key":"e_1_3_3_2_20_2","doi-asserted-by":"crossref","unstructured":"Spencer Caro Talia\u00a0B. Gillis and Scott Nelson. 2023. Modernizing Fair Lending. SSRN Electronic Journal (2023).","DOI":"10.2139\/ssrn.4728095"},{"key":"e_1_3_3_2_21_2","unstructured":"CFPB. 2022. CFPB Circular 2022-03: Adverse Action Notification Requirements in Connection with Credit Decisions Based on Complex Algorithms."},{"key":"e_1_3_3_2_22_2","volume-title":"Robustness of zero\/few-shot learning in foundation models (R0-FoMo)","author":"Chao Patrick","year":"2024","unstructured":"Patrick Chao, Alexander Robey, Edgar Dobriban, Hamed Hassani, George\u00a0J. Pappas, and Eric Wong. 2024. Jailbreaking Black Box Large Language Models in Twenty Queries. In Robustness of zero\/few-shot learning in foundation models (R0-FoMo)."},{"key":"e_1_3_3_2_23_2","volume-title":"Forty-first International Conference on Machine Learning","author":"Chiang Wei-Lin","year":"2024","unstructured":"Wei-Lin Chiang, Lianmin Zheng, Ying Sheng, Anastasios\u00a0Nikolas Angelopoulos, Tianle Li, Dacheng Li, Banghua Zhu, Hao Zhang, Michael\u00a0I. Jordan, Joseph\u00a0E. Gonzalez, and Ion Stoica. 2024. Chatbot Arena: An Open Platform for Evaluating LLMs by Human Preference. In Forty-first International Conference on Machine Learning."},{"key":"e_1_3_3_2_24_2","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV51070.2023.00283"},{"key":"e_1_3_3_2_25_2","unstructured":"Karl Cobbe Vineet Kosaraju Mohammad Bavarian Mark Chen Heewoo Jun Lukasz Kaiser Matthias Plappert Jerry Tworek Jacob Hilton Reiichiro Nakano Christopher Hesse and John Schulman. 2021. Training Verifiers to Solve Math Word Problems. arXiv:https:\/\/arXiv.org\/abs\/2110.14168 (2021)."},{"key":"e_1_3_3_2_26_2","unstructured":"A. Cooper Solon Barocas Christopher De\u00a0Sa and Siddhartha Sen. 2023. Variance Self-Consistency and Arbitrariness in Fair Classification. arXiv:https:\/\/arXiv.org\/abs\/2301.11562 (2023)."},{"key":"e_1_3_3_2_27_2","unstructured":"Jeffrey Dastin. 2018. Amazon scraps secret AI recruiting tool that showed bias against women. Reuters (2018)."},{"key":"e_1_3_3_2_28_2","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2023.emnlp-main.421"},{"key":"e_1_3_3_2_29_2","doi-asserted-by":"crossref","unstructured":"Guido Noto\u00a0La Diega and Leonardo\u00a0C.T. Bezerra. 2024. Can there be responsible AI without AI liability? Incentivizing generative AI safety through ex-post tort liability under the EU AI liability directive. International Journal of Law and Information Technology (2024).","DOI":"10.1093\/ijlit\/eaae021"},{"key":"e_1_3_3_2_30_2","volume-title":"Advances in Neural Information Processing Systems","author":"Ding Frances","year":"2021","unstructured":"Frances Ding, Moritz Hardt, John Miller, and Ludwig Schmidt. 2021. Retiring Adult: New Datasets for Fair Machine Learning. In Advances in Neural Information Processing Systems."},{"key":"e_1_3_3_2_31_2","unstructured":"Directive 43\/EC. 2000. Directive 2000\/43\/EC implementing the principle of equal treatment between persons irrespective of racial or ethnic origin."},{"key":"e_1_3_3_2_32_2","unstructured":"Directive 54\/EC. 2006. Directive 2006\/54\/EC on the implementation of the principle of equal opportunities and equal treatment of men and women in matters of employment and occupation (recast)."},{"key":"e_1_3_3_2_33_2","unstructured":"Directive 78\/EC. 2000. Directive 2000\/78\/EC establishing a general framework for equal treatment in employment and occupation."},{"key":"e_1_3_3_2_34_2","unstructured":"DSIT. 2024. The Model for Responsible Innovation."},{"key":"e_1_3_3_2_35_2","unstructured":"Abhimanyu Dubey Abhinav Jauhri Abhinav Pandey Abhishek Kadian Ahmad Al-Dahle Aiesha Letman Akhil Mathur Alan Schelten Amy Yang Angela Fan Anirudh Goyal Anthony Hartshorn Aobo Yang Archi Mitra Archie Sravankumar Artem Korenev Arthur Hinsvark Arun Rao Aston Zhang Aurelien Rodriguez Austen Gregerson Ava Spataru Baptiste Roziere Bethany Biron Binh Tang Bobbie Chern Charlotte Caucheteux Chaya Nayak Chloe Bi Chris Marra Chris McConnell Christian Keller Christophe Touret Chunyang Wu Corinne Wong Cristian\u00a0Canton Ferrer Cyrus Nikolaidis Damien Allonsius Daniel Song Danielle Pintz Danny Livshits David Esiobu Dhruv Choudhary Dhruv Mahajan Diego Garcia-Olano Diego Perino Dieuwke Hupkes Egor Lakomkin Ehab AlBadawy Elina Lobanova Emily Dinan Eric\u00a0Michael Smith Filip Radenovic Frank Zhang Gabriel Synnaeve Gabrielle Lee Georgia\u00a0Lewis Anderson Graeme Nail Gregoire Mialon Guan Pang Guillem Cucurell Hailey Nguyen Hannah Korevaar Hu Xu Hugo Touvron Iliyan Zarov Imanol\u00a0Arrieta Ibarra Isabel Kloumann Ishan Misra Ivan Evtimov Jade Copet Jaewon Lee Jan Geffert Jana Vranes Jason Park Jay Mahadeokar Jeet Shah Jelmer van\u00a0der Linde Jennifer Billock Jenny Hong Jenya Lee Jeremy Fu Jianfeng Chi Jianyu Huang Jiawen Liu Jie Wang Jiecao Yu Joanna Bitton Joe Spisak Jongsoo Park Joseph Rocca Joshua Johnstun Joshua Saxe Junteng Jia Kalyan\u00a0Vasuden Alwala Kartikeya Upasani Kate Plawiak Ke Li Kenneth Heafield Kevin Stone Khalid El-Arini Krithika Iyer Kshitiz Malik Kuenley Chiu Kunal Bhalla Lauren Rantala-Yeary Laurens van\u00a0der Maaten Lawrence Chen Liang Tan Liz Jenkins Louis Martin Lovish Madaan Lubo Malo Lukas Blecher Lukas Landzaat Luke de Oliveira Madeline Muzzi Mahesh Pasupuleti Mannat Singh Manohar Paluri Marcin Kardas Mathew Oldham Mathieu Rita Maya Pavlova Melanie Kambadur Mike Lewis Min Si Mitesh\u00a0Kumar Singh Mona Hassan Naman Goyal Narjes Torabi Nikolay Bashlykov Nikolay Bogoychev Niladri Chatterji Olivier Duchenne Onur \u00c7elebi Patrick Alrassy Pengchuan Zhang Pengwei Li Petar Vasic Peter Weng Prajjwal Bhargava Pratik Dubal Praveen Krishnan Punit\u00a0Singh Koura Puxin Xu Qing He Qingxiao Dong Ragavan Srinivasan Raj Ganapathy Ramon Calderer Ricardo\u00a0Silveira Cabral Robert Stojnic Roberta Raileanu Rohit Girdhar Rohit Patel Romain Sauvestre Ronnie Polidoro Roshan Sumbaly Ross Taylor Ruan Silva Rui Hou Rui Wang Saghar Hosseini Sahana Chennabasappa Sanjay Singh Sean Bell Seohyun\u00a0Sonia Kim Sergey Edunov Shaoliang Nie Sharan Narang Sharath Raparthy Sheng Shen Shengye Wan Shruti Bhosale Shun Zhang Simon Vandenhende Soumya Batra Spencer Whitman Sten Sootla Stephane Collot Suchin Gururangan Sydney Borodinsky Tamar Herman Tara Fowler Tarek Sheasha Thomas Georgiou Thomas Scialom Tobias Speckbacher Todor Mihaylov Tong Xiao Ujjwal Karn Vedanuj Goswami Vibhor Gupta Vignesh Ramanathan Viktor Kerkez Vincent Gonguet Virginie Do Vish Vogeti Vladan Petrovic Weiwei Chu Wenhan Xiong Wenyin Fu Whitney Meers Xavier Martinet Xiaodong Wang Xiaoqing\u00a0Ellen Tan Xinfeng Xie Xuchao Jia Xuewei Wang Yaelle Goldschlag Yashesh Gaur Yasmine Babaei Yi Wen Yiwen Song Yuchen Zhang Yue Li Yuning Mao Zacharie\u00a0Delpierre Coudert Zheng Yan Zhengxing Chen Zoe Papakipos Aaditya Singh Aaron Grattafiori Abha Jain Adam Kelsey Adam Shajnfeld Adithya Gangidi Adolfo Victoria Ahuva Goldstand Ajay Menon Ajay Sharma Alex Boesenberg Alex Vaughan Alexei Baevski Allie Feinstein Amanda Kallet Amit Sangani Anam Yunus Andrei Lupu Andres Alvarado Andrew Caples Andrew Gu Andrew Ho Andrew Poulton Andrew Ryan Ankit Ramchandani Annie Franco Aparajita Saraf Arkabandhu Chowdhury Ashley Gabriel Ashwin Bharambe Assaf Eisenman Azadeh Yazdan Beau James Ben Maurer Benjamin Leonhardi Bernie Huang Beth Loyd Beto\u00a0De Paola Bhargavi Paranjape Bing Liu Bo Wu Boyu Ni Braden Hancock Bram Wasti Brandon Spence Brani Stojkovic Brian Gamido Britt Montalvo Carl Parker Carly Burton Catalina Mejia Changhan Wang Changkyu Kim Chao Zhou Chester Hu Ching-Hsiang Chu Chris Cai Chris Tindal Christoph Feichtenhofer Damon Civin Dana Beaty Daniel Kreymer Daniel Li Danny Wyatt David Adkins David Xu Davide Testuggine Delia David Devi Parikh Diana Liskovich Didem Foss Dingkang Wang Duc Le Dustin Holland Edward Dowling Eissa Jamil Elaine Montgomery Eleonora Presani Emily Hahn Emily Wood Erik Brinkman Esteban Arcaute Evan Dunbar Evan Smothers Fei Sun Felix Kreuk Feng Tian Firat Ozgenel Francesco Caggioni Francisco Guzm\u00e1n Frank Kanayet Frank Seide Gabriela\u00a0Medina Florez Gabriella Schwarz Gada Badeer Georgia Swee Gil Halpern Govind Thattai Grant Herman Grigory Sizov Guangyi Zhang Guna Lakshminarayanan Hamid Shojanazeri Han Zou Hannah Wang Hanwen Zha Haroun Habeeb Harrison Rudolph Helen Suk Henry Aspegren Hunter Goldman Ibrahim Damlaj Igor Molybog Igor Tufanov Irina-Elena Veliche Itai Gat Jake Weissman James Geboski James Kohli Japhet Asher Jean-Baptiste Gaya Jeff Marcus Jeff Tang Jennifer Chan Jenny Zhen Jeremy Reizenstein Jeremy Teboul Jessica Zhong Jian Jin Jingyi Yang Joe Cummings Jon Carvill Jon Shepard Jonathan McPhie Jonathan Torres Josh Ginsburg Junjie Wang Kai Wu Kam\u00a0Hou U Karan Saxena Karthik Prasad Kartikay Khandelwal Katayoun Zand Kathy Matosich Kaushik Veeraraghavan Kelly Michelena Keqian Li Kun Huang Kunal Chawla Kushal Lakhotia Kyle Huang Lailin Chen Lakshya Garg Lavender A Leandro Silva Lee Bell Lei Zhang Liangpeng Guo Licheng Yu Liron Moshkovich Luca Wehrstedt Madian Khabsa Manav Avalani Manish Bhatt Maria Tsimpoukelli Martynas Mankus Matan Hasson Matthew Lennie Matthias Reso Maxim Groshev Maxim Naumov Maya Lathi Meghan Keneally Michael\u00a0L. Seltzer Michal Valko Michelle Restrepo Mihir Patel Mik Vyatskov Mikayel Samvelyan Mike Clark Mike Macey Mike Wang Miquel\u00a0Jubert Hermoso Mo Metanat Mohammad Rastegari Munish Bansal Nandhini Santhanam Natascha Parks Natasha White Navyata Bawa Nayan Singhal Nick Egebo Nicolas Usunier Nikolay\u00a0Pavlovich Laptev Ning Dong Ning Zhang Norman Cheng Oleg Chernoguz Olivia Hart Omkar Salpekar Ozlem Kalinli Parkin Kent Parth Parekh Paul Saab Pavan Balaji Pedro Rittner Philip Bontrager Pierre Roux Piotr Dollar Polina Zvyagina Prashant Ratanchandani Pritish Yuvraj Qian Liang Rachad Alao Rachel Rodriguez Rafi Ayub Raghotham Murthy Raghu Nayani Rahul Mitra Raymond Li Rebekkah Hogan Robin Battey Rocky Wang Rohan Maheswari Russ Howes Ruty Rinott Sai\u00a0Jayesh Bondu Samyak Datta Sara Chugh Sara Hunt Sargun Dhillon Sasha Sidorov Satadru Pan Saurabh Verma Seiji Yamamoto Sharadh Ramaswamy Shaun Lindsay Shaun Lindsay Sheng Feng Shenghao Lin Shengxin\u00a0Cindy Zha Shiva Shankar Shuqiang Zhang Shuqiang Zhang Sinong Wang Sneha Agarwal Soji Sajuyigbe Soumith Chintala Stephanie Max Stephen Chen Steve Kehoe Steve Satterfield Sudarshan Govindaprasad Sumit Gupta Sungmin Cho Sunny Virk Suraj Subramanian Sy Choudhury Sydney Goldman Tal Remez Tamar Glaser Tamara Best Thilo Kohler Thomas Robinson Tianhe Li Tianjun Zhang Tim Matthews Timothy Chou Tzook Shaked Varun Vontimitta Victoria Ajayi Victoria Montanez Vijai Mohan Vinay\u00a0Satish Kumar Vishal Mangla V\u00edtor Albiero Vlad Ionescu Vlad Poenaru Vlad\u00a0Tiberiu Mihailescu Vladimir Ivanov Wei Li Wenchen Wang Wenwen Jiang Wes Bouaziz Will Constable Xiaocheng Tang Xiaofang Wang Xiaojian Wu Xiaolan Wang Xide Xia Xilun Wu Xinbo Gao Yanjun Chen Ye Hu Ye Jia Ye Qi Yenda Li Yilin Zhang Ying Zhang Yossi Adi Youngjin Nam Yu Wang Yuchen Hao Yundi Qian Yuzi He Zach Rait Zachary DeVito Zef Rosnbrick Zhaoduo Wen Zhenyu Yang and Zhiwei Zhao. 2024. The Llama 3 Herd of Models. arXiv:https:\/\/arXiv.org\/abs\/2407.21783 (2024)."},{"key":"e_1_3_3_2_36_2","unstructured":"ECOA. 1974. Equal Credit Opportunity Act."},{"key":"e_1_3_3_2_37_2","unstructured":"EEOC. 2023. Select Issues: Assessing Adverse Impact in Software Algorithms and Artificial Intelligence."},{"key":"e_1_3_3_2_38_2","unstructured":"European Commission. 2024. AI Pact."},{"key":"e_1_3_3_2_39_2","unstructured":"European Union. 2023. Regulation of the European Parliament and of the Council Laying Down Harmonized Rules on Artificial Intelligence (Artificial Intelligence Act) and Amending Certain Union Legislative Acts."},{"key":"e_1_3_3_2_40_2","volume-title":"Proceedings of the AAAI\/ACM Conference on AI, Ethics, and Society","author":"Feffer Michael","year":"2024","unstructured":"Michael Feffer, Anusha Sinha, Wesley Deng, Zachary Lipton, and Hoda Heidari. 2024. Red-Teaming for Generative AI: Silver Bullet or Security Theater?. In Proceedings of the AAAI\/ACM Conference on AI, Ethics, and Society."},{"key":"e_1_3_3_2_41_2","doi-asserted-by":"crossref","unstructured":"Julien Ferry Ulrich A\u00efvodji S\u00e9bastien Gambs Marie-Jos\u00e9 Huguet and Mohamed Siala. 2022. Improving fairness generalization through a sample-robust optimization method. Mach. Learn. (2022).","DOI":"10.1007\/s10994-022-06191-y"},{"key":"e_1_3_3_2_42_2","unstructured":"FHA. 1968. Fair Housing Act."},{"key":"e_1_3_3_2_43_2","unstructured":"Johann\u00a0D. Gaebler Sharad Goel Aziz Huq and Prasanna Tambe. 2024. Auditing the Use of Language Models to Guide Hiring Decisions. arXiv:https:\/\/arXiv.org\/abs\/2404.03086 (2024). arXiv:https:\/\/arXiv.org\/abs\/2404.03086"},{"key":"e_1_3_3_2_44_2","doi-asserted-by":"publisher","DOI":"10.1145\/3593013.3594116"},{"key":"e_1_3_3_2_45_2","unstructured":"Deep Ganguli Liane Lovitt Jackson Kernion Amanda Askell Yuntao Bai Saurav Kadavath Ben Mann Ethan Perez Nicholas Schiefer Kamal Ndousse Andy Jones Sam Bowman Anna Chen Tom Conerly Nova DasSarma Dawn Drain Nelson Elhage Sheer El-Showk Stanislav Fort Zac Hatfield-Dodds Tom Henighan Danny Hernandez Tristan Hume Josh Jacobson Scott Johnston Shauna Kravec Catherine Olsson Sam Ringer Eli Tran-Johnson Dario Amodei Tom Brown Nicholas Joseph Sam McCandlish Chris Olah Jared Kaplan and Jack Clark. 2022. Red Teaming Language Models to Reduce Harms: Methods Scaling Behaviors and Lessons Learned. arXiv:https:\/\/arXiv.org\/abs\/2209.07858 (2022)."},{"key":"e_1_3_3_2_46_2","doi-asserted-by":"publisher","DOI":"10.1145\/3600211.3604672"},{"key":"e_1_3_3_2_47_2","volume-title":"International Conference on Learning Representations","author":"Giguere Stephen","year":"2022","unstructured":"Stephen Giguere, Blossom Metevier, Yuriy Brun, Philip\u00a0S. Thomas, Scott Niekum, and Bruno\u00a0Castro da Silva. 2022. Fairness Guarantees under Demographic Shift. In International Conference on Learning Representations."},{"key":"e_1_3_3_2_48_2","unstructured":"Talia\u00a0B Gillis. 2021. The input fallacy. Minnesota Law Review (2021)."},{"key":"e_1_3_3_2_49_2","doi-asserted-by":"publisher","DOI":"10.1145\/3630106.3658912"},{"key":"e_1_3_3_2_50_2","doi-asserted-by":"crossref","unstructured":"Philipp Hacker. 2018. Teaching Fairness to Artificial Intelligence: Existing and Novel Strategies against Algorithmic Discrimination under EU Law. Common Market Law Review (2018).","DOI":"10.54648\/COLA2018095"},{"key":"e_1_3_3_2_51_2","doi-asserted-by":"crossref","unstructured":"Philipp Hacker Brent Mittelstadt Frederik\u00a0Zuiderveen Borgesius and Sandra Wachter. 2024. Generative Discrimination: What Happens When Generative AI Exhibits Bias and What Can Be Done About It. arXiv:https:\/\/arXiv.org\/abs\/2407.10329 (2024).","DOI":"10.2139\/ssrn.4877398"},{"key":"e_1_3_3_2_52_2","unstructured":"Amit Haim Alejandro Salinas and Julian Nyarko. 2024. What\u2019s in a Name? Auditing Large Language Models for Race and Gender Bias. arXiv:https:\/\/arXiv.org\/abs\/2402.14875 (2024)."},{"key":"e_1_3_3_2_53_2","unstructured":"Vernon Toh\u00a0Yan Han Rishabh Bhardwaj and Soujanya Poria. 2024. Ruby Teaming: Improving Quality Diversity Search with Memory for Automated Red Teaming. arXiv:https:\/\/arXiv.org\/abs\/2406.11654 (2024)."},{"key":"e_1_3_3_2_54_2","unstructured":"Laura Hanu and Unitary team. 2020. Detoxify."},{"key":"e_1_3_3_2_55_2","volume-title":"Advances in Neural Information Processing Systems","author":"Hardt Moritz","year":"2016","unstructured":"Moritz Hardt, Eric Price, and Nathan Srebro. 2016. Equality of Opportunity in Supervised Learning. In Advances in Neural Information Processing Systems."},{"key":"e_1_3_3_2_56_2","volume-title":"International Conference on Learning Representations","author":"Hendrycks Dan","year":"2021","unstructured":"Dan Hendrycks, Collin Burns, Steven Basart, Andy Zou, Mantas Mazeika, Dawn Song, and Jacob Steinhardt. 2021. Measuring Massive Multitask Language Understanding. In International Conference on Learning Representations."},{"key":"e_1_3_3_2_57_2","volume-title":"Advances in Neural Information Processing Systems","author":"Heusel Martin","year":"2017","unstructured":"Martin Heusel, Hubert Ramsauer, Thomas Unterthiner, Bernhard Nessler, and Sepp Hochreiter. 2017. GANs Trained by a Two Time-Scale Update Rule Converge to a Local Nash Equilibrium. In Advances in Neural Information Processing Systems."},{"key":"e_1_3_3_2_58_2","doi-asserted-by":"crossref","unstructured":"John\u00a0J. Horton. 2023. Large Language Models as Simulated Economic Agents: What Can We Learn from Homo Silicus? arXiv:https:\/\/arXiv.org\/abs\/2301.07543 (2023).","DOI":"10.3386\/w31122"},{"key":"e_1_3_3_2_59_2","unstructured":"Wenyue Hua Xianjun Yang Mingyu Jin Wei Cheng Ruixiang Tang and Yongfeng Zhang. 2024. TrustAgent: Towards Safe and Trustworthy LLM-based Agents through Agent Constitution. arXiv:https:\/\/arXiv.org\/abs\/2402.01586 (2024)."},{"key":"e_1_3_3_2_60_2","unstructured":"Bojian Jiang Yi Jing Tianhao Shen Qing Yang and Deyi Xiong. 2024. DART: Deep Adversarial Automated Red Teaming for LLM Safety. arXiv:https:\/\/arXiv.org\/abs\/2407.03876 (2024)."},{"key":"e_1_3_3_2_61_2","unstructured":"Joint Statement. 2024. Joint Statement on Enforcement Efforts Against Discrimination and Bias in Automated Systems."},{"key":"e_1_3_3_2_62_2","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2024.findings-acl.29"},{"key":"e_1_3_3_2_63_2","doi-asserted-by":"publisher","DOI":"10.1145\/3582269.3615599"},{"key":"e_1_3_3_2_64_2","unstructured":"Nathan Lambert Valentina Pyatkin Jacob Morrison LJ Miranda Bill\u00a0Yuchen Lin Khyathi Chandu Nouha Dziri Sachin Kumar Tom Zick Yejin Choi Noah\u00a0A. Smith and Hannaneh Hajishirzi. 2024. RewardBench: Evaluating Reward Models for Language Modeling. arXiv:https:\/\/arXiv.org\/abs\/2403.13787 (2024)."},{"key":"e_1_3_3_2_65_2","volume-title":"Advances in Neural Information Processing Systems","author":"Li Guanlin","year":"2024","unstructured":"Guanlin Li, Kangjie Chen, Shudong Zhang, Jie Zhang, and Tianwei Zhang. 2024. ART: Automatic Red-teaming for Text-to-Image Models to Protect Benign Users. In Advances in Neural Information Processing Systems."},{"key":"e_1_3_3_2_66_2","volume-title":"Proceedings of the 42nd Annual Meeting of the Association for Computational Linguistics","author":"Lin Chin-Yew","year":"2004","unstructured":"Chin-Yew Lin. 2004. ROUGE: A Package for Automatic Evaluation of Summaries. In Proceedings of the 42nd Annual Meeting of the Association for Computational Linguistics."},{"key":"e_1_3_3_2_67_2","volume-title":"Proceedings of the 2023 Conference on Empirical Methods in Natural Language Processing","author":"Lin Zi","year":"2023","unstructured":"Zi Lin, Zihan Wang, Yongqi Tong, Yangkun Wang, Yuxin Guo, Yujia Wang, and Jingbo Shang. 2023. ToxicChat: Unveiling Hidden Challenges of Toxicity Detection in Real-World User-AI Conversation. In Proceedings of the 2023 Conference on Empirical Methods in Natural Language Processing."},{"key":"e_1_3_3_2_68_2","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-73347-5_6"},{"key":"e_1_3_3_2_69_2","volume-title":"Advances in Neural Information Processing Systems","author":"Luccioni Sasha","year":"2023","unstructured":"Sasha Luccioni, Christopher Akiki, Margaret Mitchell, and Yacine Jernite. 2023. Stable Bias: Evaluating Societal Representations in Diffusion Models. In Advances in Neural Information Processing Systems."},{"key":"e_1_3_3_2_70_2","unstructured":"Kristian Lum Jacy\u00a0Reese Anthis Chirag Nagpal and Alexander D\u2019Amour. 2024. Bias in Language Models: Beyond Trick Tests and Toward RUTEd Evaluation. arXiv:https:\/\/arXiv.org\/abs\/2402.12649 (2024)."},{"key":"e_1_3_3_2_71_2","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2024.emnlp-main.41"},{"key":"e_1_3_3_2_72_2","doi-asserted-by":"crossref","unstructured":"NIST. 2023. Artificial Intelligence Risk Management Framework (AI RMF) 1.0.","DOI":"10.6028\/NIST.AI.100-1.jpn"},{"key":"e_1_3_3_2_73_2","volume-title":"Generative AI Guidelines: Artificial Intelligence Risk Management Framework (AI RMF) 1.0","year":"2024","unstructured":"NIST. 2024. Generative AI Guidelines: Artificial Intelligence Risk Management Framework (AI RMF) 1.0. Technical Report."},{"key":"e_1_3_3_2_74_2","unstructured":"OMB. 2023. Advancing Governance Innovation and Risk Management for Agency Use of Artificial Intelligence."},{"key":"e_1_3_3_2_75_2","unstructured":"OMB. 2024. Advancing Governance Innovation and Risk Management for Agency Use of Artificial Intelligence."},{"key":"e_1_3_3_2_76_2","unstructured":"OpenAI Josh Achiam Steven Adler Sandhini Agarwal Lama Ahmad Ilge Akkaya Florencia\u00a0Leoni Aleman Diogo Almeida Janko Altenschmidt Sam Altman Shyamal Anadkat Red Avila Igor Babuschkin Suchir Balaji Valerie Balcom Paul Baltescu Haiming Bao Mohammad Bavarian Jeff Belgum Irwan Bello Jake Berdine Gabriel Bernadett-Shapiro Christopher Berner Lenny Bogdonoff Oleg Boiko Madelaine Boyd Anna-Luisa Brakman Greg Brockman Tim Brooks Miles Brundage Kevin Button Trevor Cai Rosie Campbell Andrew Cann Brittany Carey Chelsea Carlson Rory Carmichael Brooke Chan Che Chang Fotis Chantzis Derek Chen Sully Chen Ruby Chen Jason Chen Mark Chen Ben Chess Chester Cho Casey Chu Hyung\u00a0Won Chung Dave Cummings Jeremiah Currier Yunxing Dai Cory Decareaux Thomas Degry Noah Deutsch Damien Deville Arka Dhar David Dohan Steve Dowling Sheila Dunning Adrien Ecoffet Atty Eleti Tyna Eloundou David Farhi Liam Fedus Niko Felix Sim\u00f3n\u00a0Posada Fishman Juston Forte Isabella Fulford Leo Gao Elie Georges Christian Gibson Vik Goel Tarun Gogineni Gabriel Goh Rapha Gontijo-Lopes Jonathan Gordon Morgan Grafstein Scott Gray Ryan Greene Joshua Gross Shixiang\u00a0Shane Gu Yufei Guo Chris Hallacy Jesse Han Jeff Harris Yuchen He Mike Heaton Johannes Heidecke Chris Hesse Alan Hickey Wade Hickey Peter Hoeschele Brandon Houghton Kenny Hsu Shengli Hu Xin Hu Joost Huizinga Shantanu Jain Shawn Jain Joanne Jang Angela Jiang Roger Jiang Haozhun Jin Denny Jin Shino Jomoto Billie Jonn Heewoo Jun Tomer Kaftan \u0141ukasz Kaiser Ali Kamali Ingmar Kanitscheider Nitish\u00a0Shirish Keskar Tabarak Khan Logan Kilpatrick Jong\u00a0Wook Kim Christina Kim Yongjik Kim Jan\u00a0Hendrik Kirchner Jamie Kiros Matt Knight Daniel Kokotajlo \u0141ukasz Kondraciuk Andrew Kondrich Aris Konstantinidis Kyle Kosic Gretchen Krueger Vishal Kuo Michael Lampe Ikai Lan Teddy Lee Jan Leike Jade Leung Daniel Levy Chak\u00a0Ming Li Rachel Lim Molly Lin Stephanie Lin Mateusz Litwin Theresa Lopez Ryan Lowe Patricia Lue Anna Makanju Kim Malfacini Sam Manning Todor Markov Yaniv Markovski Bianca Martin Katie Mayer Andrew Mayne Bob McGrew Scott\u00a0Mayer McKinney Christine McLeavey Paul McMillan Jake McNeil David Medina Aalok Mehta Jacob Menick Luke Metz Andrey Mishchenko Pamela Mishkin Vinnie Monaco Evan Morikawa Daniel Mossing Tong Mu Mira Murati Oleg Murk David M\u00e9ly Ashvin Nair Reiichiro Nakano Rajeev Nayak Arvind Neelakantan Richard Ngo Hyeonwoo Noh Long Ouyang Cullen O\u2019Keefe Jakub Pachocki Alex Paino Joe Palermo Ashley Pantuliano Giambattista Parascandolo Joel Parish Emy Parparita Alex Passos Mikhail Pavlov Andrew Peng Adam Perelman Filipe de Avila Belbute\u00a0Peres Michael Petrov Henrique\u00a0Ponde de Oliveira\u00a0Pinto Michael Pokorny Michelle Pokrass Vitchyr\u00a0H. Pong Tolly Powell Alethea Power Boris Power Elizabeth Proehl Raul Puri Alec Radford Jack Rae Aditya Ramesh Cameron Raymond Francis Real Kendra Rimbach Carl Ross Bob Rotsted Henri Roussez Nick Ryder Mario Saltarelli Ted Sanders Shibani Santurkar Girish Sastry Heather Schmidt David Schnurr John Schulman Daniel Selsam Kyla Sheppard Toki Sherbakov Jessica Shieh Sarah Shoker Pranav Shyam Szymon Sidor Eric Sigler Maddie Simens Jordan Sitkin Katarina Slama Ian Sohl Benjamin Sokolowsky Yang Song Natalie Staudacher Felipe\u00a0Petroski Such Natalie Summers Ilya Sutskever Jie Tang Nikolas Tezak Madeleine\u00a0B. Thompson Phil Tillet Amin Tootoonchian Elizabeth Tseng Preston Tuggle Nick Turley Jerry Tworek Juan Felipe\u00a0Cer\u00f3n Uribe Andrea Vallone Arun Vijayvergiya Chelsea Voss Carroll Wainwright Justin\u00a0Jay Wang Alvin Wang Ben Wang Jonathan Ward Jason Wei CJ Weinmann Akila Welihinda Peter Welinder Jiayi Weng Lilian Weng Matt Wiethoff Dave Willner Clemens Winter Samuel Wolrich Hannah Wong Lauren Workman Sherwin Wu Jeff Wu Michael Wu Kai Xiao Tao Xu Sarah Yoo Kevin Yu Qiming Yuan Wojciech Zaremba Rowan Zellers Chong Zhang Marvin Zhang Shengjia Zhao Tianhao Zheng Juntang Zhuang William Zhuk and Barret Zoph. 2024. GPT-4 Technical Report. arXiv:https:\/\/arXiv.org\/abs\/2303.08774 (2024)."},{"key":"e_1_3_3_2_77_2","volume-title":"Proceedings of the 40th Annual Meeting of the Association for Computational Linguistics","author":"Papineni Kishore","year":"2002","unstructured":"Kishore Papineni, Salim Roukos, Todd Ward, and Wei-Jing Zhu. 2002. Bleu: a Method for Automatic Evaluation of Machine Translation. In Proceedings of the 40th Annual Meeting of the Association for Computational Linguistics."},{"key":"e_1_3_3_2_78_2","doi-asserted-by":"publisher","DOI":"10.1145\/3586183.3606763"},{"key":"e_1_3_3_2_79_2","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2022.emnlp-main.225"},{"key":"e_1_3_3_2_80_2","doi-asserted-by":"crossref","unstructured":"Marcelo Prates Pedro Avelar and Lu\u00eds Lamb. 2020. Assessing gender bias in machine translation: a case study with Google Translate. Neural Computing and Applications (2020).","DOI":"10.1007\/s00521-019-04144-6"},{"key":"e_1_3_3_2_81_2","doi-asserted-by":"publisher","DOI":"10.1145\/3576915.3616679"},{"key":"e_1_3_3_2_82_2","doi-asserted-by":"publisher","DOI":"10.1145\/3630106.3658913"},{"key":"e_1_3_3_2_83_2","volume-title":"Advances in Neural Information Processing Systems","author":"Rafailov Rafael","year":"2023","unstructured":"Rafael Rafailov, Archit Sharma, Eric Mitchell, Christopher\u00a0D. Manning, Stefano Ermon, and Chelsea Finn. 2023. Direct Preference Optimization: Your Language Model is Secretly a Reward Model. In Advances in Neural Information Processing Systems."},{"key":"e_1_3_3_2_84_2","unstructured":"Inioluwa\u00a0Deborah Raji Emily\u00a0M. Bender Amandalynne Paullada Emily Denton and Alex Hanna. 2021. AI and the Everything in the Whole Wide World Benchmark. arxiv:https:\/\/arXiv.org\/abs\/2111.15366\u00a0[cs.LG] https:\/\/arxiv.org\/abs\/2111.15366"},{"key":"e_1_3_3_2_85_2","unstructured":"Reuters. 2024. EEOC says Workday covered by anti-bias laws in AI discrimination case."},{"key":"e_1_3_3_2_86_2","volume-title":"Advances in Neural Information Processing Systems","author":"Samvelyan Mikayel","year":"2024","unstructured":"Mikayel Samvelyan, Sharath\u00a0Chandra Raparthy, Andrei Lupu, Eric Hambro, Aram\u00a0H. Markosyan, Manish Bhatt, Yuning Mao, Minqi Jiang, Jack Parker-Holder, Jakob\u00a0Nicolaus Foerster, Tim Rockt\u00e4schel, and Roberta Raileanu. 2024. Rainbow Teaming: Open-Ended Generation of Diverse Adversarial Prompts. In Advances in Neural Information Processing Systems."},{"key":"e_1_3_3_2_87_2","volume-title":"Socially Responsible Language Modelling Research","author":"Shah Rusheb","year":"2023","unstructured":"Rusheb Shah, Quentin\u00a0Feuillade Montixi, Soroush Pour, Arush Tagade, and Javier Rando. 2023. Scalable and Transferable Black-Box Jailbreaks for Language Models via Persona Modulation. In Socially Responsible Language Modelling Research."},{"key":"e_1_3_3_2_88_2","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/P19-1164"},{"key":"e_1_3_3_2_89_2","unstructured":"Title VII. 1964. Title VII of the Civil Rights Act of 1964. 42 U.S.C. \u00a7\u00a7 2000e to 2000e-17."},{"key":"e_1_3_3_2_90_2","unstructured":"Uniform Guidelines. 1978. Uniform Guidelines on Employee Selection Procedures. 29 C.F.R. \u00a7 1607 43 Federal Regulation 38 290. Adopted by multiple federal agencies under their authorities on August 25 1978."},{"key":"e_1_3_3_2_91_2","doi-asserted-by":"publisher","DOI":"10.1145\/3194770.3194776"},{"key":"e_1_3_3_2_92_2","unstructured":"Anthony W\u00a0Flores Kristin Bechtel and Christopher Lowenkamp. 2016. False Positives False Negatives and False Analyses: A Rejoinder to \u201cMachine Bias: There\u2019s Software Used Across the Country to Predict Future Criminals. And it\u2019s Biased Against Blacks.\u201d. Federal probation (2016)."},{"key":"e_1_3_3_2_93_2","volume-title":"Findings of the Association for Computational Linguistics","author":"Wan Yixin","year":"2023","unstructured":"Yixin Wan, George Pu, Jiao Sun, Aparna Garimella, Kai-Wei Chang, and Nanyun Peng. 2023. \u201cKelly is a Warm Person, Joseph is a Role Model\u201d: Gender Biases in LLM-Generated Reference Letters. In Findings of the Association for Computational Linguistics."},{"key":"e_1_3_3_2_94_2","unstructured":"White House. 2022. Blueprint for an AI Bill of Rights: Making Automated Systems Work for the American People."},{"key":"e_1_3_3_2_95_2","unstructured":"White House. 2023. Ensuring Safe Secure and Trustworthy AI."},{"key":"e_1_3_3_2_96_2","unstructured":"White House. 2023. Executive Order 14110: Advancing American Leadership in Artificial Intelligence."},{"key":"e_1_3_3_2_97_2","doi-asserted-by":"crossref","unstructured":"Kyra Wilson and Aylin Caliskan. 2024. Gender Race and Intersectional Bias in Resume Screening via Language Model Retrieval. arXiv:https:\/\/arXiv.org\/abs\/2407.20371 (2024).","DOI":"10.1609\/aies.v7i1.31748"},{"key":"e_1_3_3_2_98_2","unstructured":"Minghao Wu and Alham\u00a0Fikri Aji. 2023. Style Over Substance: Evaluation Biases for Large Language Models. arXiv:https:\/\/arXiv.org\/abs\/2307.03025 (2023)."},{"key":"e_1_3_3_2_99_2","doi-asserted-by":"crossref","unstructured":"Alice Xiang. 2024. Fairness & Privacy in an Age of Generative AI. Science and Technology Law Review (2024).","DOI":"10.52214\/stlr.v25i2.12765"},{"key":"e_1_3_3_2_100_2","volume-title":"International Conference on Learning Representations","author":"Yang Zonghan","year":"2023","unstructured":"Zonghan Yang, Xiaoyuan Yi, Peng Li, Yang Liu, and Xing Xie. 2023. Unified Detoxifying and Debiasing in Language Generation via Inference-time Adaptive Optimization. In International Conference on Learning Representations."},{"key":"e_1_3_3_2_101_2","volume-title":"Advances in Neural Information Processing Systems","author":"Zheng Lianmin","year":"2023","unstructured":"Lianmin Zheng, Wei-Lin Chiang, Ying Sheng, Siyuan Zhuang, Zhanghao Wu, Yonghao Zhuang, Zi Lin, Zhuohan Li, Dacheng Li, Eric\u00a0P. Xing, Hao Zhang, Joseph\u00a0E. Gonzalez, and Ion Stoica. 2023. Judging LLM-as-a-Judge with MT-Bench and Chatbot Arena. In Advances in Neural Information Processing Systems."},{"key":"e_1_3_3_2_102_2","unstructured":"Mi Zhou Vibhanshu Abhishek Timothy Derdenger Jaymo Kim and Kannan Srinivasan. 2024. Bias in generative ai. arXiv:https:\/\/arXiv.org\/abs\/2403.02726 (2024)."},{"key":"e_1_3_3_2_103_2","unstructured":"Thomas\u00a0P. Zollo Andrew Wei\u00a0Tung Siah Naimeng Ye Ang Li and Hongseok Namkoong. 2024. PersonalLLM: Tailoring LLMs to Individual Preferences. arXiv:https:\/\/arXiv.org\/abs\/2409.20296 (2024)."}],"event":{"name":"FAccT '25: The 2025 ACM Conference on Fairness, Accountability, and Transparency","location":"Athens Greece","acronym":"FAccT '25"},"container-title":["Proceedings of the 2025 ACM Conference on Fairness, Accountability, and Transparency"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3715275.3732067","content-type":"application\/pdf","content-version":"vor","intended-application":"syndication"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3715275.3732067","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,24]],"date-time":"2025-06-24T11:19:52Z","timestamp":1750763992000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3715275.3732067"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,6,23]]},"references-count":102,"alternative-id":["10.1145\/3715275.3732067","10.1145\/3715275"],"URL":"https:\/\/doi.org\/10.1145\/3715275.3732067","relation":{},"subject":[],"published":{"date-parts":[[2025,6,23]]},"assertion":[{"value":"2025-06-23","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}