{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,14]],"date-time":"2026-04-14T00:55:03Z","timestamp":1776128103789,"version":"3.50.1"},"publisher-location":"New York, NY, USA","reference-count":47,"publisher":"ACM","license":[{"start":{"date-parts":[[2023,8,8]],"date-time":"2023-08-08T00:00:00Z","timestamp":1691452800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"funder":[{"name":"NSF","award":["1763734"],"award-info":[{"award-number":["1763734"]}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2023,8,8]]},"DOI":"10.1145\/3600211.3604712","type":"proceedings-article","created":{"date-parts":[[2023,8,29]],"date-time":"2023-08-29T18:41:37Z","timestamp":1693334497000},"page":"913-926","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":60,"title":["Supporting Human-AI Collaboration in Auditing LLMs with LLMs"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-0820-4115","authenticated-orcid":false,"given":"Charvi","family":"Rastogi","sequence":"first","affiliation":[{"name":"Machine Learning Department, Carnegie Mellon University, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-3301-1297","authenticated-orcid":false,"given":"Marco","family":"Tulio Ribeiro","sequence":"additional","affiliation":[{"name":"Microsoft Research, USA"}]},{"ORCID":"https:\/\/orcid.org\/0009-0002-2126-5115","authenticated-orcid":false,"given":"Nicholas","family":"King","sequence":"additional","affiliation":[{"name":"Microsoft Research, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-5442-1359","authenticated-orcid":false,"given":"Harsha","family":"Nori","sequence":"additional","affiliation":[{"name":"Microsoft Research, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-3294-7288","authenticated-orcid":false,"given":"Saleema","family":"Amershi","sequence":"additional","affiliation":[{"name":"Microsoft Research, USA"}]}],"member":"320","published-online":{"date-parts":[[2023,8,29]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"publisher","DOI":"10.1609\/aimag.v35i4.2513"},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.1145\/2046396.2046416"},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1145\/2700832"},{"key":"e_1_3_2_1_4_1","unstructured":"Azure. 2022. Azure Cognitive Services: Text Analytics. https:\/\/azure.microsoft.com\/en-us\/products\/cognitive-services\/text-analytics Accessed on 03\/08\/23."},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1145\/3449148"},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.acl-main.485"},{"key":"e_1_3_2_1_7_1","volume-title":"On the opportunities and risks of foundation models. arXiv preprint arXiv:2108.07258","author":"Bommasani Rishi","year":"2021","unstructured":"Rishi Bommasani, Drew\u00a0A Hudson, Ehsan Adeli, Russ Altman, Simran Arora, Sydney von Arx, Michael\u00a0S Bernstein, Jeannette Bohg, Antoine Bosselut, Emma Brunskill, 2021. On the opportunities and risks of foundation models. arXiv preprint arXiv:2108.07258 (2021)."},{"key":"e_1_3_2_1_8_1","volume-title":"Language models are few-shot learners. Advances in neural information processing systems 33","author":"Brown Tom","year":"2020","unstructured":"Tom Brown, Benjamin Mann, Nick Ryder, Melanie Subbiah, Jared\u00a0D Kaplan, Prafulla Dhariwal, Arvind Neelakantan, Pranav Shyam, Girish Sastry, Amanda Askell, 2020. Language models are few-shot learners. Advances in neural information processing systems 33 (2020), 1877\u20131901."},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1145\/3479569"},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.1145\/3542921"},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.1145\/3172944.3172950"},{"key":"e_1_3_2_1_12_1","volume-title":"Whose Ground Truth? Accounting for Individual and Collective Identities Underlying Dataset Annotation. arXiv preprint arXiv:2112.04554","author":"Denton Emily","year":"2021","unstructured":"Emily Denton, Mark D\u00edaz, Ian Kivlichan, Vinodkumar Prabhakaran, and Rachel Rosen. 2021. Whose Ground Truth? Accounting for Individual and Collective Identities Underlying Dataset Annotation. arXiv preprint arXiv:2112.04554 (2021)."},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1145\/3491102.3517441"},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.23919\/MIPRO.2018.8400040"},{"key":"e_1_3_2_1_15_1","volume-title":"\u201cstress test","author":"Field Hayden","year":"2022","unstructured":"Hayden Field. 2022. How Microsoft and Google use AI red teams to \u201cstress test\u201d their systems. https:\/\/www.emergingtechbrew.com\/stories\/2022\/06\/14\/how-microsoft-and-google-use-ai-red-teams-to-stress-test-their-system Accessed on 03\/08\/23."},{"key":"e_1_3_2_1_16_1","volume-title":"Generative Language Models and Automated Influence Operations: Emerging Threats and Potential Mitigations. arXiv preprint arXiv:2301.04246","author":"Goldstein A","year":"2023","unstructured":"Josh\u00a0A Goldstein, Girish Sastry, Micah Musser, Renee DiResta, Matthew Gentzel, and Katerina Sedova. 2023. Generative Language Models and Automated Influence Operations: Emerging Threats and Potential Mitigations. arXiv preprint arXiv:2301.04246 (2023)."},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.1145\/3411764.3445423"},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1145\/302979.303030"},{"key":"e_1_3_2_1_19_1","unstructured":"Erik Jones and Jacob Steinhardt. 2022. Capturing Failures of Large Language Models via Human Cognitive Biases. In Advances in Neural Information Processing Systems Alice\u00a0H. Oh Alekh Agarwal Danielle Belgrave and Kyunghyun Cho (Eds.). https:\/\/openreview.net\/forum?id=fcO9Cgn-X-R"},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1"},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1"},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1"},{"key":"e_1_3_2_1_23_1","volume-title":"AutoBiasTest: Controllable Sentence Generation for Automated and Open-Ended Social Bias Testing in Language Models. arXiv preprint arXiv:2302.07371","author":"Kocielnik Rafal","year":"2023","unstructured":"Rafal Kocielnik, Shrimai Prabhumoye, Vivian Zhang, R\u00a0Michael Alvarez, and Anima Anandkumar. 2023. AutoBiasTest: Controllable Sentence Generation for Automated and Open-Ended Social Bias Testing in Language Models. arXiv preprint arXiv:2302.07371 (2023)."},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"publisher","DOI":"10.1145\/2556288.2557238"},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v31i1.10821"},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.1145\/3555625"},{"key":"e_1_3_2_1_27_1","doi-asserted-by":"publisher","DOI":"10.1145\/3359174"},{"key":"e_1_3_2_1_28_1","unstructured":"Yusuf Mehdi. 2023. Reinventing search with a new AI-powered Microsoft Bing and Edge your copilot for the web. https:\/\/blogs.microsoft.com\/blog\/2023\/02\/07\/reinventing-search-with-a-new-ai-powered-microsoft-bing-and-edge-your-copilot-for-the-web\/ Accessed on 03\/16\/23."},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.1561\/1100000083"},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2022.emnlp-main.225"},{"key":"e_1_3_2_1_31_1","volume-title":"Red teaming language models with language models. arXiv preprint arXiv:2202.03286","author":"Perez Ethan","year":"2022","unstructured":"Ethan Perez, Saffron Huang, Francis Song, Trevor Cai, Roman Ring, John Aslanides, Amelia Glaese, Nat McAleese, and Geoffrey Irving. 2022. Red teaming language models with language models. arXiv preprint arXiv:2202.03286 (2022)."},{"key":"e_1_3_2_1_32_1","unstructured":"Sundar Pichai. 2023. An important next step on our AI journey. https:\/\/blog.google\/technology\/ai\/bard-google-ai-search-updates\/ Accessed on 03\/16\/23."},{"key":"e_1_3_2_1_33_1","volume-title":"Proceedings of international conference on intelligence analysis, Vol.\u00a05.","author":"Pirolli Peter","year":"2005","unstructured":"Peter Pirolli and Stuart Card. 2005. The sensemaking process and leverage points for analyst technology as identified through cognitive task analysis. In Proceedings of international conference on intelligence analysis, Vol.\u00a05."},{"key":"e_1_3_2_1_34_1","doi-asserted-by":"publisher","DOI":"10.1145\/3306618.3314244"},{"key":"e_1_3_2_1_35_1","doi-asserted-by":"publisher","DOI":"10.1145\/3351095.3372873"},{"key":"e_1_3_2_1_36_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2022.acl-long.230"},{"key":"e_1_3_2_1_37_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.acl-main.442"},{"key":"e_1_3_2_1_38_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-02197-8"},{"key":"e_1_3_2_1_39_1","volume-title":"Auditing algorithms: Research methods for detecting discrimination on internet platforms. Data and discrimination: converting critical concerns into productive inquiry 22","author":"Sandvig Christian","year":"2014","unstructured":"Christian Sandvig, Kevin Hamilton, Karrie Karahalios, and Cedric Langbort. 2014. Auditing algorithms: Research methods for detecting discrimination on internet platforms. Data and discrimination: converting critical concerns into productive inquiry 22 (2014), 4349\u20134357."},{"key":"e_1_3_2_1_40_1","volume-title":"Emilio Garcia","author":"Shelby Renee","year":"2022","unstructured":"Renee Shelby, Shalaleh Rismani, Kathryn Henne, AJung Moon, Negar Rostamzadeh, Paul Nicholas, N\u2019Mah Yilla, Jess Gallegos, Andrew Smart, Emilio Garcia, 2022. Sociotechnical Harms: Scoping a Taxonomy for Harm Reduction. arXiv preprint arXiv:2210.05791 (2022)."},{"key":"e_1_3_2_1_41_1","doi-asserted-by":"publisher","DOI":"10.1145\/3479577"},{"key":"e_1_3_2_1_42_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2022.emnlp-main.625"},{"key":"e_1_3_2_1_43_1","volume-title":"The what-if tool: Interactive probing of machine learning models","author":"Wexler James","year":"2019","unstructured":"James Wexler, Mahima Pushkarna, Tolga Bolukbasi, Martin Wattenberg, Fernanda Vi\u00e9gas, and Jimbo Wilson. 2019. The what-if tool: Interactive probing of machine learning models. IEEE transactions on visualization and computer graphics 26, 1 (2019), 56\u201365."},{"key":"e_1_3_2_1_44_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/P19-1073"},{"key":"e_1_3_2_1_45_1","doi-asserted-by":"publisher","DOI":"10.1145\/3491102.3517582"},{"key":"e_1_3_2_1_46_1","doi-asserted-by":"publisher","DOI":"10.1145\/3544548.3581388"},{"key":"e_1_3_2_1_47_1","volume-title":"Advances in Neural Information Processing Systems, S.\u00a0Bengio, H.\u00a0Wallach, H.\u00a0Larochelle, K.\u00a0Grauman, N.\u00a0Cesa-Bianchi, and R.\u00a0Garnett (Eds.). Vol.\u00a031. Curran Associates","author":"Zhao Shengjia","year":"2018","unstructured":"Shengjia Zhao, Hongyu Ren, Arianna Yuan, Jiaming Song, Noah Goodman, and Stefano Ermon. 2018. Bias and Generalization in Deep Generative Models: An Empirical Study. In Advances in Neural Information Processing Systems, S.\u00a0Bengio, H.\u00a0Wallach, H.\u00a0Larochelle, K.\u00a0Grauman, N.\u00a0Cesa-Bianchi, and R.\u00a0Garnett (Eds.). Vol.\u00a031. Curran Associates, Inc.https:\/\/proceedings.neurips.cc\/paper\/2018\/file\/5317b6799188715d5e00a638a4278901-Paper.pdf"}],"event":{"name":"AIES '23: AAAI\/ACM Conference on AI, Ethics, and Society","location":"Montr\u00e9al QC Canada","acronym":"AIES '23","sponsor":["SIGAI ACM Special Interest Group on Artificial Intelligence"]},"container-title":["Proceedings of the 2023 AAAI\/ACM Conference on AI, Ethics, and Society"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3600211.3604712","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3600211.3604712","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T16:37:39Z","timestamp":1750178259000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3600211.3604712"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023,8,8]]},"references-count":47,"alternative-id":["10.1145\/3600211.3604712","10.1145\/3600211"],"URL":"https:\/\/doi.org\/10.1145\/3600211.3604712","relation":{},"subject":[],"published":{"date-parts":[[2023,8,8]]},"assertion":[{"value":"2023-08-29","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}