{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,13]],"date-time":"2026-04-13T13:48:59Z","timestamp":1776088139513,"version":"3.50.1"},"publisher-location":"New York, NY, USA","reference-count":29,"publisher":"ACM","license":[{"start":{"date-parts":[[2027,4,13]],"date-time":"2027-04-13T00:00:00Z","timestamp":1807574400000},"content-version":"vor","delay-in-days":365,"URL":"http:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"name":"National Science Foundation Graduate Research Fellowship Program","award":["DGE2140739"],"award-info":[{"award-number":["DGE2140739"]}]},{"DOI":"10.13039\/100000161","name":"National Institute of Standards and Technology","doi-asserted-by":"publisher","award":["60NANB24D231"],"award-info":[{"award-number":["60NANB24D231"]}],"id":[{"id":"10.13039\/100000161","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2026,4,13]]},"DOI":"10.1145\/3772363.3778702","type":"proceedings-article","created":{"date-parts":[[2026,4,13]],"date-time":"2026-04-13T01:55:28Z","timestamp":1776045328000},"page":"1-5","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":0,"title":["Human Expertise for AI Red-Teaming and Scalable Evaluation"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0009-0005-6407-6981","authenticated-orcid":false,"given":"Alice","family":"Qian","sequence":"first","affiliation":[{"name":"Human-Computer Interaction Institute, Carnegie Mellon University, Pittsburgh, Pennsylvania, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-3299-3169","authenticated-orcid":false,"given":"Srravya","family":"Chandhiramowuli","sequence":"additional","affiliation":[{"name":"Institute for Design Informatics, University of Edinburgh, Edinburgh, United Kingdom"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-9489-4188","authenticated-orcid":false,"given":"Laura","family":"Dabbish","sequence":"additional","affiliation":[{"name":"Human-Computer Interaction Institute, Carnegie Mellon University, Pittsburgh, Pennsylvania, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-5364-3718","authenticated-orcid":false,"given":"Hong","family":"Shen","sequence":"additional","affiliation":[{"name":"Human-Computer Interaction Institute, Carnegie Mellon University, Pittsburgh, Pennsylvania, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-6311-3967","authenticated-orcid":false,"given":"Alex S","family":"Taylor","sequence":"additional","affiliation":[{"name":"Design Informatics, University of Edinburgh, Edinburgh, Scotland, United Kingdom"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-5037-5017","authenticated-orcid":false,"given":"Ding","family":"Wang","sequence":"additional","affiliation":[{"name":"Google, Atlanta, Georgia, USA"}]},{"ORCID":"https:\/\/orcid.org\/0009-0008-0918-5589","authenticated-orcid":false,"given":"Theodora","family":"Skeadas","sequence":"additional","affiliation":[{"name":"Humane Intelligence, Cambridge, Massachusetts, USA"}]},{"ORCID":"https:\/\/orcid.org\/0009-0007-7099-5444","authenticated-orcid":false,"given":"Bolor-Erdene","family":"Jagdagdorj","sequence":"additional","affiliation":[{"name":"Microsoft, seattle, Washington, USA"}]}],"member":"320","published-online":{"date-parts":[[2026,4,13]]},"reference":[{"key":"e_1_3_3_2_2_2","volume-title":"Parents of teenager who took his own life sue OpenAI","author":"News BBC","year":"2025","unstructured":"BBC News. 2025. Parents of teenager who took his own life sue OpenAI. https:\/\/www.bbc.com\/news\/articles\/cgerwp7rdlvo"},{"key":"e_1_3_3_2_3_2","unstructured":"Joseph R.\u00a0JR Biden. 2023. Executive Order on the Safe Secure and Trustworthy Development and Use of Artificial Intelligence. https:\/\/bidenwhitehouse.archives.gov\/briefing-room\/presidential-actions\/2023\/10\/30\/executive-order-on-the-safe-secure-and-trustworthy-development-and-use-of-artificial-intelligence\/"},{"key":"e_1_3_3_2_4_2","volume-title":"In the Colony of Blooded Eyes: A Trilogy on Working Conditions as a Content Moderator in Germany","author":"Bozorg Sakine\u00a0Mohamadi","year":"2024","unstructured":"Sakine\u00a0Mohamadi Bozorg. 2024. In the Colony of Blooded Eyes: A Trilogy on Working Conditions as a Content Moderator in Germany. Data Workers\u2019 Inquiry. https:\/\/data-workers.org\/sakine\/"},{"key":"e_1_3_3_2_5_2","volume-title":"Parents of teens who died by suicide after AI chatbot interactions testify in Congress","author":"News CBS","year":"2025","unstructured":"CBS News. 2025. Parents of teens who died by suicide after AI chatbot interactions testify in Congress. https:\/\/www.cbsnews.com\/news\/ai-chatbots-teens-suicide-parents-testify-congress\/"},{"key":"e_1_3_3_2_6_2","doi-asserted-by":"publisher","DOI":"10.1145\/2145204.2145396"},{"key":"e_1_3_3_2_7_2","doi-asserted-by":"publisher","DOI":"10.1145\/3706598.3713477"},{"key":"e_1_3_3_2_8_2","first-page":"421","volume-title":"Proceedings of the AAAI\/ACM Conference on AI, Ethics, and Society","volume":"7","author":"Feffer Michael","year":"2024","unstructured":"Michael Feffer, Anusha Sinha, Wesley\u00a0H Deng, Zachary\u00a0C Lipton, and Hoda Heidari. 2024. Red-Teaming for generative AI: Silver bullet or security theater?. In Proceedings of the AAAI\/ACM Conference on AI, Ethics, and Society , Vol.\u00a07. 421\u2013437."},{"key":"e_1_3_3_2_9_2","doi-asserted-by":"crossref","unstructured":"H. Frluckaj L. Dabbish D.\u00a0G. Widder H.\u00a0S. Qiu and J.\u00a0D. Herbsleb. 2022. Gender and participation in open source software development. Proceedings of the ACM on Human-Computer Interaction 6 CSCW2 (2022) 1\u201331.","DOI":"10.1145\/3555190"},{"key":"e_1_3_3_2_10_2","volume-title":"Content Moderation: The harrowing, traumatizing job that left many African data workers with mental health issues and drug dependency","author":"Gebrekidan Fasica\u00a0Berhane","year":"2024","unstructured":"Fasica\u00a0Berhane Gebrekidan. 2024. Content Moderation: The harrowing, traumatizing job that left many African data workers with mental health issues and drug dependency. Data Workers\u2019 Inquiry. https:\/\/data-workers.org\/fasica\/"},{"key":"e_1_3_3_2_11_2","doi-asserted-by":"crossref","unstructured":"Shagun Jhaver Amy Bruckman and Eric Gilbert. 2019. Does transparency in moderation really matter? User behavior after content removal explanations on reddit. Proceedings of the ACM on Human-Computer Interaction 3 CSCW (2019) 1\u201327.","DOI":"10.1145\/3359252"},{"key":"e_1_3_3_2_12_2","doi-asserted-by":"publisher","DOI":"10.1145\/3678884.3681825"},{"key":"e_1_3_3_2_13_2","doi-asserted-by":"crossref","unstructured":"R. Li P. Pandurangan H. Frluckaj and L. Dabbish. 2021. Code of conduct conversations in open source software projects on GitHub. Proceedings of the ACM on Human-computer Interaction 5 CSCW1 (2021) 1\u201331.","DOI":"10.1145\/3449093"},{"key":"e_1_3_3_2_14_2","unstructured":"Jacob Metcalf Elizabeth\u00a0Anne Watkins Emanuel Moss and Madeleine\u00a0Clare Elish. 2023. Recommendations for Using Red-Teaming for AI Accountability. https:\/\/datasociety.net\/wp-content\/uploads\/2023\/10\/Recommendations-for-Using-Red-Teaming-for-AI-Accountability-PolicyBrief.pdf Policy Brief."},{"key":"e_1_3_3_2_15_2","volume-title":"Meet the hackers who are trying to make AI go rogue","author":"Oremus Will","year":"2023","unstructured":"Will Oremus. 2023. Meet the hackers who are trying to make AI go rogue. https:\/\/www.washingtonpost.com\/technology\/2023\/08\/08\/ai-red-team-defcon\/"},{"key":"e_1_3_3_2_16_2","doi-asserted-by":"publisher","DOI":"10.1145\/3715275.3732120"},{"key":"e_1_3_3_2_17_2","doi-asserted-by":"crossref","unstructured":"Alice Qian Judith Amores Hong Shen Mary Czerwinski Mary\u00a0L Gray and Jina Suh. 2025. AURA: Amplifying Understanding Resilience and Awareness for Responsible AI Content Work. Proceedings of the ACM on Human-Computer Interaction 9 CSCW2 (2025) 1\u201345.","DOI":"10.1145\/3710931"},{"key":"e_1_3_3_2_18_2","doi-asserted-by":"crossref","unstructured":"Alice Qian Judith Amores Hong Shen Mary Czerwinski Mary\u00a0L Gray and Jina Suh. 2025. AURA: Amplifying understanding resilience and awareness for responsible AI content work. Proceedings of the ACM on Human-Computer Interaction 9 2 (2025) 1\u201345.","DOI":"10.1145\/3710931"},{"key":"e_1_3_3_2_19_2","first-page":"712","volume-title":"Companion Publication of the 2024 Conference on Computer-Supported Cooperative Work and Social Computing","author":"Qian Alice","year":"2024","unstructured":"Alice Qian, Ryland Shaw, Jacy\u00a0Reese Anthis, Ashlee Milton, Emily Tseng, Jina Suh, Lama Ahmad, Ram Shankar\u00a0Siva Kumar, Julian Posada, Benjamin Shestakofsky, et\u00a0al. 2024. The human factor in ai red teaming: Perspectives from social and collaborative computing. In Companion Publication of the 2024 Conference on Computer-Supported Cooperative Work and Social Computing. 712\u2013715."},{"key":"e_1_3_3_2_20_2","unstructured":"Alice Qian Ryland Shaw Laura Dabbish Jina Suh and Hong Shen. 2025. Locating Risk: Task Designers and the Challenge of Risk Disclosure in RAI Content Work. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2505.24246 (2025)."},{"key":"e_1_3_3_2_21_2","doi-asserted-by":"crossref","unstructured":"Alice Qian Jina Suh Mary\u00a0L Gray and Hong Shen. 2025. Effective Automation to Support the Human Infrastructure in AI Red Teaming. Interactions 32 4 (2025) 58\u201361.","DOI":"10.1145\/3731866"},{"key":"e_1_3_3_2_22_2","volume-title":"The Unknown Women of Content Moderation","author":"Ranta Botlhokwa","year":"2024","unstructured":"Botlhokwa Ranta. 2024. The Unknown Women of Content Moderation. Data Workers\u2019 Inquiry. https:\/\/data-workers.org\/ranta\/"},{"key":"e_1_3_3_2_23_2","volume-title":"Behind the screen","author":"Roberts Sarah\u00a0T","year":"2019","unstructured":"Sarah\u00a0T Roberts. 2019. Behind the screen. Yale University Press."},{"key":"e_1_3_3_2_24_2","unstructured":"Reva Schwartz Gabriella Waters Razvan Amironesei Craig Greenberg Jon Fiscus Patrick Hall Anya Jones Shomik Jain Afzal Godil Kristen Greene et\u00a0al. 2024. The Assessing Risks and Impacts of AI (ARIA) Program Evaluation Design Document. (2024)."},{"key":"e_1_3_3_2_25_2","doi-asserted-by":"crossref","unstructured":"Hong Shen Alicia DeVos Motahhare Eslami and Kenneth Holstein. 2021. Everyday algorithm auditing: Understanding the power of everyday users in surfacing harmful algorithmic behaviors. Proceedings of the ACM on Human-Computer Interaction 5 CSCW2 (2021) 1\u201329.","DOI":"10.1145\/3479577"},{"key":"e_1_3_3_2_26_2","unstructured":"Ranjit Singh Borhane Blili-Hamelin Carol Anderson Emnet Tafesse Briana Vecchione Beth Duckles and Jacob Metcalf. 2025. Red-Teaming in the Public Interest. New York: Data & Society Research Institute (2025)."},{"key":"e_1_3_3_2_27_2","doi-asserted-by":"crossref","unstructured":"Araz Taeihagh. 2025. Governance of Generative AI. puaf001\u00a0pages.","DOI":"10.1093\/polsoc\/puaf001"},{"key":"e_1_3_3_2_28_2","doi-asserted-by":"publisher","DOI":"10.1145\/3491102.3502121"},{"key":"e_1_3_3_2_29_2","doi-asserted-by":"publisher","DOI":"10.1145\/3491102.3502121"},{"key":"e_1_3_3_2_30_2","doi-asserted-by":"publisher","DOI":"10.1145\/3531146.3533779"}],"event":{"name":"CHI EA '26: Extended Abstracts of the 2026 CHI Conference on Human Factors in Computing Systems","location":"Barcelona , Spain","acronym":"CHI EA '26","sponsor":["SIGCHI ACM Special Interest Group on Computer-Human Interaction"]},"container-title":["Proceedings of the Extended Abstracts of the 2026 CHI Conference on Human Factors in Computing Systems"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3772363.3778702","content-type":"application\/pdf","content-version":"vor","intended-application":"syndication"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3772363.3778702","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,4,13]],"date-time":"2026-04-13T12:54:07Z","timestamp":1776084847000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3772363.3778702"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026,4,13]]},"references-count":29,"alternative-id":["10.1145\/3772363.3778702","10.1145\/3772363"],"URL":"https:\/\/doi.org\/10.1145\/3772363.3778702","relation":{},"subject":[],"published":{"date-parts":[[2026,4,13]]},"assertion":[{"value":"2026-04-13","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}