{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,5,1]],"date-time":"2026-05-01T16:48:15Z","timestamp":1777654095639,"version":"3.51.4"},"publisher-location":"New York, NY, USA","reference-count":59,"publisher":"ACM","license":[{"start":{"date-parts":[[2024,6,3]],"date-time":"2024-06-03T00:00:00Z","timestamp":1717372800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2024,6,3]]},"DOI":"10.1145\/3630106.3658913","type":"proceedings-article","created":{"date-parts":[[2024,6,5]],"date-time":"2024-06-05T13:14:21Z","timestamp":1717593261000},"page":"388-406","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":11,"title":["Adversarial Nibbler: An Open Red-Teaming Method for Identifying Diverse Harms in Text-to-Image Generation"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0009-0004-3838-7498","authenticated-orcid":false,"given":"Jessica","family":"Quaye","sequence":"first","affiliation":[{"name":"Harvard University, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-1054-0516","authenticated-orcid":false,"given":"Alicia","family":"Parrish","sequence":"additional","affiliation":[{"name":"Google, United States of America"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-4691-6586","authenticated-orcid":false,"given":"Oana","family":"Inel","sequence":"additional","affiliation":[{"name":"University of Z\u00fcrich, Switzerland"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-0820-4115","authenticated-orcid":false,"given":"Charvi","family":"Rastogi","sequence":"additional","affiliation":[{"name":"Google, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-7419-5993","authenticated-orcid":false,"given":"Hannah Rose","family":"Kirk","sequence":"additional","affiliation":[{"name":"University of Oxford, United Kingdom"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-0291-6026","authenticated-orcid":false,"given":"Minsuk","family":"Kahng","sequence":"additional","affiliation":[{"name":"Google, USA"}]},{"ORCID":"https:\/\/orcid.org\/0009-0002-2016-200X","authenticated-orcid":false,"given":"Erin","family":"Van Liemt","sequence":"additional","affiliation":[{"name":"Google, USA"}]},{"ORCID":"https:\/\/orcid.org\/0009-0007-3301-7895","authenticated-orcid":false,"given":"Max","family":"Bartolo","sequence":"additional","affiliation":[{"name":"University College London, Cohere, United Kingdom"}]},{"ORCID":"https:\/\/orcid.org\/0009-0004-9094-7181","authenticated-orcid":false,"given":"Jess","family":"Tsang","sequence":"additional","affiliation":[{"name":"Google, USA"}]},{"ORCID":"https:\/\/orcid.org\/0009-0005-5414-7061","authenticated-orcid":false,"given":"Justin","family":"White","sequence":"additional","affiliation":[{"name":"Google, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-7533-625X","authenticated-orcid":false,"given":"Nathan","family":"Clement","sequence":"additional","affiliation":[{"name":"Google, United Kingdom"}]},{"ORCID":"https:\/\/orcid.org\/0009-0009-0812-6330","authenticated-orcid":false,"given":"Rafael","family":"Mosquera","sequence":"additional","affiliation":[{"name":"ML Commons, Colombia"}]},{"ORCID":"https:\/\/orcid.org\/0009-0000-4179-3076","authenticated-orcid":false,"given":"Juan","family":"Ciro","sequence":"additional","affiliation":[{"name":"ML Commons, Colombia"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-5259-7721","authenticated-orcid":false,"given":"Vijay","family":"Janapa Reddi","sequence":"additional","affiliation":[{"name":"Harvard University, United States of America"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-9402-1133","authenticated-orcid":false,"given":"Lora","family":"Aroyo","sequence":"additional","affiliation":[{"name":"Google, USA"}]}],"member":"320","published-online":{"date-parts":[[2024,6,5]]},"reference":[{"key":"e_1_3_2_1_1_1","unstructured":"[n. d.]. 4chan. https:\/\/www.4chan.org\/ [Accessed on 10\/01\/2024 ]."},{"key":"e_1_3_2_1_2_1","unstructured":"[n. d.]. Lexica. https:\/\/www.lexica.art\/ [Accessed on 10\/01\/2024 ]."},{"key":"e_1_3_2_1_3_1","unstructured":"2023. Nudity and sexual activity: publisher and creator guidelines - Guidelines for safe respectful behavior. https:\/\/www.facebook.com\/business\/help\/725672454452774?id=208060977200861 Accessed on 12\/04\/23."},{"key":"e_1_3_2_1_4_1","unstructured":"2023. The X Rules: safety privacy authenticity and more. https:\/\/help.twitter.com\/en\/rules-and-policies\/x-rules Accessed on 12\/04\/23."},{"key":"e_1_3_2_1_5_1","unstructured":"Lora Aroyo and Praveen Paritosh. [n. d.]. Uncovering Unknown Unknowns in Machine Learning. https:\/\/ai.googleblog.com\/2021\/02\/uncovering-unknown-unknowns-in-machine.html"},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"publisher","DOI":"10.1162\/tacl_a_00338"},{"key":"e_1_3_2_1_7_1","volume-title":"Proceedings of the 2022 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies. 3754\u20133767","author":"Bartolo Max","year":"2022","unstructured":"Max Bartolo, Tristan Thrush, Sebastian Riedel, Pontus Stenetorp, Robin Jia, and Douwe Kiela. 2022. Models in the Loop: Aiding Crowdworkers with Generative Annotation Assistants. In Proceedings of the 2022 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies. 3754\u20133767."},{"key":"e_1_3_2_1_8_1","volume-title":"Proceedings of the IEEE\/CVF International Conference on Computer Vision (ICCV).","author":"Basu Abhipsa","year":"2023","unstructured":"Abhipsa Basu, R\u00a0Venkatesh Babu, and Danish Pruthi. 2023. Inspecting the Geographical Representativeness of Images from Text-to-Image Models. In Proceedings of the IEEE\/CVF International Conference on Computer Vision (ICCV)."},{"key":"e_1_3_2_1_9_1","first-page":"320","article-title":"DendroMap: Visual Exploration of Large-Scale Image Datasets for Machine Learning with Treemaps","volume":"29","author":"Bertucci Donald","year":"2022","unstructured":"Donald Bertucci, Md\u00a0Montaser Hamid, Yashwanthi Anand, Anita Ruangrotsakun, Delyar Tabatabai, Melissa Perez, and Minsuk Kahng. 2022. DendroMap: Visual Exploration of Large-Scale Image Datasets for Machine Learning with Treemaps. IEEE Transactions on Visualization and Computer Graphics (VIS) 29, 1 (2022), 320\u2013330.","journal-title":"IEEE Transactions on Visualization and Computer Graphics (VIS)"},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.1145\/3593013.3594095"},{"key":"e_1_3_2_1_11_1","volume-title":"Into the LAIONs Den: Investigating Hate in Multimodal Datasets. arXiv preprint arXiv:2311.03449","author":"Birhane Abeba","year":"2023","unstructured":"Abeba Birhane, Vinay Prabhu, Sang Han, Vishnu\u00a0Naresh Boddeti, and Alexandra\u00a0Sasha Luccioni. 2023. Into the LAIONs Den: Investigating Hate in Multimodal Datasets. arXiv preprint arXiv:2311.03449 (2023)."},{"key":"e_1_3_2_1_12_1","volume-title":"Multimodal datasets: Misogyny, pornography, and malignant stereotypes. arXiv preprint arXiv:2110.01963","author":"Birhane Abeba","year":"2021","unstructured":"Abeba Birhane, Vinay\u00a0Uday Prabhu, and Emmanuel Kahembwe. 2021. Multimodal datasets: Misogyny, pornography, and malignant stereotypes. arXiv preprint arXiv:2110.01963 (2021)."},{"key":"e_1_3_2_1_13_1","volume-title":"Toward Trustworthy AI Development: Mechanisms for Supporting Verifiable Claims. arXiv preprint arXiv:2211.03759","author":"Brundage Miles","year":"2023","unstructured":"Miles Brundage, Shahar Avin, Jasmine Wang, Haydn Belfield, Gretchen Krueger, and Gillian et\u00a0al. Hadfield. 2023. Toward Trustworthy AI Development: Mechanisms for Supporting Verifiable Claims. arXiv preprint arXiv:2211.03759 (2023)."},{"key":"e_1_3_2_1_14_1","unstructured":"Cats4ML. [n. d.]. Cats4ML Challenge. https:\/\/cats4ml.humancomputation.com\/"},{"key":"e_1_3_2_1_15_1","volume-title":"Proceedings of the IEEE\/CVF International Conference on Computer Vision (ICCV).","author":"Cho Jaemin","year":"2023","unstructured":"Jaemin Cho, Abhay Zala, and Mohit Bansal. 2023. DALL-Eval: Probing the Reasoning Skills and Social Biases of Text-to-Image Generation Models. In Proceedings of the IEEE\/CVF International Conference on Computer Vision (ICCV)."},{"key":"e_1_3_2_1_16_1","unstructured":"DeepLearning.AI. [n. d.]. Data-Centric AI Competition. https:\/\/https-deeplearning-ai.github.io\/data-centric-comp\/"},{"key":"e_1_3_2_1_17_1","volume-title":"Assessing Language Model Deployment with Risk Cards. arXiv preprint arXiv:2303.18190","author":"Derczynski Leon","year":"2023","unstructured":"Leon Derczynski, Hannah\u00a0Rose Kirk, Vidhisha Balachandran, Sachin Kumar, Yulia Tsvetkov, MR Leiser, and Saif Mohammad. 2023. Assessing Language Model Deployment with Risk Cards. arXiv preprint arXiv:2303.18190 (2023)."},{"key":"e_1_3_2_1_18_1","volume-title":"\u201cstress test","author":"Field Hayden","year":"2022","unstructured":"Hayden Field. 2022. How Microsoft and Google use AI red teams to \u201cstress test\u201d their systems. https:\/\/www.emergingtechbrew.com\/stories\/2022\/06\/14\/how-microsoft-and-google-use-ai-red-teams-to-stress-test-their-system Accessed on 03\/08\/23."},{"key":"e_1_3_2_1_19_1","volume-title":"Red teaming language models to reduce harms: Methods, scaling behaviors, and lessons learned. arXiv preprint arXiv:2209.07858","author":"Ganguli Deep","year":"2022","unstructured":"Deep Ganguli, Liane Lovitt, Jackson Kernion, Amanda Askell, Yuntao Bai, Saurav Kadavath, Ben Mann, Ethan Perez, Nicholas Schiefer, Kamal Ndousse, 2022. Red teaming language models to reduce harms: Methods, scaling behaviors, and lessons learned. arXiv preprint arXiv:2209.07858 (2022)."},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.1145\/3306618.3314282"},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2021.naacl-main.324"},{"key":"e_1_3_2_1_22_1","volume-title":"What\u2019s in the Box? A Preliminary Analysis of Undesirable Content in the Common Crawl Corpus. arXiv preprint arXiv:2105.02732","author":"Luccioni Alexandra\u00a0Sasha","year":"2021","unstructured":"Alexandra\u00a0Sasha Luccioni and Joseph\u00a0D Viviano. 2021. What\u2019s in the Box? A Preliminary Analysis of Undesirable Content in the Common Crawl Corpus. arXiv preprint arXiv:2105.02732 (2021)."},{"key":"e_1_3_2_1_23_1","volume-title":"Dataperf: Benchmarks for data-centric AI development. arXiv preprint arXiv:2207.10062","author":"Mazumder Mark","year":"2022","unstructured":"Mark Mazumder, Colby Banbury, Xiaozhe Yao, Bojan Karla\u0161, William\u00a0Gaviria Rojas, Sudnya Diamos, Greg Diamos, Lynn He, Douwe Kiela, David Jurado, 2022. Dataperf: Benchmarks for data-centric AI development. arXiv preprint arXiv:2207.10062 (2022)."},{"key":"e_1_3_2_1_24_1","volume-title":"Article 115","author":"Mehrabi Ninareh","year":"2021","unstructured":"Ninareh Mehrabi, Fred Morstatter, Nripsuta Saxena, Kristina Lerman, and Aram Galstyan. 2021. A Survey on Bias and Fairness in Machine Learning. Comput. Surveys 54, 6, Article 115 (2021), 35\u00a0pages."},{"key":"e_1_3_2_1_25_1","unstructured":"Midjourney. 2023. Midjourney Documentation and User Guide. https:\/\/docs.midjourney.com\/. (Accessed on 04\/19\/2023)."},{"key":"e_1_3_2_1_26_1","volume-title":"Adversarial attacks on image generation with made-up words. arXiv preprint arXiv:2208.04135","author":"Milli\u00e8re Rapha\u00ebl","year":"2022","unstructured":"Rapha\u00ebl Milli\u00e8re. 2022. Adversarial attacks on image generation with made-up words. arXiv preprint arXiv:2208.04135 (2022)."},{"key":"e_1_3_2_1_27_1","volume-title":"Auditing large language models: A three-layered approach. arXiv preprint arXiv:2302.08500","author":"M\u00f6kander Jakob","year":"2023","unstructured":"Jakob M\u00f6kander, Jonas Schuett, Hannah\u00a0Rose Kirk, and Luciano Floridi. 2023. Auditing large language models: A three-layered approach. arXiv preprint arXiv:2302.08500 (2023)."},{"key":"e_1_3_2_1_28_1","unstructured":"Madhumita Murgia. [n. d.]. OpenAI\u2019s red team: the experts hired to \u2018break\u2019 ChatGPT. ([n. d.]). https:\/\/www.ft.com\/content\/0876687a-f8b7-4b39-b513-5fee942831e8"},{"key":"e_1_3_2_1_29_1","volume-title":"Proceedings of the 59th Annual Meeting of the Association for Computational Linguistics and the 11th International Joint Conference on Natural Language Processing 1: Long Papers","author":"Nadeem Moin","year":"2021","unstructured":"Moin Nadeem, Anna Bethke, and Siva Reddy. 2021. StereoSet: Measuring stereotypical bias in pretrained language models. Proceedings of the 59th Annual Meeting of the Association for Computational Linguistics and the 11th International Joint Conference on Natural Language Processing 1: Long Papers (2021), 5356\u20135371."},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"publisher","DOI":"10.1145\/3600211.3604711"},{"key":"e_1_3_2_1_31_1","volume-title":"DMLR: Data-centric Machine Learning Research\u2013Past, Present and Future. arXiv preprint arXiv:2311.13028","author":"Oala Luis","year":"2023","unstructured":"Luis Oala, Manil Maskey, Lilith Bat-Leah, Alicia Parrish, Nezihe Merve\u00a0G\u00fcrel, Tzu-Sheng Kuo, Yang Liu, Rotem Dror, Danilo Brajovic, Xiaozhe Yao, 2023. DMLR: Data-centric Machine Learning Research\u2013Past, Present and Future. arXiv preprint arXiv:2311.13028 (2023)."},{"key":"e_1_3_2_1_32_1","unstructured":"OpenAI. [n. d.]. DALL-E 2 System Card. https:\/\/github.com\/openai\/dalle-2-preview\/blob\/main\/system-card.md#early-work"},{"key":"e_1_3_2_1_33_1","volume-title":"Policy recommendations for dealing with ambiguity in machine vision models. arXiv preprint arXiv:2306.15777","author":"Parrish Alicia","year":"2023","unstructured":"Alicia Parrish, Sarah Laszlo, and Lora Aroyo. 2023. \" Is a picture of a bird a bird\": Policy recommendations for dealing with ambiguity in machine vision models. arXiv preprint arXiv:2306.15777 (2023)."},{"key":"e_1_3_2_1_34_1","doi-asserted-by":"publisher","DOI":"10.1145\/3539618.3592000"},{"key":"e_1_3_2_1_35_1","volume-title":"Proceedings of the 2022 Conference on Empirical Methods in Natural Language Processing (EMNLP).","author":"Perez Ethan","year":"2022","unstructured":"Ethan Perez, Saffron Huang, Francis Song, Trevor Cai, Roman Ring, John Aslanides, Amelia Glaese, Nat McAleese, and Geoffrey Irving. 2022. Red teaming language models with language models. In Proceedings of the 2022 Conference on Empirical Methods in Natural Language Processing (EMNLP)."},{"key":"e_1_3_2_1_36_1","volume-title":"Proceedings of the 59th Annual Meeting of the Association for Computational Linguistics and the 11th International Joint Conference on Natural Language Processing (Volume 1: Long Papers). 2388\u20132404","author":"Potts Christopher","year":"2021","unstructured":"Christopher Potts, Zhengxuan Wu, Atticus Geiger, and Douwe Kiela. 2021. DynaSent: A Dynamic Benchmark for Sentiment Analysis. In Proceedings of the 59th Annual Meeting of the Association for Computational Linguistics and the 11th International Joint Conference on Natural Language Processing (Volume 1: Long Papers). 2388\u20132404."},{"key":"e_1_3_2_1_37_1","volume-title":"2023 IEEE Symposium on Security and Privacy (SP). IEEE, 293\u2013310","author":"Qu Yiting","year":"2023","unstructured":"Yiting Qu, Xinlei He, Shannon Pierson, Michael Backes, Yang Zhang, and Savvas Zannettou. 2023. On the Evolution of (Hateful) Memes by Means of Multimodal Contrastive Learning. In 2023 IEEE Symposium on Security and Privacy (SP). IEEE, 293\u2013310."},{"key":"e_1_3_2_1_38_1","volume-title":"Proceedings of the 2023 ACM SIGSAC Conference on Computer and Communications Security. 3403\u20133417","author":"Qu Yiting","year":"2023","unstructured":"Yiting Qu, Xinyue Shen, Xinlei He, Michael Backes, Savvas Zannettou, and Yang Zhang. 2023. Unsafe Diffusion: On the Generation of Unsafe Images and Hateful Memes From Text-To-Image Models. In Proceedings of the 2023 ACM SIGSAC Conference on Computer and Communications Security. 3403\u20133417."},{"key":"e_1_3_2_1_39_1","volume-title":"International conference on machine learning. PMLR, 8748\u20138763","author":"Radford Alec","year":"2021","unstructured":"Alec Radford, Jong\u00a0Wook Kim, Chris Hallacy, Aditya Ramesh, Gabriel Goh, Sandhini Agarwal, Girish Sastry, Amanda Askell, Pamela Mishkin, Jack Clark, 2021. Learning transferable visual models from natural language supervision. In International conference on machine learning. PMLR, 8748\u20138763."},{"key":"e_1_3_2_1_40_1","doi-asserted-by":"publisher","DOI":"10.1145\/3351095.3372873"},{"key":"e_1_3_2_1_41_1","unstructured":"Aditya Ramesh Prafulla Dhariwal Alex Nichol Casey Chu and Mark Chen. 2022. Hierarchical Text-Conditional Image Generation with CLIP Latents. arxiv:2204.06125\u00a0[cs.CV]"},{"key":"e_1_3_2_1_42_1","volume-title":"Proceedings of the 38th International Conference on Machine Learning (ICML). PMLR, 8821\u20138831","author":"Ramesh Aditya","year":"2021","unstructured":"Aditya Ramesh, Mikhail Pavlov, Gabriel Goh, Scott Gray, Chelsea Voss, Alec Radford, Mark Chen, and Ilya Sutskever. 2021. Zero-Shot Text-to-Image Generation. In Proceedings of the 38th International Conference on Machine Learning (ICML). PMLR, 8821\u20138831."},{"key":"e_1_3_2_1_43_1","volume-title":"Red-teaming the stable diffusion safety filter. arXiv preprint arXiv:2210.04610","author":"Rando Javier","year":"2022","unstructured":"Javier Rando, Daniel Paleka, David Lindner, Lennart Heim, and Florian Tram\u00e8r. 2022. Red-teaming the stable diffusion safety filter. arXiv preprint arXiv:2210.04610 (2022)."},{"key":"e_1_3_2_1_44_1","doi-asserted-by":"publisher","DOI":"10.1145\/3600211.3604712"},{"key":"e_1_3_2_1_45_1","volume-title":"Fifth International Conference on Coordinated and Multiple Views in Exploratory Visualization. IEEE, 61\u201371","author":"Roberts C","year":"2007","unstructured":"Jonathan\u00a0C Roberts. 2007. State of the art: Coordinated & multiple views in exploratory visualization. In Fifth International Conference on Coordinated and Multiple Views in Exploratory Visualization. IEEE, 61\u201371."},{"key":"e_1_3_2_1_46_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52688.2022.01042"},{"key":"e_1_3_2_1_47_1","first-page":"36479","article-title":"Photorealistic Text-to-Image Diffusion Models with Deep Language Understanding","volume":"35","author":"Saharia Chitwan","year":"2022","unstructured":"Chitwan Saharia, William Chan, Saurabh Saxena, Lala Li, Jay Whang, Emily Denton, Seyed Kamyar\u00a0Seyed Ghasemipour, Burcu\u00a0Karagol Ayan, S.\u00a0Sara Mahdavi, Rapha\u00a0Gontijo Lopes, Tim Salimans, Jonathan Ho, David\u00a0J Fleet, and Mohammad Norouzi. 2022. Photorealistic Text-to-Image Diffusion Models with Deep Language Understanding. Advances in Neural Information Processing Systems 35 (2022), 36479\u201336494.","journal-title":"Advances in Neural Information Processing Systems"},{"key":"e_1_3_2_1_48_1","volume-title":"Analyzing Demographic Bias in Artificially Generated Facial Pictures. In Extended Abstracts of the 2020 CHI Conference on Human Factors in Computing Systems. ACM, 1\u20138.","author":"Salminen Joni","year":"2020","unstructured":"Joni Salminen, Soon-gyo Jung, Shammur Chowdhury, and Bernard\u00a0J. Jansen. 2020. Analyzing Demographic Bias in Artificially Generated Facial Pictures. In Extended Abstracts of the 2020 CHI Conference on Human Factors in Computing Systems. ACM, 1\u20138."},{"key":"e_1_3_2_1_49_1","volume-title":"Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition. 22522\u201322531","author":"Schramowski Patrick","year":"2023","unstructured":"Patrick Schramowski, Manuel Brack, Bj\u00f6rn Deiseroth, and Kristian Kersting. 2023. Safe latent diffusion: Mitigating inappropriate degeneration in diffusion models. In Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition. 22522\u201322531."},{"key":"e_1_3_2_1_50_1","volume-title":"Proceedings of the International AAAI Conference on Web and Social Media, Vol.\u00a016","author":"Shen Xinyue","year":"2022","unstructured":"Xinyue Shen, Xinlei He, Michael Backes, Jeremy Blackburn, Savvas Zannettou, and Yang Zhang. 2022. On Xing Tian and the Perseverance of Anti-China Sentiment Online. In Proceedings of the International AAAI Conference on Web and Social Media, Vol.\u00a016. 944\u2013955."},{"key":"e_1_3_2_1_51_1","doi-asserted-by":"publisher","DOI":"10.1109\/VL.1996.545307"},{"key":"e_1_3_2_1_52_1","volume-title":"Proceedings of the IEEE\/CVF International Conference on Computer Vision. 15291\u201315301","author":"Slyman Eric","year":"2023","unstructured":"Eric Slyman, Minsuk Kahng, and Stefan Lee. 2023. VLSlice: Interactive Vision-and-Language Slice Discovery. In Proceedings of the IEEE\/CVF International Conference on Computer Vision. 15291\u201315301."},{"key":"e_1_3_2_1_53_1","volume-title":"Proceedings of the 2022 Conference on Empirical Methods in Natural Language Processing (EMNLP). 9180\u20139211","author":"Smith Eric\u00a0Michael","year":"2022","unstructured":"Eric\u00a0Michael Smith, Melissa Hall, Melanie Kambadur, Eleonora Presani, and Adina Williams. 2022. \u201cI\u2019m sorry to hear that\u201d: Finding New Biases in Language Models with a Holistic Descriptor Dataset. In Proceedings of the 2022 Conference on Empirical Methods in Natural Language Processing (EMNLP). 9180\u20139211."},{"key":"e_1_3_2_1_54_1","unstructured":"Snorkel. [n. d.]. Data-centric AI: A complete primer. https:\/\/snorkel.ai\/data-centric-ai-primer\/"},{"key":"e_1_3_2_1_55_1","volume-title":"Proceedings of the 25th Conference on Computational Natural Language Learning. 552\u2013561","author":"Song Hoyun","year":"2021","unstructured":"Hoyun Song, Soo\u00a0Hyun Ryu, Huije Lee, and Jong\u00a0C Park. 2021. A large-scale comprehensive abusiveness detection dataset with multifaceted labels from reddit. In Proceedings of the 25th Conference on Computational Natural Language Learning. 552\u2013561."},{"key":"e_1_3_2_1_56_1","volume-title":"Proceedings of the 60th Annual Meeting of the Association for Computational Linguistics: System Demonstrations. 174\u2013181","author":"Thrush Tristan","year":"2022","unstructured":"Tristan Thrush, Kushal Tirumala, Anmol Gupta, Max Bartolo, Pedro Rodriguez, Tariq Kane, William\u00a0Gaviria Rojas, Peter Mattson, Adina Williams, and Douwe Kiela. 2022. Dynatask: A Framework for Creating Dynamic AI Benchmark Tasks. In Proceedings of the 60th Annual Meeting of the Association for Computational Linguistics: System Demonstrations. 174\u2013181."},{"key":"e_1_3_2_1_57_1","volume-title":"Findings of the WMT 2021 shared task on large-scale multilingual machine translation. In Proceedings of the Sixth Conference on Machine Translation. 89\u201399","author":"Wenzek Guillaume","year":"2021","unstructured":"Guillaume Wenzek, Vishrav Chaudhary, Angela Fan, Sahir Gomez, Naman Goyal, Somya Jain, Douwe Kiela, Tristan Thrush, and Francisco Guzm\u00e1n. 2021. Findings of the WMT 2021 shared task on large-scale multilingual machine translation. In Proceedings of the Sixth Conference on Machine Translation. 89\u201399."},{"key":"e_1_3_2_1_58_1","volume-title":"SneakyPrompt: Jailbreaking Text-to-image Generative Models. arXiv preprint arXiv:2305.12082","author":"Yang Yuchen","year":"2023","unstructured":"Yuchen Yang, Bo Hui, Haolin Yuan, Neil Gong, and Yinzhi Cao. 2023. SneakyPrompt: Jailbreaking Text-to-image Generative Models. arXiv preprint arXiv:2305.12082 (2023)."},{"key":"e_1_3_2_1_59_1","doi-asserted-by":"publisher","DOI":"10.1109\/TVCG.2021.3114837"}],"event":{"name":"FAccT '24: The 2024 ACM Conference on Fairness, Accountability, and Transparency","location":"Rio de Janeiro Brazil","acronym":"FAccT '24"},"container-title":["The 2024 ACM Conference on Fairness, Accountability, and Transparency"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3630106.3658913","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3630106.3658913","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,18]],"date-time":"2025-06-18T22:50:57Z","timestamp":1750287057000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3630106.3658913"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,6,3]]},"references-count":59,"alternative-id":["10.1145\/3630106.3658913","10.1145\/3630106"],"URL":"https:\/\/doi.org\/10.1145\/3630106.3658913","relation":{},"subject":[],"published":{"date-parts":[[2024,6,3]]},"assertion":[{"value":"2024-06-05","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}