{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,13]],"date-time":"2026-04-13T23:51:19Z","timestamp":1776124279767,"version":"3.50.1"},"publisher-location":"New York, NY, USA","reference-count":102,"publisher":"ACM","license":[{"start":{"date-parts":[[2023,4,19]],"date-time":"2023-04-19T00:00:00Z","timestamp":1681862400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2023,4,19]]},"DOI":"10.1145\/3544548.3580645","type":"proceedings-article","created":{"date-parts":[[2023,4,20]],"date-time":"2023-04-20T04:27:20Z","timestamp":1681964840000},"page":"1-15","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":45,"title":["A hunt for the Snark: Annotator Diversity in Data Practices"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-0152-4311","authenticated-orcid":false,"given":"Shivani","family":"Kapania","sequence":"first","affiliation":[{"name":"Google Research, India"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-6311-3967","authenticated-orcid":false,"given":"Alex S","family":"Taylor","sequence":"additional","affiliation":[{"name":"Centre for HCI Design, City, University of London, United Kingdom"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-5037-5017","authenticated-orcid":false,"given":"Ding","family":"Wang","sequence":"additional","affiliation":[{"name":"Google, Singapore"}]}],"member":"320","published-online":{"date-parts":[[2023,4,19]]},"reference":[{"key":"e_1_3_3_2_1_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1"},{"key":"e_1_3_3_2_2_1","doi-asserted-by":"publisher","DOI":"10.1145\/3308560.3317083"},{"key":"e_1_3_3_2_3_1","doi-asserted-by":"publisher","DOI":"10.1609\/aimag.v36i1.2564"},{"key":"e_1_3_3_2_4_1","doi-asserted-by":"publisher","DOI":"10.1086\/345321"},{"key":"e_1_3_3_2_5_1","doi-asserted-by":"publisher","DOI":"10.2307\/j.ctv12101zq"},{"key":"e_1_3_3_2_6_1","doi-asserted-by":"publisher","DOI":"10.1162\/tacl_a_00041"},{"key":"e_1_3_3_2_7_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.chb.2015.08.031"},{"key":"e_1_3_3_2_8_1","volume-title":"Proceedings of the 1st Conference on Fairness, Accountability and Transparency(Proceedings of Machine Learning Research, Vol.\u00a081)","author":"Buolamwini Joy","year":"2018","unstructured":"Joy Buolamwini and Timnit Gebru. 2018. Gender Shades: Intersectional Accuracy Disparities in Commercial Gender Classification. In Proceedings of the 1st Conference on Fairness, Accountability and Transparency(Proceedings of Machine Learning Research, Vol.\u00a081), Sorelle\u00a0A. Friedler and Christo Wilson (Eds.). PMLR, New York, NY, USA, 77\u201391. https:\/\/proceedings.mlr.press\/v81\/buolamwini18a.html"},{"key":"e_1_3_3_2_9_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-94-015-7372-6_1"},{"key":"e_1_3_3_2_10_1","doi-asserted-by":"publisher","DOI":"10.1126\/science.aal4230"},{"key":"e_1_3_3_2_11_1","doi-asserted-by":"publisher","DOI":"10.1145\/3491102.3501998"},{"key":"e_1_3_3_2_12_1","doi-asserted-by":"publisher","DOI":"10.48550\/arXiv.1610.07183"},{"key":"e_1_3_3_2_13_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-1-4614-8806-4_30"},{"key":"e_1_3_3_2_14_1","unstructured":"Patricia\u00a0Hill Collins and Sirma Bilge. 2020. Intersectionality. John Wiley & Sons."},{"key":"e_1_3_3_2_15_1","volume-title":"Doing design ethnography","author":"Crabtree Andrew","unstructured":"Andrew Crabtree, Mark Rouncefield, and Peter Tolmie. 2012. Doing design ethnography. Springer, London."},{"key":"e_1_3_3_2_16_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-1-4471-5346-7_1"},{"key":"e_1_3_3_2_17_1","volume-title":"Feminist legal theories","author":"Crenshaw Kimberl\u00e9","unstructured":"Kimberl\u00e9 Crenshaw. 2013. Demarginalizing the intersection of race and sex: A black feminist critique of antidiscrimination doctrine, feminist theory and antiracist politics. In Feminist legal theories. Routledge, New York, NY, USA, 23\u201351."},{"key":"e_1_3_3_2_18_1","doi-asserted-by":"publisher","DOI":"10.1145\/3148148"},{"key":"e_1_3_3_2_19_1","doi-asserted-by":"publisher","DOI":"10.1162\/tacl_a_00449"},{"key":"e_1_3_3_2_20_1","doi-asserted-by":"publisher","DOI":"10.1177\/20539517211044808"},{"key":"e_1_3_3_2_21_1","doi-asserted-by":"publisher","DOI":"10.1093\/jamia"},{"key":"e_1_3_3_2_22_1","doi-asserted-by":"publisher","DOI":"10.1145\/3531146.3533113"},{"key":"e_1_3_3_2_23_1","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v34i05.6267"},{"key":"e_1_3_3_2_24_1","doi-asserted-by":"publisher","DOI":"10.1145\/3173574.3173986"},{"key":"e_1_3_3_2_25_1","volume-title":"Adversarial Design","author":"Disalvo C.","unstructured":"C. Disalvo. 2015. Adversarial Design. MIT Press, Cambridge, MA, USA. https:\/\/books.google.co.in\/books?id=gdcGEAAAQBAJ"},{"key":"e_1_3_3_2_26_1","doi-asserted-by":"publisher","DOI":"10.1609\/hcomp.v4i1.13270"},{"key":"e_1_3_3_2_27_1","volume-title":"Diversity in big data: A review. Big data 5, 2","author":"Drosou Marina","year":"2017","unstructured":"Marina Drosou, HV Jagadish, Evaggelia Pitoura, and Julia Stoyanovich. 2017. Diversity in big data: A review. Big data 5, 2 (2017), 73\u201384."},{"key":"e_1_3_3_2_28_1","doi-asserted-by":"publisher","DOI":"10.1177\/20539517221082027"},{"key":"e_1_3_3_2_29_1","unstructured":"Vinitha Gadiraju Kane Shaun Sunipa Dev Alex Taylor Ding Wang Robin Brewer and Emily Denton. In submission. \"I wouldn\u2019t say offensive but...\": Disability-Centered Perspectives on Large Language Models. 20\u00a0pages."},{"key":"e_1_3_3_2_30_1","doi-asserted-by":"publisher","DOI":"10.1145\/3458723"},{"key":"e_1_3_3_2_31_1","volume-title":"Raw Data Is an Oxymoron","author":"Gitelman L.","unstructured":"L. Gitelman. 2013. Raw Data Is an Oxymoron. MIT Press, Cambridge, MA, USA. https:\/\/books.google.co.in\/books?id=oARaHF4D6h0C"},{"key":"e_1_3_3_2_32_1","volume-title":"Doing grounded theory: Issues and discussions. Vol.\u00a0254","author":"Glaser G","unstructured":"Barney\u00a0G Glaser. 1998. Doing grounded theory: Issues and discussions. Vol.\u00a0254. Sociology Press, Mill Valley, CA."},{"key":"e_1_3_3_2_33_1","volume-title":"Discovery of grounded theory: Strategies for qualitative research","author":"Glaser G","unstructured":"Barney\u00a0G Glaser and Anselm\u00a0L Strauss. 2017. Discovery of grounded theory: Strategies for qualitative research. Routledge, London."},{"key":"e_1_3_3_2_34_1","volume-title":"Seeing in depth. Social studies of science 25, 2","author":"Goodwin Charles","year":"1995","unstructured":"Charles Goodwin. 1995. Seeing in depth. Social studies of science 25, 2 (1995), 237\u2013274."},{"key":"e_1_3_3_2_35_1","doi-asserted-by":"publisher","DOI":"10.1145\/3491102.3502004"},{"key":"e_1_3_3_2_36_1","doi-asserted-by":"publisher","DOI":"10.1145\/3411764.3445423"},{"key":"e_1_3_3_2_37_1","doi-asserted-by":"publisher","DOI":"10.1145\/3555088"},{"key":"e_1_3_3_2_38_1","volume-title":"Ghost work: How to stop Silicon Valley from building a new global underclass","author":"Gray L","unstructured":"Mary\u00a0L Gray and Siddharth Suri. 2019. Ghost work: How to stop Silicon Valley from building a new global underclass. Eamon Dolan Books, Boston, MA."},{"key":"e_1_3_3_2_39_1","volume-title":"Representing and intervening: Introductory topics in the philosophy of natural science","author":"Ian Hacking","unstructured":"Ian Hacking 1983. Representing and intervening: Introductory topics in the philosophy of natural science. Cambridge university press."},{"key":"e_1_3_3_2_40_1","volume-title":"When multiplication doesn\u2019t equal quick addition: Examining intersectionality as a research paradigm. Perspectives on politics 5, 1","author":"Hancock Ange-Marie","year":"2007","unstructured":"Ange-Marie Hancock. 2007. When multiplication doesn\u2019t equal quick addition: Examining intersectionality as a research paradigm. Perspectives on politics 5, 1 (2007), 63\u201379."},{"key":"e_1_3_3_2_41_1","volume-title":"Situated Knowledges: The Science Question in Feminism and the Privilege of Partial Perspective. Vol.\u00a014. Feminist Studies","author":"Haraway Donna","year":"1988","unstructured":"Donna Haraway. 1988. Situated Knowledges: The Science Question in Feminism and the Privilege of Partial Perspective. Vol.\u00a014. Feminist Studies, Inc., 575\u2013599. http:\/\/www.jstor.org\/stable\/3178066"},{"key":"e_1_3_3_2_42_1","doi-asserted-by":"publisher","DOI":"10.1145\/3386296.3386305"},{"key":"e_1_3_3_2_43_1","volume-title":"Think in Public","author":"Irani Lilly","unstructured":"Lilly Irani. 2019. Justice for data janitors. In Think in Public. Columbia University Press, New York, NY, USA, 23\u201340."},{"key":"e_1_3_3_2_44_1","doi-asserted-by":"publisher","DOI":"10.1145\/3274345"},{"key":"e_1_3_3_2_45_1","volume-title":"Using mixed-methods sequential explanatory design: From theory to practice. Field methods 18, 1","author":"Ivankova V","year":"2006","unstructured":"Nataliya\u00a0V Ivankova, John\u00a0W Creswell, and Sheldon\u00a0L Stick. 2006. Using mixed-methods sequential explanatory design: From theory to practice. Field methods 18, 1 (2006), 3\u201320."},{"key":"e_1_3_3_2_46_1","doi-asserted-by":"publisher","DOI":"10.1145\/3313831.3376860"},{"key":"e_1_3_3_2_47_1","doi-asserted-by":"publisher","DOI":"10.1177\/20539517211013569"},{"key":"e_1_3_3_2_48_1","doi-asserted-by":"publisher","DOI":"10.1145\/3351095.3372829"},{"key":"e_1_3_3_2_49_1","doi-asserted-by":"publisher","DOI":"10.1038\/s42256-019-0088-2"},{"key":"e_1_3_3_2_50_1","unstructured":"Sonam Joshi. 2019. How artificial intelligence is creating jobs in India not just stealing them | India News - Times of India. https:\/\/timesofindia.indiatimes.com\/india\/how-artificial-intelligence-is-creating-jobs-in-india-not-just-stealing-them\/articleshow\/71030863.cms. Accessed on 08\/24\/2021."},{"key":"e_1_3_3_2_51_1","doi-asserted-by":"publisher","DOI":"10.1145\/2818048.2820016"},{"key":"e_1_3_3_2_52_1","doi-asserted-by":"publisher","DOI":"10.5555\/2908698.2908717"},{"key":"e_1_3_3_2_53_1","doi-asserted-by":"publisher","DOI":"10.1145\/3411764.3445261"},{"key":"e_1_3_3_2_54_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1"},{"key":"e_1_3_3_2_55_1","unstructured":"Sasha Luccioni Yacine Jernite and Meg Mitchell. 2021. Introducing the Data Measurements Tool: an Interactive Tool for Looking at Datasets. https:\/\/huggingface.co\/blog\/data-measurements-tool. (Accessed on 09\/15\/2022)."},{"key":"e_1_3_3_2_56_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1"},{"key":"e_1_3_3_2_57_1","doi-asserted-by":"publisher","DOI":"10.14778\/2535568.2448944"},{"key":"e_1_3_3_2_58_1","doi-asserted-by":"publisher","DOI":"10.1145\/2531602.2531663"},{"key":"e_1_3_3_2_59_1","doi-asserted-by":"publisher","DOI":"10.1086\/426800"},{"key":"e_1_3_3_2_60_1","unstructured":"Karishma Mehrotra. 2022. Human Touch. https:\/\/fiftytwo.in\/story\/human-touch\/. (Accessed on 09\/15\/2022)."},{"key":"e_1_3_3_2_61_1","doi-asserted-by":"publisher","DOI":"10.1145\/3555561"},{"key":"e_1_3_3_2_62_1","doi-asserted-by":"crossref","unstructured":"Milagros Miceli Julian Posada and Tianling Yang. 2022. Studying up machine learning data: Why talk about bias when we mean power?Proceedings of the ACM on Human-Computer Interaction 6 GROUP(2022) 1\u201314.","DOI":"10.1145\/3492853"},{"key":"e_1_3_3_2_63_1","doi-asserted-by":"publisher","DOI":"10.1145\/3415186"},{"key":"e_1_3_3_2_64_1","doi-asserted-by":"publisher","DOI":"10.1145\/3442188.3445880"},{"key":"e_1_3_3_2_65_1","volume-title":"Deliberative democracy or agonistic pluralism?Social research","author":"Mouffe Chantal","year":"1999","unstructured":"Chantal Mouffe. 1999. Deliberative democracy or agonistic pluralism?Social research (1999), 745\u2013758."},{"key":"e_1_3_3_2_66_1","doi-asserted-by":"publisher","DOI":"10.1145\/3290605.3300356"},{"key":"e_1_3_3_2_67_1","doi-asserted-by":"publisher","DOI":"10.1145\/3411764.3445402"},{"key":"e_1_3_3_2_68_1","unstructured":"Laura Nader. 1972. Up the anthropologist: Perspectives gained from studying up."},{"key":"e_1_3_3_2_69_1","doi-asserted-by":"publisher","DOI":"10.1038\/s41562-017-0273-4"},{"key":"e_1_3_3_2_70_1","doi-asserted-by":"publisher","DOI":"10.1145\/1958824.1958859"},{"key":"e_1_3_3_2_71_1","unstructured":"Google PAIR. 2022. Know Your Data. https:\/\/knowyourdata.withgoogle.com. (Accessed on 09\/15\/2022)."},{"key":"e_1_3_3_2_72_1","doi-asserted-by":"publisher","DOI":"10.1145\/3287560.3287567"},{"key":"e_1_3_3_2_73_1","doi-asserted-by":"publisher","DOI":"10.1145\/2998181.2998331"},{"key":"e_1_3_3_2_74_1","doi-asserted-by":"publisher","DOI":"10.1145\/3274405"},{"key":"e_1_3_3_2_75_1","doi-asserted-by":"publisher","DOI":"10.1145\/2702123.2702298"},{"key":"e_1_3_3_2_76_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1"},{"key":"e_1_3_3_2_77_1","doi-asserted-by":"publisher","DOI":"10.1145\/3531146.3533231"},{"key":"e_1_3_3_2_78_1","volume-title":"Fieldwork for design: theory and practice","author":"Randall David","unstructured":"David Randall, Richard Harper, and Mark Rouncefield. 2007. Fieldwork for design: theory and practice. Springer Science & Business Media, London."},{"key":"e_1_3_3_2_79_1","doi-asserted-by":"publisher","DOI":"10.1145\/3531146.3533239"},{"key":"e_1_3_3_2_80_1","doi-asserted-by":"publisher","DOI":"10.1145\/3411764.3445518"},{"key":"e_1_3_3_2_81_1","doi-asserted-by":"publisher","DOI":"10.1145\/3491102.3517578"},{"key":"e_1_3_3_2_82_1","doi-asserted-by":"publisher","DOI":"10.1145\/3359178"},{"key":"e_1_3_3_2_83_1","doi-asserted-by":"publisher","DOI":"10.1145\/3313831.3376290"},{"key":"e_1_3_3_2_84_1","doi-asserted-by":"publisher","DOI":"10.1145\/3274423"},{"key":"e_1_3_3_2_85_1","volume-title":"Representation: Assessing Geodiversity Issues in Open Data Sets for the Developing World.","author":"Shankar Shreya","year":"2017","unstructured":"Shreya Shankar, Yoni Halpern, Eric Breck, James Atwood, Jimbo Wilson, and D. Sculley. 2017. No Classification without Representation: Assessing Geodiversity Issues in Open Data Sets for the Developing World."},{"key":"e_1_3_3_2_86_1","doi-asserted-by":"publisher","DOI":"10.1145\/1401890.1401965"},{"key":"e_1_3_3_2_87_1","doi-asserted-by":"publisher","DOI":"10.1145\/2488388.2488489"},{"key":"e_1_3_3_2_88_1","volume-title":"Proceedings of the 2008 Conference on Empirical Methods in Natural Language Processing. Association for Computational Linguistics","author":"Snow Rion","year":"2008","unstructured":"Rion Snow, Brendan O\u2019Connor, Daniel Jurafsky, and Andrew Ng. 2008. Cheap and Fast \u2013 But is it Good? Evaluating Non-Expert Annotations for Natural Language Tasks. In Proceedings of the 2008 Conference on Empirical Methods in Natural Language Processing. Association for Computational Linguistics, Honolulu, Hawaii, 254\u2013263. https:\/\/aclanthology.org\/D08-1027"},{"key":"e_1_3_3_2_89_1","doi-asserted-by":"publisher","DOI":"10.1145\/3465416.3483305"},{"key":"e_1_3_3_2_90_1","doi-asserted-by":"publisher","DOI":"10.1145\/2702123.2702558"},{"key":"e_1_3_3_2_91_1","doi-asserted-by":"publisher","DOI":"10.1177\/0162243911423843"},{"key":"e_1_3_3_2_92_1","doi-asserted-by":"publisher","DOI":"10.1145\/3441852.3471225"},{"key":"e_1_3_3_2_93_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1"},{"key":"e_1_3_3_2_94_1","doi-asserted-by":"publisher","DOI":"10.1145\/1958824.1958906"},{"key":"e_1_3_3_2_95_1","doi-asserted-by":"publisher","DOI":"10.1145\/3491102.3502121"},{"key":"e_1_3_3_2_96_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1"},{"key":"e_1_3_3_2_97_1","doi-asserted-by":"publisher","DOI":"10.1162\/tacl_a_00240"},{"key":"e_1_3_3_2_98_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPRW.2010.5543189"},{"key":"e_1_3_3_2_99_1","doi-asserted-by":"publisher","DOI":"10.1017\/CBO9780511803932"},{"key":"e_1_3_3_2_100_1","unstructured":"Sarah\u00a0Myers West Meredith Whittaker and Kate Crawford. 2019. Discriminating systems."},{"key":"e_1_3_3_2_101_1","doi-asserted-by":"publisher","DOI":"10.1145\/3173574.3174230"},{"key":"e_1_3_3_2_102_1","doi-asserted-by":"publisher","DOI":"10.1145\/3395046"}],"event":{"name":"CHI '23: CHI Conference on Human Factors in Computing Systems","location":"Hamburg Germany","acronym":"CHI '23","sponsor":["SIGCHI ACM Special Interest Group on Computer-Human Interaction"]},"container-title":["Proceedings of the 2023 CHI Conference on Human Factors in Computing Systems"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3544548.3580645","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3544548.3580645","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T19:02:55Z","timestamp":1750186975000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3544548.3580645"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023,4,19]]},"references-count":102,"alternative-id":["10.1145\/3544548.3580645","10.1145\/3544548"],"URL":"https:\/\/doi.org\/10.1145\/3544548.3580645","relation":{},"subject":[],"published":{"date-parts":[[2023,4,19]]},"assertion":[{"value":"2023-04-19","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}