{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,13]],"date-time":"2026-04-13T21:39:58Z","timestamp":1776116398816,"version":"3.50.1"},"publisher-location":"New York, NY, USA","reference-count":59,"publisher":"ACM","license":[{"start":{"date-parts":[[2022,6,20]],"date-time":"2022-06-20T00:00:00Z","timestamp":1655683200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2022,6,21]]},"DOI":"10.1145\/3531146.3534647","type":"proceedings-article","created":{"date-parts":[[2022,6,20]],"date-time":"2022-06-20T10:27:10Z","timestamp":1655720830000},"page":"2342-2351","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":53,"title":["CrowdWorkSheets: Accounting for Individual and Collective Identities Underlying Crowdsourced Dataset Annotation"],"prefix":"10.1145","author":[{"given":"Mark","family":"D\u00edaz","sequence":"first","affiliation":[{"name":"Google, USA"}]},{"given":"Ian","family":"Kivlichan","sequence":"additional","affiliation":[{"name":"Jigsaw, Google, USA"}]},{"given":"Rachel","family":"Rosen","sequence":"additional","affiliation":[{"name":"Jigsaw, Google, USA"}]},{"given":"Dylan","family":"Baker","sequence":"additional","affiliation":[{"name":"Google, USA"}]},{"given":"Razvan","family":"Amironesei","sequence":"additional","affiliation":[{"name":"Google, USA"}]},{"given":"Vinodkumar","family":"Prabhakaran","sequence":"additional","affiliation":[{"name":"Google, USA"}]},{"given":"Remi","family":"Denton","sequence":"additional","affiliation":[{"name":"Google, USA"}]}],"member":"320","published-online":{"date-parts":[[2022,6,20]]},"reference":[{"key":"e_1_3_2_1_1_1","unstructured":"2008. Turkopticon. https:\/\/turkopticon.net\/. Accessed: 2021-07-21."},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.1145\/3025453.3025974"},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1145\/3308560.3317083"},{"key":"e_1_3_2_1_4_1","volume-title":"Crowd truth: Harnessing disagreement in crowdsourcing a relation extraction gold standard. WebSci2013. ACM","author":"Aroyo Lora","year":"2013","unstructured":"Lora Aroyo and Chris Welty. 2013. Crowd truth: Harnessing disagreement in crowdsourcing a relation extraction gold standard. WebSci2013. ACM (2013)."},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1609\/aimag.v36i1.2564"},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"publisher","DOI":"10.1162\/tacl_a_00041"},{"key":"e_1_3_2_1_7_1","volume-title":"Income security in the on-demand economy: Findings and policy lessons from a survey of crowdworkers. Comp. Lab. L. & Pol\u2019y J. 37","author":"Berg Janine","year":"2015","unstructured":"Janine Berg. 2015. Income security in the on-demand economy: Findings and policy lessons from a survey of crowdworkers. Comp. Lab. L. & Pol\u2019y J. 37 (2015), 543."},{"key":"e_1_3_2_1_8_1","first-page":"47","article-title":"The role of crowdsourcing for better governance in international development. Praxis","volume":"27","author":"Bott Maja","year":"2012","unstructured":"Maja Bott and Gregor Young. 2012. The role of crowdsourcing for better governance in international development. Praxis: The Fletcher Journal of Human Security 27, 1 (2012), 47\u201370.","journal-title":"The Fletcher Journal of Human Security"},{"key":"e_1_3_2_1_9_1","unstructured":"Kasia\u00a0S. Chmielinski Sarah Newman Matt Taylor Josh Joseph Kemi Thomas Jessica Yurkofsky and Yue\u00a0Chelsea Qiu. 2020. The Dataset Nutrition Label (2nd Gen): Leveraging Context to Mitigate Harms in Artificial Intelligence. (2020). http:\/\/securedata.lol\/ NeurIPS Workshop on Dataset Curation and Security."},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"crossref","unstructured":"J. Deng W. Dong R. Socher L.-J. Li K. Li and L. Fei-Fei. 2009. ImageNet: A Large-Scale Hierarchical Image Database. In CVPR09.","DOI":"10.1109\/CVPR.2009.5206848"},{"key":"e_1_3_2_1_11_1","volume-title":"Bringing the People Back In: Contesting Benchmark Machine Learning Datasets. ICML Workshop on Participatory Approaches to Machine Learning","author":"Denton Remi","year":"2020","unstructured":"Remi Denton, A. Hanna, Razvan Amironesei, Andrew Smart, Hilary Nicole, and M. Scheuerman. 2020. Bringing the People Back In: Contesting Benchmark Machine Learning Datasets. ICML Workshop on Participatory Approaches to Machine Learning (2020)."},{"key":"e_1_3_2_1_12_1","unstructured":"Lucas Dixon. 2018. Annotation instructions for Toxicity with sub-attributes. https:\/\/github.com\/conversationai\/conversationai.github.io\/blob\/master\/crowdsourcing_annotation_schemes\/toxicity_with_subattributes.md. Accessed: 2021-01-19."},{"key":"e_1_3_2_1_13_1","volume-title":"Disturbing Content. Medium OneZero","author":"Emerson Sarah","year":"2019","unstructured":"Sarah Emerson. 2019. \u2018I Can Hear the Suffering\u2019: Rev Exposes Freelance Transcribers to Violent, Disturbing Content. Medium OneZero (2019)."},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.1177\/2378023118820157"},{"key":"e_1_3_2_1_15_1","volume-title":"Measuring nominal scale agreement among many raters. Psychological bulletin 76, 5","author":"Fleiss L","year":"1971","unstructured":"Joseph\u00a0L Fleiss. 1971. Measuring nominal scale agreement among many raters. Psychological bulletin 76, 5 (1971), 378."},{"key":"e_1_3_2_1_16_1","volume-title":"a history of the factory and the making of the modern world","author":"Freeman Joshua\u00a0Benjamin","unstructured":"Joshua\u00a0Benjamin Freeman. 2018. Behemoth : a history of the factory and the making of the modern world. W.W. Norton & Company, Inc.,."},{"key":"e_1_3_2_1_17_1","unstructured":"Timnit Gebru Jamie Morgenstern Briana Vecchione Jennifer\u00a0Wortman Vaughan Hanna Wallach Hal Daum\u00e9\u00a0III and Kate Crawford. 2018. Datasheets for datasets. arXiv preprint arXiv:1803.09010(2018)."},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1145\/3351095.3372862"},{"key":"e_1_3_2_1_19_1","volume-title":"Community building on crowdwork platforms: Autonomy and control of online workers?Competition & Change 25, 2","author":"Gerber Christine","year":"2021","unstructured":"Christine Gerber. 2021. Community building on crowdwork platforms: Autonomy and control of online workers?Competition & Change 25, 2 (2021), 190\u2013211."},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2021.wnut-1.35"},{"key":"e_1_3_2_1_21_1","unstructured":"Amy\u00a0K Glasmeier. 2020. Living Wage Calculator. (2020). livingwage.mit.edu"},{"key":"e_1_3_2_1_22_1","volume-title":"Ghost work: How to stop Silicon Valley from building a new global underclass","author":"Gray L","unstructured":"Mary\u00a0L Gray and Siddharth Suri. 2019. Ghost work: How to stop Silicon Valley from building a new global underclass. Eamon Dolan Books."},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.1145\/3449202"},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"publisher","DOI":"10.1145\/3173574.3174023"},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.1177\/1461444820958725"},{"key":"e_1_3_2_1_26_1","unstructured":"Sarah Holland Ahmed Hosny Sarah Newman Joshua Joseph and Kasia Chmielinski. 2018. The dataset nutrition label: A framework to drive higher data quality standards. arXiv preprint arXiv:1805.03677(2018)."},{"key":"e_1_3_2_1_27_1","volume-title":"Proceedings of the Conference on Fairness, Accountability, and Transparency.","author":"Hutchinson Ben","year":"2021","unstructured":"Ben Hutchinson, Andrew Smart, Alex Hanna, Remi Denton, Christina Greer, Oddur Kjartansson, Parker Barnes, and Margaret Mitchell. 2021. Towards Accountability for Machine Learning Datasets: Practices from Software Engineering and Infrastructure. In Proceedings of the Conference on Fairness, Accountability, and Transparency."},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"publisher","DOI":"10.1145\/3442188.3445918"},{"key":"e_1_3_2_1_29_1","unstructured":"Panagiotis\u00a0G Ipeirotis. 2010. Demographics of mechanical turk. (2010)."},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"publisher","DOI":"10.1145\/2470654.2470742"},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"publisher","DOI":"10.1609\/hcomp.v6i1.13327"},{"key":"e_1_3_2_1_32_1","volume-title":"AMCIS","author":"Kocsis David","year":"2016","unstructured":"David Kocsis and Gert\u00a0Jan De\u00a0Vreede. 2016. Towards a taxonomy of ethical considerations in crowdsourcing(22nd Americas Conference on Information Systems: Surfing the IT Innovation Wave, AMCIS 2016)."},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"publisher","DOI":"10.1145\/2531602.2531663"},{"key":"e_1_3_2_1_34_1","doi-asserted-by":"publisher","DOI":"10.1145\/3415186"},{"key":"e_1_3_2_1_35_1","volume-title":"Human-in-the-Loop Machine Learning: Active learning and annotation for human-centered AI","author":"Monarch Robert\u00a0Munro","unstructured":"Robert\u00a0Munro Monarch. 2021. Human-in-the-Loop Machine Learning: Active learning and annotation for human-centered AI. Simon and Schuster."},{"key":"e_1_3_2_1_36_1","volume-title":"Proceedings of the 49th Annual Meeting of the Association for Computational Linguistics: Human Language Technologies. Association for Computational Linguistics","author":"Ovesdotter\u00a0Alm Cecilia","year":"2011","unstructured":"Cecilia Ovesdotter\u00a0Alm. 2011. Subjective Natural Language Problems: Motivations, Applications, Characterizations, and Implications. In Proceedings of the 49th Annual Meeting of the Association for Computational Linguistics: Human Language Technologies. Association for Computational Linguistics, Portland, Oregon, USA, 107\u2013112. https:\/\/aclanthology.org\/P11-2019"},{"key":"e_1_3_2_1_37_1","unstructured":"Desmond\u00a0Upton Patton Philipp Blandfort William\u00a0R Frey Michael\u00a0B Gaskell and Svebor Karaman. 2019. Annotating twitter data from vulnerable populations: Evaluating disagreement between domain experts and graduate student annotators."},{"key":"e_1_3_2_1_38_1","unstructured":"Lisa Posch Arnim Bleier Fabian Fl\u00f6ck and Markus Strohmaier. 2018. Characterizing the global crowd workforce: A cross-country comparison of crowdworker demographics. arXiv preprint arXiv:1812.05948(2018)."},{"key":"e_1_3_2_1_39_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2021.law-1.14"},{"key":"e_1_3_2_1_40_1","volume-title":"Proceedings of the 2021 ACM Conference on Fairness, Accountability, and Transparency.","author":"Pushkarna Mahima","year":"2022","unstructured":"Mahima Pushkarna, Andrew Zaldivar, and Oddur Kjartansson. 2022. Data Cards: Purposeful and Transparent Dataset Documentation for Responsible AI. In Proceedings of the 2021 ACM Conference on Fairness, Accountability, and Transparency."},{"key":"e_1_3_2_1_41_1","doi-asserted-by":"publisher","DOI":"10.1145\/1978942.1979148"},{"key":"e_1_3_2_1_42_1","doi-asserted-by":"publisher","DOI":"10.1145\/3479531"},{"key":"e_1_3_2_1_43_1","first-page":"1","article-title":"Digital refuse: Canadian garbage, commercial content moderation and the global circulation of social media\u2019s waste. Wi","volume":"10","author":"Roberts T","year":"2016","unstructured":"Sarah\u00a0T Roberts. 2016. Digital refuse: Canadian garbage, commercial content moderation and the global circulation of social media\u2019s waste. Wi: Journal of Mobile Media 10, 1 (2016), 1\u201318.","journal-title":"Journal of Mobile Media"},{"key":"e_1_3_2_1_44_1","volume-title":"Lauren Wilcox, Mahima Pushkarna, Razvan Amironesei, Jessica Schrouff, Madeleine Elish, Nyalleng Moorosi, Berk Ustun, Noah Broesti, and Katherine Heller.","author":"Rostamzadeh Negar","year":"2021","unstructured":"Negar Rostamzadeh, Subhrajit Roy, Diana Mincu, Andrew Smart, Lauren Wilcox, Mahima Pushkarna, Razvan Amironesei, Jessica Schrouff, Madeleine Elish, Nyalleng Moorosi, Berk Ustun, Noah Broesti, and Katherine Heller. 2021. Specialized Healthsheet for Healthcare Datasets. In Machine Learning for Health (ML4H)."},{"key":"e_1_3_2_1_45_1","volume-title":"Proceedings of the Ninth International Conference on Language Resources and Evaluation (LREC\u201914)","author":"Sabou Reka\u00a0Marta","unstructured":"Reka\u00a0Marta Sabou, Kalina Bontcheva, Leon Derczynski, and A. Scharl. 2014. Corpus Annotation through Crowdsourcing:Towards Best Practice Guidelines. In Proceedings of the Ninth International Conference on Language Resources and Evaluation (LREC\u201914). European Language Resources Association (ELRA), Reykjavik, Iceland."},{"key":"e_1_3_2_1_46_1","doi-asserted-by":"publisher","DOI":"10.1145\/2702123.2702508"},{"key":"e_1_3_2_1_47_1","doi-asserted-by":"publisher","DOI":"10.1145\/3442188.3445896"},{"key":"e_1_3_2_1_48_1","doi-asserted-by":"crossref","unstructured":"Morgan Klaus Scheuerman Remi Denton and Alex Hanna. 2021. Do Datasets Have Politics? Disciplinary Values in Computer Vision Dataset Development. Computer Supported Cooperative Work (CSCW)(2021).","DOI":"10.1145\/3476058"},{"key":"e_1_3_2_1_49_1","doi-asserted-by":"publisher","DOI":"10.1111\/isj.12227"},{"key":"e_1_3_2_1_50_1","doi-asserted-by":"publisher","DOI":"10.1177\/2329496516686620"},{"key":"e_1_3_2_1_51_1","doi-asserted-by":"publisher","DOI":"10.1145\/3287560.3287598"},{"key":"e_1_3_2_1_52_1","unstructured":"Alana Semuels. 2018. The internet is enabling a new kind of poorly paid hell. The Atlantic 23(2018)."},{"key":"e_1_3_2_1_53_1","doi-asserted-by":"publisher","DOI":"10.1145\/2675133.2675285"},{"key":"e_1_3_2_1_54_1","doi-asserted-by":"crossref","unstructured":"Boaz Shmueli Jan Fell Soumya Ray and Lun-Wei Ku. 2021. Beyond Fair Pay: Ethical Implications of NLP Crowdsourcing. arXiv preprint arXiv:2104.10097(2021).","DOI":"10.18653\/v1\/2021.naacl-main.295"},{"key":"e_1_3_2_1_55_1","volume-title":"Proceedings of Neural Information Processing Systems (NeurIPS), Datasets & Benchmarks Track. https:\/\/openreview.net\/pdf?id=K7ke_GZ_6N","author":"Srinivasan Ramya Malur","year":"2021","unstructured":"Ramya Malur Srinivasan, Remi Denton, Jordan Jennifer Famularo, Negar Rostamzadeh, Fernando Diaz, and Beth Coleman. 2021. Artsheets for Art Datasets. In Proceedings of Neural Information Processing Systems (NeurIPS), Datasets & Benchmarks Track. https:\/\/openreview.net\/pdf?id=K7ke_GZ_6N"},{"key":"e_1_3_2_1_56_1","doi-asserted-by":"publisher","DOI":"10.1145\/3476060"},{"key":"e_1_3_2_1_57_1","volume-title":"Proceedings of the iConference(2015)","author":"Vakharia Donna","year":"2015","unstructured":"Donna Vakharia and Matthew Lease. 2015. Beyond Mechanical Turk: An analysis of paid crowd work platforms. Proceedings of the iConference(2015), 1\u201317."},{"key":"e_1_3_2_1_58_1","volume-title":"The state of online harassment","author":"Vogels Emily","year":"2021","unstructured":"Emily Vogels. 2021. The state of online harassment. Pew Research Center (2021)."},{"key":"e_1_3_2_1_59_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1"}],"event":{"name":"FAccT '22: 2022 ACM Conference on Fairness, Accountability, and Transparency","location":"Seoul Republic of Korea","acronym":"FAccT '22","sponsor":["ACM Association for Computing Machinery"]},"container-title":["2022 ACM Conference on Fairness Accountability and Transparency"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3531146.3534647","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3531146.3534647","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,8,19]],"date-time":"2025-08-19T12:35:23Z","timestamp":1755606923000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3531146.3534647"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2022,6,20]]},"references-count":59,"alternative-id":["10.1145\/3531146.3534647","10.1145\/3531146"],"URL":"https:\/\/doi.org\/10.1145\/3531146.3534647","relation":{},"subject":[],"published":{"date-parts":[[2022,6,20]]},"assertion":[{"value":"2022-06-20","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}