{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,2,17]],"date-time":"2026-02-17T02:18:31Z","timestamp":1771294711981,"version":"3.50.1"},"reference-count":31,"publisher":"Association for Computing Machinery (ACM)","issue":"5","license":[{"start":{"date-parts":[[2024,5,1]],"date-time":"2024-05-01T00:00:00Z","timestamp":1714521600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":["Commun. ACM"],"published-print":{"date-parts":[[2024,5]]},"abstract":"<jats:p>What ethical framework should govern the interaction of ML researchers and crowdworkers?<\/jats:p>","DOI":"10.1145\/3641858","type":"journal-article","created":{"date-parts":[[2024,3,25]],"date-time":"2024-03-25T09:15:05Z","timestamp":1711358105000},"page":"52-59","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":3,"title":["Resolving the Human-Subjects Status of ML's Crowdworkers"],"prefix":"10.1145","volume":"67","author":[{"given":"Divyansh","family":"Kaushik","sequence":"first","affiliation":[{"name":"Federation of American Scientists, Washington, D.C., USA"}]},{"given":"Zachary C.","family":"Lipton","sequence":"additional","affiliation":[{"name":"Carnegie Mellon University, Pittsburgh, PA, USA"}]},{"given":"Alex John","family":"London","sequence":"additional","affiliation":[{"name":"Carnegie Mellon University (CMU), Pittsburgh, PA, USA"}]}],"member":"320","published-online":{"date-parts":[[2024,5]]},"reference":[{"key":"e_1_2_1_1_1","first-page":"1","article-title":"MasakhaNER: Named entity recognition for African languages","volume":"9","author":"Adelani D.F.","year":"2021","unstructured":"Adelani, D.F. et al. MasakhaNER: Named entity recognition for African languages. Trans. ACL 9, 1 (2021), 116--131; https:\/\/aclanthology.org\/2021.tacl-1.66.pdf.","journal-title":"Trans. ACL"},{"key":"e_1_2_1_2_1","unstructured":"Birmingham-Southern College. Do I Need IRB Approval? https:\/\/bit.ly\/47vJTAa."},{"key":"e_1_2_1_3_1","volume-title":"Proceedings of the 58th Annual Meeting of the ACL 4, (2020","author":"DeYoung J.","year":"2020","unstructured":"DeYoung, J. et al. ERASER: A benchmark to evaluate rationalized NLP models. In Proceedings of the 58th Annual Meeting of the ACL 4, (2020), 443--458; https:\/\/aclanthology.org\/2020.acl-main.408\/."},{"key":"e_1_2_1_4_1","volume-title":"Proceedings of the 2019 Conf. Empirical Methods in Natural Language Processing and the 9th Intern. Joint Conf. Natural Language Processing 2","author":"Dodge J.","year":"2019","unstructured":"Dodge, J. et al. Show your work: Improved reporting of experimental results. In Proceedings of the 2019 Conf. Empirical Methods in Natural Language Processing and the 9th Intern. Joint Conf. Natural Language Processing 2, (2019), 185--194; https:\/\/aclanthology.org\/D19-1224\/."},{"key":"e_1_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1162\/COLI_a_00057"},{"key":"e_1_2_1_6_1","volume-title":"Proceedings of the 2019 Conf. Empirical Methods in Natural Language Processing and the 9th Intern. Joint Conf. Natural Language Processing 1, 161--166; https:\/\/aclanthology.org\/D19-1107","author":"Geva M.","unstructured":"Geva, M., Goldberg, Y., and Berant, J. Are we modeling the task or the annotator? An investigation of annotator bias in natural language understanding datasets. In Proceedings of the 2019 Conf. Empirical Methods in Natural Language Processing and the 9th Intern. Joint Conf. Natural Language Processing 1, 161--166; https:\/\/aclanthology.org\/D19-1107.pdf."},{"key":"e_1_2_1_7_1","volume-title":"Proceedings of the 2020 Conf. Empirical Methods in Natural Language Processing 8, 142--152; https:\/\/aclanthology.org\/2020","author":"Hayati S.A.","unstructured":"Hayati, S.A. et al. INSPIRED: Toward sociable recommendation dialog systems. In Proceedings of the 2020 Conf. Empirical Methods in Natural Language Processing 8, 142--152; https:\/\/aclanthology.org\/2020.emnlp-main.654\/."},{"key":"e_1_2_1_8_1","volume-title":"Proceedings of the 52nd Annual Meeting of the ACL Vol 2: Short Papers, 2014","author":"Hovy D.","year":"2062","unstructured":"Hovy, D., Plank, B., and S\u00f8gaard, A. Experiments with crowdsourced re-annotation of a POS tagging data set. In Proceedings of the 52nd Annual Meeting of the ACL Vol 2: Short Papers, 2014, 377--382; https:\/\/aclanthology.org\/P14-2062.pdf."},{"key":"e_1_2_1_9_1","volume-title":"Proceedings of the 54th Annual Meeting of the ACL Vol 2: Short Papers, 2016","author":"Hovy D.","year":"2096","unstructured":"Hovy, D. and Spruit, S.L. The social impact of natural language processing. In Proceedings of the 54th Annual Meeting of the ACL Vol 2: Short Papers, 2016, 591--598; https:\/\/aclanthology.org\/P16-2096\/."},{"key":"e_1_2_1_10_1","unstructured":"Ipeirotis P. Mechanical Turk Human Subjects and IRBs 2009; https:\/\/bit.ly\/41T2m8w."},{"key":"e_1_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2021.acl-long.517"},{"key":"e_1_2_1_12_1","volume-title":"Proceedings of the 2020 Conf. Empirical Methods in Natural Language Processing, 163--170; https:\/\/aclanthology.org\/2020","author":"Khashabi D.","unstructured":"Khashabi, D., Khot, T., and Sabharwal, A. More bang for your buck: natural perturbation for robust question answering. In Proceedings of the 2020 Conf. Empirical Methods in Natural Language Processing, 163--170; https:\/\/aclanthology.org\/2020.emnlp-main.12.pdf."},{"key":"e_1_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1561\/0600000071"},{"key":"e_1_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/W17-1604"},{"key":"e_1_2_1_15_1","volume-title":"For the Common Good: Philosophical Foundations of Research Ethics","author":"London A.J.","year":"2021","unstructured":"London, A.J. For the Common Good: Philosophical Foundations of Research Ethics. Oxford University Press, Oxford, U.K., 2021."},{"key":"e_1_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.1002\/hast.1034"},{"key":"e_1_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.1002\/eahr.500071"},{"key":"e_1_2_1_18_1","unstructured":"Loyola University. Do I Need IRB Review? https:\/\/www.luc.edu\/irb\/gettingstarted\/isirbreviewrequired\/."},{"key":"e_1_2_1_19_1","volume-title":"Proceedings of the 2018 Conf. Empirical Methods in Natural Language Processing 2, 381--391; https:\/\/aclanthology.org\/D18-1260\/.","author":"Mihaylov T.","unstructured":"Mihaylov, T., Clark, P., Khot, T., and Sabharwal, A. Can a suit of armor conduct electricity? A new dataset for open book question answering. In Proceedings of the 2018 Conf. Empirical Methods in Natural Language Processing 2, 381--391; https:\/\/aclanthology.org\/D18-1260\/."},{"key":"e_1_2_1_20_1","first-page":"195","volume-title":"Proceedings of the 2020 Conf. Empirical Methods in Natural Language Processing: Findings 2, 144--160; https:\/\/aclanthology.org\/2020","author":"Nekoto W.","unstructured":"Nekoto, W. et al. Participatory research for low-resourced machine translation: A case study in African languages. In Proceedings of the 2020 Conf. Empirical Methods in Natural Language Processing: Findings 2, 144--160; https:\/\/aclanthology.org\/2020.findings-emnlp.195.pdf."},{"key":"e_1_2_1_21_1","volume-title":"Proceedings of the 2020 Empirical Methods in Natural Language Processing Systems Demonstrations, 127--134; https:\/\/aclanthology.org\/2020","author":"Ning Q.","unstructured":"Ning, Q. et al. Easy, reproducible, and quality-controlled data collection with CROWDAQ. In Proceedings of the 2020 Empirical Methods in Natural Language Processing Systems Demonstrations, 127--134; https:\/\/aclanthology.org\/2020.emnlp-demos.17.pdf."},{"key":"e_1_2_1_22_1","volume-title":"Proceedings of the 2015 Conf. Empirical Methods in Natural Language Processing 1, 120--125; https:\/\/aclanthology.org\/D15-1133","author":"P\u00e9rez-Rosas V.","unstructured":"P\u00e9rez-Rosas, V. and Mihalcea, R. Experiments in open domain deception detection. In Proceedings of the 2015 Conf. Empirical Methods in Natural Language Processing 1, 120--125; https:\/\/aclanthology.org\/D15-1133.pdf."},{"key":"e_1_2_1_23_1","volume-title":"Proceedings of the 2021 Conf. Of the North American Chapter of Theacl: Human Language Technologies 3, 758--769; https:\/\/aclanthology.org\/2021","author":"Shmueli B.","unstructured":"Shmueli, B., Fell, J., Ray, S., and Ku, L.-K. Beyond fair pay: Ethical implications of NLP crowdsourcing. In Proceedings of the 2021 Conf. Of the North American Chapter of Theacl: Human Language Technologies 3, 758--769; https:\/\/aclanthology.org\/2021.naaclmain.295.pdf."},{"key":"e_1_2_1_24_1","volume-title":"Proceedings of the 2019 Conf. North American Chapter of the ACL: Human Language Technologies, Vol 1 (Long and Short Papers) 4, 149--158; https:\/\/aclanthology.org\/N19-1421","author":"Talmor A.","unstructured":"Talmor, A., Herzig, J., Lourie, N., and Berant, J. CommonsenseQA: A question answering challenge targeting commonsense knowledge. In Proceedings of the 2019 Conf. North American Chapter of the ACL: Human Language Technologies, Vol 1 (Long and Short Papers) 4, 149--158; https:\/\/aclanthology.org\/N19-1421.pdf."},{"key":"e_1_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.5555\/3122009.3242050"},{"key":"e_1_2_1_26_1","volume-title":"Proceedings of the 2019 AAAI Conf. On Human Computation and Crowdsourcing 7, 197--206; https:\/\/ojs.aaai.org\/index.php\/HCOMP\/article\/view\/5283","author":"Whiting M.E.","unstructured":"Whiting, M.E., Hugh, G., and Bernstein, M.S. Fair work: Crowd work minimum wage with one line of code. In Proceedings of the 2019 AAAI Conf. On Human Computation and Crowdsourcing 7, 197--206; https:\/\/ojs.aaai.org\/index.php\/HCOMP\/article\/view\/5283."},{"key":"e_1_2_1_27_1","unstructured":"Whittier College. Do I Need IRB Review? https:\/\/www.whittier.edu\/academics\/researchethics\/irb\/need."},{"key":"e_1_2_1_28_1","volume-title":"Proceedings of the 2018 Conf. Of the North American Chapter of the ACL--- Human Language Technologies 1 (Long Papers), 1,112--1,122; https:\/\/aclanthology.org\/N18-1101","author":"Williams A.","unstructured":"Williams, A., Nangia, N., and Bowman, S. A broad-coverage challenge corpus for sentence understanding through inference. In Proceedings of the 2018 Conf. Of the North American Chapter of the ACL--- Human Language Technologies 1 (Long Papers), 1,112--1,122; https:\/\/aclanthology.org\/N18-1101.pdf."},{"key":"e_1_2_1_29_1","volume-title":"et al. Recipes for Safety in Open-Domain Chatbots","author":"Xu J.","year":"2020","unstructured":"Xu, J. et al. Recipes for Safety in Open-Domain Chatbots, 2020; https:\/\/arxiv.org\/abs\/2010.07079."},{"key":"e_1_2_1_30_1","volume-title":"Proceedings of the 2007 Conf. North American Chapter of the ACL---Human Language Technologies, 260--267; https:\/\/aclanthology.org\/N07-1033","author":"Zaidan O.","unstructured":"Zaidan, O., Eisner, J., and Piatko, C. Using \"Annotator Rationales\" to improve machine learning for text categorization. In Proceedings of the 2007 Conf. North American Chapter of the ACL---Human Language Technologies, 260--267; https:\/\/aclanthology.org\/N07-1033."},{"key":"e_1_2_1_31_1","volume-title":"Proceedings of the 59th Annual Meeting of the ACL and the 11th Intern. Joint Conf. Natural Language Processing 1: Long Papers, 3,214--3,225; https:\/\/aclanthology.org\/2021","author":"Zhang D.","unstructured":"Zhang, D. et al. MultiMET: A multimodal dataset for metaphor understanding. In Proceedings of the 59th Annual Meeting of the ACL and the 11th Intern. Joint Conf. Natural Language Processing 1: Long Papers, 3,214--3,225; https:\/\/aclanthology.org\/2021.acl-long.249.pdf."}],"container-title":["Communications of the ACM"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3641858","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3641858","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,8,23]],"date-time":"2025-08-23T18:37:15Z","timestamp":1755974235000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3641858"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,5]]},"references-count":31,"journal-issue":{"issue":"5","published-print":{"date-parts":[[2024,5]]}},"alternative-id":["10.1145\/3641858"],"URL":"https:\/\/doi.org\/10.1145\/3641858","relation":{},"ISSN":["0001-0782","1557-7317"],"issn-type":[{"value":"0001-0782","type":"print"},{"value":"1557-7317","type":"electronic"}],"subject":[],"published":{"date-parts":[[2024,5]]},"assertion":[{"value":"2024-05-01","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}