{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,10,15]],"date-time":"2025-10-15T10:35:32Z","timestamp":1760524532045,"version":"3.41.0"},"publisher-location":"New York, NY, USA","reference-count":70,"publisher":"ACM","license":[{"start":{"date-parts":[[2023,8,8]],"date-time":"2023-08-08T00:00:00Z","timestamp":1691452800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by-nc\/4.0\/"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2023,8,8]]},"DOI":"10.1145\/3600211.3604655","type":"proceedings-article","created":{"date-parts":[[2023,8,29]],"date-time":"2023-08-29T18:41:37Z","timestamp":1693334497000},"page":"834-844","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":1,"title":["How do you feel? Measuring User-Perceived Value for Rejecting Machine Decisions in Hate Speech Detection"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-3164-5592","authenticated-orcid":false,"given":"Philippe","family":"Lammerts","sequence":"first","affiliation":[{"name":"Delft University of Technology, Netherlands"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-0139-1061","authenticated-orcid":false,"given":"Philip","family":"Lippmann","sequence":"additional","affiliation":[{"name":"Delft University of Technology, Netherlands"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-8016-7534","authenticated-orcid":false,"given":"Yen-Chia","family":"Hsu","sequence":"additional","affiliation":[{"name":"University of Amsterdam, Netherlands"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-7591-9562","authenticated-orcid":false,"given":"Fabio","family":"Casati","sequence":"additional","affiliation":[{"name":"ServiceNow, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-0350-0313","authenticated-orcid":false,"given":"Jie","family":"Yang","sequence":"additional","affiliation":[{"name":"Delft University of Technology, USA"}]}],"member":"320","published-online":{"date-parts":[[2023,8,29]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-76941-7_11"},{"key":"e_1_3_2_1_2_1","volume-title":"Likert scales and data analyses. Quality progress 40, 7","author":"Allen I\u00a0Elaine","year":"2007","unstructured":"I\u00a0Elaine Allen and Christopher\u00a0A Seaman. 2007. Likert scales and data analyses. Quality progress 40, 7 (2007), 64\u201365."},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1145\/3331184.3331262"},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.1145\/3479158"},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"crossref","unstructured":"Emilio Balda Arash Behboodi and Rudolf Mathar. 2020. Adversarial Examples in Deep Neural Networks: An Overview. In Deep Learning: Algorithms and Applications. 31\u201365.","DOI":"10.1007\/978-3-030-31760-7_2"},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v35i13.17359"},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"publisher","DOI":"10.2307\/416793"},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/S19-2007"},{"key":"e_1_3_2_1_9_1","volume-title":"The values encoded in machine learning research. arXiv preprint arXiv:2106.15590","author":"Birhane Abeba","year":"2021","unstructured":"Abeba Birhane, Pratyusha Kalluri, Dallas Card, William Agnew, Ravit Dotan, and Michelle Bao. 2021. The values encoded in machine learning research. arXiv preprint arXiv:2106.15590 (2021)."},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.34068\/joe.50.02.48"},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.1007\/BF01420987"},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"crossref","unstructured":"William Callaghan Joslin Goh Michael Mohareb Andrew Lim and Edith Law. 2018. MechanicalHeart: A Human-Machine Framework for the Classification of Phonocardiograms. In CSCW\u201918 Vol.\u00a02. 28:1\u201328:17.","DOI":"10.1145\/3274297"},{"key":"e_1_3_2_1_13_1","volume-title":"On the Value of ML Models. arXiv preprint arXiv:2112.06775","author":"Casati Fabio","year":"2021","unstructured":"Fabio Casati, Pierre-Andr\u00e9 No\u00ebl, and Jie Yang. 2021. On the Value of ML Models. arXiv preprint arXiv:2112.06775 (2021)."},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.1145\/2675133.2675214"},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.1145\/2675133.2675214"},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.1109\/DSAA49011.2020.00058"},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.1007\/s11948-006-0065-0"},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1109\/5326.827457"},{"key":"e_1_3_2_1_19_1","volume-title":"Bert: Pre-training of deep bidirectional transformers for language understanding. arXiv preprint arXiv:1810.04805","author":"Devlin Jacob","year":"2018","unstructured":"Jacob Devlin, Ming-Wei Chang, Kenton Lee, and Kristina Toutanova. 2018. Bert: Pre-training of deep bidirectional transformers for language understanding. arXiv preprint arXiv:1810.04805 (2018)."},{"key":"e_1_3_2_1_20_1","first-page":"1065","article-title":"The consequences of error in criminal justice","volume":"128","author":"Epps Daniel","year":"2014","unstructured":"Daniel Epps. 2014. The consequences of error in criminal justice. Harv. L. Rev. 128 (2014), 1065.","journal-title":"Harv. L. Rev."},{"key":"e_1_3_2_1_21_1","volume-title":"The EU Code of conduct on countering illegal hate speech online. European Commission (May","author":"EU.","year":"2016","unstructured":"EU. 2016. The EU Code of conduct on countering illegal hate speech online. European Commission (May 2016). https:\/\/ec.europa.eu\/info\/policies\/justice-and-fundamental-rights\/combatting-discrimination\/racism-and-xenophobia\/eu-code-conduct-countering-illegal-hate-speech-online_en Visited on 07\/03\/2022."},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.1177\/0145721707308172"},{"key":"e_1_3_2_1_23_1","volume-title":"Principled artificial intelligence: Mapping consensus in ethical and rights-based approaches to principles for AI","author":"Fjeld Jessica","year":"2020","unstructured":"Jessica Fjeld, Nele Achten, Hannah Hilligoss, Adam Nagy, and Madhulika Srikumar. 2020. Principled artificial intelligence: Mapping consensus in ethical and rights-based approaches to principles for AI. Berkman Klein Center Research Publication2020-1 (2020)."},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"publisher","DOI":"10.1145\/3232676"},{"volume-title":"Countering online hate speech","author":"Gagliardone Iginio","key":"e_1_3_2_1_25_1","unstructured":"Iginio Gagliardone, Danit Gal, Thiago Alves, and Gabriela Martinez. 2015. Countering online hate speech. Unesco Publishing."},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.5555\/3295222.3295241"},{"key":"e_1_3_2_1_27_1","volume-title":"Proceedings of the 36th International Conference on Machine Learning(Proceedings of Machine Learning Research, Vol.\u00a097)","author":"Geifman Yonatan","year":"2019","unstructured":"Yonatan Geifman and Ran El-Yaniv. 2019. SelectiveNet: A Deep Neural Network with an Integrated Reject Option. In Proceedings of the 36th International Conference on Machine Learning(Proceedings of Machine Learning Research, Vol.\u00a097), Kamalika Chaudhuri and Ruslan Salakhutdinov (Eds.). PMLR, 2151\u20132159. https:\/\/proceedings.mlr.press\/v97\/geifman19a.html"},{"key":"e_1_3_2_1_28_1","volume-title":"Facebook Uses Deceptive Math to Hide Its Hate Speech Problem. Wired (Oct","author":"Giansiracusa Noah","year":"2021","unstructured":"Noah Giansiracusa. 2021. Facebook Uses Deceptive Math to Hide Its Hate Speech Problem. Wired (Oct 2021). https:\/\/www.wired.com\/story\/facebooks-deceptive-math-when-it-comes-to-hate-speech\/ Visited on 07\/03\/2022."},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.ijforecast.2019.04.016"},{"key":"e_1_3_2_1_30_1","unstructured":"Michael Wojatzki Tobias Horsmann\u00a0Darina Gold and Torsten Zesch. 2018. Do women perceive hate differently: Examining the relationship between hate speech gender and agreement judgments. (2018)."},{"key":"e_1_3_2_1_31_1","volume-title":"Advances in Neural Information Processing Systems, D.\u00a0Koller, D.\u00a0Schuurmans, Y.\u00a0Bengio, and L.\u00a0Bottou (Eds.). Vol.\u00a021. Curran Associates","author":"Grandvalet Yves","year":"2008","unstructured":"Yves Grandvalet, Alain Rakotomamonjy, Joseph Keshet, and St\u00e9phane Canu. 2008. Support Vector Machines with a Reject Option. In Advances in Neural Information Processing Systems, D.\u00a0Koller, D.\u00a0Schuurmans, Y.\u00a0Bengio, and L.\u00a0Bottou (Eds.). Vol.\u00a021. Curran Associates, Inc.https:\/\/proceedings.neurips.cc\/paper\/2008\/file\/3df1d4b96d8976ff5986393e8767f5b2-Paper.pdf"},{"key":"e_1_3_2_1_32_1","doi-asserted-by":"publisher","DOI":"10.1145\/3270101.3270103"},{"volume-title":"ICML\u201917 -","author":"Guo Chuan","key":"e_1_3_2_1_33_1","unstructured":"Chuan Guo, Geoff Pleiss, Yu Sun, and Kilian\u00a0Q. Weinberger. 2017. On Calibration of Modern Neural Networks. In ICML\u201917 - Volume 70. 1321\u20131330."},{"key":"e_1_3_2_1_34_1","volume-title":"Wannes Meert, and Jesse Davis.","author":"Hendrickx Kilian","year":"2021","unstructured":"Kilian Hendrickx, Lorenzo Perini, Dries Van\u00a0der Plas, Wannes Meert, and Jesse Davis. 2021. Machine Learning with a Reject Option: A survey. arXiv preprint arXiv:2107.11277 (2021)."},{"key":"e_1_3_2_1_35_1","volume-title":"Facebook now linked to violence in the Philippines","author":"Ingram Mathew","year":"2018","unstructured":"Mathew Ingram. 2018. Facebook now linked to violence in the Philippines, Libya, Germany, Myanmar, and India. Columbia Journalism Review (Sep 2018). https:\/\/www.cjr.org\/the_media_today\/facebook-linked-to-violence.php \"Visited on 07\/03\/2022\"."},{"volume-title":"AAMAS\u201912 -","author":"Kamar Ece","key":"e_1_3_2_1_36_1","unstructured":"Ece Kamar, Severin Hacker, and Eric Horvitz. 2012. Combining Human and Machine Intelligence in Large-scale Crowdsourcing. In AAMAS\u201912 - Volume 1 (Valencia, Spain). 467\u2013474."},{"volume-title":"Expanding AI\u2019s Impact With Organizational Learning","author":"Khodabandeh Shervin","key":"e_1_3_2_1_37_1","unstructured":"Shervin Khodabandeh, David Kiron, Fran\u00e7oise Candelon, Michael Chu, and Burt LaFountain. 2020. Expanding AI\u2019s Impact With Organizational Learning. MIT Sloan Management Review and Boston Consulting Group."},{"key":"e_1_3_2_1_38_1","first-page":"1598","article-title":"The new governors: The people, rules, and processes governing online speech","volume":"131","author":"Klonick Kate","year":"2018","unstructured":"Kate Klonick. 2018. The new governors: The people, rules, and processes governing online speech. Harvard Law Review 131 (2018), 1598.","journal-title":"Harvard Law Review"},{"key":"e_1_3_2_1_39_1","doi-asserted-by":"publisher","DOI":"10.1111\/j.1468-2958.2004.tb00738.x"},{"key":"e_1_3_2_1_40_1","doi-asserted-by":"publisher","DOI":"10.1177\/2053951718756684"},{"key":"e_1_3_2_1_41_1","doi-asserted-by":"publisher","DOI":"10.1080\/15213269.2019.1612760"},{"key":"e_1_3_2_1_42_1","doi-asserted-by":"publisher","DOI":"10.1145\/3002172"},{"key":"e_1_3_2_1_43_1","volume-title":"Hate Speech in India Nears Dangerous Levels. The New York Times (Feb","author":"Mashal Mujib","year":"2022","unstructured":"Mujib Mashal, Suhasini Raj, and Hari Kumar. 2022. As Officials Look Away, Hate Speech in India Nears Dangerous Levels. The New York Times (Feb 2022). https:\/\/www.nytimes.com\/2022\/02\/08\/world\/asia\/india-hate-speech-muslims.html Visited on 07\/03\/2022."},{"key":"e_1_3_2_1_44_1","doi-asserted-by":"publisher","DOI":"10.1073\/pnas.1813486116"},{"key":"e_1_3_2_1_45_1","doi-asserted-by":"publisher","DOI":"10.1145\/985692.985735"},{"key":"e_1_3_2_1_46_1","doi-asserted-by":"publisher","DOI":"10.1111\/j.1745-4557.1977.tb00942.x"},{"key":"e_1_3_2_1_47_1","doi-asserted-by":"publisher","unstructured":"Azadeh\u00a0Sadat Mozafari Hugo\u00a0Siqueira Gomes Wilson Le\u00e3o Steeven Janny and Christian Gagn\u00e9. 2018. Attended Temperature Scaling: A Practical Approach for Calibrating Deep Neural Networks. https:\/\/doi.org\/10.48550\/ARXIV.1810.11586","DOI":"10.48550\/ARXIV.1810.11586"},{"key":"e_1_3_2_1_48_1","volume-title":"With Posts From Myanmar\u2019s Military. The New York Times (Oct","author":"Mozur Paul","year":"2018","unstructured":"Paul Mozur. 2018. A Genocide Incited on Facebook, With Posts From Myanmar\u2019s Military. The New York Times (Oct 2018). https:\/\/www.nytimes.com\/2018\/10\/15\/technology\/myanmar-facebook-genocide.html Visited on 07\/03\/2022."},{"key":"e_1_3_2_1_49_1","volume-title":"Survey research in HCI. Ways of Knowing in HCI","author":"M\u00fcller Hendrik","year":"2014","unstructured":"Hendrik M\u00fcller, Aaron Sedley, and Elizabeth Ferrall-Nunge. 2014. Survey research in HCI. Ways of Knowing in HCI (2014), 229\u2013266."},{"key":"e_1_3_2_1_50_1","doi-asserted-by":"publisher","DOI":"10.1093\/jeea\/jvaa045"},{"key":"e_1_3_2_1_51_1","volume-title":"Proceedings of the third International Workshop on Machine Learning in Systems Biology(Proceedings of Machine Learning Research, Vol.\u00a08), Sa\u0161o D\u017eeroski, Pierre Guerts, and Juho Rousu (Eds.). PMLR","author":"Sajjad\u00a0Ahmed Nadeem Malik","year":"2009","unstructured":"Malik Sajjad\u00a0Ahmed Nadeem, Jean-Daniel Zucker, and Blaise Hanczar. 2009. Accuracy-Rejection Curves (ARCs) for Comparing Classification Methods with a Reject Option. In Proceedings of the third International Workshop on Machine Learning in Systems Biology(Proceedings of Machine Learning Research, Vol.\u00a08), Sa\u0161o D\u017eeroski, Pierre Guerts, and Juho Rousu (Eds.). PMLR, Ljubljana, Slovenia, 65\u201381. https:\/\/proceedings.mlr.press\/v8\/nadeem10a.html"},{"key":"e_1_3_2_1_52_1","doi-asserted-by":"publisher","DOI":"10.1145\/3091478.3098871"},{"key":"e_1_3_2_1_53_1","volume-title":"The Algorithmic Automation Problem: Prediction, Triage, and Human Effort. CoRR abs\/1903.12220","author":"Raghu Maithra","year":"2019","unstructured":"Maithra Raghu, Katy Blumer, Greg Corrado, Jon\u00a0M. Kleinberg, Ziad Obermeyer, and Sendhil Mullainathan. 2019. The Algorithmic Automation Problem: Prediction, Triage, and Human Effort. CoRR abs\/1903.12220 (2019). arxiv:1903.12220"},{"key":"e_1_3_2_1_54_1","doi-asserted-by":"publisher","DOI":"10.1002\/(SICI)1099-1298(199706)7:3<181::AID-CASP416>3.0.CO;2-Y"},{"key":"e_1_3_2_1_55_1","doi-asserted-by":"publisher","DOI":"10.1145\/2939672.2939778"},{"key":"e_1_3_2_1_56_1","doi-asserted-by":"crossref","unstructured":"Carlos Rodriguez Florian Daniel and Fabio Casati. 2014. Crowd-Based Mining of Reusable Process Model Patterns. In Business Process Management. 51\u201366.","DOI":"10.1007\/978-3-319-10172-9_4"},{"key":"e_1_3_2_1_57_1","doi-asserted-by":"publisher","DOI":"10.1145\/3209978.3210052"},{"key":"e_1_3_2_1_58_1","volume-title":"a distilled version of BERT: smaller, faster, cheaper and lighter. arXiv preprint arXiv:1910.01108","author":"Sanh Victor","year":"2019","unstructured":"Victor Sanh, Lysandre Debut, Julien Chaumond, and Thomas Wolf. 2019. DistilBERT, a distilled version of BERT: smaller, faster, cheaper and lighter. arXiv preprint arXiv:1910.01108 (2019)."},{"key":"e_1_3_2_1_59_1","doi-asserted-by":"publisher","unstructured":"Burcu Sayin Fabio Casati Andrea Passerini Jie Yang and Xinyue Chen. 2022. Rethinking and Recomputing the Value of ML Models. https:\/\/doi.org\/10.48550\/ARXIV.2209.15157","DOI":"10.48550\/ARXIV.2209.15157"},{"key":"e_1_3_2_1_60_1","doi-asserted-by":"publisher","unstructured":"Burcu Sayin Jie Yang Andrea Passerini and Fabio Casati. 2021. The Science of Rejection: A Research Area for Human Computation. https:\/\/doi.org\/10.48550\/ARXIV.2111.06736","DOI":"10.48550\/ARXIV.2111.06736"},{"key":"e_1_3_2_1_61_1","unstructured":"Olivia Solon. 2017. Facebook is hiring moderators. But is the job too gruesome to handle?https:\/\/www.theguardian.com\/technology\/2017\/may\/04\/facebook-content-moderators-ptsd-psychological-dangers Last accessed on 2022-06-21."},{"key":"e_1_3_2_1_62_1","volume-title":"The direct estimation of sensory magnitudes: Loudness. The American journal of psychology 69, 1","author":"Stevens Stanley\u00a0Smith","year":"1956","unstructured":"Stanley\u00a0Smith Stevens. 1956. The direct estimation of sensory magnitudes: Loudness. The American journal of psychology 69, 1 (1956), 1\u201325."},{"key":"e_1_3_2_1_63_1","first-page":"817","article-title":"Hate in cyberspace: Regulating hate speech on the Internet","volume":"38","author":"Tsesis Alexander","year":"2001","unstructured":"Alexander Tsesis. 2001. Hate in cyberspace: Regulating hate speech on the Internet. San Diego L. Rev. 38 (2001), 817.","journal-title":"San Diego L. Rev."},{"key":"e_1_3_2_1_64_1","unstructured":"Heidi Tworek and Patrick Leerssen. 2019. An Analysis of Germany\u2019s NetzDG Law."},{"key":"e_1_3_2_1_65_1","doi-asserted-by":"publisher","DOI":"10.5555\/3122009.3242050"},{"volume-title":"The Harm in Hate Speech","author":"Waldron Jeremy","key":"e_1_3_2_1_66_1","unstructured":"Jeremy Waldron. 2012. The harm in hate speech. In The Harm in Hate Speech. Harvard University Press."},{"key":"e_1_3_2_1_67_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/N16-2013"},{"key":"e_1_3_2_1_68_1","doi-asserted-by":"crossref","unstructured":"Bryan Wilder Eric Horvitz and Ece Kamar. 2020. Learning to Complement Humans. In IJCAI\u201920. 1526\u20131533.","DOI":"10.24963\/ijcai.2020\/212"},{"key":"e_1_3_2_1_69_1","doi-asserted-by":"publisher","DOI":"10.1145\/3531146.3533118"},{"key":"e_1_3_2_1_70_1","doi-asserted-by":"publisher","DOI":"10.1145\/3274463"}],"event":{"name":"AIES '23: AAAI\/ACM Conference on AI, Ethics, and Society","sponsor":["SIGAI ACM Special Interest Group on Artificial Intelligence"],"location":"Montr\u00e9al QC Canada","acronym":"AIES '23"},"container-title":["Proceedings of the 2023 AAAI\/ACM Conference on AI, Ethics, and Society"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3600211.3604655","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3600211.3604655","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T16:37:39Z","timestamp":1750178259000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3600211.3604655"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023,8,8]]},"references-count":70,"alternative-id":["10.1145\/3600211.3604655","10.1145\/3600211"],"URL":"https:\/\/doi.org\/10.1145\/3600211.3604655","relation":{},"subject":[],"published":{"date-parts":[[2023,8,8]]},"assertion":[{"value":"2023-08-29","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}