{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,7,8]],"date-time":"2025-07-08T04:11:15Z","timestamp":1751947875202,"version":"3.41.2"},"publisher-location":"New York, NY, USA","reference-count":35,"publisher":"ACM","content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2025,6,23]]},"DOI":"10.1145\/3707640.3731929","type":"proceedings-article","created":{"date-parts":[[2025,6,20]],"date-time":"2025-06-20T12:23:03Z","timestamp":1750422183000},"page":"1-12","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":0,"title":["Can't LLMs do that? Supporting Third-Party Audits under the DSA: Exploring Large Language Models for Systemic Risk Evaluation of the Digital Services Act in an Interdisciplinary Setting"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-3686-6100","authenticated-orcid":false,"given":"Marie-Therese","family":"Sekwenz","sequence":"first","affiliation":[{"name":"Multi-Actor Systems\/ Organisation and Governance \/ AI Futures Lab, TU Delft, Delft, Netherlands"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-9710-9763","authenticated-orcid":false,"given":"Rita","family":"Gsenger","sequence":"additional","affiliation":[{"name":"Weizenbaum Institut, Berlin, Germany"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-7553-5275","authenticated-orcid":false,"given":"Volker","family":"Stocker","sequence":"additional","affiliation":[{"name":"Weizenbaum Institut, Berlin, Germany"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-3958-2493","authenticated-orcid":false,"given":"Esther","family":"G\u00f6rnemann","sequence":"additional","affiliation":[{"name":"Weizenbaum Institut, Berlin, Germany"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-6612-8061","authenticated-orcid":false,"given":"Dinara","family":"Talypova","sequence":"additional","affiliation":[{"name":"IT:U, Linz, Austria"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-6667-0440","authenticated-orcid":false,"given":"Simon","family":"Parkin","sequence":"additional","affiliation":[{"name":"Multi-Actor Systems\/ Organisation and Governance \/ Cyber Security, TU Delft, Delft, Netherlands"}]},{"ORCID":"https:\/\/orcid.org\/0009-0005-1751-7350","authenticated-orcid":false,"given":"Lea","family":"Greminger","sequence":"additional","affiliation":[{"name":"Weizenbaum Institut, Berlin, Germany"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-4127-3617","authenticated-orcid":false,"given":"Georgios","family":"Smaragdakis","sequence":"additional","affiliation":[{"name":"Electrical Engineering, Mathematics and Computer Science \/Intelligent Systems \/ Cyber Security, TU Delft, Delft, Netherlands"}]}],"member":"320","published-online":{"date-parts":[[2025,6,22]]},"reference":[{"key":"e_1_3_3_1_2_2","doi-asserted-by":"crossref","unstructured":"Meysam Alizadeh Ma\u00ebl Kubli Zeynab Samei Shirin Dehghani Mohammadmasiha Zahedivafa Juan\u00a0D. Bermeo Maria Korobeynikova and Fabrizio Gilardi. 2024. Open-source LLMs for text annotation: a practical guide for model setting and fine-tuning. Journal of Computational Social Science 8 1 (Dec. 2024) 17. https:\/\/doi.org\/10.1007\/s42001-024-00345-9","DOI":"10.1007\/s42001-024-00345-9"},{"key":"e_1_3_3_1_3_2","unstructured":"Anthropic. 2025. Claude Language Model. https:\/\/claude.ai\/"},{"key":"e_1_3_3_1_4_2","doi-asserted-by":"publisher","DOI":"10.1145\/3689904.3694699"},{"key":"e_1_3_3_1_5_2","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2024.nllp-1.17"},{"key":"e_1_3_3_1_6_2","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2023.eval4nlp-1.8"},{"key":"e_1_3_3_1_7_2","doi-asserted-by":"crossref","unstructured":"Virginia Braun and Victoria Clarke. 2021. Conceptual and Design Thinking for Thematic Analysis. Qualitative Psychology 9 (May 2021) 3\u2013\u201326. https:\/\/doi.org\/10.1037\/qup0000196","DOI":"10.1037\/qup0000196"},{"key":"e_1_3_3_1_8_2","doi-asserted-by":"crossref","unstructured":"Virginia Braun and Victoria Clarke. 2021. One size fits all? What counts as quality practice in (reflexive) thematic analysis? Qualitative Research in Psychology 18 3 (July 2021) 328\u2013352. https:\/\/doi.org\/10.1080\/14780887.2020.1769238","DOI":"10.1080\/14780887.2020.1769238"},{"key":"e_1_3_3_1_9_2","unstructured":"Xiang Cheng Raveesh Mayya and Jo\u00e3o Sedoc. 2025. From Human Annotation to LLMs: SILICON Annotation Workflow for Management Research. https:\/\/doi.org\/10.48550\/arXiv.2412.14461 arXiv:https:\/\/arXiv.org\/abs\/2412.14461 [cs]."},{"key":"e_1_3_3_1_10_2","unstructured":"Robert Chew John Bollenbacher Michael Wenger Jessica Speer and Annice Kim. 2023. LLM-Assisted Content Analysis: Using Large Language Models to Support Deductive Coding. https:\/\/doi.org\/10.48550\/arXiv.2306.14924 arXiv:https:\/\/arXiv.org\/abs\/2306.14924 [cs]."},{"key":"e_1_3_3_1_11_2","doi-asserted-by":"crossref","unstructured":"Zachary Cooper William Lehr and Volker Stocker. 2024. The New Age: Legal & Economic Challenges to Copyright and Creative Economies in the Era of Generative AI. https:\/\/doi.org\/10.2139\/ssrn.5022340","DOI":"10.2139\/ssrn.5022340"},{"key":"e_1_3_3_1_12_2","doi-asserted-by":"crossref","unstructured":"Edgar Dubourg Valentin Thouzeau and Nicolas Baumard. 2024. A step-by-step method for cultural annotation by LLMs. Frontiers in Artificial Intelligence 7 (May 2024) 1\u201310. https:\/\/doi.org\/10.3389\/frai.2024.1365508 Publisher: Frontiers.","DOI":"10.3389\/frai.2024.1365508"},{"key":"e_1_3_3_1_13_2","doi-asserted-by":"publisher","DOI":"10.1145\/3708359.3712111"},{"key":"e_1_3_3_1_14_2","unstructured":"European Commission. 2022. Regulation (EU) 2022\/2065 of the European Parliament and of the Council of 19\u00a0October 2022 on a Single Market For Digital Services and amending Directive\u00a02000\/31\/EC (Digital Services Act) (Text with EEA relevance). http:\/\/data.europa.eu\/eli\/reg\/2022\/2065\/oj\/eng Legislative Body: EP CONSIL."},{"key":"e_1_3_3_1_15_2","doi-asserted-by":"crossref","unstructured":"Mirko Franco Ombretta Gaggi and Claudio\u00a0E. Palazzi. 2024. Integrating Content Moderation Systems with Large Language Models. ACM Trans. Web 0 (Oct. 2024) 1 \u2013 20. https:\/\/doi.org\/10.1145\/3700789 Just Accepted.","DOI":"10.1145\/3700789"},{"key":"e_1_3_3_1_16_2","unstructured":"Kristina Gligori\u0107 Tijana Zrnic Cinoo Lee Emmanuel\u00a0J. Cand\u00e8s and Dan Jurafsky. 2025. Can Unconfident LLM Annotations Be Used for Confident Conclusions?https:\/\/doi.org\/10.48550\/arXiv.2408.15204 arXiv:https:\/\/arXiv.org\/abs\/2408.15204 [cs]."},{"key":"e_1_3_3_1_17_2","unstructured":"Feng Gu Zongxia Li Carlos\u00a0Rafael Colon Benjamin Evans Ishani Mondal and Jordan\u00a0Lee Boyd-Graber. 2025. Large Language Models Are Effective Human Annotation Assistants But Not Good Independent Annotators. https:\/\/doi.org\/10.48550\/arXiv.2503.06778 arXiv:https:\/\/arXiv.org\/abs\/2503.06778 [cs]."},{"key":"e_1_3_3_1_18_2","unstructured":"Andrew Halterman and Katherine\u00a0A. Keith. 2025. Codebook LLMs: Evaluating LLMs as Measurement Tools for Political Science Concepts. https:\/\/doi.org\/10.48550\/arXiv.2407.10747 arXiv:https:\/\/arXiv.org\/abs\/2407.10747 [cs]."},{"key":"e_1_3_3_1_19_2","unstructured":"Hangzhou DeepSeek Artificial Intelligence Basic Technology Research Co. Ltd.2025. DeepSeek. https:\/\/www.deepseek.com\/"},{"key":"e_1_3_3_1_20_2","doi-asserted-by":"crossref","unstructured":"Lei Huang Weijiang Yu Weitao Ma Weihong Zhong Zhangyin Feng Haotian Wang Qianglong Chen Weihua Peng Xiaocheng Feng Bing Qin et\u00a0al. 2025. A survey on hallucination in large language models: Principles taxonomy challenges and open questions. ACM Transactions on Information Systems 43 2 (2025) 1\u201355.","DOI":"10.1145\/3703155"},{"key":"e_1_3_3_1_21_2","doi-asserted-by":"publisher","DOI":"10.1145\/3613905.3650828"},{"key":"e_1_3_3_1_22_2","unstructured":"Mentimeter. 2025. Menti Online Voting. https:\/\/www.menti.com\/"},{"key":"e_1_3_3_1_23_2","unstructured":"Meta AI. 2025. Large Language Model Meta AI. https:\/\/www.llama.com\/"},{"key":"e_1_3_3_1_24_2","unstructured":"Miro. 2025. Miro Collaboration Platform. https:\/\/miro.com\/"},{"key":"e_1_3_3_1_25_2","doi-asserted-by":"crossref","unstructured":"Rajiv Movva Pang\u00a0Wei Koh and Emma Pierson. 2024. Annotation alignment: Comparing LLM and human annotations of conversational safety. https:\/\/doi.org\/10.48550\/arXiv.2406.06369 arXiv:https:\/\/arXiv.org\/abs\/2406.06369 [cs].","DOI":"10.18653\/v1\/2024.emnlp-main.511"},{"key":"e_1_3_3_1_26_2","doi-asserted-by":"crossref","unstructured":"Jakob M\u00f6kander Jonas Schuett Hannah\u00a0Rose Kirk and Luciano Floridi. 2024. Auditing large language models: a three-layered approach. AI and Ethics 4 4 (Nov. 2024) 1085\u20131115. https:\/\/doi.org\/10.1007\/s43681-023-00289-2","DOI":"10.1007\/s43681-023-00289-2"},{"key":"e_1_3_3_1_27_2","doi-asserted-by":"crossref","unstructured":"Arbi\u00a0Haza Nasution and Aytu\u011f Onan. 2024. ChatGPT Label: Comparing the Quality of Human-Generated and LLM-Generated Annotations in Low-Resource Language NLP Tasks. IEEE Access 12 (2024) 71876\u201371900. https:\/\/doi.org\/10.1109\/ACCESS.2024.3402809 Conference Name: IEEE Access.","DOI":"10.1109\/ACCESS.2024.3402809"},{"key":"e_1_3_3_1_28_2","doi-asserted-by":"publisher","DOI":"10.1145\/3600211.3604712"},{"key":"e_1_3_3_1_29_2","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2023.emnlp-main.155"},{"key":"e_1_3_3_1_30_2","doi-asserted-by":"crossref","unstructured":"Zhen Tan Dawei Li Song Wang Alimohammad Beigi Bohan Jiang Amrita Bhattacharjee Mansooreh Karami Jundong Li Lu Cheng and Huan Liu. 2024. Large Language Models for Data Annotation and Synthesis: A Survey. https:\/\/doi.org\/10.48550\/arXiv.2402.13446 arXiv:https:\/\/arXiv.org\/abs\/2402.13446 [cs].","DOI":"10.18653\/v1\/2024.emnlp-main.54"},{"key":"e_1_3_3_1_31_2","doi-asserted-by":"crossref","unstructured":"Stephen\u00a0L Thaler. 1995. \u201cVirtual input\u201d phenomena within the death of a simple pattern associator. Neural Networks 8 1 (1995) 55\u201365.","DOI":"10.1016\/0893-6080(94)00065-T"},{"key":"e_1_3_3_1_32_2","unstructured":"Petter T\u00f6rnberg. 2023. How to use LLMs for Text Analysis. https:\/\/doi.org\/10.48550\/arXiv.2307.13106 arXiv:https:\/\/arXiv.org\/abs\/2307.13106 [cs]."},{"key":"e_1_3_3_1_33_2","unstructured":"Petter T\u00f6rnberg. 2024. Best Practices for Text Annotation with Large Language Models. https:\/\/doi.org\/10.48550\/arXiv.2402.05129 arXiv:https:\/\/arXiv.org\/abs\/2402.05129 [cs]."},{"key":"e_1_3_3_1_34_2","doi-asserted-by":"publisher","DOI":"10.1145\/3613904.3641960"},{"key":"e_1_3_3_1_35_2","unstructured":"Yue Zhang Yafu Li Leyang Cui Deng Cai Lemao Liu Tingchen Fu Xinting Huang Enbo Zhao Yu Zhang Yulong Chen Longyue Wang Anh\u00a0Tuan Luu Wei Bi Freda Shi and Shuming Shi. 2023. Siren\u2019s Song in the AI Ocean: A Survey on Hallucination in Large Language Models. arxiv:https:\/\/arXiv.org\/abs\/2309.01219\u00a0[cs.CL] https:\/\/arxiv.org\/abs\/2309.01219"},{"key":"e_1_3_3_1_36_2","unstructured":"Chengshuai Zhao Zhen Tan Chau-Wai Wong Xinyan Zhao Tianlong Chen and Huan Liu. 2025. SCALE: Towards Collaborative Content Analysis in Social Science with Large Language Model Agents and Human Intervention. arxiv:https:\/\/arXiv.org\/abs\/2502.10937\u00a0[cs.AI] https:\/\/arxiv.org\/abs\/2502.10937"}],"event":{"name":"CHIWORK '25 Adjunct: Adjunct Proceedings of the 4th Annual Symposium on Human-Computer Interaction for Work","acronym":"CHIWORK '25 Adjunct","location":"Amsterdam Netherlands"},"container-title":["Adjunct Proceedings of the 4th Annual Symposium on Human-Computer Interaction for Work"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3707640.3731929","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,7,7]],"date-time":"2025-07-07T15:47:57Z","timestamp":1751903277000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3707640.3731929"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,6,22]]},"references-count":35,"alternative-id":["10.1145\/3707640.3731929","10.1145\/3707640"],"URL":"https:\/\/doi.org\/10.1145\/3707640.3731929","relation":{},"subject":[],"published":{"date-parts":[[2025,6,22]]},"assertion":[{"value":"2025-06-22","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}