{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,6,4]],"date-time":"2026-06-04T20:50:44Z","timestamp":1780606244588,"version":"3.54.1"},"publisher-location":"New York, NY, USA","reference-count":52,"publisher":"ACM","license":[{"start":{"date-parts":[[2024,5,11]],"date-time":"2024-05-11T00:00:00Z","timestamp":1715385600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2024,5,11]]},"DOI":"10.1145\/3613905.3636301","type":"proceedings-article","created":{"date-parts":[[2024,5,11]],"date-time":"2024-05-11T08:15:21Z","timestamp":1715415321000},"page":"1-7","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":31,"title":["LLMs as Research Tools: Applications and Evaluations in HCI Data Work"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-4189-8040","authenticated-orcid":false,"given":"Marianne","family":"Aubin Le Qu\u00e9r\u00e9","sequence":"first","affiliation":[{"name":"Information Science, Cornell University, United States"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-7826-3298","authenticated-orcid":false,"given":"Hope","family":"Schroeder","sequence":"additional","affiliation":[{"name":"MIT, United States"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-8833-7588","authenticated-orcid":false,"given":"Casey","family":"Randazzo","sequence":"additional","affiliation":[{"name":"Rutgers University, United States"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-5992-6471","authenticated-orcid":false,"given":"Jie","family":"Gao","sequence":"additional","affiliation":[{"name":"Information Systems Technology and Design, Singapore University of Technology and Design, Singapore and Singapore-MIT Alliance for Research and Technology, Singapore"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-5831-5756","authenticated-orcid":false,"given":"Ziv","family":"Epstein","sequence":"additional","affiliation":[{"name":"Media Lab, MIT, United States"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-3105-9350","authenticated-orcid":false,"given":"Simon Tangi","family":"Perrault","sequence":"additional","affiliation":[{"name":"Information Systems Technology and Design, Singapore University of Technology and Design, Singapore"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-7510-9404","authenticated-orcid":false,"given":"David","family":"Mimno","sequence":"additional","affiliation":[{"name":"Cornell University, United States"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-1306-249X","authenticated-orcid":false,"given":"Louise","family":"Barkhuus","sequence":"additional","affiliation":[{"name":"Rutgers University, United States and IT University of Copenhagen, Denmark"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-0688-9918","authenticated-orcid":false,"given":"Hanlin","family":"Li","sequence":"additional","affiliation":[{"name":"University of Texas at Austin, United States"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2024,5,11]]},"reference":[{"key":"e_1_3_3_1_1_1","doi-asserted-by":"publisher","DOI":"10.1145\/3411763.3441311"},{"key":"e_1_3_3_1_2_1","doi-asserted-by":"publisher","DOI":"10.1017\/pan.2023.2"},{"key":"e_1_3_3_1_3_1","doi-asserted-by":"publisher","unstructured":"Mohammad Atari Mona\u00a0J. Xue Peter\u00a0S. Park Dami\u00e1n Blasi and Joseph Henrich. 2023. Which Humans?https:\/\/doi.org\/10.31234\/osf.io\/5b26t","DOI":"10.31234\/osf.io"},{"key":"e_1_3_3_1_4_1","doi-asserted-by":"publisher","DOI":"10.31235\/osf.io"},{"key":"e_1_3_3_1_5_1","doi-asserted-by":"publisher","DOI":"10.1145\/3491101.3503704"},{"key":"e_1_3_3_1_6_1","doi-asserted-by":"publisher","DOI":"10.1002\/asi.23786"},{"key":"e_1_3_3_1_7_1","doi-asserted-by":"publisher","DOI":"10.1177\/1460458219864730"},{"key":"e_1_3_3_1_8_1","doi-asserted-by":"publisher","DOI":"10.1145\/3027063.3027103"},{"key":"e_1_3_3_1_9_1","volume-title":"Algorithmic Accountability: A Primer. https:\/\/datasociety.net\/library\/algorithmic-accountability-a-primer\/ Publisher: Data & Society Research Institute.","author":"Caplan Robyn","year":"2018","unstructured":"Robyn Caplan, Joan Donovan, Lauren Hanson, and Jeanna Matthews. 2018. Algorithmic Accountability: A Primer. https:\/\/datasociety.net\/library\/algorithmic-accountability-a-primer\/ Publisher: Data & Society Research Institute."},{"key":"e_1_3_3_1_10_1","doi-asserted-by":"publisher","DOI":"10.1145\/3544549.3573826"},{"key":"e_1_3_3_1_11_1","doi-asserted-by":"publisher","DOI":"10.1145\/3491102.3517475"},{"key":"e_1_3_3_1_12_1","doi-asserted-by":"publisher","unstructured":"Felix Chopra and Ingar Haaland. 2023. Conducting Qualitative Interviews with AI. https:\/\/doi.org\/10.2139\/ssrn.4583756","DOI":"10.2139\/ssrn.4583756"},{"key":"e_1_3_3_1_13_1","doi-asserted-by":"publisher","DOI":"10.7551\/mitpress\/12255.001.0001"},{"key":"e_1_3_3_1_14_1","doi-asserted-by":"publisher","unstructured":"Stefano De\u00a0Paoli. 2023. Can Large Language Models emulate an inductive Thematic Analysis of semi-structured interviews? An exploration and provocation on the limits of the approach and the model. https:\/\/doi.org\/10.48550\/arXiv.2305.13014 arXiv:2305.13014 [cs].","DOI":"10.48550\/arXiv.2305.13014"},{"key":"e_1_3_3_1_15_1","doi-asserted-by":"crossref","unstructured":"Bosheng Ding Chengwei Qin Linlin Liu Yew\u00a0Ken Chia Shafiq Joty Boyang Li and Lidong Bing. 2023. Is GPT-3 a Good Data Annotator?http:\/\/arxiv.org\/abs\/2212.10450 arXiv:2212.10450 [cs].","DOI":"10.18653\/v1\/2023.acl-long.626"},{"key":"e_1_3_3_1_16_1","doi-asserted-by":"publisher","DOI":"10.1145\/3544549.3573832"},{"key":"e_1_3_3_1_17_1","doi-asserted-by":"publisher","unstructured":"Xiachong Feng Xiaocheng Feng Libo Qin Bing Qin and Ting Liu. 2021. Language Model as an Annotator: Exploring DialoGPT for Dialogue Summarization. https:\/\/doi.org\/10.48550\/arXiv.2105.12544 arXiv:2105.12544 [cs].","DOI":"10.48550\/arXiv.2105.12544"},{"key":"e_1_3_3_1_18_1","doi-asserted-by":"publisher","DOI":"10.48550\/arXiv.2304.07366"},{"key":"e_1_3_3_1_19_1","doi-asserted-by":"publisher","DOI":"10.31235\/osf.io"},{"key":"e_1_3_3_1_20_1","doi-asserted-by":"publisher","unstructured":"Michael Heseltine and Bernhard Clemm\u00a0von Hohenberg. 2023. Large Language Models as a Substitute for Human Experts in Annotating Political Text. https:\/\/doi.org\/10.31219\/osf.io\/cx752","DOI":"10.31219\/osf.io"},{"key":"e_1_3_3_1_21_1","unstructured":"Josiah Hester. 2023. Why is CHI in Hawaii. https:\/\/www.chiinhawaii.info"},{"key":"e_1_3_3_1_22_1","doi-asserted-by":"publisher","unstructured":"John\u00a0J. Horton. 2023. Large Language Models as Simulated Economic Agents: What Can We Learn from Homo Silicus?https:\/\/doi.org\/10.3386\/w31122","DOI":"10.3386\/w31122"},{"key":"e_1_3_3_1_23_1","doi-asserted-by":"publisher","DOI":"10.1145\/3544548.3580688"},{"key":"e_1_3_3_1_24_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1"},{"key":"e_1_3_3_1_25_1","doi-asserted-by":"publisher","DOI":"10.1145\/3323994.3369898"},{"key":"e_1_3_3_1_26_1","doi-asserted-by":"publisher","DOI":"10.2139\/ssrn.4523551"},{"key":"e_1_3_3_1_27_1","doi-asserted-by":"publisher","DOI":"10.1145\/3359174"},{"key":"e_1_3_3_1_28_1","first-page":"3","article-title":"Interrater reliability: the kappa statistic","volume":"22","author":"McHugh L.","year":"2012","unstructured":"Mary\u00a0L. McHugh. 2012. Interrater reliability: the kappa statistic. Biochemia Medica 22, 3 (Oct. 2012), 276\u2013282. https:\/\/www.ncbi.nlm.nih.gov\/pmc\/articles\/PMC3900052\/","journal-title":"Biochemia Medica"},{"key":"e_1_3_3_1_29_1","doi-asserted-by":"publisher","DOI":"10.1145\/3544549.3573794"},{"key":"e_1_3_3_1_30_1","unstructured":"Arvind Narayanan. 2023. Evaluating LLMs is a minefield. https:\/\/www.aisnakeoil.com\/p\/evaluating-llms-is-a-minefield"},{"key":"e_1_3_3_1_31_1","doi-asserted-by":"publisher","DOI":"10.1145\/3544548.3580766"},{"key":"e_1_3_3_1_32_1","doi-asserted-by":"publisher","DOI":"10.48550\/arXiv.2304.03442"},{"key":"e_1_3_3_1_33_1","doi-asserted-by":"publisher","DOI":"10.1145\/3491101.3503724"},{"key":"e_1_3_3_1_34_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1"},{"key":"e_1_3_3_1_35_1","doi-asserted-by":"publisher","DOI":"10.1145\/3544548.3581453"},{"key":"e_1_3_3_1_36_1","doi-asserted-by":"publisher","DOI":"10.31234\/osf.io"},{"key":"e_1_3_3_1_37_1","doi-asserted-by":"publisher","DOI":"10.1145\/3411764.3445591"},{"key":"e_1_3_3_1_38_1","doi-asserted-by":"publisher","unstructured":"Maarten Sap Swabha Swayamdipta Laura Vianna Xuhui Zhou Yejin Choi and Noah\u00a0A. Smith. 2022. Annotators with Attitudes: How Annotator Beliefs And Identities Bias Toxic Language Detection. https:\/\/doi.org\/10.48550\/arXiv.2111.07997 arXiv:2111.07997 [cs].","DOI":"10.48550\/arXiv.2111.07997"},{"key":"e_1_3_3_1_39_1","doi-asserted-by":"publisher","DOI":"10.1145\/3544548.3581512"},{"key":"e_1_3_3_1_40_1","volume-title":"Joon\u00a0Sung Park, and Diyi Yang.","author":"Shen Hong","year":"2023","unstructured":"Hong Shen, Tianshi Li, Toby Jia-Jun Li, Joon\u00a0Sung Park, and Diyi Yang. 2023. Shaping the Emerging Norms of Using Large Language Models in Social Computing Research. http:\/\/arxiv.org\/abs\/2307.04280 arXiv:2307.04280 [cs]."},{"key":"e_1_3_3_1_41_1","doi-asserted-by":"publisher","DOI":"10.1145\/3544549.3573802"},{"key":"e_1_3_3_1_42_1","doi-asserted-by":"publisher","unstructured":"Petter T\u00f6rnberg. 2023. ChatGPT-4 Outperforms Experts and Crowd Workers in Annotating Political Twitter Messages with Zero-Shot Learning. https:\/\/doi.org\/10.48550\/arXiv.2304.06588 arXiv:2304.06588 [cs].","DOI":"10.48550\/arXiv.2304.06588"},{"key":"e_1_3_3_1_43_1","unstructured":"Veniamin Veselovsky Manoel\u00a0Horta Ribeiro Akhil Arora Martin Josifoski Ashton Anderson and Robert West. 2023. Generating Faithful Synthetic Data with Large Language Models: A Case Study in Computational Social Science. http:\/\/arxiv.org\/abs\/2305.15041 arXiv:2305.15041 [cs]."},{"key":"e_1_3_3_1_44_1","unstructured":"Veniamin Veselovsky Manoel\u00a0Horta Ribeiro and Robert West. 2023. Artificial Artificial Artificial Intelligence: Crowd Workers Widely Use Large Language Models for Text Production Tasks. http:\/\/arxiv.org\/abs\/2306.07899 arXiv:2306.07899 [cs]."},{"key":"e_1_3_3_1_45_1","doi-asserted-by":"publisher","DOI":"10.1145\/3491101.3519614"},{"key":"e_1_3_3_1_46_1","volume-title":"Advances in Neural Information Processing Systems, Vol.\u00a032. Curran Associates","author":"Wang Alex","year":"2019","unstructured":"Alex Wang, Yada Pruksachatkun, Nikita Nangia, Amanpreet Singh, Julian Michael, Felix Hill, Omer Levy, and Samuel Bowman. 2019. SuperGLUE: A Stickier Benchmark for General-Purpose Language Understanding Systems. In Advances in Neural Information Processing Systems, Vol.\u00a032. Curran Associates, Inc.https:\/\/papers.nips.cc\/paper_files\/paper\/2019\/hash\/4496bf24afe7fab6f046bf4923da8de6-Abstract.html"},{"key":"e_1_3_3_1_47_1","doi-asserted-by":"publisher","DOI":"10.1145\/3581754.3584136"},{"key":"e_1_3_3_1_48_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1"},{"key":"e_1_3_3_1_49_1","doi-asserted-by":"publisher","DOI":"10.1145\/3173574.3173733"},{"key":"e_1_3_3_1_50_1","unstructured":"Caleb Ziems William Held Omar Shaikh Jiaao Chen Zhehao Zhang and Diyi Yang. 2023. Can Large Language Models Transform Computational Social Science?http:\/\/arxiv.org\/abs\/2305.03514 arXiv:2305.03514 [cs]."},{"key":"e_1_3_3_1_51_1","doi-asserted-by":"publisher","DOI":"10.1145\/3411764.3445328"},{"key":"e_1_3_3_1_52_1","doi-asserted-by":"publisher","DOI":"10.1145\/3411764.3445145"}],"event":{"name":"CHI '24: CHI Conference on Human Factors in Computing Systems","location":"Honolulu HI USA","acronym":"CHI '24","sponsor":["SIGCHI ACM Special Interest Group on Computer-Human Interaction","SIGACCESS ACM Special Interest Group on Accessible Computing"]},"container-title":["Extended Abstracts of the CHI Conference on Human Factors in Computing Systems"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3613905.3636301","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3613905.3636301","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,18]],"date-time":"2025-06-18T23:57:18Z","timestamp":1750291038000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3613905.3636301"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,5,11]]},"references-count":52,"alternative-id":["10.1145\/3613905.3636301","10.1145\/3613905"],"URL":"https:\/\/doi.org\/10.1145\/3613905.3636301","relation":{},"subject":[],"published":{"date-parts":[[2024,5,11]]},"assertion":[{"value":"2024-05-11","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}