{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,6,4]],"date-time":"2026-06-04T21:21:47Z","timestamp":1780608107634,"version":"3.54.1"},"publisher-location":"New York, NY, USA","reference-count":125,"publisher":"ACM","license":[{"start":{"date-parts":[[2025,4,25]],"date-time":"2025-04-25T00:00:00Z","timestamp":1745539200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2025,4,26]]},"DOI":"10.1145\/3706598.3713220","type":"proceedings-article","created":{"date-parts":[[2025,4,24]],"date-time":"2025-04-24T04:45:58Z","timestamp":1745469958000},"page":"1-17","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":16,"title":["Simulacrum of Stories: Examining Large Language Models as Qualitative Research Participants"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-0152-4311","authenticated-orcid":false,"given":"Shivani","family":"Kapania","sequence":"first","affiliation":[{"name":"School of Computer Science, Carnegie Mellon University, Pittsburgh, Pennsylvania, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-1362-554X","authenticated-orcid":false,"given":"William","family":"Agnew","sequence":"additional","affiliation":[{"name":"Human-Computer Interaction Institute, Carnegie Mellon University, Pittsburgh, Pennsylvania, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-1499-3045","authenticated-orcid":false,"given":"Motahhare","family":"Eslami","sequence":"additional","affiliation":[{"name":"School of Computer Science, Carnegie Mellon University, Pittsburgh, Pennsylvania, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-3710-4076","authenticated-orcid":false,"given":"Hoda","family":"Heidari","sequence":"additional","affiliation":[{"name":"School of Computer Science, Carnegie Mellon University, Pittsburgh, Pennsylvania, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-7888-2598","authenticated-orcid":false,"given":"Sarah E","family":"Fox","sequence":"additional","affiliation":[{"name":"Human-Computer Interaction Institute, Carnegie Mellon University, Pittsburgh, Pennsylvania, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2025,4,25]]},"reference":[{"key":"e_1_3_3_3_2_2","doi-asserted-by":"publisher","DOI":"10.1145\/3461702.3462624"},{"key":"e_1_3_3_3_3_2","doi-asserted-by":"publisher","DOI":"10.1145\/3613904.3642703"},{"key":"e_1_3_3_3_4_2","series-title":"Proceedings of Machine Learning Research","first-page":"337","volume-title":"Proceedings of the 40th International Conference on Machine Learning","volume":"202","author":"Aher Gati\u00a0V","year":"2023","unstructured":"Gati\u00a0V Aher, Rosa\u00a0I. Arriaga, and Adam\u00a0Tauman Kalai. 2023. Using Large Language Models to Simulate Multiple Humans and Replicate Human Subject Studies. In Proceedings of the 40th International Conference on Machine Learning(Proceedings of Machine Learning Research, Vol.\u00a0202), Andreas Krause, Emma Brunskill, Kyunghyun Cho, Barbara Engelhardt, Sivan Sabato, and Jonathan Scarlett (Eds.). PMLR, Hawaii, USA, 337\u2013371. https:\/\/proceedings.mlr.press\/v202\/aher23a.html"},{"key":"e_1_3_3_3_5_2","doi-asserted-by":"crossref","unstructured":"Linda Alcoff. 1991. The problem of speaking for others. 5\u201332\u00a0pages.","DOI":"10.2307\/1354221"},{"key":"e_1_3_3_3_6_2","doi-asserted-by":"crossref","unstructured":"Mike Ananny and Kate Crawford. 2018. Seeing without knowing: Limitations of the transparency ideal and its application to algorithmic accountability. New Media & Society 20 (2018) 973 \u2013 989. https:\/\/api.semanticscholar.org\/CorpusID:5001487","DOI":"10.1177\/1461444816676645"},{"key":"e_1_3_3_3_7_2","doi-asserted-by":"crossref","unstructured":"Lisa\u00a0P Argyle Ethan\u00a0C Busby Nancy Fulda Joshua\u00a0R Gubler Christopher Rytting and David Wingate. 2023. Out of one many: Using language models to simulate human samples. Political Analysis 31 3 (2023) 337\u2013351.","DOI":"10.1017\/pan.2023.2"},{"key":"e_1_3_3_3_8_2","doi-asserted-by":"crossref","unstructured":"Neda Atanasoski and Kalindi Vora. 2019. Surrogate humanity. Londres e Durham: Duke University Press. DOI 10 (2019) 9781478004455.","DOI":"10.1215\/9781478004455"},{"key":"e_1_3_3_3_9_2","unstructured":"Mark Avis. 2005. Is there an epistemology for qualitative research. 3\u201316\u00a0pages."},{"key":"e_1_3_3_3_10_2","unstructured":"Jinheon Baek Sujay\u00a0Kumar Jauhar Silviu Cucerzan and Sung\u00a0Ju Hwang. 2024. ResearchAgent: Iterative Research Idea Generation over Scientific Literature with Large Language Models. arxiv:https:\/\/arXiv.org\/abs\/2404.07738\u00a0[cs.CL] https:\/\/arxiv.org\/abs\/2404.07738"},{"key":"e_1_3_3_3_11_2","unstructured":"Yuntao Bai Saurav Kadavath Sandipan Kundu Amanda Askell Jackson Kernion Andy Jones Anna Chen Anna Goldie Azalia Mirhoseini Cameron McKinnon Carol Chen Catherine Olsson Christopher Olah Danny Hernandez Dawn Drain Deep Ganguli Dustin Li Eli Tran-Johnson Ethan Perez Jamie Kerr Jared Mueller Jeffrey Ladish Joshua Landau Kamal Ndousse Kamile Lukosuite Liane Lovitt Michael Sellitto Nelson Elhage Nicholas Schiefer Noemi Mercado Nova DasSarma Robert Lasenby Robin Larson Sam Ringer Scott Johnston Shauna Kravec Sheer\u00a0El Showk Stanislav Fort Tamera Lanham Timothy Telleen-Lawton Tom Conerly Tom Henighan Tristan Hume Samuel\u00a0R. Bowman Zac Hatfield-Dodds Ben Mann Dario Amodei Nicholas Joseph Sam McCandlish Tom Brown and Jared Kaplan. 2022. Constitutional AI: Harmlessness from AI Feedback. arxiv:https:\/\/arXiv.org\/abs\/2212.08073\u00a0[cs.CL] https:\/\/arxiv.org\/abs\/2212.08073"},{"key":"e_1_3_3_3_12_2","doi-asserted-by":"publisher","DOI":"10.1145\/2814464.2814472"},{"key":"e_1_3_3_3_13_2","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2023.acl-long.608"},{"key":"e_1_3_3_3_14_2","volume-title":"Fairness and Machine Learning: Limitations and Opportunities","author":"Barocas S.","year":"2023","unstructured":"S. Barocas, M. Hardt, and A. Narayanan. 2023. Fairness and Machine Learning: Limitations and Opportunities. MIT Press, Cambridge, USA. https:\/\/books.google.com\/books?id=ouawEAAAQBAJ"},{"key":"e_1_3_3_3_15_2","doi-asserted-by":"crossref","unstructured":"Tom\u00a0L Beauchamp. 2010. Autonomy and consent. 55\u201378\u00a0pages.","DOI":"10.1093\/acprof:oso\/9780195335149.003.0003"},{"key":"e_1_3_3_3_16_2","doi-asserted-by":"publisher","DOI":"10.1145\/3442188.3445922"},{"key":"e_1_3_3_3_17_2","doi-asserted-by":"publisher","DOI":"10.1145\/3290605.3300528"},{"key":"e_1_3_3_3_18_2","doi-asserted-by":"crossref","unstructured":"Linda Birt Suzanne Scott Debbie Cavers Christine Campbell and Fiona Walter. 2016. Member checking: a tool to enhance trustworthiness or merely a nod to validation? Qualitative health research 26 13 (2016) 1802\u20131811.","DOI":"10.1177\/1049732316654870"},{"key":"e_1_3_3_3_19_2","doi-asserted-by":"crossref","unstructured":"Su\u00a0Lin Blodgett Solon Barocas Hal Daum\u2019e and Hanna\u00a0M. Wallach. 2020. Language (Technology) is Power: A Critical Survey of \u201cBias\u201d in NLP. https:\/\/api.semanticscholar.org\/CorpusID:218971825","DOI":"10.18653\/v1\/2020.acl-main.485"},{"key":"e_1_3_3_3_20_2","unstructured":"ACM\u00a0Publications Board. 2023. ACM Policy on Authorship. https:\/\/www.acm.org\/publications\/policies\/new-acm-policy-on-authorship. (Accessed on 09\/08\/2024)."},{"key":"e_1_3_3_3_21_2","unstructured":"Daniil\u00a0A. Boiko Robert MacKnight and Gabe Gomes. 2023. Emergent autonomous scientific research capabilities of large language models. arxiv:https:\/\/arXiv.org\/abs\/2304.05332\u00a0[physics.chem-ph] https:\/\/arxiv.org\/abs\/2304.05332"},{"key":"e_1_3_3_3_22_2","volume-title":"Advances in Neural Information Processing Systems","author":"Bolukbasi Tolga","year":"2016","unstructured":"Tolga Bolukbasi, Kai-Wei Chang, James\u00a0Y Zou, Venkatesh Saligrama, and Adam\u00a0T Kalai. 2016. Man is to Computer Programmer as Woman is to Homemaker? Debiasing Word Embeddings. In Advances in Neural Information Processing Systems , D.\u00a0Lee, M.\u00a0Sugiyama, U.\u00a0Luxburg, I.\u00a0Guyon, and R.\u00a0Garnett (Eds.), Vol.\u00a029. Curran Associates, Inc., Barcelona, Spain. https:\/\/proceedings.neurips.cc\/paper_files\/paper\/2016\/file\/a486cd07e4ac3d270571622f4f316ec5-Paper.pdf"},{"key":"e_1_3_3_3_23_2","doi-asserted-by":"crossref","unstructured":"Shikha Bordia and Samuel\u00a0R. Bowman. 2019. Identifying and Reducing Gender Bias in Word-Level Language Models. arxiv:https:\/\/arXiv.org\/abs\/1904.03035\u00a0[cs.CL] https:\/\/arxiv.org\/abs\/1904.03035","DOI":"10.18653\/v1\/N19-3002"},{"key":"e_1_3_3_3_24_2","doi-asserted-by":"crossref","unstructured":"Virginia Braun and Victoria Clarke. 2012. Thematic analysis.","DOI":"10.1037\/13620-004"},{"key":"e_1_3_3_3_25_2","doi-asserted-by":"crossref","unstructured":"Virginia Braun and Victoria Clarke. 2019. Reflecting on reflexive thematic analysis. Qualitative research in sport exercise and health 11 4 (2019) 589\u2013597.","DOI":"10.1080\/2159676X.2019.1628806"},{"key":"e_1_3_3_3_26_2","unstructured":"Antonio Byrd. 2023. Truth-Telling: Critical Inquiries on LLMs and the Corpus Texts That Train Them. Composition studies 51 1 (2023) 135\u2013142."},{"key":"e_1_3_3_3_27_2","doi-asserted-by":"publisher","DOI":"10.1145\/3544549.3582749"},{"key":"e_1_3_3_3_28_2","doi-asserted-by":"publisher","unstructured":"Aylin Caliskan Joanna\u00a0J. Bryson and Arvind Narayanan. 2017. Semantics derived automatically from language corpora contain human-like biases. Science 356 6334 (2017) 183\u2013186. 10.1126\/science.aal4230 arXiv:https:\/\/www.science.org\/doi\/pdf\/10.1126\/science.aal4230","DOI":"10.1126\/science.aal4230"},{"key":"e_1_3_3_3_29_2","unstructured":"Chi-Min Chan Weize Chen Yusheng Su Jianxuan Yu Wei Xue Shanghang Zhang Jie Fu and Zhiyuan Liu. 2023. ChatEval: Towards Better LLM-based Evaluators through Multi-Agent Debate. arxiv:https:\/\/arXiv.org\/abs\/2308.07201\u00a0[cs.CL] https:\/\/arxiv.org\/abs\/2308.07201"},{"key":"e_1_3_3_3_30_2","unstructured":"Alicja Chaszczewicz Raj\u00a0Sanjay Shah Ryan Louie Bruce\u00a0A Arnow Robert Kraut and Diyi Yang. 2024. Multi-Level Feedback Generation with Large Language Models for Empowering Novice Peer Counselors. arxiv:https:\/\/arXiv.org\/abs\/2403.15482\u00a0[cs.CL] https:\/\/arxiv.org\/abs\/2403.15482"},{"key":"e_1_3_3_3_31_2","unstructured":"Cheng-Han Chiang and Hung yi Lee. 2023. Can Large Language Models Be an Alternative to Human Evaluations? arxiv:https:\/\/arXiv.org\/abs\/2305.01937\u00a0[cs.CL] https:\/\/arxiv.org\/abs\/2305.01937"},{"key":"e_1_3_3_3_32_2","doi-asserted-by":"publisher","DOI":"10.7551\/mitpress\/12255.001.0001"},{"key":"e_1_3_3_3_33_2","unstructured":"Andy Crabtree. 2024. H is for Human and How (Not) To Evaluate Qualitative Research in HCI. arxiv:https:\/\/arXiv.org\/abs\/2409.01302\u00a0[cs.HC] https:\/\/arxiv.org\/abs\/2409.01302"},{"key":"e_1_3_3_3_34_2","unstructured":"Emilia David. 2024. OpenAI strikes Reddit deal to train its AI on your posts - The Verge. https:\/\/www.theverge.com\/2024\/5\/16\/24158529\/reddit-openai-chatgpt-api-access-advertising. (Accessed on 09\/01\/2024)."},{"key":"e_1_3_3_3_35_2","doi-asserted-by":"crossref","unstructured":"Dorottya Demszky Diyi Yang David\u00a0S Yeager Christopher\u00a0J Bryan Margarett Clapper Susannah Chandhok Johannes\u00a0C Eichstaedt Cameron Hecht Jeremy Jamieson Meghann Johnson et\u00a0al. 2023. Using large language models in psychology. Nature Reviews Psychology 2 11 (2023) 688\u2013701.","DOI":"10.1038\/s44159-023-00241-5"},{"key":"e_1_3_3_3_36_2","unstructured":"Xiang Deng Yu Gu Boyuan Zheng Shijie Chen Sam Stevens Boshi Wang Huan Sun and Yu Su. 2024. Mind2web: Towards a generalist agent for the web."},{"key":"e_1_3_3_3_37_2","first-page":"30039","volume-title":"Advances in Neural Information Processing Systems","volume":"36","author":"Dubois Yann","year":"2023","unstructured":"Yann Dubois, Chen\u00a0Xuechen Li, Rohan Taori, Tianyi Zhang, Ishaan Gulrajani, Jimmy Ba, Carlos Guestrin, Percy\u00a0S Liang, and Tatsunori\u00a0B Hashimoto. 2023. AlpacaFarm: A Simulation Framework for Methods that Learn from Human Feedback. In Advances in Neural Information Processing Systems , A.\u00a0Oh, T.\u00a0Naumann, A.\u00a0Globerson, K.\u00a0Saenko, M.\u00a0Hardt, and S.\u00a0Levine (Eds.), Vol.\u00a036. Curran Associates, Inc., New Orleans, USA, 30039\u201330069. https:\/\/proceedings.neurips.cc\/paper_files\/paper\/2023\/file\/5fc47800ee5b30b8777fdd30abcaaf3b-Paper-Conference.pdf"},{"key":"e_1_3_3_3_38_2","doi-asserted-by":"publisher","DOI":"10.1145\/3334480.3382931"},{"key":"e_1_3_3_3_39_2","doi-asserted-by":"crossref","unstructured":"Jessica\u00a0L Feuston and Jed\u00a0R Brubaker. 2021. Putting tools in their place: The role of time and perspective in human-AI collaboration for qualitative analysis. Proceedings of the ACM on Human-Computer Interaction 5 CSCW2 (2021) 1\u201325.","DOI":"10.1145\/3479856"},{"key":"e_1_3_3_3_40_2","doi-asserted-by":"crossref","unstructured":"Casey Fiesler and Nicholas Proferes. 2018. \u201cParticipant\u201d perceptions of Twitter research ethics. Social Media+ Society 4 1 (2018) 2056305118763366.","DOI":"10.1177\/2056305118763366"},{"key":"e_1_3_3_3_41_2","doi-asserted-by":"publisher","DOI":"10.1093\/acprof:oso\/9780198237907.001.0001"},{"key":"e_1_3_3_3_42_2","unstructured":"Ivar Frisch and Mario Giulianelli. 2024. LLM Agents in Interaction: Measuring Personality Consistency and Linguistic Alignment in Interacting Populations of Large Language Models. arxiv:https:\/\/arXiv.org\/abs\/2402.02896\u00a0[cs.CL] https:\/\/arxiv.org\/abs\/2402.02896"},{"key":"e_1_3_3_3_43_2","unstructured":"Sidney Fussell. 2019. You no longer own your face."},{"key":"e_1_3_3_3_44_2","doi-asserted-by":"publisher","DOI":"10.1145\/3613904.3642002"},{"key":"e_1_3_3_3_45_2","doi-asserted-by":"crossref","unstructured":"Tao Ge Xin Chan Xiaoyang Wang Dian Yu Haitao Mi and Dong Yu. 2024. Scaling Synthetic Data Creation with 1 000 000 000 Personas. arxiv:https:\/\/arXiv.org\/abs\/2406.20094\u00a0[cs.CL] https:\/\/arxiv.org\/abs\/2406.20094","DOI":"10.14218\/GE.2024.00035"},{"key":"e_1_3_3_3_46_2","unstructured":"Clifford Geertz. 2008. Thick description: Toward an interpretive theory of culture. 41\u201351\u00a0pages."},{"key":"e_1_3_3_3_47_2","doi-asserted-by":"publisher","unstructured":"Fabrizio Gilardi Meysam Alizadeh and Ma\u00ebl Kubli. 2023. ChatGPT outperforms crowd workers for text-annotation tasks. 10.1073\/pnas.2305016120","DOI":"10.1073\/pnas.2305016120"},{"key":"e_1_3_3_3_48_2","doi-asserted-by":"crossref","unstructured":"Fabrizio Gilardi Meysam Alizadeh and Ma\u00ebl Kubli. 2023. ChatGPT outperforms crowd workers for text-annotation tasks. Proceedings of the National Academy of Sciences 120 30 (2023) e2305016120.","DOI":"10.1073\/pnas.2305016120"},{"key":"e_1_3_3_3_49_2","unstructured":"Taicheng Guo Xiuying Chen Yaqi Wang Ruidi Chang Shichao Pei Nitesh\u00a0V. Chawla Olaf Wiest and Xiangliang Zhang. 2024. Large Language Model based Multi-Agents: A Survey of Progress and Challenges. arxiv:https:\/\/arXiv.org\/abs\/2402.01680\u00a0[cs.CL] https:\/\/arxiv.org\/abs\/2402.01680"},{"key":"e_1_3_3_3_50_2","unstructured":"Shashank Gupta Vaishnavi Shrivastava A. Deshpande A. Kalyan Peter Clark Ashish Sabharwal and Tushar Khot. 2023. Bias Runs Deep: Implicit Reasoning Biases in Persona-Assigned LLMs. https:\/\/api.semanticscholar.org\/CorpusID:265050702"},{"key":"e_1_3_3_3_51_2","unstructured":"Andrew Halterman and Katherine\u00a0A. Keith. 2025. Codebook LLMs: Evaluating LLMs as Measurement Tools for Political Science Concepts. arxiv:https:\/\/arXiv.org\/abs\/2407.10747\u00a0[cs.CL] https:\/\/arxiv.org\/abs\/2407.10747"},{"key":"e_1_3_3_3_52_2","doi-asserted-by":"publisher","DOI":"10.1145\/3544548.3580688"},{"key":"e_1_3_3_3_53_2","first-page":"455","volume-title":"Women, science, and technology","author":"Haraway Donna","year":"2013","unstructured":"Donna Haraway. 2013. Situated knowledges: The science question in feminism and the privilege of partial perspective 1. In Women, science, and technology. Routledge, Milton Park, Oxfordshire, UK, 455\u2013472."},{"key":"e_1_3_3_3_54_2","doi-asserted-by":"publisher","DOI":"10.1109\/ACIIW59127.2023.10388188"},{"key":"e_1_3_3_3_55_2","doi-asserted-by":"crossref","unstructured":"John\u00a0J. Horton. 2023. Large Language Models as Simulated Economic Agents: What Can We Learn from Homo Silicus? arxiv:https:\/\/arXiv.org\/abs\/2301.07543\u00a0[econ.GN]","DOI":"10.3386\/w31122"},{"key":"e_1_3_3_3_56_2","doi-asserted-by":"publisher","unstructured":"Jinpeng Hu Tengteng Dong Gang Luo Hui Ma Peng Zou Xiao Sun Dan Guo Xun Yang and Meng Wang. 2024. PsycoLLM: Enhancing LLM for Psychological Understanding and Evaluation. 13\u00a0pages. 10.1109\/TCSS.2024.3497725","DOI":"10.1109\/TCSS.2024.3497725"},{"key":"e_1_3_3_3_57_2","unstructured":"Synthetic\u00a0Users Inc. 2024. Synthetic Users: user research without the headaches. https:\/\/web.archive.org\/web\/20240822224158\/https:\/\/www.syntheticusers.com\/. (Accessed on 08\/30\/2024)."},{"key":"e_1_3_3_3_58_2","unstructured":"OpinioAI International. 2024. AI Powered Research | OpinioAI. https:\/\/www.opinio.ai\/. (Accessed on 08\/30\/2024)."},{"key":"e_1_3_3_3_59_2","unstructured":"Julie Jiang and Emilio Ferrara. 2023. Social-LLM: Modeling User Behavior at Scale using Language Models and Social Network Data."},{"key":"e_1_3_3_3_60_2","unstructured":"Shivani Kapania Ruiyi Wang Toby Jia-Jun Li Tianshi Li and Hong Shen. 2024. \"I\u2019m categorizing LLM as a productivity tool\": Examining ethics of LLM use in HCI research practices. arxiv:https:\/\/arXiv.org\/abs\/2403.19876\u00a0[cs.HC] https:\/\/arxiv.org\/abs\/2403.19876"},{"key":"e_1_3_3_3_61_2","unstructured":"Luoma Ke Song Tong Peng Cheng and Kaiping Peng. 2024. Exploring the Frontiers of LLMs in Psychological Applications: A Comprehensive Review. arxiv:https:\/\/arXiv.org\/abs\/2401.01519\u00a0[cs.LG] https:\/\/arxiv.org\/abs\/2401.01519"},{"key":"e_1_3_3_3_62_2","doi-asserted-by":"publisher","DOI":"10.1145\/3677525.3678666"},{"key":"e_1_3_3_3_63_2","unstructured":"Geunwoo Kim Pierre Baldi and Stephen McAleer. 2024. Language models can solve computer tasks."},{"key":"e_1_3_3_3_64_2","unstructured":"Junsol Kim and Byungkyu Lee. 2023. AI-Augmented Surveys: Leveraging Large Language Models and Surveys for Opinion Prediction. arxiv:https:\/\/arXiv.org\/abs\/2305.09620\u00a0[cs.CL]"},{"key":"e_1_3_3_3_65_2","doi-asserted-by":"crossref","unstructured":"Shamika Klassen and Casey Fiesler. 2022. \u201cThis isn\u2019t your data friend\u201d: Black Twitter as a case study on research ethics for public data. Social Media+ Society 8 4 (2022) 20563051221144317.","DOI":"10.1177\/20563051221144317"},{"key":"e_1_3_3_3_66_2","unstructured":"Taja Kuzman Igor Mozeti\u010d and Nikola Ljube\u0161i\u0107. 2023. ChatGPT: Beginning of an End of Manual Linguistic Data Annotation? Use Case of Automatic Genre Identification. arxiv:https:\/\/arXiv.org\/abs\/2303.03953\u00a0[cs.CL]"},{"key":"e_1_3_3_3_67_2","doi-asserted-by":"publisher","DOI":"10.1145\/3613904.3642830"},{"key":"e_1_3_3_3_68_2","unstructured":"Ehsan Latif Luyang Fang Ping Ma and Xiaoming Zhai. 2024. Knowledge Distillation of LLM for Automatic Scoring of Science Education Assessments. arxiv:https:\/\/arXiv.org\/abs\/2312.15842\u00a0[cs.CL] https:\/\/arxiv.org\/abs\/2312.15842"},{"key":"e_1_3_3_3_69_2","doi-asserted-by":"publisher","DOI":"10.4324\/9780203481141"},{"key":"e_1_3_3_3_70_2","doi-asserted-by":"publisher","DOI":"10.1145\/2675133.2675147"},{"key":"e_1_3_3_3_71_2","unstructured":"Jung-Joo Lee et\u00a0al. 2012. Against method: The portability of method in human-centered design."},{"key":"e_1_3_3_3_72_2","unstructured":"Sam Levin. 2017. New AI can guess whether you\u2019re gay or straight from a photograph."},{"key":"e_1_3_3_3_73_2","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.findings-emnlp.311"},{"key":"e_1_3_3_3_74_2","series-title":"Proceedings of Machine Learning Research","first-page":"6565","volume-title":"Proceedings of the 38th International Conference on Machine Learning","volume":"139","author":"Liang Paul\u00a0Pu","year":"2021","unstructured":"Paul\u00a0Pu Liang, Chiyu Wu, Louis-Philippe Morency, and Ruslan Salakhutdinov. 2021. Towards Understanding and Mitigating Social Biases in Language Models. In Proceedings of the 38th International Conference on Machine Learning(Proceedings of Machine Learning Research, Vol.\u00a0139), Marina Meila and Tong Zhang (Eds.). PMLR, Virtual, 6565\u20136576. https:\/\/proceedings.mlr.press\/v139\/liang21a.html"},{"key":"e_1_3_3_3_75_2","unstructured":"Menglin Liu and Ge Shi. 2024. PoliPrompt: A High-Performance Cost-Effective LLM-Based Text Classification Framework for Political Science. arxiv:https:\/\/arXiv.org\/abs\/2409.01466\u00a0[cs.CL] https:\/\/arxiv.org\/abs\/2409.01466"},{"key":"e_1_3_3_3_76_2","doi-asserted-by":"publisher","DOI":"10.1145\/3613904.3642698"},{"key":"e_1_3_3_3_77_2","doi-asserted-by":"crossref","unstructured":"Ryan Louie Ananjan Nandi William Fang Cheng Chang Emma Brunskill and Diyi Yang. 2024. Roleplay-doh: Enabling Domain-Experts to Create LLM-simulated Patients via Eliciting and Adhering to Principles. arxiv:https:\/\/arXiv.org\/abs\/2407.00870\u00a0[cs.CL] https:\/\/arxiv.org\/abs\/2407.00870","DOI":"10.18653\/v1\/2024.emnlp-main.591"},{"key":"e_1_3_3_3_78_2","doi-asserted-by":"crossref","unstructured":"Andres M.\u00a0Bran Sam Cox Oliver Schilter Carlo Baldassari Andrew\u00a0D White and Philippe Schwaller. 2024. Augmenting large language models with chemistry tools. 11\u00a0pages.","DOI":"10.1038\/s42256-024-00832-8"},{"key":"e_1_3_3_3_79_2","doi-asserted-by":"publisher","DOI":"10.1115\/DETC2023-116838"},{"key":"e_1_3_3_3_80_2","doi-asserted-by":"crossref","unstructured":"Annette Markham. 2012. Fabrication as ethical practice: Qualitative inquiry in ambiguous internet contexts. Information Communication & Society 15 3 (2012) 334\u2013353.","DOI":"10.1080\/1369118X.2011.641993"},{"key":"e_1_3_3_3_81_2","doi-asserted-by":"crossref","unstructured":"Nora McDonald Sarita Schoenebeck and Andrea Forte. 2019. Reliability and inter-rater reliability in qualitative research: Norms and guidelines for CSCW and HCI practice. Proceedings of the ACM on human-computer interaction 3 CSCW (2019) 1\u201323.","DOI":"10.1145\/3359174"},{"key":"e_1_3_3_3_82_2","doi-asserted-by":"crossref","unstructured":"Milagros Miceli and Julian Posada. 2022. The Data-Production Dispositif. Proceedings of the ACM on Human-Computer Interaction 6 (2022) 1 \u2013 37. https:\/\/api.semanticscholar.org\/CorpusID:249017734","DOI":"10.1145\/3555561"},{"key":"e_1_3_3_3_83_2","unstructured":"Moin Nadeem Anna Bethke and Siva Reddy. 2020. StereoSet: Measuring stereotypical bias in pretrained language models. arxiv:https:\/\/arXiv.org\/abs\/2004.09456\u00a0[cs.CL] https:\/\/arxiv.org\/abs\/2004.09456"},{"key":"e_1_3_3_3_84_2","doi-asserted-by":"publisher","unstructured":"Peter Nagy and Gina Neff. 2024. Conjuring algorithms: Understanding the tech industry as stage magicians. New Media & Society 26 9 (2024) 4938\u20134954. 10.1177\/14614448241251789 arXiv:10.1177\/14614448241251789","DOI":"10.1177\/14614448241251789"},{"key":"e_1_3_3_3_85_2","doi-asserted-by":"publisher","unstructured":"James\u00a0L. Olive. 2014. Reflecting on the Tensions Between Emic and Etic Perspectives in Life History Research: Lessons Learned. 10.17169\/fqs-15.2.2072","DOI":"10.17169\/fqs-15.2.2072"},{"key":"e_1_3_3_3_86_2","doi-asserted-by":"publisher","DOI":"10.1007\/978-1-4939-0378-8"},{"key":"e_1_3_3_3_87_2","unstructured":"OpenAI. 2024. OpenAI and journalism | OpenAI. https:\/\/openai.com\/index\/openai-and-journalism\/. (Accessed on 09\/01\/2024)."},{"key":"e_1_3_3_3_88_2","doi-asserted-by":"publisher","DOI":"10.1145\/3586183.3606763"},{"key":"e_1_3_3_3_89_2","doi-asserted-by":"publisher","DOI":"10.1145\/3526113.3545616"},{"key":"e_1_3_3_3_90_2","doi-asserted-by":"crossref","unstructured":"Ethan Perez Sam Ringer Kamil\u0117 Luko\u0161i\u016bt\u0117 Karina Nguyen Edwin Chen Scott Heiner Craig Pettit Catherine Olsson Sandipan Kundu Saurav Kadavath Andy Jones Anna Chen Ben Mann Brian Israel Bryan Seethor Cameron McKinnon Christopher Olah Da Yan Daniela Amodei Dario Amodei Dawn Drain Dustin Li Eli Tran-Johnson Guro Khundadze Jackson Kernion James Landis Jamie Kerr Jared Mueller Jeeyoon Hyun Joshua Landau Kamal Ndousse Landon Goldberg Liane Lovitt Martin Lucas Michael Sellitto Miranda Zhang Neerav Kingsland Nelson Elhage Nicholas Joseph Noem\u00ed Mercado Nova DasSarma Oliver Rausch Robin Larson Sam McCandlish Scott Johnston Shauna Kravec Sheer\u00a0El Showk Tamera Lanham Timothy Telleen-Lawton Tom Brown Tom Henighan Tristan Hume Yuntao Bai Zac Hatfield-Dodds Jack Clark Samuel\u00a0R. Bowman Amanda Askell Roger Grosse Danny Hernandez Deep Ganguli Evan Hubinger Nicholas Schiefer and Jared Kaplan. 2022. Discovering Language Model Behaviors with Model-Written Evaluations. arxiv:https:\/\/arXiv.org\/abs\/2212.09251\u00a0[cs.CL] https:\/\/arxiv.org\/abs\/2212.09251","DOI":"10.18653\/v1\/2023.findings-acl.847"},{"key":"e_1_3_3_3_91_2","unstructured":"Colin Raffel Noam Shazeer Adam Roberts Katherine Lee Sharan Narang Michael Matena Yanqi Zhou Wei Li and Peter\u00a0J. Liu. 2020. Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer. Journal of Machine Learning Research 21 140 (2020) 1\u201367. http:\/\/jmlr.org\/papers\/v21\/20-074.html"},{"key":"e_1_3_3_3_92_2","doi-asserted-by":"crossref","unstructured":"Akshay Ravi Aaron Neinstein and Sara\u00a0G Murray. 2023. Large language models and medical education: Preparing for a rapid transformation in how trainees will learn to be doctors. ATS scholar 4 3 (2023) 282\u2013292.","DOI":"10.34197\/ats-scholar.2023-0036PS"},{"key":"e_1_3_3_3_93_2","unstructured":"Emma Roth. 2024. Google cut a deal with Reddit for AI training data - The Verge. https:\/\/www.theverge.com\/2024\/2\/22\/24080165\/google-reddit-ai-training-data. (Accessed on 09\/01\/2024)."},{"key":"e_1_3_3_3_94_2","doi-asserted-by":"publisher","DOI":"10.1145\/3411764.3445518"},{"key":"e_1_3_3_3_95_2","series-title":"Proceedings of Machine Learning Research","first-page":"29971","volume-title":"Proceedings of the 40th International Conference on Machine Learning","volume":"202","author":"Santurkar Shibani","year":"2023","unstructured":"Shibani Santurkar, Esin Durmus, Faisal Ladhak, Cinoo Lee, Percy Liang, and Tatsunori Hashimoto. 2023. Whose Opinions Do Language Models Reflect?. In Proceedings of the 40th International Conference on Machine Learning(Proceedings of Machine Learning Research, Vol.\u00a0202), Andreas Krause, Emma Brunskill, Kyunghyun Cho, Barbara Engelhardt, Sivan Sabato, and Jonathan Scarlett (Eds.). PMLR, Hawaii, USA, 29971\u201330004. https:\/\/proceedings.mlr.press\/v202\/santurkar23a.html"},{"key":"e_1_3_3_3_96_2","unstructured":"Sebastin Santy Jenny\u00a0T. Liang Ronan\u00a0Le Bras Katharina Reinecke and Maarten Sap. 2023. NLPositionality: Characterizing Design Biases of Datasets and Models. arxiv:https:\/\/arXiv.org\/abs\/2306.01943\u00a0[cs.CL] https:\/\/arxiv.org\/abs\/2306.01943"},{"key":"e_1_3_3_3_97_2","doi-asserted-by":"publisher","unstructured":"Jaromir Savelka Arav Agarwal Christopher Bogart Yifan Song and Majd Sakr. 2023. Can Generative Pre-trained Transformers (GPT) Pass Assessments in Higher Education Programming Courses? 7\u00a0pages. 10.1145\/3587102.3588792","DOI":"10.1145\/3587102.3588792"},{"key":"e_1_3_3_3_98_2","unstructured":"TA Schwandt. 1994. Constructivist interpretivist approaches to human inquiry."},{"key":"e_1_3_3_3_99_2","unstructured":"Melanie Sclar Yejin Choi Yulia Tsvetkov and Alane Suhr. 2024. Quantifying Language Models\u2019 Sensitivity to Spurious Features in Prompt Design or: How I learned to start worrying about prompt formatting. arxiv:https:\/\/arXiv.org\/abs\/2310.11324\u00a0[cs.CL] https:\/\/arxiv.org\/abs\/2310.11324"},{"key":"e_1_3_3_3_100_2","doi-asserted-by":"publisher","DOI":"10.1145\/3613904.3642414"},{"key":"e_1_3_3_3_101_2","doi-asserted-by":"publisher","DOI":"10.1145\/3600211.3604673"},{"key":"e_1_3_3_3_102_2","doi-asserted-by":"crossref","unstructured":"Arielle\u00a0Michal Silverman. 2015. The perils of playing blind: Problems with blindness simulation and a better way to teach about blindness. (2015).","DOI":"10.5241\/5-81"},{"key":"e_1_3_3_3_103_2","doi-asserted-by":"crossref","unstructured":"Arielle\u00a0M Silverman Jason\u00a0D Gwinn and Leaf Van\u00a0Boven. 2015. Stumbling in their shoes: Disability simulations reduce judged capabilities of disabled people. Social Psychological and Personality Science 6 4 (2015) 464\u2013471.","DOI":"10.1177\/1948550614559650"},{"key":"e_1_3_3_3_104_2","unstructured":"David Silverman. 2004. Qualitative research : theory method and practice. https:\/\/api.semanticscholar.org\/CorpusID:142594195"},{"key":"e_1_3_3_3_105_2","volume-title":"Qualitative literacy: A guide to evaluating ethnographic and interview research","author":"Small Mario\u00a0Luis","year":"2022","unstructured":"Mario\u00a0Luis Small and Jessica\u00a0McCrory Calarco. 2022. Qualitative literacy: A guide to evaluating ethnographic and interview research. Univ of California Press, California, USA."},{"key":"e_1_3_3_3_106_2","doi-asserted-by":"crossref","unstructured":"Robert Soden Austin Toombs and Michaelanne Thomas. 2024. Evaluating interpretive research in HCI. Interactions 31 1 (2024) 38\u201342.","DOI":"10.1145\/3633200"},{"key":"e_1_3_3_3_107_2","doi-asserted-by":"publisher","DOI":"10.1145\/1978942.1979042"},{"key":"e_1_3_3_3_108_2","unstructured":"Surendrabikram Thapa Usman Naseem and Mehwish Nasim. 2023. From humans to machines: can chatgpt-like llms effectively replace human annotators in nlp tasks."},{"key":"e_1_3_3_3_109_2","doi-asserted-by":"crossref","unstructured":"David\u00a0R Thomas. 2017. Feedback from research participants: are member checks useful in qualitative research? Qualitative research in psychology 14 1 (2017) 23\u201341.","DOI":"10.1080\/14780887.2016.1219435"},{"key":"e_1_3_3_3_110_2","doi-asserted-by":"publisher","unstructured":"Lindia Tjuatja Valerie Chen Tongshuang Wu Ameet Talwalkwar and Graham Neubig. 2024. Do LLMs Exhibit Human-like Response Biases? A Case Study in Survey Design. Transactions of the Association for Computational Linguistics 12 (09 2024) 1011\u20131026. 10.1162\/tacl_a_00685 arXiv:https:\/\/direct.mit.edu\/tacl\/article-pdf\/doi\/10.1162\/tacl_a_00685\/2468689\/tacl_a_00685.pdf","DOI":"10.1162\/tacl_a_00685"},{"key":"e_1_3_3_3_111_2","doi-asserted-by":"publisher","DOI":"10.7208\/chicago\/9780226849638.001.0001"},{"key":"e_1_3_3_3_112_2","unstructured":"Pranav\u00a0Narayanan Venkit Sanjana Gautam Ruchi Panchanadikar Ting-Hao\u00a0\u2019Kenneth\u2019 Huang and Shomir Wilson. 2023. Nationality Bias in Text Generation. arxiv:https:\/\/arXiv.org\/abs\/2302.02463\u00a0[cs.CL] https:\/\/arxiv.org\/abs\/2302.02463"},{"key":"e_1_3_3_3_113_2","unstructured":"Angelina Wang Jamie Morgenstern and John\u00a0P. Dickerson. 2024. Large language models should not replace human participants because they can misportray and flatten identity groups. arxiv:https:\/\/arXiv.org\/abs\/2402.01908\u00a0[cs.CY] https:\/\/arxiv.org\/abs\/2402.01908"},{"key":"e_1_3_3_3_114_2","unstructured":"Haochun Wang Sendong Zhao Zewen Qiang Nuwa Xi Bing Qin and Ting Liu. 2024. Beyond Direct Diagnosis: LLM-based Multi-Specialist Agent Consultation for Automatic Diagnosis. arxiv:https:\/\/arXiv.org\/abs\/2401.16107\u00a0[cs.CL] https:\/\/arxiv.org\/abs\/2401.16107"},{"key":"e_1_3_3_3_115_2","doi-asserted-by":"publisher","DOI":"10.1145\/3531146.3533088"},{"key":"e_1_3_3_3_116_2","doi-asserted-by":"crossref","unstructured":"Rebecca Willis. 2019. The use of composite narratives to present interview findings. Qualitative research 19 4 (2019) 471\u2013480.","DOI":"10.1177\/1468794118787711"},{"key":"e_1_3_3_3_117_2","unstructured":"Zhiheng Xi Wenxiang Chen Xin Guo Wei He Yiwen Ding Boyang Hong Ming Zhang Junzhe Wang Senjie Jin Enyu Zhou et\u00a0al. 2023. The rise and potential of large language model based agents: A survey."},{"key":"e_1_3_3_3_118_2","unstructured":"Yang Xiao Yi Cheng Jinlan Fu Jiashuo Wang Wenjie Li and Pengfei Liu. 2023. How far are we from believable AI agents? A framework for evaluating the believability of human behavior simulation."},{"key":"e_1_3_3_3_119_2","doi-asserted-by":"publisher","DOI":"10.1145\/3581754.3584136"},{"key":"e_1_3_3_3_120_2","unstructured":"Qisen Yang Zekun Wang Honghui Chen Shenzhi Wang Yifan Pu Xin Gao Wenhao Huang Shiji Song and Gao Huang. 2024. PsychoGAT: A Novel Psychological Measurement Paradigm through Interactive Fiction Games with LLM Agents. arxiv:https:\/\/arXiv.org\/abs\/2402.12326\u00a0[cs.CL] https:\/\/arxiv.org\/abs\/2402.12326"},{"key":"e_1_3_3_3_121_2","unstructured":"He Zhang Chuhao Wu Jingyi Xie Fiona Rubino Sydney Graver ChanMin Kim John\u00a0M Carroll and Jie Cai. 2024. When Qualitative Research Meets Large Language Model: Exploring the Potential of QualiGPT as a Tool for Qualitative Coding."},{"key":"e_1_3_3_3_122_2","doi-asserted-by":"publisher","DOI":"10.1145\/3586183.3606800"},{"key":"e_1_3_3_3_123_2","unstructured":"Zheyuan Zhang Daniel Zhang-Li Jifan Yu Linlu Gong Jinchang Zhou Zhiyuan Liu Lei Hou and Juanzi Li. 2024. Simulating classroom education with llm-empowered agents."},{"key":"e_1_3_3_3_124_2","unstructured":"Xuhui Zhou Hao Zhu Leena Mathur Ruohong Zhang Haofei Yu Zhengyang Qi Louis-Philippe Morency Yonatan Bisk Daniel Fried Graham Neubig and Maarten Sap. 2024. SOTOPIA: Interactive Evaluation for Social Intelligence in Language Agents. arxiv:https:\/\/arXiv.org\/abs\/2310.11667\u00a0[cs.AI] https:\/\/arxiv.org\/abs\/2310.11667"},{"key":"e_1_3_3_3_125_2","doi-asserted-by":"crossref","unstructured":"Caleb Ziems William Held Omar Shaikh Jiaao Chen Zhehao Zhang and Diyi Yang. 2024. Can large language models transform computational social science? Computational Linguistics 50 1 (2024) 237\u2013291.","DOI":"10.1162\/coli_a_00502"},{"key":"e_1_3_3_3_126_2","doi-asserted-by":"crossref","unstructured":"Michael Zimmer. 2020. \u201cBut the data is already public\u201d: on the ethics of research in Facebook. 229\u2013241\u00a0pages.","DOI":"10.4324\/9781003075011-17"}],"event":{"name":"CHI 2025: CHI Conference on Human Factors in Computing Systems","location":"Yokohama Japan","acronym":"CHI '25","sponsor":["SIGCHI ACM Special Interest Group on Computer-Human Interaction"]},"container-title":["Proceedings of the 2025 CHI Conference on Human Factors in Computing Systems"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3706598.3713220","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3706598.3713220","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,7,4]],"date-time":"2025-07-04T05:45:18Z","timestamp":1751607918000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3706598.3713220"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,4,25]]},"references-count":125,"alternative-id":["10.1145\/3706598.3713220","10.1145\/3706598"],"URL":"https:\/\/doi.org\/10.1145\/3706598.3713220","relation":{},"subject":[],"published":{"date-parts":[[2025,4,25]]},"assertion":[{"value":"2025-04-25","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}