{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,6,13]],"date-time":"2026-06-13T07:05:03Z","timestamp":1781334303471,"version":"3.54.1"},"publisher-location":"New York, NY, USA","reference-count":52,"publisher":"ACM","license":[{"start":{"date-parts":[[2025,4,25]],"date-time":"2025-04-25T00:00:00Z","timestamp":1745539200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"DOI":"10.13039\/100006785","name":"Google","doi-asserted-by":"publisher","award":[""],"award-info":[{"award-number":[""]}],"id":[{"id":"10.13039\/100006785","id-type":"DOI","asserted-by":"publisher"}]},{"name":"OpenAI","award":[""],"award-info":[{"award-number":[""]}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2025,4,26]]},"DOI":"10.1145\/3706599.3706690","type":"proceedings-article","created":{"date-parts":[[2025,4,23]],"date-time":"2025-04-23T20:44:19Z","timestamp":1745441059000},"page":"1-10","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":7,"title":["LLMs as Workers in Human-Computational Algorithms? Replicating Crowdsourcing Pipelines with LLMs"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-1630-0588","authenticated-orcid":false,"given":"Tongshuang","family":"Wu","sequence":"first","affiliation":[{"name":"Carnegie Mellon University, Pittsburgh, PA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-7271-9100","authenticated-orcid":false,"given":"Haiyi","family":"Zhu","sequence":"additional","affiliation":[{"name":"Carnegie Mellon University, Pittsburgh, PA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0006-7684-4981","authenticated-orcid":false,"given":"Maya","family":"Albayrak","sequence":"additional","affiliation":[{"name":"Carnegie Mellon University, Pittsburgh, PA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0003-0445-1810","authenticated-orcid":false,"given":"Alexis","family":"Axon","sequence":"additional","affiliation":[{"name":"Carnegie Mellon University, Pittsburgh, PA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-1368-1111","authenticated-orcid":false,"given":"Amanda","family":"Bertsch","sequence":"additional","affiliation":[{"name":"Carnegie Mellon University, Pittsburgh, PA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0007-0380-8397","authenticated-orcid":false,"given":"Wenxing","family":"Deng","sequence":"additional","affiliation":[{"name":"Carnegie Mellon University, Pittsburgh, PA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0004-5464-4546","authenticated-orcid":false,"given":"Ziqi","family":"Ding","sequence":"additional","affiliation":[{"name":"Carnegie Mellon University, Pittsburgh, PA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-8303-4649","authenticated-orcid":false,"given":"Boyuan","family":"Guo","sequence":"additional","affiliation":[{"name":"Carnegie Mellon University, Pittsburgh, PA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0004-0309-3393","authenticated-orcid":false,"given":"Sireesh","family":"Gururaja","sequence":"additional","affiliation":[{"name":"Carnegie Mellon University, Pittsburgh, PA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-1504-7640","authenticated-orcid":false,"given":"Tzu-Sheng","family":"Kuo","sequence":"additional","affiliation":[{"name":"Carnegie Mellon University, Pittsburgh, PA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-6722-9959","authenticated-orcid":false,"given":"Jenny T","family":"Liang","sequence":"additional","affiliation":[{"name":"Carnegie Mellon University, Pittsburgh, PA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0003-8755-649X","authenticated-orcid":false,"given":"Ryan","family":"Liu","sequence":"additional","affiliation":[{"name":"Carnegie Mellon University, Pittsburgh, PA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0001-0705-8390","authenticated-orcid":false,"given":"Ihita","family":"Mandal","sequence":"additional","affiliation":[{"name":"Carnegie Mellon University, Pittsburgh, PA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-2809-534X","authenticated-orcid":false,"given":"Jeremiah","family":"Milbauer","sequence":"additional","affiliation":[{"name":"Carnegie Mellon University, Pittsburgh, PA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0009-2660-9539","authenticated-orcid":false,"given":"Xiaolin","family":"Ni","sequence":"additional","affiliation":[{"name":"Carnegie Mellon University, Pittsburgh, PA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0008-1617-9566","authenticated-orcid":false,"given":"Namrata","family":"Padmanabhan","sequence":"additional","affiliation":[{"name":"Carnegie Mellon University, Pittsburgh, PA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0005-6448-562X","authenticated-orcid":false,"given":"Subhashini","family":"Ramkumar","sequence":"additional","affiliation":[{"name":"Carnegie Mellon University, Pittsburgh, PA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0003-9946-9422","authenticated-orcid":false,"given":"Alexis","family":"Sudjianto","sequence":"additional","affiliation":[{"name":"Carnegie Mellon University, Pittsburgh, PA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-0896-992X","authenticated-orcid":false,"given":"Jordan","family":"Taylor","sequence":"additional","affiliation":[{"name":"Carnegie Mellon University, Pittsburgh, PA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0006-1801-6061","authenticated-orcid":false,"given":"Ying-Jui","family":"Tseng","sequence":"additional","affiliation":[{"name":"Carnegie Mellon University, Pittsburgh, PA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0002-4189-8307","authenticated-orcid":false,"given":"Patricia","family":"Vaidos","sequence":"additional","affiliation":[{"name":"Carnegie Mellon University, Pittsburgh, PA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0009-1875-5827","authenticated-orcid":false,"given":"Zhijin","family":"Wu","sequence":"additional","affiliation":[{"name":"Carnegie Mellon University, Pittsburgh, PA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0008-1671-9470","authenticated-orcid":false,"given":"Wei","family":"Wu","sequence":"additional","affiliation":[{"name":"Carnegie Mellon University, Pittsburgh, PA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-5016-7296","authenticated-orcid":false,"given":"Chenyang","family":"Yang","sequence":"additional","affiliation":[{"name":"Carnegie Mellon University, Pittsburgh, PA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2025,4,25]]},"reference":[{"key":"e_1_3_3_2_2_2","unstructured":"Lisa\u00a0P Argyle Ethan\u00a0C Busby Nancy Fulda Joshua Gubler Christopher Rytting and David Wingate. 2022. Out of One Many: Using Language Models to Simulate Human Samples. ArXiv preprint abs\/2209.06899 (2022). https:\/\/arxiv.org\/abs\/2209.06899"},{"key":"e_1_3_3_2_3_2","doi-asserted-by":"publisher","DOI":"10.1145\/3411764.3445717"},{"key":"e_1_3_3_2_4_2","doi-asserted-by":"crossref","unstructured":"Michael\u00a0S Bernstein. 2013. Crowd-powered systems. KI-K\u00fcnstliche Intelligenz 27 (2013) 69\u201373.","DOI":"10.1007\/s13218-012-0233-0"},{"key":"e_1_3_3_2_5_2","doi-asserted-by":"publisher","DOI":"10.1145\/1866029.1866078"},{"key":"e_1_3_3_2_6_2","doi-asserted-by":"crossref","unstructured":"George\u00a0EP Box. 1976. Science and statistics. J. Amer. Statist. Assoc. 71 356 (1976) 791\u2013799.","DOI":"10.1080\/01621459.1976.10480949"},{"key":"e_1_3_3_2_7_2","volume-title":"Advances in Neural Information Processing Systems 33: Annual Conference on Neural Information Processing Systems 2020, NeurIPS 2020, December 6-12, 2020, virtual","author":"Brown Tom\u00a0B.","year":"2020","unstructured":"Tom\u00a0B. Brown, Benjamin Mann, Nick Ryder, Melanie Subbiah, Jared Kaplan, Prafulla Dhariwal, Arvind Neelakantan, Pranav Shyam, Girish Sastry, Amanda Askell, Sandhini Agarwal, Ariel Herbert-Voss, Gretchen Krueger, Tom Henighan, Rewon Child, Aditya Ramesh, Daniel\u00a0M. Ziegler, Jeffrey Wu, Clemens Winter, Christopher Hesse, Mark Chen, Eric Sigler, Mateusz Litwin, Scott Gray, Benjamin Chess, Jack Clark, Christopher Berner, Sam McCandlish, Alec Radford, Ilya Sutskever, and Dario Amodei. 2020. Language Models are Few-Shot Learners. In Advances in Neural Information Processing Systems 33: Annual Conference on Neural Information Processing Systems 2020, NeurIPS 2020, December 6-12, 2020, virtual, Hugo Larochelle, Marc\u2019Aurelio Ranzato, Raia Hadsell, Maria-Florina Balcan, and Hsuan-Tien Lin (Eds.). https:\/\/proceedings.neurips.cc\/paper\/2020\/hash\/1457c0d6bfcb4967418bfb8ac142f64a-Abstract.html"},{"key":"e_1_3_3_2_8_2","unstructured":"S\u00e9bastien Bubeck Varun Chandrasekaran Ronen Eldan Johannes Gehrke Eric Horvitz Ece Kamar Peter Lee Yin\u00a0Tat Lee Yuanzhi Li Scott Lundberg et\u00a0al. 2023. Sparks of artificial general intelligence: Early experiments with gpt-4. ArXiv preprint abs\/2303.12712 (2023). https:\/\/arxiv.org\/abs\/2303.12712"},{"key":"e_1_3_3_2_9_2","doi-asserted-by":"publisher","DOI":"10.1145\/2702123.2702146"},{"key":"e_1_3_3_2_10_2","unstructured":"Lydia\u00a0B Chilton James\u00a0A Landay and Daniel\u00a0S Weld. 2016. Humortools: A microtask workflow for writing news satire. El Paso Texas: ACM (2016)."},{"key":"e_1_3_3_2_11_2","doi-asserted-by":"publisher","DOI":"10.1145\/2470654.2466265"},{"key":"e_1_3_3_2_12_2","unstructured":"Tyna Eloundou Sam Manning Pamela Mishkin and Daniel Rock. 2023. Gpts are gpts: An early look at the labor market impact potential of large language models. ArXiv preprint abs\/2303.10130 (2023). https:\/\/arxiv.org\/abs\/2303.10130"},{"key":"e_1_3_3_2_13_2","doi-asserted-by":"crossref","unstructured":"Samuel\u00a0J Gershman Eric\u00a0J Horvitz and Joshua\u00a0B Tenenbaum. 2015. Computational rationality: A converging paradigm for intelligence in brains minds and machines. Science 349 6245 (2015) 273\u2013278.","DOI":"10.1126\/science.aac6076"},{"key":"e_1_3_3_2_14_2","unstructured":"Fabrizio Gilardi Meysam Alizadeh and Ma\u00ebl Kubli. 2023. Chatgpt outperforms crowd-workers for text-annotation tasks. ArXiv preprint abs\/2303.15056 (2023). https:\/\/arxiv.org\/abs\/2303.15056"},{"key":"e_1_3_3_2_15_2","unstructured":"Madeleine Grunde-McLaughlin Michelle\u00a0S Lam Ranjay Krishna Daniel\u00a0S Weld and Jeffrey Heer. 2023. Designing LLM chains by adapting techniques from crowdsourcing workflows. ArXiv preprint abs\/2312.11681 (2023). https:\/\/arxiv.org\/abs\/2312.11681"},{"key":"e_1_3_3_2_16_2","doi-asserted-by":"publisher","DOI":"10.1145\/3544548.3580688"},{"key":"e_1_3_3_2_17_2","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D15-1076"},{"key":"e_1_3_3_2_18_2","volume-title":"Advances in Neural Information Processing Systems 36: Annual Conference on Neural Information Processing Systems 2023, NeurIPS 2023, New Orleans, LA, USA, December 10 - 16, 2023","author":"Jiang Guangyuan","year":"2023","unstructured":"Guangyuan Jiang, Manjie Xu, Song-Chun Zhu, Wenjuan Han, Chi Zhang, and Yixin Zhu. 2023. Evaluating and Inducing Personality in Pre-trained Language Models. In Advances in Neural Information Processing Systems 36: Annual Conference on Neural Information Processing Systems 2023, NeurIPS 2023, New Orleans, LA, USA, December 10 - 16, 2023, Alice Oh, Tristan Naumann, Amir Globerson, Kate Saenko, Moritz Hardt, and Sergey Levine (Eds.). http:\/\/papers.nips.cc\/paper_files\/paper\/2023\/hash\/21f7b745f73ce0d1f9bcea7f40b1388e-Abstract-Conference.html"},{"key":"e_1_3_3_2_19_2","doi-asserted-by":"publisher","DOI":"10.1145\/2998181.2998196"},{"key":"e_1_3_3_2_20_2","doi-asserted-by":"publisher","DOI":"10.1145\/2047196.2047202"},{"key":"e_1_3_3_2_21_2","doi-asserted-by":"crossref","unstructured":"Jan Koco\u0144 Igor Cichecki Oliwier Kaszyca Mateusz Kochanek Dominika Szyd\u0142o Joanna Baran Julita Bielaniewicz Marcin Gruza Arkadiusz Janz Kamil Kanclerz et\u00a0al. 2023. ChatGPT: Jack of all trades master of none. Information Fusion (2023) 101861.","DOI":"10.1016\/j.inffus.2023.101861"},{"key":"e_1_3_3_2_22_2","doi-asserted-by":"publisher","DOI":"10.1145\/2145204.2145354"},{"key":"e_1_3_3_2_23_2","doi-asserted-by":"publisher","DOI":"10.21437\/SemDial.2017-5"},{"key":"e_1_3_3_2_24_2","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v25i1.8092"},{"key":"e_1_3_3_2_25_2","doi-asserted-by":"publisher","DOI":"10.1609\/hcomp.v2i1.13167"},{"key":"e_1_3_3_2_26_2","doi-asserted-by":"publisher","DOI":"10.1145\/1837885.1837907"},{"key":"e_1_3_3_2_27_2","doi-asserted-by":"publisher","DOI":"10.1145\/3544548.3580817"},{"key":"e_1_3_3_2_28_2","series-title":"Proceedings of Machine Learning Research","first-page":"22631","volume-title":"International Conference on Machine Learning, ICML 2023, 23-29 July 2023, Honolulu, Hawaii, USA","volume":"202","author":"Longpre Shayne","year":"2023","unstructured":"Shayne Longpre, Le Hou, Tu Vu, Albert Webson, Hyung\u00a0Won Chung, Yi Tay, Denny Zhou, Quoc\u00a0V. Le, Barret Zoph, Jason Wei, and Adam Roberts. 2023. The Flan Collection: Designing Data and Methods for Effective Instruction Tuning. In International Conference on Machine Learning, ICML 2023, 23-29 July 2023, Honolulu, Hawaii, USA(Proceedings of Machine Learning Research, Vol.\u00a0202), Andreas Krause, Emma Brunskill, Kyunghyun Cho, Barbara Engelhardt, Sivan Sabato, and Jonathan Scarlett (Eds.). PMLR, 22631\u201322648. https:\/\/proceedings.mlr.press\/v202\/longpre23a.html"},{"key":"e_1_3_3_2_29_2","unstructured":"Qianou Ma Tongshuang Wu and Kenneth Koedinger. 2023. Is AI the better programming partner? Human-Human Pair Programming vs. Human-AI pAIr Programming. arxiv:https:\/\/arXiv.org\/abs\/2306.05153\u00a0[cs.HC]"},{"key":"e_1_3_3_2_30_2","volume-title":"Advances in Neural Information Processing Systems 36: Annual Conference on Neural Information Processing Systems 2023, NeurIPS 2023, New Orleans, LA, USA, December 10 - 16, 2023","author":"Madaan Aman","year":"2023","unstructured":"Aman Madaan, Niket Tandon, Prakhar Gupta, Skyler Hallinan, Luyu Gao, Sarah Wiegreffe, Uri Alon, Nouha Dziri, Shrimai Prabhumoye, Yiming Yang, Shashank Gupta, Bodhisattwa\u00a0Prasad Majumder, Katherine Hermann, Sean Welleck, Amir Yazdanbakhsh, and Peter Clark. 2023. Self-Refine: Iterative Refinement with Self-Feedback. In Advances in Neural Information Processing Systems 36: Annual Conference on Neural Information Processing Systems 2023, NeurIPS 2023, New Orleans, LA, USA, December 10 - 16, 2023, Alice Oh, Tristan Naumann, Amir Globerson, Kate Saenko, Moritz Hardt, and Sergey Levine (Eds.). http:\/\/papers.nips.cc\/paper_files\/paper\/2023\/hash\/91edff07232fb1b55a505a9e9f6c0ff3-Abstract-Conference.html"},{"key":"e_1_3_3_2_31_2","doi-asserted-by":"publisher","DOI":"10.1017\/CBO9780511626388"},{"key":"e_1_3_3_2_32_2","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.emnlp-main.734"},{"key":"e_1_3_3_2_33_2","unstructured":"OpenAI. 2023. GPT-4 Technical Report. arxiv:https:\/\/arXiv.org\/abs\/2303.08774\u00a0[cs.CL]"},{"key":"e_1_3_3_2_34_2","volume-title":"Advances in Neural Information Processing Systems 35: Annual Conference on Neural Information Processing Systems 2022, NeurIPS 2022, New Orleans, LA, USA, November 28 - December 9, 2022","author":"Ouyang Long","year":"2022","unstructured":"Long Ouyang, Jeffrey Wu, Xu Jiang, Diogo Almeida, Carroll\u00a0L. Wainwright, Pamela Mishkin, Chong Zhang, Sandhini Agarwal, Katarina Slama, Alex Ray, John Schulman, Jacob Hilton, Fraser Kelton, Luke Miller, Maddie Simens, Amanda Askell, Peter Welinder, Paul\u00a0F. Christiano, Jan Leike, and Ryan Lowe. 2022. Training language models to follow instructions with human feedback. In Advances in Neural Information Processing Systems 35: Annual Conference on Neural Information Processing Systems 2022, NeurIPS 2022, New Orleans, LA, USA, November 28 - December 9, 2022, Sanmi Koyejo, S.\u00a0Mohamed, A.\u00a0Agarwal, Danielle Belgrave, K.\u00a0Cho, and A.\u00a0Oh (Eds.). http:\/\/papers.nips.cc\/paper_files\/paper\/2022\/hash\/b1efde53be364a73914f58805a001731-Abstract-Conference.html"},{"key":"e_1_3_3_2_35_2","doi-asserted-by":"crossref","unstructured":"Peter Pirolli and Stuart Card. 1999. Information foraging. Psychological review 106 4 (1999) 643.","DOI":"10.1037\/\/0033-295X.106.4.643"},{"key":"e_1_3_3_2_36_2","unstructured":"Silviu Pitis Michael\u00a0R Zhang Andrew Wang and Jimmy Ba. 2023. Boosted Prompt Ensembles for Large Language Models. ArXiv preprint abs\/2304.05970 (2023). https:\/\/arxiv.org\/abs\/2304.05970"},{"key":"e_1_3_3_2_37_2","unstructured":"Aditya Ramesh Prafulla Dhariwal Alex Nichol Casey Chu and Mark Chen. 2022. Hierarchical text-conditional image generation with clip latents. ArXiv preprint abs\/2204.06125 (2022). https:\/\/arxiv.org\/abs\/2204.06125"},{"key":"e_1_3_3_2_38_2","doi-asserted-by":"crossref","unstructured":"Daniela Retelny Michael\u00a0S Bernstein and Melissa\u00a0A Valentine. 2017. No workflow can ever be enough: How crowdsourcing workflows constrain complex work. Proceedings of the ACM on Human-Computer Interaction 1 CSCW (2017) 1\u201323.","DOI":"10.1145\/3134724"},{"key":"e_1_3_3_2_39_2","unstructured":"Marco\u00a0Tulio Ribeiro. 2023. Testing language models (and prompts) like we test software. Medium (2023). https:\/\/towardsdatascience.com\/testing-large-language-models-like-we-test-software-92745d28a359"},{"key":"e_1_3_3_2_40_2","series-title":"Proceedings of Machine Learning Research","first-page":"29971","volume-title":"International Conference on Machine Learning, ICML 2023, 23-29 July 2023, Honolulu, Hawaii, USA","volume":"202","author":"Santurkar Shibani","year":"2023","unstructured":"Shibani Santurkar, Esin Durmus, Faisal Ladhak, Cinoo Lee, Percy Liang, and Tatsunori Hashimoto. 2023. Whose Opinions Do Language Models Reflect?. In International Conference on Machine Learning, ICML 2023, 23-29 July 2023, Honolulu, Hawaii, USA(Proceedings of Machine Learning Research, Vol.\u00a0202), Andreas Krause, Emma Brunskill, Kyunghyun Cho, Barbara Engelhardt, Sivan Sabato, and Jonathan Scarlett (Eds.). PMLR, 29971\u201330004. https:\/\/proceedings.mlr.press\/v202\/santurkar23a.html"},{"key":"e_1_3_3_2_41_2","doi-asserted-by":"publisher","DOI":"10.5555\/6245"},{"key":"e_1_3_3_2_42_2","volume-title":"Advances in Neural Information Processing Systems 33: Annual Conference on Neural Information Processing Systems 2020, NeurIPS 2020, December 6-12, 2020, virtual","author":"Stiennon Nisan","year":"2020","unstructured":"Nisan Stiennon, Long Ouyang, Jeffrey Wu, Daniel\u00a0M. Ziegler, Ryan Lowe, Chelsea Voss, Alec Radford, Dario Amodei, and Paul\u00a0F. Christiano. 2020. Learning to summarize with human feedback. In Advances in Neural Information Processing Systems 33: Annual Conference on Neural Information Processing Systems 2020, NeurIPS 2020, December 6-12, 2020, virtual, Hugo Larochelle, Marc\u2019Aurelio Ranzato, Raia Hadsell, Maria-Florina Balcan, and Hsuan-Tien Lin (Eds.). https:\/\/proceedings.neurips.cc\/paper\/2020\/hash\/1f89885d556929e98d3ef9b86448f951-Abstract.html"},{"key":"e_1_3_3_2_43_2","unstructured":"Petter T\u00f6rnberg. 2023. Chatgpt-4 outperforms experts and crowd workers in annotating political twitter messages with zero-shot learning. ArXiv preprint abs\/2304.06588 (2023). https:\/\/arxiv.org\/abs\/2304.06588"},{"key":"e_1_3_3_2_44_2","unstructured":"Veniamin Veselovsky Manoel\u00a0Horta Ribeiro and Robert West. 2023. Artificial Artificial Artificial Intelligence: Crowd Workers Widely Use Large Language Models for Text Production Tasks. ArXiv preprint abs\/2306.07899 (2023). https:\/\/arxiv.org\/abs\/2306.07899"},{"key":"e_1_3_3_2_45_2","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2022.naacl-main.339"},{"key":"e_1_3_3_2_46_2","volume-title":"The Eleventh International Conference on Learning Representations, ICLR 2023, Kigali, Rwanda, May 1-5, 2023","author":"Wang Xuezhi","year":"2023","unstructured":"Xuezhi Wang, Jason Wei, Dale Schuurmans, Quoc\u00a0V. Le, Ed\u00a0H. Chi, Sharan Narang, Aakanksha Chowdhery, and Denny Zhou. 2023. Self-Consistency Improves Chain of Thought Reasoning in Language Models. In The Eleventh International Conference on Learning Representations, ICLR 2023, Kigali, Rwanda, May 1-5, 2023. OpenReview.net. https:\/\/openreview.net\/pdf?id=1PL1NIMMrw"},{"key":"e_1_3_3_2_47_2","doi-asserted-by":"crossref","unstructured":"Yun Wang Zhitao Hou Leixian Shen Tongshuang Wu Jiaqi Wang He Huang Haidong Zhang and Dongmei Zhang. 2022. Towards natural language-based visualization authoring. IEEE Transactions on Visualization and Computer Graphics 29 1 (2022) 1222\u20131232.","DOI":"10.1109\/TVCG.2022.3209357"},{"key":"e_1_3_3_2_48_2","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2023.findings-emnlp.514"},{"key":"e_1_3_3_2_49_2","doi-asserted-by":"publisher","DOI":"10.1145\/3491101.3519729"},{"key":"e_1_3_3_2_50_2","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2021.acl-long.523"},{"key":"e_1_3_3_2_51_2","doi-asserted-by":"publisher","DOI":"10.1145\/3491102.3517582"},{"key":"e_1_3_3_2_52_2","volume-title":"Advances in Neural Information Processing Systems 36: Annual Conference on Neural Information Processing Systems 2023, NeurIPS 2023, New Orleans, LA, USA, December 10 - 16, 2023","author":"Wu Zeqiu","year":"2023","unstructured":"Zeqiu Wu, Yushi Hu, Weijia Shi, Nouha Dziri, Alane Suhr, Prithviraj Ammanabrolu, Noah\u00a0A. Smith, Mari Ostendorf, and Hannaneh Hajishirzi. 2023. Fine-Grained Human Feedback Gives Better Rewards for Language Model Training. In Advances in Neural Information Processing Systems 36: Annual Conference on Neural Information Processing Systems 2023, NeurIPS 2023, New Orleans, LA, USA, December 10 - 16, 2023, Alice Oh, Tristan Naumann, Amir Globerson, Kate Saenko, Moritz Hardt, and Sergey Levine (Eds.). http:\/\/papers.nips.cc\/paper_files\/paper\/2023\/hash\/b8c90b65739ae8417e61eadb521f63d5-Abstract-Conference.html"},{"key":"e_1_3_3_2_53_2","volume-title":"Advances in Neural Information Processing Systems 36: Annual Conference on Neural Information Processing Systems 2023, NeurIPS 2023, New Orleans, LA, USA, December 10 - 16, 2023","author":"Yao Shunyu","year":"2023","unstructured":"Shunyu Yao, Dian Yu, Jeffrey Zhao, Izhak Shafran, Tom Griffiths, Yuan Cao, and Karthik Narasimhan. 2023. Tree of Thoughts: Deliberate Problem Solving with Large Language Models. In Advances in Neural Information Processing Systems 36: Annual Conference on Neural Information Processing Systems 2023, NeurIPS 2023, New Orleans, LA, USA, December 10 - 16, 2023, Alice Oh, Tristan Naumann, Amir Globerson, Kate Saenko, Moritz Hardt, and Sergey Levine (Eds.). http:\/\/papers.nips.cc\/paper_files\/paper\/2023\/hash\/271db9922b8d1f4dd7aaef84ed5ac703-Abstract-Conference.html"}],"event":{"name":"CHI EA '25: Extended Abstracts of the CHI Conference on Human Factors in Computing Systems","location":"Yokohama Japan","acronym":"CHI EA '25","sponsor":["SIGCHI ACM Special Interest Group on Computer-Human Interaction"]},"container-title":["Proceedings of the Extended Abstracts of the CHI Conference on Human Factors in Computing Systems"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3706599.3706690","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3706599.3706690","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,19]],"date-time":"2025-06-19T01:57:12Z","timestamp":1750298232000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3706599.3706690"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,4,25]]},"references-count":52,"alternative-id":["10.1145\/3706599.3706690","10.1145\/3706599"],"URL":"https:\/\/doi.org\/10.1145\/3706599.3706690","relation":{},"subject":[],"published":{"date-parts":[[2025,4,25]]},"assertion":[{"value":"2025-04-25","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}