{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,5,6]],"date-time":"2026-05-06T05:40:53Z","timestamp":1778046053978,"version":"3.51.4"},"reference-count":78,"publisher":"Springer Science and Business Media LLC","issue":"9","license":[{"start":{"date-parts":[[2025,8,8]],"date-time":"2025-08-08T00:00:00Z","timestamp":1754611200000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2025,8,8]],"date-time":"2025-08-08T00:00:00Z","timestamp":1754611200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Nat Comput Sci"],"DOI":"10.1038\/s43588-025-00843-4","type":"journal-article","created":{"date-parts":[[2025,8,8]],"date-time":"2025-08-08T09:04:40Z","timestamp":1754643880000},"page":"737-744","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":3,"title":["Arti-\u2018fickle\u2019 intelligence: using LLMs as a tool for inference in the political and social sciences"],"prefix":"10.1038","volume":"5","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-3109-2537","authenticated-orcid":false,"given":"Lisa P.","family":"Argyle","sequence":"first","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0002-8931-6348","authenticated-orcid":false,"given":"Ethan C.","family":"Busby","sequence":"additional","affiliation":[]},{"given":"Joshua R.","family":"Gubler","sequence":"additional","affiliation":[]},{"given":"Bryce","family":"Hepner","sequence":"additional","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0009-0004-1210-2538","authenticated-orcid":false,"given":"Alex","family":"Lyman","sequence":"additional","affiliation":[]},{"given":"David","family":"Wingate","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2025,8,8]]},"reference":[{"key":"843_CR1","doi-asserted-by":"publisher","unstructured":"Meincke, L., Girotra, K., Nave, G., Terwiesch, C. & Ulrich, K. T. Using large language models for idea generation in innovation. Preprint at https:\/\/doi.org\/10.2139\/ssrn.4526071 (2024).","DOI":"10.2139\/ssrn.4526071"},{"key":"843_CR2","unstructured":"Si, C., Yang, D. & Hashimoto, T. Can LLMs generate novel research ideas? A large-scale human study with 100+ NLP researchers. In 13th International Conference on Learning Representations (eds Yue, Y. et al.) 94003\u201394092 (ICLR, 2025); https:\/\/proceedings.iclr.cc\/paper_files\/paper\/2025\/file\/ea94957d81b1c1caf87ef5319fa6b467-Paper-Conference.pdf"},{"key":"843_CR3","unstructured":"Schmidgall, S. et al. Agent laboratory: using LLM agents as research assistants. Preprint at https:\/\/arxiv.org\/abs\/2501.04227 (2025)."},{"key":"843_CR4","unstructured":"Agarwal, S. et al. LitLLMs, LLMs for literature review: are we there yet? Preprint at https:\/\/arxiv.org\/abs\/2412.15249 (2025)."},{"key":"843_CR5","doi-asserted-by":"publisher","first-page":"e2723","DOI":"10.1002\/smr.2723","volume":"37","author":"M Nejjar","year":"2025","unstructured":"Nejjar, M., Zacharias, L., Stiehle, F. & Weber, I. LLMs for science: usage for code generation and data analysis. J. Softw. Evol. Process 37, e2723 (2025).","journal-title":"J. Softw. Evol. Process"},{"key":"843_CR6","doi-asserted-by":"publisher","unstructured":"T\u00f6rnberg, P. Large language models outperform expert coders and supervised classifiers at annotating political social media messages. Soc. Sci. Comput. Rev. https:\/\/doi.org\/10.1177\/08944393241286471 (2024).","DOI":"10.1177\/08944393241286471"},{"key":"843_CR7","doi-asserted-by":"publisher","first-page":"264","DOI":"10.1017\/psrm.2024.64","volume":"13","author":"JT Ornstein","year":"2025","unstructured":"Ornstein, J. T., Blasingame, E. N. & Truscott, J. S. How to train your stochastic parrot: large language models for political texts. Political Sci. Res. Methods 13, 264\u2013281 (2025).","journal-title":"Political Sci. Res. Methods"},{"key":"843_CR8","doi-asserted-by":"publisher","first-page":"205316802412362","DOI":"10.1177\/20531680241236239","volume":"11","author":"M Heseltine","year":"2024","unstructured":"Heseltine, M. & Clemm von Hohenberg, B. Large language models as a substitute for human experts in annotating political text. Res. Politics 11, 20531680241236239 (2024).","journal-title":"Res. Politics"},{"key":"843_CR9","unstructured":"Wang, Y. LLMs in political science: heralding a new era of visual analysis. Preprint at https:\/\/arxiv.org\/abs\/2403.00154 (2024)."},{"key":"843_CR10","unstructured":"Rytting, C. et al. Towards coding social science datasets with language models. Preprint at https:\/\/arxiv.org\/abs\/2306.02177 (2023)."},{"key":"843_CR11","doi-asserted-by":"publisher","first-page":"1036","DOI":"10.1017\/S0003055424000819","volume":"119","author":"YR VELEZ","year":"2025","unstructured":"VELEZ, Y. R. & LIU, P. Confronting core issues: a critical assessment of attitude polarization using tailored experiments. Am. Political Sci. Rev. 119, 1036\u20131053 (2025).","journal-title":"Am. Political Sci. Rev."},{"key":"843_CR12","doi-asserted-by":"publisher","first-page":"e2311627120","DOI":"10.1073\/pnas.2311627120","volume":"120","author":"LP Argyle","year":"2023","unstructured":"Argyle, L. P. et al. Leveraging AI for democratic discourse: chat interventions can improve online political conversations at scale. Proc. Natl Acad. Sci. USA 120, e2311627120 (2023).","journal-title":"Proc. Natl Acad. Sci. USA"},{"key":"843_CR13","doi-asserted-by":"publisher","first-page":"eadq2852","DOI":"10.1126\/science.adq2852","volume":"386","author":"MH Tessler","year":"2024","unstructured":"Tessler, M. H. et al. AI can help humans find common ground in democratic deliberation. Science 386, eadq2852 (2024).","journal-title":"Science"},{"key":"843_CR14","doi-asserted-by":"publisher","first-page":"e2403116121","DOI":"10.1073\/pnas.2403116121","volume":"121","author":"K Hackenburg","year":"2024","unstructured":"Hackenburg, K. & Margetts, H. Evaluating the persuasive influence of political microtargeting with large language models. Proc. Natl Acad. Sci. USA 121, e2403116121 (2024).","journal-title":"Proc. Natl Acad. Sci. USA"},{"key":"843_CR15","doi-asserted-by":"publisher","first-page":"148","DOI":"10.3390\/socsci12030148","volume":"12","author":"D Rozado","year":"2023","unstructured":"Rozado, D. The political biases of ChatGPT. Soc. Sci. 12, 148 (2023).","journal-title":"Soc. Sci."},{"key":"843_CR16","doi-asserted-by":"crossref","unstructured":"Park, J. S. et al. Generative agents: interactive simulacra of human behavior. In Follmer, S., Han, J., Steimle, J. and Riche, N. H. Proc. 36th Annual ACM Symposium on User Interface Software and Technology 1\u201322 (Association for Computing Machinery, 2023).","DOI":"10.1145\/3586183.3606763"},{"key":"843_CR17","doi-asserted-by":"publisher","first-page":"281","DOI":"10.1080\/00323187.2024.2335471","volume":"75","author":"A Palmer","year":"2023","unstructured":"Palmer, A. & Spirling, A. Large language models can argue in convincing ways about politics, but humans dislike AI authors: implications for governance. Political Sci. 75, 281\u2013291 (2023).","journal-title":"Political Sci."},{"key":"843_CR18","doi-asserted-by":"publisher","first-page":"337","DOI":"10.1017\/pan.2023.2","volume":"31","author":"LP Argyle","year":"2023","unstructured":"Argyle, L. P. et al. Out of one, many: using language models to simulate human samples. Political Anal. 31, 337\u2013351 (2023).","journal-title":"Political Anal."},{"key":"843_CR19","unstructured":"T\u00f6rnberg, P., Valeeva, D., Uitermark, J. & Bail, C. Simulating social media using large language models to evaluate alternative news feed algorithms. Preprint at https:\/\/arxiv.org\/abs\/2310.05984 (2023)."},{"key":"843_CR20","doi-asserted-by":"crossref","unstructured":"Sreedhar, K., Cai, A., Ma, J., Nickerson, J. V. & Chilton, L. B. Simulating cooperative prosocial behavior with multi-agent LLMs: evidence and mechanisms for AI agents to inform policy decisions. In Proc. 30th International Conference on Intelligent User Interfaces (eds Li, T. et al.) 1272\u20131286 (Association for Computing Machinery, 2025).","DOI":"10.1145\/3708359.3712149"},{"key":"843_CR21","doi-asserted-by":"publisher","first-page":"401","DOI":"10.1017\/pan.2024.5","volume":"32","author":"J Bisbee","year":"2023","unstructured":"Bisbee, J., Clinton, J. D., Dorff, C., Kenkel, B. & Larson, J. M. Synthetic replacements for human survey data? The perils of large language models. Political Anal. 32, 401\u2013416 (2023).","journal-title":"Political Anal."},{"key":"843_CR22","unstructured":"Ashokkumar, A., Hewitt, L., Ghezae, I. & Willer, R. Predicting results of social science experiments using large language models. Ethics and Psychology (6 November 2024)."},{"key":"843_CR23","doi-asserted-by":"publisher","first-page":"313","DOI":"10.1126\/science.adg7879","volume":"379","author":"HH Thorp","year":"2023","unstructured":"Thorp, H. H. ChatGPT is fun, but not an author. Science 379, 313\u2013313 (2023).","journal-title":"Science"},{"key":"843_CR24","doi-asserted-by":"publisher","first-page":"34","DOI":"10.1007\/s13347-024-00715-1","volume":"37","author":"R Van Woudenberg","year":"2024","unstructured":"Van Woudenberg, R., Ranalli, C. & Bracker, D. Authorship and ChatGPT: a conservative view. Phil. Technol. 37, 34 (2024).","journal-title":"Phil. Technol."},{"key":"843_CR25","doi-asserted-by":"publisher","first-page":"e2314021121","DOI":"10.1073\/pnas.2314021121","volume":"121","author":"CA Bail","year":"2024","unstructured":"Bail, C. A. Can generative AI improve social science? Proc. Natl Acad. Sci. USA 121, e2314021121 (2024).","journal-title":"Proc. Natl Acad. Sci. USA"},{"key":"843_CR26","doi-asserted-by":"publisher","first-page":"1108","DOI":"10.1126\/science.adi1778","volume":"380","author":"I Grossmann","year":"2023","unstructured":"Grossmann, I. et al. AI and the transformation of social science research. Science 380, 1108\u20131109 (2023).","journal-title":"Science"},{"key":"843_CR27","doi-asserted-by":"publisher","first-page":"103665","DOI":"10.1016\/j.ipm.2024.103665","volume":"61","author":"R Xu","year":"2024","unstructured":"Xu, R. et al. AI for social science and social science of AI: a survey. Inf. Process. Manag. 61, 103665 (2024).","journal-title":"Inf. Process. Manag."},{"key":"843_CR28","doi-asserted-by":"publisher","first-page":"e2412815122","DOI":"10.1073\/pnas.2412815122","volume":"122","author":"LP Argyle","year":"2025","unstructured":"Argyle, L. P., Busby, E. C., Gubler, J. R. & Wingate, D. Testing theories of political persuasion using artificial intelligence. Proc. Natl Acad. Sci. USA 122, e2412815122 (2025).","journal-title":"Proc. Natl Acad. Sci. USA"},{"key":"843_CR29","doi-asserted-by":"crossref","unstructured":"Lyman, A. et al. Balancing large language model alignment and algorithmic fidelity in social science research. Sociol. Methods Res. (2025).","DOI":"10.1177\/00491241251342008"},{"key":"843_CR30","doi-asserted-by":"publisher","first-page":"e2322420121","DOI":"10.1073\/pnas.2322420121","volume":"121","author":"RT McCoy","year":"2024","unstructured":"McCoy, R. T., Yao, S., Friedman, D., Hardy, M. D. & Griffiths, T. L. Embers of autoregression show how large language models are shaped by the problem they are trained to solve. Proc. Natl Acad. Sci. USA 121, e2322420121 (2024).","journal-title":"Proc. Natl Acad. Sci. USA"},{"key":"843_CR31","unstructured":"Liu, M. & Shi, G. Enhancing LLM-based text classification in political science: automatic prompt optimization and dynamic exemplar selection for few-shot learning. Preprint at https:\/\/arxiv.org\/abs\/2409.01466 (2024)."},{"key":"843_CR32","doi-asserted-by":"crossref","unstructured":"Atreja, S., Ashkinaze, J., Li, L., Mendelsohn, J. & Hemphill, L. What\u2019s in a prompt?: A large-scale experiment to assess the impact of prompt design on the compliance and accuracy of LLM-generated text annotations. Proc. International AAAI Conference on Web and Social Media Vol. 19, 122\u2013145 (AAAI, 2025); https:\/\/ojs.aaai.org\/index.php\/ICWSM\/article\/view\/35807","DOI":"10.1609\/icwsm.v19i1.35807"},{"key":"843_CR33","doi-asserted-by":"crossref","unstructured":"Zhuo, J., Zhang, S., Fang, X., Duan, H., Lin, D. & Kai Chen. ProSA: assessing and understanding the prompt sensitivity of LLMs. In Findings of the Association for Computational Linguistics 1950\u20131976 (Association for Computational Linguistics, 2024).","DOI":"10.18653\/v1\/2024.findings-emnlp.108"},{"key":"843_CR34","doi-asserted-by":"publisher","first-page":"413","DOI":"10.1038\/d41586-023-01295-4","volume":"616","author":"A Spirling","year":"2023","unstructured":"Spirling, A. Why open-source generative AI models are an ethical way forward for science. Nature 616, 413 (2023).","journal-title":"Nature"},{"key":"843_CR35","doi-asserted-by":"publisher","first-page":"4","DOI":"10.1038\/s42256-023-00783-6","volume":"6","author":"\u00c9 Ollion","year":"2024","unstructured":"Ollion, \u00c9., Shen, R., Macanovic, A. & Chatelain, A. The dangers of using proprietary LLMs for research. Nat. Mach. Intell. 6, 4\u20135 (2024).","journal-title":"Nat. Mach. Intell."},{"key":"843_CR36","unstructured":"Ja\u017awi\u0144ska, K. & Chandrasekar, A. AI search has a citation problem: We compared eight AI search engines. They\u2019re all bad at citing news. Columbia Journalism Review (6 March 2025)."},{"key":"843_CR37","unstructured":"Briggs, R., Mellon, J., Arel-Bundock, V. & Larson, T. We used LLMs to track methodological and substantive publication patterns in political science and they seem to do a pretty good job. Preprint at https:\/\/osf.io\/v7fe8 (2025)."},{"key":"843_CR38","doi-asserted-by":"publisher","first-page":"597","DOI":"10.1016\/j.tics.2023.04.008","volume":"27","author":"D Dillion","year":"2023","unstructured":"Dillion, D., Tandon, N., Gu, Y. & Gray, K. Can AI language models replace human participants? Trends Cogn. Sci. 27, 597\u2013600 (2023).","journal-title":"Trends Cogn. Sci."},{"key":"843_CR39","unstructured":"Li, J. et al. Can LLM already serve as a database interface? A big bench for large-scale database grounded text-to-SQLs. In 37th Conference on Neural Information Processing Systems (NeurIPS, 2023)."},{"key":"843_CR40","doi-asserted-by":"publisher","first-page":"3302","DOI":"10.14778\/3611479.3611527","volume":"16","author":"RC Fernandez","year":"2023","unstructured":"Fernandez, R. C., Elmore, A. J., Franklin, M. J., Krishnan, S. & Tan, C. How large language models will disrupt data management. Proc. VLDB Endow. 16, 3302\u20133309 (2023).","journal-title":"Proc. VLDB Endow."},{"key":"843_CR41","doi-asserted-by":"crossref","unstructured":"Xiao, C., Xu, S. X., Zhang, K., Wang, Y. & Xia, L. Evaluating reading comprehension exercises generated by LLMs: a showcase of ChatGPT in education applications. In Proc. 18th Workshop on Innovative Use of NLP for Building Educational Applications (eds Kochmar, E. et al.) 610\u2013625 (Association for Computational Linguistics, 2023); https:\/\/aclanthology.org\/2023.bea-1.52\/","DOI":"10.18653\/v1\/2023.bea-1.52"},{"key":"843_CR42","doi-asserted-by":"publisher","unstructured":"Lyu, W., Wang, Y., Chung, T. (R.), Sun, Y. & Zhang, Y. Evaluating the effectiveness of LLMs in introductory computer science education: a semester-long field study. In Proc. 11th ACM Conference on Learning @ Scale (eds Joyner, D.) 63\u201374 (Association for Computing Machinery, 2024); https:\/\/doi.org\/10.1145\/3657604.3662036","DOI":"10.1145\/3657604.3662036"},{"key":"843_CR43","doi-asserted-by":"publisher","first-page":"333","DOI":"10.1038\/s42256-023-00644-2","volume":"5","author":"S Milano","year":"2023","unstructured":"Milano, S., McGrane, J. A. & Leonelli, S. Large language models challenge the future of higher education. Nat. Mach. Intell. 5, 333\u2013334 (2023).","journal-title":"Nat. Mach. Intell."},{"key":"843_CR44","unstructured":"Yakura, H. et al. Empirical evidence of large language model\u2019s influence on human spoken communication. Preprint at https:\/\/arxiv.org\/abs\/2409.01754 (2024)."},{"key":"843_CR45","doi-asserted-by":"publisher","DOI":"10.1038\/s41598-023-30938-9","volume":"13","author":"J Hohenstein","year":"2023","unstructured":"Hohenstein, J. et al. Artificial intelligence in communication impacts language and social relationships. Sci. Rep. 13, 5487 (2023).","journal-title":"Sci. Rep."},{"key":"843_CR46","doi-asserted-by":"crossref","unstructured":"Manning, B. S., Zhu, K. & Horton, J. J. Automated Social Science: Language Models as Scientist and Subjects Technical Report (National Bureau of Economic Research, 2024).","DOI":"10.3386\/w32381"},{"key":"843_CR47","first-page":"145","volume":"18","author":"L Rossi","year":"2024","unstructured":"Rossi, L., Harrison, K. & Shklovski, I. The problems of LLM-generated data in social science research. Sociologica 18, 145\u2013168 (2024).","journal-title":"Sociologica"},{"key":"843_CR48","doi-asserted-by":"publisher","first-page":"160940692513223","DOI":"10.1177\/16094069251322346","volume":"24","author":"AS Hayes","year":"2025","unstructured":"Hayes, A. S. \u2018Conversing\u2019 with qualitative data: enhancing qualitative research through large language models (LLMs). Int. J. Qual. Methods 24, 16094069251322346 (2025).","journal-title":"Int. J. Qual. Methods"},{"key":"843_CR49","doi-asserted-by":"crossref","unstructured":"Schroeder, H., Aubin Le Qu\u00e9r\u00e9, M., Randazzo, C., Mimno, D. & Schoenebeck, S. Large language models in qualitative research: uses, tensions, and intentions. In Proc. 2025 CHI Conference on Human Factors in Computing Systems (eds Yamashita, N. et al.) 1\u201317 (Association for Computing Machinery, 2025).","DOI":"10.1145\/3706598.3713120"},{"key":"843_CR50","doi-asserted-by":"publisher","DOI":"10.1140\/epjds\/s13688-025-00548-8","volume":"14","author":"ZO Dunivin","year":"2025","unstructured":"Dunivin, Z. O. Scaling hermeneutics: a guide to qualitative coding with llms for reflexive content analysis. EPJ Data Sci. 14, 28 (2025).","journal-title":"EPJ Data Sci."},{"key":"843_CR51","doi-asserted-by":"crossref","unstructured":"Reiss, M. V. Testing the reliability of ChatGPT for text annotation and classification: a cautionary remark. Preprint at https:\/\/arxiv.org\/abs\/2304.11085 (2023).","DOI":"10.31219\/osf.io\/rvy5p"},{"key":"843_CR52","doi-asserted-by":"crossref","unstructured":"Ollion, E., Shen, R., Macanovic, A. & Chatelain, A. ChatGPT for text annotation? Mind the hype. Preprint at https:\/\/osf.io\/preprints\/socarxiv\/x58kn_v1 (2023).","DOI":"10.31235\/osf.io\/x58kn"},{"key":"843_CR53","unstructured":"Pangakis, N., Wolken, S. & Fasching, N. Automated annotation with generative AI requires validation. Preprint at https:\/\/arxiv.org\/abs\/2306.00176 (2023)."},{"key":"843_CR54","first-page":"67","volume":"18","author":"P T\u00f6rnberg","year":"2024","unstructured":"T\u00f6rnberg, P. Best practices for text annotation with large language models. Sociologica 18, 67\u201385 (2024).","journal-title":"Sociologica"},{"key":"843_CR55","doi-asserted-by":"crossref","unstructured":"Alizadeh, M. et al. Open-source LLMs for text annotation: a practical guide for model setting and fine-tuning. J. Comput. Soc. Sci. 8, (2025).","DOI":"10.1007\/s42001-024-00345-9"},{"key":"843_CR56","unstructured":"King, G., Keohane, R. O. & Verba, S. Designing Social Inquiry: Scientific Inference in Qualitative Research: New Edition (Princeton Univ. Press, 2021)."},{"key":"843_CR57","doi-asserted-by":"crossref","unstructured":"Lakatos, I. Falsification and the methodology of scientific research programmes. In Criticism and the Growth of Knowledge: Proc. International Colloquium in the Philosophy of Science (eds Lakatos, I. & Musgrave, A.) 91\u2013196 (Cambridge Univ. Press, 1970).","DOI":"10.1017\/CBO9781139171434.009"},{"key":"843_CR58","doi-asserted-by":"crossref","unstructured":"Cui, Z., Li, N. & Zhou, H. Can AI replace human subjects? A large-scale replication of psychological experiments with LLMs. Preprint at https:\/\/arxiv.org\/abs\/2409.00128 (2024).","DOI":"10.2139\/ssrn.4940173"},{"key":"843_CR59","doi-asserted-by":"crossref","unstructured":"Horton, J. J. Large Language Models as Simulated Economic Agents: What Can We Learn from Homo Silicus? Technical Report (National Bureau of Economic Research, 2023).","DOI":"10.3386\/w31122"},{"key":"843_CR60","doi-asserted-by":"publisher","first-page":"240682","DOI":"10.1098\/rsos.240682","volume":"11","author":"S Lippert","year":"2024","unstructured":"Lippert, S. et al. Can large language models help predict results from a complex behavioural science study? R. Soc. Open Sci. 11, 240682 (2024).","journal-title":"R. Soc. Open Sci."},{"key":"843_CR61","doi-asserted-by":"publisher","first-page":"e2413443122","DOI":"10.1073\/pnas.2413443122","volume":"122","author":"K Hackenburg","year":"2025","unstructured":"Hackenburg, K. et al. Scaling language model size yields diminishing returns for single-message political persuasion. Proc. Natl Acad. Sci. USA 122, e2413443122 (2025).","journal-title":"Proc. Natl Acad. Sci. USA"},{"key":"843_CR62","doi-asserted-by":"crossref","unstructured":"Szymanski, A. et al. Limitations of the LLM-as-a-judge approach for evaluating llm outputs in expert knowledge tasks. In Proc. 30th International Conference on Intelligent User Interfaces (eds Li, T. et al.) 952\u2013966 (Association for Computing Machinery, 2025).","DOI":"10.1145\/3708359.3712091"},{"key":"843_CR63","first-page":"97053","volume":"37","author":"J-T Huang","year":"2024","unstructured":"Huang, J.-T. et al. Apathetic or empathetic? Evaluating LLMs\u2019 emotional alignments with humans. Adv. Neural Inf. Process. Syst. 37, 97053\u201397087 (2024).","journal-title":"Adv. Neural Inf. Process. Syst."},{"key":"843_CR64","doi-asserted-by":"crossref","unstructured":"Amirizaniani, M., Martin, E., Sivachenko, M., Mashhadi, A. & Shah, C. Can LLMs reason like humans? Assessing theory of mind reasoning in LLMs for open-ended questions. In Proc. 33rd ACM International Conference on Information and Knowledge Management (eds Serra, E. & Spezzano, F.) 34\u201344 (Association for Computing Machinery, 2024).","DOI":"10.1145\/3627673.3679832"},{"key":"843_CR65","unstructured":"Valmeekam, K., Olmo, A., Sreedharan, S. & Kambhampati, S. Large language models still can\u2019t plan (a benchmark for LLMs on planning and reasoning about change). In NeurIPS 2022 Foundation Models for Decision Making Workshop (2022)."},{"key":"843_CR66","unstructured":"Eaton, K. How many R\u2019s in \u2018strawberry\u2019? This AI doesn\u2019t know. Inc. https:\/\/www.inc.com\/kit-eaton\/how-many-rs-in-strawberry-this-ai-cant-tell-you.html (2024)."},{"key":"843_CR67","doi-asserted-by":"crossref","unstructured":"Lu, Y., Zhu, W., Li, L., Qiao, Y. & Yuan, F. LLaMAX: Scaling linguistic horizons of LLM by enhancing translation capabilities beyond 100 languages. In Findings of the Association for Computational Linguistics: EMNLP 2024 (eds Al-Onaizan, Y. et al.) 10748\u201310772 (Association for Computational Linguistics, 2024).","DOI":"10.18653\/v1\/2024.findings-emnlp.631"},{"key":"843_CR68","unstructured":"Kaplan, J. et al. Scaling laws for neural language models. Preprint at https:\/\/arxiv.org\/abs\/2001.08361 (2020)."},{"key":"843_CR69","unstructured":"Cui, J., Chiang, W. L., Stoica, I., & Hsieh, C. J. Or-bench: an over-refusal benchmark for large language models. Preprint at https:\/\/arxiv.org\/abs\/2405.20947 (2025)."},{"key":"843_CR70","doi-asserted-by":"crossref","unstructured":"Tekgurler, M. Historical, low-resourced languages and contempo-rary AI models. In Proc. 9th Joint SIGHUM Workshop on Computational Linguistics for Cultural Heritage, Social Sciences, Humanities and Literature (eds Kazantseva, A. et al.) 227\u2013237 (Association for Computational Linguistics, 2025).","DOI":"10.18653\/v1\/2025.latechclfl-1.20"},{"key":"843_CR71","unstructured":"Kirk, R. et al. Understanding the effects of RLHF on LLM generalisation and diversity. In 12th International Conference on Learning Representations (ICLR, 2024); https:\/\/openreview.net\/forum?id=PXD3FAVHJT"},{"key":"843_CR72","doi-asserted-by":"publisher","unstructured":"Li, H., Ding, L., Fang, M. & Tao, D. Revisiting catastrophic forgetting in large language model tuning. In Findings of the Association for Computational Linguistics: EMNLP 2024 (eds Al-Onaizan, Y, et al.) 4297\u20134308 (Association for Computational Linguistics, 2024); https:\/\/doi.org\/10.18653\/v1\/2024.findings-emnlp.249","DOI":"10.18653\/v1\/2024.findings-emnlp.249"},{"key":"843_CR73","doi-asserted-by":"publisher","unstructured":"Pezeshkpour, P. & Hruschka, E. Large language models sensitivity to the order of options in multiple-choice questions. In Findings of the Association for Computational Linguistics: NAACL 2024 (eds Duh, K. et al.) 2006\u20132017 (Association for Computational Linguistics, 2024); https:\/\/doi.org\/10.18653\/v1\/2024.findings-naacl.130","DOI":"10.18653\/v1\/2024.findings-naacl.130"},{"key":"843_CR74","unstructured":"Wang, Q. et al. What limits LLM-based human simulation: LLMs or our design? Preprint at https:\/\/arxiv.org\/abs\/2501.08579 (2025)."},{"key":"843_CR75","unstructured":"Santurkar, S. et al. Whose opinions do language models reflect? In Proc. 40th International Conference on Machine Learning (eds Krause, A. et al.) 1\u201334 (ICML, 2023)."},{"key":"843_CR76","unstructured":"Boelaert, J., Coavoux, S., Ollion, \u00c9., Petev, I. & Pr\u00e4g, P. Machine bias generative large language models have a worldview of their own. Preprint at https:\/\/osf.io\/preprints\/socarxiv\/r2pnb_v2 (2025)."},{"key":"843_CR77","unstructured":"Kim, J. & Lee, B. AI-augmented surveys: leveraging large language models and surveys for opinion prediction. Preprint at https:\/\/arxiv.org\/abs\/2305.09620 (2023)."},{"key":"843_CR78","doi-asserted-by":"publisher","first-page":"e2305016120","DOI":"10.1073\/pnas.2305016120","volume":"120","author":"F Gilardi","year":"2023","unstructured":"Gilardi, F., Alizadeh, M. & Kubli, M. ChatGPT outperforms crowd-workers for text-annotation tasks. Proc. Natl Acad. Sci. USA 120, e2305016120 (2023).","journal-title":"Proc. Natl Acad. Sci. USA"}],"container-title":["Nature Computational Science"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/www.nature.com\/articles\/s43588-025-00843-4.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/www.nature.com\/articles\/s43588-025-00843-4","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/www.nature.com\/articles\/s43588-025-00843-4.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,9,24]],"date-time":"2025-09-24T03:05:12Z","timestamp":1758683112000},"score":1,"resource":{"primary":{"URL":"https:\/\/www.nature.com\/articles\/s43588-025-00843-4"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,8,8]]},"references-count":78,"journal-issue":{"issue":"9","published-online":{"date-parts":[[2025,9]]}},"alternative-id":["843"],"URL":"https:\/\/doi.org\/10.1038\/s43588-025-00843-4","relation":{},"ISSN":["2662-8457"],"issn-type":[{"value":"2662-8457","type":"electronic"}],"subject":[],"published":{"date-parts":[[2025,8,8]]},"assertion":[{"value":"31 March 2025","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"27 June 2025","order":2,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"8 August 2025","order":3,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"The authors declare no competing interests.","order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Competing interests"}}]}}