{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,3,31]],"date-time":"2026-03-31T04:26:02Z","timestamp":1774931162990,"version":"3.50.1"},"reference-count":36,"publisher":"Springer Science and Business Media LLC","issue":"1","license":[{"start":{"date-parts":[[2024,9,13]],"date-time":"2024-09-13T00:00:00Z","timestamp":1726185600000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by-nc-nd\/4.0"},{"start":{"date-parts":[[2024,9,13]],"date-time":"2024-09-13T00:00:00Z","timestamp":1726185600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by-nc-nd\/4.0"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["npj Digit. Med."],"DOI":"10.1038\/s41746-024-01251-0","type":"journal-article","created":{"date-parts":[[2024,9,13]],"date-time":"2024-09-13T21:03:12Z","timestamp":1726261392000},"update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":13,"title":["Results and implications for generative AI in a large introductory biomedical and health informatics course"],"prefix":"10.1038","volume":"7","author":[{"given":"William","family":"Hersh","sequence":"first","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0002-4668-6015","authenticated-orcid":false,"given":"Kate","family":"Fultz Hollis","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2024,9,13]]},"reference":[{"key":"1251_CR1","doi-asserted-by":"publisher","first-page":"210","DOI":"10.7326\/M23-2772","volume":"177","author":"JA Omiye","year":"2024","unstructured":"Omiye, J. A., Gui, H., Rezaei, S. J., Zou, J. & Daneshjou, R. Large Language Models in Medicine: The Potentials and Pitfalls: A Narrative Review. Ann. Intern Med. 177, 210\u2013220 (2024).","journal-title":"Ann. Intern Med"},{"key":"1251_CR2","doi-asserted-by":"publisher","first-page":"e230987","DOI":"10.1148\/radiol.230987","volume":"307","author":"R Bhayana","year":"2023","unstructured":"Bhayana, R., Bleakney, R. R. & Krishna, S. GPT-4 in Radiology: Improvements in Advanced Reasoning. Radiology 307, e230987 (2023).","journal-title":"Radiology"},{"key":"1251_CR3","doi-asserted-by":"publisher","unstructured":"Kumah-Crystal, Y., Mankowitz, S., Embi, P. & Lehmann, C. U. ChatGPT and the clinical informatics board examination: the end of unproctored maintenance of certification? J Am Med Inform Assoc ocad104 (2023) https:\/\/doi.org\/10.1093\/jamia\/ocad104.","DOI":"10.1093\/jamia\/ocad104"},{"key":"1251_CR4","doi-asserted-by":"publisher","unstructured":"Nori, H. et al. Can Generalist Foundation Models Outcompete Special-Purpose Tuning? Case Study in Medicine. Preprint at https:\/\/doi.org\/10.48550\/arXiv.2311.16452 (2023).","DOI":"10.48550\/arXiv.2311.16452"},{"key":"1251_CR5","volume":"0","author":"U Katz","year":"2024","unstructured":"Katz, U. et al. GPT versus Resident Physicians \u2014 A Benchmark Based on Official Board Scores. NEJM AI 0, AIdbp2300192 (2024).","journal-title":"NEJM AI"},{"key":"1251_CR6","doi-asserted-by":"publisher","unstructured":"Li, S. W. et al. ChatGPT outscored human candidates in a virtual objective structured clinical examination in obstetrics and gynecology. Am J Obstet Gynecol S0002-9378(23)00251-X (2023) https:\/\/doi.org\/10.1016\/j.ajog.2023.04.020.","DOI":"10.1016\/j.ajog.2023.04.020"},{"key":"1251_CR7","doi-asserted-by":"publisher","unstructured":"Dash, D. et al. Evaluation of GPT-3.5 and GPT-4 for supporting real-world information needs in healthcare delivery. Preprint at https:\/\/doi.org\/10.48550\/arXiv.2304.13714 (2023).","DOI":"10.48550\/arXiv.2304.13714"},{"key":"1251_CR8","doi-asserted-by":"publisher","first-page":"e2336483","DOI":"10.1001\/jamanetworkopen.2023.36483","volume":"6","author":"RS Goodman","year":"2023","unstructured":"Goodman, R. S. et al. Accuracy and Reliability of Chatbot Responses to Physician Questions. JAMA Netw. Open 6, e2336483 (2023).","journal-title":"JAMA Netw. Open"},{"key":"1251_CR9","doi-asserted-by":"publisher","unstructured":"Benoit, J. R. A. ChatGPT for Clinical Vignette Generation, Revision, and Evaluation. 2023.02.04.23285478 Preprint at https:\/\/doi.org\/10.1101\/2023.02.04.23285478 (2023).","DOI":"10.1101\/2023.02.04.23285478"},{"key":"1251_CR10","doi-asserted-by":"publisher","unstructured":"Eriksen, A. V., M\u00f6ller, S. & Ryg, J. Use of GPT-4 to Diagnose Complex Clinical Cases. NEJM AI (2023) https:\/\/doi.org\/10.1056\/AIp2300031.","DOI":"10.1056\/AIp2300031"},{"key":"1251_CR11","doi-asserted-by":"publisher","first-page":"78","DOI":"10.1001\/jama.2023.8288","volume":"330","author":"Z Kanjee","year":"2023","unstructured":"Kanjee, Z., Crowe, B. & Rodman, A. Accuracy of a Generative Artificial Intelligence Model in a Complex Diagnostic Challenge. JAMA 330, 78\u201380 (2023).","journal-title":"JAMA"},{"key":"1251_CR12","doi-asserted-by":"publisher","first-page":"e555","DOI":"10.1016\/S2589-7500(24)00097-9","volume":"6","author":"DM Levine","year":"2024","unstructured":"Levine, D. M. et al. The Diagnostic and Triage Accuracy of the GPT-3 Artificial Intelligence Model: An Observational Study. Lancet Digit Health. 6, e555\u2013e561 (2024).","journal-title":"Lancet Digit Health."},{"key":"1251_CR13","doi-asserted-by":"publisher","unstructured":"McDuff, D. et al. Towards Accurate Differential Diagnosis with Large Language Models. Preprint at https:\/\/doi.org\/10.48550\/arXiv.2312.00164 (2023).","DOI":"10.48550\/arXiv.2312.00164"},{"key":"1251_CR14","doi-asserted-by":"publisher","unstructured":"Tu, T. et al. Towards Conversational Diagnostic AI. Preprint at https:\/\/doi.org\/10.48550\/arXiv.2401.05654 (2024).","DOI":"10.48550\/arXiv.2401.05654"},{"key":"1251_CR15","doi-asserted-by":"publisher","DOI":"10.1001\/jamainternmed.2024.0295","author":"S Cabral","year":"2024","unstructured":"Cabral, S. et al. Clinical Reasoning of a Generative Artificial Intelligence Model Compared With Physicians. JAMA Intern Med https:\/\/doi.org\/10.1001\/jamainternmed.2024.0295 (2024).","journal-title":"JAMA Intern Med"},{"key":"1251_CR16","doi-asserted-by":"publisher","unstructured":"Mollick, E. R. & Mollick, L. Using AI to Implement Effective Teaching Strategies in Classrooms: Five Strategies, Including Prompts. SSRN Scholarly Paper at https:\/\/doi.org\/10.2139\/ssrn.4391243 (2023).","DOI":"10.2139\/ssrn.4391243"},{"key":"1251_CR17","doi-asserted-by":"publisher","first-page":"56","DOI":"10.1145\/3624720","volume":"67","author":"P Denny","year":"2024","unstructured":"Denny, P. et al. Computing Education in the Era of Generative AI. Commun. ACM 67, 56\u201367 (2024).","journal-title":"Commun. ACM"},{"key":"1251_CR18","doi-asserted-by":"publisher","first-page":"23","DOI":"10.1145\/3632523","volume":"67","author":"M Johnson","year":"2024","unstructured":"Johnson, M. Generative AI and CS Education. Commun. ACM 67, 23\u201324 (2024).","journal-title":"Commun. ACM"},{"key":"1251_CR19","doi-asserted-by":"publisher","unstructured":"Poldrack, R. A., Lu, T. & Begu\u0161, G. AI-assisted coding: Experiments with GPT-4. Preprint at https:\/\/doi.org\/10.48550\/arXiv.2304.13187 (2023).","DOI":"10.48550\/arXiv.2304.13187"},{"key":"1251_CR20","doi-asserted-by":"publisher","unstructured":"Choi, J. H., Monahan, A. & Schwarcz, D. Lawyering in the Age of Artificial Intelligence. SSRN Scholarly Paper at https:\/\/doi.org\/10.2139\/ssrn.4626276 (2023).","DOI":"10.2139\/ssrn.4626276"},{"key":"1251_CR21","doi-asserted-by":"publisher","unstructured":"Hong, S. et al. Data Interpreter: An LLM Agent For Data Science. Preprint at https:\/\/doi.org\/10.48550\/arXiv.2402.18679 (2024).","DOI":"10.48550\/arXiv.2402.18679"},{"key":"1251_CR22","doi-asserted-by":"publisher","unstructured":"Hersh, W. Search still matters: information retrieval in the era of generative AI. J Am Med Inform Assoc ocae014 (2024) https:\/\/doi.org\/10.1093\/jamia\/ocae014.","DOI":"10.1093\/jamia\/ocae014"},{"key":"1251_CR23","doi-asserted-by":"publisher","unstructured":"Wu, K. et al. How well do LLMs cite relevant medical references? An evaluation framework and analyses. Preprint at https:\/\/doi.org\/10.48550\/arXiv.2402.02008 (2024).","DOI":"10.48550\/arXiv.2402.02008"},{"key":"1251_CR24","doi-asserted-by":"publisher","first-page":"852","DOI":"10.1038\/s42256-024-00881-z","volume":"6","author":"I Augenstein","year":"2024","unstructured":"Augenstein, I. et al. Factuality challenges in the era of large language models and opportunities for fact-checking. Nat Mach Intell. 6, 852\u2013863 (2024).","journal-title":"Nat Mach Intell."},{"key":"1251_CR25","doi-asserted-by":"publisher","first-page":"198","DOI":"10.3390\/ime2030019","volume":"2","author":"TF Heston","year":"2023","unstructured":"Heston, T. F. & Khun, C. Prompt Engineering in Medical Education. Int. Med. Educ. 2, 198\u2013205 (2023).","journal-title":"Int. Med. Educ."},{"key":"1251_CR26","doi-asserted-by":"publisher","unstructured":"Mollick, E. R. & Mollick, L. Assigning AI: Seven Approaches for Students, with Prompts. SSRN Scholarly Paper at https:\/\/doi.org\/10.2139\/ssrn.4475995 (2023).","DOI":"10.2139\/ssrn.4475995"},{"key":"1251_CR27","doi-asserted-by":"publisher","first-page":"e48785","DOI":"10.2196\/48785","volume":"9","author":"C Preiksaitis","year":"2023","unstructured":"Preiksaitis, C. & Rose, C. Opportunities, Challenges, and Future Directions of Generative Artificial Intelligence in Medical Education: Scoping Review. JMIR Med Educ. 9, e48785 (2023).","journal-title":"JMIR Med Educ."},{"key":"1251_CR28","doi-asserted-by":"publisher","unstructured":"Sok, S. & Heng, K. ChatGPT for Education and Research: A Review of Benefits and Risks. SSRN Scholarly Paper at https:\/\/doi.org\/10.2139\/ssrn.4378735 (2023).","DOI":"10.2139\/ssrn.4378735"},{"key":"1251_CR29","doi-asserted-by":"publisher","unstructured":"Huschens, M., Briesch, M., Sobania, D. & Rothlauf, F. Do You Trust ChatGPT? -- Perceived Credibility of Human and AI-Generated Content. Preprint at https:\/\/doi.org\/10.48550\/arXiv.2309.02524 (2023).","DOI":"10.48550\/arXiv.2309.02524"},{"key":"1251_CR30","first-page":"93","volume":"300","author":"W Hersh","year":"2022","unstructured":"Hersh, W. Competencies and Curricula Across the Spectrum of Learners for Biomedical and Health Informatics. Stud. Health Technol. Inf. 300, 93\u2013107 (2022).","journal-title":"Stud. Health Technol. Inf."},{"key":"1251_CR31","doi-asserted-by":"publisher","first-page":"377","DOI":"10.1016\/j.ijmedinf.2007.01.003","volume":"76","author":"W Hersh","year":"2007","unstructured":"Hersh, W. & Williamson, J. Educating 10,000 informaticians by 2010: the AMIA 10x10 program. Int J. Med Inf. 76, 377\u2013382 (2007).","journal-title":"Int J. Med Inf."},{"key":"1251_CR32","doi-asserted-by":"publisher","unstructured":"Rein, D. et al. GPQA: A Graduate-Level Google-Proof Q&A Benchmark. Preprint at https:\/\/doi.org\/10.48550\/arXiv.2311.12022 (2023).","DOI":"10.48550\/arXiv.2311.12022"},{"key":"1251_CR33","doi-asserted-by":"publisher","first-page":"50","DOI":"10.1145\/3624725","volume":"67","author":"R Tang","year":"2024","unstructured":"Tang, R., Chuang, Y.-N. & Hu, X. The Science of Detecting LLM-Generated Text. Commun. ACM 67, 50\u201359 (2024).","journal-title":"Commun. ACM"},{"key":"1251_CR34","doi-asserted-by":"publisher","first-page":"100779","DOI":"10.1016\/j.patter.2023.100779","volume":"4","author":"W Liang","year":"2023","unstructured":"Liang, W., Yuksekgonul, M., Mao, Y., Wu, E. & Zou, J. GPT detectors are biased against non-native English writers. Patterns (N. Y) 4, 100779 (2023).","journal-title":"Patterns (N. Y)"},{"key":"1251_CR35","doi-asserted-by":"publisher","unstructured":"Cooper, A. & Rodman, A. AI and Medical Education \u2014 A 21st-Century Pandora\u2019s Box. New England Journal of Medicine (2023) https:\/\/doi.org\/10.1056\/NEJMp2304993.","DOI":"10.1056\/NEJMp2304993"},{"key":"1251_CR36","unstructured":"Mollick, E. The Homework Apocalypse. One Useful Thing https:\/\/www.oneusefulthing.org\/p\/the-homework-apocalypse (2023)."}],"container-title":["npj Digital Medicine"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/www.nature.com\/articles\/s41746-024-01251-0.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/www.nature.com\/articles\/s41746-024-01251-0","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/www.nature.com\/articles\/s41746-024-01251-0.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,9,13]],"date-time":"2024-09-13T21:04:56Z","timestamp":1726261496000},"score":1,"resource":{"primary":{"URL":"https:\/\/www.nature.com\/articles\/s41746-024-01251-0"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,9,13]]},"references-count":36,"journal-issue":{"issue":"1","published-online":{"date-parts":[[2024,12]]}},"alternative-id":["1251"],"URL":"https:\/\/doi.org\/10.1038\/s41746-024-01251-0","relation":{},"ISSN":["2398-6352"],"issn-type":[{"value":"2398-6352","type":"electronic"}],"subject":[],"published":{"date-parts":[[2024,9,13]]},"assertion":[{"value":"18 April 2024","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"3 September 2024","order":2,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"13 September 2024","order":3,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"The authors declare no competing interests.","order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Competing interests"}}],"article-number":"247"}}