{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,6,6]],"date-time":"2026-06-06T08:21:16Z","timestamp":1780734076365,"version":"3.54.1"},"reference-count":42,"publisher":"Oxford University Press (OUP)","issue":"9","license":[{"start":{"date-parts":[[2024,1,28]],"date-time":"2024-01-28T00:00:00Z","timestamp":1706400000000},"content-version":"vor","delay-in-days":1,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"funder":[{"DOI":"10.13039\/100000002","name":"National Institutes of Health","doi-asserted-by":"publisher","award":["R21EB029575"],"award-info":[{"award-number":["R21EB029575"]}],"id":[{"id":"10.13039\/100000002","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000002","name":"National Institutes of Health","doi-asserted-by":"publisher","award":["R21AI164100"],"award-info":[{"award-number":["R21AI164100"]}],"id":[{"id":"10.13039\/100000002","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000002","name":"National Institutes of Health","doi-asserted-by":"publisher","award":["R01LM011934"],"award-info":[{"award-number":["R01LM011934"]}],"id":[{"id":"10.13039\/100000002","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000002","name":"National Institutes of Health","doi-asserted-by":"publisher","award":["1K99LM01402"],"award-info":[{"award-number":["1K99LM01402"]}],"id":[{"id":"10.13039\/100000002","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000002","name":"National Institutes of Health","doi-asserted-by":"publisher","award":["R01AG066749"],"award-info":[{"award-number":["R01AG066749"]}],"id":[{"id":"10.13039\/100000002","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000002","name":"National Institutes of Health","doi-asserted-by":"publisher","award":["R01AG066749-03S1"],"award-info":[{"award-number":["R01AG066749-03S1"]}],"id":[{"id":"10.13039\/100000002","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000002","name":"National Institutes of Health","doi-asserted-by":"publisher","award":["R01LM013712"],"award-info":[{"award-number":["R01LM013712"]}],"id":[{"id":"10.13039\/100000002","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000002","name":"National Institutes of Health","doi-asserted-by":"publisher","award":["U01TR002062"],"award-info":[{"award-number":["U01TR002062"]}],"id":[{"id":"10.13039\/100000002","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000049","name":"National Institute on Aging","doi-asserted-by":"publisher","award":["1RF1AG072799"],"award-info":[{"award-number":["1RF1AG072799"]}],"id":[{"id":"10.13039\/100000049","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000049","name":"National Institute on Aging","doi-asserted-by":"publisher","award":["1R01AG080429"],"award-info":[{"award-number":["1R01AG080429"]}],"id":[{"id":"10.13039\/100000049","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100004917","name":"Cancer Prevention and Research Institute of Texas","doi-asserted-by":"publisher","award":["RR180012"],"award-info":[{"award-number":["RR180012"]}],"id":[{"id":"10.13039\/100004917","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000001","name":"National Science Foundation","doi-asserted-by":"publisher","award":["2124789"],"award-info":[{"award-number":["2124789"]}],"id":[{"id":"10.13039\/100000001","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2024,9,1]]},"abstract":"<jats:title>Abstract<\/jats:title>\n               <jats:sec>\n                  <jats:title>Importance<\/jats:title>\n                  <jats:p>The study highlights the potential of large language models, specifically GPT-3.5 and GPT-4, in processing complex clinical data and extracting meaningful information with minimal training data. By developing and refining prompt-based strategies, we can significantly enhance the models\u2019 performance, making them viable tools for clinical NER tasks and possibly reducing the reliance on extensive annotated datasets.<\/jats:p>\n               <\/jats:sec>\n               <jats:sec>\n                  <jats:title>Objectives<\/jats:title>\n                  <jats:p>This study quantifies the capabilities of GPT-3.5 and GPT-4 for clinical named entity recognition (NER) tasks and proposes task-specific prompts to improve their performance.<\/jats:p>\n               <\/jats:sec>\n               <jats:sec>\n                  <jats:title>Materials and Methods<\/jats:title>\n                  <jats:p>We evaluated these models on 2 clinical NER tasks: (1) to extract medical problems, treatments, and tests from clinical notes in the MTSamples corpus, following the 2010 i2b2 concept extraction shared task, and (2) to identify nervous system disorder-related adverse events from safety reports in the vaccine adverse event reporting system (VAERS). To improve the GPT models' performance, we developed a clinical task-specific prompt framework that includes (1) baseline prompts with task description and format specification, (2) annotation guideline-based prompts, (3) error analysis-based instructions, and (4) annotated samples for few-shot learning. We assessed each prompt's effectiveness and compared the models to BioClinicalBERT.<\/jats:p>\n               <\/jats:sec>\n               <jats:sec>\n                  <jats:title>Results<\/jats:title>\n                  <jats:p>Using baseline prompts, GPT-3.5 and GPT-4 achieved relaxed F1 scores of 0.634, 0.804 for MTSamples and 0.301, 0.593 for VAERS. Additional prompt components consistently improved model performance. When all 4 components were used, GPT-3.5 and GPT-4 achieved relaxed F1 socres of 0.794, 0.861 for MTSamples and 0.676, 0.736 for VAERS, demonstrating the effectiveness of our prompt framework. Although these results trail BioClinicalBERT (F1 of 0.901 for the MTSamples dataset and 0.802 for the VAERS), it is very promising considering few training samples are needed.<\/jats:p>\n               <\/jats:sec>\n               <jats:sec>\n                  <jats:title>Discussion<\/jats:title>\n                  <jats:p>The study\u2019s findings suggest a promising direction in leveraging LLMs for clinical NER tasks. However, while the performance of GPT models improved with task-specific prompts, there's a need for further development and refinement. LLMs like GPT-4 show potential in achieving close performance to state-of-the-art models like BioClinicalBERT, but they still require careful prompt engineering and understanding of task-specific knowledge. The study also underscores the importance of evaluation schemas that accurately reflect the capabilities and performance of LLMs in clinical settings.<\/jats:p>\n               <\/jats:sec>\n               <jats:sec>\n                  <jats:title>Conclusion<\/jats:title>\n                  <jats:p>While direct application of GPT models to clinical NER tasks falls short of optimal performance, our task-specific prompt framework, incorporating medical knowledge and training samples, significantly enhances GPT models' feasibility for potential clinical applications.<\/jats:p>\n               <\/jats:sec>","DOI":"10.1093\/jamia\/ocad259","type":"journal-article","created":{"date-parts":[[2024,1,28]],"date-time":"2024-01-28T11:34:59Z","timestamp":1706441699000},"page":"1812-1820","source":"Crossref","is-referenced-by-count":265,"title":["Improving large language models for clinical named entity recognition via prompt engineering"],"prefix":"10.1093","volume":"31","author":[{"ORCID":"https:\/\/orcid.org\/0009-0008-2413-5918","authenticated-orcid":false,"given":"Yan","family":"Hu","sequence":"first","affiliation":[{"name":"McWilliams School of Biomedical Informatics , Houston, TX, United States"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Qingyu","family":"Chen","sequence":"additional","affiliation":[{"name":"Section of Biomedical Informatics and Data Science, School of Medicine, Yale University , New Haven, CT, United States"},{"name":"National Center for Biotechnology Information, National Library of Medicine, National Institutes of Health , Bethesda, MD, United States"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-0322-4566","authenticated-orcid":false,"given":"Jingcheng","family":"Du","sequence":"additional","affiliation":[{"name":"McWilliams School of Biomedical Informatics , Houston, TX, United States"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Xueqing","family":"Peng","sequence":"additional","affiliation":[{"name":"Section of Biomedical Informatics and Data Science, School of Medicine, Yale University , New Haven, CT, United States"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Vipina Kuttichi","family":"Keloth","sequence":"additional","affiliation":[{"name":"Section of Biomedical Informatics and Data Science, School of Medicine, Yale University , New Haven, CT, United States"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Xu","family":"Zuo","sequence":"additional","affiliation":[{"name":"McWilliams School of Biomedical Informatics , Houston, TX, United States"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-0889-2261","authenticated-orcid":false,"given":"Yujia","family":"Zhou","sequence":"additional","affiliation":[{"name":"McWilliams School of Biomedical Informatics , Houston, TX, United States"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Zehan","family":"Li","sequence":"additional","affiliation":[{"name":"McWilliams School of Biomedical Informatics , Houston, TX, United States"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-9933-2205","authenticated-orcid":false,"given":"Xiaoqian","family":"Jiang","sequence":"additional","affiliation":[{"name":"McWilliams School of Biomedical Informatics , Houston, TX, United States"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Zhiyong","family":"Lu","sequence":"additional","affiliation":[{"name":"National Center for Biotechnology Information, National Library of Medicine, National Institutes of Health , Bethesda, MD, United States"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Kirk","family":"Roberts","sequence":"additional","affiliation":[{"name":"McWilliams School of Biomedical Informatics , Houston, TX, United States"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Hua","family":"Xu","sequence":"additional","affiliation":[{"name":"Section of Biomedical Informatics and Data Science, School of Medicine, Yale University , New Haven, CT, United States"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"286","published-online":{"date-parts":[[2024,1,27]]},"reference":[{"issue":"6","key":"2024082207522225300_ocad259-B1","doi-asserted-by":"crossref","first-page":"395","DOI":"10.1038\/nrg3208","article-title":"Mining electronic health records: towards better research applications and clinical care","volume":"13","author":"Jensen","year":"2012","journal-title":"Nat Rev Genet"},{"issue":"5","key":"2024082207522225300_ocad259-B2","doi-asserted-by":"crossref","first-page":"544","DOI":"10.1136\/amiajnl-2011-000464","article-title":"Natural language processing: an introduction","volume":"18","author":"Nadkarni","year":"2011","journal-title":"J Am Med Inform Assoc"},{"issue":"1","key":"2024082207522225300_ocad259-B3","doi-asserted-by":"crossref","first-page":"12","DOI":"10.1186\/s13326-018-0179-8","article-title":"Clinical natural language processing in languages other than English: opportunities and challenges","volume":"9","author":"N\u00e9v\u00e9ol","year":"2018","journal-title":"J Biomed Semantics"},{"key":"2024082207522225300_ocad259-B4","doi-asserted-by":"crossref","first-page":"34","DOI":"10.1016\/j.jbi.2017.11.011","article-title":"Clinical information extraction applications: a literature review","volume":"77","author":"Wang","year":"2018","journal-title":"J Biomed Inform"},{"key":"2024082207522225300_ocad259-B5","author":"Huang","year":"2015"},{"issue":"5","key":"2024082207522225300_ocad259-B6","doi-asserted-by":"crossref","first-page":"507","DOI":"10.1136\/jamia.2009.001560","article-title":"Mayo clinical Text Analysis and Knowledge Extraction System (cTAKES): architecture, component evaluation and applications","volume":"17","author":"Savova","year":"2010","journal-title":"J Am Med Inform Assoc"},{"key":"2024082207522225300_ocad259-B7","author":"Devlin","year":"2018"},{"issue":"4","key":"2024082207522225300_ocad259-B8","doi-asserted-by":"crossref","first-page":"1234","DOI":"10.1093\/bioinformatics\/btz682","article-title":"BioBERT: a pre-trained biomedical language representation model for biomedical text mining","volume":"36","author":"Lee","year":"2020","journal-title":"Bioinformatics"},{"issue":"1","key":"2024082207522225300_ocad259-B9","first-page":"1","article-title":"Domain-specific language model pretraining for biomedical natural language processing","volume":"3","author":"Gu","year":"2021","journal-title":"ACM Transactions on Computing for Healthcare (HEALTH)"},{"key":"2024082207522225300_ocad259-B10","author":"Huang","year":"2019"},{"key":"2024082207522225300_ocad259-B11","author":"OpenAI","year":"2021"},{"key":"2024082207522225300_ocad259-B12","author":"Bang","year":"2023"},{"key":"2024082207522225300_ocad259-B13","first-page":"1877","article-title":"Language models are few-shot learners","volume":"33","author":"Brown","year":"2020","journal-title":"Adv Neural Inform Process Syst"},{"key":"2024082207522225300_ocad259-B14","author":"OpenAI","year":"2023"},{"issue":"1","key":"2024082207522225300_ocad259-B15","doi-asserted-by":"crossref","first-page":"e45312","DOI":"10.2196\/45312","article-title":"How does CHATGPT perform on the United States Medical Licensing Examination? The implications of large language models for medical education and knowledge assessment","volume":"9","author":"Gilson","year":"2023","journal-title":"JMIR Med Educ"},{"issue":"2","key":"2024082207522225300_ocad259-B16","doi-asserted-by":"crossref","first-page":"e0000198","DOI":"10.1371\/journal.pdig.0000198","article-title":"Performance of ChatGPT on USMLE: Potential for AI-assisted medical education using large language models","volume":"2","author":"Kung","year":"2023","journal-title":"PLoS Digit Health"},{"key":"2024082207522225300_ocad259-B17","author":"Rao","year":"2023"},{"key":"2024082207522225300_ocad259-B18","author":"Antaki"},{"key":"2024082207522225300_ocad259-B19","author":"Jeblick","year":"2022"},{"key":"2024082207522225300_ocad259-B20","volume-title":"The AI Revolution in Medicine: GPT-4 and Beyond","author":"Lee","year":"2023","edition":"1st ed"},{"key":"2024082207522225300_ocad259-B21","year":"2023"},{"key":"2024082207522225300_ocad259-B22"},{"key":"2024082207522225300_ocad259-B23","year":"2023"},{"key":"2024082207522225300_ocad259-B24","year":"2023"},{"key":"2024082207522225300_ocad259-B25","author":"Yu","year":"2023"},{"key":"2024082207522225300_ocad259-B26","author":"Ma","year":"2023"},{"key":"2024082207522225300_ocad259-B27","year":"2023"},{"key":"2024082207522225300_ocad259-B28","author":"Ateia","year":"2023"},{"key":"2024082207522225300_ocad259-B29","author":"Chen","year":"2023"},{"issue":"5","key":"2024082207522225300_ocad259-B30","doi-asserted-by":"crossref","first-page":"552","DOI":"10.1136\/amiajnl-2011-000203","article-title":"2010 i2b2\/VA challenge on concepts, assertions, and relations in clinical text","volume":"18","author":"Uzuner","year":"2011","journal-title":"J Am Med Inform Assoc"},{"issue":"7","key":"2024082207522225300_ocad259-B31","doi-asserted-by":"crossref","first-page":"1393","DOI":"10.1093\/jamia\/ocab014","article-title":"Extracting postmarketing adverse events from safety reports in the vaccine adverse event reporting system (VAERS) using deep learning","volume":"28","author":"Du","year":"2021","journal-title":"J Am Med Inform Assoc"},{"key":"2024082207522225300_ocad259-B32","author":"Alsentzer","year":"2019"},{"key":"2024082207522225300_ocad259-B33","year":"2020"},{"key":"2024082207522225300_ocad259-B34","author":"Loshchilov","year":"2017"},{"issue":"5","key":"2024082207522225300_ocad259-B35","doi-asserted-by":"crossref","first-page":"601","DOI":"10.1136\/amiajnl-2011-000163","article-title":"A study of machine-learning-based approaches to extract clinical entities and their assertions from discharge summaries","volume":"18","author":"Jiang","year":"2011","journal-title":"J Am Med Inform Assoc"},{"key":"2024082207522225300_ocad259-B36","year":"2022"},{"key":"2024082207522225300_ocad259-B37","year":"2023"},{"key":"2024082207522225300_ocad259-B38","year":"2023"},{"key":"2024082207522225300_ocad259-B39","year":"2023"},{"key":"2024082207522225300_ocad259-B40","year":"2022"},{"key":"2024082207522225300_ocad259-B41","year":"2023"},{"key":"2024082207522225300_ocad259-B42","year":"2022"}],"container-title":["Journal of the American Medical Informatics Association"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/academic.oup.com\/jamia\/article-pdf\/31\/9\/1812\/58868277\/ocad259.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"syndication"},{"URL":"https:\/\/academic.oup.com\/jamia\/article-pdf\/31\/9\/1812\/58868277\/ocad259.pdf","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,8,22]],"date-time":"2024-08-22T12:34:43Z","timestamp":1724330083000},"score":1,"resource":{"primary":{"URL":"https:\/\/academic.oup.com\/jamia\/article\/31\/9\/1812\/7590607"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,1,27]]},"references-count":42,"journal-issue":{"issue":"9","published-online":{"date-parts":[[2024,1,27]]},"published-print":{"date-parts":[[2024,9,1]]}},"URL":"https:\/\/doi.org\/10.1093\/jamia\/ocad259","relation":{},"ISSN":["1067-5027","1527-974X"],"issn-type":[{"value":"1067-5027","type":"print"},{"value":"1527-974X","type":"electronic"}],"subject":[],"published-other":{"date-parts":[[2024,9]]},"published":{"date-parts":[[2024,1,27]]}}}