{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,9,16]],"date-time":"2026-09-16T19:04:30Z","timestamp":1789585470578,"version":"build-2803163510"},"reference-count":32,"publisher":"Oxford University Press (OUP)","issue":"9","license":[{"start":{"date-parts":[[2026,6,15]],"date-time":"2026-06-15T00:00:00Z","timestamp":1781481600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/academic.oup.com\/pages\/standard-publication-reuse-rights"}],"funder":[{"DOI":"10.13039\/501100012166","name":"National Key Research and Development Program of China","doi-asserted-by":"publisher","award":["2024YFF1207400"],"award-info":[{"award-number":["2024YFF1207400"]}],"id":[{"id":"10.13039\/501100012166","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100012166","name":"National Key Research and Development Program of China","doi-asserted-by":"publisher","award":["2023YFC3402800"],"award-info":[{"award-number":["2023YFC3402800"]}],"id":[{"id":"10.13039\/501100012166","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100012166","name":"National Key Research and Development Program of China","doi-asserted-by":"publisher","award":["2021YFA1301603"],"award-info":[{"award-number":["2021YFA1301603"]}],"id":[{"id":"10.13039\/501100012166","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["12571555"],"award-info":[{"award-number":["12571555"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["62331001"],"award-info":[{"award-number":["62331001"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["62333022"],"award-info":[{"award-number":["62333022"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["82272068"],"award-info":[{"award-number":["82272068"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["92359302"],"award-info":[{"award-number":["92359302"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["92259301"],"award-info":[{"award-number":["92259301"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100004826","name":"Beijing Natural Science Foundation","doi-asserted-by":"publisher","award":["JQ23034"],"award-info":[{"award-number":["JQ23034"]}],"id":[{"id":"10.13039\/501100004826","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100004826","name":"Beijing Natural Science Foundation","doi-asserted-by":"publisher","award":["L234056"],"award-info":[{"award-number":["L234056"]}],"id":[{"id":"10.13039\/501100004826","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100004826","name":"Beijing Natural Science Foundation","doi-asserted-by":"publisher","award":["L232097"],"award-info":[{"award-number":["L232097"]}],"id":[{"id":"10.13039\/501100004826","id-type":"DOI","asserted-by":"publisher"}]},{"name":"Young Elite Scientists Sponsorship Program","award":["2023QNRC001"],"award-info":[{"award-number":["2023QNRC001"]}]}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2026,9,1]]},"abstract":"<jats:title>Abstract<\/jats:title>\n                  <jats:sec>\n                    <jats:title>Objective<\/jats:title>\n                    <jats:p>Large language models (LLMs) have shown promise in clinical applications, yet prior studies mainly evaluated their standalone performance on benchmarks or examinations. This study assesses how LLMs support clinicians across specialties, disease contexts, experience levels and decision-making stages.<\/jats:p>\n                  <\/jats:sec>\n                  <jats:sec>\n                    <jats:title>Methods<\/jats:title>\n                    <jats:p>We evaluated 3 LLMs (Deepseek-R1, GPT-4o-mini and LLaMA-4) using 2 task types: (1) general-disease tasks spanning specialties and disease incidence levels, and (2) prostate disease scenario covering the clinical workflow, including diagnosis, treatment planning, postoperative rehabilitation and prognosis. Clinicians with different seniority completed tasks independently and repeated them after reviewing LLM-generated responses. Performance was rated by experts using a 5-point Likert scale, and differences were analyzed with the Wilcoxon signed-rank test.<\/jats:p>\n                  <\/jats:sec>\n                  <jats:sec>\n                    <jats:title>Results<\/jats:title>\n                    <jats:p>LLM assistance significantly improved clinician performance across general disease tasks (P\u2009&amp;lt;\u2009.05). In the specialty scenario, junior clinicians\u2019 scores increased by 0.579-0.723 (15.9%-20.8%) across 4 clinical stages, whereas senior clinicians\u2019 improvements ranged from 0.053-0.303 (1.3%-7.5%). With LLM support, junior clinicians surpassed senior clinicians\u2019 unaided performance in postoperative rehabilitation and prognosis stages (P\u2009&amp;lt;\u2009.05) and achieved comparable performance in diagnosis and treatment stages (P\u2009&amp;gt;\u2009.05). Performance varied among models, with Deepseek-R1 performing best in diagnostic tasks.<\/jats:p>\n                  <\/jats:sec>\n                  <jats:sec>\n                    <jats:title>Discussion<\/jats:title>\n                    <jats:p>These findings suggest that LLM assistance may provide greater benefit to less-experienced clinicians and are particularly effective in downstream decision-making stages, indicating a potential role in mitigating experience-related performance disparities.<\/jats:p>\n                  <\/jats:sec>\n                  <jats:sec>\n                    <jats:title>Conclusions<\/jats:title>\n                    <jats:p>LLMs may serve as supportive clinical-decision support tools across diseases, specialties, clinician experience levels, and workflow stages, with observed improvements in decision quality and reduced performance disparities.<\/jats:p>\n                  <\/jats:sec>","DOI":"10.1093\/jamia\/ocag053","type":"journal-article","created":{"date-parts":[[2026,4,2]],"date-time":"2026-04-02T11:52:39Z","timestamp":1775130759000},"page":"1660-1670","source":"Crossref","is-referenced-by-count":0,"title":["Augmenting clinical decision-making with large language models: evaluation across general and specialty tasks"],"prefix":"10.1093","volume":"33","author":[{"given":"Yuechen","family":"Tao","sequence":"first","affiliation":[{"name":"CAS Key Laboratory of Molecular Imaging, Institute of Automation, Chinese Academy of Sciences , Beijing, 100190,","place":["China"]},{"name":"School of Artificial Intelligence, University of Chinese Academy of Sciences , Beijing, 100049,","place":["China"]}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Xiushi","family":"Lin","sequence":"additional","affiliation":[{"name":"Department of Urology, Peking University Third Hospital , Beijing, 100191,","place":["China"]}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"He","family":"Sun","sequence":"additional","affiliation":[{"name":"School of Engineering Medicine and School of Biological Science and Medical Engineering, Beihang University , Beijing, 100191,","place":["China"]}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Wenrui","family":"Bao","sequence":"additional","affiliation":[{"name":"School of Artificial Intelligence, University of Chinese Academy of Sciences , Beijing, 100049,","place":["China"]}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Xiaobo","family":"Zhu","sequence":"additional","affiliation":[{"name":"CAS Key Laboratory of Molecular Imaging, Institute of Automation, Chinese Academy of Sciences , Beijing, 100190,","place":["China"]},{"name":"School of Artificial Intelligence, University of Chinese Academy of Sciences , Beijing, 100049,","place":["China"]}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Shiwei","family":"Wu","sequence":"additional","affiliation":[{"name":"CAS Key Laboratory of Molecular Imaging, Institute of Automation, Chinese Academy of Sciences , Beijing, 100190,","place":["China"]},{"name":"School of Computer Science and Engineering, Nanjing University of Science and Technology , Nanjing, 210094,","place":["China"]}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Yixiao","family":"Liu","sequence":"additional","affiliation":[{"name":"Department of Urology, Peking University Third Hospital , Beijing, 100191,","place":["China"]}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Ziang","family":"Li","sequence":"additional","affiliation":[{"name":"Department of Urology, Peking University Third Hospital , Beijing, 100191,","place":["China"]}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Jialong","family":"Wu","sequence":"additional","affiliation":[{"name":"Department of Urology, Peking University Third Hospital , Beijing, 100191,","place":["China"]}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Zhenkun","family":"Zhao","sequence":"additional","affiliation":[{"name":"Department of Urology, Peking University Third Hospital , Beijing, 100191,","place":["China"]}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Lei","family":"Qiu","sequence":"additional","affiliation":[{"name":"Department of Urology, Peking University Third Hospital , Beijing, 100191,","place":["China"]}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Qianyang","family":"Ni","sequence":"additional","affiliation":[{"name":"Department of Urology, Peking University Third Hospital , Beijing, 100191,","place":["China"]}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Song","family":"Zhang","sequence":"additional","affiliation":[{"name":"CAS Key Laboratory of Molecular Imaging, Institute of Automation, Chinese Academy of Sciences , Beijing, 100190,","place":["China"]},{"name":"School of Artificial Intelligence, University of Chinese Academy of Sciences , Beijing, 100049,","place":["China"]}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Zhenchao","family":"Tang","sequence":"additional","affiliation":[{"name":"School of Engineering Medicine and School of Biological Science and Medical Engineering, Beihang University , Beijing, 100191,","place":["China"]}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Zuobin","family":"Ying","sequence":"additional","affiliation":[{"name":"Faculty of Data Science, City University of Macau , Macau, 999078,","place":["China"]}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Yu","family":"An","sequence":"additional","affiliation":[{"name":"School of Engineering Medicine and School of Biological Science and Medical Engineering, Beihang University , Beijing, 100191,","place":["China"]}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Jie","family":"Tian","sequence":"additional","affiliation":[{"name":"CAS Key Laboratory of Molecular Imaging, Institute of Automation, Chinese Academy of Sciences , Beijing, 100190,","place":["China"]},{"name":"School of Artificial Intelligence, University of Chinese Academy of Sciences , Beijing, 100049,","place":["China"]}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Zhenyu","family":"Liu","sequence":"additional","affiliation":[{"name":"CAS Key Laboratory of Molecular Imaging, Institute of Automation, Chinese Academy of Sciences , Beijing, 100190,","place":["China"]},{"name":"School of Artificial Intelligence, University of Chinese Academy of Sciences , Beijing, 100049,","place":["China"]}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Jian","family":"Lu","sequence":"additional","affiliation":[{"name":"Department of Urology, Peking University Third Hospital , Beijing, 100191,","place":["China"]}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"286","published-online":{"date-parts":[[2026,6,15]]},"reference":[{"key":"2026091614320629500_ocag053-B1","doi-asserted-by":"publisher","first-page":"1808","DOI":"10.4081\/jphr.2020.1808","article-title":"Factors affecting emergency nurses\u2019 perceptions of the triage systems","volume":"9","author":"Sutriningsih","year":"2020","journal-title":"J Public Health Res"},{"key":"2026091614320629500_ocag053-B2","doi-asserted-by":"crossref","first-page":"382","DOI":"10.1136\/bmjqs-2018-008390","article-title":"Are more experienced clinicians better able to tolerate uncertainty and manage risks? A vignette study of doctors in three NHS emergency departments in England","volume":"28","author":"Lawton","journal-title":"BMJ Qual Saf"},{"key":"2026091614320629500_ocag053-B3","doi-asserted-by":"publisher","first-page":"2606","DOI":"10.1016\/j.pec.2021.07.028","article-title":"Patient and clinician experiences of uncertainty in the diagnostic process: current understanding and future directions","volume":"104","author":"Meyer","year":"2021","journal-title":"Patient Educ Couns"},{"issue":"5","key":"2026091614320629500_ocag053-B4","doi-asserted-by":"crossref","first-page":"1303","DOI":"10.7150\/thno.30309","article-title":"The applications of radiomics in precision diagnosis and treatment of oncology: opportunities and challenges","volume":"9","author":"Liu","year":"2019","journal-title":"Theranostics"},{"key":"2026091614320629500_ocag053-B5","doi-asserted-by":"publisher","author":"OpenAI","year":"2024","DOI":"10.48550\/arXiv.2410.21276"},{"key":"2026091614320629500_ocag053-B6","doi-asserted-by":"publisher","author":"Touvron","year":"2023","DOI":"10.48550\/arXiv.2302.13971"},{"key":"2026091614320629500_ocag053-B7","doi-asserted-by":"publisher","first-page":"633","DOI":"10.1038\/s41586-025-09422-z","article-title":"DeepSeek-R1 incentivizes reasoning in LLMs through reinforcement learning","volume":"645","author":"Guo","year":"2025","journal-title":"Nature"},{"key":"2026091614320629500_ocag053-B8","doi-asserted-by":"publisher","author":"Anil","year":"2023","DOI":"10.48550\/arXiv.2305.10403"},{"key":"2026091614320629500_ocag053-B9","doi-asserted-by":"publisher","author":"Bai","year":"2023","DOI":"10.48550\/arXiv.2309.16609"},{"key":"2026091614320629500_ocag053-B10","doi-asserted-by":"crossref","first-page":"60","DOI":"10.1038\/s41591-024-03425-5","article-title":"The TRIPOD-LLM reporting guideline for studies using large language models","volume":"31","author":"Gallifant","year":"2025","journal-title":"Nat Med"},{"key":"2026091614320629500_ocag053-B11","first-page":"37","year":"2022"},{"key":"2026091614320629500_ocag053-B12","doi-asserted-by":"publisher","first-page":"4324","DOI":"10.1109\/CVPRW63382.2024.00436","author":"Jia","year":"2024"},{"key":"2026091614320629500_ocag053-B13","doi-asserted-by":"crossref","first-page":"e2512994","DOI":"10.1001\/jamanetworkopen.2025.12994","article-title":"Dedicated AI expert system vs generative AI with large language model for clinical diagnoses","volume":"8","author":"Feldman","journal-title":"JAMA Network Open"},{"key":"2026091614320629500_ocag053-B14","doi-asserted-by":"publisher","first-page":"18417","DOI":"10.1609\/aaai.v38i16.29802","article-title":"Large language models are clinical reasoners: reasoning-aware diagnosis framework with prompt-generated rationales","volume":"38","author":"Kwon","year":"2024","journal-title":"AAAI"},{"key":"2026091614320629500_ocag053-B15","doi-asserted-by":"crossref","first-page":"e223312","DOI":"10.1148\/radiol.223312","article-title":"ChatGPT and the future of medical writing","volume":"307","author":"Biswas","journal-title":"Radiology"},{"key":"2026091614320629500_ocag053-B16","doi-asserted-by":"publisher","first-page":"2550","DOI":"10.1038\/s41591-025-03726-3","article-title":"Comparative benchmarking of the DeepSeek large language model on medical tasks and clinical reasoning","volume":"31","author":"Tordjman","year":"2025","journal-title":"Nat Med"},{"key":"2026091614320629500_ocag053-B17","doi-asserted-by":"publisher","first-page":"934","DOI":"10.1016\/j.scib.2025.01.034","article-title":"Large language models for diabetes training: a prospective study","volume":"70","author":"Li","year":"2025","journal-title":"Sci Bull (Beijing)"},{"key":"2026091614320629500_ocag053-B18","doi-asserted-by":"crossref","first-page":"e0000198","DOI":"10.1371\/journal.pdig.0000198","article-title":"Performance of ChatGPT on USMLE: potential for AI-assisted medical education using large language models","volume":"2","author":"Kung","journal-title":"PLOS Digital Health"},{"key":"2026091614320629500_ocag053-B19","doi-asserted-by":"publisher","first-page":"2546","DOI":"10.1038\/s41591-025-03727-2","article-title":"Benchmark evaluation of DeepSeek large language models in clinical decision-making","volume":"31","author":"Sandmann","year":"2025","journal-title":"Nat Med"},{"issue":"4","key":"2026091614320629500_ocag053-B20","article-title":"Evaluating GPT-4-based ChatGPT\u2019s clinical potential on the NEJM quiz","volume":"2","author":"Ueda","year":"2024","journal-title":"BMC Digital Health"},{"key":"2026091614320629500_ocag053-B21","doi-asserted-by":"publisher","first-page":"2050","DOI":"10.1038\/s41467-024-46411-8","article-title":"Systematic analysis of ChatGPT, Google search and Llama 2 for clinical decision support tasks","volume":"15","author":"Sandmann","year":"2024","journal-title":"Nat Commun"},{"key":"2026091614320629500_ocag053-B22","author":"Hou","year":"2024"},{"key":"2026091614320629500_ocag053-B23","doi-asserted-by":"crossref","first-page":"78","DOI":"10.1001\/jama.2023.8288","article-title":"Accuracy of a generative artificial intelligence model in a complex diagnostic challenge","volume":"330","author":"Kanjee","year":"2023","journal-title":"JAMA."},{"key":"2026091614320629500_ocag053-B24","doi-asserted-by":"crossref","first-page":"e71916","DOI":"10.2196\/71916","article-title":"Implementing large language models in health care: clinician-focused review with interactive guideline","volume":"27","author":"Li","year":"2025","journal-title":"J Med Internet Res."},{"key":"2026091614320629500_ocag053-B25","doi-asserted-by":"publisher","first-page":"172","DOI":"10.1038\/s41586-023-06291-2","article-title":"Large language models encode clinical knowledge","volume":"620","author":"Singhal","year":"2023","journal-title":"Nature"},{"key":"2026091614320629500_ocag053-B26","doi-asserted-by":"publisher","first-page":"275","DOI":"10.1007\/s10462-012-9338-y","article-title":"Mixture of experts: a literature survey","volume":"42","author":"Masoudnia","year":"2014","journal-title":"Artif Intell Rev"},{"key":"2026091614320629500_ocag053-B27","doi-asserted-by":"crossref","first-page":"63529","DOI":"10.52202\/075280-2774","article-title":"Achieving cross modal generalization with multimodal unified representation","volume":"36","author":"Xia","year":"2023","journal-title":"Adv Neural Inf Process Syst"},{"key":"2026091614320629500_ocag053-B28","doi-asserted-by":"crossref","first-page":"148","DOI":"10.1186\/s13054-023-04425-6","article-title":"Artificial hallucination: GPT on LSD?","volume":"27","author":"Beutel","year":"2023","journal-title":"Crit Care"},{"key":"2026091614320629500_ocag053-B29","doi-asserted-by":"publisher","author":"Shuster","DOI":"10.18653\/v1\/2021.findings-emnlp.320"},{"key":"2026091614320629500_ocag053-B30","doi-asserted-by":"publisher","author":"Han","year":"2023","DOI":"10.1101\/2023.11.03.23297957"},{"key":"2026091614320629500_ocag053-B31","doi-asserted-by":"crossref","first-page":"58","DOI":"10.1038\/s41746-024-01390-4","article-title":"Towards evaluating and building versatile large language models for medicine","volume":"8","author":"Wu","year":"2025","journal-title":"npj Digital Med"},{"key":"2026091614320629500_ocag053-B32","author":"Ma"}],"container-title":["Journal of the American Medical Informatics Association"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/academic.oup.com\/jamia\/advance-article-pdf\/doi\/10.1093\/jamia\/ocag053\/68524098\/ocag053.pdf","content-type":"application\/pdf","content-version":"am","intended-application":"syndication"},{"URL":"https:\/\/academic.oup.com\/jamia\/article-pdf\/33\/9\/1660\/68524098\/ocag053.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"syndication"},{"URL":"https:\/\/academic.oup.com\/jamia\/article-pdf\/33\/9\/1660\/68524098\/ocag053.pdf","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,9,16]],"date-time":"2026-09-16T18:32:15Z","timestamp":1789583535000},"score":1,"resource":{"primary":{"URL":"https:\/\/academic.oup.com\/jamia\/article\/33\/9\/1660\/8707792"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026,6,15]]},"references-count":32,"journal-issue":{"issue":"9","published-online":{"date-parts":[[2026,6,15]]},"published-print":{"date-parts":[[2026,9,1]]}},"URL":"https:\/\/doi.org\/10.1093\/jamia\/ocag053","relation":{},"ISSN":["1067-5027","1527-974X"],"issn-type":[{"value":"1067-5027","type":"print"},{"value":"1527-974X","type":"electronic"}],"subject":[],"published-other":{"date-parts":[[2026,9]]},"published":{"date-parts":[[2026,6,15]]}}}