{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,29]],"date-time":"2026-10-29T17:13:24Z","timestamp":1793294004139,"version":"4.5.0"},"reference-count":115,"publisher":"Springer Science and Business Media LLC","license":[{"start":{"date-parts":[[2026,6,17]],"date-time":"2026-06-17T00:00:00Z","timestamp":1781654400000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2026,6,17]],"date-time":"2026-06-17T00:00:00Z","timestamp":1781654400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"funder":[{"DOI":"10.13039\/100006093","name":"Patient-Centered Outcomes Research Institute","doi-asserted-by":"publisher","award":["ME-2022C1-25646"],"award-info":[{"award-number":["ME-2022C1-25646"]}],"id":[{"id":"10.13039\/100006093","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000049","name":"U.S. Department of Health & Human Services | NIH | National Institute on Aging","doi-asserted-by":"publisher","award":["RF1AG090405"],"award-info":[{"award-number":["RF1AG090405"]}],"id":[{"id":"10.13039\/100000049","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000092","name":"U.S. Department of Health & Human Services | NIH | U.S. National Library of Medicine","doi-asserted-by":"publisher","award":["R01LM014667"],"award-info":[{"award-number":["R01LM014667"]}],"id":[{"id":"10.13039\/100000092","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100008552","name":"BWH | Brigham Research Institute","doi-asserted-by":"publisher","id":[{"id":"10.13039\/100008552","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Nat. Biomed. Eng"],"DOI":"10.1038\/s41551-026-01719-2","type":"journal-article","created":{"date-parts":[[2026,6,17]],"date-time":"2026-06-17T09:04:32Z","timestamp":1781687072000},"update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":10,"title":["BRIDGE: benchmarking large language models for understanding real-world clinical practice texts"],"prefix":"10.1038","author":[{"given":"Jiageng","family":"Wu","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-0739-0390","authenticated-orcid":false,"given":"Bowen","family":"Gu","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Ren","family":"Zhou","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Kevin","family":"Xie","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Doug","family":"Snyder","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Yixing","family":"Jiang","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Valentina","family":"Carducci","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Richard","family":"Wyss","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-0299-7273","authenticated-orcid":false,"given":"Rishi J.","family":"Desai","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-5370-1746","authenticated-orcid":false,"given":"Emily","family":"Alsentzer","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-6712-6626","authenticated-orcid":false,"given":"Leo Anthony","family":"Celi","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Adam","family":"Rodman","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Sebastian","family":"Schneeweiss","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Jonathan H.","family":"Chen","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Santiago","family":"Romero-Brufau","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-4564-3715","authenticated-orcid":false,"given":"Kueiyu Joshua","family":"Lin","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-5696-363X","authenticated-orcid":false,"given":"Jie","family":"Yang","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2026,6,17]]},"reference":[{"key":"1719_CR1","doi-asserted-by":"publisher","first-page":"1930","DOI":"10.1038\/s41591-023-02448-8","volume":"29","author":"AJ Thirunavukarasu","year":"2023","unstructured":"Thirunavukarasu, A. J. et al. Large language models in medicine. Nat. Med. 29, 1930\u20131940 (2023).","journal-title":"Nat. Med."},{"key":"1719_CR2","unstructured":"Kaplan, J. et al. Scaling laws for neural language models. Preprint at https:\/\/arxiv.org\/abs\/2001.08361 (2020)."},{"key":"1719_CR3","first-page":"169","volume":"58","author":"S Zhang","year":"2026","unstructured":"Zhang, S. et al. Instruction tuning for large language models: a survey. ACM Comput. Surv. 58, 169 (2026).","journal-title":"ACM Comput. Surv."},{"key":"1719_CR4","doi-asserted-by":"publisher","first-page":"30","DOI":"10.1145\/3605943","volume":"56","author":"B Min","year":"2024","unstructured":"Min, B. et al. Recent advances in natural language processing via large pre-trained language models: a survey. ACM Comput. Surv. 56, 30 (2024).","journal-title":"ACM Comput. Surv."},{"key":"1719_CR5","doi-asserted-by":"publisher","unstructured":"Qin, C. et al. Is ChatGPT a general-purpose natural language processing task solver? In Proc. 2023 Conference on Empirical Methods in Natural Language Processing (eds Bouamor, H et al.) 1339\u20131384 (Association for Computational Linguistics, 2023); https:\/\/doi.org\/10.18653\/v1\/2023.emnlp-main.85","DOI":"10.18653\/v1\/2023.emnlp-main.85"},{"key":"1719_CR6","doi-asserted-by":"publisher","unstructured":"Huang, J. & Chang, K. C.-C. Towards reasoning in large language models: a survey. In Findings of the Association for Computational Linguistics (eds Rogers, A. et al.) 1049\u20131065 (Association for Computational Linguistics, 2023); https:\/\/doi.org\/10.18653\/v1\/2023.findings-acl.67","DOI":"10.18653\/v1\/2023.findings-acl.67"},{"key":"1719_CR7","doi-asserted-by":"publisher","first-page":"158","DOI":"10.1038\/s41746-023-00896-7","volume":"6","author":"L Tang","year":"2023","unstructured":"Tang, L. et al. Evaluating large language models on medical evidence summarization. npj Digit. Med. 6, 158 (2023).","journal-title":"npj Digit. Med."},{"key":"1719_CR8","doi-asserted-by":"publisher","first-page":"1134","DOI":"10.1038\/s41591-024-02855-5","volume":"30","author":"D Van Veen","year":"2024","unstructured":"Van Veen, D. et al. Adapted large language models can outperform medical experts in clinical text summarization. Nat. Med. 30, 1134\u20131142 (2024).","journal-title":"Nat. Med."},{"key":"1719_CR9","doi-asserted-by":"publisher","first-page":"357","DOI":"10.1038\/s41586-023-06160-y","volume":"619","author":"LY Jiang","year":"2023","unstructured":"Jiang, L. Y. et al. Health system-scale language models are all-purpose prediction engines. Nature 619, 357\u2013362 (2023).","journal-title":"Nature"},{"key":"1719_CR10","doi-asserted-by":"publisher","first-page":"1233","DOI":"10.1038\/s41591-024-03456-y","volume":"31","author":"E Goh","year":"2025","unstructured":"Goh, E. et al. GPT-4 assistance for improvement of physician performance on patient care tasks: a randomized controlled trial. Nat. Med. 31, 1233\u20131238 (2025).","journal-title":"Nat. Med."},{"key":"1719_CR11","doi-asserted-by":"publisher","first-page":"102465","DOI":"10.1016\/j.xcrm.2025.102465","volume":"6","author":"Z Duan","year":"2025","unstructured":"Duan, Z. et al. Multi-center benchmarking of large language models for clinical decision support in lung cancer screening. Cell Rep. Med. 6, 102465 (2025).","journal-title":"Cell Rep. Med."},{"key":"1719_CR12","doi-asserted-by":"publisher","first-page":"2817","DOI":"10.1007\/s00330-023-10213-1","volume":"34","author":"K Jeblick","year":"2024","unstructured":"Jeblick, K. et al. ChatGPT makes medicine easy to swallow: an exploratory case study on simplified radiology reports. Eur. Radiol. 34, 2817\u20132825 (2024).","journal-title":"Eur. Radiol."},{"key":"1719_CR13","doi-asserted-by":"publisher","first-page":"e2336483","DOI":"10.1001\/jamanetworkopen.2023.36483","volume":"6","author":"RS Goodman","year":"2023","unstructured":"Goodman, R. S. et al. Accuracy and reliability of chatbot responses to physician questions. JAMA Netw. Open 6, e2336483 (2023).","journal-title":"JAMA Netw. Open"},{"key":"1719_CR14","doi-asserted-by":"publisher","first-page":"583","DOI":"10.1016\/j.scib.2024.01.004","volume":"69","author":"B Sheng","year":"2024","unstructured":"Sheng, B. et al. Large language models for diabetes care: potentials and prospects. Sci. Bull. 69, 583\u2013588 (2024).","journal-title":"Sci. Bull."},{"key":"1719_CR15","doi-asserted-by":"publisher","first-page":"e240357","DOI":"10.1001\/jamanetworkopen.2024.0357","volume":"7","author":"J Zaretsky","year":"2024","unstructured":"Zaretsky, J. et al. Generative artificial intelligence to transform inpatient discharge summaries to patient-friendly language and format. JAMA Netw. Open 7, e240357 (2024).","journal-title":"JAMA Netw. Open"},{"key":"1719_CR16","first-page":"100905","volume":"41","author":"X Wang","year":"2023","unstructured":"Wang, X. et al. ChatGPT: promise and challenges for deployment in low- and middle-income countries. Lancet Reg. Health West. Pac. 41, 100905 (2023).","journal-title":"Lancet Reg. Health West. Pac."},{"key":"1719_CR17","doi-asserted-by":"publisher","first-page":"833","DOI":"10.1056\/NEJMra2214964","volume":"389","author":"K Ferryman","year":"2023","unstructured":"Ferryman, K., Mackintosh, M. & Ghassemi, M. Considering biased data as informative artifacts in AI-assisted health care. N. Engl. J. Med. 389, 833\u2013838 (2023).","journal-title":"N. Engl. J. Med."},{"key":"1719_CR18","doi-asserted-by":"publisher","first-page":"1561","DOI":"10.1056\/NEJMp2405999","volume":"391","author":"LG McCoy","year":"2024","unstructured":"McCoy, L. G., Manrai, A. K. & Rodman, A. Large language models and the degradation of the medical record. N. Engl. J. Med. 391, 1561\u20131564 (2024).","journal-title":"N. Engl. J. Med."},{"key":"1719_CR19","doi-asserted-by":"publisher","first-page":"175","DOI":"10.1038\/s41746-025-01543-z","volume":"8","author":"H Takita","year":"2025","unstructured":"Takita, H. et al. A systematic review and meta-analysis of diagnostic performance comparison between generative AI and physicians. npj Digit. Med. 8, 175 (2025).","journal-title":"npj Digit. Med."},{"key":"1719_CR20","doi-asserted-by":"publisher","DOI":"10.1056\/AIe2401235","volume":"2","author":"ID Raji","year":"2025","unstructured":"Raji, I. D., Daneshjou, R. & Alsentzer, E. It\u2019s time to bench the medical exam benchmark. NEJM AI 2, AIe2401235 (2025).","journal-title":"NEJM AI"},{"key":"1719_CR21","doi-asserted-by":"publisher","first-page":"6421","DOI":"10.3390\/app11146421","volume":"11","author":"D Jin","year":"2021","unstructured":"Jin, D. et al. What disease does this patient have? A large-scale open domain question answering dataset from medical exams. Appl. Sci. 11, 6421 (2021).","journal-title":"Appl. Sci."},{"key":"1719_CR22","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1016\/j.patter.2024.100943","volume":"5","author":"V Li\u00e9vin","year":"2024","unstructured":"Li\u00e9vin, V., Hother, C. E., Motzfeldt, A. G. & Winther, O. Can large language models reason about medical questions?. Patterns 5, 1\u201312 (2024).","journal-title":"Patterns"},{"key":"1719_CR23","doi-asserted-by":"publisher","unstructured":"Jin, Q., Dhingra, B., Liu, Z., Cohen, W. & Lu, X. PubMedQA: a dataset for biomedical research question answering. In Proc. 2019 Conference on Empirical Methods in Natural Language Processing and the 9th International Joint Conference on Natural Language Processing (eds Inui, K. et al.) 2567\u20132577 (Association for Computational Linguistics, 2019); https:\/\/doi.org\/10.18653\/v1\/D19-1259","DOI":"10.18653\/v1\/D19-1259"},{"key":"1719_CR24","doi-asserted-by":"publisher","DOI":"10.1038\/s41467-025-56989-2","volume":"16","author":"Q Chen","year":"2025","unstructured":"Chen, Q. et al. Benchmarking large language models for biomedical natural language processing applications and recommendations. Nat. Commun. 16, 3280 (2025).","journal-title":"Nat. Commun."},{"key":"1719_CR25","doi-asserted-by":"publisher","first-page":"14","DOI":"10.1016\/j.jbi.2017.07.012","volume":"73","author":"K Kreimeyer","year":"2017","unstructured":"Kreimeyer, K. et al. Natural language processing systems for capturing and standardizing unstructured clinical information: a systematic review. J. Biomed. Inform. 73, 14\u201329 (2017).","journal-title":"J. Biomed. Inform."},{"key":"1719_CR26","unstructured":"OpenAI. GPT-4 technical report. Preprint at https:\/\/arxiv.org\/abs\/2303.08774 (2023)."},{"key":"1719_CR27","doi-asserted-by":"publisher","first-page":"943","DOI":"10.1038\/s41591-024-03423-7","volume":"31","author":"K Singhal","year":"2025","unstructured":"Singhal, K. et al. Toward expert-level medical question answering with large language models. Nat. Med. 31, 943\u2013950 (2025).","journal-title":"Nat. Med."},{"key":"1719_CR28","doi-asserted-by":"publisher","first-page":"AIdbp2300040","DOI":"10.1056\/AIdbp2300040","volume":"1","author":"A Soroush","year":"2024","unstructured":"Soroush, A. et al. Large language models are poor medical coders\u2014benchmarking of medical code querying. NEJM AI 1, AIdbp2300040 (2024).","journal-title":"NEJM AI"},{"key":"1719_CR29","doi-asserted-by":"publisher","first-page":"313","DOI":"10.1001\/jamapediatrics.2023.5750","volume":"178","author":"J Barile","year":"2024","unstructured":"Barile, J. et al. Diagnostic accuracy of a large language model in pediatric case studies. JAMA Pediatr. 178, 313\u2013315 (2024).","journal-title":"JAMA Pediatr."},{"key":"1719_CR30","unstructured":"Arora, R. K. et al. HealthBench: evaluating large language models towards improved human health. Preprint at https:\/\/arxiv.org\/abs\/2505.08775 (2025)."},{"key":"1719_CR31","doi-asserted-by":"publisher","first-page":"e2440969","DOI":"10.1001\/jamanetworkopen.2024.40969","volume":"7","author":"E Goh","year":"2024","unstructured":"Goh, E. et al. Large language model influence on diagnostic reasoning: a randomized clinical trial. JAMA Netw. Open 7, e2440969 (2024).","journal-title":"JAMA Netw. Open"},{"key":"1719_CR32","doi-asserted-by":"publisher","first-page":"1026","DOI":"10.1001\/jamainternmed.2023.2561","volume":"183","author":"A Nayak","year":"2023","unstructured":"Nayak, A. et al. Comparison of history of present illness summaries generated by a chatbot and senior internal medicine residents. JAMA Int. Med. 183, 1026\u20131027 (2023).","journal-title":"JAMA Int. Med."},{"key":"1719_CR33","doi-asserted-by":"publisher","first-page":"2546","DOI":"10.1038\/s41591-025-03727-2","volume":"31","author":"S Sandmann","year":"2025","unstructured":"Sandmann, S. et al. Benchmark evaluation of DeepSeek large language models in clinical decision-making. Nat. Med. 31, 2546\u20132549 (2025).","journal-title":"Nat. Med."},{"key":"1719_CR34","doi-asserted-by":"publisher","first-page":"2550","DOI":"10.1038\/s41591-025-03726-3","volume":"31","author":"M Tordjman","year":"2025","unstructured":"Tordjman, M. et al. Comparative benchmarking of the DeepSeek large language model on medical tasks and clinical reasoning. Nat. Med. 31, 2550\u20132555 (2025).","journal-title":"Nat. Med."},{"key":"1719_CR35","doi-asserted-by":"publisher","first-page":"943","DOI":"10.1038\/s41591-025-04151-2","volume":"32","author":"S Bedi","year":"2026","unstructured":"Bedi, S. et al. Holistic evaluation of large language models for medical tasks with MedHELM. Nat. Med. 32, 943\u2013951 (2026).","journal-title":"Nat. Med."},{"key":"1719_CR36","doi-asserted-by":"publisher","first-page":"58","DOI":"10.1038\/s41746-024-01390-4","volume":"8","author":"C Wu","year":"2025","unstructured":"Wu, C. et al. Towards evaluating and building versatile large language models for medicine. npj Digit. Med. 8, 58 (2025).","journal-title":"npj Digit. Med."},{"key":"1719_CR37","doi-asserted-by":"publisher","DOI":"10.1038\/s41467-024-52417-z","volume":"15","author":"P Qiu","year":"2024","unstructured":"Qiu, P. et al. Towards building multilingual language model for medicine. Nat. Commun. 15, 8384 (2024).","journal-title":"Nat. Commun."},{"key":"1719_CR38","doi-asserted-by":"publisher","first-page":"e12","DOI":"10.1016\/S2589-7500(23)00225-X","volume":"6","author":"T Zack","year":"2024","unstructured":"Zack, T. et al. Assessing the potential of GPT-4 to perpetuate racial and gender biases in health care: a model evaluation study. Lancet Digit. Health 6, e12\u2013e22 (2024).","journal-title":"Lancet Digit. Health"},{"key":"1719_CR39","doi-asserted-by":"publisher","DOI":"10.2196\/51926","volume":"26","author":"X Liu","year":"2024","unstructured":"Liu, X. et al. Uncovering language disparity of ChatGPT on retinal vascular disease classification: cross-sectional study. J. Med. Internet Res. 26, e51926 (2024).","journal-title":"J. Med. Internet Res."},{"key":"1719_CR40","doi-asserted-by":"publisher","first-page":"e2338050","DOI":"10.1001\/jamanetworkopen.2023.38050","volume":"6","author":"J Kim","year":"2023","unstructured":"Kim, J., Cai, Z. R., Chen, M. L., Simard, J. F. & Linos, E. Assessing biases in medical decisions via clinician and AI chatbot responses to patient vignettes. JAMA Netw. Open 6, e2338050 (2023).","journal-title":"JAMA Netw. Open"},{"key":"1719_CR41","doi-asserted-by":"publisher","unstructured":"Zeng, Q. et al. GreenPLM: cross-lingual transfer of monolingual pre-trained language models at almost no cost. In Proc. 32nd International Joint Conference on Artificial Intelligence (ed. Elkind, E.) 6290\u20136298 (International Joint Conferences on Artificial Intelligence Organization, 2023); https:\/\/doi.org\/10.24963\/ijcai.2023\/698","DOI":"10.24963\/ijcai.2023\/698"},{"key":"1719_CR42","doi-asserted-by":"publisher","first-page":"147","DOI":"10.1038\/s41586-024-07856-5","volume":"633","author":"V Hofmann","year":"2024","unstructured":"Hofmann, V., Kalluri, P. R., Jurafsky, D. & King, S. AI generates covertly racist decisions about people based on their dialect. Nature 633, 147\u2013154 (2024).","journal-title":"Nature"},{"key":"1719_CR43","doi-asserted-by":"publisher","first-page":"e428","DOI":"10.1016\/S2589-7500(24)00061-X","volume":"6","author":"JCL Ong","year":"2024","unstructured":"Ong, J. C. L. et al. Ethical and regulatory challenges of large language models in medicine. Lancet Digit. Health 6, e428\u2013e432 (2024).","journal-title":"Lancet Digit. Health"},{"key":"1719_CR44","doi-asserted-by":"publisher","first-page":"246","DOI":"10.1038\/s41746-025-01576-4","volume":"8","author":"Y Ji","year":"2025","unstructured":"Ji, Y. et al. Mitigating the risk of health inequity exacerbated by large language models. npj Digit. Med. 8, 246 (2025).","journal-title":"npj Digit. Med."},{"key":"1719_CR45","doi-asserted-by":"publisher","first-page":"39","DOI":"10.1145\/3641289","volume":"15","author":"Y Chang","year":"2024","unstructured":"Chang, Y. et al. A survey on evaluation of large language models. ACM Trans. Intell. Syst. Technol. 15, 39 (2024).","journal-title":"ACM Trans. Intell. Syst. Technol."},{"key":"1719_CR46","doi-asserted-by":"publisher","first-page":"172","DOI":"10.1038\/s41586-023-06291-2","volume":"620","author":"K Singhal","year":"2023","unstructured":"Singhal, K. et al. Large language models encode clinical knowledge. Nature 620, 172\u2013180 (2023).","journal-title":"Nature"},{"key":"1719_CR47","doi-asserted-by":"publisher","first-page":"141","DOI":"10.1038\/s41746-025-01533-1","volume":"8","author":"Q Xie","year":"2025","unstructured":"Xie, Q. et al. Medical foundation large language models for comprehensive text analysis and beyond. npj Digit. Med. 8, 141\u2013150 (2025).","journal-title":"npj Digit. Med."},{"key":"1719_CR48","doi-asserted-by":"publisher","first-page":"932","DOI":"10.1038\/s41591-024-03416-6","volume":"31","author":"X Liu","year":"2025","unstructured":"Liu, X. et al. A generalist medical language model for disease diagnosis assistance. Nat. Med. 31, 932\u2013942 (2025).","journal-title":"Nat. Med."},{"key":"1719_CR49","unstructured":"Chiang, W.-L. et al. Chatbot arena: an open platform for evaluating LLMs by human preference. In 41st International Conference on Machine Learning (eds Salakhutdinov, R. et al.) 8359\u20138388 (JMLR, 2024)."},{"key":"1719_CR50","doi-asserted-by":"publisher","first-page":"315","DOI":"10.1001\/jama.2023.9651","volume":"330","author":"T Minssen","year":"2023","unstructured":"Minssen, T., Vayena, E. & Cohen, I. G. The challenges for regulating medical use of ChatGPT and other large language models. J. Am. Med. Assoc. 330, 315\u2013316 (2023).","journal-title":"J. Am. Med. Assoc."},{"key":"1719_CR51","doi-asserted-by":"publisher","first-page":"319","DOI":"10.1001\/jama.2024.21700","volume":"333","author":"S Bedi","year":"2025","unstructured":"Bedi, S. et al. Testing and evaluation of health care applications of large language models: a systematic review. J. Am. Med. Assoc. 333, 319\u2013328 (2025).","journal-title":"J. Am. Med. Assoc."},{"key":"1719_CR52","first-page":"100622","volume":"5","author":"Y-D Tang","year":"2024","unstructured":"Tang, Y.-D., Dong, E.-D. & Gao, W. LLMs in medicine: the need for advanced evaluation systems for disruptive technologies. Innovation 5, 100622 (2024).","journal-title":"Innovation"},{"key":"1719_CR53","doi-asserted-by":"crossref","unstructured":"Wu, J. et al. Clinical text datasets for medical artificial intelligence and large language models\u2014a systematic review. NEJM AI 1, AIra2400012 (2024).","DOI":"10.1056\/AIra2400012"},{"key":"1719_CR54","unstructured":"DeepSeek-AI et al. DeepSeek-R1: incentivizing reasoning capability in LLMs via reinforcement learning. Preprint at https:\/\/arxiv.org\/abs\/2501.12948 (2025)."},{"key":"1719_CR55","unstructured":"OpenAI et al. GPT-4o system card. Preprint at https:\/\/arxiv.org\/abs\/2410.21276 (2024)."},{"key":"1719_CR56","unstructured":"Gemini Team et al. Gemini 1.5: unlocking multimodal understanding across millions of tokens of context. Preprint at https:\/\/arxiv.org\/abs\/2403.05530 (2024)."},{"key":"1719_CR57","unstructured":"Comanici, G. et al. Gemini 2.5: pushing the frontier with advanced reasoning, multimodality, long context, and next generation agentic capabilities. Preprint at https:\/\/arxiv.org\/abs\/2507.062611 (2025)."},{"key":"1719_CR58","unstructured":"The Llama 4 herd: the beginning of a new era of natively multimodal AI innovation. Meta AI https:\/\/ai.meta.com\/blog\/llama-4-multimodal-intelligence\/ (2025)."},{"key":"1719_CR59","unstructured":"Yang, A. et al. Qwen3 technical report. Preprint at https:\/\/arxiv.org\/abs\/2505.09388 (2025)."},{"key":"1719_CR60","unstructured":"Brown, T. et al. Language models are few-shot learners. Adv. Neural Inf. Process. Syst. 33, 1877\u20131901 (2020)."},{"key":"1719_CR61","doi-asserted-by":"crossref","unstructured":"Wei, J. et al. Chain-of-thought prompting elicits reasoning in large language models. In Proc. 36th International Conference on Neural Information Processing Systems (eds Koyejo, S. et al.) 24824\u201324837 (Curran Associates Inc., 2022).","DOI":"10.52202\/068431-1800"},{"key":"1719_CR62","doi-asserted-by":"crossref","unstructured":"Chen J. et al. Towards medical complex reasoning with LLMs through medical verifiable problems. In Findings of the Association for Computational Linguistics 14552\u201314573 (Association for Computational Linguistics, 2025).","DOI":"10.18653\/v1\/2025.findings-acl.751"},{"key":"1719_CR63","unstructured":"Sellergren, A. et al. MedGemma technical report. Preprint at https:\/\/arxiv.org\/abs\/2507.05201 (2025)."},{"key":"1719_CR64","unstructured":"Baichuan-M2 Team et al. Baichuan-M2: scaling medical capability with large verifier system. Preprint at https:\/\/arxiv.org\/abs\/2509.02208 (2025)."},{"key":"1719_CR65","unstructured":"Jiang, A. Q. et al. Mistral 7B. Preprint at https:\/\/arxiv.org\/abs\/2310.06825 (2023)."},{"key":"1719_CR66","unstructured":"Qwen Team et al. Qwen2.5 technical report. Preprint at https:\/\/arxiv.org\/abs\/2412.15115 (2025)."},{"key":"1719_CR67","unstructured":"Gemma Team et al. Gemma 3 technical report. Preprint at https:\/\/arxiv.org\/abs\/2503.19786 (2025)."},{"key":"1719_CR68","unstructured":"Grattafiori, A. et al. The Llama 3 herd of models. Preprint at https:\/\/arxiv.org\/abs\/2407.21783 (2024)."},{"key":"1719_CR69","unstructured":"Nexusflow\/Athene-V2-Chat. Hugging Face https:\/\/huggingface.co\/Nexusflow\/Athene-V2-Chat (2024)."},{"key":"1719_CR70","unstructured":"OpenAI et al. gpt-oss-120b & gpt-oss-20b model card. Preprint at https:\/\/arxiv.org\/abs\/2508.10925 (2025)."},{"key":"1719_CR71","doi-asserted-by":"publisher","unstructured":"Romanov, A. & Shivade, C. Lessons from natural language inference in the clinical domain. In Proc. 2018 Conference on Empirical Methods in Natural Language Processing (eds Riloff, E., Chiang, D., Hockenmaier, J. & Tsujii, J.) 1586\u20131596 (Association for Computational Linguistics, 2018); https:\/\/doi.org\/10.18653\/v1\/D18-1187","DOI":"10.18653\/v1\/D18-1187"},{"key":"1719_CR72","doi-asserted-by":"publisher","unstructured":"Blinov, P. et al. RuMedNLI: a Russian natural language inference dataset for the clinical domain. PhysioNet https:\/\/doi.org\/10.13026\/GXZD-CF80 (2022).","DOI":"10.13026\/GXZD-CF80"},{"key":"1719_CR73","doi-asserted-by":"publisher","first-page":"165","DOI":"10.1146\/annurev-biodatasci-030421-030931","volume":"4","author":"B Percha","year":"2021","unstructured":"Percha, B. Modern clinical text mining: a guide and review. Annu. Rev. Biomed. Data Sci. 4, 165\u2013187 (2021).","journal-title":"Annu. Rev. Biomed. Data Sci."},{"key":"1719_CR74","doi-asserted-by":"publisher","first-page":"3601","DOI":"10.1038\/s41591-024-03233-x","volume":"30","author":"K Huang","year":"2024","unstructured":"Huang, K. et al. A foundation model for clinician-centered drug repurposing. Nat. Med. 30, 3601\u20133613 (2024).","journal-title":"Nat. Med."},{"key":"1719_CR75","doi-asserted-by":"publisher","first-page":"20","DOI":"10.1038\/s41746-024-01010-1","volume":"7","author":"T Savage","year":"2024","unstructured":"Savage, T., Nayak, A., Gallo, R., Rangan, E. & Chen, J. H. Diagnostic reasoning prompts reveal the potential for large language model interpretability in medicine. npj Digit. Med. 7, 20 (2024).","journal-title":"npj Digit. Med."},{"key":"1719_CR76","doi-asserted-by":"publisher","unstructured":"Kasai, J., Kasai, Y., Sakaguchi, K., Yamada, Y. & Radev, D. Evaluating GPT-4 and ChatGPT on Japanese medical licensing examinations. Preprint at https:\/\/doi.org\/10.48550\/arXiv.2303.18027 (2023).","DOI":"10.48550\/arXiv.2303.18027"},{"key":"1719_CR77","first-page":"1","volume":"63","author":"H Chen","year":"2025","unstructured":"Chen, H. et al. Large language models and global health equity: a roadmap for equitable adoption in LMICs. Lancet Reg. Health West. Pac. 63, 1\u20138 (2025).","journal-title":"Lancet Reg. Health West. Pac."},{"key":"1719_CR78","doi-asserted-by":"publisher","unstructured":"Hegselmann, S. et al. Large language models are powerful electronic health record encoders. Preprint at https:\/\/doi.org\/10.48550\/arXiv.2502.17403 (2025).","DOI":"10.48550\/arXiv.2502.17403"},{"key":"1719_CR79","doi-asserted-by":"publisher","first-page":"AIcs2401161","DOI":"10.1056\/AIcs2401161","volume":"2","author":"E Klang","year":"2025","unstructured":"Klang, E. et al. Assessing retrieval-augmented large language models for medical coding. NEJM AI 2, AIcs2401161 (2025).","journal-title":"NEJM AI"},{"key":"1719_CR80","doi-asserted-by":"publisher","first-page":"AIoa2300068","DOI":"10.1056\/AIoa2300068","volume":"1","author":"C Zakka","year":"2024","unstructured":"Zakka, C. et al. Almanac\u2014retrieval-augmented language models for clinical medicine. NEJM AI 1, AIoa2300068 (2024).","journal-title":"NEJM AI"},{"key":"1719_CR81","doi-asserted-by":"publisher","first-page":"104942","DOI":"10.1016\/j.jbi.2025.104942","volume":"172","author":"J Wu","year":"2025","unstructured":"Wu, J., Wu, X., Zheng, Y. & Yang, J. Clinical pathway-aware large language models for reliable and transparent medical dialogue. J. Biomed. Inform. 172, 104942 (2025).","journal-title":"J. Biomed. Inform."},{"key":"1719_CR82","doi-asserted-by":"publisher","first-page":"3590","DOI":"10.1038\/s41591-024-03258-2","volume":"30","author":"SR Pfohl","year":"2024","unstructured":"Pfohl, S. R. et al. A toolbox for surfacing health equity harms and biases in large language models. Nat. Med. 30, 3590\u20133600 (2024).","journal-title":"Nat. Med."},{"key":"1719_CR83","doi-asserted-by":"publisher","unstructured":"DeepSeek-AI et al. DeepSeek LLM: scaling open-source language models with longtermism. Preprint at https:\/\/doi.org\/10.48550\/arXiv.2401.02954 (2024).","DOI":"10.48550\/arXiv.2401.02954"},{"key":"1719_CR84","doi-asserted-by":"publisher","first-page":"e250040","DOI":"10.1001\/jamahealthforum.2025.0040","volume":"6","author":"TA Buckley","year":"2025","unstructured":"Buckley, T. A., Crowe, B., Abdulnour, R.-E. E., Rodman, A. & Manrai, A. K. Comparison of frontier open-source and proprietary large language models for complex diagnoses. JAMA Health Forum 6, e250040 (2025).","journal-title":"JAMA Health Forum"},{"key":"1719_CR85","doi-asserted-by":"publisher","first-page":"194","DOI":"10.1038\/s41746-022-00742-2","volume":"5","author":"X Yang","year":"2022","unstructured":"Yang, X. et al. A large language model for electronic health records. npj Digit. Med. 5, 194 (2022).","journal-title":"npj Digit. Med."},{"key":"1719_CR86","doi-asserted-by":"publisher","first-page":"2613","DOI":"10.1038\/s41591-024-03097-1","volume":"30","author":"P Hager","year":"2024","unstructured":"Hager, P. et al. Evaluation and mitigation of the limitations of large language models in clinical decision-making. Nat. Med. 30, 2613\u20132622 (2024).","journal-title":"Nat. Med."},{"key":"1719_CR87","doi-asserted-by":"publisher","first-page":"135","DOI":"10.1038\/s41746-023-00879-8","volume":"6","author":"M Wornow","year":"2023","unstructured":"Wornow, M. et al. The shaky foundations of large language models and foundation models for electronic health records. npj Digit. Med. 6, 135 (2023).","journal-title":"npj Digit. Med."},{"key":"1719_CR88","doi-asserted-by":"publisher","first-page":"bbae354","DOI":"10.1093\/bib\/bbae354","volume":"25","author":"H Meshkin","year":"2024","unstructured":"Meshkin, H. et al. Harnessing large language models\u2019 zero-shot and few-shot learning capabilities for regulatory research. Brief. Bioinform. 25, bbae354 (2024).","journal-title":"Brief. Bioinform."},{"key":"1719_CR89","doi-asserted-by":"crossref","unstructured":"Agrawal, M., Hegselmann, S., Lang, H., Kim, Y. & Sontag, D. Large language models are few-shot clinical information extractors. In Proc. 2022 Conference on Empirical Methods in Natural Language Processing (eds Goldberg, Y. et al.) 1998\u20132022 (Association for Computational Linguistics, 2022).","DOI":"10.18653\/v1\/2022.emnlp-main.130"},{"key":"1719_CR90","doi-asserted-by":"publisher","first-page":"101118","DOI":"10.1016\/j.patter.2024.101118","volume":"6","author":"L Qin","year":"2025","unstructured":"Qin, L. et al. A survey of multilingual large language models. Patterns 6, 101118 (2025).","journal-title":"Patterns"},{"key":"1719_CR91","doi-asserted-by":"publisher","first-page":"86","DOI":"10.1038\/s41746-024-01339-7","volume":"8","author":"F Liu","year":"2025","unstructured":"Liu, F. et al. A multimodal multidomain multilingual medical foundation model for zero shot clinical diagnosis. npj Digit. Med. 8, 86 (2025).","journal-title":"npj Digit. Med."},{"key":"1719_CR92","first-page":"1","volume":"57","author":"BC Das","year":"2025","unstructured":"Das, B. C., Amini, M. H. & Wu, Y. Security and privacy challenges of large language models: a survey. ACM Comput. Surv. 57, 1\u201339 (2025).","journal-title":"ACM Comput. Surv."},{"key":"1719_CR93","unstructured":"Lehman, E. et al. Do we still need clinical language models? In Proc. Conference on Health, Inference, and Learning (eds Mortazavi, B. J. et al.) 578\u2013597 (PMLR, 2023)."},{"key":"1719_CR94","doi-asserted-by":"publisher","unstructured":"Wang, X. et al. Self-consistency improves chain of thought reasoning in language models. Preprint at https:\/\/doi.org\/10.48550\/arXiv.2203.11171 (2023).","DOI":"10.48550\/arXiv.2203.11171"},{"key":"1719_CR95","unstructured":"Wu, J., Wu, X. & Yang, J. Guiding clinical reasoning with large language models via knowledge seeds. In Proc. Thirty-Third International Joint Conference on Artificial Intelligence (ed. Larson, K.) 7491\u20137499 (International Joint Conferences on Artificial Intelligence Organization, 2024)."},{"key":"1719_CR96","doi-asserted-by":"publisher","first-page":"2054","DOI":"10.1093\/jamia\/ocae079","volume":"31","author":"J Wu","year":"2024","unstructured":"Wu, J. et al. Large language models leverage external knowledge to extend clinical insight beyond language boundaries. J. Am. Med. Inform. Assoc. 31, 2054\u20132064 (2024).","journal-title":"J. Am. Med. Inform. Assoc."},{"key":"1719_CR97","unstructured":"Qwen Team. QwQ-32B: embracing the power of reinforcement learning. Qwen https:\/\/qwenlm.github.io\/blog\/qwq-32b\/ (2025)."},{"key":"1719_CR98","doi-asserted-by":"publisher","first-page":"866","DOI":"10.1001\/jama.2023.14217","volume":"330","author":"NH Shah","year":"2023","unstructured":"Shah, N. H., Entwistle, D. & Pfeffer, M. A. Creation and adoption of large language models in medicine. J. Am. Med. Assoc. 330, 866\u2013869 (2023).","journal-title":"J. Am. Med. Assoc."},{"key":"1719_CR99","doi-asserted-by":"publisher","first-page":"799","DOI":"10.1038\/s42256-023-00652-2","volume":"5","author":"A Karargyris","year":"2023","unstructured":"Karargyris, A. et al. Federated benchmarking of medical artificial intelligence with MedPerf. Nat. Mach. Intell. 5, 799\u2013810 (2023).","journal-title":"Nat. Mach. Intell."},{"key":"1719_CR100","doi-asserted-by":"publisher","first-page":"AIe2500143","DOI":"10.1056\/AIe2500143","volume":"2","author":"A Rodman","year":"2025","unstructured":"Rodman, A., Zwaan, L., Olson, A. & Manrai, A. K. When it comes to benchmarks, humans are the only way. NEJM AI 2, AIe2500143 (2025).","journal-title":"NEJM AI"},{"key":"1719_CR101","doi-asserted-by":"publisher","first-page":"577","DOI":"10.1038\/s41746-025-01965-9","volume":"8","author":"H Cui","year":"2025","unstructured":"Cui, H. et al. TIMER: temporal instruction modeling and evaluation for longitudinal clinical records. npj Digit. Med. 8, 577\u2013585 (2025).","journal-title":"npj Digit. Med."},{"key":"1719_CR102","doi-asserted-by":"publisher","first-page":"60","DOI":"10.1038\/s41591-024-03425-5","volume":"31","author":"J Gallifant","year":"2025","unstructured":"Gallifant, J. et al. The TRIPOD-LLM reporting guideline for studies using large language models. Nat. Med. 31, 60\u201369 (2025).","journal-title":"Nat. Med."},{"key":"1719_CR103","doi-asserted-by":"publisher","first-page":"367","DOI":"10.1038\/s41746-024-01366-4","volume":"7","author":"B Gu","year":"2024","unstructured":"Gu, B., Desai, R. J., Lin, K. J. & Yang, J. Probabilistic medical predictions of large language models. npj Digit. Med. 7, 367 (2024).","journal-title":"npj Digit. Med."},{"key":"1719_CR104","unstructured":"National NLP clinical challenges (n2c2). Harvard Medical School https:\/\/n2c2.dbmi.hms.harvard.edu\/home (2026)."},{"key":"1719_CR105","unstructured":"CLEF eHealth lab series. CLEF eHealth https:\/\/clefehealth.imag.fr\/clefehealth.imag.fr\/index.html (2024)."},{"key":"1719_CR106","doi-asserted-by":"publisher","first-page":"e215","DOI":"10.1161\/01.CIR.101.23.e215","volume":"101","author":"AL Goldberger","year":"2000","unstructured":"Goldberger, A. L. et al. PhysioBank, PhysioToolkit, and PhysioNet: components of a new research resource for complex physiologic signals. Circulation 101, e215\u2013e220 (2000).","journal-title":"Circulation"},{"key":"1719_CR107","unstructured":"Hugging face\u2013the AI community building the future. Hugging Face https:\/\/huggingface.co\/ (2026)."},{"key":"1719_CR108","doi-asserted-by":"publisher","unstructured":"Kwon, W. et al. Efficient memory management for large language model serving with PagedAttention. In Proc. 29th Symposium on Operating Systems Principles 611\u2013626 (Association for Computing Machinery, 2023); https:\/\/doi.org\/10.1145\/3600006.3613165","DOI":"10.1145\/3600006.3613165"},{"key":"1719_CR109","unstructured":"Lin, C.-Y. ROUGE: a package for automatic evaluation of summaries. In Text Summarization Branches Out 74\u201381 (Association for Computational Linguistics, 2004)."},{"key":"1719_CR110","doi-asserted-by":"publisher","unstructured":"Papineni, K., Roukos, S., Ward, T. & Zhu, W.-J. BLEU: a method for automatic evaluation of machine translation. In Proc. 40th Annual Meeting on Association for Computational Linguistics (eds Isabelle, P. et al.) 311\u2013318 (Association for Computational Linguistics, 2001); https:\/\/doi.org\/10.3115\/1073083.1073135","DOI":"10.3115\/1073083.1073135"},{"key":"1719_CR111","unstructured":"Zhang, T., Kishore, V., Wu, F., Weinberger, K. Q. & Artzi, Y. BERTScore: evaluating text generation with BERT. In International Conference on Learning Representations (2020)."},{"key":"1719_CR112","first-page":"2825","volume":"12","author":"F Pedregosa","year":"2011","unstructured":"Pedregosa, F. et al. Scikit-learn: machine learning in Python. J. Mach. Learn. Res. 12, 2825\u20132830 (2011).","journal-title":"J. Mach. Learn. Res."},{"key":"1719_CR113","doi-asserted-by":"publisher","unstructured":"Bird, S. NLTK: the natural language toolkit. In Proc. COLING\/ACL 2006 Interactive Presentation Sessions 69\u201372 (Association for Computational Linguistics, 2006); https:\/\/doi.org\/10.3115\/1225403.1225421","DOI":"10.3115\/1225403.1225421"},{"key":"1719_CR114","unstructured":"Google Research. Python ROUGE implementation. GitHub https:\/\/github.com\/google-research\/google-research\/tree\/master\/rouge (2026)."},{"key":"1719_CR115","unstructured":"Xu, R., Wang, Z., Fan, R.-Z. & Liu, P. Benchmarking benchmark leakage in large language models. Preprint at https:\/\/arxiv.org\/abs\/2404.18824 (2024)."}],"container-title":["Nature Biomedical Engineering"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/www.nature.com\/articles\/s41551-026-01719-2.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/www.nature.com\/articles\/s41551-026-01719-2","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/www.nature.com\/articles\/s41551-026-01719-2.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,6,23]],"date-time":"2026-06-23T15:40:38Z","timestamp":1782229238000},"score":1,"resource":{"primary":{"URL":"https:\/\/www.nature.com\/articles\/s41551-026-01719-2"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026,6,17]]},"references-count":115,"alternative-id":["1719"],"URL":"https:\/\/doi.org\/10.1038\/s41551-026-01719-2","relation":{},"ISSN":["2157-846X"],"issn-type":[{"value":"2157-846X","type":"electronic"}],"subject":[],"published":{"date-parts":[[2026,6,17]]},"assertion":[{"value":"16 June 2025","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"19 May 2026","order":2,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"17 June 2026","order":3,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"K.J.L. has received research grants from Takeda, AbbVie and UCB for projects unrelated to this study. E.A. reports consultant fees from Fourier Health. S.S. is participating in investigator-initiated grants to the Brigham and Women\u2019s Hospital from Boehringer Ingelheim, Takeda and UCB unrelated to the topic of this study; is an advisor to Aetion Inc., a software manufacturer.; and is an advisor to Temedica GmbH, a patient-oriented data generation company; his interests were declared, reviewed and approved by the Brigham and Women\u2019s Hospital in accordance with their institutional compliance policies. J.H.C. reports cofounding Reaction Explorer, that develops and licences organic chemistry education software, and receive medical expert witness fees from Sutton Pierce, Younker Hyde MacFarlane, Sykes McAllister and Elite Expert; consulting fees from ISHI Health; and honoraria or travel expenses for invited presentations by insitro, General Reinsurance Corporation, Cozeva and other industry conferences, academic institutions and health systems. All other authors declare no competing interests.","order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Competing interests"}}]}}