{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,6,26]],"date-time":"2026-06-26T13:55:56Z","timestamp":1782482156208,"version":"3.54.5"},"reference-count":41,"publisher":"Springer Science and Business Media LLC","issue":"1","license":[{"start":{"date-parts":[[2025,12,27]],"date-time":"2025-12-27T00:00:00Z","timestamp":1766793600000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by-nc-nd\/4.0"},{"start":{"date-parts":[[2026,1,22]],"date-time":"2026-01-22T00:00:00Z","timestamp":1769040000000},"content-version":"vor","delay-in-days":26,"URL":"https:\/\/creativecommons.org\/licenses\/by-nc-nd\/4.0"}],"funder":[{"DOI":"10.13039\/100000092","name":"U.S. National Library of Medicine","doi-asserted-by":"publisher","award":["R01LM011934"],"award-info":[{"award-number":["R01LM011934"]}],"id":[{"id":"10.13039\/100000092","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000092","name":"U.S. National Library of Medicine","doi-asserted-by":"publisher","award":["R01LM011934"],"award-info":[{"award-number":["R01LM011934"]}],"id":[{"id":"10.13039\/100000092","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000092","name":"U.S. National Library of Medicine","doi-asserted-by":"publisher","award":["R01LM011934"],"award-info":[{"award-number":["R01LM011934"]}],"id":[{"id":"10.13039\/100000092","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000092","name":"U.S. National Library of Medicine","doi-asserted-by":"publisher","award":["R01LM011934"],"award-info":[{"award-number":["R01LM011934"]}],"id":[{"id":"10.13039\/100000092","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000092","name":"U.S. National Library of Medicine","doi-asserted-by":"publisher","award":["R01LM011934"],"award-info":[{"award-number":["R01LM011934"]}],"id":[{"id":"10.13039\/100000092","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000092","name":"U.S. National Library of Medicine","doi-asserted-by":"publisher","award":["R01LM011934"],"award-info":[{"award-number":["R01LM011934"]}],"id":[{"id":"10.13039\/100000092","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000092","name":"U.S. National Library of Medicine","doi-asserted-by":"publisher","award":["R01LM011934"],"award-info":[{"award-number":["R01LM011934"]}],"id":[{"id":"10.13039\/100000092","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000092","name":"U.S. National Library of Medicine","doi-asserted-by":"publisher","award":["R01LM011934"],"award-info":[{"award-number":["R01LM011934"]}],"id":[{"id":"10.13039\/100000092","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000092","name":"U.S. National Library of Medicine","doi-asserted-by":"publisher","award":["R01LM011934"],"award-info":[{"award-number":["R01LM011934"]}],"id":[{"id":"10.13039\/100000092","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000092","name":"U.S. National Library of Medicine","doi-asserted-by":"publisher","award":["R01LM011934"],"award-info":[{"award-number":["R01LM011934"]}],"id":[{"id":"10.13039\/100000092","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000051","name":"National Human Genome Research Institute","doi-asserted-by":"publisher","award":["R01HG012748"],"award-info":[{"award-number":["R01HG012748"]}],"id":[{"id":"10.13039\/100000051","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000051","name":"National Human Genome Research Institute","doi-asserted-by":"publisher","award":["R01HG012748"],"award-info":[{"award-number":["R01HG012748"]}],"id":[{"id":"10.13039\/100000051","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000051","name":"National Human Genome Research Institute","doi-asserted-by":"publisher","award":["R01HG012748"],"award-info":[{"award-number":["R01HG012748"]}],"id":[{"id":"10.13039\/100000051","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000051","name":"National Human Genome Research Institute","doi-asserted-by":"publisher","award":["R01HG012748"],"award-info":[{"award-number":["R01HG012748"]}],"id":[{"id":"10.13039\/100000051","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100004917","name":"Cancer Prevention and Research Institute of Texas","doi-asserted-by":"publisher","award":["RR230020"],"award-info":[{"award-number":["RR230020"]}],"id":[{"id":"10.13039\/100004917","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100004917","name":"Cancer Prevention and Research Institute of Texas","doi-asserted-by":"publisher","award":["RR230020"],"award-info":[{"award-number":["RR230020"]}],"id":[{"id":"10.13039\/100004917","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100004917","name":"Cancer Prevention and Research Institute of Texas","doi-asserted-by":"publisher","award":["RR230020"],"award-info":[{"award-number":["RR230020"]}],"id":[{"id":"10.13039\/100004917","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100004917","name":"Cancer Prevention and Research Institute of Texas","doi-asserted-by":"publisher","award":["RR230020"],"award-info":[{"award-number":["RR230020"]}],"id":[{"id":"10.13039\/100004917","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100004917","name":"Cancer Prevention and Research Institute of Texas","doi-asserted-by":"publisher","award":["RR230020"],"award-info":[{"award-number":["RR230020"]}],"id":[{"id":"10.13039\/100004917","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000049","name":"National Institute on Aging","doi-asserted-by":"publisher","award":["R01AG072799"],"award-info":[{"award-number":["R01AG072799"]}],"id":[{"id":"10.13039\/100000049","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000049","name":"National Institute on Aging","doi-asserted-by":"publisher","award":["R01AG072799"],"award-info":[{"award-number":["R01AG072799"]}],"id":[{"id":"10.13039\/100000049","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["npj Digit. Med."],"DOI":"10.1038\/s41746-025-02253-2","type":"journal-article","created":{"date-parts":[[2025,12,27]],"date-time":"2025-12-27T13:07:11Z","timestamp":1766840831000},"update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":2,"title":["Context matching is not reasoning when performing generalized clinical evaluation of generative language models"],"prefix":"10.1038","volume":"9","author":[{"given":"Andrew","family":"Wen","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Qiuhao","family":"Lu","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Yu-Neng","family":"Chuang","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Guanchu","family":"Wang","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Jiayi","family":"Yuan","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Jiamu","family":"Zhang","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Liwei","family":"Wang","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Sunyang","family":"Fu","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Kurt D.","family":"Miller","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Heling","family":"Jia","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Steven D.","family":"Bedrick","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"William R.","family":"Hersh","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Kirk E.","family":"Roberts","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Xia","family":"Hu","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Hongfang","family":"Liu","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2025,12,27]]},"reference":[{"key":"2253_CR1","doi-asserted-by":"publisher","first-page":"1930","DOI":"10.1038\/s41591-023-02448-8","volume":"29","author":"AJ Thirunavukarasu","year":"2023","unstructured":"Thirunavukarasu, A. J. et al. Large language models in medicine. Nat. Med. 29, 1930\u20131940 (2023).","journal-title":"Nat. Med."},{"key":"2253_CR2","doi-asserted-by":"publisher","first-page":"319","DOI":"10.1001\/jama.2024.21700","volume":"333","author":"S Bedi","year":"2025","unstructured":"Bedi, S. et al. Testing and evaluation of health care applications of large language models: a systematic review. JAMA 333, 319\u2013328 (2025).","journal-title":"JAMA"},{"key":"2253_CR3","unstructured":"Yuan, J., Zhang, J., Wen, A. & Hu, X. The science of evaluating foundation models. arXiv preprint arXiv:250209670 (2025)."},{"key":"2253_CR4","unstructured":"Yosinski, J., Clune, J., Bengio, Y. & Lipson, H. How transferable are features in deep neural networks? Adv. Neural Inf. Process. Syst. 27, (2014)."},{"key":"2253_CR5","unstructured":"Wang, A. et al. GLUE: A multi-task benchmark and analysis platform for natural language understanding. In Proceedings of the 2019 International Conference on Learning Representations (2019)."},{"key":"2253_CR6","doi-asserted-by":"crossref","unstructured":"Devlin, J., Chang M.-W., Lee K. & Toutanova K. Bert: Pre-training of deep bidirectional transformers for language understanding. In Proceedings of the 2019 Conference of the North American chapter of the Association for Computational Linguistics: Human Language Technologies, volume 1 (long and short papers) 4171\u20134186 (2019).","DOI":"10.18653\/v1\/N19-1423"},{"key":"2253_CR7","doi-asserted-by":"publisher","first-page":"1314","DOI":"10.1093\/jamia\/ocz102","volume":"26","author":"Q Wei","year":"2019","unstructured":"Wei, Q. et al. Cost-aware active learning for named entity recognition in clinical text. J. Am. Med. Inf. Assoc. 26, 1314\u20131322 (2019).","journal-title":"J. Am. Med. Inf. Assoc."},{"key":"2253_CR8","doi-asserted-by":"publisher","first-page":"2632","DOI":"10.1093\/jamia\/ocae197","volume":"31","author":"J Liu","year":"2024","unstructured":"Liu, J. & Wong, Z. S. Y. Utilizing active learning strategies in machine-assisted annotation for clinical named entity recognition: a comprehensive analysis considering annotation costs and target effectiveness. J. Am. Med. Inf. Assoc. 31, 2632\u20132640 (2024).","journal-title":"J. Am. Med. Inf. Assoc."},{"key":"2253_CR9","doi-asserted-by":"crossref","unstructured":"Schick, T. & Sch\u00fctze, H. It\u2019s not just size that matters: Small language models are also few-shot learners. In Proceedings of the 2021 Conference of the North American chapter of the Association for Computational Linguistics (2021).","DOI":"10.18653\/v1\/2021.naacl-main.185"},{"key":"2253_CR10","unstructured":"Wang F. et al. A comprehensive survey of small language models in the era of large language models: Techniques, enhancements, applications, collaboration with LLMs, and trustworthiness. arXiv preprint arXiv:241103350 (2024)."},{"key":"2253_CR11","doi-asserted-by":"publisher","first-page":"240","DOI":"10.1038\/s41746-025-01653-8","volume":"8","author":"H Kim","year":"2025","unstructured":"Kim, H. et al. Small language models learn enhanced reasoning skills from medical textbooks. NPJ Digit Med 8, 240 (2025).","journal-title":"NPJ Digit Med"},{"key":"2253_CR12","doi-asserted-by":"publisher","first-page":"1418","DOI":"10.1038\/s42256-024-00944-1","volume":"6","author":"J Qiu","year":"2024","unstructured":"Qiu, J. et al. LLM-based agentic systems in medicine and healthcare. Nat. Mach. Intell. 6, 1418\u20131420 (2024).","journal-title":"Nat. Mach. Intell."},{"key":"2253_CR13","unstructured":"Chang, C.-Y. et al. MAIN-RAG: Multi-agent filtering retrieval-augmented generation. arXiv preprint arXiv:250100332 (2024)."},{"key":"2253_CR14","doi-asserted-by":"publisher","first-page":"457","DOI":"10.1016\/S0140-6736(25)00202-8","volume":"405","author":"J Zou","year":"2025","unstructured":"Zou, J. & Topol, E. J. The rise of agentic AI teammates in medicine. Lancet 405, 457 (2025).","journal-title":"Lancet"},{"key":"2253_CR15","doi-asserted-by":"crossref","unstructured":"Suura S. R. Agentic AI systems in organ health management: early detection of rejection in transplant patients. J. Neonatal Surg. 14. (2025).","DOI":"10.52783\/jns.v14.1823"},{"key":"2253_CR16","first-page":"1","volume":"18","author":"J Yang","year":"2024","unstructured":"Yang, J. et al. Harnessing the power of LMS in practice: A survey on ChatGPT and beyond. ACM Trans. Knowl. Discov. Data 18, 1\u201332 (2024).","journal-title":"ACM Trans. Knowl. Discov. Data"},{"key":"2253_CR17","doi-asserted-by":"publisher","first-page":"2245","DOI":"10.1001\/jama.2013.282328","volume":"310","author":"SA Haist","year":"2013","unstructured":"Haist, S. A., Katsufrakis, P. J. & Dillon, G. F. The evolution of the United States Medical Licensing Examination (USMLE): Enhancing assessment of practice-related competencies. JAMA 310, 2245\u20132246 (2013).","journal-title":"JAMA"},{"key":"2253_CR18","doi-asserted-by":"publisher","first-page":"6421","DOI":"10.3390\/app11146421","volume":"11","author":"D Jin","year":"2021","unstructured":"Jin, D. et al. What disease does this patient have? A large-scale open domain question answering dataset from medical exams. Appl. Sci. 11, 6421 (2021).","journal-title":"Appl. Sci."},{"key":"2253_CR19","unstructured":"Hendrycks, D. et al. Measuring massive multitask language understanding. In Proceedings of the 2021 International Conference on Learning Representations (2021)."},{"key":"2253_CR20","doi-asserted-by":"publisher","unstructured":"Raji, I. D., Daneshjou R., Alsentzer, E. In It\u2019s time to bench the medical exam benchmark. NEJM AI 2. https:\/\/doi.org\/10.1056\/AIe2401235.(2025).","DOI":"10.1056\/AIe2401235"},{"key":"2253_CR21","unstructured":"Li, W. et al. Can multiple-choice questions really be useful in detecting the abilities of LLMs? Proceedings of the 2024 Joint International Conference on Computational Linguistics, Language Resources and Evaluation (2024)."},{"key":"2253_CR22","unstructured":"Matton, K., Ness, R., Guttag, J. & Kiciman, E. Walk the Talk? Measuring the faithfulness of large language model explanations. In The 13th International Conference on Learning Representations (2025)."},{"key":"2253_CR23","unstructured":"Lanham, T. et al. Measuring faithfulness in chain-of-thought reasoning. arXiv preprint arXiv:230713702 (2023)."},{"key":"2253_CR24","doi-asserted-by":"publisher","first-page":"e45312","DOI":"10.2196\/45312","volume":"9","author":"A Gilson","year":"2023","unstructured":"Gilson, A. et al. How does ChatGPT perform on the United States Medical Licensing Examination (USMLE)? The implications of large language models for medical education and knowledge assessment. JMIR Med Educ. 9, e45312 (2023).","journal-title":"JMIR Med Educ."},{"key":"2253_CR25","doi-asserted-by":"publisher","first-page":"192","DOI":"10.1097\/ACM.0000000000005549","volume":"99","author":"V Yaneva","year":"2024","unstructured":"Yaneva, V., Baldwin, P., Jurich, D. P., Swygert, K. & Clauser, B. E. Examining ChatGPT performance on USMLE sample items and implications for assessment. Acad. Med 99, 192\u2013197 (2024).","journal-title":"Acad. Med"},{"key":"2253_CR26","unstructured":"Guo, D. et al. Deepseek-r1: Incentivizing reasoning capability in LLMs via reinforcement learning. arXiv preprint arXiv:250112948 (2025)."},{"key":"2253_CR27","unstructured":"Jaech, A. et al. OpenAI O1 system card. arXiv preprint arXiv:241216720 (2024)."},{"key":"2253_CR28","unstructured":"Agarwal, S. et al. gpt-oss-120b & gpt-oss-20b model card. arXiv preprint arXiv:250810925 (2025)."},{"key":"2253_CR29","unstructured":"Qwen Team. Qwq-32b: Embracing the power of reinforcement learning. https:\/\/qwenlm.github.io\/blog\/qwq-32b (2025)."},{"key":"2253_CR30","unstructured":"Grattafiori, A. et al. The llama 3 herd of models. arXiv preprint arXiv:240721783 (2024)."},{"key":"2253_CR31","unstructured":"Yang, A. et al. Qwen2. 5 technical report. arXiv preprint arXiv:241215115 (2024)."},{"key":"2253_CR32","unstructured":"Gema, A. P. et al. Are We Done with MMLU? arXiv preprint arXiv:240604127 (2024)."},{"key":"2253_CR33","unstructured":"Pal, A., Umapathi, L. K. & Sankarasubbu, M. Medmcqa: A large-scale multi-subject multi-choice dataset for medical domain question answering. In Conference on health, inference, and learning: PMLR. 248\u2013260.(2022)."},{"key":"2253_CR34","doi-asserted-by":"crossref","unstructured":"Chen, H., Fang, Z., Singla, Y. & Dredze, M. Benchmarking large language models on answering and explaining challenging medical questions. In Proceedings of the 2025 Conference of the Nations of the Americas Chapter of the Association for Computational Linguistics: Human Language Technologies. Volume I (Long Papers) 3563-3599 (2025).","DOI":"10.18653\/v1\/2025.naacl-long.182"},{"key":"2253_CR35","first-page":"1910","volume":"305","author":"HJ Chang","year":"2011","unstructured":"Chang, H. J. Fontanarosa PB. Introducing. JAMA Clin. Chall. JAMA 305, 1910\u20131910 (2011).","journal-title":"JAMA Clin. Chall. JAMA"},{"key":"2253_CR36","doi-asserted-by":"crossref","unstructured":"Jin, Q., Dhingra, B., Liu, Z., Cohen, W. & Lu, X. Pubmedqa: A dataset for biomedical research question answering. In Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing and the 9th International Joint Conference On Natural Language Processing (EMNLP-IJCNLP)2567\u20132577. (2019).","DOI":"10.18653\/v1\/D19-1259"},{"key":"2253_CR37","unstructured":"Beurer-Kellner, L., Fischer, M. & Vechev, M. Guiding LLMs the right way: Fast, non-invasive constrained generation. In Proceedings of the 41st International Conference on Machine Learning, Vol. 235, 3658\u20133673 (PMLR, 2024)."},{"key":"2253_CR38","doi-asserted-by":"crossref","unstructured":"Kwon, W. et al. Efficient memory management for large language model serving with paged attention. In Proceedings of the 29th Symposium on Operating Systems Principles. 611\u2013626 (2023).","DOI":"10.1145\/3600006.3613165"},{"key":"2253_CR39","doi-asserted-by":"crossref","unstructured":"Dong, Y. et al. Generalization or memorization: Data contamination and trustworthy evaluation for large language models. In Findings of the Association for Computational Linguistics: ACL 2024 (2024).","DOI":"10.18653\/v1\/2024.findings-acl.716"},{"key":"2253_CR40","doi-asserted-by":"crossref","unstructured":"Sainz, O. et al NLP evaluation in trouble: On the need to measure LLM data contamination for each benchmark. In Findings of the Association for Computational Linguistics: EMNLP 2023 (2023).","DOI":"10.18653\/v1\/2023.findings-emnlp.722"},{"key":"2253_CR41","unstructured":"Zong, Y., Yu, T., Chavhan, R., Zhao, B. & Hospedales, T. Fool your (vision and) language model with embarrassingly simple permutations. In Proceedings of the 2024 International Conference on Machine Learning (2024)."}],"container-title":["npj Digital Medicine"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/www.nature.com\/articles\/s41746-025-02253-2","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/www.nature.com\/articles\/s41746-025-02253-2.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/www.nature.com\/articles\/s41746-025-02253-2.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,1,23]],"date-time":"2026-01-23T03:03:24Z","timestamp":1769137404000},"score":1,"resource":{"primary":{"URL":"https:\/\/www.nature.com\/articles\/s41746-025-02253-2"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,12,27]]},"references-count":41,"journal-issue":{"issue":"1","published-online":{"date-parts":[[2026,12]]}},"alternative-id":["2253"],"URL":"https:\/\/doi.org\/10.1038\/s41746-025-02253-2","relation":{"has-preprint":[{"id-type":"doi","id":"10.21203\/rs.3.rs-7325383\/v1","asserted-by":"object"}]},"ISSN":["2398-6352"],"issn-type":[{"value":"2398-6352","type":"electronic"}],"subject":[],"published":{"date-parts":[[2025,12,27]]},"assertion":[{"value":"8 August 2025","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"4 December 2025","order":2,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"27 December 2025","order":3,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"Author HL is an Editorial Board Member of npj Digital Medicine. They played no role in the peer review or decision to publish this paper. Author HL declares no financial competing interests. All other authors declare no financial or non-financial competing interests.","order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Competing interests"}}],"article-number":"71"}}