{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,28]],"date-time":"2026-10-28T02:13:06Z","timestamp":1793153586840,"version":"4.5.0"},"reference-count":52,"publisher":"Elsevier BV","issue":"10","license":[{"start":{"date-parts":[[2026,10,1]],"date-time":"2026-10-01T00:00:00Z","timestamp":1790812800000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.elsevier.com\/tdm\/userlicense\/1.0\/"},{"start":{"date-parts":[[2026,10,1]],"date-time":"2026-10-01T00:00:00Z","timestamp":1790812800000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.elsevier.com\/legal\/tdmrep-license"},{"start":{"date-parts":[[2026,6,9]],"date-time":"2026-06-09T00:00:00Z","timestamp":1780963200000},"content-version":"vor","delay-in-days":0,"URL":"http:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"funder":[{"DOI":"10.13039\/100000002","name":"National Institutes of Health","doi-asserted-by":"publisher","award":["R01DA063148"],"award-info":[{"award-number":["R01DA063148"]}],"id":[{"id":"10.13039\/100000002","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100005326","name":"Yale University","doi-asserted-by":"publisher","id":[{"id":"10.13039\/100005326","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["cell.com","elsevier.com","sciencedirect.com"],"crossmark-restriction":true},"short-container-title":["Patterns"],"published-print":{"date-parts":[[2026,10]]},"DOI":"10.1016\/j.patter.2026.101610","type":"journal-article","created":{"date-parts":[[2026,7,8]],"date-time":"2026-07-08T14:59:31Z","timestamp":1783522771000},"page":"101610","update-policy":"https:\/\/doi.org\/10.1016\/elsevier_cm_policy","source":"Crossref","is-referenced-by-count":1,"title":["MedicalAgentsBench for complex medical reasoning: Comparing internalized reasoning models versus externalized agent-based frameworks"],"prefix":"10.1016","volume":"7","author":[{"given":"Yanjun","family":"Shao","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0006-2700-4513","authenticated-orcid":false,"given":"Xiangru","family":"Tang","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Jiwoong","family":"Sohn","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Jiapeng","family":"Chen","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Yuxuan","family":"Liao","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Jiayi","family":"Zhang","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Jinyu","family":"Xiang","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Fang","family":"Wu","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Yilun","family":"Zhao","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Chenglin","family":"Wu","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Wenqi","family":"Shi","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Arman","family":"Cohan","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-9746-3719","authenticated-orcid":false,"given":"Mark","family":"Gerstein","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"78","reference":[{"key":"10.1016\/j.patter.2026.101610_bib1","doi-asserted-by":"crossref","first-page":"943","DOI":"10.1038\/s41591-024-03423-7","article-title":"Toward expert-level medical question answering with large language models","volume":"31","author":"Singhal","year":"2025","journal-title":"Nat. Med."},{"key":"10.1016\/j.patter.2026.101610_bib2","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1145\/3490238","article-title":"Biomedical question answering: a survey of approaches and challenges","volume":"55","author":"Jin","year":"2022","journal-title":"ACM Comput. Surv."},{"key":"10.1016\/j.patter.2026.101610_bib3","doi-asserted-by":"crossref","first-page":"3280","DOI":"10.1038\/s41467-025-56989-2","article-title":"Benchmarking large language models for biomedical natural language processing applications and recommendations","volume":"16","author":"Chen","year":"2025","journal-title":"Nat. Commun."},{"key":"10.1016\/j.patter.2026.101610_bib4","doi-asserted-by":"crossref","first-page":"445","DOI":"10.1038\/s44222-025-00279-5","article-title":"Application of large language models in medicine","volume":"3","author":"Liu","year":"2025","journal-title":"Nat. Rev. Bioeng."},{"key":"10.1016\/j.patter.2026.101610_bib5","doi-asserted-by":"crossref","first-page":"6125","DOI":"10.1016\/j.cell.2024.09.022","article-title":"Empowering biomedical discovery with AI agents","volume":"187","author":"Gao","year":"2024","journal-title":"Cell"},{"key":"10.1016\/j.patter.2026.101610_bib6","series-title":"Proceedings of the 31st International Conference on Computational Linguistics","first-page":"10183","article-title":"AI hospital: Benchmarking large language models in a multi-agent medical interaction simulator","author":"Fan","year":"2025"},{"key":"10.1016\/j.patter.2026.101610_bib7","series-title":"Proceedings of the 2024 Conference on Empirical Methods in Natural Language Processing","first-page":"22294","article-title":"MedAdapter: Efficient test-time adaptation of large language models towards medical reasoning","author":"Shi","year":"2024"},{"key":"10.1016\/j.patter.2026.101610_bib8","doi-asserted-by":"crossref","first-page":"6421","DOI":"10.3390\/app11146421","article-title":"What disease does this patient have? A large-scale open domain question answering dataset from medical exams","volume":"11","author":"Jin","year":"2021","journal-title":"Appl. Sci."},{"key":"10.1016\/j.patter.2026.101610_bib9","series-title":"Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing","first-page":"2567","article-title":"PubMedQA: A dataset for biomedical research question answering","author":"Jin","year":"2019"},{"key":"10.1016\/j.patter.2026.101610_bib10","series-title":"Conference on Health, Inference, and Learning","first-page":"248","article-title":"MedMCQA: A large-scale multi-subject multi-choice dataset for medical domain question answering","author":"Pal","year":"2022"},{"key":"10.1016\/j.patter.2026.101610_bib11","series-title":"Proceedings of the 2018 Conference on Empirical Methods in Natural Language Processing","first-page":"2357","article-title":"emrQA: A large corpus for question answering on electronic medical records","author":"Pampari","year":"2018"},{"key":"10.1016\/j.patter.2026.101610_bib12","series-title":"Proceedings of the 20th Workshop on Biomedical Language Processing","first-page":"74","article-title":"Overview of the MEDIQA 2021 shared task on summarization in the medical domain","author":"Abacha","year":"2021"},{"key":"10.1016\/j.patter.2026.101610_bib13","article-title":"PathVQA: 30000+ questions for medical visual question answering","author":"He","year":"2020","journal-title":"arXiv"},{"key":"10.1016\/j.patter.2026.101610_bib14","series-title":"Proceedings of the 13th Language Resources and Evaluation Conference","doi-asserted-by":"crossref","first-page":"6250","DOI":"10.63317\/2rkacs78gax6","article-title":"RadQA: A question answering dataset to improve comprehension of radiology reports","author":"Soni","year":"2022"},{"key":"10.1016\/j.patter.2026.101610_bib15","doi-asserted-by":"crossref","first-page":"3867","DOI":"10.52202\/075280-0170","article-title":"EHRXQA: A multi-modal question answering dataset for electronic health records with chest x-ray images","volume":"36","author":"Bae","year":"2023","journal-title":"Adv. Neural Inf. Process. Syst."},{"key":"10.1016\/j.patter.2026.101610_bib16","series-title":"Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics","first-page":"960","article-title":"HEAD-QA: A healthcare dataset for complex reasoning","author":"Vilares","year":"2019"},{"key":"10.1016\/j.patter.2026.101610_bib17","series-title":"Proceedings of the 2024 Joint International Conference on Computational Linguistics, Language Resources and Evaluation","first-page":"11178","article-title":"MedQA-SWE: a clinical question & answer dataset for swedish","author":"Hertzberg","year":"2024"},{"key":"10.1016\/j.patter.2026.101610_bib18","series-title":"Proceedings of the 63rd Annual Meeting of the Association for Computational Linguistics","first-page":"1948","article-title":"AfriMed-QA: A pan-african, multi-specialty, medical question-answering benchmark dataset","author":"Olatunji","year":"2025"},{"key":"10.1016\/j.patter.2026.101610_bib19","doi-asserted-by":"crossref","first-page":"172","DOI":"10.1038\/s41586-023-06291-2","article-title":"Large language models encode clinical knowledge","volume":"620","author":"Singhal","year":"2023","journal-title":"Nature"},{"key":"10.1016\/j.patter.2026.101610_bib20","doi-asserted-by":"crossref","first-page":"1930","DOI":"10.1038\/s41591-023-02448-8","article-title":"Large language models in medicine","volume":"29","author":"Thirunavukarasu","year":"2023","journal-title":"Nat. Med."},{"key":"10.1016\/j.patter.2026.101610_bib21","doi-asserted-by":"crossref","DOI":"10.1016\/j.patter.2024.100943","article-title":"Can large language models reason about medical questions?","volume":"5","author":"Li\u00e9vin","year":"2024","journal-title":"Patterns"},{"key":"10.1016\/j.patter.2026.101610_bib22","doi-asserted-by":"crossref","DOI":"10.2196\/45312","article-title":"How does ChatGPT perform on the United States Medical Licensing Examination (USMLE)? the implications of large language models for medical education and knowledge assessment","volume":"9","author":"Gilson","year":"2023","journal-title":"JMIR Med. Educ."},{"key":"10.1016\/j.patter.2026.101610_bib23","article-title":"GPT-4 technical report","author":"Achiam","year":"2023","journal-title":"arXiv"},{"key":"10.1016\/j.patter.2026.101610_bib24","article-title":"MedAlpaca\u2013an open-source collection of medical conversational AI models and training data","author":"Han","year":"2023","journal-title":"arXiv"},{"key":"10.1016\/j.patter.2026.101610_bib25","doi-asserted-by":"crossref","first-page":"1833","DOI":"10.1093\/jamia\/ocae045","article-title":"PMC-LLaMA: toward building open-source language models for medicine","volume":"31","author":"Wu","year":"2024","journal-title":"J. Am. Med. Inform. Assoc."},{"key":"10.1016\/j.patter.2026.101610_bib26","article-title":"Meditron-70b: Scaling medical pretraining for large language models","author":"Chen","year":"2023","journal-title":"arXiv"},{"key":"10.1016\/j.patter.2026.101610_bib27","series-title":"Proceedings of the 2024 Conference on Empirical Methods in Natural Language Processing","first-page":"17371","article-title":"MedCoT: Medical chain of thought via hierarchical expert","author":"Liu","year":"2024"},{"key":"10.1016\/j.patter.2026.101610_bib28","doi-asserted-by":"crossref","DOI":"10.1093\/bioinformatics\/btad651","article-title":"MedCPT: Contrastive pre-trained transformers with large-scale pubmed search logs for zero-shot biomedical information retrieval","volume":"39","author":"Jin","year":"2023","journal-title":"Bioinformatics"},{"key":"10.1016\/j.patter.2026.101610_bib29","doi-asserted-by":"crossref","first-page":"i119","DOI":"10.1093\/bioinformatics\/btae238","article-title":"Improving medical reasoning through retrieval and self-reflection with retrieval-augmented LLMs","volume":"40","author":"Jeong","year":"2024","journal-title":"Bioinformatics"},{"key":"10.1016\/j.patter.2026.101610_bib30","series-title":"Findings of the Association for Computational Linguistics","first-page":"6233","article-title":"Benchmarking retrieval-augmented generation (RAG) for medicine","author":"Xiong","year":"2024"},{"key":"10.1016\/j.patter.2026.101610_bib31","series-title":"Findings of the Association for Computational Linguistics ACL 2024","first-page":"599","article-title":"MedAgents: Large language models as collaborators for zero-shot medical reasoning","author":"Tang","year":"2024"},{"key":"10.1016\/j.patter.2026.101610_bib32","first-page":"1","article-title":"MDAgents: An adaptive collaboration of llms in medical decision making","volume":"37","author":"Kim","year":"2024","journal-title":"Adv. Neural Inf. Process. Syst."},{"key":"10.1016\/j.patter.2026.101610_bib33","doi-asserted-by":"crossref","first-page":"24824","DOI":"10.52202\/068431-1800","article-title":"Chain-of-thought prompting elicits reasoning in large language models","volume":"35","author":"Wei","year":"2022","journal-title":"Adv. Neural Inf. Process. Syst."},{"key":"10.1016\/j.patter.2026.101610_bib34","series-title":"International Conference on Learning Representations","first-page":"1","article-title":"Self-consistency improves chain of thought reasoning in language models","author":"Wang","year":"2023"},{"key":"10.1016\/j.patter.2026.101610_bib37","article-title":"Can generalist foundation models outcompete special-purpose tuning? Case study in medicine","author":"Nori","year":"2023","journal-title":"arXiv"},{"key":"10.1016\/j.patter.2026.101610_bib38","series-title":"International Conference on Learning Representations","first-page":"1","article-title":"AFlow: Automating agentic workflow generation","author":"Zhang","year":"2025"},{"key":"10.1016\/j.patter.2026.101610_bib39","series-title":"Findings of the Association for Computational Linguistics: EMNLP 2025","first-page":"9017","article-title":"Self-supervised prompt optimization","author":"Xiang","year":"2025"},{"key":"10.1016\/j.patter.2026.101610_bib35","article-title":"MedicalAgentsBench: A benchmark of 862 complex clinical questions (dataset)","author":"Shao","year":"2026","journal-title":"Zenodo"},{"key":"10.1016\/j.patter.2026.101610_bib36","article-title":"MedicalAgentsBench: Evaluation frame- work and baseline implementations (code)","author":"Shao","year":"2026","journal-title":"Zenodo"},{"key":"10.1016\/j.patter.2026.101610_bib40","doi-asserted-by":"crossref","first-page":"633","DOI":"10.1038\/s41586-025-09422-z","article-title":"DeepSeek-R1 incentivizes reasoning in LLMs through reinforcement learning","volume":"645","author":"Guo","year":"2025","journal-title":"Nature"},{"key":"10.1016\/j.patter.2026.101610_bib41","article-title":"Understanding reasoning LLMs: Methods and strategies for building and refining reasoning models","author":"Raschka","year":"2025","journal-title":"Ahead of AI"},{"key":"10.1016\/j.patter.2026.101610_bib42","series-title":"Proceedings of the 2025 Conference of the Nations of the Americas Chapter of the Association for Computational Linguistics","first-page":"3563","article-title":"Benchmarking large language models on answering and explaining challenging medical questions","author":"Chen","year":"2025"},{"key":"10.1016\/j.patter.2026.101610_bib43","series-title":"Processing BioNLP Workshop","first-page":"167","article-title":"MedExQA: Medical question answering benchmark with multiple explanations","author":"Kim","year":"2024"},{"key":"10.1016\/j.patter.2026.101610_bib44","series-title":"Proceedings of the 42nd International Conference on Machine Learning vol. 267 of Proceedings of Machine Learning Research","first-page":"80961","article-title":"MedXpertQA: Benchmarking expert-level medical reasoning and understanding","author":"Zuo","year":"2025"},{"key":"10.1016\/j.patter.2026.101610_bib45","series-title":"International Conference on Learning Representations","first-page":"1","article-title":"Measuring massive multitask language understanding","author":"Hendrycks","year":"2021"},{"key":"10.1016\/j.patter.2026.101610_bib46","first-page":"1","article-title":"MMLU-Pro: A more robust and challenging multi-task language understanding benchmark","volume":"37","author":"Wang","year":"2024","journal-title":"Adv. Neural Inf. Process. Syst."},{"key":"10.1016\/j.patter.2026.101610_bib47","doi-asserted-by":"crossref","first-page":"943","DOI":"10.1038\/s41591-025-04151-2","article-title":"Holistic evaluation of large language models for medical tasks with MedHELM","volume":"32","author":"Bedi","year":"2026","journal-title":"Nat. Med."},{"key":"10.1016\/j.patter.2026.101610_bib48","doi-asserted-by":"crossref","DOI":"10.1056\/AIdbp2500144","article-title":"MedAgentBench: A virtual EHR environment to benchmark medical LLM agents","volume":"2","author":"Jiang","year":"2025","journal-title":"NEJM AI"},{"key":"10.1016\/j.patter.2026.101610_bib49","first-page":"1","article-title":"MedAgentBoard: Benchmarking multi-agent collaboration with conventional methods for diverse medical tasks","volume":"38","author":"Zhu","year":"2025","journal-title":"Adv. Neural Inf. Process. Syst."},{"key":"10.1016\/j.patter.2026.101610_bib50","series-title":"International Conference on Learning Representations","first-page":"1","article-title":"MedAgentGym: A scalable agentic training environment for code-centric reasoning in biomedical data science","author":"Xu","year":"2026"},{"key":"10.1016\/j.patter.2026.101610_bib51","article-title":"Capabilities of GPT-4 on medical challenge problems","author":"Nori","year":"2023","journal-title":"arXiv"},{"key":"10.1016\/j.patter.2026.101610_bib52","doi-asserted-by":"crossref","first-page":"210","DOI":"10.1038\/s41746-023-00958-w","article-title":"A study of generative large language model for medical research and healthcare","volume":"6","author":"Peng","year":"2023","journal-title":"NPJ Digit. Med."}],"updated-by":[{"DOI":"10.1016\/j.patter.2026.101658","type":"erratum","label":"Erratum","source":"publisher","updated":{"date-parts":[[2026,8,25]],"date-time":"2026-08-25T00:00:00Z","timestamp":1787616000000}},{"DOI":"10.1016\/j.patter.2026.101658","type":"erratum","label":"Erratum","source":"publisher","updated":{"date-parts":[[2026,9,11]],"date-time":"2026-09-11T00:00:00Z","timestamp":1789084800000}}],"container-title":["Patterns"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/api.elsevier.com\/content\/article\/PII:S2666389926001194?httpAccept=text\/xml","content-type":"text\/xml","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/api.elsevier.com\/content\/article\/PII:S2666389926001194?httpAccept=text\/plain","content-type":"text\/plain","content-version":"vor","intended-application":"text-mining"}],"deposited":{"date-parts":[[2026,10,9]],"date-time":"2026-10-09T15:48:54Z","timestamp":1791560934000},"score":1,"resource":{"primary":{"URL":"https:\/\/linkinghub.elsevier.com\/retrieve\/pii\/S2666389926001194"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026,10]]},"references-count":52,"journal-issue":{"issue":"10","published-print":{"date-parts":[[2026,10]]}},"alternative-id":["S2666389926001194"],"URL":"https:\/\/doi.org\/10.1016\/j.patter.2026.101610","relation":{},"ISSN":["2666-3899"],"issn-type":[{"value":"2666-3899","type":"print"}],"subject":[],"published":{"date-parts":[[2026,10]]},"assertion":[{"value":"Elsevier","name":"publisher","label":"This article is maintained by"},{"value":"MedicalAgentsBench for complex medical reasoning: Comparing internalized reasoning models versus externalized agent-based frameworks","name":"articletitle","label":"Article Title"},{"value":"Patterns","name":"journaltitle","label":"Journal Title"},{"value":"https:\/\/doi.org\/10.1016\/j.patter.2026.101610","name":"articlelink","label":"CrossRef DOI link to publisher maintained version"},{"value":"article","name":"content_type","label":"Content Type"},{"value":"\u00a9 2026 The Author(s). Published by Elsevier Inc.","name":"copyright","label":"Copyright"}],"article-number":"101610"}}