{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,6,15]],"date-time":"2026-06-15T00:55:07Z","timestamp":1781484907477,"version":"3.54.1"},"reference-count":18,"publisher":"Springer Science and Business Media LLC","issue":"1","license":[{"start":{"date-parts":[[2026,5,15]],"date-time":"2026-05-15T00:00:00Z","timestamp":1778803200000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0"},{"start":{"date-parts":[[2026,5,15]],"date-time":"2026-05-15T00:00:00Z","timestamp":1778803200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0"}],"funder":[{"DOI":"10.13039\/100000054","name":"National Cancer Institute","doi-asserted-by":"publisher","award":["U24CA248010"],"award-info":[{"award-number":["U24CA248010"]}],"id":[{"id":"10.13039\/100000054","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000054","name":"National Cancer Institute","doi-asserted-by":"publisher","award":["U54CA274516-01A1"],"award-info":[{"award-number":["U54CA274516-01A1"]}],"id":[{"id":"10.13039\/100000054","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100007316","name":"Division of Cancer Prevention, National Cancer Institute","doi-asserted-by":"publisher","award":["R01CA294033-01"],"award-info":[{"award-number":["R01CA294033-01"]}],"id":[{"id":"10.13039\/100007316","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000048","name":"American Cancer Society","doi-asserted-by":"publisher","award":["ACS.ASTRO-CSDG-24-1244514-01-CTPS.pc.gr.222210"],"award-info":[{"award-number":["ACS.ASTRO-CSDG-24-1244514-01-CTPS.pc.gr.222210"]}],"id":[{"id":"10.13039\/100000048","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100006093","name":"Patient-Centered Outcomes Research Institute","doi-asserted-by":"publisher","award":["ME-2024C2-37484"],"award-info":[{"award-number":["ME-2024C2-37484"]}],"id":[{"id":"10.13039\/100006093","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100006108","name":"National Center for Advancing Translational Sciences","doi-asserted-by":"publisher","award":["UM1TR004408"],"award-info":[{"award-number":["UM1TR004408"]}],"id":[{"id":"10.13039\/100006108","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["npj Digit. Med."],"DOI":"10.1038\/s41746-026-02740-0","type":"journal-article","created":{"date-parts":[[2026,5,15]],"date-time":"2026-05-15T13:51:16Z","timestamp":1778853076000},"update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":1,"title":["Large language models require a new form of oversight: capability-based monitoring"],"prefix":"10.1038","volume":"9","author":[{"given":"Katherine C.","family":"Kellogg","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Bingyang","family":"Ye","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Yifan","family":"Hu","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Guergana K.","family":"Savova","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Byron","family":"Wallace","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Danielle S.","family":"Bitterman","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2026,5,15]]},"reference":[{"key":"2740_CR1","doi-asserted-by":"publisher","first-page":"e252724","DOI":"10.1001\/jamahealthforum.2025.2724","volume":"6","author":"A Wong","year":"2025","unstructured":"Wong, A. & Sussman, J. B. Understanding model drift and its impact on health care policy. JAMA Health Forum 6, e252724 (2025).","journal-title":"JAMA Health Forum"},{"key":"2740_CR2","doi-asserted-by":"publisher","first-page":"283","DOI":"10.1056\/NEJMc2104626","volume":"385","author":"SG Finlayson","year":"2021","unstructured":"Finlayson, S. G. et al. The Clinician and Dataset Shift in Artificial Intelligence. N. Engl. J. Med. 385, 283\u2013286 (2021).","journal-title":"N. Engl. J. Med."},{"key":"2740_CR3","doi-asserted-by":"publisher","first-page":"1065","DOI":"10.1001\/jamainternmed.2021.2626","volume":"181","author":"A Wong","year":"2021","unstructured":"Wong, A. et al. External validation of a widely implemented proprietary sepsis prediction model in hospitalized patients. JAMA Intern. Med. 181, 1065\u20131070 (2021).","journal-title":"JAMA Intern. Med."},{"key":"2740_CR4","doi-asserted-by":"publisher","first-page":"1040","DOI":"10.1001\/jamainternmed.2021.3333","volume":"181","author":"AR Habib","year":"2021","unstructured":"Habib, A. R., Lin, A. L. & Grant, R. W. The epic sepsis model falls short-the importance of external validation. JAMA Intern. Med. 181, 1040\u20131041 (2021).","journal-title":"JAMA Intern. Med."},{"key":"2740_CR5","doi-asserted-by":"publisher","unstructured":"Afshar, M. et al. A Novel Playbook for Pragmatic Trial Operations to Monitor and Evaluate Ambient Artificial Intelligence in Clinical Practice. NEJM AI 2, https:\/\/doi.org\/10.1056\/aidbp2401267 (2025).","DOI":"10.1056\/aidbp2401267"},{"key":"2740_CR6","first-page":"e2528056","volume":"8","author":"JG You","year":"2025","unstructured":"You, J. G. et al. Ambient documentation technology in clinician experience of documentation burden and burnout. JAMA Netw. Open 8, e2528056 (2025).","journal-title":"Open"},{"key":"2740_CR7","doi-asserted-by":"publisher","first-page":"e379","DOI":"10.1016\/S2589-7500(24)00060-8","volume":"6","author":"S Chen","year":"2024","unstructured":"Chen, S. et al. The effect of using a large language model to respond to patient messages. Lancet Digit. Health 6, e379\u2013e381 (2024).","journal-title":"Lancet Digit. Health"},{"key":"2740_CR8","doi-asserted-by":"publisher","first-page":"1026","DOI":"10.1001\/jamapediatrics.2025.1729","volume":"179","author":"M Ray","year":"2025","unstructured":"Ray, M. et al. Evaluating a large language model in translating patient instructions to Spanish using a standardized framework. JAMA Pediatr. 179, 1026\u20131033 (2025).","journal-title":"JAMA Pediatr."},{"key":"2740_CR9","doi-asserted-by":"crossref","unstructured":"Sujan, M. et al. Human factors challenges for the safe use of artificial intelligence in patient care. BMJ Health Care Inform. 26, e100081 (2019).","DOI":"10.1136\/bmjhci-2019-100081"},{"key":"2740_CR10","doi-asserted-by":"publisher","first-page":"600","DOI":"10.1038\/s41746-025-01963-x","volume":"8","author":"M Agrawal","year":"2025","unstructured":"Agrawal, M., Chen, I. Y., Gulamali, F. & Joshi, S. The evaluation illusion of large language models in medicine. NPJ Digit. Med. 8, 600 (2025).","journal-title":"NPJ Digit. Med."},{"key":"2740_CR11","doi-asserted-by":"publisher","first-page":"916","DOI":"10.1016\/j.ijrobp.2023.11.012","volume":"118","author":"AC Moreno","year":"2024","unstructured":"Moreno, A. C. & Bitterman, D. S. Toward clinical-grade evaluation of large language models. Int. J. Radiat. Oncol. Biol. Phys. 118, 916\u2013920 (2024).","journal-title":"Int. J. Radiat. Oncol. Biol. Phys."},{"key":"2740_CR12","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1145\/3485766","volume":"55","author":"AB Sai","year":"2023","unstructured":"Sai, A. B., Mohankumar, A. K. & Khapra, M. M. A survey of evaluation metrics used for NLG systems. ACM Comput. Surv. 55, 1\u201339 (2023).","journal-title":"ACM Comput. Surv."},{"key":"2740_CR13","unstructured":"Ni, S. et al. A survey on large language model benchmarks. ArXiv abs\/2508.15361, (2025)."},{"key":"2740_CR14","doi-asserted-by":"publisher","unstructured":"Guan, H., Bates, D. & Zhou, L. Keeping Medical AI Healthy and Trustworthy: A Review of Detection and Correction Methods for System Degradation. In IEEE Transactions on Biomedical Engineering. https:\/\/doi.org\/10.1109\/TBME.2025.3642706 (2025).","DOI":"10.1109\/TBME.2025.3642706"},{"key":"2740_CR15","unstructured":"EU Artificial Intelligence Act. https:\/\/artificialintelligenceact.eu\/."},{"key":"2740_CR16","unstructured":"Request For Public Comment: Measuring and Evaluating Artificial Intelligence-enabled Medical Device Performance in the Real-World. U.S. Food and Drug Administration https:\/\/www.fda.gov\/medical-devices\/digital-health-center-excellence\/request-public-comment-measuring-and-evaluating-artificial-intelligence-enabled-medical-device?utm_medium=email&utm_source=govdelivery (2025)."},{"key":"2740_CR17","doi-asserted-by":"publisher","first-page":"241","DOI":"10.1001\/jama.2024.21451","volume":"333","author":"HJ Warraich","year":"2025","unstructured":"Warraich, H. J., Tazbaz, T. & Califf, R. M. FDA perspective on the regulation of artificial intelligence in health care and biomedicine. JAMA 333, 241\u2013247 (2025).","journal-title":"JAMA"},{"key":"2740_CR18","unstructured":"Noori, A. et al. A global log for medical AI. arXiv [cs.AI] (2025)."}],"container-title":["npj Digital Medicine"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/www.nature.com\/articles\/s41746-026-02740-0.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/www.nature.com\/articles\/s41746-026-02740-0","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/www.nature.com\/articles\/s41746-026-02740-0.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,5,15]],"date-time":"2026-05-15T13:51:21Z","timestamp":1778853081000},"score":1,"resource":{"primary":{"URL":"https:\/\/www.nature.com\/articles\/s41746-026-02740-0"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026,5,15]]},"references-count":18,"journal-issue":{"issue":"1","published-online":{"date-parts":[[2026,12]]}},"alternative-id":["2740"],"URL":"https:\/\/doi.org\/10.1038\/s41746-026-02740-0","relation":{},"ISSN":["2398-6352"],"issn-type":[{"value":"2398-6352","type":"electronic"}],"subject":[],"published":{"date-parts":[[2026,5,15]]},"assertion":[{"value":"1 December 2025","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"1 May 2026","order":2,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"15 May 2026","order":3,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"D.S.B.: Associate Editor, J.C.O. Clinical Cancer Informatics (not related to the submitted work), Associate Editor, Annals of Oncology (not related to the submitted work). Associate editor of Radiation Oncology of HemOnc.Org (not related to the submitted work) and is on the Scientific Advisory Board of Mercurial AI (not related to the submitted work) and Blue Clay Health LLC (not related to the submitted work). The authors declare no other competing interests.","order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Competing interests"}}],"article-number":"375"}}