{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,11,4]],"date-time":"2026-11-04T22:25:03Z","timestamp":1793831103929,"version":"4.6.1"},"reference-count":42,"publisher":"Elsevier BV","license":[{"start":{"date-parts":[[2025,8,1]],"date-time":"2025-08-01T00:00:00Z","timestamp":1754006400000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.elsevier.com\/tdm\/userlicense\/1.0\/"},{"start":{"date-parts":[[2025,8,1]],"date-time":"2025-08-01T00:00:00Z","timestamp":1754006400000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.elsevier.com\/legal\/tdmrep-license"},{"start":{"date-parts":[[2026,6,16]],"date-time":"2026-06-16T00:00:00Z","timestamp":1781568000000},"content-version":"am","delay-in-days":319,"URL":"http:\/\/www.elsevier.com\/open-access\/userlicense\/1.0\/"},{"start":{"date-parts":[[2025,8,1]],"date-time":"2025-08-01T00:00:00Z","timestamp":1754006400000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-017"},{"start":{"date-parts":[[2025,8,1]],"date-time":"2025-08-01T00:00:00Z","timestamp":1754006400000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"},{"start":{"date-parts":[[2025,8,1]],"date-time":"2025-08-01T00:00:00Z","timestamp":1754006400000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-012"},{"start":{"date-parts":[[2025,8,1]],"date-time":"2025-08-01T00:00:00Z","timestamp":1754006400000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2025,8,1]],"date-time":"2025-08-01T00:00:00Z","timestamp":1754006400000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-004"}],"funder":[{"DOI":"10.13039\/100006108","name":"National Center for Advancing Translational Sciences","doi-asserted-by":"publisher","award":["UL1TR003142"],"award-info":[{"award-number":["UL1TR003142"]}],"id":[{"id":"10.13039\/100006108","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000060","name":"National Institutes of Health National Institute of Allergy and Infectious Diseases","doi-asserted-by":"publisher","award":["1R01AI17812101"],"award-info":[{"award-number":["1R01AI17812101"]}],"id":[{"id":"10.13039\/100000060","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100015615","name":"National Institute on Drug Abuse Clinical Trials Network","doi-asserted-by":"publisher","award":["UG1DA015815 - CTN-0136"],"award-info":[{"award-number":["UG1DA015815 - CTN-0136"]}],"id":[{"id":"10.13039\/100015615","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000968","name":"American Heart Association Inc","doi-asserted-by":"publisher","award":["UL1TR003142"],"award-info":[{"award-number":["UL1TR003142"]}],"id":[{"id":"10.13039\/100000968","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100011098","name":"Stanford University Bio-X","doi-asserted-by":"publisher","id":[{"id":"10.13039\/100011098","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000936","name":"Gordon and Betty Moore Foundation","doi-asserted-by":"publisher","award":["12409"],"award-info":[{"award-number":["12409"]}],"id":[{"id":"10.13039\/100000936","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000002","name":"National Institutes of Health","doi-asserted-by":"publisher","id":[{"id":"10.13039\/100000002","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000026","name":"National Institute on Drug Abuse","doi-asserted-by":"publisher","id":[{"id":"10.13039\/100000026","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100020670","name":"Stanford Institute for Human-Centered Artificial Intelligence, Stanford University","doi-asserted-by":"publisher","id":[{"id":"10.13039\/100020670","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["elsevier.com","sciencedirect.com"],"crossmark-restriction":true},"short-container-title":["Journal of Biomedical Informatics"],"published-print":{"date-parts":[[2025,8]]},"DOI":"10.1016\/j.jbi.2025.104854","type":"journal-article","created":{"date-parts":[[2025,6,5]],"date-time":"2025-06-05T21:08:03Z","timestamp":1749157683000},"page":"104854","update-policy":"https:\/\/doi.org\/10.1016\/elsevier_cm_policy","source":"Crossref","is-referenced-by-count":15,"special_numbering":"C","title":["Monitoring strategies for continuous evaluation of deployed clinical prediction models"],"prefix":"10.1016","volume":"168","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-2674-6682","authenticated-orcid":false,"given":"Grace Y.E.","family":"Kim","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Conor K.","family":"Corbin","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-7873-7772","authenticated-orcid":false,"given":"Fran\u00e7ois","family":"Grolleau","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Michael","family":"Baiocchi","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Jonathan H.","family":"Chen","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"78","reference":[{"issue":"24","key":"10.1016\/j.jbi.2025.104854_b1","doi-asserted-by":"crossref","first-page":"2405","DOI":"10.1001\/jama.2019.5284","article-title":"The proliferation of reports on clinical scoring systems: issues about uptake and clinical utility","volume":"321","author":"Challener","year":"2019","journal-title":"Jama"},{"issue":"04","key":"10.1016\/j.jbi.2025.104854_b2","doi-asserted-by":"crossref","first-page":"808","DOI":"10.1055\/s-0041-1735184","article-title":"Systematic review of approaches to preserve machine learning performance in the presence of temporal dataset shift in clinical medicine","volume":"12","author":"Guo","year":"2021","journal-title":"Appl. Clin. Inform."},{"issue":"11","key":"10.1016\/j.jbi.2025.104854_b3","doi-asserted-by":"crossref","first-page":"2423","DOI":"10.1093\/jamia\/ocab118","article-title":"Developing machine learning models to personalize care levels among emergency room patients for hospital admission","volume":"28","author":"Nguyen","year":"2021","journal-title":"J. Am. Med. Inform. Assoc."},{"issue":"2","key":"10.1016\/j.jbi.2025.104854_b4","doi-asserted-by":"crossref","first-page":"339","DOI":"10.1093\/jamia\/ocv091","article-title":"OrderRex: clinical order decision support and outcome predictions by data-mining electronic medical records","volume":"23","author":"Chen","year":"2016","journal-title":"J. Am. Med. Inform. Assoc."},{"key":"10.1016\/j.jbi.2025.104854_b5","first-page":"108","article-title":"Personalized antibiograms: Machine learning for precision selection of empiric antibiotics","volume":"2020","author":"Corbin","year":"2020","journal-title":"AMIA Summits Transl. Sci. Proc."},{"issue":"1","key":"10.1016\/j.jbi.2025.104854_b6","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1038\/s43856-022-00094-8","article-title":"Personalized antibiograms for machine learning driven antibiotic selection","volume":"2","author":"Corbin","year":"2022","journal-title":"Commun. Med."},{"issue":"568","key":"10.1016\/j.jbi.2025.104854_b7","doi-asserted-by":"crossref","DOI":"10.1126\/scitranslmed.aay5067","article-title":"A decision algorithm to promote outpatient antimicrobial stewardship for uncomplicated urinary tract infection","volume":"12","author":"Kanjilal","year":"2020","journal-title":"Sci. Transl. Med."},{"issue":"10","key":"10.1016\/j.jbi.2025.104854_b8","doi-asserted-by":"crossref","first-page":"2212","DOI":"10.1093\/jamia\/ocab099","article-title":"Machine learning for initial insulin estimation in hospitalized patients","volume":"28","author":"Nguyen","year":"2021","journal-title":"J. Am. Med. Inform. Assoc."},{"issue":"1","key":"10.1016\/j.jbi.2025.104854_b9","doi-asserted-by":"crossref","first-page":"23","DOI":"10.1038\/s41746-019-0101-5","article-title":"Finding missed cases of familial hypercholesterolemia in health systems using machine learning","volume":"2","author":"Banda","year":"2019","journal-title":"Npj Digit. Med."},{"issue":"5","key":"10.1016\/j.jbi.2025.104854_b10","doi-asserted-by":"crossref","first-page":"1515","DOI":"10.1016\/j.jvs.2016.04.026","article-title":"The use of machine learning for the identification of peripheral artery disease and future mortality risk","volume":"64","author":"Ross","year":"2016","journal-title":"J. Vasc. Surg."},{"issue":"2","key":"10.1016\/j.jbi.2025.104854_b11","doi-asserted-by":"crossref","DOI":"10.1136\/bmjinnov-2019-000359","article-title":"Bridging the implementation gap of machine learning in healthcare","volume":"6","author":"Seneviratne","year":"2020","journal-title":"Bmj Innov."},{"issue":"9","key":"10.1016\/j.jbi.2025.104854_b12","doi-asserted-by":"crossref","first-page":"1532","DOI":"10.1093\/jamia\/ocad114","article-title":"DEPLOYR: a technical framework for deploying custom real-time machine learning models into the electronic medical record","volume":"30","author":"Corbin","year":"2023","journal-title":"J. Am. Med. Inform. Assoc."},{"key":"10.1016\/j.jbi.2025.104854_b13","series-title":"Researchers create guide for fair and Equitable AI in Health Care","author":"Armitage","year":"2022"},{"key":"10.1016\/j.jbi.2025.104854_b14","doi-asserted-by":"crossref","DOI":"10.1093\/jamia\/ocac078","article-title":"A framework for the oversight and local deployment of safe and high-quality prediction models","author":"Bedoya","year":"2022","journal-title":"J. Am. Med. Inform. Assoc."},{"issue":"3","key":"10.1016\/j.jbi.2025.104854_b15","doi-asserted-by":"crossref","first-page":"491","DOI":"10.1093\/jamia\/ocz192","article-title":"A governance model for the application of AI in health care","volume":"27","author":"Reddy","year":"2020","journal-title":"J. Am. Med. Inform. Assoc."},{"issue":"9","key":"10.1016\/j.jbi.2025.104854_b16","doi-asserted-by":"crossref","first-page":"1337","DOI":"10.1038\/s41591-019-0548-6","article-title":"Do no harm: a roadmap for responsible machine learning for health care","volume":"25","author":"Wiens","year":"2019","journal-title":"Nature Med."},{"key":"10.1016\/j.jbi.2025.104854_b17","doi-asserted-by":"crossref","unstructured":"J.Y. Kim, W. Boag, F. Gulamali, A. Hasan, H.D.J. Hogg, M. Lifson, D. Mulligan, M. Patel, I.D. Raji, A. Sehgal, et al., Organizational Governance of Emerging Technologies: AI Adoption in Healthcare, in: Proceedings of the 2023 ACM Conference on Fairness, Accountability, and Transparency, 2023, pp. 1396\u20131417.","DOI":"10.1145\/3593013.3594089"},{"issue":"1","key":"10.1016\/j.jbi.2025.104854_b18","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1038\/s41746-022-00611-y","article-title":"Clinical artificial intelligence quality improvement: towards continual monitoring and updating of AI algorithms in healthcare","volume":"5","author":"Feng","year":"2022","journal-title":"Npj Digit. Med."},{"issue":"3","key":"10.1016\/j.jbi.2025.104854_b19","doi-asserted-by":"crossref","first-page":"105","DOI":"10.1016\/j.dsm.2022.07.004","article-title":"Monitoring machine learning models: a categorization of challenges and methods","volume":"5","author":"Schr\u00f6der","year":"2022","journal-title":"Data Sci. Manag."},{"key":"10.1016\/j.jbi.2025.104854_b20","series-title":"Monitoring and explainability of models in production","author":"Klaise","year":"2020"},{"key":"10.1016\/j.jbi.2025.104854_b21","doi-asserted-by":"crossref","first-page":"168","DOI":"10.1016\/j.jbi.2015.10.006","article-title":"Implications of non-stationarity on predictive modeling using EHRs","volume":"58","author":"Jung","year":"2015","journal-title":"J. Biomed. Inform."},{"key":"10.1016\/j.jbi.2025.104854_b22","series-title":"Artificial Neural Networks and Machine Learning\u2013ICANN 2021: 30th International Conference on Artificial Neural Networks, Bratislava, Slovakia, September 14\u201317, 2021, Proceedings, Part IV 30","first-page":"139","article-title":"Estimating expected calibration errors","author":"Posocco","year":"2021"},{"key":"10.1016\/j.jbi.2025.104854_b23","article-title":"Net benefit approaches to the evaluation of prediction models, molecular markers, and diagnostic tests","volume":"352","author":"Vickers","year":"2016","journal-title":"Bmj"},{"issue":"14","key":"10.1016\/j.jbi.2025.104854_b24","doi-asserted-by":"crossref","first-page":"1351","DOI":"10.1001\/jama.2019.10306","article-title":"Making machine learning models clinically useful","volume":"322","author":"Shah","year":"2019","journal-title":"Jama"},{"issue":"6","key":"10.1016\/j.jbi.2025.104854_b25","doi-asserted-by":"crossref","DOI":"10.1001\/jamanetworkopen.2025.13685","article-title":"Detecting and remediating harmful data shifts for the responsible deployment of clinical AI models","volume":"8","author":"Subasri","year":"2025","journal-title":"JAMA Netw Open."},{"issue":"12","key":"10.1016\/j.jbi.2025.104854_b26","doi-asserted-by":"crossref","first-page":"1448","DOI":"10.1093\/jamia\/ocz127","article-title":"A nonparametric updating method to correct clinical prediction model drift","volume":"26","author":"Davis","year":"2019","journal-title":"J. Am. Med. Inform. Assoc."},{"issue":"18","key":"10.1016\/j.jbi.2025.104854_b27","doi-asserted-by":"crossref","DOI":"10.1161\/01.CIR.97.18.1837","article-title":"Prediction of coronary heart disease using risk factor categories","volume":"97","author":"Wilson","year":"1998","journal-title":"Circulation"},{"issue":"2","key":"10.1016\/j.jbi.2025.104854_b28","doi-asserted-by":"crossref","first-page":"263","DOI":"10.1378\/chest.09-1584","article-title":"Refining clinical risk stratification for predicting stroke and thromboembolism in atrial fibrilation using a novel risk factor-based approach","volume":"137","author":"Lip","year":"2010","journal-title":"Chest"},{"issue":"12","key":"10.1016\/j.jbi.2025.104854_b29","doi-asserted-by":"crossref","first-page":"1645","DOI":"10.1093\/jamia\/ocz145","article-title":"Prognostic models will be victims of their own success, unless\u2026","volume":"26","author":"Lenert","year":"2019","journal-title":"J. Am. Med. Inform. Assoc."},{"key":"10.1016\/j.jbi.2025.104854_b30","series-title":"International Conference on Machine Learning","first-page":"7599","article-title":"Performative prediction","author":"Perdomo","year":"2020"},{"key":"10.1016\/j.jbi.2025.104854_b31","series-title":"Performative Prediction: Theory and Practice","author":"Perdomo","year":"2023"},{"key":"10.1016\/j.jbi.2025.104854_b32","doi-asserted-by":"crossref","DOI":"10.1177\/0272989X241248612","article-title":"Feedback loop failure modes in medical diagnosis: How biases can emerge and be reinforced","author":"Aikens","year":"2024","journal-title":"Med. Decis. Making"},{"key":"10.1016\/j.jbi.2025.104854_b33","first-page":"465","article-title":"On the application of probability theory to agricultural experiments. essay on principles. Section 9","author":"Splawa-Neyman","year":"1990","journal-title":"Statist. Sci."},{"issue":"5","key":"10.1016\/j.jbi.2025.104854_b34","doi-asserted-by":"crossref","first-page":"688","DOI":"10.1037\/h0037350","article-title":"Estimating causal effects of treatments in randomized and nonrandomized studies","volume":"66","author":"Rubin","year":"1974","journal-title":"J. Educ. Psychol."},{"issue":"260","key":"10.1016\/j.jbi.2025.104854_b35","doi-asserted-by":"crossref","first-page":"663","DOI":"10.1080\/01621459.1952.10483446","article-title":"A generalization of sampling without replacement from a finite universe","volume":"47","author":"Horvitz","year":"1952","journal-title":"J. Amer. Statist. Assoc."},{"key":"10.1016\/j.jbi.2025.104854_b36","first-page":"34","article-title":"Bayesian inference for causal effects: The role of randomization","author":"Rubin","year":"1978","journal-title":"Ann. Statist."},{"issue":"1","key":"10.1016\/j.jbi.2025.104854_b37","doi-asserted-by":"crossref","first-page":"521","DOI":"10.1016\/j.patcog.2011.06.019","article-title":"A unifying view on dataset shift in classification","volume":"45","author":"Moreno-Torres","year":"2012","journal-title":"Pattern Recognit."},{"key":"10.1016\/j.jbi.2025.104854_b38","series-title":"Avoiding biased clinical machine learning model performance estimates in the presence of label selection","author":"Corbin","year":"2022"},{"key":"10.1016\/j.jbi.2025.104854_b39","series-title":"Test distribution-aware active learning: A principled approach against distribution shift and outliers","author":"Kirsch","year":"2021"},{"key":"10.1016\/j.jbi.2025.104854_b40","series-title":"On statistical bias in active learning: How and when to fix it","author":"Farquhar","year":"2021"},{"issue":"4","key":"10.1016\/j.jbi.2025.104854_b41","doi-asserted-by":"crossref","first-page":"625","DOI":"10.1093\/eurjpc\/zwab018","article-title":"Utility of the CHA2ds2-VASc score for predicting ischaemic stroke in patients with or without atrial fibrillation: a systematic review and meta-analysis","volume":"29","author":"Siddiqi","year":"2022","journal-title":"Eur. J. Prev. Cardiol."},{"issue":"10","key":"10.1016\/j.jbi.2025.104854_b42","doi-asserted-by":"crossref","first-page":"959","DOI":"10.1016\/j.jacc.2014.06.1186","article-title":"A systematic examination of the 2013 ACC\/AHA pooled cohort risk assessment tool for atherosclerotic cardiovascular disease","volume":"64","author":"Karmali","year":"2014","journal-title":"J. Am. Coll. Cardiol."}],"container-title":["Journal of Biomedical Informatics"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/api.elsevier.com\/content\/article\/PII:S1532046425000838?httpAccept=text\/xml","content-type":"text\/xml","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/api.elsevier.com\/content\/article\/PII:S1532046425000838?httpAccept=text\/plain","content-type":"text\/plain","content-version":"vor","intended-application":"text-mining"}],"deposited":{"date-parts":[[2026,5,15]],"date-time":"2026-05-15T18:04:22Z","timestamp":1778868262000},"score":1,"resource":{"primary":{"URL":"https:\/\/linkinghub.elsevier.com\/retrieve\/pii\/S1532046425000838"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,8]]},"references-count":42,"alternative-id":["S1532046425000838"],"URL":"https:\/\/doi.org\/10.1016\/j.jbi.2025.104854","relation":{},"ISSN":["1532-0464"],"issn-type":[{"value":"1532-0464","type":"print"}],"subject":[],"published":{"date-parts":[[2025,8]]},"assertion":[{"value":"Elsevier","name":"publisher","label":"This article is maintained by"},{"value":"Monitoring strategies for continuous evaluation of deployed clinical prediction models","name":"articletitle","label":"Article Title"},{"value":"Journal of Biomedical Informatics","name":"journaltitle","label":"Journal Title"},{"value":"https:\/\/doi.org\/10.1016\/j.jbi.2025.104854","name":"articlelink","label":"CrossRef DOI link to publisher maintained version"},{"value":"article","name":"content_type","label":"Content Type"},{"value":"\u00a9 2025 Elsevier Inc. All rights are reserved, including those for text and data mining, AI training, and similar technologies.","name":"copyright","label":"Copyright"}],"article-number":"104854"}}