{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,11,14]],"date-time":"2026-11-14T22:17:04Z","timestamp":1794694624485,"version":"4.9.1"},"reference-count":26,"publisher":"Springer Science and Business Media LLC","issue":"1","license":[{"start":{"date-parts":[[2026,2,26]],"date-time":"2026-02-26T00:00:00Z","timestamp":1772064000000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by-nc-nd\/4.0"},{"start":{"date-parts":[[2026,2,26]],"date-time":"2026-02-26T00:00:00Z","timestamp":1772064000000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by-nc-nd\/4.0"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Crit Care"],"DOI":"10.1186\/s13054-025-05725-9","type":"journal-article","created":{"date-parts":[[2026,2,26]],"date-time":"2026-02-26T13:10:22Z","timestamp":1772111422000},"update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":2,"title":["Continuous Evaluation Frameworks for Retrospective Evaluation of Clinical Machine Learning Models"],"prefix":"10.1186","volume":"30","author":[{"given":"Annamalai","family":"Natarajan","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Yale","family":"Chang","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Brian","family":"McLaverty","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Lasith","family":"Adhikari","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Erina","family":"Ghosh","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2026,2,26]]},"reference":[{"issue":"6","key":"5725_CR1","doi-asserted-by":"publisher","first-page":"1059","DOI":"10.1097\/SLA.0000000000002665","volume":"269","author":"B Bartkowiak","year":"2019","unstructured":"Bartkowiak B, Snyder AM, Benjamin A, Schneider A, Twu NM, Churpek MM, et al. Validating the electronic cardiac arrest risk triage (eCART) score for risk stratification of surgical inpatients in the postoperative setting: retrospective cohort study. Ann Surg. 2019;269(6):1059\u201363.","journal-title":"Ann Surg"},{"key":"5725_CR2","doi-asserted-by":"publisher","first-page":"99","DOI":"10.1016\/j.resuscitation.2017.10.026","volume":"122","author":"E Ghosh","year":"2018","unstructured":"Ghosh E, Eshelman L, Yang L, Carlson E, Lord B. Early deterioration indicator: data-driven approach to detecting deterioration in general ward. Resuscitation. 2018;122:99\u2013105.","journal-title":"Resuscitation"},{"issue":"4","key":"5725_CR3","doi-asserted-by":"publisher","first-page":"663","DOI":"10.1097\/ALN.0000000000002300","volume":"129","author":"F Hatib","year":"2018","unstructured":"Hatib F, Jian Z, Buddi S, Lee C, Settels J, Sibert K, et al. Machine-learning algorithm to predict hypotension based on high-fidelity arterial pressure waveform analysis. Anesthesiology. 2018;129(4):663\u201374.","journal-title":"Anesthesiology"},{"key":"5725_CR4","doi-asserted-by":"publisher","DOI":"10.1097\/CCM.0000000000004145","author":"MA Reyna","year":"2020","unstructured":"Reyna MA, Josef CS, Jeter R, Westover MB, Nemati S, Clifford GD, et al. Early prediction of sepsis from clinical data: the PhysioNet\/Computing in cardiology challenge 2019. Crit Care Med. 2020. https:\/\/doi.org\/10.1097\/CCM.0000000000004145.","journal-title":"Crit Care Med"},{"issue":"1","key":"5725_CR5","doi-asserted-by":"publisher","first-page":"661","DOI":"10.1186\/s13054-020-03379-3","volume":"24","author":"JH Yoon","year":"2020","unstructured":"Yoon JH, Jeanselme V, Dubrawski A, Hravnak M, Pinsky MR, Clermont G. Prediction of hypotension events with physiologic vital sign signatures in the intensive care unit. Crit Care. 2020;24(1):661.","journal-title":"Crit Care"},{"issue":"6","key":"5725_CR6","doi-asserted-by":"publisher","first-page":"973","DOI":"10.1007\/s10877-019-00277-0","volume":"33","author":"JH Yoon","year":"2019","unstructured":"Yoon JH, Mu L, Chen L, Dubrawski A, Hravnak M, Pinsky MR, et al. Predicting tachycardia as a surrogate for instability in the intensive care unit. J Clin Monit Comput. 2019;33(6):973\u201385.","journal-title":"J Clin Monit Comput"},{"key":"5725_CR7","doi-asserted-by":"publisher","unstructured":"Evaluation of Sepsis Prediction. Models before Onset of Treatment | NEJM AI [Internet]. [cited 2025 Sept 17]. Available from: https:\/\/ai.nejm.org\/doi\/full\/https:\/\/doi.org\/10.1056\/AIoa2300032","DOI":"10.1056\/AIoa2300032"},{"issue":"8","key":"5725_CR8","doi-asserted-by":"publisher","first-page":"1065","DOI":"10.1001\/jamainternmed.2021.2626","volume":"181","author":"A Wong","year":"2021","unstructured":"Wong A, Otles E, Donnelly JP, Krumm A, McCullough J, DeTroyer-Cooley O, et al. External validation of a widely implemented proprietary sepsis prediction model in hospitalized patients. JAMA Intern Med. 2021;181(8):1065\u201370.","journal-title":"JAMA Intern Med"},{"issue":"5","key":"5725_CR9","first-page":"96","volume":"17","author":"B Benson","year":"2023","unstructured":"Benson B, Lee S, Belle A. Evaluation of the analytic for hemodynamic instability as a prediction tool for early identification of patient deterioration. Int J Health Med Eng. 2023;17(5):96\u2013100.","journal-title":"Int J Health Med Eng"},{"key":"5725_CR10","unstructured":"MIMIC-IV. a freely accessible electronic health record dataset | Scientific Data [Internet]. [cited 2025 Sept 17]. Available from: https:\/\/www.nature.com\/articles\/s41597-022-01899-x"},{"key":"5725_CR11","doi-asserted-by":"crossref","unstructured":"Morrill J, Kormilitzin A, Nevado-Holgado A, Swaminathan S, Howison S, Lyons T. The Signature-Based Model for Early Detection of Sepsis From Electronic Health Records in the Intensive Care Unit. In: 2019 Computing in Cardiology (CinC) [Internet]. 2019 [cited 2025 Sept 17]. p. Page 1-Page 4. Available from: https:\/\/ieeexplore.ieee.org\/document\/9005805","DOI":"10.22489\/CinC.2019.014"},{"issue":"1","key":"5725_CR12","doi-asserted-by":"publisher","DOI":"10.1016\/j.isci.2021.103651","volume":"25","author":"HF Deng","year":"2022","unstructured":"Deng HF, Sun MW, Wang Y, Zeng J, Yuan T, Li T, et al. Evaluating machine learning models for sepsis prediction: a systematic review of methodologies. iScience. 2022;25(1):103651.","journal-title":"iScience"},{"key":"5725_CR13","doi-asserted-by":"publisher","first-page":"79","DOI":"10.1016\/j.compbiomed.2019.04.027","volume":"109","author":"C Barton","year":"2019","unstructured":"Barton C, Chettipally U, Zhou Y, Jiang Z, Lynn-Palevsky A, Le S, et al. Evaluation of a machine learning algorithm for up to 48-hour advance prediction of sepsis using six vital signs. Comput Biol Med. 2019;109:79\u201384.","journal-title":"Comput Biol Med"},{"issue":"3","key":"5725_CR14","doi-asserted-by":"publisher","first-page":"383","DOI":"10.1007\/s00134-019-05872-y","volume":"46","author":"LM Fleuren","year":"2020","unstructured":"Fleuren LM, Klausch TLT, Zwager CL, Schoonmade LJ, Guo T, Roggeveen LF, et al. Machine learning for the prediction of sepsis: a systematic review and meta-analysis of diagnostic test accuracy. Intensive Care Med. 2020;46(3):383\u2013400.","journal-title":"Intensive Care Med"},{"key":"5725_CR15","doi-asserted-by":"publisher","DOI":"10.1016\/j.artmed.2021.102036","volume":"113","author":"SP Shashikumar","year":"2021","unstructured":"Shashikumar SP, Josef CS, Sharma A, Nemati S. DeepAISE - an interpretable and recurrent neural survival model for early prediction of sepsis. Artif Intell Med. 2021;113:102036.","journal-title":"Artif Intell Med"},{"key":"5725_CR16","unstructured":"Jiang X, Cooper GF, Neill DB. Generalized AMOC Curves For Evaluation and Improvement of Event Surveillance. AMIA Annu Symp Proc. 2009;2009:281\u20135."},{"issue":"1","key":"5725_CR17","doi-asserted-by":"publisher","first-page":"25","DOI":"10.4097\/kja.21209","volume":"75","author":"FS Nahm","year":"2022","unstructured":"Nahm FS. Receiver operating characteristic curve: overview and practical use for clinicians. Korean J Anesthesiol. 2022;75(1):25\u201336.","journal-title":"Korean J Anesthesiol"},{"issue":"3","key":"5725_CR18","doi-asserted-by":"publisher","DOI":"10.1186\/cc13868","volume":"18","author":"V Beck","year":"2014","unstructured":"Beck V, Chateau D, Bryson GL, Pisipati A, Zanotti S, Parrillo JE, et al. Timing of vasopressor initiation and mortality in septic shock: a cohort study. Crit Care. 2014;18(3):R97.","journal-title":"Crit Care"},{"issue":"5","key":"5725_CR19","doi-asserted-by":"publisher","DOI":"10.1186\/s13054-014-0532-y","volume":"18","author":"X Bai","year":"2014","unstructured":"Bai X, Yu W, Ji W, Lin Z, Tan S, Duan K, et al. Early versus delayed administration of norepinephrine in patients with septic shock. Crit Care. 2014;18(5):532.","journal-title":"Crit Care"},{"key":"5725_CR20","doi-asserted-by":"publisher","first-page":"145","DOI":"10.1016\/j.jcrc.2019.11.004","volume":"55","author":"D Colon Hidalgo","year":"2020","unstructured":"Colon Hidalgo D, Patel J, Masic D, Park D, Rech MA. Delayed vasopressor initiation is associated with increased mortality in patients with septic shock. J Crit Care. 2020;55:145\u20138.","journal-title":"J Crit Care"},{"key":"5725_CR21","unstructured":"The Third International Consensus Definitions for Sepsis and Septic Shock (Sepsis-3.) | Critical Care Medicine | JAMA | JAMA Network [Internet]. [cited 2025 Sept 17]. Available from: https:\/\/jamanetwork.com\/journals\/jama\/fullarticle\/2492881"},{"key":"5725_CR22","doi-asserted-by":"publisher","unstructured":"Time to Treatment and Mortality during Mandated Emergency Care for Sepsis. | New England Journal of Medicine [Internet]. [cited 2025 Sept 17]. Available from: https:\/\/www.nejm.org\/doi\/full\/https:\/\/doi.org\/10.1056\/NEJMoa1703058","DOI":"10.1056\/NEJMoa1703058"},{"issue":"6","key":"5725_CR23","doi-asserted-by":"publisher","first-page":"1589","DOI":"10.1097\/01.CCM.0000217961.75225.E9","volume":"34","author":"A Kumar","year":"2006","unstructured":"Kumar A, Roberts D, Wood KE, Light B, Parrillo JE, Sharma S, et al. Duration of hypotension before initiation of effective antimicrobial therapy is the critical determinant of survival in human septic shock. Crit Care Med. 2006;34(6):1589\u201396.","journal-title":"Crit Care Med"},{"issue":"1","key":"5725_CR24","doi-asserted-by":"publisher","first-page":"32","DOI":"10.1093\/jamia\/ocx084","volume":"25","author":"AE Johnson","year":"2018","unstructured":"Johnson AE, Stone DJ, Celi LA, Pollard TJ. The MIMIC code repository: enabling reproducibility in critical care research. J Am Med Inform Assoc. 2018;25(1):32\u20139.","journal-title":"J Am Med Inform Assoc"},{"key":"5725_CR25","unstructured":"Baddam S, Burns B. Systemic Inflammatory Response Syndrome. In: StatPearls [Internet]. Treasure Island (FL): StatPearls Publishing; 2025 [cited 2025 Sept 17]. Available from: http:\/\/www.ncbi.nlm.nih.gov\/books\/NBK547669\/"},{"issue":"17","key":"5725_CR26","doi-asserted-by":"publisher","first-page":"1629","DOI":"10.1056\/NEJMoa1415236","volume":"372","author":"KM Kaukonen","year":"2015","unstructured":"Kaukonen KM, Bailey M, Pilcher D, Cooper DJ, Bellomo R. Systemic inflammatory response syndrome criteria in defining severe sepsis. N Engl J Med. 2015;372(17):1629\u201338.","journal-title":"N Engl J Med"}],"container-title":["Critical Care"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1186\/s13054-025-05725-9.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1186\/s13054-025-05725-9","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1186\/s13054-025-05725-9.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,2,26]],"date-time":"2026-02-26T13:10:25Z","timestamp":1772111425000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1186\/s13054-025-05725-9"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026,2,26]]},"references-count":26,"journal-issue":{"issue":"1","published-online":{"date-parts":[[2026,12]]}},"alternative-id":["5725"],"URL":"https:\/\/doi.org\/10.1186\/s13054-025-05725-9","relation":{},"ISSN":["1364-8535"],"issn-type":[{"value":"1364-8535","type":"electronic"}],"subject":[],"published":{"date-parts":[[2026,2,26]]},"assertion":[{"value":"3 July 2025","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"13 October 2025","order":2,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"26 February 2026","order":3,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Declarations"}},{"value":"Not Applicable.","order":2,"name":"Ethics","group":{"name":"EthicsHeading","label":"Ethics approval and consent to participate"}},{"value":"We give consent for publication.","order":3,"name":"Ethics","group":{"name":"EthicsHeading","label":"Consent for publication"}},{"value":"The authors declare no competing interests.","order":4,"name":"Ethics","group":{"name":"EthicsHeading","label":"Competing interests"}}],"article-number":"99"}}