{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,3,18]],"date-time":"2026-03-18T19:12:57Z","timestamp":1773861177537,"version":"3.50.1"},"reference-count":23,"publisher":"Springer Science and Business Media LLC","issue":"1","license":[{"start":{"date-parts":[[2023,5,6]],"date-time":"2023-05-06T00:00:00Z","timestamp":1683331200000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0"},{"start":{"date-parts":[[2023,5,6]],"date-time":"2023-05-06T00:00:00Z","timestamp":1683331200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["npj Digit. Med."],"DOI":"10.1038\/s41746-023-00832-9","type":"journal-article","created":{"date-parts":[[2023,5,6]],"date-time":"2023-05-06T11:01:39Z","timestamp":1683370899000},"update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":62,"title":["Perspectives on validation of clinical predictive algorithms"],"prefix":"10.1038","volume":"6","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-3473-3398","authenticated-orcid":false,"given":"Anne A. H.","family":"de Hond","sequence":"first","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0001-5576-9878","authenticated-orcid":false,"given":"Vaibhavi B.","family":"Shah","sequence":"additional","affiliation":[]},{"given":"Ilse M. J.","family":"Kant","sequence":"additional","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0003-1613-7450","authenticated-orcid":false,"given":"Ben","family":"Van Calster","sequence":"additional","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0002-7787-0122","authenticated-orcid":false,"given":"Ewout W.","family":"Steyerberg","sequence":"additional","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0001-6553-3455","authenticated-orcid":false,"given":"Tina","family":"Hernandez-Boussard","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2023,5,6]]},"reference":[{"key":"832_CR1","doi-asserted-by":"publisher","first-page":"582","DOI":"10.1038\/s41591-021-01312-x","volume":"27","author":"E Wu","year":"2021","unstructured":"Wu, E. et al. How medical AI devices are evaluated: limitations and recommendations from an analysis of FDA approvals. Nat. Med. 27, 582\u2013584 (2021).","journal-title":"Nat. Med."},{"key":"832_CR2","doi-asserted-by":"publisher","DOI":"10.1038\/s41746-020-00336-w","volume":"3","author":"S Kakarmath","year":"2020","unstructured":"Kakarmath, S. et al. Best practices for authors of healthcare-related artificial intelligence manuscripts. npj Digital Med. 3, 134 (2020).","journal-title":"npj Digital Med."},{"key":"832_CR3","doi-asserted-by":"publisher","first-page":"1925","DOI":"10.1093\/eurheartj\/ehu207","volume":"35","author":"EW Steyerberg","year":"2014","unstructured":"Steyerberg, E. W. & Vergouwe, Y. Towards better clinical prediction models: seven steps for development and an ABCD for validation. Eur. Heart J. 35, 1925\u20131931 (2014).","journal-title":"Eur. Heart J."},{"key":"832_CR4","doi-asserted-by":"publisher","DOI":"10.1186\/s12916-019-1466-7","volume":"17","author":"B Van Calster","year":"2019","unstructured":"Van Calster, B. et al. Calibration: the Achilles heel of predictive analytics. BMC Med. 17, 230 (2019).","journal-title":"BMC Med."},{"key":"832_CR5","doi-asserted-by":"publisher","first-page":"i6","DOI":"10.1136\/bmj.i6","volume":"352","author":"AJ Vickers","year":"2016","unstructured":"Vickers, A. J., Van Calster, B. & Steyerberg, E. W. Net benefit approaches to the evaluation of prediction models, molecular markers, and diagnostic tests. BMJ 352, i6 (2016).","journal-title":"BMJ"},{"key":"832_CR6","doi-asserted-by":"crossref","unstructured":"Harrell, F. Multivariable modeling strategies. In: Regression Modeling Strategies. Springer Series in Statistics. (Springer, Cham., 2015).","DOI":"10.1007\/978-3-319-19425-7"},{"key":"832_CR7","doi-asserted-by":"crossref","unstructured":"Steyerberg, E. W. Clinical prediction models (Springer Nature, 2009).","DOI":"10.1007\/978-0-387-77244-8"},{"key":"832_CR8","doi-asserted-by":"crossref","unstructured":"Efron, B. & Tibshirani, R. J. An introduction to the bootstrap (CRC press, 1994).","DOI":"10.1201\/9780429246593"},{"key":"832_CR9","doi-asserted-by":"publisher","first-page":"e489","DOI":"10.1016\/S2589-7500(20)30186-2","volume":"2","author":"J Futoma","year":"2020","unstructured":"Futoma, J., Simons, M., Panch, T., Doshi-Velez, F. & Celi, L. A. The myth of generalisability in clinical research and machine learning in health care. Lancet Digital Health 2, e489\u2013e492 (2020).","journal-title":"Lancet Digital Health"},{"key":"832_CR10","doi-asserted-by":"publisher","unstructured":"Wan, B., Caffo, B. & Vedula, S. S. A unified framework on generalizability of clinical prediction models. Front. Artif. Intell. 5, https:\/\/doi.org\/10.3389\/frai.2022.872720 (2022).","DOI":"10.3389\/frai.2022.872720"},{"key":"832_CR11","doi-asserted-by":"publisher","first-page":"291","DOI":"10.1097\/CCM.0000000000005758","volume":"51","author":"AAH de Hond","year":"2023","unstructured":"de Hond, A. A. H. et al. Predicting readmission or death after discharge from the ICU: external validation and retraining of a machine learning model. Crit. Care Med. 51, 291\u2013300 (2023).","journal-title":"Crit. Care Med."},{"key":"832_CR12","doi-asserted-by":"publisher","first-page":"76","DOI":"10.1016\/j.jclinepi.2016.05.007","volume":"79","author":"PC Austin","year":"2016","unstructured":"Austin, P. C. et al. Geographic and temporal validity of prediction models: different approaches were useful to examine model performance. J. Clin. Epidemiol. 79, 76\u201385 (2016).","journal-title":"J. Clin. Epidemiol."},{"key":"832_CR13","doi-asserted-by":"publisher","first-page":"4290","DOI":"10.1002\/sim.8296","volume":"38","author":"EW Steyerberg","year":"2019","unstructured":"Steyerberg, E. W., Nieboer, D., Debray, T. P. A. & van Houwelingen, H. C. Assessment of heterogeneity in an individual participant data meta-analysis of prediction models: an overview and illustration. Stat. Med 38, 4290\u20134309 (2019).","journal-title":"Stat. Med"},{"key":"832_CR14","doi-asserted-by":"publisher","first-page":"279","DOI":"10.1016\/j.jclinepi.2014.06.018","volume":"68","author":"TP Debray","year":"2015","unstructured":"Debray, T. P. et al. A new framework to enhance the interpretation of external validation studies of clinical prediction models. J. Clin. Epidemiol. 68, 279\u2013289 (2015).","journal-title":"J. Clin. Epidemiol."},{"key":"832_CR15","doi-asserted-by":"publisher","DOI":"10.1186\/s41512-019-0060-y","volume":"3","author":"LE Cowley","year":"2019","unstructured":"Cowley, L. E., Farewell, D. M., Maguire, S. & Kemp, A. M. Methodological standards for the development and evaluation of clinical prediction rules: a review of the literature. Diagnostic Progn. Res. 3, 16 (2019).","journal-title":"Diagnostic Progn. Res."},{"key":"832_CR16","doi-asserted-by":"publisher","first-page":"m1328","DOI":"10.1136\/bmj.m1328","volume":"369","author":"L Wynants","year":"2020","unstructured":"Wynants, L. et al. Prediction models for diagnosis and prognosis of covid-19: systematic review and critical appraisal. BMJ 369, m1328 (2020).","journal-title":"BMJ"},{"key":"832_CR17","doi-asserted-by":"publisher","first-page":"e008487","DOI":"10.1161\/CIRCOUTCOMES.121.008487","volume":"15","author":"G Gulati","year":"2022","unstructured":"Gulati, G. et al. Generalizability of cardiovascular disease clinical prediction models: 158 independent external validations of 104 unique models. Circ. Cardiovasc. Qual. Outcomes 15, e008487 (2022).","journal-title":"Circ. Cardiovasc. Qual. Outcomes"},{"key":"832_CR18","doi-asserted-by":"publisher","first-page":"e489","DOI":"10.1016\/S2589-7500(20)30186-2","volume":"2","author":"J Futoma","year":"2020","unstructured":"Futoma, J., Simons, M., Panch, T., Doshi-Velez, F. & Celi, L. A. The myth of generalisability in clinical research and machine learning in health care. Lancet Digit Health 2, e489\u2013e492 (2020).","journal-title":"Lancet Digit Health"},{"key":"832_CR19","doi-asserted-by":"publisher","first-page":"939","DOI":"10.1097\/ALN.0000000000003223","volume":"132","author":"ML Burns","year":"2020","unstructured":"Burns, M. L. & Kheterpal, S. Machine learning comes of age: local impact versus national generalizability. Anesthesiology 132, 939\u2013941 (2020).","journal-title":"Anesthesiology"},{"key":"832_CR20","doi-asserted-by":"publisher","DOI":"10.1038\/s41746-021-00549-7","volume":"5","author":"AAH de Hond","year":"2022","unstructured":"de Hond, A. A. H. et al. Guidelines and quality criteria for artificial intelligence-based prediction models in healthcare: a scoping review. npj Digital Med. 5, 2 (2022).","journal-title":"npj Digital Med."},{"key":"832_CR21","doi-asserted-by":"publisher","DOI":"10.1186\/s41512-022-00136-8","volume":"6","author":"M Sperrin","year":"2022","unstructured":"Sperrin, M., Riley, R. D., Collins, G. S. & Martin, G. P. Targeted validation: validating clinical prediction models in their intended population and setting. Diagnostic Progn. Res. 6, 24 (2022).","journal-title":"Diagnostic Progn. Res."},{"key":"832_CR22","doi-asserted-by":"publisher","DOI":"10.1186\/s12916-023-02779-w","volume":"21","author":"B Van Calster","year":"2023","unstructured":"Van Calster, B., Steyerberg, E. W., Wynants, L. & van Smeden, M. There is no such thing as a validated prediction model. BMC Med. 21, 70 (2023).","journal-title":"BMC Med."},{"key":"832_CR23","doi-asserted-by":"publisher","first-page":"1142","DOI":"10.1016\/j.eururo.2014.11.025","volume":"67","author":"GS Collins","year":"2015","unstructured":"Collins, G. S., Reitsma, J. B., Altman, D. G. & Moons, K. G. M. Transparent reporting of a multivariable prediction model for individual prognosis or diagnosis (TRIPOD): the TRIPOD statement. Eur. Urol. 67, 1142\u20131151 (2015).","journal-title":"Eur. Urol."}],"container-title":["npj Digital Medicine"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/www.nature.com\/articles\/s41746-023-00832-9.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/www.nature.com\/articles\/s41746-023-00832-9","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/www.nature.com\/articles\/s41746-023-00832-9.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2023,5,6]],"date-time":"2023-05-06T11:03:46Z","timestamp":1683371026000},"score":1,"resource":{"primary":{"URL":"https:\/\/www.nature.com\/articles\/s41746-023-00832-9"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023,5,6]]},"references-count":23,"journal-issue":{"issue":"1","published-online":{"date-parts":[[2023,12]]}},"alternative-id":["832"],"URL":"https:\/\/doi.org\/10.1038\/s41746-023-00832-9","relation":{},"ISSN":["2398-6352"],"issn-type":[{"value":"2398-6352","type":"electronic"}],"subject":[],"published":{"date-parts":[[2023,5,6]]},"assertion":[{"value":"10 January 2023","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"28 April 2023","order":2,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"6 May 2023","order":3,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"The authors declare no competing interests.","order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Competing interests"}}],"article-number":"86"}}