{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,2,11]],"date-time":"2026-02-11T19:15:47Z","timestamp":1770837347595,"version":"3.50.1"},"reference-count":16,"publisher":"Oxford University Press (OUP)","issue":"2","license":[{"start":{"date-parts":[[2022,11,21]],"date-time":"2022-11-21T00:00:00Z","timestamp":1668988800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/academic.oup.com\/pages\/standard-publication-reuse-rights"}],"funder":[{"DOI":"10.13039\/100000054","name":"National Cancer Institute","doi-asserted-by":"publisher","award":["K08-CA-263541"],"award-info":[{"award-number":["K08-CA-263541"]}],"id":[{"id":"10.13039\/100000054","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000054","name":"National Cancer Institute","doi-asserted-by":"publisher","award":["R01-HL138306"],"award-info":[{"award-number":["R01-HL138306"]}],"id":[{"id":"10.13039\/100000054","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000054","name":"National Cancer Institute","doi-asserted-by":"publisher","award":["UL1-TR001878"],"award-info":[{"award-number":["UL1-TR001878"]}],"id":[{"id":"10.13039\/100000054","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000002","name":"National Institutes of Health","doi-asserted-by":"publisher","id":[{"id":"10.13039\/100000002","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2023,1,18]]},"abstract":"<jats:title>Abstract<\/jats:title>\n               <jats:p>Sudden changes in health care utilization during the severe acute respiratory syndrome coronavirus 2 (SARS-CoV-2) pandemic may have impacted the performance of clinical predictive models that were trained prior to the pandemic. In this study, we evaluated the performance over time of a machine learning, electronic health record-based mortality prediction algorithm currently used in clinical practice to identify patients with cancer who may benefit from early advance care planning conversations. We show that during the pandemic period, algorithm identification of high-risk patients had a substantial and sustained decline. Decreases in laboratory utilization during the peak of the pandemic may have contributed to drift. Calibration and overall discrimination did not markedly decline during the pandemic. This argues for careful attention to the performance and retraining of predictive algorithms that use inputs from the pandemic period.<\/jats:p>","DOI":"10.1093\/jamia\/ocac221","type":"journal-article","created":{"date-parts":[[2022,11,21]],"date-time":"2022-11-21T21:52:02Z","timestamp":1669067522000},"page":"348-354","source":"Crossref","is-referenced-by-count":13,"title":["Performance drift in a mortality prediction algorithm among patients with cancer during the SARS-CoV-2 pandemic"],"prefix":"10.1093","volume":"30","author":[{"given":"Ravi B","family":"Parikh","sequence":"first","affiliation":[{"name":"Department of Medicine, Perelman School of Medicine, University of Pennsylvania , Philadelphia, Pennsylvania, USA"},{"name":"Department of Medical Ethics and Health Policy, Perelman School of Medicine, University of Pennsylvania , Philadelphia, Pennsylvania, USA"},{"name":"Corporal Michael J. Crescenz VA Medical Center , Philadelphia, Pennsylvania, USA"}]},{"given":"Yichen","family":"Zhang","sequence":"additional","affiliation":[{"name":"Department of Medical Ethics and Health Policy, Perelman School of Medicine, University of Pennsylvania , Philadelphia, Pennsylvania, USA"}]},{"given":"Likhitha","family":"Kolla","sequence":"additional","affiliation":[{"name":"Department of Medicine, Perelman School of Medicine, University of Pennsylvania , Philadelphia, Pennsylvania, USA"},{"name":"Department of Biostatistics, Epidemiology and Informatics, Perelman School of Medicine, University of Pennsylvania , Philadelphia, Pennsylvania, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-7290-2183","authenticated-orcid":false,"given":"Corey","family":"Chivers","sequence":"additional","affiliation":[{"name":"Department of Medicine, Perelman School of Medicine, University of Pennsylvania , Philadelphia, Pennsylvania, USA"}]},{"given":"Katherine R","family":"Courtright","sequence":"additional","affiliation":[{"name":"Department of Medicine, Perelman School of Medicine, University of Pennsylvania , Philadelphia, Pennsylvania, USA"}]},{"given":"Jingsan","family":"Zhu","sequence":"additional","affiliation":[{"name":"Department of Medical Ethics and Health Policy, Perelman School of Medicine, University of Pennsylvania , Philadelphia, Pennsylvania, USA"}]},{"given":"Amol S","family":"Navathe","sequence":"additional","affiliation":[{"name":"Department of Medicine, Perelman School of Medicine, University of Pennsylvania , Philadelphia, Pennsylvania, USA"},{"name":"Department of Medical Ethics and Health Policy, Perelman School of Medicine, University of Pennsylvania , Philadelphia, Pennsylvania, USA"},{"name":"Corporal Michael J. Crescenz VA Medical Center , Philadelphia, Pennsylvania, USA"}]},{"given":"Jinbo","family":"Chen","sequence":"additional","affiliation":[{"name":"Department of Biostatistics, Epidemiology and Informatics, Perelman School of Medicine, University of Pennsylvania , Philadelphia, Pennsylvania, USA"}]}],"member":"286","published-online":{"date-parts":[[2022,11,21]]},"reference":[{"key":"2023011811024075300_ocac221-B1","first-page":"1109","article-title":"Developing predictive models using electronic medical records: challenges and pitfalls","volume":"2013","author":"Paxton","year":"2013","journal-title":"AMIA Annu Symp Proc"},{"issue":"3","key":"2023011811024075300_ocac221-B2","doi-asserted-by":"crossref","first-page":"e045343","DOI":"10.1136\/bmjopen-2020-045343","article-title":"Impact of COVID-19 pandemic on utilisation of healthcare services: a systematic review","volume":"11","author":"Moynihan","year":"2021","journal-title":"BMJ Open"},{"issue":"6","key":"2023011811024075300_ocac221-B3","doi-asserted-by":"crossref","first-page":"1052","DOI":"10.1093\/jamia\/ocx030","article-title":"Calibration drift in regression and machine learning models for acute kidney injury","volume":"24","author":"Davis","year":"2017","journal-title":"J Am Med Inform Assoc"},{"issue":"8","key":"2023011811024075300_ocac221-B4","doi-asserted-by":"crossref","first-page":"1065","DOI":"10.1001\/jamainternmed.2021.2626","article-title":"External validation of a widely implemented proprietary sepsis prediction model in hospitalized patients","volume":"181","author":"Wong","year":"2021","journal-title":"JAMA Intern Med"},{"issue":"12","key":"2023011811024075300_ocac221-B5","doi-asserted-by":"crossref","first-page":"e204759","DOI":"10.1001\/jamaoncol.2020.4759","article-title":"Effect of integrating machine learning mortality estimates with behavioral nudges to clinicians on serious illness conversations among patients with cancer: a stepped-wedge cluster randomized clinical trial","volume":"6","author":"Manz","year":"2020","journal-title":"JAMA Oncol"},{"issue":"11","key":"2023011811024075300_ocac221-B6","doi-asserted-by":"crossref","first-page":"1723","DOI":"10.1001\/jamaoncol.2020.4331","article-title":"Validation of a machine learning algorithm to predict 180-day mortality for outpatients with cancer","volume":"6","author":"Manz","year":"2020","journal-title":"JAMA Oncol"},{"issue":"10","key":"2023011811024075300_ocac221-B7","doi-asserted-by":"crossref","first-page":"e1915997","DOI":"10.1001\/jamanetworkopen.2019.15997","article-title":"Machine learning approaches to predict 6-month mortality among patients with cancer","volume":"2","author":"Parikh","year":"2019","journal-title":"JAMA Netw Open"},{"issue":"1","key":"2023011811024075300_ocac221-B8","first-page":"348","article-title":"Interrupted time series regression for the evaluation of public health interventions: a tutorial","volume":"46","author":"Bernal","year":"2017","journal-title":"Int J Epidemiol"},{"key":"2023011811024075300_ocac221-B9","doi-asserted-by":"crossref","DOI":"10.1201\/9780429246593","volume-title":"An Introduction to the Bootstrap","author":"Efron","year":"1994"},{"key":"2023011811024075300_ocac221-B10","doi-asserted-by":"crossref","DOI":"10.1017\/CBO9780511802843","volume-title":"Bootstrap Methods and Their Application","author":"Davison","year":"1997"},{"issue":"1","key":"2023011811024075300_ocac221-B11","doi-asserted-by":"crossref","first-page":"128","DOI":"10.1097\/EDE.0b013e3181c30fb2","article-title":"Assessing the performance of prediction models: a framework for traditional and novel measures","volume":"21","author":"Steyerberg","year":"2010","journal-title":"Epidemiology"},{"issue":"11","key":"2023011811024075300_ocac221-B12","doi-asserted-by":"crossref","first-page":"e2135286","DOI":"10.1001\/jamanetworkopen.2021.35286","article-title":"Quantification of sepsis model alerts in 24 US hospitals before and during the COVID-19 pandemic","volume":"4","author":"Wong","year":"2021","journal-title":"JAMA Netw Open"},{"issue":"7","key":"2023011811024075300_ocac221-B13","doi-asserted-by":"crossref","first-page":"623","DOI":"10.1001\/jama.2021.24935","article-title":"Proactive vs reactive machine learning in health care: lessons from the COVID-19 pandemic","volume":"327","author":"Luo","year":"2022","journal-title":"JAMA"},{"issue":"7883","key":"2023011811024075300_ocac221-B14","doi-asserted-by":"crossref","first-page":"108","DOI":"10.1038\/s41586-021-04014-z","article-title":"Efficient and targeted COVID-19 border testing via reinforcement learning","volume":"599","author":"Bastani","year":"2021","journal-title":"Nature"},{"issue":"4","key":"2023011811024075300_ocac221-B15","doi-asserted-by":"crossref","first-page":"375","DOI":"10.1001\/jamacardio.2019.0198","article-title":"Evaluation of mortality data from the social security administration death master file for clinical research","volume":"4","author":"Navar","year":"2019","journal-title":"JAMA Cardiol"},{"key":"2023011811024075300_ocac221-B16","doi-asserted-by":"crossref","first-page":"e2200019","DOI":"10.1200\/CCI.22.00019","article-title":"Accuracy of electronic medical record follow-up data for estimating the survival time of patients with cancer","volume":"6","author":"Gensheimer","year":"2022","journal-title":"JCO Clin Cancer Inform"}],"container-title":["Journal of the American Medical Informatics Association"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/academic.oup.com\/jamia\/article-pdf\/30\/2\/348\/48754819\/ocac221.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"syndication"},{"URL":"https:\/\/academic.oup.com\/jamia\/article-pdf\/30\/2\/348\/48754819\/ocac221.pdf","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2023,1,18]],"date-time":"2023-01-18T11:03:08Z","timestamp":1674039788000},"score":1,"resource":{"primary":{"URL":"https:\/\/academic.oup.com\/jamia\/article\/30\/2\/348\/6835770"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2022,11,21]]},"references-count":16,"journal-issue":{"issue":"2","published-online":{"date-parts":[[2022,11,21]]},"published-print":{"date-parts":[[2023,1,18]]}},"URL":"https:\/\/doi.org\/10.1093\/jamia\/ocac221","relation":{},"ISSN":["1067-5027","1527-974X"],"issn-type":[{"value":"1067-5027","type":"print"},{"value":"1527-974X","type":"electronic"}],"subject":[],"published-other":{"date-parts":[[2023,2,1]]},"published":{"date-parts":[[2022,11,21]]}}}