{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,9,30]],"date-time":"2026-09-30T22:49:57Z","timestamp":1790808597459,"version":"4.1.0"},"reference-count":58,"publisher":"Oxford University Press (OUP)","issue":"10","license":[{"start":{"date-parts":[[2023,7,10]],"date-time":"2023-07-10T00:00:00Z","timestamp":1688947200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"funder":[{"name":"VA Center for Innovation to Implementation"},{"DOI":"10.13039\/100000002","name":"NIH","doi-asserted-by":"publisher","award":["DP1LM014278"],"award-info":[{"award-number":["DP1LM014278"]}],"id":[{"id":"10.13039\/100000002","id-type":"DOI","asserted-by":"publisher"}]},{"name":"Stanford HAI Hoffman-Yee Research Grant"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2023,9,25]]},"abstract":"<jats:title>Abstract<\/jats:title>\n                  <jats:p>Clinical decision support (CDS) systems powered by predictive models have the potential to improve the accuracy and efficiency of clinical decision-making. However, without sufficient validation, these systems have the potential to mislead clinicians and harm patients. This is especially true for CDS systems used by opioid prescribers and dispensers, where a flawed prediction can directly harm patients. To prevent these harms, regulators and researchers have proposed guidance for validating predictive models and CDS systems. However, this guidance is not universally followed and is not required by law. We call on CDS developers, deployers, and users to hold these systems to higher standards of clinical and technical validation. We provide a case study on two CDS systems deployed on a national scale in the United States for predicting a patient\u2019s risk of adverse opioid-related events: the Stratification Tool for Opioid Risk Mitigation (STORM), used by the Veterans Health Administration, and NarxCare, a commercial system.<\/jats:p>","DOI":"10.1093\/jamia\/ocad110","type":"journal-article","created":{"date-parts":[[2023,7,1]],"date-time":"2023-07-01T08:35:54Z","timestamp":1688200554000},"page":"1741-1746","source":"Crossref","is-referenced-by-count":9,"title":["A call for better validation of opioid overdose risk algorithms"],"prefix":"10.1093","volume":"30","author":[{"given":"Duncan C","family":"McElfresh","sequence":"first","affiliation":[{"name":"Department of Health Policy, Stanford University , Stanford, California, USA"},{"name":"Program Evaluation Resource Center, Office of Mental Health and Suicide Prevention, US Department of Veterans Affairs , Menlo Park, California, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Lucia","family":"Chen","sequence":"additional","affiliation":[{"name":"Department of Health Policy, Stanford University , Stanford, California, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Elizabeth","family":"Oliva","sequence":"additional","affiliation":[{"name":"Program Evaluation Resource Center, Office of Mental Health and Suicide Prevention, US Department of Veterans Affairs , Menlo Park, California, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Vilija","family":"Joyce","sequence":"additional","affiliation":[{"name":"Program Evaluation Resource Center, Office of Mental Health and Suicide Prevention, US Department of Veterans Affairs , Menlo Park, California, USA"},{"name":"Health Economics Resource Center, US Department of Veterans Affairs , Menlo Park, California, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Sherri","family":"Rose","sequence":"additional","affiliation":[{"name":"Department of Health Policy, Stanford University , Stanford, California, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Suzanne","family":"Tamang","sequence":"additional","affiliation":[{"name":"Program Evaluation Resource Center, Office of Mental Health and Suicide Prevention, US Department of Veterans Affairs , Menlo Park, California, USA"},{"name":"Department of Medicine, Stanford University , Stanford, California, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"286","published-online":{"date-parts":[[2023,7,10]]},"reference":[{"issue":"7494","key":"2023092720004797700_ocad110-B1","doi-asserted-by":"crossref","first-page":"765","DOI":"10.1136\/bmj.38398.500764.8F","article-title":"Improving clinical practice using clinical decision support systems: a systematic review of trials to identify features critical to success","volume":"330","author":"Kawamoto","year":"2005","journal-title":"BMJ"},{"issue":"1","key":"2023092720004797700_ocad110-B2","doi-asserted-by":"crossref","first-page":"29","DOI":"10.7326\/0003-4819-157-1-201207030-00450","article-title":"Effect of clinical decision-support systems","volume":"157","author":"Bright","year":"2012","journal-title":"Ann Intern Med"},{"issue":"4","key":"2023092720004797700_ocad110-B3","doi-asserted-by":"crossref","first-page":"408","DOI":"10.1016\/j.neunet.2005.10.007","article-title":"The use of artificial neural networks in decision support in cancer: a systematic review","volume":"19","author":"Lisboa","year":"2006","journal-title":"Neural Networks"},{"key":"2023092720004797700_ocad110-B4","doi-asserted-by":"crossref","first-page":"49","DOI":"10.1016\/j.cmpb.2018.06.006","article-title":"Predictive models for hospital readmission risk: a systematic review of methods","volume":"164","author":"Artetxe","year":"2018","journal-title":"Comput Methods Programs Biomed"},{"issue":"1","key":"2023092720004797700_ocad110-B5","doi-asserted-by":"crossref","first-page":"17","DOI":"10.1038\/s41746-020-0221-y","article-title":"An overview of clinical decision support systems: benefits, risks, and strategies for success","volume":"3","author":"Sutton","year":"2020","journal-title":"NPJ Digit Med"},{"issue":"1","key":"2023092720004797700_ocad110-B6","doi-asserted-by":"crossref","first-page":"2","DOI":"10.1038\/s41746-021-00549-7","article-title":"Guidelines and quality criteria for artificial intelligence-based prediction models in healthcare: a scoping review","volume":"5","author":"de Hond","year":"2022","journal-title":"NPJ Digit Med"},{"issue":"8","key":"2023092720004797700_ocad110-B7","doi-asserted-by":"crossref","first-page":"e2227779","DOI":"10.1001\/jamanetworkopen.2022.27779","article-title":"Assessment of adherence to reporting guidelines by commonly used clinical prediction models from a single vendor: a systematic review","volume":"5","author":"Lu","year":"2022","journal-title":"JAMA Netw Open"},{"key":"2023092720004797700_ocad110-B8","author":"International Medical Device Regulators Forum, Software as a Medical Device (SaMD) Working Group","year":"2014"},{"key":"2023092720004797700_ocad110-B9","author":"U.S. Department of Health and Human Services, Food and Drug Administration","year":"2017"},{"key":"2023092720004797700_ocad110-B10","author":"U.S. Food and Drug Administration","year":"2022"},{"issue":"5","key":"2023092720004797700_ocad110-B11","doi-asserted-by":"crossref","first-page":"983","DOI":"10.1093\/jamia\/ocac002","article-title":"Trends in the conduct and reporting of clinical prediction model development and validation: a systematic review","volume":"29","author":"Yang","year":"2022","journal-title":"J Am Med Inform Assoc"},{"issue":"5","key":"2023092720004797700_ocad110-B12","doi-asserted-by":"crossref","first-page":"206","DOI":"10.1038\/s42256-019-0048-x","article-title":"Stop explaining black box machine learning models for high stakes decisions and use interpretable models instead","volume":"1","author":"Rudin","year":"2019","journal-title":"Nat Mach Intell"},{"issue":"2","key":"2023092720004797700_ocad110-B13","doi-asserted-by":"crossref","first-page":"285","DOI":"10.2337\/dc15-0509","article-title":"Racial differences in the performance of existing risk prediction models for incident type 2 diabetes: the cardia study","volume":"39","author":"Lacy","year":"2016","journal-title":"Diabetes Care"},{"issue":"2","key":"2023092720004797700_ocad110-B14","doi-asserted-by":"crossref","first-page":"167","DOI":"10.1001\/amajethics.2019.167","article-title":"Can ai help reduce disparities in general medical and mental health care?","volume":"21","author":"Chen","year":"2019","journal-title":"AMA J Ethics"},{"issue":"1","key":"2023092720004797700_ocad110-B15","doi-asserted-by":"crossref","first-page":"e100456","DOI":"10.1136\/bmjhci-2021-100456","article-title":"Resampling to address inequities in predictive modeling of suicide deaths","volume":"29","author":"Reeves","year":"2022","journal-title":"BMJ Health Care Inform"},{"issue":"6464","key":"2023092720004797700_ocad110-B16","doi-asserted-by":"crossref","first-page":"447","DOI":"10.1126\/science.aax2342","article-title":"Dissecting racial bias in an algorithm used to manage the health of populations","volume":"366","author":"Obermeyer","year":"2019","journal-title":"Science"},{"issue":"25","key":"2023092720004797700_ocad110-B17","doi-asserted-by":"crossref","first-page":"2477","DOI":"10.1056\/NEJMc2029240","article-title":"Racial bias in pulse oximetry measurement","volume":"383","author":"Sjoding","year":"2020","journal-title":"N Engl J Med"},{"issue":"2","key":"2023092720004797700_ocad110-B18","doi-asserted-by":"crossref","first-page":"268","DOI":"10.1053\/j.ajkd.2021.08.003","article-title":"A unifying approach for GFR estimation: recommendations of the NKF-ASN task force on reassessing the inclusion of race in diagnosing kidney disease","volume":"79","author":"Delgado","year":"2022","journal-title":"Am J Kidney Dis"},{"issue":"1","key":"2023092720004797700_ocad110-B19","doi-asserted-by":"crossref","first-page":"84","DOI":"10.1093\/humrep\/dev268","article-title":"Predicting the chance of live birth for women undergoing IVF: a novel pretreatment counselling tool","volume":"31","author":"Dhillon","year":"2016","journal-title":"Hum Reprod"},{"key":"2023092720004797700_ocad110-B20","doi-asserted-by":"crossref","first-page":"e4181","DOI":"10.1136\/bmj.e4181","article-title":"Predicting the 10 year risk of cardiovascular disease in the United Kingdom: independent and external validation of an updated version of QRISK2","volume":"344","author":"Collins","year":"2012","journal-title":"BMJ"},{"key":"2023092720004797700_ocad110-B21","author":"Spencer","year":"2022"},{"key":"2023092720004797700_ocad110-B22","doi-asserted-by":"crossref","first-page":"74","DOI":"10.1016\/j.trsl.2021.03.012","article-title":"Assessing opioid overdose risk: a review of clinical prediction models utilizing patient-level data","volume":"234","author":"Tseregounis","year":"2021","journal-title":"Transl Res"},{"key":"2023092720004797700_ocad110-B23","author":"Appriss Health","year":"2019"},{"key":"2023092720004797700_ocad110-B24","author":"Sacco","year":"2018"},{"key":"2023092720004797700_ocad110-B25","article-title":"Dosing discrimination: regulating PDMP risk scores","volume":"47","author":"Oliva","year":"2021","journal-title":"110 California Law Review"},{"key":"2023092720004797700_ocad110-B26","author":"Whalen","year":"2020"},{"issue":"5","key":"2023092720004797700_ocad110-B27","doi-asserted-by":"crossref","first-page":"437","DOI":"10.1111\/papr.12097","article-title":"Prevalence of diagnosed opioid abuse and its economic burden in the Veterans Health Administration","volume":"14","author":"Baser","year":"2014","journal-title":"Pain Pract"},{"issue":"4","key":"2023092720004797700_ocad110-B28","doi-asserted-by":"crossref","first-page":"393","DOI":"10.1097\/MLR.0b013e318202aa27","article-title":"Accidental poisoning mortality among patients in the Department of Veterans Affairs Health System","volume":"49","author":"Bohnert","year":"2011","journal-title":"Med Care"},{"key":"2023092720004797700_ocad110-B29","author":"Miotto","year":"2022"},{"key":"2023092720004797700_ocad110-B30","first-page":"272","author":"Kilby","year":"2021"},{"key":"2023092720004797700_ocad110-B31","author":"Szalavitz","year":"2021"},{"key":"2023092720004797700_ocad110-B32","author":"Appriss Health","year":"2022"},{"key":"2023092720004797700_ocad110-B33","doi-asserted-by":"crossref","first-page":"109090","DOI":"10.1016\/j.drugalcdep.2021.109090","article-title":"How prescription drug monitoring programs influence clinical decision-making: a mixed methods systematic review and meta-analysis","volume":"228","author":"Picco","year":"2021","journal-title":"Drug Alcohol Depend"},{"key":"2023092720004797700_ocad110-B34","doi-asserted-by":"crossref","first-page":"103856","DOI":"10.1016\/j.drugpo.2022.103856","article-title":"How do patient, pharmacist and medication characteristics and prescription drug monitoring program alerts influence pharmacists\u2019 decisions to dispense opioids? a randomised controlled factorial experiment","volume":"109","author":"Picco","year":"2022","journal-title":"Int J Drug Policy"},{"issue":"1","key":"2023092720004797700_ocad110-B35","doi-asserted-by":"crossref","first-page":"34","DOI":"10.1037\/ser0000099","article-title":"Development and applications of the Veterans Health Administration\u2019s stratification tool for opioid risk mitigation (storm) to improve opioid safety and prevent overdose and suicide","volume":"14","author":"Oliva","year":"2017","journal-title":"Psychol Serv"},{"issue":"1","key":"2023092720004797700_ocad110-B36","doi-asserted-by":"crossref","first-page":"5","DOI":"10.1186\/s13012-019-0852-z","article-title":"Protocol for evaluating the nationwide implementation of the VA Stratification Tool for Opioid Risk Management (STORM)","volume":"14","author":"Chinman","year":"2019","journal-title":"Implement Sci"},{"key":"2023092720004797700_ocad110-B37","author":"International Organization for Standardization","year":"2016"},{"key":"2023092720004797700_ocad110-B38","author":"Global Harmonization Task Force International Medical Device Regulators Forum","year":"2010"},{"issue":"1","key":"2023092720004797700_ocad110-B39","doi-asserted-by":"crossref","first-page":"123","DOI":"10.1146\/annurev-biodatasci-092820-114757","article-title":"Ethical machine learning in healthcare","volume":"4","author":"Chen","year":"2021","journal-title":"Annu Rev Biomed Data Sci"},{"issue":"10","key":"2023092720004797700_ocad110-B40","doi-asserted-by":"crossref","first-page":"e006556","DOI":"10.1161\/CIRCOUTCOMES.120.006556","article-title":"Recommendations for reporting machine learning analyses in clinical research","volume":"13","author":"Stevens","year":"2020","journal-title":"Circ Cardiovasc Qual Outcomes"},{"issue":"6","key":"2023092720004797700_ocad110-B41","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1145\/3457607","article-title":"A survey on bias and fairness in machine learning","volume":"54","author":"Mehrabi","year":"2022","journal-title":"ACM Comput Surv"},{"issue":"9","key":"2023092720004797700_ocad110-B42","doi-asserted-by":"crossref","first-page":"1337","DOI":"10.1038\/s41591-019-0548-6","article-title":"Do no harm: a roadmap for responsible machine learning for health care","volume":"25","author":"Wiens","year":"2019","journal-title":"Nat Med"},{"key":"2023092720004797700_ocad110-B43","author":"Huizenga","year":"2016"},{"key":"2023092720004797700_ocad110-B44","doi-asserted-by":"crossref","first-page":"109067","DOI":"10.1016\/j.drugalcdep.2021.109067","article-title":"Validation and threshold identification of a prescription drug monitoring program clinical opioid risk metric with the who alcohol, smoking, and substance involvement screening test","volume":"228","author":"Cochran","year":"2021","journal-title":"Drug Alcohol Depend"},{"key":"2023092720004797700_ocad110-B45","doi-asserted-by":"crossref","first-page":"108854","DOI":"10.1016\/j.drugalcdep.2021.108854","article-title":"Opioid deaths involving concurrent benzodiazepine use: assessing risk factors through the analysis of prescription drug monitoring data and postmortem toxicology","volume":"225","author":"Bannon","year":"2021","journal-title":"Drug Alcohol Depend"},{"issue":"9","key":"2023092720004797700_ocad110-B46","doi-asserted-by":"crossref","first-page":"1957","DOI":"10.1097\/CORR.0000000000001745","article-title":"Narxcare scores greater than 300 are associated with adverse outcomes after primary tha","volume":"479","author":"Emara","year":"2021","journal-title":"Clin Orthop Relat Res"},{"issue":"6","key":"2023092720004797700_ocad110-B47","doi-asserted-by":"crossref","first-page":"e2113977","DOI":"10.1001\/jamanetworkopen.2021.13977","article-title":"Exploration of overdose risk score and postoperative complications and health care use after total knee arthroplasty","volume":"4","author":"Emara","year":"2021","journal-title":"JAMA Netw Open"},{"issue":"12","key":"2023092720004797700_ocad110-B48","doi-asserted-by":"crossref","first-page":"9313","DOI":"10.1007\/s00464-022-09205-x","article-title":"Patient characteristics and outcomes among bariatric surgery patients with high narcotic overdose scores","volume":"36","author":"Yang","year":"2022","journal-title":"Surg Endosc"},{"issue":"2","key":"2023092720004797700_ocad110-B49","doi-asserted-by":"crossref","first-page":"375","DOI":"10.1007\/s11606-022-07617-y","article-title":"Effect of a predictive analytics-targeted program in patients on opioids: a stepped-wedge cluster randomized controlled trial","volume":"38","author":"Strombotne","year":"2023","journal-title":"J Gen Intern Med"},{"issue":"5","key":"2023092720004797700_ocad110-B50","doi-asserted-by":"crossref","first-page":"870","DOI":"10.1111\/add.16110","article-title":"Effect of a Veterans Health Administration mandate to case review patients with opioid prescriptions on mortality among patients with opioid use disorder: a secondary analysis of the storm randomized control trial","volume":"118","author":"Auty","year":"2023","journal-title":"Addiction"},{"issue":"1","key":"2023092720004797700_ocad110-B51","doi-asserted-by":"crossref","first-page":"48","DOI":"10.1186\/s13012-020-01005-y","article-title":"Tracking implementation strategies in the randomized rollout of a Veterans Affairs national opioid risk management initiative","volume":"15","author":"Rogal","year":"2020","journal-title":"Implement Sci"},{"issue":"14","key":"2023092720004797700_ocad110-B52","doi-asserted-by":"crossref","first-page":"3746","DOI":"10.1007\/s11606-022-07622-1","article-title":"Randomized policy evaluation of the Veterans Health Administration Stratification Tool for Opioid Risk Mitigation (STORM)","volume":"37","author":"Minegishi","year":"2022","journal-title":"J Gen Intern Med"},{"issue":"1","key":"2023092720004797700_ocad110-B53","doi-asserted-by":"crossref","first-page":"14","DOI":"10.1080\/08897077.2018.1540376","article-title":"Randomized program evaluation of the Veterans Health Administration Stratification Tool for Opioid Risk Mitigation (STORM): a research and clinical operations partnership to examine effectiveness","volume":"40","author":"Minegishi","year":"2019","journal-title":"Subst Abus"},{"issue":"1","key":"2023092720004797700_ocad110-B54","doi-asserted-by":"crossref","first-page":"3","DOI":"10.23876\/j.krcp.2017.36.1.3","article-title":"Medical big data: promise and challenges","volume":"36","author":"Hyung-Jin","year":"2017","journal-title":"Kidney Res Clin Pract"},{"key":"2023092720004797700_ocad110-B55","author":"Tamang","year":"2021"},{"key":"2023092720004797700_ocad110-B56","author":"Halamka","year":"2021"},{"key":"2023092720004797700_ocad110-B57","doi-asserted-by":"publisher","DOI":"10.3389\/fdgth.2022.931439","article-title":"Governance of clinical ai applications to facilitate safe and equitable deployment in a large health system: key elements and early successes","volume":"4","author":"Liao","year":"2022","journal-title":"Front Digit Health"},{"key":"2023092720004797700_ocad110-B58","author":"Armitage","year":"2022"}],"container-title":["Journal of the American Medical Informatics Association"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/academic.oup.com\/jamia\/article-pdf\/30\/10\/1741\/51770167\/ocad110.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"syndication"},{"URL":"https:\/\/academic.oup.com\/jamia\/article-pdf\/30\/10\/1741\/51770167\/ocad110.pdf","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2023,9,27]],"date-time":"2023-09-27T16:34:04Z","timestamp":1695832444000},"score":1,"resource":{"primary":{"URL":"https:\/\/academic.oup.com\/jamia\/article\/30\/10\/1741\/7222347"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023,7,10]]},"references-count":58,"journal-issue":{"issue":"10","published-online":{"date-parts":[[2023,7,10]]},"published-print":{"date-parts":[[2023,9,25]]}},"URL":"https:\/\/doi.org\/10.1093\/jamia\/ocad110","relation":{},"ISSN":["1067-5027","1527-974X"],"issn-type":[{"value":"1067-5027","type":"print"},{"value":"1527-974X","type":"electronic"}],"subject":[],"published-other":{"date-parts":[[2023,10,1]]},"published":{"date-parts":[[2023,7,10]]}}}