{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,11]],"date-time":"2026-04-11T02:27:10Z","timestamp":1775874430103,"version":"3.50.1"},"reference-count":50,"publisher":"Springer Science and Business Media LLC","issue":"1","license":[{"start":{"date-parts":[[2025,2,17]],"date-time":"2025-02-17T00:00:00Z","timestamp":1739750400000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by-nc-nd\/4.0"},{"start":{"date-parts":[[2025,2,17]],"date-time":"2025-02-17T00:00:00Z","timestamp":1739750400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by-nc-nd\/4.0"}],"funder":[{"DOI":"10.13039\/100000092","name":"U.S. National Library of Medicine","doi-asserted-by":"publisher","award":["T15LM007092"],"award-info":[{"award-number":["T15LM007092"]}],"id":[{"id":"10.13039\/100000092","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000002","name":"National Institutes of Health","doi-asserted-by":"publisher","award":["UL1TR003142"],"award-info":[{"award-number":["UL1TR003142"]}],"id":[{"id":"10.13039\/100000002","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["npj Digit. Med."],"DOI":"10.1038\/s41746-025-01506-4","type":"journal-article","created":{"date-parts":[[2025,2,18]],"date-time":"2025-02-18T02:11:42Z","timestamp":1739844702000},"update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":26,"title":["Clinical trials informed framework for real world clinical implementation and deployment of artificial intelligence applications"],"prefix":"10.1038","volume":"8","author":[{"given":"Jacqueline G.","family":"You","sequence":"first","affiliation":[]},{"given":"Tina","family":"Hernandez-Boussard","sequence":"additional","affiliation":[]},{"given":"Michael A.","family":"Pfeffer","sequence":"additional","affiliation":[]},{"given":"Adam","family":"Landman","sequence":"additional","affiliation":[]},{"given":"Rebecca G.","family":"Mishuris","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2025,2,17]]},"reference":[{"key":"1506_CR1","doi-asserted-by":"publisher","DOI":"10.1038\/s41746-020-00333-z","volume":"3","author":"B Mesko","year":"2020","unstructured":"Mesko, B. & G\u00f6r\u00f6g, M. A short guide for medical professionals in the era of artificial intelligence. npj Digit. Med. 3, 126 (2020).","journal-title":"npj Digit. Med."},{"key":"1506_CR2","unstructured":"European Parliament. EU AI Act: first regulation on artificial intelligence. European Parliament https:\/\/www.europarl.europa.eu\/topics\/en\/article\/20230601STO93804\/eu-ai-act-first-regulation-on-artificial-intelligence (2024)."},{"key":"1506_CR3","unstructured":"Regulation (EU) 2024\/1689 of the European Parliament and of the Council of 13 June 2024 Laying down Harmonised Rules on Artificial Intelligence and Amending Regulations (EC) No 300\/2008, (EU) No 167\/2013, (EU) No 168\/2013, (EU) 2018\/858, (EU) 2018\/1139 and (EU) 2019\/2144 and Directives 2014\/90\/EU, (EU) 2016\/797 and (EU) 2020\/1828 (Artificial Intelligence Act). (2024)."},{"key":"1506_CR4","unstructured":"The White House. Executive Order on the Safe, Secure, and Trustworthy Development and Use of Artificial Intelligence (2023)."},{"key":"1506_CR5","unstructured":"Office of the National Coordinator for Health Information Technology, Department of Health and Human Services. Health Data, Technology, and Interoperability: Certification Program Updates, Algorithm Transparency, and Information Sharing. 45 CFR \u00a7 170, 171 (2024)."},{"key":"1506_CR6","unstructured":"World Health Organization. Executive Summary: Ethics and Governance of Artificial Intelligence for Health (2021)."},{"key":"1506_CR7","doi-asserted-by":"publisher","first-page":"491","DOI":"10.1093\/jamia\/ocz192","volume":"27","author":"S Reddy","year":"2020","unstructured":"Reddy, S., Allan, S., Coghlan, S. & Cooper, P. A governance model for the application of AI in health care. J. Am. Med. Inform. Assoc. 27, 491\u2013497 (2020).","journal-title":"J. Am. Med. Inform. Assoc."},{"key":"1506_CR8","doi-asserted-by":"publisher","first-page":"2730","DOI":"10.1093\/jamia\/ocae209","volume":"31","author":"S Labkoff","year":"2024","unstructured":"Labkoff, S. et al. Toward a responsible future: recommendations for AI-enabled clinical decision support. J. Am. Med. Inform. Assoc. 31, 2730\u20132739 (2024).","journal-title":"J. Am. Med. Inform. Assoc."},{"key":"1506_CR9","first-page":"28","volume":"4","author":"F Li","year":"2023","unstructured":"Li, F., Ruijs, N. & Lu, Y. Ethics & AI: A systematic review on ethical concerns and related strategies for designing with AI in. Healthc. AI 4, 28\u201353 (2023).","journal-title":"Healthc. AI"},{"key":"1506_CR10","doi-asserted-by":"publisher","unstructured":"Garba-Sani, Z., Farinacci-Roberts, C., Essien, A. & Yracheta, J. M. A.C.C.E.S.S. AI: a new framework for advancing health equity in health care AI. Health Affairs https:\/\/doi.org\/10.1377\/forefront.20240424.369302 (2024).","DOI":"10.1377\/forefront.20240424.369302"},{"key":"1506_CR11","doi-asserted-by":"publisher","first-page":"300","DOI":"10.1353\/hpu.2021.0065","volume":"32","author":"I Dankwa-Mullan","year":"2021","unstructured":"Dankwa-Mullan, I. et al. A Proposed framework on integrating health equity and racial justice into the artificial intelligence development lifecycle. J. Health Care Poor Underserved 32, 300\u2013317 (2021).","journal-title":"J. Health Care Poor Underserved"},{"key":"1506_CR12","doi-asserted-by":"publisher","first-page":"3188","DOI":"10.1007\/s11606-021-06846-x","volume":"36","author":"CR Clark","year":"2021","unstructured":"Clark, C. R. et al. Health care equity in the use of advanced analytics and artificial intelligence technologies in primary care. J. Gen. Intern. Med. 36, 3188\u20133193 (2021).","journal-title":"J. Gen. Intern. Med."},{"key":"1506_CR13","doi-asserted-by":"publisher","first-page":"2147","DOI":"10.1093\/jamia\/ocae055","volume":"31","author":"JA Rodriguez","year":"2024","unstructured":"Rodriguez, J. A., Alsentzer, E. & Bates, D. W. Leveraging large language models to foster equity in healthcare. J. Am. Med. Inform. Assoc. 31, 2147\u20132150 (2024).","journal-title":"J. Am. Med. Inform. Assoc."},{"key":"1506_CR14","unstructured":"Ishizaki, K. AI model lifecycle management: overview. IBM https:\/\/www.ibm.com\/think\/topics\/ai-lifecycle (2020)."},{"key":"1506_CR15","doi-asserted-by":"publisher","first-page":"e549","DOI":"10.1016\/S2589-7500(20)30219-3","volume":"2","author":"S Cruz Rivera","year":"2020","unstructured":"Cruz Rivera, S., Liu, X., Chan, A.-W., Denniston, A. K. & Calvert, M. J. Guidelines for clinical trial protocols for interventions involving artificial intelligence: the SPIRIT-AI extension. Lancet Digit. Health 2, e549\u2013e560 (2020).","journal-title":"Lancet Digit. Health"},{"key":"1506_CR16","doi-asserted-by":"publisher","first-page":"1351","DOI":"10.1038\/s41591-020-1037-7","volume":"26","author":"S Cruz Rivera","year":"2020","unstructured":"Cruz Rivera, S., Liu, X., Chan, A.-W., Denniston, A. K. & Calvert, M. J. Guidelines for clinical trial protocols for interventions involving artificial intelligence: the SPIRIT-AI extension. Nat. Med. 26, 1351\u20131363 (2020).","journal-title":"Nat. Med."},{"key":"1506_CR17","doi-asserted-by":"publisher","first-page":"m3210","DOI":"10.1136\/bmj.m3210","volume":"370","author":"SC Rivera","year":"2020","unstructured":"Rivera, S. C., Liu, X., Chan, A.-W., Denniston, A. K. & Calvert, M. J. Guidelines for clinical trial protocols for interventions involving artificial intelligence: the SPIRIT-AI extension. BMJ 370, m3210 (2020).","journal-title":"BMJ"},{"key":"1506_CR18","doi-asserted-by":"publisher","first-page":"e537","DOI":"10.1016\/S2589-7500(20)30218-1","volume":"2","author":"X Liu","year":"2020","unstructured":"Liu, X., Cruz Rivera, S., Moher, D., Calvert, M. J. & Denniston, A. K. Reporting guidelines for clinical trial reports for interventions involving artificial intelligence: the CONSORT-AI extension. Lancet Digit Health 2, e537\u2013e548 (2020).","journal-title":"Lancet Digit Health"},{"key":"1506_CR19","doi-asserted-by":"publisher","first-page":"m3164","DOI":"10.1136\/bmj.m3164","volume":"370","author":"X Liu","year":"2020","unstructured":"Liu, X., Rivera, S. C., Moher, D., Calvert, M. J. & Denniston, A. K. Reporting guidelines for clinical trial reports for interventions involving artificial intelligence: the CONSORT-AI extension. BMJ 370, m3164 (2020).","journal-title":"BMJ"},{"key":"1506_CR20","doi-asserted-by":"publisher","first-page":"1364","DOI":"10.1038\/s41591-020-1034-x","volume":"26","author":"X Liu","year":"2020","unstructured":"Liu, X., Cruz Rivera, S., Moher, D., Calvert, M. J. & Denniston, A. K. Reporting guidelines for clinical trial reports for interventions involving artificial intelligence: the CONSORT-AI extension. Nat. Med 26, 1364\u20131374 (2020).","journal-title":"Nat. Med"},{"key":"1506_CR21","doi-asserted-by":"publisher","DOI":"10.1016\/j.artmed.2024.102861","volume":"151","author":"P Esmaeilzadeh","year":"2024","unstructured":"Esmaeilzadeh, P. Challenges and strategies for wide-scale artificial intelligence (AI) deployment in healthcare practices: a perspective for healthcare organizations. Artif. Intell. Med. 151, 102861 (2024).","journal-title":"Artif. Intell. Med."},{"key":"1506_CR22","doi-asserted-by":"publisher","first-page":"2000052","DOI":"10.1002\/aisy.202000052","volume":"2","author":"D Higgins","year":"2020","unstructured":"Higgins, D. & Madai, V. I. From bit to bedside: a practical framework for artificial intelligence product development in healthcare. Adv. Intell. Syst. 2, 2000052 (2020).","journal-title":"Adv. Intell. Syst."},{"key":"1506_CR23","unstructured":"Food and Drug Administration. Step 3: clinical research. FDA https:\/\/www.fda.gov\/patients\/drug-development-process\/step-3-clinical-research (2018)."},{"key":"1506_CR24","unstructured":"American Medical Informatics Association. AMIA 2024 artificial intelligence evaluation showcase. AMIA https:\/\/amia.org\/education-events\/amia-2024-artificial-intelligence-evaluation-showcase (2024)."},{"key":"1506_CR25","doi-asserted-by":"publisher","first-page":"744","DOI":"10.1016\/j.jbi.2013.06.009","volume":"46","author":"M Peleg","year":"2013","unstructured":"Peleg, M. Computer-interpretable clinical guidelines: a methodological review. J. Biomed. Inf. 46, 744\u2013763 (2013).","journal-title":"J. Biomed. Inf."},{"key":"1506_CR26","doi-asserted-by":"publisher","first-page":"83","DOI":"10.1016\/j.jbi.2017.02.008","volume":"68","author":"J Fox","year":"2017","unstructured":"Fox, J. Cognitive systems at the point of care: The CREDO program. J. Biomed. Inform. 68, 83\u201395 (2017).","journal-title":"J. Biomed. Inform."},{"key":"1506_CR27","unstructured":"Nestor, B. et al. Preparing a clinical support model for silent mode in general internal medicine. In Proceedings of the 5th Machine Learning for Healthcare Conference Vol. 126 (eds Doshi-Velez, F. et al.) 950\u2013972 (PMLR, 2020)."},{"key":"1506_CR28","doi-asserted-by":"publisher","DOI":"10.1056\/AIoa2400181","volume":"1","author":"O Unlu","year":"2024","unstructured":"Unlu, O. et al. Retrieval-augmented generation\u2013enabled GPT-4 for clinical trial screening. NEJM AI 1, AIoa2400181 (2024).","journal-title":"NEJM AI"},{"key":"1506_CR29","doi-asserted-by":"publisher","DOI":"10.1038\/s41746-023-00832-9","volume":"6","author":"AAH de Hond","year":"2023","unstructured":"de Hond, A. A. H. et al. Perspectives on validation of clinical predictive algorithms. npj Digit. Med. 6, 86 (2023).","journal-title":"npj Digit. Med."},{"key":"1506_CR30","doi-asserted-by":"publisher","first-page":"738","DOI":"10.1016\/j.annemergmed.2022.11.012","volume":"81","author":"FZ Dadabhoy","year":"2023","unstructured":"Dadabhoy, F. Z. et al. Prospective external validation of a commercial model predicting the likelihood of inpatient admission from the emergency department. Ann. Emerg. Med. 81, 738\u2013748 (2023).","journal-title":"Ann. Emerg. Med."},{"key":"1506_CR31","doi-asserted-by":"publisher","first-page":"70","DOI":"10.1016\/j.ajem.2023.01.054","volume":"67","author":"G Bunney","year":"2023","unstructured":"Bunney, G. et al. Beyond chest pain: Incremental value of other variables to identify patients for an early ECG. Am. J. Emerg. Med. 67, 70\u201378 (2023).","journal-title":"Am. J. Emerg. Med."},{"key":"1506_CR32","doi-asserted-by":"crossref","unstructured":"Callahan, A. et al. Standing on FURM ground: a framework for evaluating fair, useful, and reliable AI models in health care systems. NEJM Catal. 5, CAT.24.0131 (2024).","DOI":"10.1056\/CAT.24.0131"},{"key":"1506_CR33","doi-asserted-by":"publisher","DOI":"10.1038\/s41597-021-01110-7","volume":"9","author":"E R\u00f6\u00f6sli","year":"2022","unstructured":"R\u00f6\u00f6sli, E., Bozkurt, S. & Hernandez-Boussard, T. Peeking into a black box, the fairness and generalizability of a MIMIC-III benchmarking model. Sci. Data 9, 24 (2022).","journal-title":"Sci. Data"},{"key":"1506_CR34","unstructured":"Agency for Healthcare Research and Quality (US). Criteria for Distinguishing Effectiveness from Efficacy Trials in Systematic Reviews. https:\/\/www.ncbi.nlm.nih.gov\/books\/NBK44029\/ (2006)."},{"key":"1506_CR35","doi-asserted-by":"publisher","unstructured":"Achiam, O. J. et al. GPT-4 Technical Report. in. https:\/\/doi.org\/10.48550\/arXiv.2303.08774 (2023).","DOI":"10.48550\/arXiv.2303.08774"},{"key":"1506_CR36","doi-asserted-by":"publisher","first-page":"e243201","DOI":"10.1001\/jamanetworkopen.2024.3201","volume":"7","author":"P Garcia","year":"2024","unstructured":"Garcia, P. et al. Artificial intelligence\u2013generated draft replies to patient inbox messages. JAMA Netw. Open 7, e243201\u2013e243201 (2024).","journal-title":"JAMA Netw. Open"},{"key":"1506_CR37","doi-asserted-by":"publisher","first-page":"e2422399","DOI":"10.1001\/jamanetworkopen.2024.22399","volume":"7","author":"WR Small","year":"2024","unstructured":"Small, W. R. et al. Large language model\u2013based responses to patients\u2019 in-basket messages. JAMA Netw. Open 7, e2422399\u2013e2422399 (2024).","journal-title":"JAMA Netw. Open"},{"key":"1506_CR38","doi-asserted-by":"publisher","first-page":"103611","DOI":"10.1016\/j.jbi.2020.103611","volume":"112","author":"SE Davis","year":"2020","unstructured":"Davis, S. E., Greevy, R. A. J., Lasko, T. A., Walsh, C. G. & Matheny, M. E. Detection of calibration drift in clinical prediction models to inform model updating. J. Biomed. Inf. 112, 103611 (2020).","journal-title":"J. Biomed. Inf."},{"key":"1506_CR39","doi-asserted-by":"publisher","first-page":"37","DOI":"10.1093\/jamia\/ocy139","volume":"26","author":"S Aaron","year":"2019","unstructured":"Aaron, S., McEvoy, D. S., Ray, S., Hickman, T.-T. T. & Wright, A. Cranky comments: detecting clinical decision support malfunctions through free-text override reasons. J. Am. Med. Inform. Assoc. 26, 37\u201343 (2019).","journal-title":"J. Am. Med. Inform. Assoc."},{"key":"1506_CR40","doi-asserted-by":"publisher","first-page":"1050","DOI":"10.1093\/jamia\/ocac027","volume":"29","author":"AB McCoy","year":"2022","unstructured":"McCoy, A. B. et al. Clinician collaboration to improve clinical decision support: the Clickbusters initiative. J. Am. Med. Inform. Assoc. 29, 1050\u20131059 (2022).","journal-title":"J. Am. Med. Inform. Assoc."},{"key":"1506_CR41","unstructured":"National Association of Community Health Centers & American Academy of Family Physicians. Closing the Primary Care Gap: How Community Health Centers Can Address the Nation\u2019s Primary Care Crisis. https:\/\/www.nachc.org\/wp-content\/uploads\/2023\/06\/Closing-the-Primary-Care-Gap_Full-Report_2023_digital-final.pdf (2023)."},{"key":"1506_CR42","unstructured":"Your Clinical Decision Support Software: Is It a Medical Device? US Food and Drug Administration https:\/\/www.fda.gov\/medical-devices\/software-medical-device-samd\/your-clinical-decision-support-software-it-medical-device (2022)."},{"key":"1506_CR43","unstructured":"The Device Development Process. US Food and Drug Administration https:\/\/www.fda.gov\/patients\/learn-about-drug-and-device-approvals\/device-development-process (2018)."},{"key":"1506_CR44","doi-asserted-by":"publisher","first-page":"AIp2400223","DOI":"10.1056\/AIp2400223","volume":"1","author":"CA Longhurst","year":"2024","unstructured":"Longhurst, C. A., Singh, K., Chopra, A., Atreja, A. & Brownstein, J. S. A call for artificial intelligence implementation science centers to evaluate clinical effectiveness. NEJM AI 1, AIp2400223 (2024).","journal-title":"NEJM AI"},{"key":"1506_CR45","doi-asserted-by":"publisher","first-page":"e2345050","DOI":"10.1001\/jamanetworkopen.2023.45050","volume":"6","author":"MH Chin","year":"2023","unstructured":"Chin, M. H. et al. Guiding principles to address the impact of algorithm bias on racial and ethnic disparities in health and health care. JAMA Netw. Open 6, e2345050\u2013e2345050 (2023).","journal-title":"JAMA Netw. Open"},{"key":"1506_CR46","doi-asserted-by":"publisher","DOI":"10.1038\/s41746-023-00873-0","volume":"6","author":"B Mesk\u00f3","year":"2023","unstructured":"Mesk\u00f3, B. & Topol, E. J. The imperative for regulatory oversight of large language models (or generative AI) in healthcare. npj Digit. Med. 6, 120 (2023).","journal-title":"npj Digit. Med."},{"key":"1506_CR47","unstructured":"MIT Work of the Future. About Us. MIT Work of the Future https:\/\/workofthefuture-taskforce.mit.edu\/mission\/ (2024)."},{"key":"1506_CR48","unstructured":"Coalition for Health AI, Inc. Our Purpose. CHAI https:\/\/chai.org\/our-purpose\/ (2024)."},{"key":"1506_CR49","unstructured":"Stanford Medicine. Responsible AI for Safe and Equitable Health. RAISE Health https:\/\/med.stanford.edu\/raisehealth (2024)."},{"key":"1506_CR50","unstructured":"Stanford University. Stanford University Human-Centered Artificial Intelligence. https:\/\/hai.stanford.edu\/ (2024)."}],"container-title":["npj Digital Medicine"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/www.nature.com\/articles\/s41746-025-01506-4.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/www.nature.com\/articles\/s41746-025-01506-4","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/www.nature.com\/articles\/s41746-025-01506-4.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,2,18]],"date-time":"2025-02-18T02:11:55Z","timestamp":1739844715000},"score":1,"resource":{"primary":{"URL":"https:\/\/www.nature.com\/articles\/s41746-025-01506-4"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,2,17]]},"references-count":50,"journal-issue":{"issue":"1","published-online":{"date-parts":[[2025,12]]}},"alternative-id":["1506"],"URL":"https:\/\/doi.org\/10.1038\/s41746-025-01506-4","relation":{},"ISSN":["2398-6352"],"issn-type":[{"value":"2398-6352","type":"electronic"}],"subject":[],"published":{"date-parts":[[2025,2,17]]},"assertion":[{"value":"28 September 2024","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"9 February 2025","order":2,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"17 February 2025","order":3,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"A.L. is a consultant for the Abbott Medical Device Cybersecurity Council. The other authors have no competing financial or non-financial interests.","order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Competing interests"}}],"article-number":"107"}}