{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,3]],"date-time":"2026-07-03T03:06:43Z","timestamp":1783048003686,"version":"3.54.6"},"reference-count":26,"publisher":"Oxford University Press (OUP)","issue":"5","license":[{"start":{"date-parts":[[2025,3,21]],"date-time":"2025-03-21T00:00:00Z","timestamp":1742515200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/academic.oup.com\/pages\/standard-publication-reuse-rights"}],"funder":[{"DOI":"10.13039\/100000133","name":"Agency for Healthcare Research and Quality","doi-asserted-by":"publisher","award":["#: 1R03HS030307-01"],"award-info":[{"award-number":["#: 1R03HS030307-01"]}],"id":[{"id":"10.13039\/100000133","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2025,5,1]]},"abstract":"<jats:title>Abstract<\/jats:title>\n               <jats:sec>\n                  <jats:title>Objectives<\/jats:title>\n                  <jats:p>The objective of this work is to demonstrate the value of simulation testing for rapidly evaluating artificial intelligence (AI) products.<\/jats:p>\n               <\/jats:sec>\n               <jats:sec>\n                  <jats:title>Materials and Methods<\/jats:title>\n                  <jats:p>Researcher-physician teams simulated the use of 2 Ambient Digital Scribe (ADS) products by reading scripts of outpatient encounters while using both products, yielding a total of 44 draft notes. Time to edit, perceived amount of effort and editing, and errors in the AI-generated draft notes were analyzed.<\/jats:p>\n               <\/jats:sec>\n               <jats:sec>\n                  <jats:title>Results<\/jats:title>\n                  <jats:p>Ambient Digital Scribe Product A draft notes took significantly longer to edit, had fewer omissions, and more additions and irrelevant or misplaced text errors than ADS Product B. Ambient Digital Scribe Product A was rated as performing better for most encounters.<\/jats:p>\n               <\/jats:sec>\n               <jats:sec>\n                  <jats:title>Discussion<\/jats:title>\n                  <jats:p>Artificial intelligence-enabled products are being rapidly developed and implemented into practice, outpacing safety concerns. Simulation testing can efficiently identify safety issues.<\/jats:p>\n               <\/jats:sec>\n               <jats:sec>\n                  <jats:title>Conclusion<\/jats:title>\n                  <jats:p>Simulation testing is a crucial first step to take when evaluating AI-enabled technologies.<\/jats:p>\n               <\/jats:sec>","DOI":"10.1093\/jamia\/ocaf052","type":"journal-article","created":{"date-parts":[[2025,3,23]],"date-time":"2025-03-23T08:18:21Z","timestamp":1742717901000},"page":"928-931","source":"Crossref","is-referenced-by-count":7,"title":["The value of simulation testing for the evaluation of ambient digital scribes: a case report"],"prefix":"10.1093","volume":"32","author":[{"given":"Joshua M","family":"Biro","sequence":"first","affiliation":[{"name":"MedStar Health National Center for Human Factors in Healthcare , Washington, DC 20008,","place":["United States"]}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Jessica L","family":"Handley","sequence":"additional","affiliation":[{"name":"MedStar Health National Center for Human Factors in Healthcare , Washington, DC 20008,","place":["United States"]}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"James","family":"Mickler","sequence":"additional","affiliation":[{"name":"Georgetown University Medical Center , Washington, DC 20007,","place":["United States"]}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Sahithi","family":"Reddy","sequence":"additional","affiliation":[{"name":"Georgetown University Medical Center , Washington, DC 20007,","place":["United States"]}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Varsha","family":"Kottamasu","sequence":"additional","affiliation":[{"name":"MedStar Health Institute for Innovation , Washington, DC 20008,","place":["United States"]}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Raj M","family":"Ratwani","sequence":"additional","affiliation":[{"name":"MedStar Health National Center for Human Factors in Healthcare , Washington, DC 20008,","place":["United States"]},{"name":"Georgetown University School of Medicine , Washington, DC 20007,","place":["United States"]}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Nathan K","family":"Cobb","sequence":"additional","affiliation":[{"name":"Georgetown University Medical Center , Washington, DC 20007,","place":["United States"]},{"name":"MedStar Health Institute for Innovation , Washington, DC 20008,","place":["United States"]}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"286","published-online":{"date-parts":[[2025,3,21]]},"reference":[{"key":"2025042203524730300_ocaf052-B1","doi-asserted-by":"publisher","first-page":"689","DOI":"10.1186\/s12909-023-04698-z","article-title":"Revolutionizing healthcare: the role of artificial intelligence in clinical practice","volume":"23","author":"Alowais","year":"2023","journal-title":"BMC Med Educ"},{"key":"2025042203524730300_ocaf052-B2","doi-asserted-by":"publisher","first-page":"1379","DOI":"10.1007\/s11606-019-04961-4","article-title":"Using predictive analytics to guide patient care and research in a national health system","volume":"34","author":"Nelson","year":"2019","journal-title":"J Gen Intern Med"},{"key":"2025042203524730300_ocaf052-B3","first-page":"1j","article-title":"Predictive model based on health data analysis for risk of readmission in disease-specific cohorts","volume":"18","author":"Ansari","year":"2021","journal-title":"Perspect Health Inf Manag"},{"key":"2025042203524730300_ocaf052-B4","doi-asserted-by":"publisher","first-page":"86","DOI":"10.1111\/cts.12884","article-title":"Precision medicine, AI, and the future of personalized health care","volume":"14","author":"Johnson","year":"2021","journal-title":"Clin Transl Sci"},{"key":"2025042203524730300_ocaf052-B5","doi-asserted-by":"publisher","first-page":"472","DOI":"10.1186\/s12967-020-02658-5","article-title":"Precision medicine in the era of artificial intelligence: implications in chronic disease management","volume":"18","author":"Subramanian","year":"2020","journal-title":"J Transl Med"},{"key":"2025042203524730300_ocaf052-B6","doi-asserted-by":"publisher","first-page":"89","DOI":"10.1038\/s41586-019-1799-6","article-title":"International evaluation of an AI system for breast cancer screening","volume":"577","author":"McKinney","year":"2020","journal-title":"Nature"},{"key":"2025042203524730300_ocaf052-B7","doi-asserted-by":"publisher","first-page":"e138","DOI":"10.1016\/S2589-7500(20)30003-0","article-title":"Changes in cancer detection and false-positive recall in mammography using artificial intelligence: a retrospective, multireader study","volume":"2","author":"Kim","year":"2020","journal-title":"Lancet Digit Health"},{"key":"2025042203524730300_ocaf052-B8","volume-title":"Clinical Documentation Strategies 2023 - Examining Which Options Best Fit Your Needs","author":"Hickenlooper","year":"2023"},{"key":"2025042203524730300_ocaf052-B9","author":"Elation Health Team","year":"2023"},{"key":"2025042203524730300_ocaf052-B10","author":"Nuance Communications","year":"2024"},{"key":"2025042203524730300_ocaf052-B11","doi-asserted-by":"publisher","first-page":"100860","DOI":"10.1016\/j.xcrm.2022.100860","article-title":"Evidence synthesis, digital scribes, and translational challenges for artificial intelligence in healthcare","volume":"3","author":"Coiera","year":"2022","journal-title":"Cell Rep Med"},{"key":"2025042203524730300_ocaf052-B12","doi-asserted-by":"publisher","first-page":"1296","DOI":"10.1038\/s41591-023-02341-4","article-title":"ChatGPT is not the solution to physicians\u2019 documentation burden","volume":"29","author":"Preiksaitis","year":"2023","journal-title":"Nat Med"},{"key":"2025042203524730300_ocaf052-B13","author":"Biden","year":"2023"},{"key":"2025042203524730300_ocaf052-B14","doi-asserted-by":"publisher","first-page":"447","DOI":"10.1126\/science.aax2342","article-title":"Dissecting racial bias in an algorithm used to manage the health of populations","volume":"366","author":"Obermeyer","year":"2019","journal-title":"Science"},{"key":"2025042203524730300_ocaf052-B15","doi-asserted-by":"publisher","first-page":"1065","DOI":"10.1001\/jamainternmed.2021.2626","article-title":"External validation of a widely implemented proprietary sepsis prediction model in hospitalized patients","volume":"181","author":"Wong","year":"2021","journal-title":"JAMA Intern Med"},{"key":"2025042203524730300_ocaf052-B16","doi-asserted-by":"publisher","first-page":"1669","DOI":"10.1080\/00140139.2013.838643","article-title":"SEIPS 2.0: a human factors framework for studying and improving the work of healthcare professionals and patients","volume":"56","author":"Holden","year":"2013","journal-title":"Ergonomics"},{"key":"2025042203524730300_ocaf052-B17","volume-title":"Black-Box Testing: Techniques for Functional Testing of Software and Systems","author":"Beizer","year":"1995"},{"key":"2025042203524730300_ocaf052-B18","doi-asserted-by":"publisher","first-page":"255","DOI":"10.1016\/j.jcrc.2014.10.007","article-title":"The effectiveness of an intensive care quick reference checklist manual\u2014A randomized simulation-based trial","volume":"30","author":"Just","year":"2015","journal-title":"J Crit Care"},{"key":"2025042203524730300_ocaf052-B19","doi-asserted-by":"publisher","first-page":"12","DOI":"10.1016\/j.jcjq.2021.09.013","article-title":"A simulation systems testing program using HFMEA methodology can effectively identify and mitigate latent safety threats for a new on-site helipad","volume":"48","author":"Holmes","year":"2022","journal-title":"Jt Comm J Qual Patient Saf"},{"key":"2025042203524730300_ocaf052-B20","doi-asserted-by":"publisher","first-page":"103","DOI":"10.1097\/SIH.0b013e3181c82c0a","article-title":"Development of a simulation environment to study emergency department information technology","volume":"5","author":"Pennathur","year":"2010","journal-title":"Simul Healthc"},{"key":"2025042203524730300_ocaf052-B21","doi-asserted-by":"publisher","first-page":"329","DOI":"10.1136\/bmjqs-2019-009857","article-title":"Application of human factors to improve usability of clinical decision support for diagnostic decision-making: a scenario-based simulation study","volume":"29","author":"Carayon","year":"2020","journal-title":"BMJ Qual Saf"},{"key":"2025042203524730300_ocaf052-B22","doi-asserted-by":"publisher","first-page":"e287","DOI":"10.1136\/amiajnl-2013-002045","article-title":"Applying human factors principles to alert design increases efficiency and reduces prescribing errors in a scenario-based simulation","volume":"21","author":"Russ","year":"2014","journal-title":"J Am Med Inform Assoc"},{"key":"2025042203524730300_ocaf052-B23","doi-asserted-by":"publisher","first-page":"626","DOI":"10.1177\/0018720811426427","article-title":"Improving medication management through the redesign of the hospital code cart medication drawer","volume":"53","author":"Rousek","year":"2011","journal-title":"Hum Factors"},{"key":"2025042203524730300_ocaf052-B24","doi-asserted-by":"publisher","first-page":"252","DOI":"10.1037\/0033-295X.87.3.252","article-title":"Recognizing: the judgment of previous occurrence","volume":"87","author":"Mandler","year":"1980","journal-title":"Psychol Rev"},{"key":"2025042203524730300_ocaf052-B25","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1207\/s15327108ijap0301_1","article-title":"Performance consequences of automation-induced complacency","volume":"3","author":"Parasuraman","year":"1993","journal-title":"Int J Aviat Psychol"},{"key":"2025042203524730300_ocaf052-B26","doi-asserted-by":"publisher","first-page":"697","DOI":"10.1016\/S1071-5819(03)00038-7","article-title":"The role of trust in automation reliance","volume":"58","author":"Dzindolet","year":"2003","journal-title":"Int J Hum Comput Stud"}],"container-title":["Journal of the American Medical Informatics Association"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/academic.oup.com\/jamia\/article-pdf\/32\/5\/928\/62520461\/ocaf052.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"syndication"},{"URL":"https:\/\/academic.oup.com\/jamia\/article-pdf\/32\/5\/928\/62520461\/ocaf052.pdf","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,4,22]],"date-time":"2025-04-22T07:52:57Z","timestamp":1745308377000},"score":1,"resource":{"primary":{"URL":"https:\/\/academic.oup.com\/jamia\/article\/32\/5\/928\/8090036"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,3,21]]},"references-count":26,"journal-issue":{"issue":"5","published-online":{"date-parts":[[2025,3,21]]},"published-print":{"date-parts":[[2025,5,1]]}},"URL":"https:\/\/doi.org\/10.1093\/jamia\/ocaf052","relation":{},"ISSN":["1067-5027","1527-974X"],"issn-type":[{"value":"1067-5027","type":"print"},{"value":"1527-974X","type":"electronic"}],"subject":[],"published-other":{"date-parts":[[2025,5]]},"published":{"date-parts":[[2025,3,21]]}}}