{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,12]],"date-time":"2026-04-12T23:13:35Z","timestamp":1776035615923,"version":"3.50.1"},"reference-count":78,"publisher":"Oxford University Press (OUP)","issue":"1","license":[{"start":{"date-parts":[[2025,1,10]],"date-time":"2025-01-10T00:00:00Z","timestamp":1736467200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"funder":[{"DOI":"10.13039\/501100001381","name":"National Research Foundation, Singapore","doi-asserted-by":"publisher","award":["AISG-GC-2019-001-2B"],"award-info":[{"award-number":["AISG-GC-2019-001-2B"]}],"id":[{"id":"10.13039\/501100001381","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100001381","name":"National Research Foundation, Singapore","doi-asserted-by":"publisher","award":["AISG2-TC-2022-004"],"award-info":[{"award-number":["AISG2-TC-2022-004"]}],"id":[{"id":"10.13039\/501100001381","id-type":"DOI","asserted-by":"publisher"}]},{"name":"RIE2025 Industry Alignment Fund"},{"name":"Cisco-NUS Accelerated Digital Economy Corporate Laboratory","award":["I2101E0002"],"award-info":[{"award-number":["I2101E0002"]}]},{"DOI":"10.13039\/100022902","name":"Saw Swee Hock School of Public Health","doi-asserted-by":"publisher","award":["24-0180-A0001-0"],"award-info":[{"award-number":["24-0180-A0001-0"]}],"id":[{"id":"10.13039\/100022902","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2026,1,1]]},"abstract":"<jats:title>Abstract<\/jats:title>\n                  <jats:sec>\n                    <jats:title>Objectives<\/jats:title>\n                    <jats:p>This study introduces Smart Imitator (SI), a 2-phase reinforcement learning (RL) solution enhancing personalized treatment policies in healthcare, addressing challenges from imperfect clinician data and complex environments.<\/jats:p>\n                  <\/jats:sec>\n                  <jats:sec>\n                    <jats:title>Materials and Methods<\/jats:title>\n                    <jats:p>Smart Imitator\u2019s first phase uses adversarial cooperative imitation learning with a novel sample selection schema to categorize clinician policies from optimal to nonoptimal. The second phase creates a parameterized reward function to guide the learning of superior treatment policies through RL. Smart Imitator\u2019s effectiveness was validated on 2 datasets: a sepsis dataset with 19\u00a0711 patient trajectories and a diabetes dataset with 7234 trajectories.<\/jats:p>\n                  <\/jats:sec>\n                  <jats:sec>\n                    <jats:title>Results<\/jats:title>\n                    <jats:p>Extensive quantitative and qualitative experiments showed that SI significantly outperformed state-of-the-art baselines in both datasets. For sepsis, SI reduced estimated mortality rates by 19.6% compared to the best baseline. For diabetes, SI reduced HbA1c-High rates by 12.2%. The learned policies aligned closely with successful clinical decisions and deviated strategically when necessary. These deviations aligned with recent clinical findings, suggesting improved outcomes.<\/jats:p>\n                  <\/jats:sec>\n                  <jats:sec>\n                    <jats:title>Discussion<\/jats:title>\n                    <jats:p>Smart Imitator advances RL applications by addressing challenges such as imperfect data and environmental complexities, demonstrating effectiveness within the tested conditions of sepsis and diabetes. Further validation across diverse conditions and exploration of additional RL algorithms are needed to enhance precision and generalizability.<\/jats:p>\n                  <\/jats:sec>\n                  <jats:sec>\n                    <jats:title>Conclusion<\/jats:title>\n                    <jats:p>This study shows potential in advancing personalized healthcare learning from clinician behaviors to improve treatment outcomes. Its methodology offers a robust approach for adaptive, personalized strategies in various complex and uncertain environments.<\/jats:p>\n                  <\/jats:sec>","DOI":"10.1093\/jamia\/ocae320","type":"journal-article","created":{"date-parts":[[2024,12,23]],"date-time":"2024-12-23T15:32:12Z","timestamp":1734967932000},"page":"49-66","source":"Crossref","is-referenced-by-count":3,"title":["Smart Imitator: Learning from Imperfect Clinical Decisions"],"prefix":"10.1093","volume":"33","author":[{"given":"Dilruk","family":"Perera","sequence":"first","affiliation":[{"name":"Institute of Data Science, National University of Singapore , 117602,","place":["Singapore"]},{"name":"Saw Swee Hock School of Public Health, National University of Singapore , 117549,","place":["Singapore"]}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Siqi","family":"Liu","sequence":"additional","affiliation":[{"name":"Institute of Data Science, National University of Singapore , 117602,","place":["Singapore"]},{"name":"NUS Graduate School\u2014ISEP , 119077,","place":["Singapore"]}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Kay Choong","family":"See","sequence":"additional","affiliation":[{"name":"Yong Loo Lin School of Medicine, National University of Singapore , 117597,","place":["Singapore"]}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Mengling","family":"Feng","sequence":"additional","affiliation":[{"name":"Institute of Data Science, National University of Singapore , 117602,","place":["Singapore"]},{"name":"Saw Swee Hock School of Public Health, National University of Singapore , 117549,","place":["Singapore"]}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"286","published-online":{"date-parts":[[2025,1,10]]},"reference":[{"key":"2026010211542871300_ocae320-B1","author":"Raghu","year":"2017"},{"key":"2026010211542871300_ocae320-B2"},{"key":"2026010211542871300_ocae320-B3","author":"Gottesman"},{"key":"2026010211542871300_ocae320-B4","first-page":"2447","author":"Wang","year":"2018"},{"key":"2026010211542871300_ocae320-B5","first-page":"1","author":"Perera","year":"2023"},{"key":"2026010211542871300_ocae320-B6","doi-asserted-by":"crossref","first-page":"e18477","DOI":"10.2196\/18477","article-title":"Reinforcement learning for clinical decision support in critical care: a comprehensive review","volume":"22","author":"Liu","year":"2020","journal-title":"J Med Internet Res"},{"key":"2026010211542871300_ocae320-B7","author":"Riachi"},{"key":"2026010211542871300_ocae320-B8","author":"Mnih","year":"2013"},{"key":"2026010211542871300_ocae320-B9","doi-asserted-by":"crossref","first-page":"529","DOI":"10.1038\/nature14236","article-title":"Human- level control through deep reinforcement learning","volume":"518","author":"Mnih","year":"2015","journal-title":"Nature"},{"key":"2026010211542871300_ocae320-B10","doi-asserted-by":"crossref","first-page":"1455","DOI":"10.1002\/sim.2022","article-title":"An experimental design for the development of adaptive treatment strategies","volume":"24","author":"Murphy","year":"2005","journal-title":"Stat Med"},{"key":"2026010211542871300_ocae320-B11","doi-asserted-by":"publisher","first-page":"4055","DOI":"10.1002\/sim.10161","article-title":"A sequential, multiple assignment, randomized trial design with a tailoring function","volume":"43","author":"Hartman","year":"2024","journal-title":"Stat Med"},{"key":"2026010211542871300_ocae320-B12","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1016\/j.jet.2004.03.008","article-title":"On the convergence of reinforcement learning","volume":"122","author":"Beggs","year":"2005","journal-title":"J Econ Theory"},{"key":"2026010211542871300_ocae320-B13","doi-asserted-by":"crossref","first-page":"426","DOI":"10.1513\/AnnalsATS.201809-646PS","article-title":"Improving sepsis treatment by embracing diagnostic uncertainty","volume":"16","author":"Prescott","year":"2019","journal-title":"Ann Am Thorac Soc"},{"key":"2026010211542871300_ocae320-B14","doi-asserted-by":"crossref","first-page":"ii21","DOI":"10.1136\/bmjqs-2012-001615","article-title":"The incidence of diagnostic error in medicine","volume":"22","author":"Graber","year":"2013","journal-title":"BMJ Qual Saf"},{"key":"2026010211542871300_ocae320-B15","doi-asserted-by":"crossref","first-page":"41","DOI":"10.1093\/biomet\/70.1.41","article-title":"The central role of the propensity score in observational studies for causal effects","volume":"70","author":"Rosenbaum","year":"1983","journal-title":"Biometrika"},{"key":"2026010211542871300_ocae320-B16","volume-title":"Clinical Practice Guidelines We Can Trust","author":"Institute of Medicine (US) Committee on Standards for Developing Trustworthy Clinical Practice Guidelines","year":"2011"},{"key":"2026010211542871300_ocae320-B17","doi-asserted-by":"crossref","first-page":"550","DOI":"10.1097\/00001648-200009000-00011","article-title":"Marginal structural models and causal inference in epidemiology","volume":"11","author":"Robins","year":"2000","journal-title":"Epidemiology"},{"key":"2026010211542871300_ocae320-B18","doi-asserted-by":"crossref","first-page":"1149","DOI":"10.1093\/aje\/kwj149","article-title":"Variable selection for propensity score models","volume":"163","author":"Brookhart","year":"2006","journal-title":"Am J Epidemiol"},{"key":"2026010211542871300_ocae320-B19","doi-asserted-by":"crossref","first-page":"1458","DOI":"10.1001\/jama.282.15.1458","article-title":"Why don\u2019t physicians follow clinical practice guidelines? A framework for improvement","volume":"282","author":"Cabana","year":"1999","journal-title":"JAMA"},{"key":"2026010211542871300_ocae320-B20","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1145\/3477600","article-title":"Reinforcement learning in healthcare: a survey","volume":"55","author":"Yu","year":"2023","journal-title":"ACM Comput Surv"},{"key":"2026010211542871300_ocae320-B21","doi-asserted-by":"crossref","first-page":"16","DOI":"10.1038\/s41591-018-0310-5","article-title":"Guidelines for reinforcement learning in healthcare","volume":"25","author":"Gottesman","year":"2019","journal-title":"Nat Med"},{"key":"2026010211542871300_ocae320-B22","first-page":"38","author":"Zhang","year":"2022:"},{"key":"2026010211542871300_ocae320-B23","first-page":"663","author":"Ng","year":"2000"},{"key":"2026010211542871300_ocae320-B24","first-page":"1785","author":"Wang","year":"2020"},{"key":"2026010211542871300_ocae320-B25","doi-asserted-by":"crossref","first-page":"S2","DOI":"10.1016\/j.amjmed.2008.01.001","article-title":"Overconfidence as a cause of diagnostic error in medicine","volume":"121","author":"Berner","year":"2008","journal-title":"Am J Med"},{"key":"2026010211542871300_ocae320-B26","doi-asserted-by":"crossref","first-page":"768","DOI":"10.1136\/bmj.320.7237.768","article-title":"Human error: models and management","volume":"320","author":"Reason","year":"2000","journal-title":"BMJ"},{"key":"2026010211542871300_ocae320-B27","doi-asserted-by":"crossref","first-page":"1124","DOI":"10.1126\/science.185.4157.1124","article-title":"Judgment under uncertainty: heuristics and biases","volume":"185","author":"Tversky","year":"1974","journal-title":"Science"},{"key":"2026010211542871300_ocae320-B28","doi-asserted-by":"crossref","first-page":"568","DOI":"10.1177\/0278364920903104","article-title":"Human motion analysis in medical robotics via high-dimensional inverse reinforcement learning","volume":"39","author":"Li","year":"2020","journal-title":"Int J Rob Res"},{"key":"2026010211542871300_ocae320-B29","author":"Shahryari","year":"2017"},{"key":"2026010211542871300_ocae320-B30","author":"Zheng","year":"2014"},{"key":"2026010211542871300_ocae320-B31","author":"Zhou","year":"2022"},{"key":"2026010211542871300_ocae320-B32","doi-asserted-by":"crossref","first-page":"331","DOI":"10.1111\/1467-9868.00389","article-title":"Optimal dynamic treatment regimes","volume":"65","author":"Murphy","year":"2003","journal-title":"J R Stat Soc Series B Stat Methodol"},{"key":"2026010211542871300_ocae320-B33","first-page":"189","author":"Robins","year":"2004"},{"key":"2026010211542871300_ocae320-B34","doi-asserted-by":"crossref","DOI":"10.1201\/9780429192692","volume-title":"Dynamic Treatment Regimes: Statistical Methods for Precision Medicine","author":"Tsiatis","year":"2019"},{"key":"2026010211542871300_ocae320-B35","doi-asserted-by":"crossref","first-page":"447","DOI":"10.1111\/j.1541-0420.2006.00686.x","article-title":"Estimation of optimal dynamic antipsychotic treatment regimes using iterations of Q-learning","volume":"63","author":"Moodie","year":"2007","journal-title":"Biometrics"},{"key":"2026010211542871300_ocae320-B36","doi-asserted-by":"crossref","first-page":"478","DOI":"10.1037\/a0029373","article-title":"Q-learning: a data analysis method for constructing adaptive interventions","volume":"17","author":"Nahum-Shani","year":"2012","journal-title":"Psychol Methods"},{"key":"2026010211542871300_ocae320-B37","doi-asserted-by":"crossref","first-page":"501","DOI":"10.1093\/biomet\/asv028","article-title":"Tree-based methods for individualized treatment regimes","volume":"102","author":"Laber","year":"2015","journal-title":"Biometrika"},{"key":"2026010211542871300_ocae320-B38","author":"Wang"},{"key":"2026010211542871300_ocae320-B39","doi-asserted-by":"crossref","first-page":"1641","DOI":"10.1038\/s41591-018-0253-x","article-title":"Individualized sepsis treatment using reinforcement learning","volume":"24","author":"Saria","year":"2018","journal-title":"Nat Med"},{"key":"2026010211542871300_ocae320-B40","doi-asserted-by":"crossref","first-page":"1716","DOI":"10.1038\/s41591-018-0213-5","article-title":"The artificial intelligence clinician learns optimal treatment strategies for sepsis in intensive care","volume":"24","author":"Komorowski","year":"2018","journal-title":"Nat Med"},{"key":"2026010211542871300_ocae320-B41","first-page":"147","author":"Raghu","year":"2017"},{"key":"2026010211542871300_ocae320-B42","author":"Raghu","year":"2019"},{"key":"2026010211542871300_ocae320-B43","first-page":"313","author":"Patil","year":"2014"},{"key":"2026010211542871300_ocae320-B44","author":"Torabi"},{"key":"2026010211542871300_ocae320-B45","doi-asserted-by":"crossref","first-page":"219","DOI":"10.1109\/THMS.2019.2895753","article-title":"Modeling interaction structure for robot imitation learning of human social behavior","volume":"49","author":"Doering","year":"2019","journal-title":"IEEE Trans Human-Mach Syst"},{"key":"2026010211542871300_ocae320-B46","first-page":"756","author":"Shon","year":"2007"},{"key":"2026010211542871300_ocae320-B47","first-page":"251","author":"Hawke","year":"2020"},{"key":"2026010211542871300_ocae320-B48","first-page":"1","author":"Harmer","year":"2018"},{"key":"2026010211542871300_ocae320-B49","doi-asserted-by":"crossref","first-page":"537","DOI":"10.1098\/rstb.2002.1258","article-title":"Computational approaches to motor learning by imitation","volume":"358","author":"Schaal","year":"2003","journal-title":"Philos Trans R Soc Lond B Biol Sci"},{"key":"2026010211542871300_ocae320-B50","first-page":"1","author":"Abbeel","year":"2004"},{"key":"2026010211542871300_ocae320-B51","doi-asserted-by":"crossref","first-page":"469","DOI":"10.1016\/j.robot.2008.10.024","article-title":"A survey of robot learning from demonstration","volume":"57","author":"Argall","year":"2009","journal-title":"Rob Auton Syst"},{"key":"2026010211542871300_ocae320-B52","first-page":"4572","author":"Ho","year":"2016"},{"key":"2026010211542871300_ocae320-B53","first-page":"2672","author":"Goodfellow"},{"key":"2026010211542871300_ocae320-B54","first-page":"2469","author":"Kang","year":"2018"},{"key":"2026010211542871300_ocae320-B55","first-page":"6818","author":"Wu","year":"2019"},{"key":"2026010211542871300_ocae320-B56","first-page":"78148","author":"Shah","year":"2022"},{"key":"2026010211542871300_ocae320-B57","first-page":"4950","author":"Torabi","year":"2018"},{"key":"2026010211542871300_ocae320-B58","first-page":"627","author":"Ross","year":"2011"},{"key":"2026010211542871300_ocae320-B59","first-page":"24725","author":"Xu","year":"2022"},{"key":"2026010211542871300_ocae320-B60","first-page":"279","author":"Watkins","year":"1992"},{"key":"2026010211542871300_ocae320-B61","doi-asserted-by":"crossref","first-page":"1670","DOI":"10.1097\/CCM.0b013e31819fcf68","article-title":"Serum lactate is associated with mortality in severe sepsis independent of organ failure and shock","volume":"37","author":"Mikkelsen","year":"2009","journal-title":"Crit Care Med"},{"key":"2026010211542871300_ocae320-B62","doi-asserted-by":"crossref","first-page":"739","DOI":"10.1016\/j.jelectrocard.2017.08.013","article-title":"Early sepsis detection in critical care patients using multiscale blood pressure and heart rate dynamics","volume":"50","author":"Shashikumar","year":"2017","journal-title":"J Electrocardiol"},{"key":"2026010211542871300_ocae320-B63","doi-asserted-by":"crossref","first-page":"95","DOI":"10.4137\/BMI.S38440","article-title":"Significance of HbA1c test in diagnosis and prognosis of diabetic patients","volume":"11","author":"Sherwani","year":"2016","journal-title":"Biomark Insights"},{"key":"2026010211542871300_ocae320-B64","doi-asserted-by":"crossref","first-page":"1350","DOI":"10.1111\/j.1464-5491.2012.03762.x","article-title":"Use of HbA1c in the diagnosis of diabetes mellitus in the UK. The implementation of World Health Organization guidance 2011","volume":"29","author":"John","year":"2012","journal-title":"Diabet Med"},{"key":"2026010211542871300_ocae320-B65","doi-asserted-by":"crossref","first-page":"90","DOI":"10.1001\/jama.2014.5804","article-title":"Hospital deaths in patients with sepsis from 2 independent cohorts","volume":"312","author":"Liu","year":"2014","journal-title":"JAMA"},{"key":"2026010211542871300_ocae320-B66","first-page":"i1585","article-title":"Sepsis: pathophysiology and clinical management","volume":"353","author":"Gotts","year":"2016","journal-title":"BMJ (Clinical Research ed.)"},{"key":"2026010211542871300_ocae320-B67","doi-asserted-by":"crossref","first-page":"561","DOI":"10.1111\/aas.12479","article-title":"The demise of early goal-directed therapy for severe sepsis and septic shock","volume":"59","author":"Marik","year":"2015","journal-title":"Acta Anaesthesiol Scand"},{"key":"2026010211542871300_ocae320-B68","first-page":"1","volume-title":"Global Report on Diabetes","author":"World Health Organization","year":"2016"},{"key":"2026010211542871300_ocae320-B69","doi-asserted-by":"crossref","first-page":"801","DOI":"10.1001\/jama.2016.0287","article-title":"The third international consensus definitions for sepsis and septic shock (Sepsis- 3)","volume":"315","author":"Singer","year":"2016","journal-title":"JAMA"},{"key":"2026010211542871300_ocae320-B70","first-page":"1","author":"Liu","year":"2020"},{"key":"2026010211542871300_ocae320-B71","author":"Cheng","year":"2019:320-331"},{"key":"2026010211542871300_ocae320-B72","author":"Prasad"},{"key":"2026010211542871300_ocae320-B73","author":"Thomas","year":"2015"},{"key":"2026010211542871300_ocae320-B74","author":"Jiang","year":"2016:652-661"},{"key":"2026010211542871300_ocae320-B75","author":"Dud\u00edk","year":"1104"},{"key":"2026010211542871300_ocae320-B76","author":"M\u00fcllner","year":"2004"},{"key":"2026010211542871300_ocae320-B77","first-page":"101836","author":"Oroojeni Mohammad Javad","year":"2015"},{"key":"2026010211542871300_ocae320-B78","first-page":"1","author":"Xu","year":"2021"}],"container-title":["Journal of the American Medical Informatics Association"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/academic.oup.com\/jamia\/article-pdf\/33\/1\/49\/61411081\/ocae320.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"syndication"},{"URL":"https:\/\/academic.oup.com\/jamia\/article-pdf\/33\/1\/49\/61411081\/ocae320.pdf","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,1,2]],"date-time":"2026-01-02T16:54:47Z","timestamp":1767372887000},"score":1,"resource":{"primary":{"URL":"https:\/\/academic.oup.com\/jamia\/article\/33\/1\/49\/7951506"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,1,10]]},"references-count":78,"journal-issue":{"issue":"1","published-online":{"date-parts":[[2025,1,10]]},"published-print":{"date-parts":[[2026,1,1]]}},"URL":"https:\/\/doi.org\/10.1093\/jamia\/ocae320","relation":{},"ISSN":["1067-5027","1527-974X"],"issn-type":[{"value":"1067-5027","type":"print"},{"value":"1527-974X","type":"electronic"}],"subject":[],"published-other":{"date-parts":[[2026,1]]},"published":{"date-parts":[[2025,1,10]]}}}