{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,3,21]],"date-time":"2026-03-21T08:58:58Z","timestamp":1774083538067,"version":"3.50.1"},"reference-count":40,"publisher":"Oxford University Press (OUP)","issue":"4","license":[{"start":{"date-parts":[[2024,1,19]],"date-time":"2024-01-19T00:00:00Z","timestamp":1705622400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/academic.oup.com\/pages\/standard-publication-reuse-rights"}],"funder":[{"DOI":"10.13039\/100015521","name":"Stanford Maternal and Child Health Research Institute","doi-asserted-by":"publisher","id":[{"id":"10.13039\/100015521","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000025","name":"National Institute of Mental Health","doi-asserted-by":"publisher","id":[{"id":"10.13039\/100000025","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000002","name":"National Institutes of Health","doi-asserted-by":"publisher","award":["K23MH128455"],"award-info":[{"award-number":["K23MH128455"]}],"id":[{"id":"10.13039\/100000002","id-type":"DOI","asserted-by":"publisher"}]},{"name":"Advanced Data and Analytics for Behavioral and Social Sciences Research","award":["TADA-BSSR"],"award-info":[{"award-number":["TADA-BSSR"]}]},{"DOI":"10.13039\/100000002","name":"NIH","doi-asserted-by":"publisher","id":[{"id":"10.13039\/100000002","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000050","name":"National Heart, Lung, and Blood Institute","doi-asserted-by":"publisher","award":["1T32HL151323"],"award-info":[{"award-number":["1T32HL151323"]}],"id":[{"id":"10.13039\/100000050","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2024,4,3]]},"abstract":"<jats:title>Abstract<\/jats:title>\n                  <jats:sec>\n                    <jats:title>Objective<\/jats:title>\n                    <jats:p>To measure pediatrician adherence to evidence-based guidelines in the treatment of young children with attention-deficit\/hyperactivity disorder (ADHD) in a diverse healthcare system using natural language processing (NLP) techniques.<\/jats:p>\n                  <\/jats:sec>\n                  <jats:sec>\n                    <jats:title>Materials and Methods<\/jats:title>\n                    <jats:p>We extracted structured and free-text data from electronic health records (EHRs) of all office visits (2015-2019) of children aged 4-6 years in a community-based primary healthcare network in California, who had \u22651 visits with an ICD-10 diagnosis of ADHD. Two pediatricians annotated clinical notes of the first ADHD visit for 423 patients. Inter-annotator agreement (IAA) was assessed for the recommendation for the first-line behavioral treatment (F-measure\u2009=\u20090.89). Four pre-trained language models, including BioClinical Bidirectional Encoder Representations from Transformers (BioClinicalBERT), were used to identify behavioral treatment recommendations using a 70\/30 train\/test split. For temporal validation, we deployed BioClinicalBERT on 1,020 unannotated notes from other ADHD visits and well-care visits; all positively classified notes (n\u2009=\u200953) and 5% of negatively classified notes (n\u2009=\u200950) were manually reviewed.<\/jats:p>\n                  <\/jats:sec>\n                  <jats:sec>\n                    <jats:title>Results<\/jats:title>\n                    <jats:p>Of 423 patients, 313 (74%) were male; 298 (70%) were privately insured; 138 (33%) were White; 61 (14%) were Hispanic. The BioClinicalBERT model trained on the first ADHD visits achieved F1\u2009=\u20090.76, precision\u2009=\u20090.81, recall\u2009=\u20090.72, and AUC\u2009=\u20090.81 [0.72-0.89]. Temporal validation achieved F1\u2009=\u20090.77, precision\u2009=\u20090.68, and recall\u2009=\u20090.88. Fairness analysis revealed low model performance in publicly insured patients (F1\u2009=\u20090.53).<\/jats:p>\n                  <\/jats:sec>\n                  <jats:sec>\n                    <jats:title>Conclusion<\/jats:title>\n                    <jats:p>Deploying pre-trained language models on a variable set of clinical notes accurately captured pediatrician adherence to guidelines in the treatment of children with ADHD. Validating this approach in other patient populations is needed to achieve equitable measurement of quality of care at scale and improve clinical care for mental health conditions.<\/jats:p>\n                  <\/jats:sec>","DOI":"10.1093\/jamia\/ocae001","type":"journal-article","created":{"date-parts":[[2024,1,4]],"date-time":"2024-01-04T00:24:15Z","timestamp":1704327855000},"page":"949-957","source":"Crossref","is-referenced-by-count":12,"title":["Measuring quality-of-care in treatment of young children with attention-deficit\/hyperactivity disorder using pre-trained language models"],"prefix":"10.1093","volume":"31","author":[{"ORCID":"https:\/\/orcid.org\/0000-0001-8739-189X","authenticated-orcid":false,"given":"Malvika","family":"Pillai","sequence":"first","affiliation":[{"name":"Department of Biomedical Data Science, Stanford University School of Medicine , Stanford, CA 94305, United States"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Jose","family":"Posada","sequence":"additional","affiliation":[{"name":"Computer Science Department, University of the North , Barranquilla 080020, Colombia"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Rebecca M","family":"Gardner","sequence":"additional","affiliation":[{"name":"Department of Epidemiology and Population Health, Stanford University School of Medicine , Stanford, CA 94305, United States"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-6553-3455","authenticated-orcid":false,"given":"Tina","family":"Hernandez-Boussard","sequence":"additional","affiliation":[{"name":"Department of Biomedical Data Science, Stanford University School of Medicine , Stanford, CA 94305, United States"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-5134-3300","authenticated-orcid":false,"given":"Yair","family":"Bannett","sequence":"additional","affiliation":[{"name":"Division of Developmental-Behavioral Pediatrics, Department of Pediatrics, Stanford University School of Medicine , Stanford, CA 94304, United States"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"286","published-online":{"date-parts":[[2024,1,19]]},"reference":[{"issue":"6","key":"2024040320065110700_ocae001-B1","doi-asserted-by":"crossref","first-page":"584","DOI":"10.1177\/0009922812439621","article-title":"Attention-deficit\/hyperactivity disorder among children and adolescents in the United States: trend in diagnosis and use of pharmacotherapy by gender","volume":"51","author":"Sclar","year":"2012","journal-title":"Clin Pediatr (Phila)"},{"issue":"2","key":"2024040320065110700_ocae001-B2","doi-asserted-by":"crossref","first-page":"199","DOI":"10.1080\/15374416.2017.1417860","article-title":"Prevalence of parent-reported ADHD diagnosis and associated treatment among U.S. children and adolescents, 2016","volume":"47","author":"Danielson","year":"2018","journal-title":"J Clin Child Adolesc Psychol"},{"issue":"81","key":"2024040320065110700_ocae001-B3","first-page":"1","article-title":"Diagnostic experiences of children with attention-deficit\/hyperactivity disorder","author":"Visser","year":"2015","journal-title":"Natl Health Stat Rep"},{"key":"2024040320065110700_ocae001-B4","doi-asserted-by":"crossref","first-page":"82","DOI":"10.1016\/j.ambp.2006.05.005","article-title":"Academic and educational outcomes of children with ADHD","volume":"7(suppl 1)","author":"Loe","year":"2007","journal-title":"Ambul Pediatr"},{"issue":"5","key":"2024040320065110700_ocae001-B5","doi-asserted-by":"crossref","first-page":"e1584","DOI":"10.1542\/peds.2012-0974","article-title":"Interventions for preschool children at high risk for ADHD: a comparative effectiveness review","volume":"131","author":"Charach","year":"2013","journal-title":"Pediatrics"},{"issue":"2","key":"2024040320065110700_ocae001-B6","doi-asserted-by":"crossref","DOI":"10.1542\/peds.2019-0038","article-title":"School readiness in preschoolers with symptoms of attention-deficit\/hyperactivity disorder","volume":"144","author":"Perrin","year":"2019","journal-title":"Pediatrics"},{"issue":"6","key":"2024040320065110700_ocae001-B7","doi-asserted-by":"crossref","first-page":"1423","DOI":"10.1016\/j.jpeds.2015.02.018","article-title":"Treatment of attention deficit\/hyperactivity disorder among children with special health care needs","volume":"166","author":"Visser","year":"2015","journal-title":"J Pediatr"},{"issue":"269","key":"2024040320065110700_ocae001-B8","first-page":"1","article-title":"Physician office visits for attention-deficit\/hyperactivity disorder in children and adolescents aged 4-17 years: United States, 2012-2013","author":"Albert","year":"2017","journal-title":"NCHS Data Brief"},{"issue":"4","key":"2024040320065110700_ocae001-B9","doi-asserted-by":"crossref","first-page":"1033","DOI":"10.1542\/peds.108.4.1033","article-title":"Clinical practice guideline: treatment of the school-aged child with attention-deficit\/hyperactivity disorder","volume":"108","author":"Perrin","year":"2001","journal-title":"Pediatrics."},{"issue":"5","key":"2024040320065110700_ocae001-B10","doi-asserted-by":"crossref","first-page":"1007","DOI":"10.1542\/peds.2011-2654","article-title":"ADHD: clinical practice guideline for the diagnosis, evaluation, and treatment of attention-deficit\/hyperactivity disorder in children and adolescents","volume":"128","author":"Wolraich","year":"2011","journal-title":"Pediatrics"},{"issue":"4","key":"2024040320065110700_ocae001-B11","doi-asserted-by":"crossref","DOI":"10.1542\/peds.2019-2528","article-title":"Clinical practice guideline for the diagnosis, evaluation, and treatment of attention-deficit\/hyperactivity disorder in children and adolescents","volume":"144","author":"Wolraich","year":"2019","journal-title":"Pediatrics"},{"issue":"1","key":"2024040320065110700_ocae001-B12","doi-asserted-by":"crossref","first-page":"184","DOI":"10.1080\/15374410701818681","article-title":"Evidence-based psychosocial treatments for attention-deficit\/hyperactivity disorder","volume":"37","author":"Pelham","year":"2008","journal-title":"J Clin Child Adolesc Psychol"},{"issue":"4","key":"2024040320065110700_ocae001-B13","doi-asserted-by":"crossref","first-page":"396","DOI":"10.1080\/15374416.2015.1105138","article-title":"Treatment sequencing for childhood ADHD: a multiple-randomization study of adaptive medication and behavioral interventions","volume":"45","author":"Pelham","year":"2016","journal-title":"J Clin Child Adolesc Psychol"},{"key":"2024040320065110700_ocae001-B14","article-title":"Rate of pediatrician recommendations for behavioral treatment for preschoolers with attention-deficit\/hyperactivity disorder diagnosis or related symptoms","author":"Bannett","year":"2021","journal-title":"JAMA Pediatr"},{"issue":"8","key":"2024040320065110700_ocae001-B15","doi-asserted-by":"crossref","first-page":"905","DOI":"10.1111\/jcpp.12204","article-title":"Racial\/ethnic disparities in ADHD diagnosis by kindergarten entry","volume":"55","author":"Morgan","year":"2014","journal-title":"J Child Psychol Psychiatry"},{"key":"2024040320065110700_ocae001-B16","doi-asserted-by":"crossref","first-page":"155","DOI":"10.1016\/j.jpeds.2019.06.025","article-title":"Factors associated with attention deficit hyperactivity disorder medication use in community care settings","volume":"213","author":"Kamimura-Nishimura","year":"2019","journal-title":"J Pediatr"},{"issue":"3","key":"2024040320065110700_ocae001-B17","doi-asserted-by":"crossref","first-page":"256","DOI":"10.1016\/j.acap.2017.10.003","article-title":"Receipt of medication and behavioral therapy among a national sample of school-age children diagnosed with attention-deficit\/hyperactivity disorder","volume":"18","author":"Walls","year":"2018","journal-title":"Acad Pediatr"},{"issue":"8","key":"2024040320065110700_ocae001-B18","doi-asserted-by":"crossref","first-page":"735","DOI":"10.1016\/j.jaac.2011.05.006","article-title":"Gaps in quality measures for child mental health care: an opportunity for a collaborative agenda","volume":"50","author":"Zima","year":"2011","journal-title":"J Am Acad Child Adolesc Psychiatry"},{"key":"2024040320065110700_ocae001-B19","doi-asserted-by":"crossref","first-page":"S38","DOI":"10.1542\/peds.2012-1427e","article-title":"National quality measures for child mental health care: background, progress, and next steps","volume":"131(suppl 1)","author":"Zima","year":"2013","journal-title":"Pediatrics"},{"key":"2024040320065110700_ocae001-B20","author":"National Committee for Quality Assurance"},{"issue":"3","key":"2024040320065110700_ocae001-B21","doi-asserted-by":"crossref","first-page":"401","DOI":"10.1377\/hlthaff.2015.1258","article-title":"US physician practices spend more than $15.4 Billion annually to report quality measures","volume":"35","author":"Casalino","year":"2016","journal-title":"Health Aff (Millwood)"},{"issue":"13","key":"2024040320065110700_ocae001-B22","doi-asserted-by":"crossref","first-page":"1219","DOI":"10.1001\/jama.2017.11525","article-title":"Measuring the cost of quality measurement: a missing link in quality strategy","volume":"318","author":"Schuster","year":"2017","journal-title":"JAMA"},{"issue":"6","key":"2024040320065110700_ocae001-B23","doi-asserted-by":"crossref","first-page":"1136","DOI":"10.1542\/peds.2014-1500","article-title":"Variability in ADHD care in community-based pediatrics","volume":"134","author":"Epstein","year":"2014","journal-title":"Pediatrics."},{"issue":"8","key":"2024040320065110700_ocae001-B24","doi-asserted-by":"crossref","first-page":"573","DOI":"10.1097\/DBP.0000000000000490","article-title":"Distance-learning, ADHD quality improvement in primary care: a cluster-randomized trial","volume":"38","author":"Fiks","year":"2017","journal-title":"J Dev Behav Pediatr"},{"issue":"1","key":"2024040320065110700_ocae001-B25","first-page":"5","article-title":"Enhanced quality measurement event detection: an application to physician reporting","volume":"5","author":"Tamang","year":"2017","journal-title":"EGEMS (Wash DC)"},{"issue":"4","key":"2024040320065110700_ocae001-B26","doi-asserted-by":"crossref","first-page":"816","DOI":"10.1158\/1055-9965.EPI-19-0873","article-title":"Leveraging digital data to inform and improve quality cancer care","volume":"29","author":"Hernandez-Boussard","year":"2020","journal-title":"Cancer Epidemiol Biomarkers Prev"},{"issue":"12","key":"2024040320065110700_ocae001-B27","doi-asserted-by":"crossref","first-page":"2011","DOI":"10.1093\/jamia\/ocaa088","article-title":"MINIMAR (MINimum Information for Medical AI Reporting): developing reporting standards for artificial intelligence in health care","volume":"27","author":"Hernandez-Boussard","year":"2020","journal-title":"J Am Med Inform Assoc"},{"issue":"2","key":"2024040320065110700_ocae001-B28","doi-asserted-by":"crossref","first-page":"280","DOI":"10.1016\/j.acap.2020.04.009","article-title":"Attention-deficit\/hyperactivity disorder in 2- to 5-year-olds: a primary care network experience","volume":"21","author":"Bannett","year":"2020","journal-title":"Acad Pediatr"},{"issue":"13","key":"2024040320065110700_ocae001-B29","doi-asserted-by":"crossref","first-page":"1647","DOI":"10.1177\/1087054716672337","article-title":"Validation of the use of electronic health records for classification of ADHD status","volume":"23","author":"Gruschow","year":"2016","journal-title":"J Atten Disord"},{"issue":"3","key":"2024040320065110700_ocae001-B30","doi-asserted-by":"crossref","first-page":"331","DOI":"10.1093\/jamia\/ocx132","article-title":"CLAMP\u2014a toolkit for efficiently building customized clinical natural language processing pipelines","volume":"25","author":"Soysal","year":"2017","journal-title":"J Am Med Inform Assoc"},{"issue":"3","key":"2024040320065110700_ocae001-B31","doi-asserted-by":"crossref","first-page":"296","DOI":"10.1197\/jamia.M1733","article-title":"Agreement, the f-measure, and reliability in information retrieval","volume":"12","author":"Hripcsak","year":"2005","journal-title":"J Am Med Inform Assoc"},{"issue":"1","key":"2024040320065110700_ocae001-B32","doi-asserted-by":"crossref","first-page":"24","DOI":"10.1038\/s41597-021-01110-7","article-title":"Peeking into a black box, the fairness and generalizability of a MIMIC-III benchmarking model","volume":"9","author":"R\u00f6\u00f6sli","year":"2022","journal-title":"Sci Data"},{"issue":"2","key":"2024040320065110700_ocae001-B33","doi-asserted-by":"crossref","first-page":"314","DOI":"10.1016\/j.acap.2022.07.012","article-title":"Continuity of care in primary care for young children with chronic conditions","volume":"23","author":"Bannett","year":"2023","journal-title":"Acad Pediatr"},{"issue":"1","key":"2024040320065110700_ocae001-B34","doi-asserted-by":"crossref","first-page":"7","DOI":"10.1007\/s44176-022-00006-z","article-title":"A brief review on algorithmic fairness","volume":"1","author":"Wang","year":"2022","journal-title":"MSE"},{"key":"2024040320065110700_ocae001-B35","doi-asserted-by":"crossref","first-page":"203","DOI":"10.1016\/B978-0-12-818438-7.00009-5","volume-title":"Artificial Intelligence in Healthcare","author":"Jeddi","year":"2020"},{"key":"2024040320065110700_ocae001-B36","doi-asserted-by":"crossref","first-page":"1249","DOI":"10.1162\/tacl_a_00425","article-title":"Quantifying social biases in NLP: a generalization and empirical comparison of extrinsic fairness metrics","volume":"9","author":"Czarnowska","year":"2021","journal-title":"Trans Assoc Comput Linguist"},{"issue":"1","key":"2024040320065110700_ocae001-B37","doi-asserted-by":"crossref","first-page":"190","DOI":"10.1093\/jamia\/ocaa210","article-title":"Bias at warp speed: how AI may contribute to the disparities gap in the time of COVID-19","volume":"28","author":"R\u00f6\u00f6sli","year":"2020","journal-title":"J Am Med Inform Assoc"},{"key":"2024040320065110700_ocae001-B38","author":"Orgad","year":"2022"},{"key":"2024040320065110700_ocae001-B39","author":"Dev","year":"2022"},{"issue":"3","key":"2024040320065110700_ocae001-B40","doi-asserted-by":"crossref","first-page":"295","DOI":"10.1097\/00005650-199903000-00009","article-title":"Grading the graders: how hospitals in California and New York perceive and interpret their report cards","volume":"37","author":"Romano","year":"1999","journal-title":"Med Care"}],"container-title":["Journal of the American Medical Informatics Association"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/academic.oup.com\/jamia\/advance-article-pdf\/doi\/10.1093\/jamia\/ocae001\/57148541\/ocae001.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"syndication"},{"URL":"https:\/\/academic.oup.com\/jamia\/advance-article-pdf\/doi\/10.1093\/jamia\/ocae001\/57148541\/ocae001.pdf","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,4,3]],"date-time":"2024-04-03T16:07:25Z","timestamp":1712160445000},"score":1,"resource":{"primary":{"URL":"https:\/\/academic.oup.com\/jamia\/article\/31\/4\/949\/7582076"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,1,19]]},"references-count":40,"journal-issue":{"issue":"4","published-online":{"date-parts":[[2024,1,19]]},"published-print":{"date-parts":[[2024,4,3]]}},"URL":"https:\/\/doi.org\/10.1093\/jamia\/ocae001","relation":{"has-preprint":[{"id-type":"doi","id":"10.1101\/2023.06.12.23291071","asserted-by":"object"}]},"ISSN":["1067-5027","1527-974X"],"issn-type":[{"value":"1067-5027","type":"print"},{"value":"1527-974X","type":"electronic"}],"subject":[],"published-other":{"date-parts":[[2024,4,1]]},"published":{"date-parts":[[2024,1,19]]}}}