{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,1,21]],"date-time":"2026-01-21T04:00:16Z","timestamp":1768968016185,"version":"3.49.0"},"reference-count":17,"publisher":"Oxford University Press (OUP)","issue":"7","license":[{"start":{"date-parts":[[2023,5,22]],"date-time":"2023-05-22T00:00:00Z","timestamp":1684713600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/academic.oup.com\/pages\/standard-publication-reuse-rights"}],"funder":[{"DOI":"10.13039\/100000002","name":"National Institutes of Health","doi-asserted-by":"publisher","award":["OT2HL161847"],"award-info":[{"award-number":["OT2HL161847"]}],"id":[{"id":"10.13039\/100000002","id-type":"DOI","asserted-by":"publisher"}]},{"name":"Researching COVID to Enhance Recovery"},{"name":"CD2H\u2014The National COVID Cohort Collaborative","award":["U24TR002306"],"award-info":[{"award-number":["U24TR002306"]}]},{"DOI":"10.13039\/100000002","name":"National Institutes of Health","doi-asserted-by":"publisher","award":["1 OT2 OD026549"],"award-info":[{"award-number":["1 OT2 OD026549"]}],"id":[{"id":"10.13039\/100000002","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000002","name":"National Institutes of Health","doi-asserted-by":"publisher","award":["1 OT2 OD026554"],"award-info":[{"award-number":["1 OT2 OD026554"]}],"id":[{"id":"10.13039\/100000002","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000002","name":"National Institutes of Health","doi-asserted-by":"publisher","award":["1 OT2 OD026557"],"award-info":[{"award-number":["1 OT2 OD026557"]}],"id":[{"id":"10.13039\/100000002","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000002","name":"National Institutes of Health","doi-asserted-by":"publisher","award":["1 OT2 OD026556"],"award-info":[{"award-number":["1 OT2 OD026556"]}],"id":[{"id":"10.13039\/100000002","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000002","name":"National Institutes of Health","doi-asserted-by":"publisher","award":["1 OT2 OD026550"],"award-info":[{"award-number":["1 OT2 OD026550"]}],"id":[{"id":"10.13039\/100000002","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000002","name":"National Institutes of Health","doi-asserted-by":"publisher","award":["1 OT2 OD 026552"],"award-info":[{"award-number":["1 OT2 OD 026552"]}],"id":[{"id":"10.13039\/100000002","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000002","name":"National Institutes of Health","doi-asserted-by":"publisher","award":["1 OT2 OD026553"],"award-info":[{"award-number":["1 OT2 OD026553"]}],"id":[{"id":"10.13039\/100000002","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000002","name":"National Institutes of Health","doi-asserted-by":"publisher","award":["1 OT2 OD026548"],"award-info":[{"award-number":["1 OT2 OD026548"]}],"id":[{"id":"10.13039\/100000002","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000002","name":"National Institutes of Health","doi-asserted-by":"publisher","award":["1 OT2 OD026551"],"award-info":[{"award-number":["1 OT2 OD026551"]}],"id":[{"id":"10.13039\/100000002","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000002","name":"National Institutes of Health","doi-asserted-by":"publisher","award":["1 OT2 OD026555"],"award-info":[{"award-number":["1 OT2 OD026555"]}],"id":[{"id":"10.13039\/100000002","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000002","name":"National Institutes of Health","doi-asserted-by":"publisher","award":["IAA: AOD21037"],"award-info":[{"award-number":["IAA: AOD21037"]}],"id":[{"id":"10.13039\/100000002","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000002","name":"National Institutes of Health","doi-asserted-by":"publisher","award":["AOD22003"],"award-info":[{"award-number":["AOD22003"]}],"id":[{"id":"10.13039\/100000002","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000002","name":"National Institutes of Health","doi-asserted-by":"publisher","award":["AOD16037"],"award-info":[{"award-number":["AOD16037"]}],"id":[{"id":"10.13039\/100000002","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000002","name":"National Institutes of Health","doi-asserted-by":"publisher","award":["AOD21041"],"award-info":[{"award-number":["AOD21041"]}],"id":[{"id":"10.13039\/100000002","id-type":"DOI","asserted-by":"publisher"}]},{"name":"Federally Qualified Health Centers","award":["HHSN 263201600085U"],"award-info":[{"award-number":["HHSN 263201600085U"]}]},{"name":"Data and Research Center","award":["5 U2C OD023196"],"award-info":[{"award-number":["5 U2C OD023196"]}]},{"name":"Data and Research Center","award":["1 U24 OD023121"],"award-info":[{"award-number":["1 U24 OD023121"]}]},{"name":"The Participant Center","award":["U24 OD023176"],"award-info":[{"award-number":["U24 OD023176"]}]},{"name":"Participant Technology Systems Center","award":["1 U24 OD023163"],"award-info":[{"award-number":["1 U24 OD023163"]}]},{"name":"Communications and Engagement","award":["3 OT2 OD023205"],"award-info":[{"award-number":["3 OT2 OD023205"]}]},{"name":"Communications and Engagement","award":["3 OT2 OD023206"],"award-info":[{"award-number":["3 OT2 OD023206"]}]},{"name":"Community Partners","award":["1 OT2 OD025277"],"award-info":[{"award-number":["1 OT2 OD025277"]}]},{"name":"Community Partners","award":["3 OT2 OD025315"],"award-info":[{"award-number":["3 OT2 OD025315"]}]},{"name":"Community Partners","award":["1 OT2 OD025337"],"award-info":[{"award-number":["1 OT2 OD025337"]}]},{"name":"Community Partners","award":["1 OT2 OD025276"],"award-info":[{"award-number":["1 OT2 OD025276"]}]}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2023,6,20]]},"abstract":"<jats:title>Abstract<\/jats:title>\n               <jats:p>Machine learning (ML)-driven computable phenotypes are among the most challenging to share and reproduce. Despite this difficulty, the urgent public health considerations around Long COVID make it especially important to ensure the rigor and reproducibility of Long COVID phenotyping algorithms such that they can be made available to a broad audience of researchers. As part of the NIH Researching COVID to Enhance Recovery (RECOVER) Initiative, researchers with the National COVID Cohort Collaborative (N3C) devised and trained an ML-based phenotype to identify patients highly probable to have Long COVID. Supported by RECOVER, N3C and NIH\u2019s All of Us study partnered to reproduce the output of N3C\u2019s trained model in the All of Us data enclave, demonstrating model extensibility in multiple environments. This case study in ML-based phenotype reuse illustrates how open-source software best practices and cross-site collaboration can de-black-box phenotyping algorithms, prevent unnecessary rework, and promote open science in informatics.<\/jats:p>","DOI":"10.1093\/jamia\/ocad077","type":"journal-article","created":{"date-parts":[[2023,5,23]],"date-time":"2023-05-23T05:58:25Z","timestamp":1684821505000},"page":"1305-1312","source":"Crossref","is-referenced-by-count":17,"title":["De-black-boxing health AI: demonstrating reproducible machine learning computable phenotypes using the N3C-RECOVER Long COVID model in the <i>All of Us<\/i> data repository"],"prefix":"10.1093","volume":"30","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-6840-9756","authenticated-orcid":false,"given":"Emily R","family":"Pfaff","sequence":"first","affiliation":[{"name":"Department of Medicine, University of North Carolina at Chapel Hill School of Medicine , Chapel Hill, North Carolina, USA"}]},{"given":"Andrew T","family":"Girvin","sequence":"additional","affiliation":[{"name":"Palantir Technologies , Denver, Colorado, USA"}]},{"given":"Miles","family":"Crosskey","sequence":"additional","affiliation":[{"name":"CoVar Applied Technologies , Durham, North Carolina, USA"}]},{"given":"Srushti","family":"Gangireddy","sequence":"additional","affiliation":[{"name":"Department of Biomedical Informatics, Vanderbilt University Medical Center , Nashville, Tennessee, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-0019-3087","authenticated-orcid":false,"given":"Hiral","family":"Master","sequence":"additional","affiliation":[{"name":"Vanderbilt Institute for Clinical and Translational Research, Vanderbilt University Medical Center , Nashville, Tennessee, USA"}]},{"given":"Wei-Qi","family":"Wei","sequence":"additional","affiliation":[{"name":"Department of Biomedical Informatics, Vanderbilt University Medical Center , Nashville, Tennessee, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-0342-1965","authenticated-orcid":false,"given":"V Eric","family":"Kerchberger","sequence":"additional","affiliation":[{"name":"Department of Medicine, Division of Allergy, Pulmonary & Critical Care Medicine, Vanderbilt University Medical Center , Nashville, Tennessee, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-5586-9940","authenticated-orcid":false,"given":"Mark","family":"Weiner","sequence":"additional","affiliation":[{"name":"Department of Medicine, Weill Cornell Medicine , New York, USA"}]},{"given":"Paul A","family":"Harris","sequence":"additional","affiliation":[{"name":"Department of Biomedical Informatics, Vanderbilt University Medical Center , Nashville, Tennessee, USA"}]},{"given":"Melissa","family":"Basford","sequence":"additional","affiliation":[{"name":"Vanderbilt Institute for Clinical and Translational Research, Vanderbilt University Medical Center , Nashville, Tennessee, USA"}]},{"given":"Chris","family":"Lunt","sequence":"additional","affiliation":[{"name":"National Institutes of Health , Bethesda, Maryland, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-5437-2545","authenticated-orcid":false,"given":"Christopher G","family":"Chute","sequence":"additional","affiliation":[{"name":"Johns Hopkins Schools of Medicine, Public Health, and Nursing. Baltimore , Maryland, USA"}]},{"given":"Richard A","family":"Moffitt","sequence":"additional","affiliation":[{"name":"Departments of Hematology and Medical Oncology and Biomedical Informatics, Emory University , Atlanta, Georgia, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-9114-8737","authenticated-orcid":false,"given":"Melissa","family":"Haendel","sequence":"additional","affiliation":[{"name":"Department of Biomedical Informatics, University of Colorado Anschutz Medical Campus , Denver, Colorado, USA"}]},{"name":"N3C and RECOVER Consortia","sequence":"additional","affiliation":[]}],"member":"286","published-online":{"date-parts":[[2023,5,22]]},"reference":[{"key":"2024061209211931500_ocad077-B1","year":"2021"},{"key":"2024061209211931500_ocad077-B2","doi-asserted-by":"publisher","DOI":"10.1038\/d41586-022-01702-2","volume-title":"How Common is Long COVID? Why Studies Give Different Answers","author":"Ledford","year":"2022"},{"key":"2024061209211931500_ocad077-B3","doi-asserted-by":"crossref","first-page":"e532","DOI":"10.1016\/S2589-7500(22)00048-6","article-title":"Identifying who has long COVID in the USA: a machine learning approach using N3C data","volume":"4","author":"Pfaff","year":"2022","journal-title":"Lancet Digit Health"},{"key":"2024061209211931500_ocad077-B4","doi-asserted-by":"publisher","author":"Hill","year":"2022","DOI":"10.1101\/2022.08.15.22278603"},{"key":"2024061209211931500_ocad077-B5","doi-asserted-by":"publisher","author":"Daniel Brannock","year":"2022","DOI":"10.1101\/2022.10.06.22280795"},{"key":"2024061209211931500_ocad077-B6","doi-asserted-by":"publisher","author":"Sidky","year":"2022","DOI":"10.1101\/2022.11.09.22282142"},{"key":"2024061209211931500_ocad077-B7","doi-asserted-by":"crossref","first-page":"1220","DOI":"10.1093\/jamia\/ocv112","article-title":"Desiderata for computable representations of electronic health records-driven phenotype algorithms","volume":"22","author":"Mo","year":"2015","journal-title":"J Am Med Inform Assoc"},{"key":"2024061209211931500_ocad077-B8","doi-asserted-by":"crossref","first-page":"668","DOI":"10.1056\/NEJMsr1809937","article-title":"The \u2018all of us\u2019 research program","volume":"381","author":"The All of Us Research Program Investigators","year":"2019","journal-title":"N Engl J Med"},{"key":"2024061209211931500_ocad077-B9","doi-asserted-by":"crossref","first-page":"609","DOI":"10.1093\/jamia\/ocab217","article-title":"Synergies between centralized and federated approaches to data quality: a report from the national COVID cohort collaborative","volume":"29","author":"Pfaff","year":"2021","journal-title":"J Am Med Inform Assoc"},{"key":"2024061209211931500_ocad077-B10","doi-asserted-by":"crossref","first-page":"427","DOI":"10.1093\/jamia\/ocaa196","article-title":"The National COVID Cohort Collaborative (N3C): rationale, design, infrastructure, and deployment","volume":"28","author":"Haendel","year":"2021","journal-title":"J Am Med Inform Assoc"},{"key":"2024061209211931500_ocad077-B11","author":"NCTraCSIDSci\/n3c-longcovid","year":"2022"},{"key":"2024061209211931500_ocad077-B12","doi-asserted-by":"crossref","first-page":"e0234962","DOI":"10.1371\/journal.pone.0234962","article-title":"Diversity and inclusion for the all of us research program: a scoping review","volume":"15","author":"Mapes","year":"2020","journal-title":"PLoS One"},{"key":"2024061209211931500_ocad077-B13","doi-asserted-by":"publisher","DOI":"10.1038\/d41586-022-00858-1","volume-title":"Breaking into the Black Box of Artificial Intelligence","author":"Savage","year":"2022"},{"key":"2024061209211931500_ocad077-B14"},{"key":"2024061209211931500_ocad077-B15","doi-asserted-by":"crossref","first-page":"427","DOI":"10.1093\/jamia\/ocac235","article-title":"Characterizing variability of electronic health record-driven phenotype definitions","volume":"30","author":"Brandt","year":"2022","journal-title":"J Am Med Inform Assoc"},{"key":"2024061209211931500_ocad077-B16","doi-asserted-by":"publisher","author":"Lorman","year":"2022","DOI":"10.1101\/2022.12.22.22283791"},{"key":"2024061209211931500_ocad077-B17","doi-asserted-by":"crossref","first-page":"719","DOI":"10.1007\/s10994-020-05877-5","volume":"109","author":"Bekker","year":"2020","journal-title":"Mach Learn"}],"container-title":["Journal of the American Medical Informatics Association"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/academic.oup.com\/jamia\/article-pdf\/30\/7\/1305\/58202585\/ocad077.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"syndication"},{"URL":"https:\/\/academic.oup.com\/jamia\/article-pdf\/30\/7\/1305\/58202585\/ocad077.pdf","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,6,12]],"date-time":"2024-06-12T09:21:56Z","timestamp":1718184116000},"score":1,"resource":{"primary":{"URL":"https:\/\/academic.oup.com\/jamia\/article\/30\/7\/1305\/7176021"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023,5,22]]},"references-count":17,"journal-issue":{"issue":"7","published-online":{"date-parts":[[2023,5,22]]},"published-print":{"date-parts":[[2023,6,20]]}},"URL":"https:\/\/doi.org\/10.1093\/jamia\/ocad077","relation":{},"ISSN":["1067-5027","1527-974X"],"issn-type":[{"value":"1067-5027","type":"print"},{"value":"1527-974X","type":"electronic"}],"subject":[],"published-other":{"date-parts":[[2023,7,1]]},"published":{"date-parts":[[2023,5,22]]}}}