{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,7]],"date-time":"2026-04-07T16:02:57Z","timestamp":1775577777291,"version":"3.50.1"},"reference-count":27,"publisher":"Oxford University Press (OUP)","issue":"9","license":[{"start":{"date-parts":[[2024,4,30]],"date-time":"2024-04-30T00:00:00Z","timestamp":1714435200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/academic.oup.com\/pages\/standard-publication-reuse-rights"}],"funder":[{"DOI":"10.13039\/100000050","name":"National Heart, Lung, & Blood Institute","doi-asserted-by":"crossref","award":["R01HL155915"],"award-info":[{"award-number":["R01HL155915"]}],"id":[{"id":"10.13039\/100000050","id-type":"DOI","asserted-by":"crossref"}]},{"name":"National Center for Advancing Clinical Sciences","award":["UL1TR004419"],"award-info":[{"award-number":["UL1TR004419"]}]},{"DOI":"10.13039\/100000002","name":"National Institutes of Health","doi-asserted-by":"publisher","award":["K23HL153775"],"award-info":[{"award-number":["K23HL153775"]}],"id":[{"id":"10.13039\/100000002","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000862","name":"Doris Duke Charitable Foundation","doi-asserted-by":"publisher","award":["2022060"],"award-info":[{"award-number":["2022060"]}],"id":[{"id":"10.13039\/100000862","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100002491","name":"Bristol-Myers Squibb","doi-asserted-by":"publisher","id":[{"id":"10.13039\/100002491","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100004191","name":"Novo Nordisk","doi-asserted-by":"publisher","id":[{"id":"10.13039\/501100004191","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2024,9,1]]},"abstract":"<jats:title>Abstract<\/jats:title>\n               <jats:sec>\n                  <jats:title>Objectives<\/jats:title>\n                  <jats:p>The study developed framework that leverages an open-source Large Language Model (LLM) to enable clinicians to ask plain-language questions about a patient\u2019s entire echocardiogram report history. This approach is intended to streamline the extraction of clinical insights from multiple echocardiogram reports, particularly in patients with complex cardiac diseases, thereby enhancing both patient care and research efficiency.<\/jats:p>\n               <\/jats:sec>\n               <jats:sec>\n                  <jats:title>Materials and Methods<\/jats:title>\n                  <jats:p>Data from over 10 years were collected, comprising echocardiogram reports from patients with more than 10 echocardiograms on file at the Mount Sinai Health System. These reports were converted into a single document per patient for analysis, broken down into snippets and relevant snippets were retrieved using text similarity measures. The LLaMA-2 70B model was employed for analyzing the text using a specially crafted prompt. The model\u2019s performance was evaluated against ground-truth answers created by faculty cardiologists.<\/jats:p>\n               <\/jats:sec>\n               <jats:sec>\n                  <jats:title>Results<\/jats:title>\n                  <jats:p>The study analyzed 432 reports from 37 patients for a total of 100 question-answer pairs. The LLM correctly answered 90% questions, with accuracies of 83% for temporality, 93% for severity assessment, 84% for intervention identification, and 100% for diagnosis retrieval. Errors mainly stemmed from the LLM\u2019s inherent limitations, such as misinterpreting numbers or hallucinations.<\/jats:p>\n               <\/jats:sec>\n               <jats:sec>\n                  <jats:title>Conclusion<\/jats:title>\n                  <jats:p>The study demonstrates the feasibility and effectiveness of using a local, open-source LLM for querying and interpreting echocardiogram report data. This approach offers a significant improvement over traditional keyword-based searches, enabling more contextually relevant and semantically accurate responses; in turn showing promise in enhancing clinical decision-making and research by facilitating more efficient access to complex patient data.<\/jats:p>\n               <\/jats:sec>","DOI":"10.1093\/jamia\/ocae085","type":"journal-article","created":{"date-parts":[[2024,4,30]],"date-time":"2024-04-30T16:41:29Z","timestamp":1714495289000},"page":"2097-2102","source":"Crossref","is-referenced-by-count":23,"title":["Local large language models for privacy-preserving accelerated review of historic echocardiogram reports"],"prefix":"10.1093","volume":"31","author":[{"given":"Akhil","family":"Vaid","sequence":"first","affiliation":[{"name":"The Charles Bronfman Institute for Personalized Medicine, Icahn School of Medicine at Mount Sinai , New York, NY 10029, United States"},{"name":"The Division of Data Driven and Digital Medicine (D3M), Department of Medicine, Icahn School of Medicine at Mount Sinai , New York, NY 10029, United States"}]},{"given":"Son Q","family":"Duong","sequence":"additional","affiliation":[{"name":"The Charles Bronfman Institute for Personalized Medicine, Icahn School of Medicine at Mount Sinai , New York, NY 10029, United States"},{"name":"Division of Pediatric Cardiology, Icahn School of Medicine at Mount Sinai , New York, NY 10029, United States"}]},{"given":"Joshua","family":"Lampert","sequence":"additional","affiliation":[{"name":"The Charles Bronfman Institute for Personalized Medicine, Icahn School of Medicine at Mount Sinai , New York, NY 10029, United States"},{"name":"Helmsley Electrophysiology Center, Icahn School of Medicine at Mount Sinai , New York, NY 10029, United States"}]},{"given":"Patricia","family":"Kovatch","sequence":"additional","affiliation":[{"name":"Department of Genetics and Genomic Sciences, Icahn School of Medicine at Mount Sinai , New York, NY 10029, United States"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-4946-6533","authenticated-orcid":false,"given":"Robert","family":"Freeman","sequence":"additional","affiliation":[{"name":"Department of Population Health Science and Policy, Institute for Healthcare Delivery Science, Icahn School of Medicine at Mount Sinai , New York, NY 10029, United States"}]},{"given":"Edgar","family":"Argulian","sequence":"additional","affiliation":[{"name":"The Zena and Michael A. Wiener Cardiovascular Institute, Icahn School of Medicine at Mount Sinai , New York, NY 10029, United States"},{"name":"Mount Sinai Heart, Icahn School of Medicine at Mount Sinai , New York, NY 10029, United States"}]},{"given":"Lori","family":"Croft","sequence":"additional","affiliation":[{"name":"The Zena and Michael A. Wiener Cardiovascular Institute, Icahn School of Medicine at Mount Sinai , New York, NY 10029, United States"},{"name":"Mount Sinai Heart, Icahn School of Medicine at Mount Sinai , New York, NY 10029, United States"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-7487-1990","authenticated-orcid":false,"given":"Stamatios","family":"Lerakis","sequence":"additional","affiliation":[{"name":"The Zena and Michael A. Wiener Cardiovascular Institute, Icahn School of Medicine at Mount Sinai , New York, NY 10029, United States"},{"name":"Mount Sinai Heart, Icahn School of Medicine at Mount Sinai , New York, NY 10029, United States"}]},{"given":"Martin","family":"Goldman","sequence":"additional","affiliation":[{"name":"The Zena and Michael A. Wiener Cardiovascular Institute, Icahn School of Medicine at Mount Sinai , New York, NY 10029, United States"},{"name":"Mount Sinai Heart, Icahn School of Medicine at Mount Sinai , New York, NY 10029, United States"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-9467-6199","authenticated-orcid":false,"given":"Rohan","family":"Khera","sequence":"additional","affiliation":[{"name":"Section of Cardiovascular Medicine, Department of Internal Medicine, Yale School of Medicine , New Haven, CT 06510, United States"}]},{"given":"Girish N","family":"Nadkarni","sequence":"additional","affiliation":[{"name":"The Charles Bronfman Institute for Personalized Medicine, Icahn School of Medicine at Mount Sinai , New York, NY 10029, United States"},{"name":"The Division of Data Driven and Digital Medicine (D3M), Department of Medicine, Icahn School of Medicine at Mount Sinai , New York, NY 10029, United States"}]}],"member":"286","published-online":{"date-parts":[[2024,4,30]]},"reference":[{"issue":"20","key":"2024082207515394000_ocae085-B1","doi-asserted-by":"crossref","first-page":"1592","DOI":"10.1136\/heartjnl-2021-319725","article-title":"Artificial intelligence for the echocardiographic assessment of valvular heart disease","volume":"108","author":"Nedadur","year":"2022","journal-title":"Heart"},{"issue":"4","key":"2024082207515394000_ocae085-B2","doi-asserted-by":"crossref","first-page":"222","DOI":"10.1007\/s12574-021-00531-y","article-title":"Differences in echocardiography interpretation techniques among trainees and expert readers","volume":"19","author":"Anderson","year":"2021","journal-title":"J Echocardiogr"},{"issue":"2","key":"2024082207515394000_ocae085-B3","doi-asserted-by":"crossref","first-page":"48","DOI":"10.1109\/MCI.2014.2307227","article-title":"Jumping NLP curves: a review of natural language processing research [Review Article]","volume":"9","author":"Cambria","year":"2014","journal-title":"IEEE Comput Intell Mag"},{"key":"2024082207515394000_ocae085-B4","author":"Kaddour","year":"2023"},{"key":"2024082207515394000_ocae085-B5","author":"Zhao","year":"2023"},{"key":"2024082207515394000_ocae085-B6","author":"Wei","year":"2021"},{"issue":"3","key":"2024082207515394000_ocae085-B7","doi-asserted-by":"crossref","first-page":"e102","DOI":"10.1016\/S2589-7500(23)00023-7","article-title":"ChatGPT: friend or foe?","volume":"5","author":"The Lancet Digital H","year":"2023","journal-title":"Lancet Digit Health"},{"issue":"5","key":"2024082207515394000_ocae085-B8","doi-asserted-by":"crossref","first-page":"1122","DOI":"10.1109\/JAS.2023.123618","article-title":"A brief overview of ChatGPT: the history, status quo and potential future development","volume":"10","author":"Wu","year":"2023","journal-title":"IEEE\/CAA J Autom Sin"},{"key":"2024082207515394000_ocae085-B9","author":"Huang","year":"2023"},{"issue":"12","key":"2024082207515394000_ocae085-B10","doi-asserted-by":"crossref","first-page":"e855","DOI":"10.1016\/S2589-7500(23)00202-9","article-title":"Using fine-tuned large language models to parse clinical notes in musculoskeletal pain disorders","volume":"5","author":"Vaid","year":"2023","journal-title":"Lancet Digit Health"},{"key":"2024082207515394000_ocae085-B11","author":"Touvron","year":"2023"},{"key":"2024082207515394000_ocae085-B12","author":"Bai","year":"2022"},{"issue":"9","key":"2024082207515394000_ocae085-B13","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1145\/3560815","article-title":"Pre-train, prompt, and predict: a systematic survey of prompting methods in natural language processing","volume":"55","author":"Liu","year":"2023","journal-title":"ACM Comput Surv"},{"key":"2024082207515394000_ocae085-B14","author":"Conneau","year":"2018"},{"key":"2024082207515394000_ocae085-B15","author":"Wang","year":"2022"},{"key":"2024082207515394000_ocae085-B16","author":"Muennighoff","year":"2022"},{"issue":"1-2","key":"2024082207515394000_ocae085-B17","doi-asserted-by":"crossref","first-page":"91","DOI":"10.1016\/j.mcm.2010.07.022","article-title":"Cosine similarity measures for intuitionistic fuzzy sets and their applications","volume":"53","author":"Ye","year":"2011","journal-title":"Math Comput Model"},{"key":"2024082207515394000_ocae085-B18","author":"Wolf","year":"2019"},{"key":"2024082207515394000_ocae085-B19","author":"Reimers","year":"2019"},{"issue":"9","key":"2024082207515394000_ocae085-B20","first-page":"1","article-title":"Pandas: a foundational Python library for data analysis and statistics","volume":"14","author":"McKinney","year":"2011","journal-title":"Python High Performance Sci Comput"},{"key":"2024082207515394000_ocae085-B21","author":"Liu","year":"2018"},{"key":"2024082207515394000_ocae085-B22"},{"issue":"4","key":"2024082207515394000_ocae085-B23","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1145\/3447772","article-title":"Knowledge graphs","volume":"54","author":"Hogan","year":"2021","journal-title":"ACM Comput Surv"},{"key":"2024082207515394000_ocae085-B24","author":"Wang","year":"2023"},{"key":"2024082207515394000_ocae085-B25","author":"Noever","year":"2023"},{"key":"2024082207515394000_ocae085-B26","first-page":"38087","author":"Xiao","year":"2023"},{"key":"2024082207515394000_ocae085-B27","author":"Poli","year":"2023"}],"container-title":["Journal of the American Medical Informatics Association"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/academic.oup.com\/jamia\/article-pdf\/31\/9\/2097\/58868167\/ocae085.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"syndication"},{"URL":"https:\/\/academic.oup.com\/jamia\/article-pdf\/31\/9\/2097\/58868167\/ocae085.pdf","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,8,22]],"date-time":"2024-08-22T11:46:05Z","timestamp":1724327165000},"score":1,"resource":{"primary":{"URL":"https:\/\/academic.oup.com\/jamia\/article\/31\/9\/2097\/7660230"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,4,30]]},"references-count":27,"journal-issue":{"issue":"9","published-online":{"date-parts":[[2024,4,30]]},"published-print":{"date-parts":[[2024,9,1]]}},"URL":"https:\/\/doi.org\/10.1093\/jamia\/ocae085","relation":{},"ISSN":["1067-5027","1527-974X"],"issn-type":[{"value":"1067-5027","type":"print"},{"value":"1527-974X","type":"electronic"}],"subject":[],"published-other":{"date-parts":[[2024,9]]},"published":{"date-parts":[[2024,4,30]]}}}