{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,5,10]],"date-time":"2026-05-10T11:11:04Z","timestamp":1778411464101,"version":"3.51.4"},"reference-count":62,"publisher":"Oxford University Press (OUP)","issue":"3","license":[{"start":{"date-parts":[[2025,1,11]],"date-time":"2025-01-11T00:00:00Z","timestamp":1736553600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by-nc-nd\/4.0\/"}],"funder":[{"DOI":"10.13039\/100000002","name":"National Institutes of Health","doi-asserted-by":"publisher","id":[{"id":"10.13039\/100000002","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100008460","name":"National Center for Complementary and Integrative Health","doi-asserted-by":"publisher","award":["R01AT009457"],"award-info":[{"award-number":["R01AT009457"]}],"id":[{"id":"10.13039\/100008460","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100008460","name":"National Center for Complementary and Integrative Health","doi-asserted-by":"publisher","award":["U01AT012871"],"award-info":[{"award-number":["U01AT012871"]}],"id":[{"id":"10.13039\/100008460","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000049","name":"National Institute on Aging","doi-asserted-by":"publisher","award":["R01AG078154"],"award-info":[{"award-number":["R01AG078154"]}],"id":[{"id":"10.13039\/100000049","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000054","name":"National Cancer Institute","doi-asserted-by":"publisher","award":["R01CA287413"],"award-info":[{"award-number":["R01CA287413"]}],"id":[{"id":"10.13039\/100000054","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000062","name":"National Institute of Diabetes and Digestive and Kidney Diseases","doi-asserted-by":"publisher","award":["R01DK115629"],"award-info":[{"award-number":["R01DK115629"]}],"id":[{"id":"10.13039\/100000062","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100006545","name":"National Institute on Minority Health and Health Disparities","doi-asserted-by":"publisher","award":["1R21MD019134-01"],"award-info":[{"award-number":["1R21MD019134-01"]}],"id":[{"id":"10.13039\/100006545","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2025,3,1]]},"abstract":"<jats:title>Abstract<\/jats:title>\n               <jats:sec>\n                  <jats:title>Objective<\/jats:title>\n                  <jats:p>To develop an advanced multi-task large language model (LLM) framework for extracting diverse types of information about dietary supplements (DSs) from clinical records.<\/jats:p>\n               <\/jats:sec>\n               <jats:sec>\n                  <jats:title>Methods<\/jats:title>\n                  <jats:p>We focused on 4 core DS information extraction tasks: named entity recognition (2\u00a0949 clinical sentences), relation extraction (4\u00a0892 sentences), triple extraction (2\u00a0949 sentences), and usage classification (2\u00a0460 sentences). To address these tasks, we introduced the retrieval-augmented multi-task information extraction (RAMIE) framework, which incorporates: (1) instruction fine-tuning with task-specific prompts; (2) multi-task training of LLMs to enhance storage efficiency and reduce training costs; and (3) retrieval-augmented generation, which retrieves similar examples from the training set to improve task performance. We compared the performance of RAMIE to LLMs with instruction fine-tuning alone and conducted an ablation study to evaluate the individual contributions of multi-task learning and retrieval-augmented generation to overall performance improvements.<\/jats:p>\n               <\/jats:sec>\n               <jats:sec>\n                  <jats:title>Results<\/jats:title>\n                  <jats:p>Using the RAMIE framework, Llama2-13B achieved an F1 score of 87.39 on the named entity recognition task, reflecting a 3.51% improvement. It also excelled in the relation extraction task with an F1 score of 93.74, a 1.15% improvement. For the triple extraction task, Llama2-7B achieved an F1 score of 79.45, representing a significant 14.26% improvement. MedAlpaca-7B delivered the highest F1 score of 93.45 on the usage classification task, with a 0.94% improvement. The ablation study highlighted that while multi-task learning improved efficiency with a minor trade-off in performance, the inclusion of retrieval-augmented generation significantly enhanced overall accuracy across tasks.<\/jats:p>\n               <\/jats:sec>\n               <jats:sec>\n                  <jats:title>Conclusion<\/jats:title>\n                  <jats:p>The RAMIE framework demonstrates substantial improvements in multi-task information extraction for DS-related data from clinical records.<\/jats:p>\n               <\/jats:sec>","DOI":"10.1093\/jamia\/ocaf002","type":"journal-article","created":{"date-parts":[[2025,1,11]],"date-time":"2025-01-11T20:04:13Z","timestamp":1736625853000},"page":"545-554","source":"Crossref","is-referenced-by-count":13,"title":["RAMIE: retrieval-augmented multi-task information extraction with large language models on dietary supplements"],"prefix":"10.1093","volume":"32","author":[{"given":"Zaifu","family":"Zhan","sequence":"first","affiliation":[{"name":"Department of Electrical and Computer Engineering, University of Minnesota , Minneapolis, MN 55455,","place":["United States"]}]},{"given":"Shuang","family":"Zhou","sequence":"additional","affiliation":[{"name":"Division of Computational Health Sciences, Department of Surgery, University of Minnesota , Minneapolis, MN 55455,","place":["United States"]}]},{"given":"Mingchen","family":"Li","sequence":"additional","affiliation":[{"name":"Division of Computational Health Sciences, Department of Surgery, University of Minnesota , Minneapolis, MN 55455,","place":["United States"]}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-8258-3585","authenticated-orcid":false,"given":"Rui","family":"Zhang","sequence":"additional","affiliation":[{"name":"Division of Computational Health Sciences, Department of Surgery, University of Minnesota , Minneapolis, MN 55455,","place":["United States"]}]}],"member":"286","published-online":{"date-parts":[[2025,1,11]]},"reference":[{"key":"2025021811401897700_ocaf002-B1","author":"Council for Responsible Nutrition"},{"key":"2025021811401897700_ocaf002-B2","doi-asserted-by":"crossref","first-page":"91","DOI":"10.1080\/10590500902885676","article-title":"Quality assurance and safety of herbal dietary supplements","volume":"27","author":"Fu","year":"2009","journal-title":"J Environ Sci Health C Environ Carcinog Ecotoxicol Rev"},{"key":"2025021811401897700_ocaf002-B3","doi-asserted-by":"crossref","first-page":"230","DOI":"10.1080\/10810730.2010.529493","article-title":"Influence of the dietary supplement health and education act on consumer beliefs about the safety and effectiveness of dietary supplements","volume":"16","author":"Dodge","year":"2011","journal-title":"J Health Commun"},{"key":"2025021811401897700_ocaf002-B4","doi-asserted-by":"crossref","first-page":"393","DOI":"10.1016\/j.fct.2010.11.014","article-title":"Mission impossible? Regulatory and enforcement issues to ensure safety of dietary supplements","volume":"49","author":"Petroczi","year":"2011","journal-title":"Food Chem Toxicol"},{"key":"2025021811401897700_ocaf002-B5","author":"U.S. Food and Drug Administration","year":"2023"},{"key":"2025021811401897700_ocaf002-B6","doi-asserted-by":"crossref","first-page":"e183337","DOI":"10.1001\/jamanetworkopen.2018.3337","article-title":"Unapproved pharmaceutical ingredients included in dietary supplements associated with us Food and Drug Administration warnings","volume":"1","author":"Tucker","year":"2018","journal-title":"JAMA Netw Open"},{"key":"2025021811401897700_ocaf002-B7","doi-asserted-by":"crossref","first-page":"280","DOI":"10.1017\/S0007114517003452","article-title":"Effect of long-term nutraceutical and dietary supplement use on cognition in the elderly: a 10-year systematic review of randomised controlled trials","volume":"119","author":"D\u2019Cunha","year":"2018","journal-title":"Br J Nutr"},{"key":"2025021811401897700_ocaf002-B8","doi-asserted-by":"crossref","first-page":"845","DOI":"10.1016\/j.jacbts.2019.10.008","article-title":"Limitations of animal studies for predicting toxicity in clinical trials: is it time to rethink our current approach?","volume":"4","author":"Van Norman","year":"2019","journal-title":"JACC Basic Transl Sci"},{"key":"2025021811401897700_ocaf002-B9","doi-asserted-by":"crossref","first-page":"101","DOI":"10.1016\/S0140-6736(03)12227-1","article-title":"Adverse events associated with dietary supplements: an observational study","volume":"361","author":"Palmer","year":"2003","journal-title":"Lancet"},{"key":"2025021811401897700_ocaf002-B10","doi-asserted-by":"crossref","first-page":"431","DOI":"10.1177\/1060028017744316","article-title":"Dietary supplement adverse event report data from the FDA Center for Food Safety and Applied Nutrition Adverse Event Reporting System (CAERS), 2004-2013","volume":"52","author":"Timbo","year":"2018","journal-title":"Ann Pharmacother"},{"key":"2025021811401897700_ocaf002-B11","first-page":"370","article-title":"Detecting signals of interactions between warfarin and dietary supplements in electronic health records","volume":"245","author":"Fan","year":"2017","journal-title":"Stud Health Technol Inform"},{"key":"2025021811401897700_ocaf002-B12","first-page":"1361","article-title":"Evaluating term coverage of herbal and dietary supplements in electronic health records","volume":"2015","author":"Zhang","year":"2015","journal-title":"AMIA Annu Symp Proc"},{"key":"2025021811401897700_ocaf002-B13","doi-asserted-by":"crossref","first-page":"15","DOI":"10.1186\/s12911-018-0626-6","article-title":"Using natural language processing methods to classify use status of dietary supplements in clinical notes","volume":"18","author":"Fan","year":"2018","journal-title":"BMC Med Inform Decis Mak"},{"key":"2025021811401897700_ocaf002-B14","doi-asserted-by":"crossref","first-page":"1589","DOI":"10.1109\/JBHI.2017.2767063","article-title":"Deep EHR: a survey of recent advances in deep learning techniques for electronic health record (EHR) analysis","volume":"22","author":"Shickel","year":"2018","journal-title":"IEEE J Biomed Health Inform"},{"key":"2025021811401897700_ocaf002-B15","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1007\/s00392-016-1025-6","article-title":"Electronic health records to facilitate clinical research","volume":"106","author":"Cowie","year":"2017","journal-title":"Clin Res Cardiol"},{"key":"2025021811401897700_ocaf002-B16","doi-asserted-by":"crossref","first-page":"1628","DOI":"10.1056\/NEJMsa0900592","article-title":"Use of electronic health records in U.S. hospitals","volume":"360","author":"Jha","year":"2009","journal-title":"N Engl J Med"},{"key":"2025021811401897700_ocaf002-B17","doi-asserted-by":"crossref","first-page":"569","DOI":"10.1093\/jamia\/ocaa218","article-title":"Deep learning approaches for extracting adverse events and indications of dietary supplements from clinical text","volume":"28","author":"Fan","year":"2021","journal-title":"J Am Med Inform Assoc"},{"key":"2025021811401897700_ocaf002-B18","doi-asserted-by":"crossref","first-page":"2801","DOI":"10.1109\/JBHI.2020.3048700","article-title":"Multiple embeddings enhanced multi-graph neural networks for Chinese healthcare named entity recognition","volume":"25","author":"Lee","year":"2021","journal-title":"IEEE J Biomed Health Inform"},{"key":"2025021811401897700_ocaf002-B19","doi-asserted-by":"crossref","first-page":"2010","DOI":"10.1093\/jamia\/ocae147","article-title":"LEAP: LLM instruction-example adaptive prompting framework for biomedical relation extraction","volume":"31","author":"Zhou","year":"2024","journal-title":"J Am Med Inform Assoc"},{"key":"2025021811401897700_ocaf002-B20","author":"Li","year":"2024"},{"key":"2025021811401897700_ocaf002-B21","doi-asserted-by":"crossref","first-page":"1073","DOI":"10.1111\/imj.14442","article-title":"Ambiguous medical abbreviation study: challenges and opportunities","volume":"50","author":"Holper","year":"2020","journal-title":"Intern Med J"},{"key":"2025021811401897700_ocaf002-B22","doi-asserted-by":"crossref","first-page":"149","DOI":"10.1038\/s41597-021-00929-4","article-title":"A deep database of medical abbreviations and acronyms for natural language processing","volume":"8","author":"Grossman Liu","year":"2021","journal-title":"Sci Data"},{"key":"2025021811401897700_ocaf002-B23","doi-asserted-by":"crossref","first-page":"246","DOI":"10.1093\/jamiaopen\/ooz007","article-title":"Using word embeddings to expand terminology of dietary supplements on clinical notes","volume":"2","author":"Fan","year":"2019","journal-title":"JAMIA Open"},{"key":"2025021811401897700_ocaf002-B24","doi-asserted-by":"crossref","first-page":"539","DOI":"10.1093\/jamia\/ocz216","article-title":"iDISK: the integrated dietary supplements knowledge base","volume":"27","author":"Rizvi","year":"2020","journal-title":"J Am Med Inform Assoc"},{"key":"2025021811401897700_ocaf002-B25","first-page":"37","author":"Dang","year":"2020"},{"key":"2025021811401897700_ocaf002-B26","doi-asserted-by":"crossref","first-page":"153","DOI":"10.1186\/s12911-022-01888-5","article-title":"A conversational agent system for dietary supplements use","volume":"22","author":"Singh","year":"2022","journal-title":"BMC Med Inform Decis Mak"},{"key":"2025021811401897700_ocaf002-B27","author":"Schutte","year":"2021"},{"key":"2025021811401897700_ocaf002-B28","first-page":"513","author":"Zhou","year":"2021"},{"key":"2025021811401897700_ocaf002-B29","author":"Radford","year":"2018"},{"key":"2025021811401897700_ocaf002-B30","first-page":"9","article-title":"Language models are unsupervised multitask learners","volume":"1","author":"Radford","year":"2019","journal-title":"OpenAI Blog"},{"key":"2025021811401897700_ocaf002-B31","author":"Brown","year":"2020"},{"key":"2025021811401897700_ocaf002-B32","author":"Touvron","year":"2023"},{"key":"2025021811401897700_ocaf002-B33","doi-asserted-by":"crossref","first-page":"91","DOI":"10.58948\/2331-3528.2083","article-title":"Large language models: AI\u2019s legal revolution","volume":"44","author":"Bent","year":"2023","journal-title":"Pace Law Rev"},{"key":"2025021811401897700_ocaf002-B34","author":"Zhou","year":"2024"},{"key":"2025021811401897700_ocaf002-B35","first-page":"82","volume-title":"Proc Mach Learn Res","author":"Goel","year":"2023"},{"key":"2025021811401897700_ocaf002-B36","author":"Monajatipoor","year":"2024"},{"key":"2025021811401897700_ocaf002-B37","first-page":"17772","author":"Chen","year":"2024"},{"key":"2025021811401897700_ocaf002-B38","first-page":"4407","author":"Sun","year":"2024"},{"key":"2025021811401897700_ocaf002-B39","author":"Zhang","year":"2024"},{"key":"2025021811401897700_ocaf002-B40","doi-asserted-by":"crossref","first-page":"426","DOI":"10.1093\/jamia\/ocad216","article-title":"Complementary and integrative health information in the literature: its lexicon and named entity recognition","volume":"31","author":"Zhou","year":"2024","journal-title":"J Am Med Inform Assoc"},{"key":"2025021811401897700_ocaf002-B41","first-page":"14524","author":"Zhao","year":"2021"},{"key":"2025021811401897700_ocaf002-B42","author":"Zheng","year":"2017"},{"key":"2025021811401897700_ocaf002-B43","first-page":"6190","author":"Qin","year":"2022"},{"key":"2025021811401897700_ocaf002-B44","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1145\/3679013","article-title":"Multi-task learning in natural language processing: an overview","volume":"56","author":"Chen","year":"2024","journal-title":"ACM Comput Surv"},{"key":"2025021811401897700_ocaf002-B45","author":"Ruder","year":"2017"},{"key":"2025021811401897700_ocaf002-B46","first-page":"521","author":"Kang","year":"2011"},{"key":"2025021811401897700_ocaf002-B47","author":"Yin","year":"2024"},{"key":"2025021811401897700_ocaf002-B48","doi-asserted-by":"crossref","first-page":"2108","DOI":"10.1093\/jamia\/ocab126","article-title":"MT-clinical BERT: scaling clinical information extraction with multitask learning","volume":"28","author":"Mulyar","year":"2021","journal-title":"J Am Med Inform Assoc"},{"key":"2025021811401897700_ocaf002-B49","first-page":"9459","article-title":"Retrieval-augmented generation for knowledge-intensive NLP tasks","volume":"33","author":"Lewis","year":"2020","journal-title":"Adv Neural Inf Process Syst"},{"key":"2025021811401897700_ocaf002-B50","author":"Wu","year":"2024"},{"key":"2025021811401897700_ocaf002-B51","author":"Hou","year":"2024"},{"key":"2025021811401897700_ocaf002-B52","author":"Jiang","year":"2023"},{"key":"2025021811401897700_ocaf002-B53","author":"Hugging Face","year":"2024"},{"key":"2025021811401897700_ocaf002-B54","author":"Labrak","year":"2024"},{"key":"2025021811401897700_ocaf002-B55","doi-asserted-by":"crossref","first-page":"1833","DOI":"10.1093\/jamia\/ocae045","article-title":"PMC-LLaMA: toward building open-source language models for medicine","volume":"31","author":"Wu","year":"2024","journal-title":"J Am Med Inform Assoc"},{"key":"2025021811401897700_ocaf002-B56","author":"Han","year":"2023"},{"key":"2025021811401897700_ocaf002-B57","doi-asserted-by":"crossref","first-page":"btad651","DOI":"10.1093\/bioinformatics\/btad651","article-title":"MedCPT: contrastive pre-trained transformers with large-scale PubMed search logs for zero-shot biomedical information retrieval","volume":"39","author":"Jin","year":"2023","journal-title":"Bioinformatics"},{"key":"2025021811401897700_ocaf002-B58","author":"Izacard","year":"2021"},{"key":"2025021811401897700_ocaf002-B59","author":"Xu","year":"2024"},{"key":"2025021811401897700_ocaf002-B60","first-page":"22631","article-title":"The flan collection: designing data and methods for effective instruction tuning","volume":"941","author":"Longpre","year":"2023","journal-title":"Proc Mach Learn Res"},{"key":"2025021811401897700_ocaf002-B61","author":"Zhang","year":"2023"},{"key":"2025021811401897700_ocaf002-B62","author":"Hu","year":"2021"}],"container-title":["Journal of the American Medical Informatics Association"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/academic.oup.com\/jamia\/article-pdf\/32\/3\/545\/61415205\/ocaf002.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"syndication"},{"URL":"https:\/\/academic.oup.com\/jamia\/article-pdf\/32\/3\/545\/61415205\/ocaf002.pdf","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,2,18]],"date-time":"2025-02-18T11:40:39Z","timestamp":1739878839000},"score":1,"resource":{"primary":{"URL":"https:\/\/academic.oup.com\/jamia\/article\/32\/3\/545\/7951915"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,1,11]]},"references-count":62,"journal-issue":{"issue":"3","published-online":{"date-parts":[[2025,1,11]]},"published-print":{"date-parts":[[2025,3,1]]}},"URL":"https:\/\/doi.org\/10.1093\/jamia\/ocaf002","relation":{},"ISSN":["1067-5027","1527-974X"],"issn-type":[{"value":"1067-5027","type":"print"},{"value":"1527-974X","type":"electronic"}],"subject":[],"published-other":{"date-parts":[[2025,3]]},"published":{"date-parts":[[2025,1,11]]}}}