{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,23]],"date-time":"2026-10-23T03:41:09Z","timestamp":1792726869375,"version":"4.5.0"},"reference-count":24,"publisher":"Oxford University Press (OUP)","issue":"5","license":[{"start":{"date-parts":[[2026,10,23]],"date-time":"2026-10-23T00:00:00Z","timestamp":1792713600000},"content-version":"vor","delay-in-days":44,"URL":"https:\/\/creativecommons.org\/licenses\/by-nc\/4.0\/"}],"funder":[{"DOI":"10.13039\/100006108","name":"National Center for Advancing Translational Sciences","doi-asserted-by":"publisher","award":["TL1TR002344"],"award-info":[{"award-number":["TL1TR002344"]}],"id":[{"id":"10.13039\/100006108","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2026,9,9]]},"abstract":"<jats:title>Abstract<\/jats:title>\n                  <jats:sec>\n                    <jats:title>Objectives<\/jats:title>\n                    <jats:p>To evaluate performance and efficiency of a lightweight open-source reasoning large language model (LLM) against an expert-derived reference standard for screening scientific publication databases.<\/jats:p>\n                  <\/jats:sec>\n                  <jats:sec>\n                    <jats:title>Materials and Methods<\/jats:title>\n                    <jats:p>Two expert reviewers screened subsets of database records across 3 stages: (1) base eligibility (n\u2009=\u2009200), (2) obesity focus (n\u2009=\u2009200), and (3) women of reproductive age focus (n\u2009=\u2009150). The LLM (gpt-oss-20b) screened all records (n\u2009=\u200920\u00a0118). Performance (F1-score and sensitivity [recall]) and efficiency (average screening rate) were assessed.<\/jats:p>\n                  <\/jats:sec>\n                  <jats:sec>\n                    <jats:title>Results<\/jats:title>\n                    <jats:p>Interreader agreement was near-perfect (\u03ba: 0.99-1.00; all P\u2009&amp;lt;\u2009.001). LLM performance was high: F1-score, 0.83-0.94; sensitivity, 90%-95%; specificity, 86%-99%; accuracy, 89%-94%; precision, 77%-99%; and negative predictive value (NPV), 89%-96%. LLM screening was faster than reviewers (19.4 vs \u223c1.0 records\/min).<\/jats:p>\n                  <\/jats:sec>\n                  <jats:sec>\n                    <jats:title>Discussion<\/jats:title>\n                    <jats:p>The LLM achieved high performance and efficiency relative to an expert-derived reference standard for screening.<\/jats:p>\n                  <\/jats:sec>\n                  <jats:sec>\n                    <jats:title>Conclusion<\/jats:title>\n                    <jats:p>The use of lightweight open-source reasoning LLMs shows promise to support screening in evidence synthesis workflows.<\/jats:p>\n                  <\/jats:sec>","DOI":"10.1093\/jamiaopen\/ooag210","type":"journal-article","created":{"date-parts":[[2026,10,23]],"date-time":"2026-10-23T03:32:38Z","timestamp":1792726358000},"source":"Crossref","is-referenced-by-count":0,"title":["Lightweight open-source reasoning large language model for evidence synthesis screening"],"prefix":"10.1093","volume":"9","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-9951-2270","authenticated-orcid":false,"given":"David M","family":"D\u00e1vila-Garc\u00eda","sequence":"first","affiliation":[{"name":"Institute for Informatics, Data Science & Biostatistics, Washington University School of Medicine in St. Louis , St. Louis, MO 63110,","place":["United States"]}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-6948-0127","authenticated-orcid":false,"given":"Amanda","family":"Gilbert","sequence":"additional","affiliation":[{"name":"Prevention Research Center, Washington University Bursky School of Public Health , St. Louis, MO 63130,","place":["United States"]}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Adam B","family":"Wilcox","sequence":"additional","affiliation":[{"name":"Institute for Informatics, Data Science & Biostatistics, Washington University School of Medicine in St. Louis , St. Louis, MO 63110,","place":["United States"]}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"286","published-online":{"date-parts":[[2026,10,23]]},"reference":[{"key":"2026102223323347900_ooag210-B1","doi-asserted-by":"publisher","first-page":"133","DOI":"10.1016\/j.jclinepi.2017.10.014","article-title":"Same family, different species: methodological conduct and quality varies according to purpose for five types of knowledge synthesis","volume":"96","author":"Tricco","year":"2018","journal-title":"J Clin Epidemiol"},{"key":"2026102223323347900_ooag210-B2","doi-asserted-by":"publisher","author":"Borah","year":"2017","DOI":"10.1136\/bmjopen-2016-012545"},{"key":"2026102223323347900_ooag210-B3","doi-asserted-by":"publisher","first-page":"287","DOI":"10.1016\/j.jclinepi.2021.05.019","article-title":"Resource use during systematic review production varies widely: a scoping review","volume":"139","author":"Nussbaumer-Streit","year":"2021","journal-title":"J Clin Epidemiol"},{"key":"2026102223323347900_ooag210-B4","doi-asserted-by":"publisher","first-page":"95","DOI":"10.1186\/s40001-022-00717-9","article-title":"Changing trends in clinical research literature on PubMed database from 1991 to 2020","volume":"27","author":"Zhao","year":"2022","journal-title":"Eur J Med Res"},{"key":"2026102223323347900_ooag210-B5","doi-asserted-by":"publisher","first-page":"142","DOI":"10.1186\/s12874-021-01335-5","article-title":"Evaluating the relationship between citation set size, team size and screening methods used in systematic reviews: a cross-sectional study","volume":"21","author":"O\u2019Hearn","year":"2021","journal-title":"BMC Med Res Methodol"},{"key":"2026102223323347900_ooag210-B6","volume-title":"Responsible Use of AI in Evidence SynthEsis (RAISE): Recommendations and Guidance","author":"Thomas","year":"2026"},{"key":"2026102223323347900_ooag210-B7"},{"key":"2026102223323347900_ooag210-B8","doi-asserted-by":"publisher","first-page":"210","DOI":"10.1186\/s13643-016-0384-4","article-title":"Rayyan\u2014a web and mobile app for systematic reviews","volume":"5","author":"Ouzzani","year":"2016","journal-title":"Syst Rev"},{"key":"2026102223323347900_ooag210-B9","doi-asserted-by":"publisher","first-page":"125","DOI":"10.1038\/s42256-020-00287-7","article-title":"An open source machine learning framework for efficient and transparent systematic reviews","volume":"3","author":"van de Schoot","year":"2021","journal-title":"Nat Mach Intell"},{"key":"2026102223323347900_ooag210-B10","doi-asserted-by":"publisher","first-page":"109808","DOI":"10.1016\/j.compbiomed.2025.109808","article-title":"Natural language processing techniques applied to the electronic health record in clinical research and practice\u2014an introduction to methodologies","volume":"188","author":"Clay","year":"2025","journal-title":"Comput Biol Med"},{"key":"2026102223323347900_ooag210-B11","first-page":"518","article-title":"Exploring ChatGPT 3.5 for structured data extraction from oncological notes","volume":"2025","author":"Skyles","year":"2025","journal-title":"AMIA Jt Summits Transl Sci Proc"},{"key":"2026102223323347900_ooag210-B12","doi-asserted-by":"publisher","first-page":"e797\u20138","DOI":"10.1016\/j.ijrobp.2025.06.3860","article-title":"Leveraging a large language model to extract structured data elements from unstructured radiology and pathology reports","volume":"123","author":"Zhao","year":"2025","journal-title":"Int J Radiat Oncol"},{"key":"2026102223323347900_ooag210-B13","author":"Elsevier Research Data Services","year":"2026"},{"key":"2026102223323347900_ooag210-B14","doi-asserted-by":"publisher","first-page":"ooaf182","DOI":"10.1093\/jamiaopen\/ooaf182","article-title":"Lightweight open-source large language models versus cTAKES for information extraction from discharge summaries: tobacco smoking status test case","volume":"9","author":"D\u00e1vila-Garc\u00eda","year":"2026","journal-title":"JAMIA Open"},{"key":"2026102223323347900_ooag210-B15","doi-asserted-by":"publisher","DOI":"10.7936\/e792-d231","article-title":"Supporting health equity in obesity prevention for women of reproductive age","author":"Gilbert","year":"2025","journal-title":"Brown Sch Theses Diss"},{"key":"2026102223323347900_ooag210-B16","doi-asserted-by":"publisher","first-page":"83","DOI":"10.1186\/s13643-023-02240-2","article-title":"Protocol for a scoping review of health equity frameworks and models applied in empirical studies of chronic disease prevention and control","volume":"12","author":"Walsh-Bailey","year":"2023","journal-title":"Syst Rev"},{"key":"2026102223323347900_ooag210-B17","year":"2025"},{"key":"2026102223323347900_ooag210-B18","author":"CDC","year":"2025"},{"key":"2026102223323347900_ooag210-B19","doi-asserted-by":"publisher","first-page":"99","DOI":"10.1016\/S1532-0464(02)00500-2","article-title":"Measuring agreement in medical informatics reliability studies","volume":"35","author":"Hripcsak","year":"2002","journal-title":"J Biomed Inform"},{"key":"2026102223323347900_ooag210-B20","author":"OpenAI","year":"2025"},{"key":"2026102223323347900_ooag210-B21","doi-asserted-by":"publisher","first-page":"100931","DOI":"10.1016\/j.landig.2025.100931","article-title":"Reasoning-driven large language models in medicine: opportunities, challenges, and the road ahead","volume":"8","author":"Wang","year":"2026","journal-title":"Lancet Digit Health"},{"key":"2026102223323347900_ooag210-B22","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1136\/jamia.2002.0090001","article-title":"Reference standards, judges, and comparison subjects: roles for experts in evaluating system performance","volume":"9","author":"Hripcsak","year":"2002","journal-title":"J Am Med Inform Assoc"},{"key":"2026102223323347900_ooag210-B23","author":"Adam","year":"2026"},{"key":"2026102223323347900_ooag210-B24","doi-asserted-by":"publisher","first-page":"337","DOI":"10.1016\/S0263-7863(98)00069-6","article-title":"Project management: cost, time and quality, two best guesses and a phenomenon, its time to accept other success criteria","volume":"17","author":"Atkinson","year":"1999","journal-title":"Int J Proj Manag"}],"container-title":["JAMIA Open"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/academic.oup.com\/jamiaopen\/article-pdf\/9\/5\/ooag210\/72376942\/ooag210.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"syndication"},{"URL":"https:\/\/academic.oup.com\/jamiaopen\/article-pdf\/9\/5\/ooag210\/72376942\/ooag210.pdf","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,10,23]],"date-time":"2026-10-23T03:32:47Z","timestamp":1792726367000},"score":1,"resource":{"primary":{"URL":"https:\/\/academic.oup.com\/jamiaopen\/article\/doi\/10.1093\/jamiaopen\/ooag210\/8980878"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026,9,9]]},"references-count":24,"journal-issue":{"issue":"5","published-print":{"date-parts":[[2026,9,9]]}},"URL":"https:\/\/doi.org\/10.1093\/jamiaopen\/ooag210","relation":{},"ISSN":["2574-2531"],"issn-type":[{"value":"2574-2531","type":"electronic"}],"subject":[],"published-other":{"date-parts":[[2026,10]]},"published":{"date-parts":[[2026,9,9]]},"article-number":"ooag210"}}