{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,6,3]],"date-time":"2026-06-03T11:12:53Z","timestamp":1780485173412,"version":"3.54.1"},"reference-count":37,"publisher":"Springer Science and Business Media LLC","issue":"1","license":[{"start":{"date-parts":[[2024,9,9]],"date-time":"2024-09-09T00:00:00Z","timestamp":1725840000000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by-nc-nd\/4.0"},{"start":{"date-parts":[[2024,9,9]],"date-time":"2024-09-09T00:00:00Z","timestamp":1725840000000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by-nc-nd\/4.0"}],"funder":[{"DOI":"10.13039\/100000092","name":"U.S. Department of Health & Human Services | NIH | U.S. National Library of Medicine","doi-asserted-by":"publisher","award":["R01LM014344"],"award-info":[{"award-number":["R01LM014344"]}],"id":[{"id":"10.13039\/100000092","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000002","name":"U.S. Department of Health & Human Services | National Institutes of Health","doi-asserted-by":"publisher","award":["UL1TR001873"],"award-info":[{"award-number":["UL1TR001873"]}],"id":[{"id":"10.13039\/100000002","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["npj Digit. Med."],"DOI":"10.1038\/s41746-024-01239-w","type":"journal-article","created":{"date-parts":[[2024,9,9]],"date-time":"2024-09-09T21:02:25Z","timestamp":1725915745000},"update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":65,"title":["Closing the gap between open source and commercial large language models for medical evidence summarization"],"prefix":"10.1038","volume":"7","author":[{"given":"Gongbo","family":"Zhang","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-1268-7239","authenticated-orcid":false,"given":"Qiao","family":"Jin","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0002-7457-7075","authenticated-orcid":false,"given":"Yiliang","family":"Zhou","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-8224-0424","authenticated-orcid":false,"given":"Song","family":"Wang","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Betina","family":"Idnay","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Yiming","family":"Luo","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Elizabeth","family":"Park","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-1418-3103","authenticated-orcid":false,"given":"Jordan G.","family":"Nestor","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Matthew E.","family":"Spotnitz","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-6900-5596","authenticated-orcid":false,"given":"Ali","family":"Soroush","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"suffix":"Jr.","given":"Thomas R.","family":"Campion","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-9998-916X","authenticated-orcid":false,"given":"Zhiyong","family":"Lu","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-9624-0214","authenticated-orcid":false,"given":"Chunhua","family":"Weng","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-9309-8331","authenticated-orcid":false,"given":"Yifan","family":"Peng","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2024,9,9]]},"reference":[{"key":"1239_CR1","doi-asserted-by":"publisher","first-page":"1593","DOI":"10.1038\/s41591-023-02366-9","volume":"29","author":"Y Peng","year":"2023","unstructured":"Peng, Y., Rousseau, J. F., Shortliffe, E. H. & Weng, C. AI-generated text may have a role in evidence-based medicine. Nat. Med. 29, 1593\u20131594 (2023).","journal-title":"Nat. Med."},{"key":"1239_CR2","doi-asserted-by":"publisher","first-page":"1887","DOI":"10.1056\/NEJM200006223422507","volume":"342","author":"J Concato","year":"2000","unstructured":"Concato, J., Shah, N. & Horwitz, R. I. Randomized, controlled trials, observational studies, and the hierarchy of research designs. N. Engl. J. Med. 342, 1887\u20131892 (2000).","journal-title":"N. Engl. J. Med."},{"key":"1239_CR3","doi-asserted-by":"publisher","first-page":"e012545","DOI":"10.1136\/bmjopen-2016-012545","volume":"7","author":"R Borah","year":"2017","unstructured":"Borah, R., Brown, A. W., Capers, P. L. & Kaiser, K. A. Analysis of the time and workers needed to conduct systematic reviews of medical interventions using data from the PROSPERO registry. BMJ Open 7, e012545 (2017).","journal-title":"BMJ Open"},{"key":"1239_CR4","unstructured":"ClinicalTrials.gov. U.S. National Library of Medicine. Available at: https:\/\/clinicaltrials.gov (Accessed: 4 September 2024)."},{"key":"1239_CR5","doi-asserted-by":"publisher","first-page":"790","DOI":"10.1016\/j.recesp.2021.06.016","volume":"74","author":"MJ Page","year":"2021","unstructured":"Page, M. J. et al. The PRISMA 2020 statement: an updated guideline for reporting systematic reviews. Rev. Esp. Cardiol. 74, 790\u2013799 (2021).","journal-title":"Rev. Esp. Cardiol."},{"key":"1239_CR6","first-page":"605","volume":"2021","author":"BC Wallace","year":"2021","unstructured":"Wallace, B. C., Saha, S., Soboczenski, F. & Marshall, I. J. Generating (Factual?) narrative summaries of RCTs: experiments with neural multi-document summarization. AMIA Jt. Summits Transl. Sci. Proc. 2021, 605\u2013614 (2021).","journal-title":"AMIA Jt. Summits Transl. Sci. Proc."},{"key":"1239_CR7","doi-asserted-by":"publisher","first-page":"158","DOI":"10.1038\/s41746-023-00896-7","volume":"6","author":"L Tang","year":"2023","unstructured":"Tang, L. et al. Evaluating large language models on medical evidence summarization. NPJ Digit. Med. 6, 158 (2023).","journal-title":"NPJ Digit. Med."},{"key":"1239_CR8","doi-asserted-by":"publisher","first-page":"35","DOI":"10.1613\/jair.991","volume":"17","author":"R Barzilay","year":"2002","unstructured":"Barzilay, R. & Elhadad, N. Inferring strategies for sentence ordering in multidocument news summarization. J. Artif. Intell. Res. 17, 35\u201355 (2002).","journal-title":"J. Artif. Intell. Res."},{"key":"1239_CR9","doi-asserted-by":"publisher","first-page":"938","DOI":"10.1093\/jamia\/ocv032","volume":"22","author":"R Pivovarov","year":"2015","unstructured":"Pivovarov, R. & Elhadad, N. Automated methods for the summarization of electronic health records. J. Am. Med. Inform. Assoc. 22, 938\u2013947 (2015).","journal-title":"J. Am. Med. Inform. Assoc."},{"key":"1239_CR10","doi-asserted-by":"publisher","first-page":"358","DOI":"10.1093\/bib\/bbm045","volume":"8","author":"P Zweigenbaum","year":"2007","unstructured":"Zweigenbaum, P., Demner-Fushman, D., Yu, H. & Cohen, K. B. Frontiers of biomedical text mining: current progress. Brief. Bioinform. 8, 358\u2013375 (2007).","journal-title":"Brief. Bioinform."},{"key":"1239_CR11","unstructured":"Li, F. et al. Structure-aware review mining and summarization. In Proc. 23rd International Conference on Computational Linguistics (Coling 2010) (eds. Huang, C.-R. & Jurafsky, D.) 653\u2013661 (Coling 2010 Organizing Committee, Beijing, 2010)."},{"key":"1239_CR12","doi-asserted-by":"publisher","first-page":"63","DOI":"10.1162\/coli.2007.33.1.63","volume":"33","author":"D Demner-Fushman","year":"2007","unstructured":"Demner-Fushman, D. & Lin, J. J. Answering clinical questions with knowledge-based and statistical techniques. Comput. Linguist. 33, 63\u2013103 (2007).","journal-title":"Comput. Linguist."},{"key":"1239_CR13","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1145\/3458754","volume":"3","author":"Y Gu","year":"2021","unstructured":"Gu, Y. et al. Domain-specific language model pretraining for biomedical natural language processing. ACM Trans. Comput. Healthc. 3, 1\u201323 (2021).","journal-title":"ACM Trans. Comput. Healthc."},{"key":"1239_CR14","doi-asserted-by":"publisher","unstructured":"Guo, M. et al. LongT5: Efficient Text-To-Text Transformer for Long Sequences. Findings of the Association for Computational Linguistics: NAACL 2022, 724\u2013736. https:\/\/doi.org\/10.18653\/v1\/2022.findings-naacl.55 (2022).","DOI":"10.18653\/v1\/2022.findings-naacl.55"},{"key":"1239_CR15","doi-asserted-by":"crossref","unstructured":"Xiao, W., Beltagy, I., Carenini, G. & Cohan, A. PRIMERA: pyramid-based masked sentence pre-training for multi-document summarization. In Proc. of the 60th Annual Meeting of the Association for Computational Linguistics (Vol 1: Long Papers) 5245\u20135263 (ACL 2022).","DOI":"10.18653\/v1\/2022.acl-long.360"},{"key":"1239_CR16","unstructured":"Zhang, J., Zhao, Y., Saleh, M. & Liu, P. PEGASUS: pre-training with extracted gap-sentences for abstractive summarization. In Proc. 37th International Conference on Machine Learning (eds. Iii, H. D. & Singh, A.) 11328\u201311339 (PMLR, 2020)."},{"key":"1239_CR17","doi-asserted-by":"publisher","unstructured":"Lewis, M. et al. BART: Denoising Sequence-to-Sequence Pre-training for Natural Language Generation, Translation, and Comprehension. In Proc. of the 58th Annual Meeting of the Association for Computational Linguistics 7871\u20137880. (ACL 2020), https:\/\/doi.org\/10.18653\/v1\/2020.acl-main.703.","DOI":"10.18653\/v1\/2020.acl-main.703"},{"key":"1239_CR18","doi-asserted-by":"publisher","unstructured":"Devlin, J., Chang, M.-W., Lee, K., & Toutanova, K. BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding. In Proc. of the 2019 Conference of the North American Chapter of the Association for Computational Linguistics 4171\u20134186. (NAACL-HLT 2019), https:\/\/doi.org\/10.18653\/v1\/n19-1423.","DOI":"10.18653\/v1\/n19-1423"},{"key":"1239_CR19","doi-asserted-by":"crossref","unstructured":"Mrabet, Y. & Demner-Fushman, D. HOLMS: alternative summary evaluation with large language models. In Proc. 28th International Conference on Computational Linguistics (eds. Scott, D., Bel, N. & Zong, C.) 5679\u20135688 (International Committee on Computational Linguistics, Barcelona, Spain (Online), 2020).","DOI":"10.18653\/v1\/2020.coling-main.498"},{"key":"1239_CR20","doi-asserted-by":"publisher","first-page":"172","DOI":"10.1038\/s41586-023-06291-2","volume":"620","author":"K Singhal","year":"2023","unstructured":"Singhal, K. et al. Large language models encode clinical knowledge. Nature 620, 172\u2013180 (2023).","journal-title":"Nature"},{"key":"1239_CR21","doi-asserted-by":"publisher","first-page":"e12","DOI":"10.1016\/S2589-7500(23)00225-X","volume":"6","author":"T Zack","year":"2024","unstructured":"Zack, T. et al. Assessing the potential of GPT-4 to perpetuate racial and gender biases in health care: a model evaluation study. Lancet Digit. Health 6, e12\u2013e22 (2024).","journal-title":"Lancet Digit. Health"},{"key":"1239_CR22","doi-asserted-by":"publisher","first-page":"btae075","DOI":"10.1093\/bioinformatics\/btae075","volume":"40","author":"Q Jin","year":"2024","unstructured":"Jin, Q., Yang, Y., Chen, Q. & Lu, Z. GeneGPT: Augmenting large language models with domain tools for improved access to biomedical information. Bioinform. 40, btae075 (2024).","journal-title":"Bioinform."},{"key":"1239_CR23","doi-asserted-by":"publisher","unstructured":"Jiang, A. Q. et al. Mixtral of experts. Preprint at https:\/\/doi.org\/10.48550\/arXiv.2401.04088 (2024).","DOI":"10.48550\/arXiv.2401.04088"},{"key":"1239_CR24","first-page":"27730","volume":"35","author":"L Ouyang","year":"2022","unstructured":"Ouyang, L. et al. Training language models to follow instructions with human feedback. Adv Neural Inf Process Syst 35, 27730\u201327744 (2022).","journal-title":"Adv Neural Inf Process Syst"},{"key":"1239_CR25","doi-asserted-by":"publisher","unstructured":"Touvron, H. et al. Llama 2: open foundation and fine-tuned chat models. Preprint at https:\/\/doi.org\/10.48550\/arXiv.2307.09288 (2023).","DOI":"10.48550\/arXiv.2307.09288"},{"key":"1239_CR26","doi-asserted-by":"publisher","unstructured":"OpenAI, R. Gpt-4 technical report. Preprint at https:\/\/doi.org\/10.48550\/arxiv.2303.08774 (2023).","DOI":"10.48550\/arxiv.2303.08774"},{"key":"1239_CR27","doi-asserted-by":"publisher","first-page":"1422","DOI":"10.1126\/science.aab2374","volume":"348","author":"BA Nosek","year":"2015","unstructured":"Nosek, B. A. et al. Promoting an open research culture. Science 348, 1422\u20131425 (2015).","journal-title":"Science"},{"key":"1239_CR28","doi-asserted-by":"publisher","first-page":"104640","DOI":"10.1016\/j.jbi.2024.104640","volume":"153","author":"G Zhang","year":"2024","unstructured":"Zhang, G. et al. Leveraging generative AI for clinical evidence synthesis needs to ensure trustworthiness. J. Biomed. Inform. 153, 104640 (2024).","journal-title":"J. Biomed. Inform."},{"key":"1239_CR29","doi-asserted-by":"publisher","unstructured":"Gutierrez, B. J., et al. Thinking about GPT-3 In-Context Learning for Biomedical IE? Think Again. Findings of the Association for Computational Linguistics: EMNLP 2022, 4497\u20134512. https:\/\/doi.org\/10.18653\/v1\/2022.findings-emnlp.329 (2022).","DOI":"10.18653\/v1\/2022.findings-emnlp.329"},{"key":"1239_CR30","doi-asserted-by":"publisher","DOI":"10.1038\/s41467-022-34938-7","volume":"13","author":"T Tadros","year":"2022","unstructured":"Tadros, T., Krishnan, G. P., Ramyaa, R. & Bazhenov, M. Sleep-like unsupervised replay reduces catastrophic forgetting in artificial neural networks. Nat. Commun. 13, 7742 (2022).","journal-title":"Nat. Commun."},{"key":"1239_CR31","unstructured":"Hu, E. J. et al. LoRA: Low-Rank Adaptation of Large Language Models. The Tenth International Conference on Learning Representations (ICLR 2022)."},{"key":"1239_CR32","unstructured":"The Cochrane Library. https:\/\/www.cochranelibrary.com\/."},{"key":"1239_CR33","doi-asserted-by":"publisher","first-page":"391","DOI":"10.1162\/tacl_a_00373","volume":"9","author":"AR Fabbri","year":"2021","unstructured":"Fabbri, A. R. et al. SummEval: Re-evaluating Summarization Evaluation. Trans. Assoc. Comput. Linguistics 9, 391\u2013409 (2021).","journal-title":"Trans. Assoc. Comput. Linguistics"},{"key":"1239_CR34","unstructured":"Wolf, T. et al. Transformers: state-of-the-art natural language processing. In Proc. 2020 Conference on Empirical Methods in Natural Language Processing: System Demonstrations (eds. Liu, Q. & Schlangen, D.) 38\u201345 (Association for Computational Linguistics, Online, 2020)."},{"key":"1239_CR35","unstructured":"Paszke, A. et al. Automatic differentiation in PyTorch (2017)."},{"key":"1239_CR36","unstructured":"Mangrulkar, S. et al. PEFT: State-of-the-Art Parameter-Efficient Fine-Tuning Methods. https:\/\/github.com\/huggingface\/peft (2022)."},{"key":"1239_CR37","doi-asserted-by":"publisher","first-page":"1163","DOI":"10.1093\/jamia\/ocae065","volume":"31","author":"G Zhang","year":"2024","unstructured":"Zhang, G. et al. A span-based model for extracting overlapping PICO entities from randomized controlled trial publications. J. Am. Med. Inform. Assoc. 31, 1163\u20131171 (2024).","journal-title":"J. Am. Med. Inform. Assoc."}],"container-title":["npj Digital Medicine"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/www.nature.com\/articles\/s41746-024-01239-w.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/www.nature.com\/articles\/s41746-024-01239-w","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/www.nature.com\/articles\/s41746-024-01239-w.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,9,9]],"date-time":"2024-09-09T21:15:44Z","timestamp":1725916544000},"score":1,"resource":{"primary":{"URL":"https:\/\/www.nature.com\/articles\/s41746-024-01239-w"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,9,9]]},"references-count":37,"journal-issue":{"issue":"1","published-online":{"date-parts":[[2024,12]]}},"alternative-id":["1239"],"URL":"https:\/\/doi.org\/10.1038\/s41746-024-01239-w","relation":{},"ISSN":["2398-6352"],"issn-type":[{"value":"2398-6352","type":"electronic"}],"subject":[],"published":{"date-parts":[[2024,9,9]]},"assertion":[{"value":"15 February 2024","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"29 August 2024","order":2,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"9 September 2024","order":3,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"The authors declare no competing interests.","order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Competing interests"}}],"article-number":"239"}}