{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,26]],"date-time":"2026-10-26T23:32:30Z","timestamp":1793057550893,"version":"4.5.0"},"reference-count":59,"publisher":"Massachusetts Medical Society","issue":"2","content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["NEJM AI"],"published-print":{"date-parts":[[2024,1,25]]},"DOI":"10.1056\/aioa2300068","type":"journal-article","created":{"date-parts":[[2024,1,25]],"date-time":"2024-01-25T09:00:22Z","timestamp":1706173222000},"source":"Crossref","is-referenced-by-count":438,"title":["Almanac \u2014 Retrieval-Augmented Language Models for Clinical Medicine"],"prefix":"10.1056","volume":"1","author":[{"ORCID":"https:\/\/orcid.org\/0000-0001-8446-2349","authenticated-orcid":false,"given":"Cyril","family":"Zakka","sequence":"first","affiliation":[{"name":"Department of Cardiothoracic Surgery, Stanford Medicine, Stanford, CA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-0453-9041","authenticated-orcid":false,"given":"Rohan","family":"Shad","sequence":"additional","affiliation":[{"name":"Division of Cardiovascular Surgery, Penn Medicine, Philadelphia"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-3438-3313","authenticated-orcid":false,"given":"Akash","family":"Chaurasia","sequence":"additional","affiliation":[{"name":"Department of Computer Science, Stanford University, Stanford, CA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-9993-3817","authenticated-orcid":false,"given":"Alex R.","family":"Dalal","sequence":"additional","affiliation":[{"name":"Department of Cardiothoracic Surgery, Stanford Medicine, Stanford, CA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-2313-2685","authenticated-orcid":false,"given":"Jennifer L.","family":"Kim","sequence":"additional","affiliation":[{"name":"Department of Cardiothoracic Surgery, Stanford Medicine, Stanford, CA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-4911-6437","authenticated-orcid":false,"given":"Michael","family":"Moor","sequence":"additional","affiliation":[{"name":"Department of Computer Science, Stanford University, Stanford, CA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-7953-5109","authenticated-orcid":false,"given":"Robyn","family":"Fong","sequence":"additional","affiliation":[{"name":"Department of Cardiothoracic Surgery, Stanford Medicine, Stanford, CA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0008-8316-6765","authenticated-orcid":false,"given":"Curran","family":"Phillips","sequence":"additional","affiliation":[{"name":"Department of Cardiothoracic Surgery, Stanford Medicine, Stanford, CA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-4024-3691","authenticated-orcid":false,"given":"Kevin","family":"Alexander","sequence":"additional","affiliation":[{"name":"Division of Cardiovascular Medicine, Stanford Medicine, Stanford, CA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-9418-9577","authenticated-orcid":false,"given":"Euan","family":"Ashley","sequence":"additional","affiliation":[{"name":"Division of Cardiovascular Medicine, Stanford Medicine, Stanford, CA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-3020-5280","authenticated-orcid":false,"given":"Jack","family":"Boyd","sequence":"additional","affiliation":[{"name":"Department of Cardiothoracic Surgery, Stanford Medicine, Stanford, CA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0007-9366-754X","authenticated-orcid":false,"given":"Kathleen","family":"Boyd","sequence":"additional","affiliation":[{"name":"Department of Pediatrics, Stanford Medicine, Stanford, CA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-5474-3243","authenticated-orcid":false,"given":"Karen","family":"Hirsch","sequence":"additional","affiliation":[{"name":"Department of Neurology, Stanford Medicine, Stanford, CA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-8972-8051","authenticated-orcid":false,"given":"Curt","family":"Langlotz","sequence":"additional","affiliation":[{"name":"Department of Radiology and Biomedical Informatics, Stanford Medicine, Stanford, CA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0007-8868-6447","authenticated-orcid":false,"given":"Rita","family":"Lee","sequence":"additional","affiliation":[{"name":"Department of Cardiothoracic Surgery, Stanford Medicine, Stanford, CA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-0984-4600","authenticated-orcid":false,"given":"Joanna","family":"Melia","sequence":"additional","affiliation":[{"name":"Division of Gastroenterology and Hepatology, Johns Hopkins Medicine, Baltimore"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-2152-2847","authenticated-orcid":false,"given":"Joanna","family":"Nelson","sequence":"additional","affiliation":[{"name":"Division of Infectious Diseases, Stanford Medicine, Stanford, CA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-3469-6571","authenticated-orcid":false,"given":"Karim","family":"Sallam","sequence":"additional","affiliation":[{"name":"Division of Cardiovascular Medicine, Stanford Medicine, Stanford, CA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0000-8950-5790","authenticated-orcid":false,"given":"Stacey","family":"Tullis","sequence":"additional","affiliation":[{"name":"Department of Cardiothoracic Surgery, Stanford Medicine, Stanford, CA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-0601-3542","authenticated-orcid":false,"given":"Melissa Ann","family":"Vogelsong","sequence":"additional","affiliation":[{"name":"Division of Anesthesia, Stanford Medicine, Stanford, CA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-9125-3027","authenticated-orcid":false,"given":"John Patrick","family":"Cunningham","sequence":"additional","affiliation":[{"name":"Department of Statistics, Columbia University, New York"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-3548-2578","authenticated-orcid":false,"given":"William","family":"Hiesinger","sequence":"additional","affiliation":[{"name":"Department of Cardiothoracic Surgery, Stanford Medicine, Stanford, CA"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"150","reference":[{"key":"e_1_3_5_2_2","unstructured":"Brown TB Mann B Ryder N et\u00a0al. Language models are few-shot learners. July 22 2020 (https:\/\/arxiv.org\/abs\/2005.14165). Preprint."},{"key":"e_1_3_5_3_2","unstructured":"Chen M Tworek J Jun H et\u00a0al. Evaluating large language models trained on code. July 14 2021 (https:\/\/arxiv.org\/abs\/2107.03374). Preprint."},{"key":"e_1_3_5_4_2","unstructured":"Wei C Xie SM Ma T. Why do pretrained language models help in downstream tasks? An analysis of head and prompt tuning. June 2021 (https:\/\/arxiv.org\/abs\/2106.09226). Preprint."},{"key":"e_1_3_5_5_2","unstructured":"Devlin J Chang M-W Lee K Toutanova K. BERT: pre-training of deep bidirectional transformers for language understanding. October 2018 (https:\/\/arxiv.org\/abs\/1810.04805). Preprint."},{"key":"e_1_3_5_6_2","unstructured":"Wei J Tay Y Bommasani R et\u00a0al. Emergent abilities of large language models. June 2022 (https:\/\/arxiv.org\/abs\/2206.07682). Preprint."},{"key":"e_1_3_5_7_2","unstructured":"Hoffmann J Borgeaud S Mensch A et\u00a0al. Training compute-optimal large language models. March 2022 (https:\/\/arxiv.org\/abs\/2203.15556). Preprint."},{"key":"e_1_3_5_8_2","unstructured":"Rae JW Borgeaud S Cai T et\u00a0al. Scaling language models: methods analysis and insights from Training Gopher. December 2021 (https:\/\/arxiv.org\/abs\/2112.11446). Preprint."},{"key":"e_1_3_5_9_2","doi-asserted-by":"publisher","DOI":"10.1145\/3571730"},{"key":"e_1_3_5_10_2","doi-asserted-by":"crossref","unstructured":"Raunak V Menezes A Junczys-Dowmunt M. The curious case of hallucinations in neural machine translation. April 2021 (https:\/\/arxiv.org\/abs\/2104.06683). Preprint.","DOI":"10.18653\/v1\/2021.naacl-main.92"},{"key":"e_1_3_5_11_2","unstructured":"Liang PP Wu C Morency L-P Salakhutdinov R. Towards understanding and mitigating social biases in language models. June 2021 (https:\/\/arxiv.org\/abs\/2106.13219). Preprint."},{"key":"e_1_3_5_12_2","doi-asserted-by":"crossref","unstructured":"Swinger N De-Arteaga M Heffernan NT Leiserson MD Kalai AT. What are the biases in my word embedding? December 2018 (https:\/\/arxiv.org\/abs\/1812.08769). Preprint.","DOI":"10.1145\/3306618.3314270"},{"key":"e_1_3_5_13_2","unstructured":"Ouyang L Wu J Jiang X et\u00a0al. Training language models to follow instructions with human feedback. March 2022 (https:\/\/arxiv.org\/abs\/2203.02155). Preprint."},{"key":"e_1_3_5_14_2","unstructured":"Bai Y Kadavath S Kundu S et\u00a0al. Constitutional AI: harmlessness from AI feedback. December 2022 (https:\/\/arxiv.org\/abs\/2212.08073). Preprint."},{"key":"e_1_3_5_15_2","unstructured":"Zhou Y Muresanu AI Han Z et\u00a0al. Large language models are human-level prompt engineers. November 2022 (https:\/\/arxiv.org\/abs\/2211.01910). Preprint."},{"key":"e_1_3_5_16_2","doi-asserted-by":"crossref","unstructured":"Reynolds L McDonell K. Prompt programming for large language models: beyond the few-shot paradigm. February 2021 (https:\/\/arxiv.org\/abs\/2102.07350). Preprint.","DOI":"10.1145\/3411763.3451760"},{"key":"e_1_3_5_17_2","unstructured":"Thoppilan R De Freitas D Hall J et\u00a0al. LaMDA: language models for dialog applications. January 2022 (https:\/\/arxiv.org\/abs\/2201.08239). Preprint."},{"key":"e_1_3_5_18_2","doi-asserted-by":"crossref","unstructured":"Hartvigsen T Gabriel S Palangi H Sap M Ray D Kamar E. ToxiGen: a large-scale machine-generated dataset for adversarial and implicit hate speech detection. March2022 (https:\/\/arxiv.org\/abs\/2203.09509). Preprint.","DOI":"10.18653\/v1\/2022.acl-long.234"},{"key":"e_1_3_5_19_2","unstructured":"Carlini N Tramer F Wallace E et\u00a0al. Extracting training data from large language models. December 2020 (https:\/\/arxiv.org\/abs\/2012.07805). Preprint."},{"key":"e_1_3_5_20_2","unstructured":"Christiano P Leike J Brown TB Martic M Legg S Amodei D. Deep reinforcement learning from human preferences. February 2023 (https:\/\/arxiv.org\/abs\/1706.03741)."},{"key":"e_1_3_5_21_2","doi-asserted-by":"publisher","DOI":"10.1111\/j.1525-1497.2004.30306.x"},{"key":"e_1_3_5_22_2","doi-asserted-by":"publisher","DOI":"10.1093\/bib\/bbac409"},{"key":"e_1_3_5_23_2","doi-asserted-by":"crossref","unstructured":"Beltagy I Lo K Cohan A. SciBERT: a pretrained language model for scientific text. In: Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing and the 9th International Joint Conference on Natural Language Processing (EMNLP-IJCNLP).2019;3615-3620. DOI: 10.18653\/v1\/D19-1371.","DOI":"10.18653\/v1\/D19-1371"},{"key":"e_1_3_5_24_2","doi-asserted-by":"crossref","unstructured":"Shin H-C Zhang Y Bakhturina E et\u00a0al. BioMegatron: larger biomedical domain language model. October 2020 (https:\/\/arxiv.org\/abs\/2010.06060). Preprint.","DOI":"10.18653\/v1\/2020.emnlp-main.379"},{"key":"e_1_3_5_25_2","doi-asserted-by":"publisher","DOI":"10.1093\/bioinformatics\/btz682"},{"key":"e_1_3_5_26_2","doi-asserted-by":"publisher","DOI":"10.1145\/3458754"},{"key":"e_1_3_5_27_2","unstructured":"Papanikolaou Y Pierleoni A. DARE: data augmented relation extraction with GPT-2. April 2020 (https:\/\/arxiv.org\/abs\/2004.13845). Preprint."},{"key":"e_1_3_5_28_2","unstructured":"Hong Z Ajith A Pauloski G et\u00a0al. ScholarBERT: bigger is not always better. May 2022 (https:\/\/arxiv.org\/abs\/2205.11342). Preprint."},{"key":"e_1_3_5_29_2","unstructured":"Lehman E Hernandez E Mahajan D et\u00a0al. Do we still need clinical language models? February 2023 (https:\/\/arxiv.org\/abs\/2302.08091). Preprint."},{"key":"e_1_3_5_30_2","unstructured":"Yang X Chen A PourNejatian N et\u00a0al. GatorTron: a large clinical language model to unlock patient information from unstructured electronic health records. December 2022 (https:\/\/arxiv.org\/abs\/2203.03540). Preprint."},{"key":"e_1_3_5_31_2","unstructured":"Singhal K Azizi S Tu T et\u00a0al. Large language models encode clinical knowledge. December 2022 (https:\/\/arxiv.org\/abs\/2212.13138). Preprint."},{"key":"e_1_3_5_32_2","unstructured":"Taylor R Kardas M Cucurull G et\u00a0al. Galactica: a large language model for science. November 2022 (https:\/\/arxiv.org\/abs\/2211.09085). Preprint."},{"key":"e_1_3_5_33_2","unstructured":"Lewis P Perez E Piktus A et\u00a0al. Retrieval-augmented generation for knowledge-intensive NLP tasks. NIPS\u201920: Proceedings of the 34th International Conference on Neural Information Processing Systems December 2020 (https:\/\/proceedings.neurips.cc\/paper\/2020\/file\/6b493230205f780e1bc26945df7481e5-Paper.pdf)."},{"key":"e_1_3_5_34_2","unstructured":"Borgeaud S Mensch A Hoffmann J et\u00a0al. Improving language models by retrieving from trillions of tokens. International Conference on Machine Learning (ICML) 2022."},{"key":"e_1_3_5_35_2","doi-asserted-by":"publisher","DOI":"10.1186\/s12859-019-2607-x"},{"key":"e_1_3_5_36_2","doi-asserted-by":"crossref","unstructured":"Naik A Parasa S Feldman S Wang LL Hope T. Literature augmented clinical outcome prediction. November 2022 (https:\/\/arxiv.org\/abs\/2111.08374).","DOI":"10.18653\/v1\/2022.findings-naacl.33"},{"key":"e_1_3_5_37_2","doi-asserted-by":"publisher","DOI":"10.1162\/tacl_a_00605"},{"key":"e_1_3_5_38_2","unstructured":"Nakano R Hilton J Balaji S et\u00a0al. WebGPT: browser-assisted question-answering with human feedback. December 2021 (https:\/\/arxiv.org\/abs\/2112.09332). Preprint."},{"key":"e_1_3_5_39_2","unstructured":"Schick T Dwivedi-Yu J Dess\u00ec R et\u00a0al. Toolformer: language models can teach themselves to use tools. February 2023 (https:\/\/arxiv.org\/abs\/2302.04761). Preprint."},{"key":"e_1_3_5_40_2","unstructured":"Li\u00e9vin V Hother CE Winther O. Can large language models reason about medical questions? July 2022 (https:\/\/arxiv.org\/abs\/2207.08143). Preprint."},{"key":"e_1_3_5_41_2","unstructured":"Pal A Umapathi LK Sankarasubbu M. MedMCQA: a large-scale multi-subject multi-choice dataset for medical domain question answering. March 2022 (https:\/\/arxiv.org\/abs\/2203.14371)."},{"key":"e_1_3_5_42_2","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D19-1259"},{"key":"e_1_3_5_43_2","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2023.emnlp-main.308"},{"key":"e_1_3_5_44_2","doi-asserted-by":"publisher","DOI":"10.1097\/ACM.0b013e3181ffacdb"},{"key":"e_1_3_5_45_2","doi-asserted-by":"publisher","DOI":"10.1097\/ACM.0000000000004227"},{"key":"e_1_3_5_46_2","doi-asserted-by":"publisher","DOI":"10.4300\/JGME-D-19-00782.1"},{"key":"e_1_3_5_47_2","first-page":"47","article-title":"Correlation between housestaff performance on the United States Medical Licensing Examination and standardized patient encounters","volume":"72","year":"2005","unstructured":"Rifkin WD, Rifkin A. Correlation between housestaff performance on the United States Medical Licensing Examination and standardized patient encounters. Mt Sinai J Med 2005;72:47-49.","journal-title":"Mt Sinai J Med"},{"key":"e_1_3_5_48_2","doi-asserted-by":"publisher","DOI":"10.2147\/OAEM.S389868"},{"key":"e_1_3_5_49_2","doi-asserted-by":"publisher","DOI":"10.52544\/2642-7184(1)2001"},{"key":"e_1_3_5_50_2","unstructured":"Malkov YA Yashunin DA. Efficient and robust approximate nearest neighbor search using Hierarchical Navigable Small World graphs. March 2016 (https:\/\/arxiv.org\/abs\/1603.09320). Preprint."},{"key":"e_1_3_5_51_2","unstructured":"GitHub. Qdrant: open-source vector similarity search engine with segment storage and distributed serving. May 2020 (https:\/\/github.com\/qdrant\/qdrant)."},{"key":"e_1_3_5_52_2","unstructured":"MDCalc. MDCalc \u2014 medical calculators equations scores and guidelines (https:\/\/www.mdcalc.com)."},{"key":"e_1_3_5_53_2","unstructured":"National Center for Biotechnology Information. PubMed. National Library of Medicine. 2023 (https:\/\/pubmed.ncbi.nlm.nih.gov)."},{"key":"e_1_3_5_54_2","unstructured":"Wolters Kluwer. UpToDate. 2023 (https:\/\/www.uptodate.com)."},{"key":"e_1_3_5_55_2","unstructured":"BMJ Publishing Group. BMJ Best Practice. 2023 (http:\/\/bestpractice.bmj.com)."},{"key":"e_1_3_5_56_2","doi-asserted-by":"crossref","unstructured":"Papineni K Roukos S Ward T Zhu W-J. BLEU: a method for automatic evaluation of machine translation. Annual Meeting of the Association for Computational Linguistics (ACL) Philadelphia PA July 2002.","DOI":"10.3115\/1073083.1073135"},{"key":"e_1_3_5_57_2","unstructured":"Yang C Wang X Lu Y et\u00a0al. Large language models as optimizers. September 2023 (https:\/\/arxiv.org\/abs\/2309.03409). Preprint."},{"key":"e_1_3_5_58_2","doi-asserted-by":"crossref","unstructured":"Ben Abacha A Agichtein E Pinter Y Demner-Fushman D. Overview of the Medical Question Answering Task at TREC 2017 LiveQA. Text Retrieval Conference 2017 (https:\/\/api.semanticscholar.org\/CorpusID:3902472).","DOI":"10.6028\/NIST.SP.500-324.qa-overview"},{"key":"e_1_3_5_59_2","unstructured":"Yasunaga M Bosselut A Ren H et\u00a0al. Deep bidirectional language-knowledge graph pretraining. October 2022 (https:\/\/arxiv.org\/abs\/2210.09338)."},{"key":"e_1_3_5_60_2","doi-asserted-by":"crossref","unstructured":"Jin D Pan E Oufattole N Weng W-H Fang H Szolovits P. What disease does this patient have? A large-scale open domain question answering dataset from medical exams. September 2020 (https:\/\/arxiv.org\/abs\/2009.13081).","DOI":"10.20944\/preprints202105.0498.v1"}],"container-title":["NEJM AI"],"original-title":[],"language":"en","deposited":{"date-parts":[[2024,11,8]],"date-time":"2024-11-08T23:39:13Z","timestamp":1731109153000},"score":1,"resource":{"primary":{"URL":"https:\/\/ai.nejm.org\/doi\/10.1056\/AIoa2300068"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,1,25]]},"references-count":59,"journal-issue":{"issue":"2","published-print":{"date-parts":[[2024,1,25]]}},"alternative-id":["10.1056\/AIoa2300068"],"URL":"https:\/\/doi.org\/10.1056\/aioa2300068","relation":{},"ISSN":["2836-9386"],"issn-type":[{"value":"2836-9386","type":"print"}],"subject":[],"published":{"date-parts":[[2024,1,25]]}}}