{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,11]],"date-time":"2026-10-11T07:51:48Z","timestamp":1791705108512,"version":"4.3.4"},"reference-count":16,"publisher":"Radiological Society of North America (RSNA)","issue":"2","funder":[{"DOI":"10.13039\/100000002","name":"National INstitutes of Health","doi-asserted-by":"publisher","id":[{"id":"10.13039\/100000002","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000092","name":"National Library of Medicine","doi-asserted-by":"publisher","award":["1ZIALM010021"],"award-info":[{"award-number":["1ZIALM010021"]}],"id":[{"id":"10.13039\/100000092","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["Radiology"],"published-print":{"date-parts":[[2025,8,1]]},"abstract":"<jats:p>Multimodal large language models have significantly improved in assessing RSNA Case of the Day Questions over the past year, with OpenAI\u2019s latest models outperforming those from Google and Meta; notably, OpenAI o1 had an accuracy comparable to expert radiologists.<\/jats:p>","DOI":"10.1148\/radiol.250617","type":"journal-article","created":{"date-parts":[[2025,8,12]],"date-time":"2025-08-12T13:50:36Z","timestamp":1755006636000},"source":"Crossref","is-referenced-by-count":19,"title":["One Year On: Assessing Progress of Multimodal Large Language Model                     Performance on RSNA 2024 Case of the Day Questions"],"prefix":"10.1148","volume":"316","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-3968-1707","authenticated-orcid":false,"given":"Benjamin","family":"Hou","sequence":"first","affiliation":[{"name":"Division of Intramural Research, National Library of                     Medicine, National Institutes of Health, Bethesda, Md"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-9975-9994","authenticated-orcid":false,"given":"Pritam","family":"Mukherjee","sequence":"additional","affiliation":[{"name":"Imaging Biomarkers and Computer Aided Diagnosis Lab,                     Clinical Center, National Institutes of Health, 8600 Rockville Pike, Bldg 38A                     Lister Hill, Bethesda, MD 20894"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Vivek","family":"Batheja","sequence":"additional","affiliation":[{"name":"Imaging Biomarkers and Computer Aided Diagnosis Lab,                     Clinical Center, National Institutes of Health, 8600 Rockville Pike, Bldg 38A                     Lister Hill, Bethesda, MD 20894"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-6475-814X","authenticated-orcid":false,"given":"Kenneth C.","family":"Wang","sequence":"additional","affiliation":[{"name":"Baltimore VA Medical Center, Baltimore, Md"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Ronald M.","family":"Summers","sequence":"additional","affiliation":[{"name":"Imaging Biomarkers and Computer Aided Diagnosis Lab,                     Clinical Center, National Institutes of Health, 8600 Rockville Pike, Bldg 38A                     Lister Hill, Bethesda, MD 20894"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-9998-916X","authenticated-orcid":false,"given":"Zhiyong","family":"Lu","sequence":"additional","affiliation":[{"name":"Division of Intramural Research, National Library of                     Medicine, National Institutes of Health, Bethesda, Md"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"1577","reference":[{"key":"r1","unstructured":"Achiam J, Adler S, Agarwal S, et                             al. GPT-4 Technical                         Report. arXiv 2023. Preprint posted online                         March 15, 2023; doi:10.48550\/arxiv.2303.08774."},{"key":"r2","unstructured":"Team G, Georgiev P, Lei VI, et                             al. Gemini 1.5: Unlocking                         multimodal understanding across millions of tokens of                         context. arXiv 2024. Preprint posted online                         March 8, 2024; doi:10.48550\/arxiv.2403.05530."},{"key":"r4","unstructured":"Groeneveld D, Beltagy I, Walsh P, et                             al. OLMo: Accelerating the                         Science of Language Models. arXiv 2024.                         Preprint posted online February 1, 2024; doi:10.48550\/arxiv.2402.00838."},{"key":"r5","unstructured":"Schuhmann C, Beaumont R, Vencu R, et                             al. LAION-5B: An open                         large-scale dataset for training next generation image-text                         models. arXiv 2022. Preprint posted online                         October 16, 2022; doi:10.48550\/arxiv.2210.08402."},{"key":"r7","unstructured":"Wolf T, Debut L, Sanh V, et                             al. HuggingFace\u2019s                         Transformers: State-of-the-art Natural Language                         Processing. arXiv 2019. Preprint posted                         online October 9, 2019; doi:10.48550\/arxiv.1910.03771."},{"key":"r10","unstructured":"Chiang WL, Zheng L, Sheng Y, et                             al. Chatbot Arena: An Open                         Platform for Evaluating LLMs by Human                         Preference. arXiv 2024. Preprint posted                         online March 7, 2024; doi:10.48550\/arxiv.2403.04132."},{"key":"r11","doi-asserted-by":"publisher","DOI":"10.1148\/radiol.240609"},{"key":"r13","doi-asserted-by":"publisher","DOI":"10.1093\/biomet\/26.4.404"},{"issue":"2","key":"r14","first-page":"65","volume":"6","author":"Holm S","year":"1979","journal-title":"Scand J Stat"},{"key":"r15","unstructured":"Rein D, Hou BL, Stickland AC, et                             al. GPQA: A Graduate-Level                         Google-Proof Q*A Benchmark. arXiv                         2023. Preprint posted online November 20, 2023; doi:10.48550\/arxiv.2311.12022."},{"key":"r16","author":"Pal A","year":"2024","journal-title":"Hugging Face;"},{"key":"r17","doi-asserted-by":"publisher","DOI":"10.3390\/app11146421"},{"key":"r18","doi-asserted-by":"crossref","unstructured":"Jin Q, Dhingra B, Liu Z, Cohen WW. PubMedQA:                         A Dataset for Biomedical Research Question                         Answering. arXiv 2019. Preprint posted                         online September 13, 2019; doi:10.48550\/arxiv.1909.06146.","DOI":"10.18653\/v1\/D19-1259"},{"key":"r19","unstructured":"Pal A, Umapathi LK, Sankarasubbu M. MedMCQA:                         A Large-scale Multi-Subject Multi-Choice Dataset for Medical domain Question                         Answering. arXiv 2022. Preprint posted                         online March 27, 2022; doi:10.48550\/arxiv.2203.14371."},{"key":"r20","unstructured":"Hendrycks D, Burns C, Basart S, et                             al. Measuring Massive                         Multitask Language Understanding. arXiv                         2020. Preprint posted online September 7, 2020; doi:10.48550\/arxiv.2009.03300."},{"key":"r21","doi-asserted-by":"publisher","DOI":"10.1038\/s41746-024-01185-7"}],"container-title":["Radiology"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/pubs.rsna.org\/doi\/pdf\/10.1148\/radiol.250617","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,8,12]],"date-time":"2025-08-12T13:50:38Z","timestamp":1755006638000},"score":1,"resource":{"primary":{"URL":"http:\/\/pubs.rsna.org\/doi\/10.1148\/radiol.250617"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,8,1]]},"references-count":16,"journal-issue":{"issue":"2","published-print":{"date-parts":[[2025,8,1]]}},"alternative-id":["10.1148\/radiol.250617"],"URL":"https:\/\/doi.org\/10.1148\/radiol.250617","relation":{},"ISSN":["0033-8419","1527-1315"],"issn-type":[{"value":"0033-8419","type":"print"},{"value":"1527-1315","type":"electronic"}],"subject":[],"published":{"date-parts":[[2025,8,1]]},"article-number":"e250617"}}