{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,7]],"date-time":"2026-07-07T15:37:09Z","timestamp":1783438629623,"version":"3.54.6"},"reference-count":114,"publisher":"Tsinghua University Press","issue":"2","content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["Big Data Min. Anal."],"published-print":{"date-parts":[[2025,4]]},"DOI":"10.26599\/bdma.2024.9020090","type":"journal-article","created":{"date-parts":[[2025,1,28]],"date-time":"2025-01-28T18:53:58Z","timestamp":1738090438000},"page":"496-517","source":"Crossref","is-referenced-by-count":17,"title":["Large Language Model for Medical Images: A Survey of Taxonomy, Systematic Review, and Future Trends"],"prefix":"10.26599","volume":"8","author":[{"given":"Peng","family":"Wang","sequence":"first","affiliation":[{"name":"School of Computer Science and Engineering, Central South University,Changsha,China,410083"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Wenpeng","family":"Lu","sequence":"additional","affiliation":[{"name":"Qilu University of Technology (Shandong Academy of Sciences),Key Laboratory of Computing Power Network and Information Security Affiliated with Ministry of Education,Jinan,China,250353"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Chunlin","family":"Lu","sequence":"additional","affiliation":[{"name":"School of Computer Science and Engineering, Central South University,Changsha,China,410083"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Ruoxi","family":"Zhou","sequence":"additional","affiliation":[{"name":"School of Computer Science and Engineering, Central South University,Changsha,China,410083"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Min","family":"Li","sequence":"additional","affiliation":[{"name":"School of Computer Science and Engineering, Central South University,Changsha,China,410083"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Libo","family":"Qin","sequence":"additional","affiliation":[{"name":"School of Computer Science and Engineering, Central South University,Changsha,China,410083"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"11138","reference":[{"key":"ref1","doi-asserted-by":"publisher","DOI":"10.1109\/JPROC.2021.3054390"},{"key":"ref2","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-33128-3_1"},{"key":"ref3","doi-asserted-by":"publisher","DOI":"10.1016\/j.media.2017.07.005"},{"key":"ref4","article-title":"A survey of large language models","author":"Zhao","year":"2024","journal-title":"arXiv preprint"},{"key":"ref5","article-title":"Large language models meet NLP: A survey","author":"Qin","year":"2024","journal-title":"arXiv preprint"},{"key":"ref6","article-title":"Multilingual large language model: A survey of resources, taxonomy and frontiers","author":"Qin","year":"2024","journal-title":"arXiv preprint"},{"key":"ref7","doi-asserted-by":"publisher","DOI":"10.1007\/s11704-024-40231-1"},{"key":"ref8","doi-asserted-by":"publisher","DOI":"10.1145\/3690642"},{"key":"ref9","article-title":"Cro Prompt: Cross-task interactive prompting for zero-shot spoken language understanding","author":"Qin","year":"2024","journal-title":"arXiv preprint"},{"key":"ref10","doi-asserted-by":"publisher","DOI":"10.1038\/s41591-023-02448-8"},{"key":"ref11","article-title":"A survey of large language models in medicine: Progress, application, and challenge","author":"Zhou","year":"2024","journal-title":"arXiv preprint"},{"key":"ref12","first-page":"353","article-title":"Med-flamingo: A multimodal medical few-shot learner","volume-title":"Proc. 3rd Machine Learning for Health Symp.","author":"Moor"},{"key":"ref13","doi-asserted-by":"publisher","DOI":"10.1038\/s41746-023-00952-2"},{"key":"ref14","article-title":"A systematic evaluation of GPT -4 V\u2019s multimodal capability for medical image analysis","author":"Li","year":"2024","journal-title":"arXiv preprint"},{"key":"ref15","article-title":"Holistic evaluation of GPT-4V for biomedical imaging","author":"Liu","year":"2023","journal-title":"arXiv preprint"},{"key":"ref16","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52729.2023.00325"},{"key":"ref17","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v33i01.3301590"},{"key":"ref18","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v34i07.6989"},{"key":"ref19","doi-asserted-by":"publisher","DOI":"10.1016\/j.compbiomed.2020.103761"},{"key":"ref20","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-20059-5_1"},{"key":"ref21","article-title":"Chat CAD: Interactive computer-aided diagnosis on medical image using large language models","author":"Wang","year":"2023","journal-title":"arXiv preprint"},{"key":"ref22","doi-asserted-by":"publisher","DOI":"10.1109\/TMI.2024.3398350"},{"key":"ref23","article-title":"The dawn of LMMs: Preliminary explorations with GPT-4V(ision)","author":"Yang","year":"2023","journal-title":"arXiv preprint"},{"key":"ref24","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2024.clinicalnlp-1.3"},{"key":"ref25","article-title":"Multimodal large language models are generalist medical image interpreters","author":"Han","year":"2023","journal-title":"medRxiv preprint"},{"key":"ref26","article-title":"OphGLM: Training an ophthalmology large language-and-vision assistant based on instructions and dialogue","author":"Gao","year":"2023","journal-title":"arXiv preprint"},{"key":"ref27","volume-title":"Visual med-alpaca: A parameter-efficient biomedical LLM with visual capabilities","author":"Shu","year":"2023"},{"key":"ref28","author":"Shi","year":"2023","journal-title":"FFA-GPT: An interactive visual question answering system for fundus fluorescein angiography"},{"key":"ref29","article-title":"Med BLIP: Bootstrapping language-image pre-training from 3D medical images and texts","author":"Chen","year":"2023","journal-title":"arXiv preprint"},{"key":"ref30","article-title":"Towards generalist foundation model for radiology by leveraging web-scale 2D&3D medical data","author":"Wu","year":"2023","journal-title":"arXiv preprint"},{"key":"ref31","volume-title":"Large multimodal model for real-world radiology report generation","author":"Zhao","year":"2023"},{"key":"ref32","article-title":"MAIRA-1: A specialised large multimodal model for radiology report generation","author":"Hyland","year":"2024","journal-title":"arXiv preprint"},{"key":"ref33","author":"Tu","year":"2023","journal-title":"Towards generalist biomedical AI, arXiv preprint"},{"key":"ref34","article-title":"Customizing general-purpose foundation models for medical report generation","author":"Yang","year":"2023","journal-title":"arXiv preprint"},{"key":"ref35","article-title":"PMC-VQA: Visual instruction tuning for medical visual question answering","author":"Zhang","year":"2024","journal-title":"arXiv preprint"},{"key":"ref36","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-43904-9_70"},{"key":"ref37","doi-asserted-by":"publisher","DOI":"10.1016\/j.metrad.2023.100033"},{"key":"ref38","article-title":"Med XChat: Bridging CXR modalities with a unified multimodal large model","author":"Yang","year":"2024","journal-title":"arXiv preprint"},{"key":"ref39","doi-asserted-by":"publisher","DOI":"10.1016\/j.media.2024.103248"},{"key":"ref40","article-title":"Pe FoMed: Parameter efficient fine-tuning on multimodal large language models for medical visual question answering","author":"He","year":"2024","journal-title":"arXiv preprint"},{"key":"ref41","article-title":"SkinGPT-4: An interactive dermatology diagnostic system with visual large language model","author":"Zhou","year":"2023","journal-title":"arXiv preprint"},{"key":"ref42","article-title":"CephGPT-4: An interactive multimodal cephalometric measurement and diagnostic system with visual large language model","author":"Ma","year":"2023","journal-title":"arXiv preprint"},{"key":"ref43","article-title":"Path Asst: Redefining pathology through generative foundation AI assistant for pathology","author":"Sun","year":"2024","journal-title":"arXiv preprint"},{"key":"ref44","article-title":"CXR-LLAVA: A multimodal large language model for interpreting chest X-ray images","author":"Lee","year":"2024","journal-title":"arXiv preprint"},{"key":"ref45","article-title":"LLaVA-Med: Training a large language-and-vision assistant for biomedicine in one day","author":"Li","year":"2023","journal-title":"arXiv preprint"},{"key":"ref46","article-title":"ELIXR: Towards a general purpose x-ray artificial intelligence system through alignment of large language models and radiology vision encoders","author":"Xu","year":"2023","journal-title":"arXiv preprint"},{"key":"ref47","volume-title":"Towards automated healthcare: Deep vision and large language models for radiology report generation","author":"Tian","year":"2023"},{"key":"ref48","article-title":"Transferring pre-trained large language-image model for medical image captioning","volume-title":"Proc. CLEF2023: Conf. and Labs of the Evaluation Forum","author":"Zhou"},{"key":"ref49","doi-asserted-by":"publisher","DOI":"10.2196\/32690"},{"key":"ref50","article-title":"Xray GPT: Chest radiographs summarization using medical vision-language models","author":"Thawkar","year":"2023","journal-title":"arXiv preprint"},{"key":"ref51","article-title":"Qilin-Med-VL: Towards Chinese large vision-language model for general healthcare","author":"Liu","year":"2023","journal-title":"arXiv preprint"},{"key":"ref52","article-title":"LLM itself can read and generate CXR images","author":"Lee","year":"2024","journal-title":"arXiv preprint"},{"key":"ref53","article-title":"Effectively fine-tune to improve large multimodal models for radiology report generation","author":"Lu","year":"2023","journal-title":"arXiv preprint"},{"key":"ref54","doi-asserted-by":"publisher","DOI":"10.1145\/3688005"},{"key":"ref55","article-title":"Lite GPT: Large vision-language model for joint chest X-ray localization and classification task","author":"Le-Duc","year":"2024","journal-title":"arXiv preprint"},{"key":"ref56","article-title":"Mini GPT-Med: Large language model as a general interface for radiology diagnosis","author":"Alkhaldi","year":"2024","journal-title":"arXiv preprint"},{"key":"ref57","article-title":"Gemini Team Google, Gemini: A family of highly capable multimodal models","year":"2024","journal-title":"arXiv preprint"},{"key":"ref58","first-page":"23716","article-title":"Flamingo: A visual language model for few-shot learning","volume-title":"Proc. 36th Int. Conf. Neural Information Processing Systems","author":"Alayrac"},{"key":"ref59","article-title":"GLM-130B: An open bilingual pre-trained model","author":"Zeng","year":"2023","journal-title":"arXiv preprint"},{"key":"ref60","article-title":"LLaMA: Open and efficient foundation language models","author":"Touvron","year":"2023","journal-title":"arXiv preprint"},{"key":"ref61","first-page":"12888","article-title":"BLIP: Bootstrapping language-image pre-training for unified vision-language understanding and generation","volume-title":"Proc. 39th Int. Conf. Machine Learning","author":"Li"},{"key":"ref62","article-title":"Llama 2: Open foundation and fine-tuned chat models","author":"Touvron","year":"2023","journal-title":"arXiv preprint"},{"key":"ref63","article-title":"OpenFlamingo: An open-source framework for training large autoregressive vision-language models","author":"Awadalla","year":"2023","journal-title":"arXiv preprint"},{"key":"ref64","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52733.2024.02484"},{"key":"ref65","article-title":"PaLM-E: An embodied multimodal language model","author":"Driess","year":"2023","journal-title":"arXiv preprint"},{"key":"ref66","article-title":"BLIP-2: Bootstrapping language-image pre-training with frozen image encoders and large language models","author":"Li","year":"2023","journal-title":"arXiv preprint"},{"key":"ref67","first-page":"8748","article-title":"Learning transferable visual models from natural language supervision","volume-title":"Proc. 38th Int. Conf. Machine Learning, Virtual Event","author":"Radford"},{"key":"ref68","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52688.2022.01042"},{"key":"ref69","article-title":"MiniGPT -4: Enhancing vision-language understanding with advanced large language models","author":"Zhu","year":"2023","journal-title":"arXiv preprint"},{"key":"ref70","article-title":"MiniGPT-v2: Large language model as a unified interface for vision-language multi-task learning","author":"Chen","year":"2023","journal-title":"arXiv preprint"},{"key":"ref71","first-page":"34892","article-title":"Visual instruction tuning","volume-title":"Proc. 37th Int. Conf. Neural Information Processing Systems","author":"Liu"},{"key":"ref72","article-title":"An image is worth 16\u00d716 words: Transformers for image recognition at scale","author":"Dosovitskiy","year":"2021","journal-title":"arXiv preprint"},{"key":"ref73","article-title":"OPT: Open pre-trained transformer language models","author":"Zhang","year":"2022","journal-title":"arXiv preprint"},{"key":"ref74","article-title":"Instruct BLIP: Towards general-purpose vision-language models with instruction tuning","author":"Dai","year":"2023","journal-title":"arXiv preprint"},{"key":"ref75","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2022.emnlp-main.256"},{"key":"ref76","volume-title":"Vicuna: An open-source chatbot impressing GPT-4 with 90%* ChatGPT quality","author":"Chiang","year":"2023"},{"key":"ref77","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-43907-0_36"},{"key":"ref78","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-34048-2_34"},{"key":"ref79","article-title":"Textbooks are all you need","author":"Gunasekar","year":"2023","journal-title":"arXiv preprint"},{"key":"ref80","article-title":"Agentinstruct: Toward generative teaching with agentic flows","author":"Mitra","year":"2024","journal-title":"arXiv preprint"},{"key":"ref81","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2024.acl-demos.9"},{"key":"ref82","article-title":"Knowledge editing for large language models: A survey","author":"Wang","year":"2024","journal-title":"arXiv preprint"},{"key":"ref83","article-title":"Siren\u2019s song in the AI ocean: A survey on hallucination in large language models","author":"Zhang","year":"2023","journal-title":"arXiv preprint"},{"key":"ref84","article-title":"Scalable extraction of training data from (production) language models","author":"Nasr","year":"2023","journal-title":"arXiv preprint"},{"key":"ref85","article-title":"Federated optimization: Distributed machine learning for on-device intelligence","author":"Kone\u010dn\u00fd","year":"2016","journal-title":"arXiv preprint"},{"key":"ref86","doi-asserted-by":"publisher","DOI":"10.1186\/s12880-015-0068-x"},{"key":"ref87","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2023.emnlp-main.163"},{"key":"ref88","article-title":"Lang Bridge: Multilingual reasoning without multilingual supervision","author":"Yoon","year":"2024","journal-title":"arXiv preprint"},{"key":"ref89","article-title":"Med Agents: Large language models as collaborators for zero-shot medical reasoning","author":"Tang","year":"2024","journal-title":"arXiv preprint"},{"key":"ref90","first-page":"24824","article-title":"Chain-of-thought prompting elicits reasoning in large language models","volume-title":"Proc. 36th Int. Conf. Neural Information Processing Systems","author":"Wei"},{"key":"ref91","article-title":"Automatic chain of thought prompting in large language models","author":"Zhang","year":"2022","journal-title":"arXiv preprint"},{"key":"ref92","article-title":"Program of thoughts prompting: Disentangling computation from reasoning for numerical reasoning tasks","author":"Chen","year":"2023","journal-title":"arXiv preprint"},{"key":"ref93","article-title":"Large language model guided tree-of-thought","author":"Long","year":"2023","journal-title":"arXiv preprint"},{"key":"ref94","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2024.findings-emnlp.388"},{"key":"ref95","doi-asserted-by":"publisher","DOI":"10.1038\/s41746-024-01029-4"},{"key":"ref96","first-page":"53366","article-title":"NExT-GPT: Any-to-any multimodal LLM","volume-title":"Proc. 41st Int. Conf. Machine Learning","author":"Wu"},{"key":"ref97","doi-asserted-by":"publisher","DOI":"10.1016\/j.media.2023.102802"},{"key":"ref98","doi-asserted-by":"publisher","DOI":"10.1016\/j.media.2021.102125"},{"key":"ref99","doi-asserted-by":"publisher","DOI":"10.1007\/s11036-020-01672-7"},{"key":"ref100","doi-asserted-by":"publisher","DOI":"10.1111\/1754-9485.13261"},{"key":"ref101","doi-asserted-by":"publisher","DOI":"10.1016\/j.compbiomed.2020.104115"},{"key":"ref102","doi-asserted-by":"publisher","DOI":"10.1016\/j.media.2021.101985"},{"key":"ref103","doi-asserted-by":"publisher","DOI":"10.1109\/TBME.2021.3117407"},{"key":"ref104","doi-asserted-by":"publisher","DOI":"10.1016\/j.media.2019.03.009"},{"key":"ref105","doi-asserted-by":"publisher","DOI":"10.7717\/peerj-cs.1045"},{"key":"ref106","doi-asserted-by":"publisher","DOI":"10.3390\/s20185097"},{"key":"ref107","doi-asserted-by":"publisher","DOI":"10.1016\/j.media.2022.102470"},{"key":"ref108","doi-asserted-by":"publisher","DOI":"10.3390\/jimaging6060052"},{"key":"ref109","doi-asserted-by":"publisher","DOI":"10.3115\/1073083.1073135"},{"key":"ref110","first-page":"74","article-title":"ROUGE: A package for automatic evaluation of summaries","volume-title":"Proc. Text Summarization Branches Out","author":"Lin"},{"key":"ref111","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2015.7299087"},{"key":"ref112","first-page":"65","article-title":"METEOR: An automatic metric for MT evaluation with improved correlation with human judgments","volume-title":"Proc. ACL Workshop on Intrinsic and Extrinsic Evaluation Measures for Machine Translation and\/or Summarization","author":"Banerjee"},{"key":"ref113","doi-asserted-by":"publisher","DOI":"10.1109\/JBHI.2022.3207502"},{"key":"ref114","doi-asserted-by":"publisher","DOI":"10.1016\/j.media.2023.103021"}],"container-title":["Big Data Mining and Analytics"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx8\/8254253\/10856852\/10856853.pdf?arnumber=10856853","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,1,29]],"date-time":"2025-01-29T19:16:37Z","timestamp":1738178197000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/10856853\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,4]]},"references-count":114,"journal-issue":{"issue":"2"},"URL":"https:\/\/doi.org\/10.26599\/bdma.2024.9020090","relation":{},"ISSN":["2096-0654","2097-406X"],"issn-type":[{"value":"2096-0654","type":"print"},{"value":"2097-406X","type":"electronic"}],"subject":[],"published":{"date-parts":[[2025,4]]}}}