{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,6,23]],"date-time":"2026-06-23T20:59:21Z","timestamp":1782248361702,"version":"3.54.5"},"reference-count":57,"publisher":"Springer Science and Business Media LLC","issue":"7","license":[{"start":{"date-parts":[[2025,6,19]],"date-time":"2025-06-19T00:00:00Z","timestamp":1750291200000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2025,6,19]],"date-time":"2025-06-19T00:00:00Z","timestamp":1750291200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"funder":[{"name":"Shanghai Artificial intelligence laboratory"},{"name":"Shanghai Artificial intelligence laboratory, the Centre for Perceptual and Interactive Intelligence (CPll) Ltd under the Innovation and Technology Commission (lTC)'s InnoHK."}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Nat Comput Sci"],"DOI":"10.1038\/s43588-025-00818-5","type":"journal-article","created":{"date-parts":[[2025,6,19]],"date-time":"2025-06-19T05:03:02Z","timestamp":1750309382000},"page":"524-533","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":6,"title":["Cost-effective instruction learning for pathology vision and language analysis"],"prefix":"10.1038","volume":"5","author":[{"given":"Kaitao","family":"Chen","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-5171-778X","authenticated-orcid":false,"given":"Mianxin","family":"Liu","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Fang","family":"Yan","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-6024-3854","authenticated-orcid":false,"given":"Lei","family":"Ma","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Xiaoming","family":"Shi","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Lilong","family":"Wang","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Xiaosong","family":"Wang","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Lifeng","family":"Zhu","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Zhe","family":"Wang","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Mu","family":"Zhou","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-8719-448X","authenticated-orcid":false,"given":"Shaoting","family":"Zhang","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2025,6,19]]},"reference":[{"key":"818_CR1","unstructured":"Zhang, Y. et al. Data-centric foundation models in computational healthcare: a survey. Preprint at https:\/\/arxiv.org\/abs\/2401.02458 (2024)."},{"key":"818_CR2","doi-asserted-by":"publisher","unstructured":"van Sonsbeek, T., Derakhshani, M. M., Najdenkoska, I., Snoek, C. G. M. & Worring, M. Open-ended medical visual question answering through prefix tuning of language models. In Medical Image Computing and Computer Assisted Intervention \u2013 MICCAI 2023 Lecture Notes in Computer Science, vol 14224 (eds Greenspan, H. et al.) https:\/\/doi.org\/10.1007\/978-3-031-43904-9_70 (Springer, Cham, 2023).","DOI":"10.1007\/978-3-031-43904-9_70"},{"key":"818_CR3","doi-asserted-by":"publisher","unstructured":"Li, P., Liu, G., Tan, L., Liao, J, & Zhong, S. Self-supervised vision-language pretraining for medial visual question answering. In IEEE 20th International Symposium on Biomedical Imaging https:\/\/doi.org\/10.1109\/ISBI53787.2023.10230743 (IEEE, 2023).","DOI":"10.1109\/ISBI53787.2023.10230743"},{"key":"818_CR4","doi-asserted-by":"publisher","first-page":"1930","DOI":"10.1038\/s41591-023-02448-8","volume":"29","author":"AJ Thirunavukarasu","year":"2023","unstructured":"Thirunavukarasu, A. J. et al. Large language models in medicine. Nat. Med. 29, 1930\u20131940 (2023).","journal-title":"Nat. Med."},{"key":"818_CR5","doi-asserted-by":"publisher","first-page":"172","DOI":"10.1038\/s41586-023-06291-2","volume":"620","author":"K Singhal","year":"2023","unstructured":"Singhal, K. et al. Large language models encode clinical knowledge. Nature 620, 172\u2013180 (2023).","journal-title":"Nature"},{"key":"818_CR6","unstructured":"Wei, J. et al. Chain-of-thought prompting elicits reasoning in large language models. In 36th Conference on Neural Information Processing Systems (NeurIPS 2022) https:\/\/openreview.net\/pdf?id=_VjQlMeSB_J (NeurIPS, 2022)."},{"key":"818_CR7","unstructured":"Schulman, J. et al. ChatGPT: optimizing language models for dialogue. OpenAI Blog https:\/\/openai.com\/index\/chatgpt (2022)."},{"key":"818_CR8","unstructured":"Achiam, J. et al. GPT-4 technical report. Preprint at https:\/\/arxiv.org\/abs\/2303.08774 (2023)."},{"key":"818_CR9","unstructured":"Liu, H., Li, C., Wu, Q. & Lee, Y. J. Visual instruction tuning. In 37th Conference on Neural Information Processing Systems (NeurIPS 2023) https:\/\/proceedings.neurips.cc\/paper_files\/paper\/2023\/file\/6dcf277ea32ce3288914faf369fe6de0-Paper-Conference.pdf (NeurIPS, 2023)."},{"key":"818_CR10","unstructured":"Alayrac, J.-B. et al. Flamingo: a visual language model for few-shot learning. In 36th Conference on Neural Information Processing Systems (NeurIPS 2022) https:\/\/papers.neurips.cc\/paper_files\/paper\/2022\/file\/960a172bc7fbf0177ccccbb411a7d800-Paper-Conference.pdf (NeurIPS, 2022)."},{"key":"818_CR11","unstructured":"Koh, J. Y., Salakhutdinov, R. & Fried, D. Grounding language models to images for multimodal generation. In International Conference on Machine Learning 17283\u201317300 (PMLR, 2023)."},{"key":"818_CR12","unstructured":"Ye, Q. et al. mPLUG-Owl: modularization empowers large language models with multimodality. Preprint at https:\/\/arxiv.org\/abs\/2304.14178 (2023)."},{"key":"818_CR13","doi-asserted-by":"publisher","first-page":"930","DOI":"10.1038\/s44222-023-00096-8","volume":"1","author":"AH Song","year":"2023","unstructured":"Song, A. H. et al. Artificial intelligence for digital and computational pathology. Nat. Rev. Bioeng. 1, 930\u2013949 (2023).","journal-title":"Nat. Rev. Bioeng."},{"key":"818_CR14","doi-asserted-by":"publisher","first-page":"1579","DOI":"10.1016\/S0140-6736(20)30226-9","volume":"395","author":"N Schwalbe","year":"2020","unstructured":"Schwalbe, N. & Wahl, B. Artificial intelligence and the future of global health. The Lancet 395, 1579\u20131586 (2020).","journal-title":"The Lancet"},{"key":"818_CR15","doi-asserted-by":"publisher","first-page":"23","DOI":"10.1038\/s41379-021-00919-2","volume":"35","author":"V Baxi","year":"2022","unstructured":"Baxi, V., Edwards, R., Montalto, M. & Saha, S. Digital pathology and artificial intelligence in translational medicine and clinical practice. Mod. Pathol. 35, 23\u201332 (2022).","journal-title":"Mod. Pathol."},{"key":"818_CR16","doi-asserted-by":"publisher","unstructured":"Wang, X. et al. Editorial for special issue on foundation models for medical image analysis. Med. Image Anal. https:\/\/doi.org\/10.1016\/j.media.2024.103389 (2024).","DOI":"10.1016\/j.media.2024.103389"},{"key":"818_CR17","doi-asserted-by":"publisher","first-page":"102996","DOI":"10.1016\/j.media.2023.102996","volume":"91","author":"S Zhang","year":"2024","unstructured":"Zhang, S. & Metaxas, D. On the challenges and perspectives of foundation models for medical image analysis. Med. Image Anal. 91, 102996 (2024).","journal-title":"Med. Image Anal."},{"key":"818_CR18","unstructured":"Li, C. et al. LLaVA-Med: training a large language-and-vision assistant for biomedicine in one day. In 37th Conference on Neural Information Processing Systems (NeurIPS 2023) https:\/\/papers.neurips.cc\/paper_files\/paper\/2023\/file\/5abcdf8ecdcacba028c6662789194572-Paper-Datasets_and_Benchmarks.pdf (NeurIPS, 2023)."},{"key":"818_CR19","doi-asserted-by":"crossref","unstructured":"Seyfioglu, M. S., Ikezogwo, W. O., Ghezloo, F., Krishna, R. & Shapiro, L. Quilt-LLaVA: visual instruction tuning by extracting localized narratives from open-source histopathology videos. In Proc. IEEE\/CVF Conference on Computer Vision and Pattern Recognition 13183\u201313192 (IEEE, 2024).","DOI":"10.1109\/CVPR52733.2024.01252"},{"key":"818_CR20","doi-asserted-by":"publisher","unstructured":"Wu, C. et al. PMC-LLaMA: toward building open-source language models for medicine. J. Am. Med. Inform. Assoc. https:\/\/doi.org\/10.1093\/jamia\/ocae045 (2024).","DOI":"10.1093\/jamia\/ocae045"},{"key":"818_CR21","unstructured":"Moor, M. et al. Med-flamingo: a multimodal medical few-shot learner. In Proc. 3rd Machine Learning for Health Symposium 353\u2013367 (PMLR, 2023)."},{"key":"818_CR22","doi-asserted-by":"publisher","first-page":"970","DOI":"10.1038\/s41586-024-07894-z","volume":"634","author":"X Wang","year":"2024","unstructured":"Wang, X. et al. A pathology foundation model for cancer diagnosis and prognosis prediction. Nature 634, 970\u2013978 (2024).","journal-title":"Nature"},{"key":"818_CR23","doi-asserted-by":"publisher","first-page":"863","DOI":"10.1038\/s41591-024-02856-4","volume":"30","author":"MY Lu","year":"2024","unstructured":"Lu, M. Y. et al. A visual-language foundation model for computational pathology. Nat. Med. 30, 863\u2013874 (2024).","journal-title":"Nat. Med."},{"key":"818_CR24","doi-asserted-by":"publisher","first-page":"466","DOI":"10.1038\/s41586-024-07618-3","volume":"634","author":"MY Lu","year":"2024","unstructured":"Lu, M. Y. et al. A multimodal generative AI copilot for human pathology. Nature 634, 466\u2013473 (2024).","journal-title":"Nature"},{"key":"818_CR25","unstructured":"Xu, Y. et al. A multimodal knowledge-enhanced whole-slide pathology foundation model. Preprint at https:\/\/arxiv.org\/abs\/2407.15362 (2024)."},{"key":"818_CR26","unstructured":"Zhang, S. et al. Large-scale domain-specific pretraining for biomedical vision-language processing. Preprint at https:\/\/arxiv.org\/abs\/2303.00915 (2023)."},{"key":"818_CR27","unstructured":"Ikezogwo, W. O. et al. Quilt-1m: One million image-text pairs for histopathology. In 37th Conference on Neural Information Processing Systems (NeurIPS 2023) https:\/\/proceedings.neurips.cc\/paper_files\/paper\/2023\/file\/775ec578876fa6812c062644964b9870-Paper-Datasets_and_Benchmarks.pdf (NeurIPS, 2023)."},{"key":"818_CR28","doi-asserted-by":"publisher","first-page":"2307","DOI":"10.1038\/s41591-023-02504-3","volume":"29","author":"Z Huang","year":"2023","unstructured":"Huang, Z., Bianchi, F., Yuksekgonul, M., Montine, T. J. & Zou, J. A visual\u2013language foundation model for pathology image analysis using medical twitter. Nat. Med. 29, 2307\u20132316 (2023).","journal-title":"Nat. Med."},{"key":"818_CR29","unstructured":"Radford, A. et al. Learning transferable visual models from natural language supervision. In Proc. International Conference on Machine Learning 8748\u20138763 (PMLR, 2021)."},{"key":"818_CR30","doi-asserted-by":"crossref","unstructured":"Gao, Y., Gu, D., Zhou, M. & Metaxas, D. Aligning human knowledge with visual concepts towards explainable medical image classification. In International Conference on Medical Image Computing and Computer-Assisted Intervention 46\u201356 (Springer, 2024).","DOI":"10.1007\/978-3-031-72117-5_5"},{"key":"818_CR31","doi-asserted-by":"crossref","unstructured":"Zhang, Y. et al. Text-guided foundation model adaptation for pathological image classification. In Proc. International Conference on Medical Image Computing and Computer-Assisted Intervention 272\u2013282 (Springer, 2023).","DOI":"10.1007\/978-3-031-43904-9_27"},{"key":"818_CR32","doi-asserted-by":"crossref","unstructured":"Ding, K., Zhou, M., Metaxas, D. N. & Zhang, S. Pathology-and-genomics multimodal transformer for survival outcome prediction. In Proc. International Conference on Medical Image Computing and Computer-Assisted Intervention 622\u2013631 (Springer, 2023).","DOI":"10.1007\/978-3-031-43987-2_60"},{"key":"818_CR33","unstructured":"Li, J., Li, D., Savarese, S. & Hoi, S. BLIP-2: bootstrapping language-image pre-training with frozen image encoders and large language models. In International Conference on Machine Learning 19730\u201319742 (PMLR, 2023)."},{"key":"818_CR34","unstructured":"He, X. et al. Pathvqa: 30000+ questions for medical visual question answering. Preprint at https:\/\/arxiv.org\/abs\/2003.10286 (2020)."},{"key":"818_CR35","unstructured":"Dubey, A. et al. The Llama 3 herd of models. Preprint at https:\/\/arxiv.org\/abs\/2407.21783 (2024)."},{"key":"818_CR36","doi-asserted-by":"publisher","first-page":"102444","DOI":"10.1016\/j.media.2022.102444","volume":"79","author":"X Chen","year":"2022","unstructured":"Chen, X. et al. Recent advances and clinical applications of deep learning in medical image analysis. Med. Image Anal. 79, 102444 (2022).","journal-title":"Med. Image Anal."},{"key":"818_CR37","doi-asserted-by":"publisher","DOI":"10.1038\/s41467-023-40687-y","volume":"14","author":"Q Chang","year":"2023","unstructured":"Chang, Q. et al. Mining multi-center heterogeneous medical data with distributed synthetic learning. Nat. Commun. 14, 5510 (2023).","journal-title":"Nat. Commun."},{"key":"818_CR38","doi-asserted-by":"crossref","unstructured":"Graikos, A. et al. Learned representation-guided diffusion models for large-image generation. In Proc. IEEE\/CVF Conference on Computer Vision and Pattern Recognition 8532\u20138542 (IEEE, 2024).","DOI":"10.1109\/CVPR52733.2024.00815"},{"key":"818_CR39","doi-asserted-by":"publisher","DOI":"10.1038\/s41597-023-02125-y","volume":"10","author":"K Ding","year":"2023","unstructured":"Ding, K. et al. A large-scale synthetic pathological dataset for deep learning-enabled segmentation of breast cancer. Sci. Data 10, 231 (2023).","journal-title":"Sci. Data"},{"key":"818_CR40","unstructured":"Sun, Y. et al. PathGen-1.6M: 1.6 million pathology image-text pairs generation through multi-agent collaboration. In Thirteenth International Conference on Learning Representations (ICLR 2025) https:\/\/openreview.net\/pdf?id=rFpZnn11gj (ICLR, 2025)."},{"key":"818_CR41","first-page":"1","volume":"25","author":"HW Chung","year":"2024","unstructured":"Chung, H. W. et al. Scaling instruction-finetuned language models. J. Mach. Learn. Res. 25, 1\u201353 (2024).","journal-title":"J. Mach. Learn. Res."},{"key":"818_CR42","doi-asserted-by":"crossref","unstructured":"Liu, H., Li, C., Li, Y. & Lee, Y. J. Improved baselines with visual instruction tuning. In Proc. IEEE\/CVF Conference on Computer Vision and Pattern Recognition 26296\u201326306 (IEEE, 2024).","DOI":"10.1109\/CVPR52733.2024.02484"},{"key":"818_CR43","unstructured":"Peng, B., Li, C., He, P., Galley, M. & Gao, J. Instruction tuning with GPT-4. Preprint at https:\/\/arxiv.org\/abs\/2304.03277 (2023)."},{"key":"818_CR44","unstructured":"Zhou, C. et al. LIMA: less is more for alignment. In 37th Conference on Neural Information Processing Systems (NeurIPS 2023) https:\/\/proceedings.neurips.cc\/paper_files\/paper\/2023\/file\/ac662d74829e4407ce1d126477f4a03a-Paper-Conference.pdf (NeurIPS, 2023)."},{"key":"818_CR45","unstructured":"Wei, J. et al. Finetuned language models are zero-shot learners. In Tenth International Conference on Learning Representations https:\/\/openreview.net\/pdf?id=gEZrGCozdqR (ICLR, 2022)."},{"key":"818_CR46","doi-asserted-by":"crossref","unstructured":"Chen, P., Zhu, C., Zheng, S., Li, H., & Yang, L. WSI-VQA: interpreting whole slide images by generative visual question answering. In Proc. European Conference on Computer Vision 401\u2013417 (2025).","DOI":"10.1007\/978-3-031-72764-1_23"},{"key":"818_CR47","doi-asserted-by":"publisher","first-page":"787","DOI":"10.1016\/S2589-7500(22)00168-6","volume":"4","author":"K Ding","year":"2022","unstructured":"Ding, K., Zhou, M., Wang, H., Zhang, S. & Metaxas, D. N. Spatially aware graph neural networks and cross-level molecular profile prediction in colon cancer histopathology: a retrospective multi-cohort study. Lancet Digit. Health 4, 787\u2013795 (2022).","journal-title":"Lancet Digit. Health"},{"key":"818_CR48","unstructured":"Hu, E.J. et al. LoRA: low-rank adaptation of large language models. In Tenth International Conference on Learning Representations https:\/\/openreview.net\/pdf?id=nZeVKeeFYf9 (ICLR, 2022)."},{"key":"818_CR49","unstructured":"Jin, Y. et al. Efficient multimodal large language models: a survey. Preprint at https:\/\/arxiv.org\/abs\/2405.10739 (2024)."},{"key":"818_CR50","unstructured":"Liu, H. et al. Few-shot parameter-efficient fine-tuning is better and cheaper than in-context learning. In 36th Conference on Neural Information Processing Systems (NeurIPS 2022) https:\/\/openreview.net\/pdf?id=rBCvMG-JsPd (2022)."},{"key":"818_CR51","unstructured":"Vaswani, A. et al. Attention is all you need. In 31st Conference on Neural Information Processing Systems (NIPS 2017) https:\/\/papers.nips.cc\/paper_files\/paper\/2017\/file\/3f5ee243547dee91fbd053c1c4a845aa-Paper.pdf (2017)."},{"key":"818_CR52","unstructured":"Li, J., Li, D., Xiong, C. & Hoi, S. BLIP: bootstrapping language-image pre-training for unified vision-language understanding and generation. In Proc. International Conference on Machine Learning 12888\u201312900 (PLMR, 2022)."},{"key":"818_CR53","doi-asserted-by":"crossref","unstructured":"Fang, Y. et al. EVA: exploring the limits of masked visual representation learning at scale. In Proc. IEEE\/CVF Conference on Computer Vision and Pattern Recognition 19358\u201319369 (IEEE, 2023).","DOI":"10.1109\/CVPR52729.2023.01855"},{"key":"818_CR54","unstructured":"Chiang, W.-L. et al. Vicuna: an open-source chatbot impressing GPT-4 with 90% chatgpt quality. LMSYS Org https:\/\/vicuna.lmsys.org (2023)."},{"key":"818_CR55","doi-asserted-by":"publisher","first-page":"380","DOI":"10.3390\/bioengineering10030380","volume":"10","author":"Y Bazi","year":"2023","unstructured":"Bazi, Y., Rahhal, M. M. A., Bashmal, L. & Zuair, M. Vision\u2013language model for visual question answering in medical imagery. Bioengineering 10, 380 (2023).","journal-title":"Bioengineering"},{"key":"818_CR56","doi-asserted-by":"crossref","unstructured":"Liu, Y., Wang, Z., Xu, D. & Zhou, L. Q2ATransformer: improving medical VQA via an answer querying decoder. In Proc. International Conference on Information Processing in Medical Imaging 445\u2013456 (Springer Nature, 2023).","DOI":"10.1007\/978-3-031-34048-2_34"},{"key":"818_CR57","doi-asserted-by":"publisher","unstructured":"Chen, K. et al. Cost-effective instruction learning for pathology vision and language analysis. Zenodo https:\/\/doi.org\/10.5281\/zenodo.15081542 (2025).","DOI":"10.5281\/zenodo.15081542"}],"container-title":["Nature Computational Science"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/www.nature.com\/articles\/s43588-025-00818-5.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/www.nature.com\/articles\/s43588-025-00818-5","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/www.nature.com\/articles\/s43588-025-00818-5.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,9,6]],"date-time":"2025-09-06T19:27:55Z","timestamp":1757186875000},"score":1,"resource":{"primary":{"URL":"https:\/\/www.nature.com\/articles\/s43588-025-00818-5"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,6,19]]},"references-count":57,"journal-issue":{"issue":"7","published-online":{"date-parts":[[2025,7]]}},"alternative-id":["818"],"URL":"https:\/\/doi.org\/10.1038\/s43588-025-00818-5","relation":{},"ISSN":["2662-8457"],"issn-type":[{"value":"2662-8457","type":"electronic"}],"subject":[],"published":{"date-parts":[[2025,6,19]]},"assertion":[{"value":"12 September 2024","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"13 May 2025","order":2,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"19 June 2025","order":3,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"The authors declare no competing interests.","order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Competing interests"}}]}}