{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,1,27]],"date-time":"2026-01-27T18:44:53Z","timestamp":1769539493939,"version":"3.49.0"},"reference-count":104,"publisher":"Institute of Electrical and Electronics Engineers (IEEE)","license":[{"start":{"date-parts":[[2026,1,1]],"date-time":"2026-01-01T00:00:00Z","timestamp":1767225600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/ieeexplore.ieee.org\/Xplorehelp\/downloads\/license-information\/IEEE.html"},{"start":{"date-parts":[[2026,1,1]],"date-time":"2026-01-01T00:00:00Z","timestamp":1767225600000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2026,1,1]],"date-time":"2026-01-01T00:00:00Z","timestamp":1767225600000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"}],"funder":[{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["62422603"],"award-info":[{"award-number":["62422603"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100021171","name":"Basic and Applied Basic Research Foundation of Guangdong Province","doi-asserted-by":"publisher","award":["2024B0101050003"],"award-info":[{"award-number":["2024B0101050003"]}],"id":[{"id":"10.13039\/501100021171","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100019014","name":"Shenzhen Science and Technology Program","doi-asserted-by":"publisher","award":["ZDSYS20230626091203008"],"award-info":[{"award-number":["ZDSYS20230626091203008"]}],"id":[{"id":"10.13039\/501100019014","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["IEEE Trans. on Image Process."],"published-print":{"date-parts":[[2026]]},"DOI":"10.1109\/tip.2025.3649356","type":"journal-article","created":{"date-parts":[[2026,1,16]],"date-time":"2026-01-16T20:52:20Z","timestamp":1768596740000},"page":"858-871","source":"Crossref","is-referenced-by-count":0,"title":["Vision Enhancing LLMs: Empowering Multimodal Knowledge Storage and Sharing in LLMs"],"prefix":"10.1109","volume":"35","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-4819-2489","authenticated-orcid":false,"given":"Yunxin","family":"Li","sequence":"first","affiliation":[{"name":"School of Computer Science and Technology, Harbin Institute of Technology, Shenzhen, China"}]},{"given":"Zhenyu","family":"Liu","sequence":"additional","affiliation":[{"name":"School of Computer Science and Technology, Harbin Institute of Technology, Shenzhen, China"}]},{"ORCID":"https:\/\/orcid.org\/0009-0000-0640-6990","authenticated-orcid":false,"given":"Baotian","family":"Hu","sequence":"additional","affiliation":[{"name":"School of Computer Science and Technology, Harbin Institute of Technology, Shenzhen, China"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-8676-1190","authenticated-orcid":false,"given":"Wei","family":"Wang","sequence":"additional","affiliation":[{"name":"School of Cyber Science and Technology, Sun Yat-sen University, Shenzhen Campus, Shenzhen, China"}]},{"given":"Yuxin","family":"Ding","sequence":"additional","affiliation":[{"name":"School of Computer Science and Technology, Harbin Institute of Technology, Shenzhen, China"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-7141-708X","authenticated-orcid":false,"given":"Xiaochun","family":"Cao","sequence":"additional","affiliation":[{"name":"School of Cyber Science and Technology, Sun Yat-sen University, Shenzhen Campus, Shenzhen, China"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-3895-5510","authenticated-orcid":false,"given":"Min","family":"Zhang","sequence":"additional","affiliation":[{"name":"School of Computer Science and Technology, Harbin Institute of Technology, Shenzhen, China"}]}],"member":"263","reference":[{"key":"ref1","doi-asserted-by":"publisher","DOI":"10.1093\/nsr\/nwae403"},{"key":"ref2","article-title":"PaLM-E: An embodied multimodal language model","volume-title":"Proc. ICML","author":"Driess"},{"key":"ref3","doi-asserted-by":"publisher","DOI":"10.1109\/TMM.2024.3428317"},{"key":"ref4","article-title":"MPLUG-owl: Modularization empowers large language models with multimodality","volume-title":"Proc. ICML","author":"Ye"},{"key":"ref5","article-title":"MiniGPT-4: Enhancing vision-language understanding with advanced large language models","volume-title":"Proc. ICLR","author":"Zhu"},{"key":"ref6","first-page":"34892","article-title":"Visual instruction tuning","volume-title":"Proc. NeurIPS","author":"Liu"},{"key":"ref7","article-title":"BLIP-2: Bootstrapping language-image pre-training with frozen image encoders and large language models","volume-title":"Proc. ICML","author":"Li"},{"key":"ref8","article-title":"GPT-4 technical report","author":"Achiam","year":"2023","journal-title":"arXiv:2303.08774"},{"key":"ref9","article-title":"LLaVAR: Enhanced visual instruction tuning for text-rich image understanding","author":"Zhang","year":"2023","journal-title":"arXiv:2306.17107"},{"key":"ref10","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-72658-3_13"},{"key":"ref11","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2023.findings-emnlp.285"},{"key":"ref12","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2022.emnlp-main.765"},{"key":"ref13","article-title":"LoRA: Low-rank adaptation of large language models","author":"Hu","year":"2021","journal-title":"arXiv:2106.09685"},{"key":"ref14","first-page":"1877","article-title":"Language models are few-shot learners","volume-title":"Proc. NeurIPS","volume":"33","author":"Brown"},{"key":"ref15","first-page":"27730","article-title":"Training language models to follow instructions with human feedback","volume-title":"Proc. NeurIPS","author":"Ouyang"},{"key":"ref16","article-title":"LLaMA-adapter V2: Parameter-efficient visual instruction model","author":"Gao","year":"2023","journal-title":"arXiv:2304.15010"},{"key":"ref17","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2019.00156"},{"key":"ref18","doi-asserted-by":"publisher","DOI":"10.48550\/ARXIV.1706.03762"},{"key":"ref19","doi-asserted-by":"publisher","DOI":"10.1145\/3560815"},{"key":"ref20","first-page":"15706","article-title":"Better & faster large language models via multi-token prediction","volume-title":"Proc. ICML","author":"Gloeckle"},{"key":"ref21","first-page":"2835","article-title":"Large language models on tabular data\u2014A survey","author":"Fang","year":"2024","journal-title":"TMLR"},{"key":"ref22","first-page":"50358","article-title":"Scaling data-constrained language models","volume-title":"Proc. NeurIPS","author":"Muennighoff"},{"key":"ref23","doi-asserted-by":"publisher","DOI":"10.1109\/TNNLS.2023.3323491"},{"key":"ref24","doi-asserted-by":"publisher","DOI":"10.1109\/TIP.2023.3345652"},{"key":"ref25","doi-asserted-by":"publisher","DOI":"10.1109\/TIP.2024.3359045"},{"key":"ref26","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2022.acl-long.196"},{"key":"ref27","doi-asserted-by":"publisher","DOI":"10.1109\/TIP.2024.3468884"},{"key":"ref28","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/P19-1180"},{"key":"ref29","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2022.naacl-main.326"},{"key":"ref30","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2023.acl-long.601"},{"key":"ref31","doi-asserted-by":"publisher","DOI":"10.1109\/TIP.2024.3379900"},{"key":"ref32","doi-asserted-by":"publisher","DOI":"10.1109\/TIP.2022.3197972"},{"key":"ref33","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/P19-1472"},{"key":"ref34","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/P18-1082"},{"key":"ref35","doi-asserted-by":"publisher","DOI":"10.3115\/1220575.1220617"},{"key":"ref36","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2021.naacl-main.457"},{"key":"ref37","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v35i16.17675"},{"key":"ref38","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2023.findings-eacl.5"},{"key":"ref39","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2021.acl-long.353"},{"key":"ref40","first-page":"1","article-title":"Linearly mapping from image to text space","volume-title":"Proc. ICLR","author":"Merullo"},{"key":"ref41","article-title":"Perception, reason, think, and plan: A survey on large multimodal reasoning models","author":"Li","year":"2025","journal-title":"arXiv:2505.04921"},{"key":"ref42","article-title":"Qwen-VL: A frontier large vision-language model with versatile abilities","author":"Bai","year":"2023","journal-title":"arXiv:2308.12966"},{"key":"ref43","article-title":"Crosslingual generalization through multitask finetuning","author":"Muennighoff","year":"2022","journal-title":"arXiv:2211.01786"},{"key":"ref44","article-title":"Scaling instruction-finetuned language models","author":"Won Chung","year":"2022","journal-title":"arXiv:2210.11416"},{"key":"ref45","article-title":"LLaMA-adapter: Efficient fine-tuning of language models with zero-init attention","author":"Zhang","year":"2023","journal-title":"arXiv:2303.16199"},{"key":"ref46","doi-asserted-by":"publisher","DOI":"10.1109\/TPAMI.2025.3532688"},{"key":"ref47","article-title":"Llama 2: Open foundation and fine-tuned chat models","author":"Touvron","year":"2023","journal-title":"arXiv:2307.09288"},{"key":"ref48","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52733.2024.02527"},{"key":"ref49","doi-asserted-by":"crossref","DOI":"10.1109\/CVPR52733.2024.01311","article-title":"Honeybee: Locality-enhanced projector for multimodal LLM","volume-title":"Proc. CVPR","author":"Cha"},{"key":"ref50","first-page":"1028","article-title":"The all-seeing project: Towards panoptic visual recognition and understanding of the open world","volume-title":"Proc. ICLR","author":"Wang"},{"key":"ref51","article-title":"LLaMA-VID: An image is worth 2 tokens in large language models","author":"Li","year":"2023","journal-title":"arXiv:2311.17043"},{"key":"ref52","article-title":"MiniGPT-5: Interleaved vision-and-language generation via generative vokens","author":"Zheng","year":"2023","journal-title":"arXiv:2310.02239"},{"key":"ref53","article-title":"InternLM-XComposer2: Mastering free-form text-image composition and comprehension in vision-language large model","author":"Dong","year":"2024","journal-title":"arXiv:2401.16420"},{"key":"ref54","article-title":"MobileVLM v2: Faster and stronger baseline for vision language model","author":"Chu","year":"2024","journal-title":"arXiv:2402.03766"},{"key":"ref55","article-title":"Safety fine-tuning at (almost) no cost: A baseline for vision large language models","author":"Zong","year":"2024","journal-title":"arXiv:2402.02207"},{"key":"ref56","article-title":"MoE-LLaVA: Mixture of experts for large vision-language models","author":"Lin","year":"2024","journal-title":"arXiv:2401.15947"},{"key":"ref57","article-title":"LLaVA-NeXT: Improved reasoning, OCR, and world knowledge","author":"Liu","year":"2024"},{"key":"ref58","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2023.acl-long.754"},{"key":"ref59","first-page":"10088","article-title":"QLoRA: Efficient finetuning of quantized LLMs","volume-title":"Proc. NeurIPS","author":"Dettmers"},{"key":"ref60","article-title":"LoftQ: LoRA-fine-tuning-aware quantization for large language models","author":"Li","year":"2023","journal-title":"arXiv:2310.08659"},{"key":"ref61","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-58577-8_8"},{"key":"ref62","first-page":"23716","article-title":"Flamingo: A visual language model for few-shot learning","volume-title":"Proc. NeurIPS","author":"Alayrac"},{"key":"ref63","first-page":"23318","article-title":"Unifying architectures, tasks, and modalities through a simple sequence-to-sequence learning framework","volume-title":"Proc. ICML","author":"Wang"},{"key":"ref64","doi-asserted-by":"publisher","DOI":"10.1038\/s43586-021-00018-1"},{"key":"ref65","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR46437.2021.00356"},{"key":"ref66","article-title":"Microsoft COCO captions: Data collection and evaluation server","author":"Chen","year":"2015","journal-title":"arXiv:1504.00325"},{"key":"ref67","doi-asserted-by":"publisher","DOI":"10.1109\/iccv.2015.303"},{"key":"ref68","first-page":"23124","article-title":"WizardLM: Empowering large language models to follow complex instructions","volume-title":"Proc. ICLR","author":"Xu"},{"key":"ref69","article-title":"Measuring massive multitask language understanding","volume-title":"Proc. ICLR","author":"Hendrycks"},{"key":"ref70","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v34i05.6239"},{"key":"ref71","first-page":"4149","article-title":"CommonsenseQA: A question answering challenge targeting commonsense knowledge","volume-title":"Proc. NAACL","author":"Talmor"},{"key":"ref72","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D18-1260"},{"key":"ref73","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2021.findings-acl.131"},{"key":"ref74","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/d19-1454"},{"key":"ref75","doi-asserted-by":"publisher","DOI":"10.1162\/tacl_a_00370"},{"key":"ref76","article-title":"Kosmos-2: Grounding multimodal large language models to the world","author":"Peng","year":"2023","journal-title":"arXiv:2306.14824"},{"key":"ref77","article-title":"Vicuna: An open-source chatbot impressing GPT-4 with 90% ChatGPT quality","author":"Chiang","year":"2023"},{"key":"ref78","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2015.279"},{"key":"ref79","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2019.00331"},{"key":"ref80","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2019.00439"},{"key":"ref81","doi-asserted-by":"publisher","DOI":"10.1109\/ICDAR.2019.00156"},{"key":"ref82","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2019.00851"},{"key":"ref83","doi-asserted-by":"publisher","DOI":"10.1109\/WACV48630.2021.00225"},{"key":"ref84","doi-asserted-by":"publisher","DOI":"10.1007\/s11432-024-4235-6"},{"key":"ref85","article-title":"The RefinedWeb dataset for falcon LLM: Outperforming curated corpora with web data, and web data only","author":"Penedo","year":"2023","journal-title":"arXiv:2306.01116"},{"key":"ref86","article-title":"ImageBind-LLM: Multi-modality instruction tuning","author":"Han","year":"2023","journal-title":"arXiv:2309.03905"},{"key":"ref87","first-page":"17283","article-title":"Grounding language models to images for multimodal generation","volume-title":"Proc. ICML","author":"Koh"},{"key":"ref88","article-title":"InstructBLIP: Towards general-purpose vision-language models with instruction tuning","volume-title":"Proc. NeurIPS","author":"Dai"},{"key":"ref89","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV51070.2023.01092"},{"key":"ref90","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52729.2023.02238"},{"key":"ref91","article-title":"Adam: A method for stochastic optimization","author":"Kingma","year":"2014","journal-title":"arXiv:1412.6980"},{"key":"ref92","first-page":"9694","article-title":"Align before fuse: Vision and language representation learning with momentum distillation","volume-title":"Proc. NeurIPS","author":"Li"},{"key":"ref93","article-title":"FILIP: Fine-grained interactive language-image pre-training","author":"Yao","year":"2021","journal-title":"arXiv:2111.07783"},{"key":"ref94","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2021.acl-long.42"},{"key":"ref95","article-title":"Florence: A new foundation model for computer vision","author":"Yuan","year":"2021","journal-title":"arXiv:2111.11432"},{"key":"ref96","article-title":"CoCa: Contrastive captioners are image-text foundation models","author":"Yu","year":"2022","journal-title":"arXiv:2205.01917"},{"key":"ref97","article-title":"GIT: A generative image-to-text transformer for vision and language","author":"Wang","year":"2022","journal-title":"arXiv:2205.14100"},{"key":"ref98","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52733.2024.02484"},{"key":"ref99","article-title":"MultiModal-GPT: A vision and language model for dialogue with humans","author":"Gong","year":"2023","journal-title":"arXiv:2305.04790"},{"key":"ref100","article-title":"PandaGPT: One model to instruction-follow them all","author":"Su","year":"2023","journal-title":"arXiv:2305.16355"},{"key":"ref101","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2022.acl-long.26"},{"key":"ref102","doi-asserted-by":"publisher","DOI":"10.1109\/TPAMI.2025.3571946"},{"key":"ref103","article-title":"Shikra: Unleashing multimodal LLM\u2019s referential dialogue magic","author":"Chen","year":"2023","journal-title":"arXiv:2306.15195"},{"key":"ref104","article-title":"LAION-400M: Open dataset of CLIP-filtered 400 million image-text pairs","author":"Schuhmann","year":"2021","journal-title":"arXiv:2111.02114"}],"container-title":["IEEE Transactions on Image Processing"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx8\/83\/11355710\/11353361.pdf?arnumber=11353361","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,1,27]],"date-time":"2026-01-27T06:07:18Z","timestamp":1769494038000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/11353361\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026]]},"references-count":104,"URL":"https:\/\/doi.org\/10.1109\/tip.2025.3649356","relation":{},"ISSN":["1057-7149","1941-0042"],"issn-type":[{"value":"1057-7149","type":"print"},{"value":"1941-0042","type":"electronic"}],"subject":[],"published":{"date-parts":[[2026]]}}}