{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,17]],"date-time":"2026-07-17T04:16:29Z","timestamp":1784261789251,"version":"3.55.0"},"reference-count":94,"publisher":"Institute of Electrical and Electronics Engineers (IEEE)","license":[{"start":{"date-parts":[[2026,1,1]],"date-time":"2026-01-01T00:00:00Z","timestamp":1767225600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/ieeexplore.ieee.org\/Xplorehelp\/downloads\/license-information\/IEEE.html"},{"start":{"date-parts":[[2026,1,1]],"date-time":"2026-01-01T00:00:00Z","timestamp":1767225600000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2026,1,1]],"date-time":"2026-01-01T00:00:00Z","timestamp":1767225600000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"}],"funder":[{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["62332002"],"award-info":[{"award-number":["62332002"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["62425101"],"award-info":[{"award-number":["62425101"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]},{"name":"Shenzhen Science and Technology Program","award":["KQTD20240729102051063"],"award-info":[{"award-number":["KQTD20240729102051063"]}]}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["IEEE Trans. Multimedia"],"published-print":{"date-parts":[[2026]]},"DOI":"10.1109\/tmm.2026.3654458","type":"journal-article","created":{"date-parts":[[2026,1,15]],"date-time":"2026-01-15T20:50:42Z","timestamp":1768510242000},"page":"4408-4419","source":"Crossref","is-referenced-by-count":20,"title":["MoE-LLaVA: Mixture of Experts for Large Vision-Language Models"],"prefix":"10.1109","volume":"28","author":[{"ORCID":"https:\/\/orcid.org\/0009-0003-4805-9730","authenticated-orcid":false,"given":"Bin","family":"Lin","sequence":"first","affiliation":[{"name":"College of Information Engineering, Peking Univercity Shenzhen Graduate School, Shenzhen, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Zhenyu","family":"Tang","sequence":"additional","affiliation":[{"name":"College of Information Engineering, Peking Univercity Shenzhen Graduate School, Shenzhen, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Yang","family":"Ye","sequence":"additional","affiliation":[{"name":"College of Information Engineering, Peking Univercity Shenzhen Graduate School, Shenzhen, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-0081-4106","authenticated-orcid":false,"given":"Jinfa","family":"Huang","sequence":"additional","affiliation":[{"name":"Department of Computer Science, University of Rochester Rochester, NY, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Junwu","family":"Zhang","sequence":"additional","affiliation":[{"name":"College of Information Engineering, Peking Univercity Shenzhen Graduate School, Shenzhen, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Yatian","family":"Pang","sequence":"additional","affiliation":[{"name":"College of Information Engineering, Peking Univercity Shenzhen Graduate School, Shenzhen, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-9287-6410","authenticated-orcid":false,"given":"Peng","family":"Jin","sequence":"additional","affiliation":[{"name":"College of Information Engineering, Peking Univercity Shenzhen Graduate School, Shenzhen, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0005-3418-085X","authenticated-orcid":false,"given":"Munan","family":"Ning","sequence":"additional","affiliation":[{"name":"College of Information Engineering, Peking Univercity Shenzhen Graduate School, Shenzhen, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-4516-9729","authenticated-orcid":false,"given":"Jiebo","family":"Luo","sequence":"additional","affiliation":[{"name":"Department of Computer Science, University of Rochester Rochester, NY, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-2120-5588","authenticated-orcid":false,"given":"Li","family":"Yuan","sequence":"additional","affiliation":[{"name":"College of Information Engineering, Peking Univercity Shenzhen Graduate School, Shenzhen, China"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"263","reference":[{"issue":"8","key":"ref1","article-title":"Language models are unsupervised multitask learners","volume":"1","author":"Radford","year":"2019","journal-title":"OpenAI Blog"},{"key":"ref2","volume-title":"Natural Language Understanding","author":"Allen","year":"1988"},{"key":"ref3","doi-asserted-by":"publisher","DOI":"10.1073\/pnas.92.22.9977"},{"key":"ref4","doi-asserted-by":"publisher","DOI":"10.1145\/3554727"},{"key":"ref5","first-page":"121","article-title":"Natural language generation","volume-title":"Handbook of Natural Language Processing","volume":"2","author":"McDonald","year":"2010"},{"key":"ref6","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52729.2023.01839"},{"key":"ref7","article-title":"Sus-chat: Instruction tuning done right","year":"2023"},{"key":"ref8","article-title":"Building the next generation of open-source and bilingual LLMs","year":"2023"},{"key":"ref9","article-title":"Aquila2-34b","year":"2023"},{"key":"ref10","first-page":"1877","article-title":"Language models are few-shot learners","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","volume":"33","author":"Brown","year":"2020"},{"key":"ref11","article-title":"GLM-130B: An open bilingual pre-trained model","volume-title":"Proc. 11th Int. Conf. Learn. Representations","author":"Zeng","year":"2022"},{"key":"ref12","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.emnlp-main.700"},{"key":"ref13","article-title":"Palm 2 technical report","author":"Anil","year":"2023"},{"key":"ref14","article-title":"GPT-4 technical report","year":"2023"},{"key":"ref15","doi-asserted-by":"publisher","DOI":"10.1162\/neco.1991.3.1.79"},{"key":"ref16","article-title":"Learning factored representations in a deep mixture of experts","author":"Eigen","year":"2013"},{"issue":"1","key":"ref17","first-page":"5232","article-title":"Switch transformers: Scaling to trillion parameter models with simple and efficient sparsity","volume":"23","author":"Fedus","year":"2022","journal-title":"J. Mach. Learn. Res."},{"key":"ref18","article-title":"Mistral 7B","author":"Jiang","year":"2023"},{"key":"ref19","article-title":"Mixtral of experts","author":"Jiang","year":"2024"},{"key":"ref20","article-title":"Llama 2: Open foundation and fine-tuned chat models","author":"Touvron","year":"2023"},{"key":"ref21","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2023.emnlp-main.20"},{"key":"ref22","first-page":"8583","article-title":"Scaling vision with sparse mixture of experts","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","volume":"34","author":"Riquelme","year":"2021"},{"key":"ref23","article-title":"Llama: Open and efficient foundation language models","author":"Touvron","year":"2023"},{"key":"ref24","doi-asserted-by":"publisher","DOI":"10.52202\/068431-1800"},{"key":"ref25","first-page":"19730","article-title":"BLIP-2: Bootstrapping language-image pre-training with frozen image encoders and large language models","volume-title":"Proc. Int. Conf. Mach. learn.","author":"Li","year":"2023"},{"key":"ref26","article-title":"Grounding language models to images for multimodal generation","author":"Koh","year":"2023"},{"key":"ref27","article-title":"Aligning large multi-modal model with robust instruction tuning","author":"Liu","year":"2023"},{"key":"ref28","doi-asserted-by":"publisher","DOI":"10.52202\/075280-1516"},{"key":"ref29","article-title":"LLaVAr: Enhanced visual instruction tuning for text-rich image understanding","author":"Zhang","year":"2023"},{"key":"ref30","article-title":"SViT: Scaling up visual instruction tuning","author":"Zhao","year":"2023"},{"key":"ref31","article-title":"Qwen-Vl: A frontier large vision-language model with versatile abilities","author":"Bai","year":"2023"},{"key":"ref32","article-title":"MiniGPT-V2: Large language model as a unified interface for vision-language multi-task learning","author":"Chen","year":"2023"},{"key":"ref33","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52733.2024.02484"},{"key":"ref34","first-page":"24185","article-title":"InternVL: Scaling up vision foundation models and aligning for generic visual-linguistic tasks","volume-title":"Proc. IEEE\/CVF Conf. Comput. Vis. Pattern Recognit.","author":"Chen","year":"2024"},{"key":"ref35","article-title":"InternLM-XComposer: A vision-language large model for advanced text-image comprehension and composition","author":"Zhang","year":"2023"},{"key":"ref36","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2024.emnlp-main.342"},{"key":"ref37","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52733.2024.01311"},{"key":"ref38","doi-asserted-by":"publisher","DOI":"10.1093\/oed\/6775979618"},{"key":"ref39","article-title":"BuboGPT: Enabling visual grounding in multi-modal LLMs","author":"Zhao","year":"2023"},{"key":"ref40","article-title":"InternGPT: Solving vision-centric tasks by interacting with ChatGPT beyond language","author":"Liu","year":"2023"},{"key":"ref41","first-page":"61501","article-title":"VisionLLM: Large language model is also an open-ended decoder for vision-centric tasks","volume":"36","author":"Wang","year":"2023","journal-title":"Adv. Neural Inf. Process. Syst."},{"key":"ref42","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2023.emnlp-main.876"},{"key":"ref43","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52733.2024.01236"},{"key":"ref44","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52688.2022.01170"},{"key":"ref45","first-page":"17612","article-title":"Mind the gap: Understanding the modality gap in multi-modal contrastive representation learning","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","volume":"35","author":"Liang","year":"2022"},{"key":"ref46","first-page":"32897","article-title":"VLMO: Unified vision-language pre-training with mixture-of-modality-experts","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","volume":"35","author":"Bao","year":"2022"},{"key":"ref47","article-title":"Multiway-adapater: Adapting large-scale multi-modal models for scalable image-text retrieval","author":"Long","year":"2023"},{"key":"ref48","article-title":"ROME: Role-aware mixture-of-expert transformer for text-to-video retrieval","author":"Satar","year":"2022"},{"key":"ref49","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2023.acl-long.749"},{"key":"ref50","doi-asserted-by":"publisher","DOI":"10.52202\/068431-0193"},{"key":"ref51","article-title":"Cot-mote: Exploring contextual masked auto-encoder pre-training with mixture-of-textual-experts for passage retrieval","author":"Ma","year":"2023"},{"key":"ref52","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2021.findings-emnlp.304"},{"key":"ref53","article-title":"Outrageously large neural networks: The sparsely-gated mixture-of-experts layer","author":"Shazeer","year":"2017"},{"key":"ref54","article-title":"GShard: Scaling giant models with conditional computation and automatic sharding","author":"Lepikhin","year":"2020"},{"key":"ref55","doi-asserted-by":"publisher","DOI":"10.1109\/ipdpsw55747.2022.00171"},{"key":"ref56","article-title":"Sparse upcycling: Training mixture-of-experts from dense checkpoints","author":"Komatsuzaki","year":"2022"},{"key":"ref57","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v38i2.27872"},{"key":"ref58","first-page":"9564","article-title":"Multimodal contrastive learning with liMOE: The language-image mixture of experts","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","volume":"35","author":"Mustafa","year":"2022"},{"key":"ref59","article-title":"LLaVA-Mole: Sparse mixture of LoRa experts for mitigating data conflicts in instruction finetuning mLLMs","author":"Chen","year":"2024"},{"key":"ref60","doi-asserted-by":"publisher","DOI":"10.1109\/tip.2026.3678087"},{"key":"ref61","doi-asserted-by":"publisher","DOI":"10.1145\/3805622.3810616"},{"key":"ref62","doi-asserted-by":"publisher","DOI":"10.52202\/079017-1330"},{"key":"ref63","article-title":"Kimi-Vl technical report","author":"Team","year":"2025"},{"key":"ref64","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2024.acl-srw.8"},{"key":"ref65","article-title":"DeepSeek-Vl2: Mixture-of-experts vision-language models for advanced multimodal understanding","author":"Wu","year":"2024"},{"key":"ref66","article-title":"Aria: An open multimodal native mixture-of-experts model","author":"Li","year":"2024"},{"key":"ref67","doi-asserted-by":"publisher","DOI":"10.1145\/3777411"},{"key":"ref68","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/P19-1176"},{"key":"ref69","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/p19-1176"},{"key":"ref70","article-title":"To see is to believe: Prompting GPT-4V for better visual instruction tuning","author":"Wang","year":"2023"},{"key":"ref71","article-title":"Mimic-It: Multi-modal in-context instruction tuning","author":"Li","year":"2023"},{"key":"ref72","first-page":"8748","article-title":"Learning transferable visual models from natural language supervision","volume-title":"Proc. 38th Int. Conf. Mach. Learn.","author":"Radford","year":"2021"},{"key":"ref73","article-title":"Gaussian error linear units (GELUs)","author":"Hendrycks","year":"2016"},{"key":"ref74","article-title":"Vicuna: An open-source chatbot impressing GPT-4 with 90% ChatGPT quality","author":"Chiang","year":"2023"},{"key":"ref75","article-title":"Stable LM 2 1.6b","author":"Team"},{"key":"ref76","article-title":"Qwen technical report","author":"Bai","year":"2023"},{"key":"ref77","article-title":"Phi-2: The surprising power of small language models","year":"2023"},{"key":"ref78","article-title":"MobileVLM: A fast, reproducible and strong vision language assistant for mobile devices","author":"Chu","year":"2023"},{"key":"ref79","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2017.670"},{"key":"ref80","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2019.00686"},{"key":"ref81","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2018.00380"},{"key":"ref82","doi-asserted-by":"publisher","DOI":"10.52202\/068431-0182"},{"key":"ref83","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2019.00851"},{"key":"ref84","article-title":"MME: A comprehensive evaluation benchmark for multimodal large language models","volume-title":"Proc. 39th Annu. Conf. Neural Inf. Process. Syst. Datasets Benchmarks Track","author":"Fu","year":"2025"},{"key":"ref85","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-72658-3_13"},{"key":"ref86","first-page":"57730","article-title":"MM-VET: Evaluating large multimodal models for integrated capabilities","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Yu","year":"2024"},{"key":"ref87","doi-asserted-by":"publisher","DOI":"10.52202\/075280-3138"},{"key":"ref88","article-title":"TinyGPT-V: Efficient multimodal large language model via small backbones","volume-title":"Proc. 2nd Workshop Adv. Neural Network Training: Comput. Efficiency, Scalability, Resour. Optim.","author":"Yuan","year":"2024"},{"key":"ref89","doi-asserted-by":"publisher","DOI":"10.1145\/3688863.3689575"},{"key":"ref90","article-title":"MPlug-Owl: Modularization empowers large language models with multimodality","author":"Ye","year":"2023"},{"key":"ref91","article-title":"Multimodal-GPT: A vision and language model for dialogue with humans","author":"Gong","year":"2023"},{"key":"ref92","article-title":"MiniGPT-4: Enhancing vision-language understanding with advanced large language models","volume-title":"Proc. 12th Int. Conf. Learn. Representations","author":"Zhu","year":"2023"},{"key":"ref93","doi-asserted-by":"publisher","DOI":"10.52202\/075280-2142"},{"key":"ref94","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-72643-9_22"}],"container-title":["IEEE Transactions on Multimedia"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx8\/6046\/11342315\/11353922.pdf?arnumber=11353922","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,6,19]],"date-time":"2026-06-19T19:40:59Z","timestamp":1781898059000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/11353922\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026]]},"references-count":94,"URL":"https:\/\/doi.org\/10.1109\/tmm.2026.3654458","relation":{},"ISSN":["1520-9210","1941-0077"],"issn-type":[{"value":"1520-9210","type":"print"},{"value":"1941-0077","type":"electronic"}],"subject":[],"published":{"date-parts":[[2026]]}}}