{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,6,4]],"date-time":"2026-06-04T16:02:08Z","timestamp":1780588928573,"version":"3.54.1"},"reference-count":228,"publisher":"Springer Science and Business Media LLC","issue":"11","license":[{"start":{"date-parts":[[2025,8,30]],"date-time":"2025-08-30T00:00:00Z","timestamp":1756512000000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2025,8,30]],"date-time":"2025-08-30T00:00:00Z","timestamp":1756512000000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Int J Comput Vis"],"published-print":{"date-parts":[[2025,11]]},"DOI":"10.1007\/s11263-025-02572-7","type":"journal-article","created":{"date-parts":[[2025,8,30]],"date-time":"2025-08-30T15:45:09Z","timestamp":1756568709000},"page":"8151-8189","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":7,"title":["Visual Instruction Tuning towards General-Purpose Multimodal Large Language Model: A Survey"],"prefix":"10.1007","volume":"133","author":[{"given":"Jiaxing","family":"Huang","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Jingyi","family":"Zhang","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Kai","family":"Jiang","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Han","family":"Qiu","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Xiaoqin","family":"Zhang","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Ling","family":"Shao","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-6766-2506","authenticated-orcid":false,"given":"Shijian","family":"Lu","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Dacheng","family":"Tao","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2025,8,30]]},"reference":[{"key":"2572_CR1","unstructured":". Lu, H., Liu, W., Zhang, B., Wang, B., Dong, K., Liu, B., Sun, J., Ren, T., Li, Z., Yang, H., et al. (2024). Deepseek-vl: towards real-world vision-language understanding. arXiv preprint arXiv:2403.05525."},{"key":"2572_CR2","doi-asserted-by":"crossref","unstructured":"Agrawal, H., Desai, K., Wang, Y., Chen, X., Jain, R., Johnson, M., Batra, D., Parikh, D., Lee, S., & Anderson, P. (2019). Nocaps: Novel object captioning at scale. In: Proceedings of the IEEE\/CVF International Conference on Computer Vision, pp. 8948\u20138957.","DOI":"10.1109\/ICCV.2019.00904"},{"key":"2572_CR3","doi-asserted-by":"crossref","unstructured":"An, W., Tian, F., Leng, S., Nie, J., Lin, H., Wang, Q., Dai, G., Chen, P., & Lu, S. (2024). Agla: Mitigating object hallucinations in large vision-language models with assembly of global and local attention. arXiv preprint arXiv:2406.12718.","DOI":"10.1109\/CVPR52734.2025.02784"},{"key":"2572_CR4","doi-asserted-by":"crossref","unstructured":"Azuma, D., Miyanishi, T., Kurita, S., & Kawanabe, M. (2022). Scanqa: 3d question answering for spatial scene understanding. In: Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition, pp. 19129\u201319139.","DOI":"10.1109\/CVPR52688.2022.01854"},{"key":"2572_CR5","unstructured":"Bai, J., Bai, S., Yang, S., Wang, S., Tan, S., Wang, P., Lin, J., Zhou, C., & Zhou, J. (2023). Qwen-vl: A versatile vision-language model for understanding, localization, text reading, and beyond. 3(1). arXiv preprint arXiv:2308.12966."},{"key":"2572_CR6","doi-asserted-by":"publisher","first-page":"6833","DOI":"10.52202\/079017-0219","volume":"37","author":"Z Bai","year":"2024","unstructured":"Bai, Z., He, T., Mei, H., Wang, P., Gao, Z., Chen, J., Zhang, Z., & Shou, M. Z. (2024). One token to seg them all: Language instructed reasoning segmentation in videos. Advances in Neural Information Processing Systems, 37, 6833\u20136859.","journal-title":"Advances in Neural Information Processing Systems"},{"key":"2572_CR7","doi-asserted-by":"crossref","unstructured":"Cai, M., Liu, H., Karthik\u00a0Mustikovela, S., Meyer, G.P., Chai, Y., Park, D., & Lee, Y.J. (2023). Making large multimodal models understand arbitrary visual prompts. arXiv e-prints, 2312.","DOI":"10.1109\/CVPR52733.2024.01227"},{"key":"2572_CR8","unstructured":"Cao, M., Liu, Y., Liu, Y., Wang, T., Dong, J., Ding, H., Zhang, X., Reid, I., & Liang, X. (2024). Continual llava: Continual instruction tuning in large vision-language models. arXiv preprint arXiv:2411.02564."},{"key":"2572_CR9","doi-asserted-by":"crossref","unstructured":"Cha, J., Kang, W., Mun, J., & Roh, B. (2024). Honeybee: Locality-enhanced projector for multimodal llm. In: Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition, pp. 13817\u201313827.","DOI":"10.1109\/CVPR52733.2024.01311"},{"key":"2572_CR10","doi-asserted-by":"crossref","unstructured":"Chen, Q., & Hong, Y. (2024). Medblip: Bootstrapping language-image pre-training from 3d medical images and texts. In: Proceedings of the Asian Conference on Computer Vision, pp. 2404\u20132420.","DOI":"10.1007\/978-981-96-0908-6_6"},{"key":"2572_CR11","doi-asserted-by":"crossref","unstructured":"Chen, S., Chen, X., Zhang, C., Li, M., Yu, G., Fei, H., Zhu, H., Fan, J., & Chen, T. (2024). Ll3da: Visual interactive instruction tuning for omni-3d understanding reasoning and planning. In: Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition, pp. 26428\u201326438","DOI":"10.1109\/CVPR52733.2024.02496"},{"key":"2572_CR12","unstructured":"Chen, Z., Duan, Y., Wang, W., He, J., Lu, T., Dai, J., & Qiao, Y. (2022). Vision transformer adapter for dense predictions. arXiv preprint arXiv:2205.08534."},{"key":"2572_CR13","unstructured":"Chen, X., Fang, H., Lin, T.-Y., Vedantam, R., Gupta, S., Doll\u00e1r, P., & Zitnick, C.L. (2015). Microsoft coco captions: Data collection and evaluation server. arXiv preprint arXiv:1504.00325."},{"key":"2572_CR14","unstructured":"Chen, F., Han, M., Zhao, H., Zhang, Q., Shi, J., Xu, S., & XLLM, B.X. (2023). bootstrapping advanced large language models by treating multi-modalities as foreign languages. corr. 10.48550. arXiv preprint arXiv:2305.04160."},{"key":"2572_CR15","unstructured":"Chen, S., Lan, X., Yuan, Y., Jie, Z., & Ma, L. (2024). Timemarker: A versatile video-llm for long and short video understanding with superior temporal localization ability. arXiv preprint arXiv:2411.18211."},{"key":"2572_CR16","doi-asserted-by":"crossref","unstructured":"Chen, Y.-C., Li, L., Yu, L., El\u00a0Kholy, A., Ahmed, F., Gan, Z., Cheng, Y., & Liu, J. (2020). Uniter: Universal image-text representation learning. In: European Conference on Computer Vision, pp. 104\u2013120. Springer.","DOI":"10.1007\/978-3-030-58577-8_7"},{"key":"2572_CR17","unstructured":"Chen, C., Qin, R., Luo, F., Mi, X., Li, P., Sun, M., & Liu, Y. (2023). Position-enhanced visual instruction tuning for multimodal large language models. arXiv preprint arXiv:2308.13437."},{"key":"2572_CR18","unstructured":"Chen, W.-G., Spiridonova, I., Yang, J., Gao, J., & Li, C. (2023). Llava-interactive: An all-in-one demo for image chat, segmentation, generation and editing. arXiv preprint arXiv:2311.00571."},{"key":"2572_CR19","unstructured":"Chen, Z., Varma, M., Delbrouck, J.-B., Paschali, M., Blankemeier, L., Van\u00a0Veen, D., Valanarasu, J.M.J., Youssef, A., Cohen, J.P., Reis, E.P., et al. (2024). Chexagent: Towards a foundation model for chest x-ray interpretation. arXiv preprint arXiv:2401.12208."},{"key":"2572_CR20","unstructured":"Chen, W., Wang, H., Chen, J., Zhang, Y., Wang, H., Li, S., Zhou, X., & Wang, W.Y. (2019). Tabfact: A large-scale dataset for table-based fact verification. arXiv preprint arXiv:1909.02164."},{"key":"2572_CR21","doi-asserted-by":"crossref","unstructured":"Chen, Z., Wu, J., Wang, W., Su, W., Chen, G., Xing, S., Zhong, M., Zhang, Q., Zhu, X., & Lu, L. (2024). Internvl: Scaling up vision foundation models and aligning for generic visual-linguistic tasks. In: Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition, pp. 24185\u201324198.","DOI":"10.1109\/CVPR52733.2024.02283"},{"key":"2572_CR22","unstructured":"Chen, K., Zhang, Z., Zeng, W., Zhang, R., Zhu, F., & Zhao, R. (2023). Shikra: Unleashing multimodal llm\u2019s referential dialogue magic. arXiv preprint arXiv:2306.15195."},{"key":"2572_CR23","unstructured":"Chen, J., Zhu, D., Shen, X., Li, X., Liu, Z., Zhang, P., Krishnamoorthi, R., Chandra, V., Xiong, Y., & Elhoseiny, M. (2023). Minigpt-v2: large language model as a unified interface for vision-language multi-task learning. arXiv preprint arXiv:2310.09478."},{"key":"2572_CR24","first-page":"17745","volume":"38","author":"D Chen","year":"2024","unstructured":"Chen, D., Liu, J., Dai, W., & Wang, B. (2024). Visual instruction tuning with polite flamingo. In: Proceedings of the AAAI Conference on Artificial Intelligence, 38, 17745\u201317753.","journal-title":"In: Proceedings of the AAAI Conference on Artificial Intelligence"},{"key":"2572_CR25","unstructured":"Cheng, Z., Leng, S., Zhang, H., Xin, Y., Li, X., Chen, G., Zhu, Y., Zhang, W., Luo, Z., Zhao, D., et al. (2024). Videollama 2: Advancing spatial-temporal modeling and audio understanding in video-llms. arXiv preprint arXiv:2406.07476."},{"key":"2572_CR26","doi-asserted-by":"publisher","first-page":"110805","DOI":"10.52202\/079017-3518","volume":"37","author":"Z Cheng","year":"2024","unstructured":"Cheng, Z., Cheng, Z.-Q., He, J.-Y., Wang, K., Lin, Y., Lian, Z., Peng, X., & Hauptmann, A. (2024). Emotion-llama: Multimodal emotion recognition and reasoning with instruction tuning. Advances in Neural Information Processing Systems, 37, 110805\u2013110853.","journal-title":"Advances in Neural Information Processing Systems"},{"key":"2572_CR27","unstructured":"Chu, X., Qiao, L., Lin, X., Xu, S., Yang, Y., Hu, Y., Wei, F., Zhang, X., Zhang, B., Wei, X., et al. (2023). Mobilevlm: A fast, reproducible and strong vision language assistant for mobile devices. arXiv preprint arXiv:2312.16886."},{"key":"2572_CR28","doi-asserted-by":"crossref","unstructured":"Conde, M.V., Geigle, G., & Timofte, R. (2024). Instructir: High-quality image restoration following human instructions. In: European Conference on Computer Vision, pp. 1\u201321. Springer.","DOI":"10.1007\/978-3-031-72764-1_1"},{"key":"2572_CR29","doi-asserted-by":"crossref","unstructured":"Cui, H., Mao, L., Liang, X., Zhang, J., Ren, H., Li, Q., Li, X., & Yang, C. (2024). Biomedical visual instruction tuning with clinician preference alignment. arXiv preprint arXiv:2406.13173.","DOI":"10.52202\/079017-3057"},{"key":"2572_CR30","doi-asserted-by":"crossref","unstructured":"Dai, A., Chang, A.X., Savva, M., Halber, M., Funkhouser, T., & Nie\u00dfner, M. (2017). Scannet: Richly-annotated 3d reconstructions of indoor scenes. In: Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition, pp. 5828\u20135839.","DOI":"10.1109\/CVPR.2017.261"},{"key":"2572_CR31","unstructured":"Dai, W., Li, J., Li, D., Tiong, A.M.H., Zhao, J., Wang, W., Li, B., Fung, P., & Hoi, S. (2023). InstructBLIP: Towards General-purpose Vision-Language Models with Instruction Tuning."},{"key":"2572_CR32","doi-asserted-by":"crossref","unstructured":"Deitke, M., Schwenk, D., Salvador, J., Weihs, L., Michel, O., VanderBilt, E., Schmidt, L., Ehsani, K., Kembhavi, A., & Farhadi, A. (2023). Objaverse: A universe of annotated 3d objects. In: Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition, pp. 13142\u201313153.","DOI":"10.1109\/CVPR52729.2023.01263"},{"key":"2572_CR33","doi-asserted-by":"crossref","unstructured":"Deng, J., Dong, W., Socher, R., Li, L.-J., Li, K., & Fei-Fei, L. (2009). Imagenet: A large-scale hierarchical image database. In: 2009 IEEE Conference on Computer Vision and Pattern Recognition, pp. 248\u2013255. Ieee.","DOI":"10.1109\/CVPR.2009.5206848"},{"key":"2572_CR34","unstructured":"Dettmers, T., Pagnoni, A., Holtzman, A., & Zettlemoyer, L. (2023). Qlora: Efficient finetuning of quantized llms. arXiv preprint arXiv:2305.14314"},{"key":"2572_CR35","unstructured":"Dong, R., Han, C., Peng, Y., Qi, Z., Ge, Z., Yang, J., Zhao, L., Sun, J., Zhou, H., Wei, H., et al. (2023). Dreamllm: Synergistic multimodal comprehension and creation. arXiv preprint arXiv:2309.11499."},{"key":"2572_CR36","unstructured":"Dosovitskiy, A., Beyer, L., Kolesnikov, A., Weissenborn, D., Zhai, X., Unterthiner, T., Dehghani, M., Minderer, M., Heigold, G., Gelly, S., et al. (2020). An image is worth 16x16 words: Transformers for image recognition at scale. arXiv preprint arXiv:2010.11929."},{"key":"2572_CR37","unstructured":"Emma, D., Jared, Jorge, & Cukierski, W. (2015). Diabetic Retinopathy Detection. Kaggle. https:\/\/kaggle.com\/competitions\/diabetic-retinopathy-detection."},{"issue":"2","key":"2572_CR38","doi-asserted-by":"publisher","first-page":"303","DOI":"10.1007\/s11263-009-0275-4","volume":"88","author":"M Everingham","year":"2010","unstructured":"Everingham, M., Van Gool, L., Williams, C. K., Winn, J., & Zisserman, A. (2010). The pascal visual object classes (voc) challenge. International journal of computer vision, 88(2), 303\u2013338.","journal-title":"International journal of computer vision"},{"issue":"10","key":"2572_CR39","doi-asserted-by":"publisher","first-page":"2828","DOI":"10.1109\/TMI.2022.3172773","volume":"41","author":"H Fang","year":"2022","unstructured":"Fang, H., Li, F., Fu, H., Sun, X., Cao, X., Lin, F., Son, J., Kim, S., Quellec, G., & Matta, S. (2022). Adam challenge: Detecting age-related macular degeneration from fundus images. IEEE Transactions on Medical Imaging, 41(10), 2828\u20132847.","journal-title":"IEEE Transactions on Medical Imaging"},{"key":"2572_CR40","doi-asserted-by":"crossref","unstructured":"Fei-Fei, L., Fergus, R., & Perona, P. (2004). Learning generative visual models from few training examples: An incremental bayesian approach tested on 101 object categories. In: 2004 Conference on Computer Vision and Pattern Recognition Workshop, pp. 178\u2013178. IEEE","DOI":"10.1109\/CVPR.2004.383"},{"key":"2572_CR41","unstructured":"Fu, C., Chen, P., Shen, Y., Qin, Y., Zhang, M., Lin, X., Yang, J., Zheng, X., Li, K., Sun, X., et al. (2023). Mme: A comprehensive evaluation benchmark for multimodal large language models. arXiv preprint arXiv:2306.13394."},{"key":"2572_CR42","doi-asserted-by":"crossref","unstructured":"Gao, W., Deng, Z., Niu, Z., Rong, F., Chen, C., Gong, Z., Zhang, W., Xiao, D., Li, F., Cao, Z., et al. (2023). Ophglm: Training an ophthalmology large language-and-vision assistant based on instructions and dialogue. arXiv preprint arXiv:2306.12174.","DOI":"10.1016\/j.artmed.2024.103001"},{"key":"2572_CR43","unstructured":"Gao, P., Geng, S., Zhang, R., Ma, T., Fang, R., Zhang, Y., Li, H., & Qiao, Y. (2021). Clip-adapter: Better vision-language models with feature adapters. arXiv preprint arXiv:2110.04544."},{"key":"2572_CR44","unstructured":"Gao, P., Han, J., Zhang, R., Lin, Z., Geng, S., Zhou, A., Zhang, W., Lu, P., He, C., Yue, X., et al. (2023). Llama-adapter v2: Parameter-efficient visual instruction model. arXiv preprint arXiv:2304.15010."},{"key":"2572_CR45","unstructured":"Gao, D., Ji, L., Zhou, L., Lin, K.Q., Chen, J., Fan, Z., & Shou, M.Z. (2023) Assistgpt: A general multi-modal assistant that can plan, execute, inspect, and learn. arXiv preprint arXiv:2306.08640."},{"key":"2572_CR46","unstructured":"Ge, Y., Zhao, S., Zeng, Z., Ge, Y., Li, C., Wang, X., & Shan, Y. (2023). Making llama see and draw with seed tokenizer. arXiv preprint arXiv:2310.01218."},{"key":"2572_CR47","unstructured":"Gong, T., Lyu, C., Zhang, S., Wang, Y., Zheng, M., Zhao, Q., Liu, K., Zhang, W., Luo, P., & Chen, K. (2023). Multimodal-gpt: A vision and language model for dialogue with humans. arXiv preprint arXiv:2305.04790."},{"key":"2572_CR48","doi-asserted-by":"crossref","unstructured":"Goyal, Y., Khot, T., Summers-Stay, D., Batra, D., & Parikh, D. (2017). Making the v in vqa matter: Elevating the role of image understanding in visual question answering. In: Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition, pp. 6904\u20136913.","DOI":"10.1109\/CVPR.2017.670"},{"issue":"4","key":"2572_CR49","doi-asserted-by":"publisher","first-page":"1498","DOI":"10.1109\/TITS.2012.2208909","volume":"13","author":"J Greenhalgh","year":"2012","unstructured":"Greenhalgh, J., & Mirmehdi, M. (2012). Real-time detection and recognition of road traffic signs. IEEE transactions on intelligent transportation systems, 13(4), 1498\u20131506.","journal-title":"IEEE transactions on intelligent transportation systems"},{"key":"2572_CR50","unstructured":"Guo, Z., Zhang, R., Zhu, X., Tang, Y., Ma, X., Han, J., Chen, K., Gao, P., Li, X., Li, H., et al. (2023). Point-bind & point-llm: Aligning point cloud with multi-modality for 3d understanding, generation, and instruction following. arXiv preprint arXiv:2309.00615."},{"key":"2572_CR51","doi-asserted-by":"crossref","unstructured":"Gupta, A., Dollar, P., & Girshick, R. (2019). Lvis: A dataset for large vocabulary instance segmentation. In: Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition, pp. 5356\u20135364.","DOI":"10.1109\/CVPR.2019.00550"},{"key":"2572_CR52","doi-asserted-by":"crossref","unstructured":"Gurari, D., Li, Q., Stangl, A.J., Guo, A., Lin, C., Grauman, K., Luo, J., & Bigham, J.P. (2018). Vizwiz grand challenge: Answering visual questions from blind people. In: Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition, pp. 3608\u20133617.","DOI":"10.1109\/CVPR.2018.00380"},{"key":"2572_CR53","doi-asserted-by":"crossref","unstructured":"Han, J., Gong, K., Zhang, Y., Wang, J., Zhang, K., Lin, D., Qiao, Y., Gao, P., & Yue, X. (2024). Onellm: One framework to align all modalities with language. In: Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition, pp. 26584\u201326595.","DOI":"10.1109\/CVPR52733.2024.02510"},{"key":"2572_CR54","doi-asserted-by":"crossref","unstructured":"Han, C., Wang, Q., Dianat, S.A., Rabbani, M., Rao, R.M., Fang, Y., Guan, Q., Huang, L., & Liu, D. (2024). Amd: Automatic multi-step distillation of large-scale vision models. In: European Conference on Computer Vision, pp. 431\u2013450. Springer.","DOI":"10.1007\/978-3-031-73650-6_25"},{"key":"2572_CR55","unstructured":"Han, J., Zhang, R., Shao, W., Gao, P., Xu, P., Xiao, H., Zhang, K., Liu, C., Wen, S., Guo, Z., et al. (2023). Imagebind-llm: Multi-modality instruction tuning. arXiv preprint arXiv:2309.03905."},{"key":"2572_CR56","doi-asserted-by":"crossref","unstructured":"He, X., Cai, Z., Wei, W., Zhang, Y., Mou, L., Xing, E., & Xie, P. (2020). Pathological visual question answering. arXiv preprint arXiv:2010.12435.","DOI":"10.36227\/techrxiv.13127537.v1"},{"key":"2572_CR57","unstructured":"He, J., Guo, H., Tang, M., & Wang, J. (2023). Continual instruction tuning for large multimodal models. arXiv preprint arXiv:2311.16206."},{"key":"2572_CR58","unstructured":"He, J., Li, P., Liu, G., Zhao, Z., & Zhong, S. (2024). Pefomed: Parameter efficient fine-tuning on multimodal large language models for medical visual question answering. arXiv preprint arXiv:2401.02797."},{"key":"2572_CR59","doi-asserted-by":"crossref","unstructured":"He, K., Zhang, X., Ren, S., & Sun, J. (2016). Deep residual learning for image recognition. In: Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition, pp. 770\u2013778.","DOI":"10.1109\/CVPR.2016.90"},{"key":"2572_CR60","doi-asserted-by":"crossref","unstructured":"Hong, W., Wang, W., Lv, Q., Xu, J., Yu, W., Ji, J., Wang, Y., Wang, Z., Dong, Y., & Ding, M. (2024). Cogagent: A visual language model for gui agents. In: Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition, pp. 14281\u201314290.","DOI":"10.1109\/CVPR52733.2024.01354"},{"key":"2572_CR61","first-page":"20482","volume":"36","author":"Y Hong","year":"2023","unstructured":"Hong, Y., Zhen, H., Chen, P., Zheng, S., Du, Y., Chen, Z., & Gan, C. (2023). 3d-llm: Injecting the 3d world into large language models. Advances in Neural Information Processing Systems, 36, 20482\u201320494.","journal-title":"Advances in Neural Information Processing Systems"},{"key":"2572_CR62","doi-asserted-by":"crossref","unstructured":"Horawalavithana, S., Munikoti, S., Stewart, I., & Kvinge, H. (2023). Scitune: Aligning large language models with scientific multimodal instructions. arXiv preprint arXiv:2307.01139.","DOI":"10.18653\/v1\/2024.nlp4science-1.7"},{"key":"2572_CR63","unstructured":"Howard, A.G., Zhu, M., Chen, B., Kalenichenko, D., Wang, W., Weyand, T., Andreetto, M., & Adam, H. (2017). Mobilenets: Efficient convolutional neural networks for mobile vision applications. arXiv preprint arXiv:1704.04861."},{"key":"2572_CR64","doi-asserted-by":"crossref","unstructured":"Hu, A., Shi, Y., Xu, H., Ye, J., Ye, Q., Yan, M., Li, C., Qian, Q., Zhang, J., & Huang, F. (2024). mplug-paperowl: Scientific diagram analysis with the multimodal large language model. In: Proceedings of the 32nd ACM International Conference on Multimedia, pp. 6929\u20136938.","DOI":"10.1145\/3664647.3681294"},{"key":"2572_CR65","doi-asserted-by":"crossref","unstructured":"Hu, W., Xu, Y., Li, Y., Li, W., Chen, Z., & Tu, Z. (2024). Bliva: A simple multimodal llm for better handling of text-rich visual questions. In: Proceedings of the AAAI Conference on Artificial Intelligence, vol. 38, pp. 2256\u20132264.","DOI":"10.1609\/aaai.v38i3.27999"},{"issue":"2","key":"2572_CR66","first-page":"3","volume":"1","author":"EJ Hu","year":"2022","unstructured":"Hu, E. J., Shen, Y., Wallis, P., Allen-Zhu, Z., Li, Y., Wang, S., Wang, L., & Chen, W. (2022). Lora: Low-rank adaptation of large language models. ICLR, 1(2), 3.","journal-title":"ICLR"},{"key":"2572_CR67","unstructured":"Huang, Y., Meng, Z., Liu, F., Su, Y., Collier, N., & Lu, Y. (2023). Sparkles: Unlocking chats across multiple images for multimodal instruction-following models. arXiv preprint arXiv:2308.16463."},{"key":"2572_CR68","doi-asserted-by":"crossref","unstructured":"Huang, B., Wang, X., Chen, H., Song, Z., & Zhu, W. (2024). Vtimellm: Empower llm to grasp video moments. In: Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition, pp. 14271\u201314280.","DOI":"10.1109\/CVPR52733.2024.01353"},{"key":"2572_CR69","unstructured":"Huang, J., Yong, S., Ma, X., Linghu, X., Li, P., Wang, Y., Li, Q., Zhu, S.-C., Jia, B., & Huang, S. (2023). An embodied generalist agent in 3d world. arXiv preprint arXiv:2311.12871."},{"key":"2572_CR70","doi-asserted-by":"crossref","unstructured":"Huang, K., Zou, H., Xi, Y., Wang, B., Xie, Z., & Yu, L. (2024). Ivtp: Instruction-guided visual token pruning for large vision-language models. In: European Conference on Computer Vision, pp. 214\u2013230. Springer.","DOI":"10.1007\/978-3-031-72643-9_13"},{"key":"2572_CR71","doi-asserted-by":"crossref","unstructured":"Hudson, D.A., & Manning, C.D. (2019). Gqa: A new dataset for real-world visual reasoning and compositional question answering. In: Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition, pp. 6700\u20136709.","DOI":"10.1109\/CVPR.2019.00686"},{"key":"2572_CR72","doi-asserted-by":"crossref","unstructured":"Jang, Y., Song, Y., Yu, Y., Kim, Y., & Kim, G. (2017). Tgif-qa: Toward spatio-temporal reasoning in visual question answering. In: Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition, pp. 2758\u20132766.","DOI":"10.1109\/CVPR.2017.149"},{"key":"2572_CR73","doi-asserted-by":"crossref","unstructured":"Jia, M., Tang, L., Chen, B.-C., Cardie, C., Belongie, S., Hariharan, B., & Lim, S.-N. (2022). Visual prompt tuning. arXiv preprint arXiv:2203.12119.","DOI":"10.1007\/978-3-031-19827-4_41"},{"key":"2572_CR74","unstructured":"Jia, C., Yang, Y., Xia, Y., Chen, Y.-T., Parekh, Z., Pham, H., Le, Q., Sung, Y.-H., Li, Z., & Duerig, T. (2021). Scaling up visual and vision-language representation learning with noisy text supervision. In: International Conference on Machine Learning, pp. 4904\u20134916.. PMLR"},{"key":"2572_CR75","doi-asserted-by":"crossref","unstructured":"Jiang, Y., Schoop, E., Swearngin, A., & Nichols, J. (2025). Iluvui: Instruction-tuned language-vision modeling of uis from machine conversations. In: Proceedings of the 30th International Conference on Intelligent User Interfaces, pp. 861\u2013877.","DOI":"10.1145\/3708359.3712129"},{"key":"2572_CR76","unstructured":"Jin, Y., Li, J., Liu, Y., Gu, T., Wu, K., Jiang, Z., He, M., Zhao, B., Tan, X., Gan, Z., et al. (2024). Efficient multimodal large language models: A survey. arXiv preprint arXiv:2405.10739."},{"key":"2572_CR77","doi-asserted-by":"crossref","unstructured":"Kazemzadeh, S., Ordonez, V., Matten, M., & Berg, T. (2014). Referitgame: Referring to objects in photographs of natural scenes. In: Proceedings of the 2014 Conference on Empirical Methods in Natural Language Processing (EMNLP), pp. 787\u2013798.","DOI":"10.3115\/v1\/D14-1086"},{"key":"2572_CR78","unstructured":"Kim, C.D., Kim, B., Lee, H., & Kim, G. (2019). Audiocaps: Generating captions for audios in the wild. In: Proceedings of the 2019 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, Volume 1 (Long and Short Papers), pp. 119\u2013132."},{"key":"2572_CR79","doi-asserted-by":"crossref","unstructured":"Kim, Y., Mo, S., Kim, M., Lee, K., Lee, J., & Shin, J. (2024). Discovering and mitigating visual biases through keyword explanation. In: Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition, pp. 11082\u201311092.","DOI":"10.1109\/CVPR52733.2024.01054"},{"key":"2572_CR80","doi-asserted-by":"crossref","unstructured":"Kirillov, A., Mintun, E., Ravi, N., Mao, H., Rolland, C., Gustafson, L., Xiao, T., Whitehead, S., Berg, A.C., & Lo, W.-Y. (2023). Segment anything. In: Proceedings of the IEEE\/CVF International Conference on Computer Vision, pp. 4015\u20134026.","DOI":"10.1109\/ICCV51070.2023.00371"},{"key":"2572_CR81","unstructured":"Krause, J., Deng, J., Stark, M., & Fei-Fei, L. (2013). Collecting a large-scale dataset of fine-grained cars."},{"issue":"1","key":"2572_CR82","doi-asserted-by":"publisher","first-page":"32","DOI":"10.1007\/s11263-016-0981-7","volume":"123","author":"R Krishna","year":"2017","unstructured":"Krishna, R., Zhu, Y., Groth, O., Johnson, J., Hata, K., Kravitz, J., Chen, S., Kalantidis, Y., Li, L.-J., & Shamma, D. A. (2017). Visual genome: Connecting language and vision using crowdsourced dense image annotations. International journal of computer vision, 123(1), 32\u201373.","journal-title":"International journal of computer vision"},{"issue":"6","key":"2572_CR83","doi-asserted-by":"publisher","first-page":"84","DOI":"10.1145\/3065386","volume":"60","author":"A Krizhevsky","year":"2017","unstructured":"Krizhevsky, A., Sutskever, I., & Hinton, G. E. (2017). Imagenet classification with deep convolutional neural networks. Communications of the ACM, 60(6), 84\u201390.","journal-title":"Communications of the ACM"},{"key":"2572_CR84","doi-asserted-by":"crossref","unstructured":"Lai, B., Dai, X., Chen, L., Pang, G., Rehg, J.M., & Liu, M. (2024). Lego: L earning ego centric action frame generation via visual instruction tuning. In: European Conference on Computer Vision, pp. 135\u2013155. Springer.","DOI":"10.1007\/978-3-031-72673-6_8"},{"key":"2572_CR85","doi-asserted-by":"crossref","unstructured":"Lai, X., Tian, Z., Chen, Y., Li, Y., Yuan, Y., Liu, S., & Jia, J. (2024). Lisa: Reasoning segmentation via large language model. In: Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition, pp. 9579\u20139589.","DOI":"10.1109\/CVPR52733.2024.00915"},{"issue":"1","key":"2572_CR86","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1038\/sdata.2018.251","volume":"5","author":"JJ Lau","year":"2018","unstructured":"Lau, J. J., Gayen, S., Ben Abacha, A., & Demner-Fushman, D. (2018). A dataset of clinically generated visual questions and answers about radiology images. Scientific data, 5(1), 1\u201310.","journal-title":"Scientific data"},{"key":"2572_CR87","doi-asserted-by":"crossref","unstructured":"Li, Y., Du, Y., Zhou, K., Wang, J., Zhao, W.X., & Wen, J.-R. (2023). Evaluating object hallucination in large vision-language models. arXiv preprint arXiv:2305.10355.","DOI":"10.18653\/v1\/2023.emnlp-main.20"},{"key":"2572_CR88","unstructured":"Li, C., Ge, Y., Li, D., & Shan, Y. (2023). Vision-language instruction tuning: A review and analysis. arXiv preprint arXiv:2311.08172."},{"key":"2572_CR89","unstructured":"Li, K., He, Y., Wang, Y., Li, Y., Wang, W., Luo, P., Wang, Y., Wang, L., & Qiao, Y. (2023). Videochat: Chat-centric video understanding. arXiv preprint arXiv:2305.06355."},{"key":"2572_CR90","doi-asserted-by":"crossref","unstructured":"Li, Y., Hu, B., Chen, X., Ma, L., Xu, Y., & Zhang, M. (2024). Lmeye: An interactive perception network for large language models. IEEE Transactions on Multimedia.","DOI":"10.1109\/TMM.2024.3428317"},{"key":"2572_CR91","doi-asserted-by":"crossref","unstructured":"Li, H., Li, S., Cai, D., Wang, L., Liu, L., Watanabe, T., Yang, Y., & Shi, S. (2023). Textbind: Multi-turn interleaved multimodal instruction-following. arXiv preprint arXiv:2309.08637.","DOI":"10.18653\/v1\/2024.findings-acl.537"},{"key":"2572_CR92","unstructured":"Li, J., Li, D., Savarese, S., & Hoi, S. (2023). Blip-2: Bootstrapping language-image pre-training with frozen image encoders and large language models. In: International Conference on Machine Learning, pp. 19730\u201319742. PMLR."},{"key":"2572_CR93","unstructured":"Li, J., Pan, K., Ge, Z., Gao, M., Zhang, H., Ji, W., Zhang, W., Chua, T.-S., Tang, S., & Zhuang, Y. (2023). Fine-tuning multimodal llms to follow zero-shot demonstrative instructions. 3. arXiv preprint arXiv:2308.04152."},{"key":"2572_CR94","doi-asserted-by":"crossref","unstructured":"Li, B., Wang, R., Wang, G., Ge, Y., Ge, Y., & Shan, Y. (2023). Seed-bench: Benchmarking multimodal llms with generative comprehension. arXiv preprint arXiv:2307.16125.","DOI":"10.1109\/CVPR52733.2024.01263"},{"key":"2572_CR95","unstructured":"Li, L., Yin, Y., Li, S., Chen, L., Wang, P., Ren, S., Li, M., Yang, Y., Xu, J., Sun, X., et al. (2023). M3it: A large-scale dataset towards multi-modal multilingual instruction tuning. arXiv preprint arXiv:2306.04387."},{"key":"2572_CR96","doi-asserted-by":"crossref","unstructured":"Li, X., Yin, X., Li, C., Zhang, P., Hu, X., Zhang, L., Wang, L., Hu, H., Dong, L., & Wei, F. (2020). Oscar: Object-semantics aligned pre-training for vision-language tasks. In: Computer Vision\u2013ECCV 2020: 16th European Conference, Glasgow, UK, August 23\u201328, 2020, Proceedings, Part XXX 16, pp. 121\u2013137. Springer.","DOI":"10.1007\/978-3-030-58577-8_8"},{"key":"2572_CR97","unstructured":"Li, B., Zhang, Y., Chen, L., Wang, J., Pu, F., Yang, J., Li, C., & Liu, Z. (2023). MIMIC-IT: Multi-Modal In-Context Instruction Tuning."},{"key":"2572_CR98","unstructured":"Li, B., Zhang, Y., Chen, L., Wang, J., Yang, J., & Liu, Z. (2023). Otter: A multi-modal model with in-context instruction tuning. arXiv preprint arXiv:2305.03726."},{"key":"2572_CR99","unstructured":"Li, B., Zhang, P., Yang, J., Zhang, Y., Pu, F., & Liu, Z. (2023). Otterhd: A high-resolution multi-modality model. arXiv preprint arXiv:2311.04219."},{"key":"2572_CR100","doi-asserted-by":"crossref","unstructured":"Li, Y., Zhang, C., Yu, G., Wang, Z., Fu, B., Lin, G., Shen, C., Chen, L., & Wei, Y. (2023). Stablellava: Enhanced visual instruction tuning with synthesized image-dialogue data. arXiv preprint arXiv:2308.10253.","DOI":"10.18653\/v1\/2024.findings-acl.864"},{"key":"2572_CR101","unstructured":"Li, B., Zhang, K., Zhang, H., Guo, D., Zhang, R., Li, F., Zhang, Y., Liu, Z., & Li, C. May (2024). Llava-next: Stronger llms supercharge multimodal capabilities in the wild."},{"key":"2572_CR102","doi-asserted-by":"publisher","first-page":"511","DOI":"10.1016\/j.ins.2019.06.011","volume":"501","author":"T Li","year":"2019","unstructured":"Li, T., Gao, Y., Wang, K., Guo, S., Liu, H., & Kang, H. (2019). Diagnostic assessment of deep learning algorithms for diabetic retinopathy screening. Information Sciences, 501, 511\u2013522.","journal-title":"Information Sciences"},{"key":"2572_CR103","first-page":"28541","volume":"36","author":"C Li","year":"2023","unstructured":"Li, C., Wong, C., Zhang, S., Usuyama, N., Liu, H., Yang, J., Naumann, T., Poon, H., & Gao, J. (2023). Llava-med: Training a large language-and-vision assistant for biomedicine in one day. Advances in Neural Information Processing Systems, 36, 28541\u201328564.","journal-title":"Advances in Neural Information Processing Systems"},{"key":"2572_CR104","first-page":"109","volume":"35","author":"D Lian","year":"2022","unstructured":"Lian, D., Zhou, D., Feng, J., & Wang, X. (2022). Scaling & shifting your features: A new baseline for efficient model tuning. Advances in Neural Information Processing Systems, 35, 109\u2013123.","journal-title":"Advances in Neural Information Processing Systems"},{"key":"2572_CR105","doi-asserted-by":"crossref","unstructured":"Lin, T.-Y., Maire, M., Belongie, S., Hays, J., Perona, P., Ramanan, D., Doll\u00e1r, P., & Zitnick, C.L. (2014). Microsoft coco: Common objects in context. In: European Conference on Computer Vision, pp. 740\u2013755. Springer.","DOI":"10.1007\/978-3-319-10602-1_48"},{"key":"2572_CR106","unstructured":"Lin, B., Tang, Z., Ye, Y., Cui, J., Zhu, B., Jin, P., Zhang, J., Ning, M., & Yuan, L. (2024). Moe-llava: Mixture of experts for large vision-language models. arXiv preprint arXiv:2401.15947."},{"key":"2572_CR107","doi-asserted-by":"crossref","unstructured":"Liu, Y., Duan, H., Zhang, Y., Li, B., Zhang, S., Zhao, W., Yuan, Y., Wang, J., He, C., Liu, Z., et al. (2023). Mmbench: Is your multi-modal model an all-around player? arXiv preprint arXiv:2307.06281.","DOI":"10.1007\/978-3-031-72658-3_13"},{"key":"2572_CR108","doi-asserted-by":"crossref","unstructured":"Liu, H., Li, C., Li, Y., Lee, Y.J. (2024). Improved baselines with visual instruction tuning. In: Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition, pp. 26296\u201326306.","DOI":"10.1109\/CVPR52733.2024.02484"},{"key":"2572_CR109","unstructured":"Liu, F., Lin, K., Li, L., Wang, J., Yacoob, Y., & Wang, L. (2024). Mitigating hallucination in large multi-modal models via robust instruction tuning. In: The Twelfth International Conference on Learning Representations. https:\/\/openreview.net\/forum?id=J44HfH4JCg."},{"key":"2572_CR110","doi-asserted-by":"crossref","unstructured":"Liu, Z., Liu, B., Wang, J., Dong, Y., Chen, G., Rao, Y., Krishna, R., & Lu, J. (2024). Efficient inference of vision instruction-following models with elastic cache. In: European Conference on Computer Vision, pp. 54\u201369. Springer.","DOI":"10.1007\/978-3-031-72643-9_4"},{"key":"2572_CR111","unstructured":"Liu, J., Wang, Z., Ye, Q., Chong, D., Zhou, P., & Hua, Y. (2023). Qilin-med-vl: Towards chinese large vision-language model for general healthcare. arXiv preprint arXiv:2310.17956."},{"key":"2572_CR112","unstructured":"Liu, Y., Yang, B., Liu, Q., Li, Z., Ma, Z., Zhang, S., & Bai, X. (2024). Textmonkey: An ocr-free large multimodal model for understanding document. arXiv preprint arXiv:2403.04473."},{"key":"2572_CR113","doi-asserted-by":"crossref","unstructured":"Liu, B., Zhan, L.-M., Xu, L., Ma, L., Yang, Y., & Wu, X.-M. (2021). Slake: A semantically-labeled knowledge-enhanced dataset for medical visual question answering. In: 2021 IEEE 18th International Symposium on Biomedical Imaging (ISBI), pp. 1650\u20131654. IEEE.","DOI":"10.1109\/ISBI48211.2021.9434010"},{"key":"2572_CR114","first-page":"34892","volume":"36","author":"H Liu","year":"2023","unstructured":"Liu, H., Li, C., Wu, Q., & Lee, Y. J. (2023). Visual instruction tuning. Advances in neural information processing systems, 36, 34892\u201334916.","journal-title":"Advances in neural information processing systems"},{"key":"2572_CR115","first-page":"2507","volume":"35","author":"P Lu","year":"2022","unstructured":"Lu, P., Mishra, S., Xia, T., Qiu, L., Chang, K.-W., Zhu, S.-C., Tafjord, O., Clark, P., & Kalyan, A. (2022). Learn to explain: Multimodal reasoning via thought chains for science question answering. Advances in Neural Information Processing Systems, 35, 2507\u20132521.","journal-title":"Advances in Neural Information Processing Systems"},{"key":"2572_CR116","unstructured":"Luo, R., Zhao, Z., Yang, M., Dong, J., Qiu, M., Lu, P., Wang, T., & Wei, Z. (2023). Valley: Video assistant with large language model enhanced ability. arXiv preprint arXiv:2306.07207."},{"key":"2572_CR117","first-page":"29615","volume":"36","author":"G Luo","year":"2023","unstructured":"Luo, G., Zhou, Y., Ren, T., Chen, S., Sun, X., & Ji, R. (2023). Cheap and quick: Efficient vision-language instruction tuning for large language models. Advances in Neural Information Processing Systems, 36, 29615\u201329627.","journal-title":"Advances in Neural Information Processing Systems"},{"key":"2572_CR118","unstructured":"Lyu, C., Wu, M., Wang, L., Huang, X., Liu, B., Du, Z., Shi, S., & Tu, Z. (2023). Macaw-llm: Multi-modal language modeling with image, audio, video, and text integration. arXiv preprint arXiv:2306.09093."},{"key":"2572_CR119","doi-asserted-by":"crossref","unstructured":"Ma, Y., Cao, Y., Sun, J., Pavone, M., & Xiao, C. (2023). Dolphins: Multimodal language model for driving. arXiv preprint arXiv:2312.00438.","DOI":"10.1007\/978-3-031-72995-9_23"},{"key":"2572_CR120","doi-asserted-by":"crossref","unstructured":"Maaz, M., Rasheed, H., Khan, S., & Khan, F. (2024). Video-chatgpt: Towards detailed video understanding via large vision and language models. In: Proceedings of the 62nd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers), pp. 12585\u201312602.","DOI":"10.18653\/v1\/2024.acl-long.679"},{"key":"2572_CR121","unstructured":"Mani, A., Yoo, N., Hinthorn, W., & Russakovsky, O. (2020). Point and ask: Incorporating pointing into visual question answering. arXiv preprint arXiv:2011.13681."},{"key":"2572_CR122","doi-asserted-by":"crossref","unstructured":"Mao, J., Huang, J., Toshev, A., Camburu, O., Yuille, A.L., & Murphy, K. (2016). Generation and comprehension of unambiguous object descriptions. In: Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition, pp. 11\u201320.","DOI":"10.1109\/CVPR.2016.9"},{"key":"2572_CR123","doi-asserted-by":"crossref","unstructured":"Marino, K., Rastegari, M., Farhadi, A., & Mottaghi, R. (2019). Ok-vqa: A visual question answering benchmark requiring external knowledge. In: Proceedings of the IEEE\/cvf Conference on Computer Vision and Pattern Recognition, pp. 3195\u20133204.","DOI":"10.1109\/CVPR.2019.00331"},{"key":"2572_CR124","doi-asserted-by":"crossref","unstructured":"Masry, A., Long, D.X., Tan, J.Q., Joty, S., & Hoque, E. (2022). Chartqa: A benchmark for question answering about charts with visual and logical reasoning. arXiv preprint arXiv:2203.10244.","DOI":"10.18653\/v1\/2022.findings-acl.177"},{"key":"2572_CR125","doi-asserted-by":"crossref","unstructured":"Mathew, M., Karatzas, D., & Jawahar, C. (2021). Docvqa: A dataset for vqa on document images. In: Proceedings of the IEEE\/CVF Winter Conference on Applications of Computer Vision, pp. 2200\u20132209.","DOI":"10.1109\/WACV48630.2021.00225"},{"key":"2572_CR126","doi-asserted-by":"crossref","unstructured":"Meng, F., Shao, W., Lu, Q., Gao, P., Zhang, K., Qiao, Y., & Luo, P. (2024). Chartassisstant: A universal chart multimodal language model via chart-to-table pre-training and multitask instruction tuning. arXiv preprint arXiv:2401.02384.","DOI":"10.18653\/v1\/2024.findings-acl.463"},{"key":"2572_CR127","doi-asserted-by":"crossref","unstructured":"Moon, S., Madotto, A., Lin, Z., Nagarajan, T., Smith, M., Jain, S., Yeh, C.-F., Murugesan, P., Heidari, P., & Liu, Y. (2024). Anymal: An efficient and scalable any-modality augmented language model. In: Proceedings of the 2024 Conference on Empirical Methods in Natural Language Processing: Industry Track, pp. 1314\u20131332.","DOI":"10.18653\/v1\/2024.emnlp-industry.98"},{"key":"2572_CR128","unstructured":"Moor, M., Huang, Q., Wu, S., Yasunaga, M., Dalmia, Y., Leskovec, J., Zakka, C., Reis, E.P., & Rajpurkar, P. (2023). Med-flamingo: a multimodal medical few-shot learner. In: Machine Learning for Health (ML4H), pp. 353\u2013367. PMLR."},{"key":"2572_CR129","first-page":"25081","volume":"36","author":"Y Mu","year":"2023","unstructured":"Mu, Y., Zhang, Q., Hu, M., Wang, W., Ding, M., Jin, J., Wang, B., Dai, J., Qiao, Y., & Luo, P. (2023). Embodiedgpt: Vision-language pre-training via embodied chain of thought. Advances in Neural Information Processing Systems, 36, 25081\u201325094.","journal-title":"Advances in Neural Information Processing Systems"},{"key":"2572_CR130","doi-asserted-by":"crossref","unstructured":"Nilsback, M.-E., & Zisserman, A. (2008). Automated flower classification over a large number of classes. In: 2008 Sixth Indian Conference on Computer Vision, Graphics & Image Processing, pp. 722\u2013729. IEEE.","DOI":"10.1109\/ICVGIP.2008.47"},{"key":"2572_CR131","unstructured":"Niu, D., Sharma, Y., Biamby, G., Quenum, J., Bai, Y., Shi, B., Darrell, T., & Herzig, R. (2024). Llarva: Vision-action instruction tuning enhances robot learning. arXiv preprint arXiv:2406.11815."},{"key":"2572_CR132","unstructured":"OpenAI. (2023). GPT-4 Technical Report."},{"key":"2572_CR133","unstructured":"OpenAI: ChatGPT. https:\/\/openai.com\/blog\/chatgpt. Accessed: 2023-09-15 (2020)."},{"key":"2572_CR134","doi-asserted-by":"crossref","unstructured":"Panagopoulou, A., Xue, L., Yu, N., Li, J., Li, D., Joty, S., Xu, R., Savarese, S., Xiong, C., & Niebles, J.C. (2024). X-instructblip: A framework for aligning image, 3d, audio, video to llms and its emergent cross-modal reasoning. In: European Conference on Computer Vision, pp. 177\u2013197. Springer.","DOI":"10.1007\/978-3-031-72995-9_11"},{"key":"2572_CR135","doi-asserted-by":"crossref","unstructured":"Parkhi, O.M., Vedaldi, A., Zisserman, A., & Jawahar, C. (2012). Cats and dogs. In: 2012 IEEE Conference on Computer Vision and Pattern Recognition, pp. 3498\u20133505. IEEE.","DOI":"10.1109\/CVPR.2012.6248092"},{"key":"2572_CR136","doi-asserted-by":"crossref","unstructured":"Perazzi, F., Pont-Tuset, J., McWilliams, B., Van\u00a0Gool, L., Gross, M., & Sorkine-Hornung, A. (2016). A benchmark dataset and evaluation methodology for video object segmentation. In: Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition, pp. 724\u2013732.","DOI":"10.1109\/CVPR.2016.85"},{"key":"2572_CR137","doi-asserted-by":"crossref","unstructured":"Pi, R., Gao, J., Diao, S., Pan, R., Dong, H., Zhang, J., Yao, L., Han, J., Xu, H., & Kong, L. (2023). Detgpt: Detect what you need via reasoning. In: Proceedings of the 2023 Conference on Empirical Methods in Natural Language Processing, pp. 14172\u201314189.","DOI":"10.18653\/v1\/2023.emnlp-main.876"},{"key":"2572_CR138","unstructured":"Qi, C.R., Su, H., Mo, K., & Guibas, L.J. (2017). Pointnet: Deep learning on point sets for 3d classification and segmentation. In: Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition, pp. 652\u2013660."},{"key":"2572_CR139","unstructured":"Radford, A., Kim, J.W., Hallacy, C., Ramesh, A., Goh, G., Agarwal, S., Sastry, G., Askell, A., Mishkin, P., & Clark, J. (2021). Learning transferable visual models from natural language supervision. In: International Conference on Machine Learning, pp. 8748\u20138763. PMLR."},{"key":"2572_CR140","unstructured":"Radford, A., Kim, J.W., Xu, T., Brockman, G., McLeavey, C., & Sutskever, I. (2023). Robust speech recognition via large-scale weak supervision. In: International Conference on Machine Learning, pp. 28492\u201328518. PMLR."},{"key":"2572_CR141","doi-asserted-by":"crossref","unstructured":"Rasheed, H., Maaz, M., Shaji, S., Shaker, A., Khan, S., Cholakkal, H., Anwer, R.M., Xing, E., Yang, M.-H., & Khan, F.S. (2024). Glamm: Pixel grounding large multimodal model. In: Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition, pp. 13009\u201313018.","DOI":"10.1109\/CVPR52733.2024.01236"},{"key":"2572_CR142","doi-asserted-by":"crossref","unstructured":"Redmon, J., Divvala, S., Girshick, R., & Farhadi, A. (2016). You only look once: Unified, real-time object detection. In: Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition, pp. 779\u2013788.","DOI":"10.1109\/CVPR.2016.91"},{"key":"2572_CR143","unstructured":"Ren, S., He, K., Girshick, R., & Sun, J. (2015). Faster r-cnn: Towards real-time object detection with region proposal networks. Advances in neural information processing systems 28."},{"key":"2572_CR144","doi-asserted-by":"crossref","unstructured":"Ren, S., Yao, L., Li, S., Sun, X., & Hou, L. (2024). Timechat: A time-sensitive multimodal large language model for long video understanding. In: Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition, pp. 14313\u201314323.","DOI":"10.1109\/CVPR52733.2024.01357"},{"key":"2572_CR145","unstructured":"Schuhmann, C., Vencu, R., Beaumont, R., Kaczmarczyk, R., Mullis, C., Katta, A., Coombes, T., Jitsev, J., & Komatsuzaki, A. (2021). Laion-400m: Open dataset of clip-filtered 400 million image-text pairs. arXiv preprint arXiv:2111.02114."},{"key":"2572_CR146","doi-asserted-by":"crossref","unstructured":"Seth, A., Hemani, M., & Agarwal, C. (2023). Dear: Debiasing vision-language models with additive residuals. In: Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition, pp. 6820\u20136829.","DOI":"10.1109\/CVPR52729.2023.00659"},{"key":"2572_CR147","unstructured":"Shen, X., Xiong, Y., Zhao, C., Wu, L., Chen, J., Zhu, C., Liu, Z., Xiao, F., Varadarajan, B., Bordes, F., et al. (2024). Longvu: Spatiotemporal adaptive compression for long video-language understanding. arXiv preprint arXiv:2410.17434."},{"key":"2572_CR148","unstructured":"Simonyan, K., & Zisserman, A. (2014). Very deep convolutional networks for large-scale image recognition. arXiv preprint arXiv:1409.1556."},{"key":"2572_CR149","doi-asserted-by":"crossref","unstructured":"Singh, A., Natarajan, V., Shah, M., Jiang, Y., Chen, X., Batra, D., Parikh, D., & Rohrbach, M. (2019). Towards vqa models that can read. In: Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition, pp. 8317\u20138326.","DOI":"10.1109\/CVPR.2019.00851"},{"key":"2572_CR150","first-page":"92983","volume":"37","author":"G Somepalli","year":"2024","unstructured":"Somepalli, G., Chowdhury, A., Geiping, J., Basri, R., Goldstein, T., & Jacobs, D. (2024). Calvin: Improved contextual video captioning via instruction tuning. Advances in Neural Information Processing Systems, 37, 92983\u201393010.","journal-title":"Advances in Neural Information Processing Systems"},{"key":"2572_CR151","doi-asserted-by":"crossref","unstructured":"Steitz, J.-M.O., & Roth, S. (2024). Adapters strike back. In: Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition, pp. 23449\u201323459.","DOI":"10.1109\/CVPR52733.2024.02213"},{"key":"2572_CR152","unstructured":"Su, Y., Lan, T., Li, H., Xu, J., Wang, Y., & Cai, D. (2023). Pandagpt: One model to instruction-follow them all. In: Proceedings of the 1st Workshop on Taming Large Language Models: Controllability in the Era of Interactive Assistants!, pp. 11\u201323."},{"key":"2572_CR153","unstructured":"Su, W., Zhu, X., Cao, Y., Li, B., Lu, L., Wei, F., & Dai, J. (2019). Vl-bert: Pre-training of generic visual-linguistic representations. arXiv preprint arXiv:1908.08530."},{"key":"2572_CR154","unstructured":"Sun, G., Jin, M., Wang, Z., Wang, C.-L., Ma, S., Wang, Q., Geng, T., Wu, Y.N., Zhang, Y., & Liu, D. (2024). Visual agents as fast and slow thinkers. arXiv preprint arXiv:2408.08862."},{"key":"2572_CR155","unstructured":"Sun, G., Yu, W., Tang, C., Chen, X., Tan, T., Li, W., Lu, L., Ma, Z., Wang, Y., & Zhang, C. (2024). video-salmonn: Speech-enhanced audio-visual large language models. arXiv preprint arXiv:2406.15704."},{"key":"2572_CR156","doi-asserted-by":"crossref","unstructured":"Sun, Y., Zhu, C., Zheng, S., Zhang, K., Sun, L., Shui, Z., Zhang, Y., Li, H., & Yang, L. (2024). Pathasst: A generative foundation ai assistant towards artificial general intelligence of pathology. In: Proceedings of the AAAI Conference on Artificial Intelligence, vol. 38, pp. 5034\u20135042.","DOI":"10.1609\/aaai.v38i5.28308"},{"key":"2572_CR157","first-page":"12991","volume":"35","author":"Y-L Sung","year":"2022","unstructured":"Sung, Y.-L., Cho, J., & Bansal, M. (2022). Lst: Ladder side-tuning for parameter and memory efficient transfer learning. Advances in Neural Information Processing Systems, 35, 12991\u201313005.","journal-title":"Advances in Neural Information Processing Systems"},{"key":"2572_CR158","doi-asserted-by":"crossref","unstructured":"Tanaka, R., Nishida, K., & Yoshida, S. (2021). Visualmrc: Machine reading comprehension on document images. In: Proceedings of the AAAI Conference on Artificial Intelligence, vol. 35, pp. 13878\u201313888.","DOI":"10.1609\/aaai.v35i15.17635"},{"key":"2572_CR159","unstructured":"Team, G., Anil, R., Borgeaud, S., Alayrac, J.-B., Yu, J., Soricut, R., Schalkwyk, J., Dai, A.M., Hauth, A., Millican, K., et al. (2023). Gemini: a family of highly capable multimodal models. arXiv preprint arXiv:2312.11805."},{"key":"2572_CR160","first-page":"9565","volume":"37","author":"C Tian","year":"2024","unstructured":"Tian, C., Shi, Z., Guo, Z., Li, L., & Xu, C.-Z. (2024). Hydralora: An asymmetric lora architecture for efficient fine-tuning. Advances in Neural Information Processing Systems, 37, 9565\u20139584.","journal-title":"Advances in Neural Information Processing Systems"},{"key":"2572_CR161","unstructured":"Tong, S., Brown, E., Wu, P., Woo, S., Middepogu, M., Akula, S.C., Yang, J., Yang, S., Iyer, A., Pan, X., et al. (2024). Cambrian-1: A fully open, vision-centric exploration of multimodal llms. arXiv preprint arXiv:2406.16860."},{"key":"2572_CR162","unstructured":"Touvron, H., Lavril, T., Izacard, G., Martinet, X., Lachaux, M.-A., Lacroix, T., Rozi\u00e8re, B., Goyal, N., Hambro, E., Azhar, F., et al. (2023). Llama: Open and efficient foundation language models. arXiv preprint arXiv:2302.13971."},{"issue":"3","key":"2572_CR163","first-page":"2300138","volume":"1","author":"T Tu","year":"2024","unstructured":"Tu, T., Azizi, S., Driess, D., Schaekermann, M., Amin, M., Chang, P.-C., Carroll, A., Lau, C., Tanno, R., & Ktena, I. (2024). Towards generalist biomedical ai. NEJM AI, 1(3), 2300138.","journal-title":"Towards generalist biomedical ai. NEJM AI"},{"key":"2572_CR164","first-page":"2","volume":"30","author":"A Vaswani","year":"2017","unstructured":"Vaswani, A., Shazeer, N., Parmar, N., Uszkoreit, J., Jones, L., Gomez, A. N., Kaiser, \u0141, & Polosukhin, I. (2017). Attention is all you need. Advances in neural information processing systems, 30, 2.","journal-title":"Advances in neural information processing systems"},{"key":"2572_CR165","doi-asserted-by":"publisher","first-page":"7068349","DOI":"10.1155\/2018\/7068349","volume":"2018","author":"A Voulodimos","year":"2018","unstructured":"Voulodimos, A., Doulamis, N., Doulamis, A., Protopapadakis, E., et al. (2018). Deep learning for computer vision: A brief review. Computational intelligence and neuroscience, 2018, 7068349.","journal-title":"Computational intelligence and neuroscience"},{"key":"2572_CR166","unstructured":"Wang, P., Bai, S., Tan, S., Wang, S., Fan, Z., Bai, J., Chen, K., Liu, X., Wang, J., Ge, W., et al. (2024). Qwen2-vl: Enhancing vision-language model\u2019s perception of the world at any resolution. arXiv preprint arXiv:2409.12191."},{"key":"2572_CR167","doi-asserted-by":"crossref","unstructured":"Wang, D., Cui, J., Li, M., Lin, W., Chen, B., & Zhang, H. (2024). Instruction tuning-free visual token complement for multimodal llms. In: European Conference on Computer Vision, pp. 446\u2013462. Springer.","DOI":"10.1007\/978-3-031-73004-7_26"},{"key":"2572_CR168","doi-asserted-by":"crossref","unstructured":"Wang, T., Liu, Y., Liang, J.C., Cui, Y., Mao, Y., Nie, S., Liu, J., Feng, F., Xu, Z., Han, C., et al. (2024). M2pt: Multimodal prompt tuning for zero-shot instruction learning. arXiv preprint arXiv:2409.15657.","DOI":"10.18653\/v1\/2024.emnlp-main.218"},{"key":"2572_CR169","unstructured":"Wang, T., Liu, D., Liang, J.C., Yang, W., Wang, Q., Han, C., Luo, J., & Tang, R. (2024). Exploring the adversarial vulnerabilities of vision-language-action models in robotics. arXiv preprint arXiv:2411.13587."},{"key":"2572_CR170","unstructured":"Wang, W., Shi, M., Li, Q., Wang, W., Huang, Z., Xing, L., Chen, Z., Li, H., Zhu, X., Cao, Z., et al. (2023). The all-seeing project: Towards panoptic visual recognition and understanding of the open world. arXiv preprint arXiv:2308.01907."},{"key":"2572_CR171","doi-asserted-by":"crossref","unstructured":"Wang, X., Wu, J., Chen, J., Li, L., Wang, Y.-F., & Wang, W.Y. (2019). Vatex: A large-scale, high-quality multilingual dataset for video-and-language research. In: Proceedings of the IEEE\/CVF International Conference on Computer Vision, pp. 4581\u20134591.","DOI":"10.1109\/ICCV.2019.00468"},{"key":"2572_CR172","unstructured":"Wang, Y., Zhang, R., Wang, H., Bhattacharya, U., Fu, Y., & Wu, G. (2023). Vaquita: Enhancing alignment in llm-assisted video understanding. arXiv preprint arXiv:2312.02310."},{"key":"2572_CR173","first-page":"61501","volume":"36","author":"W Wang","year":"2023","unstructured":"Wang, W., Chen, Z., Chen, X., Wu, J., Zhu, X., Zeng, G., Luo, P., Lu, T., Zhou, J., & Qiao, Y. (2023). Visionllm: Large language model is also an open-ended decoder for vision-centric tasks. Advances in Neural Information Processing Systems, 36, 61501\u201361513.","journal-title":"Advances in Neural Information Processing Systems"},{"key":"2572_CR174","first-page":"121475","volume":"37","author":"W Wang","year":"2024","unstructured":"Wang, W., Lv, Q., Yu, W., Hong, W., Qi, J., Wang, Y., Ji, J., Yang, Z., Zhao, L., & XiXuan, S. (2024). Cogvlm: Visual expert for pretrained language models. Advances in Neural Information Processing Systems, 37, 121475\u2013121499.","journal-title":"Advances in Neural Information Processing Systems"},{"key":"2572_CR175","first-page":"5309","volume":"38","author":"B Wang","year":"2024","unstructured":"Wang, B., Wu, F., Han, X., Peng, J., Zhong, H., Zhang, P., Dong, X., Li, W., Li, W., & Wang, J. (2024). Vigc: Visual instruction generation and correction. In: Proceedings of the AAAI Conference on Artificial Intelligence, 38, 5309\u20135317.","journal-title":"In: Proceedings of the AAAI Conference on Artificial Intelligence"},{"key":"2572_CR176","unstructured":"Wei, F., Zhang, X., Zhang, A., Zhang, B., & Chu, X. (2023). Lenna: Language enhanced reasoning detection assistant. arXiv preprint arXiv:2312.02433."},{"key":"2572_CR177","doi-asserted-by":"crossref","unstructured":"Weng, Z., Gao, Z., Andrews, J., & Zhao, J. (2024). Images speak louder than words: Understanding and mitigating bias in vision-language model from a causal mediation perspective. arXiv preprint arXiv:2407.02814.","DOI":"10.18653\/v1\/2024.emnlp-main.878"},{"key":"2572_CR178","unstructured":"Wu, S., Fei, H., Qu, L., Ji, W., & Chua, T.-S. (2024). Next-gpt: Any-to-any multimodal llm. In: Forty-first International Conference on Machine Learning."},{"key":"2572_CR179","doi-asserted-by":"crossref","unstructured":"Wu, J., Gan, W., Chen, Z., Wan, S., & Philip, S.Y. (2023). Multimodal large language models: A survey. In: 2023 IEEE International Conference on Big Data (BigData), pp. 2247\u20132256. IEEE.","DOI":"10.1109\/BigData59044.2023.10386743"},{"key":"2572_CR180","unstructured":"Wu, Z., Song, S., Khosla, A., Yu, F., Zhang, L., Tang, X., & Xiao, J. (2015). 3d shapenets: A deep representation for volumetric shapes. In: Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition, pp. 1912\u20131920."},{"key":"2572_CR181","doi-asserted-by":"crossref","unstructured":"Wu, J., Xiong, Y., Li, X., Xia, Y., Wang, R., Wang, Y., Yu, T., Kim, S., Rossi, R.A., Yao, L., et al. (2025). Mitigating visual knowledge forgetting in mllm instruction-tuning via modality-decoupled gradient descent. arXiv preprint arXiv:2502.11740.","DOI":"10.18653\/v1\/2025.findings-emnlp.123"},{"key":"2572_CR182","doi-asserted-by":"crossref","unstructured":"Xiao, Z., Chen, Y., Zhang, L., Yao, J., Wu, Z., Yu, X., Pan, Y., Zhao, L., Ma, C., Liu, X., et al. (2023). Instruction-vit: Multi-modal prompts for instruction learning in vit. arXiv preprint arXiv:2305.00201.","DOI":"10.1016\/j.inffus.2023.102204"},{"key":"2572_CR183","unstructured":"Xie, Y., Zhou, C., Gao, L., Wu, J., Li, X., Zhou, H.-Y., Liu, S., Xing, L., Zou, J., Xie, C., et al. (2024). Medtrinity-25m: A large-scale multimodal dataset with multigranular annotations for medicine. arXiv preprint arXiv:2408.02900."},{"key":"2572_CR184","first-page":"92012","volume":"37","author":"Y Xing","year":"2024","unstructured":"Xing, Y., Li, Y., Laptev, I., & Lu, S. (2024). Mitigating object hallucination via concentric causal attention. Advances in Neural Information Processing Systems, 37, 92012\u201392035.","journal-title":"Advances in Neural Information Processing Systems"},{"key":"2572_CR185","doi-asserted-by":"crossref","unstructured":"Xu, Z., Feng, C., Shao, R., Ashby, T., Shen, Y., Jin, D., Cheng, Y., Wang, Q., & Huang, L. (2024). Vision-flan: Scaling human-labeled tasks in visual instruction tuning. arXiv preprint arXiv:2402.11690.","DOI":"10.18653\/v1\/2024.findings-acl.905"},{"key":"2572_CR186","doi-asserted-by":"crossref","unstructured":"Xu, Z., Shen, Y., & Huang, L. (2023). Multiinstruct: Improving multi-modal zero-shot learning via instruction tuning. In: Proceedings of the 61st Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers), pp. 11445\u201311465.","DOI":"10.18653\/v1\/2023.acl-long.641"},{"key":"2572_CR187","doi-asserted-by":"crossref","unstructured":"Xu, R., Wang, X., Wang, T., Chen, Y., Pang, J., & Lin, D. (2024). Pointllm: Empowering large language models to understand point clouds. In: European Conference on Computer Vision, pp. 131\u2013147. Springer.","DOI":"10.1007\/978-3-031-72698-9_8"},{"key":"2572_CR188","doi-asserted-by":"crossref","unstructured":"Xu, Z., Zhang, Y., Xie, E., Zhao, Z., Guo, Y., Wong, K.-Y.K., Li, Z., & Zhao, H. (2024). Drivegpt4: Interpretable end-to-end autonomous driving via large language model. IEEE Robotics and Automation Letters.","DOI":"10.1109\/LRA.2024.3440097"},{"key":"2572_CR189","doi-asserted-by":"crossref","unstructured":"Xu, D., Zhao, Z., Xiao, J., Wu, F., Zhang, H., He, X., & Zhuang, Y. (2017). Video question answering via gradually refined attention over appearance and motion. In: Proceedings of the 25th ACM International Conference on Multimedia, pp. 1645\u20131653.","DOI":"10.1145\/3123266.3123427"},{"key":"2572_CR190","unstructured":"Yang, B., Raza, A., Zou, Y., & Zhang, T. (2023). Customizing general-purpose foundation models for medical report generation. arXiv preprint arXiv:2306.05642."},{"key":"2572_CR191","first-page":"71995","volume":"36","author":"R Yang","year":"2023","unstructured":"Yang, R., Song, L., Li, Y., Zhao, S., Ge, Y., Li, X., & Shan, Y. (2023). Gpt4tools: Teaching large language model to use tools via self-instruction. Advances in Neural Information Processing Systems, 36, 71995\u201372007.","journal-title":"Advances in Neural Information Processing Systems"},{"key":"2572_CR192","unstructured":"Ye, J., Hu, A., Xu, H., Ye, Q., Yan, M., Dan, Y., Zhao, C., Xu, G., Li, C., Tian, J., et al. (2023). mplug-docowl: Modularized multimodal large language model for document understanding. arXiv preprint arXiv:2307.02499."},{"key":"2572_CR193","doi-asserted-by":"crossref","unstructured":"Ye, Q., Xu, H., Ye, J., Yan, M., Hu, A., Liu, H., Qian, Q., Zhang, J., & Huang, F. (2024). mplug-owl2: Revolutionizing multi-modal large language model with modality collaboration. In: Proceedings of the Ieee\/cvf Conference on Computer Vision and Pattern Recognition, pp. 13040\u201313051.","DOI":"10.1109\/CVPR52733.2024.01239"},{"key":"2572_CR194","doi-asserted-by":"crossref","unstructured":"Yin, S., Fu, C., Zhao, S., Li, K., Sun, X., Xu, T., & Chen, E. (2024). A survey on multimodal large language models. National Science Review, 403.","DOI":"10.1093\/nsr\/nwae403"},{"key":"2572_CR195","first-page":"26650","volume":"36","author":"Z Yin","year":"2023","unstructured":"Yin, Z., Wang, J., Cao, J., Shi, Z., Liu, D., Li, M., Huang, X., Wang, Z., Sheng, L., & Bai, L. (2023). Lamm: Language-assisted multi-modal instruction-tuning dataset, framework, and benchmark. Advances in Neural Information Processing Systems, 36, 26650\u201326685.","journal-title":"Advances in Neural Information Processing Systems"},{"key":"2572_CR196","first-page":"67034","volume":"37","author":"M Yoo","year":"2024","unstructured":"Yoo, M., Jang, J., Park, W.-J., & Woo, H. (2024). Exploratory retrieval-augmented planning for continual embodied instruction following. Advances in Neural Information Processing Systems, 37, 67034\u201367060.","journal-title":"Advances in Neural Information Processing Systems"},{"key":"2572_CR197","unstructured":"You, H., Zhang, H., Gan, Z., Du, X., Zhang, B., Wang, Z., Cao, L., Chang, S.-F., & Yang, Y. (2023). Ferret: Refer and ground anything anywhere at any granularity. arXiv preprint arXiv:2310.07704."},{"key":"2572_CR198","doi-asserted-by":"publisher","first-page":"67","DOI":"10.1162\/tacl_a_00166","volume":"2","author":"P Young","year":"2014","unstructured":"Young, P., Lai, A., Hodosh, M., & Hockenmaier, J. (2014). From image descriptions to visual denotations: New similarity metrics for semantic inference over event descriptions. Transactions of the Association for Computational Linguistics, 2, 67\u201378.","journal-title":"Transactions of the Association for Computational Linguistics"},{"key":"2572_CR199","doi-asserted-by":"crossref","unstructured":"Yu, X., Tang, L., Rao, Y., Huang, T., Zhou, J., & Lu, J. (2021). Point-bert: Pre-training 3d point cloud transformers with masked point modeling. arXiv preprint arXiv:2111.14819.","DOI":"10.1109\/CVPR52688.2022.01871"},{"key":"2572_CR200","doi-asserted-by":"crossref","unstructured":"Yu, Z., Xu, D., Yu, J., Yu, T., Zhao, Z., Zhuang, Y., & Tao, D. (2019). Activitynet-qa: A dataset for understanding complex web videos via question answering. In: Proceedings of the AAAI Conference on Artificial Intelligence, vol. 33, pp. 9127\u20139134.","DOI":"10.1609\/aaai.v33i01.33019127"},{"key":"2572_CR201","unstructured":"Yu, W., Yang, Z., Li, L., Wang, J., Lin, K., Liu, Z., Wang, X., & Wang, L. (2023). Mm-vet: Evaluating large multimodal models for integrated capabilities. arXiv preprint arXiv:2308.02490."},{"key":"2572_CR202","unstructured":"Yuan, W., Duan, J., Blukis, V., Pumacay, W., Krishna, R., Murali, A., Mousavian, A., & Fox, D. (2024). Robopoint: A vision-language model for spatial affordance prediction for robotics. arXiv preprint arXiv:2406.10721."},{"key":"2572_CR203","doi-asserted-by":"crossref","unstructured":"Yuan, Y., Li, W., Liu, J., Tang, D., Luo, X., Qin, C., Zhang, L., & Zhu, J. (2024). Osprey: Pixel understanding with visual instruction tuning. In: Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition, pp. 28202\u201328211.","DOI":"10.1109\/CVPR52733.2024.02664"},{"key":"2572_CR204","first-page":"5552","volume":"37","author":"R Zeng","year":"2024","unstructured":"Zeng, R., Han, C., Wang, Q., Wu, C., Geng, T., Huangg, L., Wu, Y. N., & Liu, D. (2024). Visual fourier prompt tuning. Advances in Neural Information Processing Systems, 37, 5552\u20135585.","journal-title":"Advances in Neural Information Processing Systems"},{"key":"2572_CR205","unstructured":"Zhai, Y., Tong, S., Li, X., Cai, M., Qu, Q., Lee, Y.J., & Ma, Y. (2024). Investigating the catastrophic forgetting in multimodal large language model fine-tuning. In: Conference on Parsimony and Learning, pp. 202\u2013227. PMLR."},{"key":"2572_CR206","unstructured":"Zhang, S., Dong, L., Li, X., Zhang, S., Sun, X., Wang, S., Li, J., Hu, R., Zhang, T., Wu, F., et al. (2023). Instruction tuning for large language models: A survey. arXiv preprint arXiv:2308.10792"},{"key":"2572_CR207","unstructured":"Zhang, R., Fang, R., Gao, P., Zhang, W., Li, K., Dai, J., Qiao, Y., & Li, H. (2021). Tip-adapter: Training-free clip-adapter for better vision-language modeling. arXiv preprint arXiv:2111.03930."},{"key":"2572_CR208","doi-asserted-by":"crossref","unstructured":"Zhang, J., Huang, J., Jin, S., & Lu, S. (2024). Vision-language models for vision tasks: A survey. IEEE Transactions on Pattern Analysis and Machine Intelligence.","DOI":"10.1109\/TPAMI.2024.3369699"},{"key":"2572_CR209","doi-asserted-by":"crossref","unstructured":"Zhang, H., Li, X., & Bing, L. (2023). Video-llama: An instruction-tuned audio-visual language model for video understanding. In: Proceedings of the 2023 Conference on Empirical Methods in Natural Language Processing: System Demonstrations, pp. 543\u2013553.","DOI":"10.18653\/v1\/2023.emnlp-demo.49"},{"key":"2572_CR210","unstructured":"Zhang, T., Li, X., Fei, H., Yuan, H., Wu, S., Ji, S., Loy, C.C., & Yan, S. (2024). Omg-llava: Bridging image-level, object-level, pixel-level reasoning and understanding. arXiv preprint arXiv:2406.19389."},{"key":"2572_CR211","unstructured":"Zhang, S., Sun, P., Chen, S., Xiao, M., Shao, W., Zhang, W., Chen, K., & Luo, P. (2023). Gpt4roi: Instruction tuning large language model on region-of-interest. arXiv preprint arXiv:2307.03601."},{"key":"2572_CR212","unstructured":"Zhang, P., Wang, X.D.B., Cao, Y., Xu, C., Ouyang, L., Zhao, Z., Ding, S., Zhang, S., Duan, H., Yan, H., et al. (2023). Internlm-xcomposer: A vision-language large model for advanced text-image comprehension and composition. arXiv preprint arXiv:2309.15112."},{"key":"2572_CR213","doi-asserted-by":"crossref","unstructured":"Zhang, J., Wang, T., Zhang, H., Lu, P., & Zheng, F. (2024). Reflective instruction tuning: Mitigating hallucinations in large vision-language models. In: European Conference on Computer Vision, pp. 196\u2013213. Springer.","DOI":"10.1007\/978-3-031-73113-6_12"},{"key":"2572_CR214","unstructured":"Zhang, X., Wu, C., Zhao, Z., Lin, W., Zhang, Y., Wang, Y., & Xie, W. (2023). Pmc-vqa: Visual instruction tuning for medical visual question answering. arXiv preprint arXiv:2305.10415."},{"key":"2572_CR215","doi-asserted-by":"crossref","unstructured":"Zhang, D., Yu, Y., Li, C., Dong, J., Su, D., Chu, C., & Yu, D. (2024). Mm-llms: Recent advances in multimodal large language models. arXiv preprint arXiv:2401.13601.","DOI":"10.18653\/v1\/2024.findings-acl.738"},{"key":"2572_CR216","unstructured":"Zhang, Y., Zhang, R., Gu, J., Zhou, Y., Lipka, N., Yang, D., & Sun, T. (2023). Llavar: Enhanced visual instruction tuning for text-rich image understanding. arXiv preprint arXiv:2306.17107."},{"issue":"10","key":"2572_CR217","doi-asserted-by":"publisher","first-page":"7789","DOI":"10.1109\/JIOT.2020.3039359","volume":"8","author":"J Zhang","year":"2020","unstructured":"Zhang, J., & Tao, D. (2020). Empowering things with intelligence: a survey of the progress, challenges, and opportunities in artificial intelligence of things. IEEE Internet of Things Journal, 8(10), 7789\u20137817.","journal-title":"IEEE Internet of Things Journal"},{"key":"2572_CR218","unstructured":"Zhao, Z., Guo, L., Yue, T., Chen, S., Shao, S., Zhu, X., Yuan, Z., & Liu, J. (2023). Chatbridge: Bridging modalities with large language model as a language catalyst. arXiv preprint arXiv:2305.16103."},{"key":"2572_CR219","unstructured":"Zhao, Y., Lin, Z., Zhou, D., Huang, Z., Feng, J., & Kang, B. (2023). Bubogpt: Enabling visual grounding in multi-modal llms. arXiv preprint arXiv:2307.08581."},{"key":"2572_CR220","unstructured":"Zhao, B., Wu, B., & Huang, T. (2023). Svit: Scaling up visual instruction tuning. arXiv preprint arXiv:2307.04087."},{"key":"2572_CR221","unstructured":"Zhao, L., Yu, E., Ge, Z., Yang, J., Wei, H., Zhou, H., Sun, J., Peng, Y., Dong, R., Han, C., et al. (2023). Chatspot: Bootstrapping multimodal llms via precise referring instruction tuning. arXiv preprint arXiv:2307.09474."},{"key":"2572_CR222","unstructured":"Zheng, L., Chiang, W.-L., Sheng, Y., Zhuang, S., Wu, Z., Zhuang, Y., Lin, Z., Li, Z., Li, D., Xing, E.P., Zhang, H., Gonzalez, J.E., & Stoica, I. (2023). Judging LLM-as-a-judge with MT-Bench and Chatbot Arena."},{"key":"2572_CR223","unstructured":"Zheng, J., Ma, Q., Liu, Z., Wu, B., & Feng, H. (2024). Beyond anti-forgetting: Multimodal continual instruction tuning with positive forward transfer. arXiv preprint arXiv:2401.09181."},{"key":"2572_CR224","doi-asserted-by":"crossref","unstructured":"Zhong, Y., Yang, J., Zhang, P., Li, C., Codella, N., Li, L.H., Zhou, L., Dai, X., Yuan, L., & Li, Y. (2022). Regionclip: Region-based language-image pretraining. In: Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition, pp. 16793\u201316803.","DOI":"10.1109\/CVPR52688.2022.01629"},{"key":"2572_CR225","doi-asserted-by":"crossref","unstructured":"Zhou, F., Liu, X., Zeng, Q., Li, Z., & Xiao, H. (2024). Sigphi-med: A lightweight vision-language assistant for biomedicine. Available at SSRN 4988925.","DOI":"10.2139\/ssrn.4988925"},{"issue":"9","key":"2572_CR226","doi-asserted-by":"publisher","first-page":"2337","DOI":"10.1007\/s11263-022-01653-1","volume":"130","author":"K Zhou","year":"2022","unstructured":"Zhou, K., Yang, J., Loy, C. C., & Liu, Z. (2022). Learning to prompt for vision-language models. International Journal of Computer Vision, 130(9), 2337\u20132348.","journal-title":"International Journal of Computer Vision"},{"issue":"1","key":"2572_CR227","doi-asserted-by":"publisher","first-page":"5649","DOI":"10.1038\/s41467-024-50043-3","volume":"15","author":"J Zhou","year":"2024","unstructured":"Zhou, J., He, X., Sun, L., Xu, J., Chen, X., Chu, Y., Zhou, L., Liao, X., Zhang, B., & Afvari, S. (2024). Pre-trained multimodal large language model enhances dermatological diagnosis using skingpt-4. Nature Communications, 15(1), 5649.","journal-title":"Nature Communications"},{"key":"2572_CR228","unstructured":"Zhu, D., Chen, J., Shen, X., Li, X., & Elhoseiny, M. (2024). Minigpt-4: Enhancing vision-language understanding with advanced large language models. 12th International Conference on Learning Representations, ICLR 2024."}],"container-title":["International Journal of Computer Vision"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s11263-025-02572-7.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s11263-025-02572-7\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s11263-025-02572-7.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,11,12]],"date-time":"2025-11-12T06:31:43Z","timestamp":1762929103000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s11263-025-02572-7"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,8,30]]},"references-count":228,"journal-issue":{"issue":"11","published-print":{"date-parts":[[2025,11]]}},"alternative-id":["2572"],"URL":"https:\/\/doi.org\/10.1007\/s11263-025-02572-7","relation":{},"ISSN":["0920-5691","1573-1405"],"issn-type":[{"value":"0920-5691","type":"print"},{"value":"1573-1405","type":"electronic"}],"subject":[],"published":{"date-parts":[[2025,8,30]]},"assertion":[{"value":"26 December 2024","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"20 August 2025","order":2,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"30 August 2025","order":3,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}}]}}