{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,1,13]],"date-time":"2026-01-13T14:47:23Z","timestamp":1768315643391,"version":"3.49.0"},"publisher-location":"Cham","reference-count":50,"publisher":"Springer Nature Switzerland","isbn-type":[{"value":"9783031726422","type":"print"},{"value":"9783031726439","type":"electronic"}],"license":[{"start":{"date-parts":[[2024,11,22]],"date-time":"2024-11-22T00:00:00Z","timestamp":1732233600000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2024,11,22]],"date-time":"2024-11-22T00:00:00Z","timestamp":1732233600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2025]]},"DOI":"10.1007\/978-3-031-72643-9_23","type":"book-chapter","created":{"date-parts":[[2024,11,21]],"date-time":"2024-11-21T20:48:57Z","timestamp":1732222137000},"page":"388-404","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":10,"title":["Eyes Closed, Safety on: Protecting Multimodal LLMs via\u00a0Image-to-Text Transformation"],"prefix":"10.1007","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-1352-794X","authenticated-orcid":false,"given":"Yunhao","family":"Gou","sequence":"first","affiliation":[]},{"given":"Kai","family":"Chen","sequence":"additional","affiliation":[]},{"given":"Zhili","family":"Liu","sequence":"additional","affiliation":[]},{"given":"Lanqing","family":"Hong","sequence":"additional","affiliation":[]},{"given":"Hang","family":"Xu","sequence":"additional","affiliation":[]},{"given":"Zhenguo","family":"Li","sequence":"additional","affiliation":[]},{"given":"Dit-Yan","family":"Yeung","sequence":"additional","affiliation":[]},{"given":"James T.","family":"Kwok","sequence":"additional","affiliation":[]},{"given":"Yu","family":"Zhang","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2024,11,22]]},"reference":[{"key":"23_CR1","unstructured":"Alayrac, J.B., et al.: Flamingo: a visual language model for few-shot learning. arXiv preprint arxiv:2204.14198 (2022)"},{"key":"23_CR2","unstructured":"Bagdasaryan, E., Hsieh, T.Y., Nassi, B., Shmatikov, V.: (ab) using images and sounds for indirect instruction injection in multi-modal LLMs. arXiv preprint arXiv:2307.10490 (2023)"},{"key":"23_CR3","unstructured":"Bai, J., et al.: Qwen-VL: a frontier large vision-language model with versatile abilities. arXiv preprint arXiv:2308.12966 (2023)"},{"key":"23_CR4","unstructured":"Bailey, L., Ong, E., Russell, S., Emmons, S.: Image hijacks: adversarial images can control generative models at runtime. arXiv preprint arXiv:2309.00236 (2023)"},{"key":"23_CR5","unstructured":"Chen, K., et\u00a0al.: Gaining wisdom from setbacks: aligning large language models via mistake analysis. arXiv preprint arXiv:2310.10477 (2023)"},{"key":"23_CR6","doi-asserted-by":"crossref","unstructured":"Chen, L., et al.: ShareGPT4V: improving large multi-modal models with better captions. arXiv preprint arXiv:2311.12793 (2023)","DOI":"10.1007\/978-3-031-72643-9_22"},{"key":"23_CR7","unstructured":"Chen, Y., Mendes, E., Das, S., Xu, W., Ritter, A.: Can language models be instructed to protect personal information? arXiv preprint arXiv:2310.02224 (2023)"},{"key":"23_CR8","doi-asserted-by":"crossref","unstructured":"Chen, Y., Sikka, K., Cogswell, M., Ji, H., Divakaran, A.: Dress: instructing large vision-language models to align and interact with humans via natural language feedback. arXiv preprint arXiv:2311.10081 (2023)","DOI":"10.1109\/CVPR52733.2024.01350"},{"key":"23_CR9","unstructured":"Chiang, W.L., et al.: Vicuna: an open-source chatbot impressing GPT-4 with 90%* chatGPT quality (2023). https:\/\/lmsys.org\/blog\/2023-03-30-vicuna\/"},{"key":"23_CR10","unstructured":"Dai, J., et al.: Safe RLHF: safe reinforcement learning from human feedback. arXiv preprint arXiv:2310.12773 (2023)"},{"key":"23_CR11","unstructured":"Dai, W., et al.: InstructBLIP: towards general-purpose vision-language models with instruction tuning. arXiv preprint arxiv:2305.06500 (2023)"},{"key":"23_CR12","unstructured":"Dong, Y., et al.: How robust is Google\u2019s bard to adversarial image attacks? arXiv preprint arXiv:2309.11751 (2023)"},{"key":"23_CR13","unstructured":"Fu, C., et\u00a0al.: MME: a comprehensive evaluation benchmark for multimodal large language models. arXiv preprint arXiv:2306.13394 (2023)"},{"key":"23_CR14","unstructured":"Fu, X., et al.: Misusing tools in large language models with visual adversarial examples. arXiv preprint arXiv:2310.03185 (2023)"},{"key":"23_CR15","unstructured":"Gong, Y., et al.: FigStep: jailbreaking large vision-language models via typographic visual prompts. arXiv preprint arXiv:2311.05608 (2023)"},{"key":"23_CR16","unstructured":"Gou, Y., et al.: Mixture of cluster-conditional LoRA experts for vision-language instruction tuning. arXiv preprint arXiv:2312.12379 (2023)"},{"key":"23_CR17","unstructured":"Jiang, A.Q., et\u00a0al.: Mixtral of experts. arXiv preprint arXiv:2401.04088 (2024)"},{"key":"23_CR18","doi-asserted-by":"crossref","unstructured":"Li, M., Li, L., Yin, Y., Ahmed, M., Liu, Z., Liu, Q.: Red teaming visual language models. arXiv preprint arXiv:2401.12915 (2024)","DOI":"10.18653\/v1\/2024.findings-acl.198"},{"key":"23_CR19","unstructured":"Li, Y., et\u00a0al.: Automated evaluation of large vision-language models on self-driving corner cases. arXiv preprint arXiv:2404.10595 (2024)"},{"key":"23_CR20","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"740","DOI":"10.1007\/978-3-319-10602-1_48","volume-title":"Computer Vision \u2013 ECCV 2014","author":"T-Y Lin","year":"2014","unstructured":"Lin, T.-Y., et al.: Microsoft COCO: common objects in context. In: Fleet, D., Pajdla, T., Schiele, B., Tuytelaars, T. (eds.) ECCV 2014. LNCS, vol. 8693, pp. 740\u2013755. Springer, Cham (2014). https:\/\/doi.org\/10.1007\/978-3-319-10602-1_48"},{"key":"23_CR21","unstructured":"Liu, H., Sferrazza, C., Abbeel, P.: Languages are rewards: hindsight finetuning using human feedback. arXiv preprint arXiv:2302.02676 (2023)"},{"key":"23_CR22","doi-asserted-by":"crossref","unstructured":"Liu, H., Li, C., Li, Y., Lee, Y.J.: Improved baselines with visual instruction tuning. arXiv preprint arXiv:2310.03744 (2023)","DOI":"10.1109\/CVPR52733.2024.02484"},{"key":"23_CR23","unstructured":"Liu, X., Zhu, Y., Lan, Y., Yang, C., Qiao, Y.: Query-relevant images jailbreak large multi-modal models. arXiv preprint arXiv:2311.17600 (2023)"},{"key":"23_CR24","doi-asserted-by":"crossref","unstructured":"Liu, X., Zhu, Y., Lan, Y., Yang, C., Qiao, Y.: Safety of multimodal large language models on images and text. arXiv preprint arXiv:2402.00357 (2024)","DOI":"10.24963\/ijcai.2024\/901"},{"key":"23_CR25","doi-asserted-by":"crossref","unstructured":"Liu, Y., et al.: MMBench: is your multi-modal model an all-around player? arXiv preprint arXiv:2307.06281 (2023)","DOI":"10.1007\/978-3-031-72658-3_13"},{"key":"23_CR26","unstructured":"Liu, Z., et\u00a0al.: Mixture of insightful experts (mote): the synergy of thought chains and expert mixtures in self-alignment. arXiv preprint arXiv:2405.00557 (2024)"},{"key":"23_CR27","unstructured":"Luo, H., Gu, J., Liu, F., Torr, P.: An image is worth 1000 lies: transferability of adversarial images across prompts on vision-language models. In: ICLR (2024)"},{"key":"23_CR28","unstructured":"Ouyang, L., et\u00a0al.: Training language models to follow instructions with human feedback. In: NeruIPS (2022)"},{"key":"23_CR29","doi-asserted-by":"crossref","unstructured":"Pi, R., et al.: MLLM-protector: ensuring MLLM\u2019s safety without hurting performance. arXiv preprint arXiv:2401.02906 (2024)","DOI":"10.18653\/v1\/2024.emnlp-main.895"},{"key":"23_CR30","doi-asserted-by":"crossref","unstructured":"Qi, X., Huang, K., Panda, A., Wang, M., Mittal, P.: Visual adversarial examples jailbreak large language models. arXiv preprint arXiv:2306.13213 (2023)","DOI":"10.1609\/aaai.v38i19.30150"},{"key":"23_CR31","unstructured":"Rafailov, R., Sharma, A., Mitchell, E., Manning, C.D., Ermon, S., Finn, C.: Direct preference optimization: your language model is secretly a reward model. In: NeurIPS (2023)"},{"key":"23_CR32","doi-asserted-by":"crossref","unstructured":"Rombach, R., Blattmann, A., Lorenz, D., Esser, P., Ommer, B.: High-resolution image synthesis with latent diffusion models. In: CVPR (2022)","DOI":"10.1109\/CVPR52688.2022.01042"},{"key":"23_CR33","unstructured":"Saunders, W., et al.: Self-critiquing models for assisting human evaluators. arXiv preprint arXiv:2206.05802 (2022)"},{"key":"23_CR34","doi-asserted-by":"crossref","unstructured":"Schlarmann, C., Hein, M.: On the adversarial robustness of multi-modal foundation models. In: ICCV (2023)","DOI":"10.1109\/ICCVW60793.2023.00395"},{"key":"23_CR35","unstructured":"Shayegani, E., Dong, Y., Abu-Ghazaleh, N.: Plug and pray: exploiting off-the-shelf components of multi-modal models. arXiv preprint arXiv:2307.14539 (2023)"},{"key":"23_CR36","unstructured":"Sun, H., Zhang, Z., Deng, J., Cheng, J., Huang, M.: Safety assessment of Chinese large language models. arXiv preprint arXiv:2304.10436 (2023)"},{"key":"23_CR37","unstructured":"Taori, R., et al.: Stanford alpaca: an instruction-following LLaMA model (2023). https:\/\/github.com\/tatsu-lab\/stanford_alpaca"},{"key":"23_CR38","unstructured":"Touvron, H., et al.: LLaMA: open and efficient foundation language models. arXiv preprint arXiv:2302.13971 (2023)"},{"key":"23_CR39","unstructured":"Touvron, H., et\u00a0al.: LLaMA 2: open foundation and fine-tuned chat models. arXiv preprint arXiv:2307.09288 (2023)"},{"key":"23_CR40","unstructured":"Tu, H., et al.: How many unicorns are in this image? a safety evaluation benchmark for vision LLMs. arXiv preprint arXiv:2311.16101 (2023)"},{"key":"23_CR41","doi-asserted-by":"crossref","unstructured":"Wang, P., et al.: InferAligner: inference-time alignment for harmlessness through cross-model guidance. arXiv preprint arXiv:2401.11206 (2024)","DOI":"10.18653\/v1\/2024.emnlp-main.585"},{"key":"23_CR42","unstructured":"Wang, Y., et al.: Aligning large language models with human: a survey. arXiv preprint arXiv:2307.12966 (2023)"},{"key":"23_CR43","doi-asserted-by":"crossref","unstructured":"Wu, P., Xie, S.: V*: guided visual search as a core mechanism in multimodal LLMs. arXiv preprint arXiv:2312.14135 (2023)","DOI":"10.1109\/CVPR52733.2024.01243"},{"key":"23_CR44","unstructured":"Wu, Y., Li, X., Liu, Y., Zhou, P., Sun, L.: Jailbreaking GPT-4V via self-adversarial attacks with system prompts. arXiv preprint arXiv:2311.09127 (2023)"},{"key":"23_CR45","doi-asserted-by":"crossref","unstructured":"Ye, Q., et al.: mPLUG-Owl2: revolutionizing multi-modal large language model with modality collaboration. arXiv preprint arXiv:2311.04257 (2023)","DOI":"10.1109\/CVPR52733.2024.01239"},{"key":"23_CR46","unstructured":"Yu, W., et al.: MM-Vet: evaluating large multimodal models for integrated capabilities. arXiv preprint arXiv:2308.02490 (2023)"},{"key":"23_CR47","unstructured":"Zhang, P., et\u00a0al.: InterNLM-XComposer: a vision-language large model for advanced text-image comprehension and composition. arXiv preprint arXiv:2309.15112 (2023)"},{"key":"23_CR48","unstructured":"Zhang, Z., Zhang, A., Li, M., Zhao, H., Karypis, G., Smola, A.: Multimodal chain-of-thought reasoning in language models. arXiv preprint arXiv:2302.00923 (2023)"},{"key":"23_CR49","unstructured":"Zhao, Y., et al.: On evaluating adversarial robustness of large vision-language models. In: Advances in Neural Information Processing Systems, vol. 36 (2024)"},{"key":"23_CR50","unstructured":"Zong, Y., Bohdal, O., Yu, T., Yang, Y., Hospedales, T.: Safety fine-tuning at (almost) no cost: a baseline for vision large language models. arXiv preprint arXiv:2402.02207 (2024)"}],"container-title":["Lecture Notes in Computer Science","Computer Vision \u2013 ECCV 2024"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-031-72643-9_23","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,12,1]],"date-time":"2024-12-01T20:06:11Z","timestamp":1733083571000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-3-031-72643-9_23"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,11,22]]},"ISBN":["9783031726422","9783031726439"],"references-count":50,"URL":"https:\/\/doi.org\/10.1007\/978-3-031-72643-9_23","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"value":"0302-9743","type":"print"},{"value":"1611-3349","type":"electronic"}],"subject":[],"published":{"date-parts":[[2024,11,22]]},"assertion":[{"value":"22 November 2024","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"ECCV","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"European Conference on Computer Vision","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Milan","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Italy","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2024","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"29 September 2024","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"4 October 2024","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"18","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"eccv2024","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"https:\/\/eccv2024.ecva.net\/","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}}]}}