{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,14]],"date-time":"2026-07-14T03:46:14Z","timestamp":1784000774616,"version":"3.55.0"},"publisher-location":"Cham","reference-count":72,"publisher":"Springer Nature Switzerland","isbn-type":[{"value":"9783031726606","type":"print"},{"value":"9783031726613","type":"electronic"}],"license":[{"start":{"date-parts":[[2024,11,27]],"date-time":"2024-11-27T00:00:00Z","timestamp":1732665600000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2024,11,27]],"date-time":"2024-11-27T00:00:00Z","timestamp":1732665600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2025]]},"DOI":"10.1007\/978-3-031-72661-3_5","type":"book-chapter","created":{"date-parts":[[2024,11,26]],"date-time":"2024-11-26T07:51:37Z","timestamp":1732607497000},"page":"77-94","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":15,"title":["AdaShield : Safeguarding Multimodal Large Language Models from\u00a0Structure-Based Attack via\u00a0Adaptive Shield Prompting"],"prefix":"10.1007","author":[{"given":"Yu","family":"Wang","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Xiaogeng","family":"Liu","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Yu","family":"Li","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Muhao","family":"Chen","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Chaowei","family":"Xiao","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2024,11,27]]},"reference":[{"key":"5_CR1","unstructured":"Achiam, J., et\u00a0al.: GPT-4 technical report. arXiv preprint arXiv:2303.08774 (2023)"},{"key":"5_CR2","unstructured":"Awadalla, A., et al.: OpenFlamingo: An Open-Source Framework for Training Large Autoregressive Vision-Language Models. arXiv preprint arXiv:2308.01390 (2023)"},{"key":"5_CR3","unstructured":"Bai, J., et al.: Qwen-VL: A Versatile Vision-Language Model for Understanding, Localization, Text Reading, and Beyond. arXiv preprint arXiv:2308.12966 (2023)"},{"key":"5_CR4","unstructured":"Cao, H., Liu, Z., Lu, X., Yao, Y., Li, Y.: Instructmol: multi-modal integration for building a versatile and reliable molecular assistant in drug discovery. arXiv preprint arXiv:2311.16208 (2023)"},{"key":"5_CR5","doi-asserted-by":"crossref","unstructured":"Cao, H., et al.: Presto: progressive pretraining enhances synthetic chemistry outcomes. arXiv preprint arXiv:2406.13193 (2024)","DOI":"10.18653\/v1\/2024.findings-emnlp.597"},{"key":"5_CR6","unstructured":"Carlini, N., et al.: Are aligned neural networks adversarially aligned? (2023)"},{"key":"5_CR7","doi-asserted-by":"crossref","unstructured":"Cha, S., Lee, J., Lee, Y., Yang, C.: Visually Dehallucinative Instruction Generation: Know What You Don\u2019t Know. arXiv preprint arXiv:2303.16199 (2024)","DOI":"10.1109\/ICASSP48485.2024.10446658"},{"key":"5_CR8","unstructured":"Chen, J., et al.: MiniGPT-V2: large language model as a unified interface for vision-language multi-task learning. arXiv preprint arXiv:2310.09478 (2023)"},{"key":"5_CR9","unstructured":"Chen, K., Zhang, Z., Zeng, W., Zhang, R., Zhu, F., Zhao, R.: Shikra: Unleashing Multimodal LLM\u2019s Referential Dialogue Magic. arXiv preprint arXiv:2306.15195 (2023)"},{"key":"5_CR10","doi-asserted-by":"crossref","unstructured":"Chen, Y., Sikka, K., Cogswell, M., Ji, H., Divakaran, A.: DRESS: Instructing Large Vision-Language Models to Align and Interact with Humans via Natural Language Feedback. arXiv preprint arXiv:2311.10081 (2023)","DOI":"10.1109\/CVPR52733.2024.01350"},{"key":"5_CR11","doi-asserted-by":"crossref","unstructured":"Costa, J.C., Roxo, T., Proen\u00e7a, H., In\u00e1cio, P.R.M.: How Deep Learning Sees the World: A Survey on Adversarial Attacksn and Defenses. arXiv preprint arXiv:2305.10862 (2023)","DOI":"10.1109\/ACCESS.2024.3395118"},{"key":"5_CR12","unstructured":"Dong, X., et al.: InternLM-XComposer2: Mastering Free-form Text-Image Composition and Comprehension in Vision-Language Large Model. arXiv preprint arXiv:2401.16420 (2024)"},{"key":"5_CR13","unstructured":"Dong, Y., et al.: How Robust is Google\u2019s Bard to Adversarial Image Attacks? arXiv preprint arXiv:2309.11751 (2023)"},{"key":"5_CR14","doi-asserted-by":"crossref","unstructured":"Dong, Z., Zhou, Z., Yang, C., Shao, J., Qiao, Y.: Attacks, Defenses and Evaluations for LLM Conversation Safety: A Survey. arXiv preprint arXiv:2402.09283 (2024)","DOI":"10.18653\/v1\/2024.naacl-long.375"},{"key":"5_CR15","unstructured":"Fu, C., et al.: MME: A Comprehensive Evaluation Benchmark for Multimodal Large Language Models. arXiv preprint arXiv:2306.13394 (2023)"},{"key":"5_CR16","unstructured":"Fu, C., et al.: A Challenger to GPT-4V? Early Explorations of Gemini in Visual Expertise. arXiv preprint arXiv:2312.12436 (2023)"},{"key":"5_CR17","unstructured":"Ge, J., Luo, H., Qian, S., Gan, Y., Fu, J., Zhan, S.: Chain of Thought Prompt Tuning in Vision Language Models. arXiv preprint arXiv:2304.07919 (2023)"},{"key":"5_CR18","unstructured":"Gong, Y., et al.: FigStep: Jailbreaking Large Vision-language Models via Typographic Visual Prompts. arXiv preprint arXiv:2311.05608 (2023)"},{"key":"5_CR19","unstructured":"Gu, X., et al.: Agent Smith: A Single Image Can Jailbreak One Million Multimodal LLM Agents Exponentially Fast. arXiv preprint arXiv:2402.08567 (2024)"},{"key":"5_CR20","unstructured":"Guo, P., Yang, Z., Lin, X., Zhao, Q., Zhang, Q.: PuriDefense: Randomized Local Implicit Adversarial Purification for Defending Black-box Query-based Attacks. arXiv preprint arXiv:2401.10586 (2024)"},{"key":"5_CR21","unstructured":"Han, D., Jia, X., Bai, Y., Gu, J., Liu, Y., Cao, X.: OT-Attack: Enhancing Adversarial Transferability of Vision-Language Models via Optimal Transport Optimization. arXiv preprint arXiv:2312.04403 (2023)"},{"key":"5_CR22","unstructured":"Ji, Y., et al.: Large Language Models as Automated Aligners for benchmarking Vision-Language Models. arXiv preprint arXiv:2311.14580 (2023)"},{"key":"5_CR23","unstructured":"Kojima, T., Gu, S.S., Reid, M., Matsuo, Y., Iwasawa, Y.: Large language models are zero-shot reasoners. In: NeurIPS (2022)"},{"key":"5_CR24","unstructured":"Kurakin, A., Goodfellow, I.J., Bengio, S.: Adversarial machine learning at scale. In: ICLR (2017)"},{"key":"5_CR25","doi-asserted-by":"crossref","unstructured":"Li, H., et al.: Freestyleret: retrieving images from style-diversified queries. arXiv preprint arXiv:2312.02428 (2023)","DOI":"10.1007\/978-3-031-73337-6_15"},{"key":"5_CR26","unstructured":"Li, J., Li, D., Savarese, S., Hoi, S.: BLIP-2: bootstrapping language-image pre-training with frozen image encoders and large language models. In: ICML (2023)"},{"key":"5_CR27","unstructured":"Li, L., et al.: Silkie: Preference Distillation for Large Visual Language Models. arXiv preprint arXiv:2312.10665 (2023)"},{"key":"5_CR28","doi-asserted-by":"crossref","unstructured":"Li, M., Li, L., Yin, Y., Ahmed, M., Liu, Z., Liu, Q.: Red Teaming Visual Language Models. arXiv preprint arXiv:2401.12915 (2024)","DOI":"10.18653\/v1\/2024.findings-acl.198"},{"key":"5_CR29","doi-asserted-by":"crossref","unstructured":"Lin, B., Zhu, B., Ye, Y., Ning, M., Jin, P., Yuan, L.: Video-LLaVA: Learning United Visual Representation by Alignment Before Projection. arXiv preprint arXiv:2311.10122 (2023)","DOI":"10.18653\/v1\/2024.emnlp-main.342"},{"key":"5_CR30","unstructured":"Liu, H., et al.: A Survey on Hallucination in Large Vision-Language Models. arXiv preprint arXiv:2402.00253 (2024)"},{"key":"5_CR31","unstructured":"Liu, H., Li, C., Wu, Q., Lee, Y.J.: Visual Instruction Tuning (2023)"},{"key":"5_CR32","unstructured":"Liu, M., Roy, S., Li, W., Zhong, Z., Sebe, N., Ricci, E.: Democratizing fine-grained visual recognition with large language models. In: ICLR (2024)"},{"key":"5_CR33","unstructured":"Liu, S., et al.: Multi-modal Molecule Structure-text Model for Text-based Retrieval and Editing. arXiv preprint arXiv:2212.10789 (2024)"},{"key":"5_CR34","unstructured":"Liu, X., et al.: AgentBench: evaluating LLMs as agents. In: ICLR (2024)"},{"key":"5_CR35","unstructured":"Liu, X., Xu, N., Chen, M., Xiao, C.: Generating stealthy jailbreak prompts on aligned large language models. In: ICLR (2024)"},{"key":"5_CR36","unstructured":"Liu, X., Zhu, Y., Lan, Y., Yang, C., Qiao, Y.: Query-Relevant Images Jailbreak Large Multi-Modal Models (2023)"},{"key":"5_CR37","doi-asserted-by":"crossref","unstructured":"Liu, X., Zhu, Y., Lan, Y., Yang, C., Qiao, Y.: Safety of Multimodal Large Language Models on Images and Text. arXiv preprint arXiv:2402.00357 (2024)","DOI":"10.24963\/ijcai.2024\/901"},{"key":"5_CR38","doi-asserted-by":"crossref","unstructured":"Lu, X., et al.: Moleculeqa: A dataset to evaluate factual accuracy in molecular comprehension. arXiv preprint arXiv:2403.08192 (2024)","DOI":"10.18653\/v1\/2024.findings-emnlp.216"},{"key":"5_CR39","unstructured":"Lyu, H., et al.: GPT-4v(ision) as a social media analysis engine. arXiv preprint arXiv:2311.07547 (2023)"},{"key":"5_CR40","doi-asserted-by":"crossref","unstructured":"Mao, C., Chiquier, M., Wang, H., Yang, J., Vondrick, C.: Adversarial attacks are reversible with natural supervision. In: ICCV (2021)","DOI":"10.1109\/ICCV48922.2021.00070"},{"key":"5_CR41","unstructured":"Meta: Llama usage policy (2023). Accessed 10 2023"},{"key":"5_CR42","unstructured":"Naveed, H., et al.: A Comprehensive Overview of Large Language Models. arXiv preprint arXiv:2307.06435 (2024)"},{"key":"5_CR43","unstructured":"Niu, Z., Ren, H., Gao, X., Hua, G., Jin, R.: Jailbreaking Attack against Multimodal Large Language Model. arXiv preprint arXiv:2402.02309 (2024)"},{"key":"5_CR44","unstructured":"OpenAI: OpenAI usage policy (2023). Accessed 10 2023"},{"key":"5_CR45","doi-asserted-by":"crossref","unstructured":"Pi, R., et al.: MLLM-Protector: Ensuring MLLM\u2019s Safety without Hurting Performance. arXiv preprint arXiv:2401.02906 (2024)","DOI":"10.18653\/v1\/2024.emnlp-main.895"},{"key":"5_CR46","doi-asserted-by":"crossref","unstructured":"Qi, X., Huang, K., Panda, A., Henderson, P., Wang, M., Mittal, P.: Visual Adversarial Examples Jailbreak Aligned Large Language Models. arXiv preprint arXiv:2306.13213 (2023)","DOI":"10.1609\/aaai.v38i19.30150"},{"key":"5_CR47","unstructured":"Rizwan, N., Bhaskar, P., Das, M., Majhi, S.S., Saha, P., Mukherjee, A.: Zero shot VLMs for hate meme detection: Are we there yet? arXiv preprint arXiv:2402.12198 (2024)"},{"key":"5_CR48","doi-asserted-by":"crossref","unstructured":"Schlarmann, C., Hein, M.: On the adversarial robustness of multi-modal foundation models. In: ICCV (2023)","DOI":"10.1109\/ICCVW60793.2023.00395"},{"key":"5_CR49","unstructured":"Shayegani, E., Dong, Y., Abu-Ghazaleh, N.: Jailbreak in pieces: Compositional Adversarial Attacks on Multi-Modal Language Models. arXiv preprint arXiv:2307.14539 (2023)"},{"key":"5_CR50","unstructured":"Shayegani, E., Mamun, M.A.A., Fu, Y., Zaree, P., Dong, Y., Abu-Ghazaleh, N.: Survey of vulnerabilities in large language models revealed by adversarial attacks. arXiv preprint arXiv:2310.10844 (2023)"},{"key":"5_CR51","doi-asserted-by":"crossref","unstructured":"Sun, Z., et al.: Aligning Large Multimodal Models with Factually Augmented RLHF. arXiv preprint arXiv:2309.14525 (2023)","DOI":"10.18653\/v1\/2024.findings-acl.775"},{"key":"5_CR52","unstructured":"Wang, B., et al.: DecodingTrust: A Comprehensive Assessment of Trustworthiness in GPT Models. arXiv preprint arXiv:2306.11698 (2024)"},{"key":"5_CR53","unstructured":"Wang, W., et al.: CogVLM: visual expert for pretrained language models. arXiv preprint arXiv:2311.03079 (2023)"},{"key":"5_CR54","unstructured":"Wei, T., et al.: Skywork: A More Open Bilingual Foundation Model. arXiv preprint arXiv:2310.19341 (2023)"},{"key":"5_CR55","unstructured":"Yang, J., Zhang, H., Li, F., Zou, X., Li, C., Gao, J.: Set-of-Mark Prompting Unleashes Extraordinary Visual Grounding in GPT-4V. arXiv preprint arXiv:2310.11441 (2023)"},{"key":"5_CR56","unstructured":"Ye, Q., et al.: mPLUG-Owl: modularization empowers large language models with multimodality. arXiv preprint arXiv:2304.14178 (2023)"},{"key":"5_CR57","doi-asserted-by":"crossref","unstructured":"Ye, Q., et al.: mPLUG-Owl2: Revolutionizing Multi-modal Large Language Model with Modality Collaboration. arXiv preprint arXiv:2311.04257 (2023)","DOI":"10.1109\/CVPR52733.2024.01239"},{"key":"5_CR58","unstructured":"Yin, S., et al.: A Survey on Multimodal Large Language Models. arXiv preprint arXiv:2306.13549 (2023)"},{"key":"5_CR59","unstructured":"Yin, S., et al.: Woodpecker: Hallucination Correction for Multimodal Large Language Models. arXiv preprint arXiv:2310.16045 (2023)"},{"key":"5_CR60","doi-asserted-by":"crossref","unstructured":"Yu, T., et al.: RLHF-V: Towards Trustworthy MLLMs via Behavior Alignment from Fine-grained Correctional Human Feedback. arXiv preprint arXiv:2312.00849 (2023)","DOI":"10.1109\/CVPR52733.2024.01310"},{"key":"5_CR61","unstructured":"Yu, W., et al.: MM-Vet: Evaluating Large Multimodal Models for Integrated Capabilities. arXiv preprint arXiv:2308.02490 (2023)"},{"key":"5_CR62","doi-asserted-by":"crossref","unstructured":"Zhang, D., et al.: MM-LLMs: Recent Advances in MultiModal Large Language Models. arXiv preprint arXiv:2401.13601 (2024)","DOI":"10.18653\/v1\/2024.findings-acl.738"},{"key":"5_CR63","doi-asserted-by":"crossref","unstructured":"Zhang, H., Li, X., Bing, L.: Video-LLaMA: An Instruction-tuned Audio-Visual Language Model for Video Understanding. arXiv preprint arXiv:2306.02858 (2023)","DOI":"10.18653\/v1\/2023.emnlp-demo.49"},{"key":"5_CR64","unstructured":"Zhang, R., et al.: LLaMA-Adapter: Efficient Finetuning of Language Models with Zero-init Attention. arXiv preprint arXiv:2303.16199 (2023)"},{"key":"5_CR65","unstructured":"Zhang, X., et al.: A mutation-based method for multi-modal jailbreaking attack detection. arXiv preprint arXiv:2312.10766 (2023)"},{"key":"5_CR66","doi-asserted-by":"crossref","unstructured":"Zhang, X., Li, R., Yu, J., Xu, Y., Li, W., Zhang, J.: Editguard: versatile image watermarking for tamper localization and copyright protection. In: Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition, pp. 11964\u201311974 (2024)","DOI":"10.1109\/CVPR52733.2024.01137"},{"key":"5_CR67","unstructured":"Zhao, Y., et al.: On evaluating adversarial robustness of large vision-language models. In: NeurIPS (2023)"},{"key":"5_CR68","unstructured":"Zheng, C., et al.: Prompt-Driven LLM Safeguarding via Directed Representation Optimization. arXiv preprint arXiv:2401.18018 (2024)"},{"key":"5_CR69","unstructured":"Zheng, G., Yang, B., Tang, J., Zhou, H.Y., Yang, S.: DDCoT: duty-distinct chain-of-thought prompting for multimodal reasoning in language models. In: NeurIPS (2023)"},{"key":"5_CR70","unstructured":"Zhou, H., et al.: A Survey of Large Language Models in Medicine: Progress, Application, and Challenge. arXiv preprint arXiv:2311.05112 (2023)"},{"key":"5_CR71","unstructured":"Zhu, B., et al.: LanguageBind: Extending Video-Language Pretraining to N-modality by Language-based Semantic Alignment. arXiv preprint arXiv:2310.01852 (2023)"},{"key":"5_CR72","unstructured":"Zong, Y., Bohdal, O., Yu, T., Yang, Y., Timothy, H.: Safety Fine-Tuning at (Almost) No Cost: A Baseline for Vision Large Language Models. arXiv preprint arXiv:2402.02207 (2024)"}],"container-title":["Lecture Notes in Computer Science","Computer Vision \u2013 ECCV 2024"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-031-72661-3_5","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,12,2]],"date-time":"2024-12-02T00:40:27Z","timestamp":1733100027000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-3-031-72661-3_5"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,11,27]]},"ISBN":["9783031726606","9783031726613"],"references-count":72,"URL":"https:\/\/doi.org\/10.1007\/978-3-031-72661-3_5","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"value":"0302-9743","type":"print"},{"value":"1611-3349","type":"electronic"}],"subject":[],"published":{"date-parts":[[2024,11,27]]},"assertion":[{"value":"27 November 2024","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"This paper contains offensive content that may be disturbing to some readers.","order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Disclaimer"}},{"value":"ECCV","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"European Conference on Computer Vision","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Milan","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Italy","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2024","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"29 September 2024","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"4 October 2024","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"18","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"eccv2024","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"https:\/\/eccv2024.ecva.net\/","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}}]}}