{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,12,22]],"date-time":"2025-12-22T22:22:33Z","timestamp":1766442153303,"version":"3.48.0"},"publisher-location":"New York, NY, USA","reference-count":49,"publisher":"ACM","license":[{"start":{"date-parts":[[2026,11,22]],"date-time":"2026-11-22T00:00:00Z","timestamp":1795305600000},"content-version":"vor","delay-in-days":368,"URL":"http:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"DOI":"10.13039\/100000001","name":"NSF (National Science Foundation)","doi-asserted-by":"publisher","award":["2405136,2406572"],"award-info":[{"award-number":["2405136,2406572"]}],"id":[{"id":"10.13039\/100000001","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2025,11,19]]},"DOI":"10.1145\/3719027.3765124","type":"proceedings-article","created":{"date-parts":[[2025,11,22]],"date-time":"2025-11-22T23:33:16Z","timestamp":1763854396000},"page":"4423-4437","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":0,"title":["You Can't Steal Nothing: Mitigating Prompt Leakages in LLMs via System Vectors"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0009-0007-1973-8186","authenticated-orcid":false,"given":"Bochuan","family":"Cao","sequence":"first","affiliation":[{"name":"The Pennsylvania State University, State College, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-1671-7183","authenticated-orcid":false,"given":"Changjiang","family":"Li","sequence":"additional","affiliation":[{"name":"Palo Alto Networks, Santa Clara, USA"}]},{"ORCID":"https:\/\/orcid.org\/0009-0004-1993-912X","authenticated-orcid":false,"given":"Yuanpu","family":"Cao","sequence":"additional","affiliation":[{"name":"The Pennsylvania State University, State College, USA"}]},{"ORCID":"https:\/\/orcid.org\/0009-0006-0730-210X","authenticated-orcid":false,"given":"Yameng","family":"Ge","sequence":"additional","affiliation":[{"name":"The Pennsylvania State University, State College, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-4927-5833","authenticated-orcid":false,"given":"Ting","family":"Wang","sequence":"additional","affiliation":[{"name":"Stony Brook University, Stony Brook, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-1486-4526","authenticated-orcid":false,"given":"Jinghui","family":"Chen","sequence":"additional","affiliation":[{"name":"The Pennsylvania State University, State College, USA"}]}],"member":"320","published-online":{"date-parts":[[2025,11,22]]},"reference":[{"key":"e_1_3_2_1_1_1","volume-title":"https:\/\/www.anthropic.com\/news\/claude-3-5-sonnet","author":"Sonnet Claude","year":"2024","unstructured":"Anthropic. 2024. Claude 3.5 Sonnet. (2024). https:\/\/www.anthropic.com\/news\/claude-3-5-sonnet"},{"key":"e_1_3_2_1_2_1","volume-title":"Defending against alignment-breaking attacks via robustly aligned llm. arXiv preprint arXiv:2309.14348","author":"Cao Bochuan","year":"2023","unstructured":"Bochuan Cao, Yuanpu Cao, Lu Lin, and Jinghui Chen. 2023. Defending against alignment-breaking attacks via robustly aligned llm. arXiv preprint arXiv:2309.14348 (2023)."},{"key":"e_1_3_2_1_3_1","volume-title":"Personalized Steering of Large Language Models: Versatile Steering Vectors Through Bi-directional Preference Optimization. arXiv preprint arXiv:2406.00045","author":"Cao Yuanpu","year":"2024","unstructured":"Yuanpu Cao, Tianrong Zhang, Bochuan Cao, Ziyi Yin, Lu Lin, Fenglong Ma, and Jinghui Chen. 2024. Personalized Steering of Large Language Models: Versatile Steering Vectors Through Bi-directional Preference Optimization. arXiv preprint arXiv:2406.00045 (2024)."},{"key":"e_1_3_2_1_4_1","volume-title":"StruQ: Defending against prompt injection with structured queries. arXiv preprint arXiv:2402.06363","author":"Chen Sizhe","year":"2024","unstructured":"Sizhe Chen, Julien Piet, Chawin Sitawarin, and David Wagner. 2024a. StruQ: Defending against prompt injection with structured queries. arXiv preprint arXiv:2402.06363 (2024)."},{"key":"e_1_3_2_1_5_1","volume-title":"Aligning llms to be robust against prompt injection. arXiv preprint arXiv:2410.05451","author":"Chen Sizhe","year":"2024","unstructured":"Sizhe Chen, Arman Zharmagambetov, Saeed Mahloujifar, Kamalika Chaudhuri, and Chuan Guo. 2024b. Aligning llms to be robust against prompt injection. arXiv preprint arXiv:2410.05451 (2024)."},{"key":"e_1_3_2_1_6_1","volume-title":"Prompt injection: Parameterization of fixed inputs. arXiv preprint arXiv:2206.11349","author":"Choi Eunbi","year":"2022","unstructured":"Eunbi Choi, Yongrae Jo, Joel Jang, and Minjoon Seo. 2022. Prompt injection: Parameterization of fixed inputs. arXiv preprint arXiv:2206.11349 (2022)."},{"key":"e_1_3_2_1_7_1","volume-title":"Botchat: Evaluating llms' capabilities of having multi-turn dialogues. arXiv preprint arXiv:2310.13650","author":"Duan Haodong","year":"2023","unstructured":"Haodong Duan, Jueqi Wei, Chonghua Wang, Hongwei Liu, Yixiao Fang, Songyang Zhang, Dahua Lin, and Kai Chen. 2023. Botchat: Evaluating llms' capabilities of having multi-turn dialogues. arXiv preprint arXiv:2310.13650 (2023)."},{"key":"e_1_3_2_1_8_1","unstructured":"Abhimanyu Dubey Abhinav Jauhri Abhinav Pandey Abhishek Kadian Ahmad Al-Dahle Aiesha Letman Akhil Mathur Alan Schelten Amy Yang Angela Fan et al. 2024. The llama 3 herd of models. arXiv preprint arXiv:2407.21783 (2024)."},{"key":"e_1_3_2_1_9_1","volume-title":"Aligning language models with preferences through f-divergence minimization. arXiv preprint arXiv:2302.08215","author":"Go Dongyoung","year":"2023","unstructured":"Dongyoung Go, Tomasz Korbak, Germ\u00e1n Kruszewski, Jos Rozen, Nahyeon Ryu, and Marc Dymetman. 2023. Aligning language models with preferences through f-divergence minimization. arXiv preprint arXiv:2302.08215 (2023)."},{"key":"e_1_3_2_1_10_1","volume-title":"Measuring massive multitask language understanding. arXiv preprint arXiv:2009.03300","author":"Hendrycks Dan","year":"2020","unstructured":"Dan Hendrycks, Collin Burns, Steven Basart, Andy Zou, Mantas Mazeika, Dawn Song, and Jacob Steinhardt. 2020. Measuring massive multitask language understanding. arXiv preprint arXiv:2009.03300 (2020)."},{"key":"e_1_3_2_1_11_1","volume-title":"Lora: Low-rank adaptation of large language models. arXiv preprint arXiv:2106.09685","author":"Hu Edward J","year":"2021","unstructured":"Edward J Hu, Yelong Shen, Phillip Wallis, Zeyuan Allen-Zhu, Yuanzhi Li, Shean Wang, Lu Wang, and Weizhu Chen. 2021. Lora: Low-rank adaptation of large language models. arXiv preprint arXiv:2106.09685 (2021)."},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.1145\/3658644.3670370"},{"key":"e_1_3_2_1_13_1","volume-title":"Micah Goldblum, Aniruddha Saha, Jonas Geiping, and Tom Goldstein.","author":"Jain Neel","year":"2023","unstructured":"Neel Jain, Avi Schwarzschild, Yuxin Wen, Gowthami Somepalli, John Kirchenbauer, Ping yeh Chiang, Micah Goldblum, Aniruddha Saha, Jonas Geiping, and Tom Goldstein. 2023. Baseline Defenses for Adversarial Attacks Against Aligned Language Models. arXiv:2309.00614 [cs.LG]"},{"key":"e_1_3_2_1_14_1","volume-title":"Diego de las Casas, Florian Bressand, Gianna Lengyel, Guillaume Lample, Lucile Saulnier, et al.","author":"Jiang Albert Q","year":"2023","unstructured":"Albert Q Jiang, Alexandre Sablayrolles, Arthur Mensch, Chris Bamford, Devendra Singh Chaplot, Diego de las Casas, Florian Bressand, Gianna Lengyel, Guillaume Lample, Lucile Saulnier, et al., 2023. Mistral 7B. arXiv preprint arXiv:2310.06825 (2023)."},{"key":"e_1_3_2_1_15_1","volume-title":"Llms get lost in multi-turn conversation. arXiv preprint arXiv:2505.06120","author":"Laban Philippe","year":"2025","unstructured":"Philippe Laban, Hiroaki Hayashi, Yingbo Zhou, and Jennifer Neville. 2025. Llms get lost in multi-turn conversation. arXiv preprint arXiv:2505.06120 (2025)."},{"key":"e_1_3_2_1_16_1","volume-title":"Advances in Neural Information Processing Systems","volume":"36","author":"Li Kenneth","year":"2024","unstructured":"Kenneth Li, Oam Patel, Fernanda Vi\u00e9gas, Hanspeter Pfister, and Martin Wattenberg. 2024. Inference-time intervention: Eliciting truthful answers from a language model. Advances in Neural Information Processing Systems, Vol. 36 (2024)."},{"key":"e_1_3_2_1_17_1","volume-title":"In-context vectors: Making in context learning more effective and controllable through latent space steering. arXiv preprint arXiv:2311.06668","author":"Liu Sheng","year":"2023","unstructured":"Sheng Liu, Haotian Ye, Lei Xing, and James Zou. 2023c. In-context vectors: Making in context learning more effective and controllable through latent space steering. arXiv preprint arXiv:2311.06668 (2023)."},{"key":"e_1_3_2_1_18_1","unstructured":"Yi Liu Gelei Deng Yuekang Li Kailong Wang Zihao Wang Xiaofeng Wang Tianwei Zhang Yepang Liu Haoyu Wang Yan Zheng et al. 2023a. Prompt Injection attack against LLM-integrated Applications. arXiv preprint arXiv:2306.05499 (2023)."},{"key":"e_1_3_2_1_19_1","volume-title":"Prompt injection attacks and defenses in llm-integrated applications. arXiv preprint arXiv:2310.12815","author":"Liu Yupei","year":"2023","unstructured":"Yupei Liu, Yuqi Jia, Runpeng Geng, Jinyuan Jia, and Neil Zhenqiang Gong. 2023b. Prompt injection attacks and defenses in llm-integrated applications. arXiv preprint arXiv:2310.12815 (2023)."},{"key":"e_1_3_2_1_20_1","volume-title":"33rd USENIX Security Symposium (USENIX Security 24)","author":"Liu Yupei","year":"2024","unstructured":"Yupei Liu, Yuqi Jia, Runpeng Geng, Jinyuan Jia, and Neil Zhenqiang Gong. 2024a. Formalizing and benchmarking prompt injection attacks and defenses. In 33rd USENIX Security Symposium (USENIX Security 24). 1831-1847."},{"key":"e_1_3_2_1_21_1","volume-title":"33rd USENIX Security Symposium (USENIX Security 24)","author":"Liu Yupei","year":"2024","unstructured":"Yupei Liu, Yuqi Jia, Runpeng Geng, Jinyuan Jia, and Neil Zhenqiang Gong. 2024b. Formalizing and benchmarking prompt injection attacks and defenses. In 33rd USENIX Security Symposium (USENIX Security 24). 1831-1847."},{"key":"e_1_3_2_1_22_1","unstructured":"Ilya Loshchilov Frank Hutter et al. 2017. Fixing weight decay regularization in adam. arXiv preprint arXiv:1711.05101 Vol. 5 (2017)."},{"key":"e_1_3_2_1_23_1","volume-title":"Language model inversion. arXiv preprint arXiv:2311.13647","author":"Morris John X","year":"2023","unstructured":"John X Morris, Wenting Zhao, Justin T Chiu, Vitaly Shmatikov, and Alexander M Rush. 2023. Language model inversion. arXiv preprint arXiv:2311.13647 (2023)."},{"key":"e_1_3_2_1_24_1","volume-title":"A Brief Report on LawGPT 1.0: A Virtual Legal Assistant Based on GPT-3. arXiv preprint arXiv:2302.05729","author":"Nguyen Ha-Thanh","year":"2023","unstructured":"Ha-Thanh Nguyen. 2023. A Brief Report on LawGPT 1.0: A Virtual Legal Assistant Based on GPT-3. arXiv preprint arXiv:2302.05729 (2023)."},{"key":"e_1_3_2_1_25_1","unstructured":"OpenAI. 2023. GPT-4 Technical Report. arXiv:2303.08774 [cs.CL]"},{"key":"e_1_3_2_1_26_1","volume-title":"Steering llama 2 via contrastive activation addition. arXiv preprint arXiv:2312.06681","author":"Panickssery Nina","year":"2023","unstructured":"Nina Panickssery, Nick Gabrieli, Julian Schulz, Meg Tong, Evan Hubinger, and Alexander Matt Turner. 2023. Steering llama 2 via contrastive activation addition. arXiv preprint arXiv:2312.06681 (2023)."},{"key":"e_1_3_2_1_27_1","volume-title":"Llmmap: Fingerprinting for large language models. arXiv preprint arXiv:2407.15847","author":"Pasquini Dario","year":"2024","unstructured":"Dario Pasquini, Evgenios M Kornaropoulos, and Giuseppe Ateniese. 2024. Llmmap: Fingerprinting for large language models. arXiv preprint arXiv:2407.15847 (2024)."},{"key":"e_1_3_2_1_28_1","volume-title":"Ignore previous prompt: Attack techniques for language models. arXiv preprint arXiv:2211.09527","author":"Perez F\u00e1bio","year":"2022","unstructured":"F\u00e1bio Perez and Ian Ribeiro. 2022. Ignore previous prompt: Attack techniques for language models. arXiv preprint arXiv:2211.09527 (2022)."},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-70879-4_6"},{"key":"e_1_3_2_1_30_1","volume-title":"Fine-tuning aligned language models compromises safety, even when users do not intend to! arXiv preprint arXiv:2310.03693","author":"Qi Xiangyu","year":"2023","unstructured":"Xiangyu Qi, Yi Zeng, Tinghao Xie, Pin-Yu Chen, Ruoxi Jia, Prateek Mittal, and Peter Henderson. 2023. Fine-tuning aligned language models compromises safety, even when users do not intend to! arXiv preprint arXiv:2310.03693 (2023)."},{"key":"e_1_3_2_1_31_1","volume-title":"Advances in Neural Information Processing Systems","volume":"36","author":"Rafailov Rafael","year":"2024","unstructured":"Rafael Rafailov, Archit Sharma, Eric Mitchell, Christopher D Manning, Stefano Ermon, and Chelsea Finn. 2024. Direct preference optimization: Your language model is secretly a reward model. Advances in Neural Information Processing Systems, Vol. 36 (2024)."},{"key":"e_1_3_2_1_32_1","volume-title":"Yossi Adi, Jingyu Liu, Romain Sauvestre, Tal Remez, et al.","author":"Roziere Baptiste","year":"2023","unstructured":"Baptiste Roziere, Jonas Gehring, Fabian Gloeckle, Sten Sootla, Itai Gat, Xiaoqing Ellen Tan, Yossi Adi, Jingyu Liu, Romain Sauvestre, Tal Remez, et al., 2023. Code llama: Open foundation models for code. arXiv preprint arXiv:2308.12950 (2023)."},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"publisher","DOI":"10.1145\/3658644.3690291"},{"key":"e_1_3_2_1_34_1","volume-title":"Extracting latent steering vectors from pretrained language models. arXiv preprint arXiv:2205.05124","author":"Subramani Nishant","year":"2022","unstructured":"Nishant Subramani, Nivedita Suresh, and Matthew E Peters. 2022. Extracting latent steering vectors from pretrained language models. arXiv preprint arXiv:2205.05124 (2022)."},{"key":"e_1_3_2_1_35_1","unstructured":"Gemini Team Rohan Anil Sebastian Borgeaud Jean-Baptiste Alayrac Jiahui Yu Radu Soricut Johan Schalkwyk Andrew M Dai Anja Hauth Katie Millican et al. 2023. Gemini: a family of highly capable multimodal models. arXiv preprint arXiv:2312.11805 (2023)."},{"key":"e_1_3_2_1_36_1","volume-title":"Kabilan Elangovan, Laura Gutierrez, Ting Fang Tan, and Daniel Shu Wei Ting.","author":"Thirunavukarasu Arun James","year":"2023","unstructured":"Arun James Thirunavukarasu, Darren Shu Jeng Ting, Kabilan Elangovan, Laura Gutierrez, Ting Fang Tan, and Daniel Shu Wei Ting. 2023. Large language models in medicine. Nature medicine (2023), 1-11."},{"key":"e_1_3_2_1_37_1","unstructured":"Hugo Touvron Louis Martin Kevin Stone Peter Albert Amjad Almahairi Yasmine Babaei Nikolay Bashlykov Soumya Batra Prajjwal Bhargava Shruti Bhosale et al. 2023. Llama 2: Open foundation and fine-tuned chat models. arXiv preprint arXiv:2307.09288 (2023)."},{"key":"e_1_3_2_1_38_1","volume-title":"Activation addition: Steering language models without optimization. arXiv e-prints","author":"Turner Alexander Matt","year":"2023","unstructured":"Alexander Matt Turner, Lisa Thiergart, Gavin Leech, David Udell, Juan J Vazquez, Ulisse Mini, and Monte MacDiarmid. 2023. Activation addition: Steering language models without optimization. arXiv e-prints (2023), arXiv-2308."},{"key":"e_1_3_2_1_39_1","volume-title":"Backdoor activation attack: Attack large language models using activation steering for safety-alignment. arXiv preprint arXiv:2311.09433","author":"Wang Haoran","year":"2023","unstructured":"Haoran Wang and Kai Shu. 2023. Backdoor activation attack: Attack large language models using activation steering for safety-alignment. arXiv preprint arXiv:2311.09433 (2023)."},{"key":"e_1_3_2_1_40_1","volume-title":"Jailbreak and guard aligned language models with only few in-context demonstrations. arXiv preprint arXiv:2310.06387","author":"Wei Zeming","year":"2023","unstructured":"Zeming Wei, Yifei Wang, and Yisen Wang. 2023. Jailbreak and guard aligned language models with only few in-context demonstrations. arXiv preprint arXiv:2310.06387 (2023)."},{"key":"e_1_3_2_1_41_1","unstructured":"Simon Willison. 2024. Delimiters won't save you from prompt injection."},{"key":"e_1_3_2_1_42_1","unstructured":"Shijie Wu Ozan Irsoy Steven Lu Vadim Dabravolski Mark Dredze Sebastian Gehrmann Prabhanjan Kambadur David Rosenberg and Gideon Mann. 2023. BloombergGPT: A Large Language Model for Finance. arXiv:2303.17564 [cs.LG]"},{"key":"e_1_3_2_1_43_1","volume-title":"Pheng Ann Heng, and Wai Lam","author":"Yang Haoran","year":"2024","unstructured":"Haoran Yang, Yumeng Zhang, Jiaqi Xu, Hongyuan Lu, Pheng Ann Heng, and Wai Lam. 2024. Unveiling the generalization power of fine-tuned large language models. arXiv preprint arXiv:2403.09162 (2024)."},{"key":"e_1_3_2_1_44_1","volume-title":"Benchmarking and defending against indirect prompt injection attacks on large language models. arXiv preprint arXiv:2312.14197","author":"Yi Jingwei","year":"2023","unstructured":"Jingwei Yi, Yueqi Xie, Bin Zhu, Emre Kiciman, Guangzhong Sun, Xing Xie, and Fangzhao Wu. 2023. Benchmarking and defending against indirect prompt injection attacks on large language models. arXiv preprint arXiv:2312.14197 (2023)."},{"key":"e_1_3_2_1_45_1","volume-title":"Assessing prompt injection risks in 200 custom gpts. arXiv preprint arXiv:2311.11538","author":"Yu Jiahao","year":"2023","unstructured":"Jiahao Yu, Yuhang Wu, Dong Shu, Mingyu Jin, and Xinyu Xing. 2023. Assessing prompt injection risks in 200 custom gpts. arXiv preprint arXiv:2311.11538 (2023)."},{"key":"e_1_3_2_1_46_1","volume-title":"On the Safety of Open-Sourced Large Language Models: Does Alignment Really Prevent Them From Being Misused? arXiv preprint arXiv:2310.01581","author":"Zhang Hangfan","year":"2023","unstructured":"Hangfan Zhang, Zhimeng Guo, Huaisheng Zhu, Bochuan Cao, Lu Lin, Jinyuan Jia, Jinghui Chen, and Dinghao Wu. 2023a. On the Safety of Open-Sourced Large Language Models: Does Alignment Really Prevent Them From Being Misused? arXiv preprint arXiv:2310.01581 (2023)."},{"key":"e_1_3_2_1_47_1","volume-title":"Defending large language models against jailbreaking attacks through goal prioritization. arXiv preprint arXiv:2311.09096","author":"Zhang Zhexin","year":"2023","unstructured":"Zhexin Zhang, Junxiao Yang, Pei Ke, Fei Mi, Hongning Wang, and Minlie Huang. 2023b. Defending large language models against jailbreaking attacks through goal prioritization. arXiv preprint arXiv:2311.09096 (2023)."},{"key":"e_1_3_2_1_48_1","unstructured":"Andy Zou Long Phan Sarah Chen James Campbell Phillip Guo Richard Ren Alexander Pan Xuwang Yin Mantas Mazeika Ann-Kathrin Dombrowski et al. 2023a. Representation engineering: A top-down approach to ai transparency. arXiv preprint arXiv:2310.01405 (2023)."},{"key":"e_1_3_2_1_49_1","volume-title":"Universal and transferable adversarial attacks on aligned language models. arXiv preprint arXiv:2307.15043","author":"Zou Andy","year":"2023","unstructured":"Andy Zou, Zifan Wang, J Zico Kolter, and Matt Fredrikson. 2023b. Universal and transferable adversarial attacks on aligned language models. arXiv preprint arXiv:2307.15043 (2023)."}],"event":{"name":"CCS '25: ACM SIGSAC Conference on Computer and Communications Security","sponsor":["SIGSAC ACM Special Interest Group on Security, Audit, and Control"],"location":"Taipei Taiwan","acronym":"CCS '25"},"container-title":["Proceedings of the 2025 ACM SIGSAC Conference on Computer and Communications Security"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3719027.3765124","content-type":"application\/pdf","content-version":"vor","intended-application":"syndication"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3719027.3765124","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,12,22]],"date-time":"2025-12-22T22:20:26Z","timestamp":1766442026000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3719027.3765124"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,11,19]]},"references-count":49,"alternative-id":["10.1145\/3719027.3765124","10.1145\/3719027"],"URL":"https:\/\/doi.org\/10.1145\/3719027.3765124","relation":{},"subject":[],"published":{"date-parts":[[2025,11,19]]},"assertion":[{"value":"2025-11-22","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}