{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,8,2]],"date-time":"2025-08-02T14:31:19Z","timestamp":1754145079466,"version":"3.41.2"},"publisher-location":"New York, NY, USA","reference-count":32,"publisher":"ACM","content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2025,1,10]]},"DOI":"10.1145\/3725899.3725911","type":"proceedings-article","created":{"date-parts":[[2025,7,11]],"date-time":"2025-07-11T11:33:04Z","timestamp":1752233584000},"page":"72-78","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":0,"title":["A MoE-based Safety Fine-tuning Method for Multimodal Large Language Models"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0009-0003-7096-1739","authenticated-orcid":false,"given":"Runjia","family":"Zhang","sequence":"first","affiliation":[{"name":"University of Electronic Science and Technology of China, Chengdu, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0009-0006-0917-3399","authenticated-orcid":false,"given":"Qi","family":"Xue","sequence":"additional","affiliation":[{"name":"University of Electronic Science and Technology of China, Chengdu, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0009-0006-1901-7728","authenticated-orcid":false,"given":"Nanxin","family":"Zhang","sequence":"additional","affiliation":[{"name":"Tianfu Jiangxi Laboratory, Chengdu, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0009-0007-9891-2348","authenticated-orcid":false,"given":"Yu","family":"Wang","sequence":"additional","affiliation":[{"name":"Tianfu Jiangxi Laboratory, Chengdu, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-3720-4379","authenticated-orcid":false,"given":"Xiurui","family":"Xie","sequence":"additional","affiliation":[{"name":"University of Electronic Science and Technology of China, Chengdu, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-4839-0234","authenticated-orcid":false,"given":"Dongyang","family":"Zhang","sequence":"additional","affiliation":[{"name":"University of Electronic Science and Technology of China, Chengdu, China"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2025,7,11]]},"reference":[{"key":"e_1_3_3_1_1_2","doi-asserted-by":"crossref","unstructured":"Yin S. Fu C. Zhao S. Li K. Sun X. Xu T. and Chen E. 2024. A survey on multimodal large language models.\u00a0National Science Review p.nwae403.","DOI":"10.1093\/nsr\/nwae403"},{"key":"e_1_3_3_1_2_2","unstructured":"Achiam Josh et al. \"Gpt-4 technical report.\"\u00a0arXiv preprint arXiv:2303.08774\u00a0(2023)."},{"key":"e_1_3_3_1_3_2","unstructured":"Templeton A. 2024.\u00a0Scaling monosemanticity: Extracting interpretable features from claude 3 sonnet. Anthropic."},{"key":"e_1_3_3_1_4_2","unstructured":"Team G. Anil R. Borgeaud S. Alayrac J.B. Yu J. Soricut R. Schalkwyk J. Dai A.M. Hauth A. Millican K. and Silver D. 2023. Gemini: a family of highly capable multimodal models.\u00a0arXiv preprint arXiv:2312.11805."},{"key":"e_1_3_3_1_5_2","doi-asserted-by":"crossref","unstructured":"Bewersdorff A. Hartmann C. Hornberger M. Se\u00dfler K. Bannert M. Kasneci E. Kasneci G. Zhai X. and Nerdel C. 2024. Taking the next step with generative artificial intelligence: The transformative role of multimodal large language models in science education.\u00a0arXiv preprint arXiv:2401.00832.","DOI":"10.1016\/j.lindif.2024.102601"},{"volume-title":"The impact of multimodal large language models on health care's future.\u00a0Journal of medical Internet research,\u00a025, p.e52865","author":"Mesk\u00f3 B.","key":"e_1_3_3_1_6_2","unstructured":"Mesk\u00f3, B., 2023. The impact of multimodal large language models on health care's future.\u00a0Journal of medical Internet research,\u00a025, p.e52865."},{"key":"e_1_3_3_1_7_2","unstructured":"Koh J.Y. Fried D. and Salakhutdinov R.R. 2024. Generating images with multimodal language models.\u00a0Advances in Neural Information Processing Systems \u00a036."},{"key":"e_1_3_3_1_8_2","unstructured":"Chung H.W. Hou L. Longpre S. Zoph B. Tay Y. Fedus W. Li Y. Wang X. Dehghani M. Brahma S. and Webson A. 2024. Scaling instruction-finetuned language models.\u00a0Journal of Machine Learning Research \u00a025(70) pp.1-53."},{"key":"e_1_3_3_1_9_2","unstructured":"Ouyang L. Wu J. Jiang X. Almeida D. Wainwright C. Mishkin P. Zhang C. Agarwal S. Slama K. Ray A. and Schulman J. 2022. Training language models to follow instructions with human feedback.\u00a0Advances in neural information processing systems \u00a035 pp.27730-27744."},{"key":"e_1_3_3_1_10_2","unstructured":"Qi X. Zeng Y. Xie T. Chen P.Y. Jia R. Mittal P. and Henderson P. 2023. Fine-tuning aligned language models compromises safety even when users do not intend to!.\u00a0arXiv preprint arXiv:2310.03693."},{"key":"e_1_3_3_1_11_2","volume-title":"Minicpm-v: A gpt-4v level mllm on your phone.\u00a0arXiv preprint arXiv:2408.01800.","author":"Yao Y.","year":"2024","unstructured":"Yao, Y., Yu, T., Zhang, A., Wang, C., Cui, J., Zhu, H., Cai, T., Li, H., Zhao, W., He, Z. and Chen, Q., 2024. Minicpm-v: A gpt-4v level mllm on your phone.\u00a0arXiv preprint arXiv:2408.01800."},{"volume-title":"In\u00a0Proceedings of the IEEE\/CVF conference on computer vision and pattern recognition\u00a0(pp. 8317-8326)","author":"Singh A.","key":"e_1_3_3_1_12_2","unstructured":"Singh, A., Natarajan, V., Shah, M., Jiang, Y., Chen, X., Batra, D., Parikh, D. and Rohrbach, M., 2019. Towards vqa models that can read. In\u00a0Proceedings of the IEEE\/CVF conference on computer vision and pattern recognition\u00a0(pp. 8317-8326)."},{"key":"e_1_3_3_1_13_2","volume-title":"Vaccine: Perturbation-aware alignment for large language model.\u00a0arXiv preprint arXiv:2402.01109.","author":"Huang T.","year":"2024","unstructured":"Huang, T., Hu, S. and Liu, L., 2024. Vaccine: Perturbation-aware alignment for large language model.\u00a0arXiv preprint arXiv:2402.01109."},{"key":"e_1_3_3_1_14_2","unstructured":"Liu G. Lin W. Huang T. Mo R. Mu Q. and Shen L. 2024. Targeted vaccine: Safety alignment for large language models against harmful fine-tuning via layer-wise perturbation.\u00a0arXiv preprint arXiv:2410.09760."},{"key":"e_1_3_3_1_15_2","unstructured":"Rosati D. Wehner J. Williams K. Bartoszcze \u0141. Atanasov D. Gonzales R. Majumdar S. Maple C. Sajjad H. and Rudzicz F. 2024. Representation noising effectively prevents harmful fine-tuning on LLMs.\u00a0arXiv preprint arXiv:2405.14577."},{"key":"e_1_3_3_1_16_2","unstructured":"Wang J. Li J. Li Y. Qi X. Chen M. Hu J. Li Y. Li B. and Xiao C. 2024. Mitigating fine-tuning jailbreak attack with backdoor enhanced alignment.\u00a0arXiv preprint arXiv:2402.14968."},{"key":"e_1_3_3_1_17_2","unstructured":"Hsu C.Y. Tsai Y.L. Lin C.H. Chen P.Y. Yu C.M. and Huang C.Y. 2024. Safe LoRA: the Silver Lining of Reducing Safety Risks when Fine-tuning Large Language Models.\u00a0arXiv preprint arXiv:2405.16833."},{"key":"e_1_3_3_1_18_2","volume-title":"Antidote: Post-fine-tuning safety alignment for large language models against harmful fine-tuning.\u00a0arXiv preprint arXiv:2408.09600.","author":"Huang T.","year":"2024","unstructured":"Huang, T., Bhattacharya, G., Joshi, P., Kimball, J. and Liu, L., 2024. Antidote: Post-fine-tuning safety alignment for large language models against harmful fine-tuning.\u00a0arXiv preprint arXiv:2408.09600."},{"key":"e_1_3_3_1_19_2","doi-asserted-by":"crossref","unstructured":"Czejdo C. and Bhattacharya S. 2022. Increasing Accessibility of Language Models with Multi-stage Information Extraction.\u00a0Journal of Advances in Information Technology Vol \u00a013(2).","DOI":"10.12720\/jait.13.2.181-185"},{"key":"e_1_3_3_1_20_2","doi-asserted-by":"crossref","unstructured":"Sujaini H. Cahyawijaya S. and Putra A.B. 2023. Analysis of Language Model Role in Improving Machine Translation Accuracy for Extremely Low Resource Languages.\u00a0Journal of Advances in Information Technology \u00a014(5).","DOI":"10.12720\/jait.14.5.1073-1081"},{"key":"e_1_3_3_1_21_2","doi-asserted-by":"crossref","unstructured":"Hong P.P. Khanh H.H. Vinh N.N. Trung N.N. Quoc A.N. and Ngoc H.T. 2024. Deep Learning-Based Lane-Keeping Assist System for Self-Driving Cars Using Transfer Learning and Fine Tuning.\u00a0Journal of Advances in Information Technology \u00a015(3).","DOI":"10.12720\/jait.15.3.322-329"},{"key":"e_1_3_3_1_22_2","unstructured":"Fedus W. Zoph B. and Shazeer N. 2022. Switch transformers: Scaling to trillion parameter models with simple and efficient sparsity.\u00a0Journal of Machine Learning Research \u00a023(120) pp.1-39."},{"key":"e_1_3_3_1_23_2","volume-title":"Gshard: Scaling giant models with conditional computation and automatic sharding.\u00a0arXiv preprint arXiv:2006.16668.","author":"Lepikhin D.","year":"2020","unstructured":"Lepikhin, D., Lee, H., Xu, Y., Chen, D., Firat, O., Huang, Y., Krikun, M., Shazeer, N. and Chen, Z., 2020. Gshard: Scaling giant models with conditional computation and automatic sharding.\u00a0arXiv preprint arXiv:2006.16668."},{"key":"e_1_3_3_1_24_2","unstructured":"Shazeer N. Mirhoseini A. Maziarz K. Davis A. Le Q. Hinton G. and Dean J. 2017. Outrageously large neural networks: The sparsely-gated mixture-of-experts layer.\u00a0arXiv preprint arXiv:1701.06538."},{"key":"e_1_3_3_1_25_2","unstructured":"Mustafa B. Riquelme C. Puigcerver J. Jenatton R. and Houlsby N. 2022. Multimodal contrastive learning with limoe: the language-image mixture of experts.\u00a0Advances in Neural Information Processing Systems \u00a035 pp.9564-9576."},{"key":"e_1_3_3_1_26_2","unstructured":"Riquelme C. Puigcerver J. Mustafa B. Neumann M. Jenatton R. Susano Pinto A. Keysers D. and Houlsby N. 2021. Scaling vision with sparse mixture of experts.\u00a0Advances in Neural Information Processing Systems \u00a034 pp.8583-8595."},{"key":"e_1_3_3_1_27_2","doi-asserted-by":"crossref","unstructured":"Shen S. Yao Z. Li C. Darrell T. Keutzer K. and He Y. 2023. Scaling vision-language models with sparse mixture of experts.\u00a0arXiv preprint arXiv:2303.07226.","DOI":"10.18653\/v1\/2023.findings-emnlp.758"},{"volume-title":"In\u00a0Proceedings of the 62nd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)\u00a0(pp. 1932-1945)","author":"Dou S.","key":"e_1_3_3_1_28_2","unstructured":"Dou, S., Zhou, E., Liu, Y., Gao, S., Shen, W., Xiong, L., Zhou, Y., Wang, X., Xi, Z., Fan, X. and Pu, S., 2024, August. LoRAMoE: Alleviating world knowledge forgetting in large language models via MoE-style plugin. In\u00a0Proceedings of the 62nd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)\u00a0(pp. 1932-1945)."},{"key":"e_1_3_3_1_29_2","doi-asserted-by":"crossref","unstructured":"Jacobs R.A. Jordan M.I. Nowlan S.J. and Hinton G.E. 1991. Adaptive mixtures of local experts.\u00a0Neural computation \u00a03(1) pp.79-87.","DOI":"10.1162\/neco.1991.3.1.79"},{"key":"e_1_3_3_1_30_2","unstructured":"Zhang Y. Chen L. Zheng G. Gao Y. Zheng R. Fu J. Yin Z. Jin S. Qiao Y. Huang X. and Zhao F. 2024. SPA-VL: A Comprehensive Safety Preference Alignment Dataset for Vision Language Model.\u00a0arXiv preprint arXiv:2406.12030."},{"key":"e_1_3_3_1_31_2","unstructured":"Zong Y. Bohdal O. Yu T. Yang Y. and Hospedales T. 2024. Safety fine-tuning at (almost) no cost: A baseline for vision large language models.\u00a0arXiv preprint arXiv:2402.02207."},{"key":"e_1_3_3_1_32_2","volume-title":"Beavertails: Towards improved safety alignment of llm via a human-preference dataset.\u00a0Advances in Neural Information Processing Systems,\u00a036.","author":"Ji J.","year":"2024","unstructured":"Ji, J., Liu, M., Dai, J., Pan, X., Zhang, C., Bian, C., Chen, B., Sun, R., Wang, Y. and Yang, Y., 2024. Beavertails: Towards improved safety alignment of llm via a human-preference dataset.\u00a0Advances in Neural Information Processing Systems,\u00a036."}],"event":{"name":"ICSIM 2025: 2025 The 8th International Conference on Software Engineering and Information Management","acronym":"ICSIM 2025","location":"Singapore Singapore"},"container-title":["Proceedings of the 2025 8th International Conference on Software Engineering and Information Management"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3725899.3725911","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,7,16]],"date-time":"2025-07-16T11:35:52Z","timestamp":1752665752000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3725899.3725911"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,1,10]]},"references-count":32,"alternative-id":["10.1145\/3725899.3725911","10.1145\/3725899"],"URL":"https:\/\/doi.org\/10.1145\/3725899.3725911","relation":{},"subject":[],"published":{"date-parts":[[2025,1,10]]},"assertion":[{"value":"2025-07-11","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}