{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,9,4]],"date-time":"2026-09-04T03:14:19Z","timestamp":1788491659148,"version":"build-2803163510"},"reference-count":29,"publisher":"Springer Science and Business Media LLC","issue":"3","license":[{"start":{"date-parts":[[2026,5,1]],"date-time":"2026-05-01T00:00:00Z","timestamp":1777593600000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2026,5,1]],"date-time":"2026-05-01T00:00:00Z","timestamp":1777593600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["J. Comput. Sci. Technol."],"published-print":{"date-parts":[[2026,5]]},"DOI":"10.1007\/s11390-026-5705-z","type":"journal-article","created":{"date-parts":[[2026,9,4]],"date-time":"2026-09-04T02:42:57Z","timestamp":1788489777000},"page":"924-935","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":0,"title":["Safety Evaluation for Large Language Models with Metamorphic Testing: An Empirical Study"],"prefix":"10.1007","volume":"41","author":[{"given":"Ying","family":"Xing","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Jia-Qi","family":"Huang","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Hai-Jiao","family":"Zhao","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Long","family":"Zhang","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Wen-Jin","family":"Li","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2026,9,4]]},"reference":[{"key":"5705_CR1","unstructured":"Shi D, Shen T, Huang Y, Li Z, Leng Y, Jin R, Liu C, Wu X, Guo Z, Yu L, Shi L, Jiang B, Xiong D. Large language model safety: A holistic survey. arXiv: 2412.17686, 2024. https:\/\/arxiv.org\/abs\/2412.17686, May 2026."},{"key":"5705_CR2","doi-asserted-by":"publisher","first-page":"23790","DOI":"10.1609\/aaai.v38i21.30566","volume-title":"Proc. the 38th AAAI Conference on Artificial Intelligence","author":"S Goyal","year":"2024","unstructured":"Goyal S, Hira M, Mishra S, Goyal S, Goel A, Dadu N, DB K, Mehta S, Madaan N. LLMGuard: Guarding against unsafe LLM behavior. In Proc. the 38th AAAI Conference on Artificial Intelligence, Feb. 2024, pp.23790\u201323792. DOI: https:\/\/doi.org\/10.1609\/aaai.v38i21.30566."},{"key":"5705_CR3","doi-asserted-by":"publisher","first-page":"7101","DOI":"10.18653\/v1\/2025.findings-naacl.395","volume-title":"Proc. the 2025 Findings of the Association for Computational Linguistics","author":"Q Zhan","year":"2025","unstructured":"Zhan Q, Fang R, Panchal H S, Kang D. Adaptive attacks break defenses against indirect prompt injection attacks on LLM agents. In Proc. the 2025 Findings of the Association for Computational Linguistics, Apr. 2025, pp.7101\u20137117. DOI: https:\/\/doi.org\/10.18653\/v1\/2025.findings-naacl.395."},{"key":"5705_CR4","doi-asserted-by":"publisher","first-page":"27617","DOI":"10.1609\/aaai.v39i26.34975","volume-title":"Proc. the 39th AAAI Conference on Artificial Intelligence","author":"P R\u00f6ttger","year":"2025","unstructured":"R\u00f6ttger P, Pernisi F, Vidgen B, Hovy D. SafetyPrompts: A systematic review of open datasets for evaluating and improving large language model safety. In Proc. the 39th AAAI Conference on Artificial Intelligence, Feb. 2025, pp.27617\u201327627. DOI: https:\/\/doi.org\/10.1609\/aaai.v39i26.34975."},{"key":"5705_CR5","doi-asserted-by":"publisher","first-page":"1152","DOI":"10.1109\/ICMLA61862.2024.00177","volume-title":"Proc. the 2024 International Conference on Machine Learning and Applications","author":"T Elvira","year":"2024","unstructured":"Elvira T, Procko T T, Vonderhaar L, Ochoa O. Exploring testing methods for large language models. In Proc. the 2024 International Conference on Machine Learning and Applications, Dec. 2024, pp.1152\u20131157. DOI: https:\/\/doi.org\/10.1109\/ICMLA61862.2024.00177."},{"key":"5705_CR6","doi-asserted-by":"publisher","first-page":"3214","DOI":"10.18653\/v1\/2022.acl-long.229","volume-title":"Proc. the 60th Annual Meeting of the Association for Computational Linguistics","author":"S Lin","year":"2022","unstructured":"Lin S, Hilton J, Evans O. TruthfulQA: Measuring how models mimic human falsehoods. In Proc. the 60th Annual Meeting of the Association for Computational Linguistics, May 2022, pp.3214\u20133252. DOI: https:\/\/doi.org\/10.18653\/v1\/2022.acl-long.229."},{"key":"5705_CR7","unstructured":"Cheng Q, Sun T, Zhang W, Wang S, Liu X, Zhang M, He J, Huang M, Yin Z, Chen K, Qiu X. Evaluating hallucinations in Chinese large language models. arXiv: 2310.03368, 2023. https:\/\/arxiv.org\/abs\/2310.03368, May 2026."},{"key":"5705_CR8","doi-asserted-by":"publisher","first-page":"13538","DOI":"10.18653\/v1\/2023.acllong.757","volume-title":"Proc. the 61st Annual Meeting of the Association for Computational Linguistics","author":"J Zhao","year":"2023","unstructured":"Zhao J, Fang M, Shi Z, Li Y, Chen L, Pechenizkiy M. CHBias: Bias evaluation and mitigation of Chinese conversational language models. In Proc. the 61st Annual Meeting of the Association for Computational Linguistics, Jul. 2023, pp.13538\u201313556. DOI: https:\/\/doi.org\/10.18653\/v1\/2023.acllong.757."},{"key":"5705_CR9","unstructured":"Sun H, Zhang Z, Deng J, Cheng J, Huang M. Safety assessment of Chinese large language models. arXiv: 2304.10436, 2023. https:\/\/arxiv.org\/abs\/2304.10436, May 2026."},{"key":"5705_CR10","doi-asserted-by":"publisher","first-page":"3106","DOI":"10.18653\/v1\/2024.findings-acl.184","volume-title":"Proc. the 2024 Findings of the Association for Computational Linguistics","author":"Y Wang","year":"2024","unstructured":"Wang Y, Zhai Z, Li H, Han X, Lin S, Zhang Z, Zhao A, Nakov P, Baldwin T. A Chinese dataset for evaluating the safeguards in large language models. In Proc. the 2024 Findings of the Association for Computational Linguistics, Aug. 2024, pp.3106\u20133119. DOI: https:\/\/doi.org\/10.18653\/v1\/2024.findings-acl.184."},{"key":"5705_CR11","doi-asserted-by":"publisher","first-page":"5865","DOI":"10.18653\/v1\/2024.findings-acl.349","volume-title":"Proc. the 2024 Findings of the Association for Computational Linguistics","author":"W Wang","year":"2024","unstructured":"Wang W, Tu Z, Chen C, Yuan Y, Huang J T, Jiao W, Lyu M. All languages matter: On the multilingual safety of LLMs. In Proc. the 2024 Findings of the Association for Computational Linguistics, Aug. 2024, pp.5865\u20135877. DOI: https:\/\/doi.org\/10.18653\/v1\/2024.findings-acl.349."},{"key":"5705_CR12","doi-asserted-by":"publisher","first-page":"15537","DOI":"10.18653\/v1\/2024.acl-long.830","volume-title":"Proc. the 62nd Annual Meeting of the Association for Computational Linguistics","author":"Z Zhang","year":"2024","unstructured":"Zhang Z, Lei L, Wu L, Sun R, Huang Y, Long C, Liu X, Lei X, Tang J, Huang M. SafetyBench: Evaluating the safety of large language models. In Proc. the 62nd Annual Meeting of the Association for Computational Linguistics, Aug. 2024, pp.15537\u201315553. DOI: https:\/\/doi.org\/10.18653\/v1\/2024.acl-long.830."},{"key":"5705_CR13","unstructured":"Zhang W, Lei X, Liu Z, An M, Yang B, Zhao K, Wang K, Lian S. ChiSafetyBench: A Chinese hierarchical safety benchmark for large language models. arXiv: 2406.10311, 2024. https:\/\/arxiv.org\/abs\/2406.10311, May 2026."},{"key":"5705_CR14","doi-asserted-by":"publisher","DOI":"10.5555\/3737916.3741826","volume-title":"Proc. the 38th International Conference on Neural Information Processing Systems","author":"Y Mou","year":"2024","unstructured":"Mou Y, Zhang S, Ye W. SG-Bench: Evaluating LLM safety generalization across diverse tasks and prompt types. In Proc. the 38th International Conference on Neural Information Processing Systems, Dec. 2024, Article No. 3910. DOI: https:\/\/doi.org\/10.5555\/3737916.3741826."},{"key":"5705_CR15","doi-asserted-by":"publisher","first-page":"3923","DOI":"10.18653\/v1\/2024.findings-acl.235","volume-title":"Proc. the 2024 Findings of the Association for Computational Linguistics","author":"L Li","year":"2024","unstructured":"Li L, Dong B, Wang R, Hu X, Zuo W, Lin D, Qiao Y, Shao J. SALAD-Bench: A hierarchical and comprehensive safety benchmark for large language models. In Proc. the 2024 Findings of the Association for Computational Linguistics, Aug. 2024, pp.3923\u20133954. DOI: https:\/\/doi.org\/10.18653\/v1\/2024.findings-acl.235."},{"key":"5705_CR16","doi-asserted-by":"publisher","first-page":"95","DOI":"10.1007\/978-3-031-43412-9_6","volume-title":"Proc. the European Conference on Machine Learning and Knowledge Discovery in Databases: Research Track","author":"Y Yang","year":"2023","unstructured":"Yang Y, Huang P, Cao J, Ma F, Zhang J, Li J. Quantifying robustness to adversarial word substitutions. In Proc. the European Conference on Machine Learning and Knowledge Discovery in Databases: Research Track, Sept. 2023, pp.95\u2013112. DOI: https:\/\/doi.org\/10.1007\/978-3-031-43412-9_6."},{"key":"5705_CR17","unstructured":"Yang Y, Huang P, Cao J, Li J, Lin Y, Dong J S, Ma F, Zhang J. A prompting-based approach for adversarial example generation and robustness enhancement. arXiv: 2203.10714, 2022. https:\/\/arxiv.org\/abs\/2203.10714, May 2026."},{"key":"5705_CR18","doi-asserted-by":"publisher","first-page":"10785","DOI":"10.1609\/aaai.v36i10.21324","volume-title":"Proc. the 36th AAAI Conference on Artificial Intelligence","author":"P Huang","year":"2022","unstructured":"Huang P, Yang Y, Jia F, Liu M, Ma F, Zhang J. Word level robustness enhancement: Fight perturbation with perturbation. In Proc. the 36th AAAI Conference on Artificial Intelligence, Feb. 2022, pp.10785\u201310793. DOI: https:\/\/doi.org\/10.1609\/aaai.v36i10.21324."},{"key":"5705_CR19","doi-asserted-by":"publisher","first-page":"2387","DOI":"10.1109\/ICSE48619.2023.00200","volume-title":"Proc. the 45th IEEE\/ACM International Conference on Software Engineering","author":"W Wang","year":"2023","unstructured":"Wang W, Huang J T, Wu W, Zhang J, Huang Y, Li S, He P, Lyu M R. MTTM: Metamorphic testing for textual content moderation software. In Proc. the 45th IEEE\/ACM International Conference on Software Engineering, May 2023, pp.2387\u20132399. DOI: https:\/\/doi.org\/10.1109\/ICSE48619.2023.00200."},{"key":"5705_CR20","doi-asserted-by":"publisher","DOI":"10.1145\/3551349.3556953","volume-title":"Proc. the 37th IEEE\/ACM International Conference on Automated Software Engineering","author":"Q Shen","year":"2022","unstructured":"Shen Q, Chen J, Zhang J M, Wang H, Liu S, Tian M. Natural test generation for precise testing of question answering software. In Proc. the 37th IEEE\/ACM International Conference on Automated Software Engineering, Oct. 2022, Article No. 71. DOI: https:\/\/doi.org\/10.1145\/3551349.3556953."},{"key":"5705_CR21","doi-asserted-by":"publisher","unstructured":"Li N, Li Y, Liu Y, Shi L, Wang K, Wang H. Drowzee: Metamorphic testing for fact-conflicting hallucination detection in large language models. Proceedings of the ACM on Programming Languages, 2024, 8(OOPSLA2): Article No. 336. DOI: https:\/\/doi.org\/10.1145\/3689776.","DOI":"10.1145\/3689776"},{"key":"5705_CR22","unstructured":"Zhang Y, Cai Y, Zuo X, Luan X, Wang K, Hou Z, Zhang Y, Wei Z, Sun M, Sun J, Sun J, Dong J S. The fusion of large language models and formal methods for trustworthy AI agents: A roadmap. arXiv: 2412.06512, 2024. https:\/\/arxiv.org\/abs\/2412.06512, May 2026."},{"key":"5705_CR23","unstructured":"Xiao M, Xiao Y, Ji S, Cai H, Xue L, Zhang P. Automated robustness testing for LLM-based NLP software. arXiv: 2412.21016, 2024. https:\/\/arxiv.org\/abs\/2412.21016, May 2026."},{"key":"5705_CR24","doi-asserted-by":"publisher","unstructured":"Yang B, Al Mamun A, Zhang J, Uddin G. Hallucination detection in large language models with metamorphic relations. Proceedings of the ACM on Software Engineering, 2025, 2(FSE): Article No. FSE020. DOI: https:\/\/doi.org\/10.1145\/3715735.","DOI":"10.1145\/3715735"},{"key":"5705_CR25","doi-asserted-by":"publisher","first-page":"117","DOI":"10.1109\/ICST60714.2024.00019","volume-title":"Proc. the 2024 IEEE Conference on Software Testing, Verification and Validation","author":"S Hyun","year":"2024","unstructured":"Hyun S, Guo M, Babar M A. METAL: Metamorphic testing framework for analyzing large-language model qualities. In Proc. the 2024 IEEE Conference on Software Testing, Verification and Validation, May 2024, pp.117\u2013128. DOI: https:\/\/doi.org\/10.1109\/ICST60714.2024.00019."},{"key":"5705_CR26","doi-asserted-by":"publisher","first-page":"12","DOI":"10.1145\/3650105.3652297","volume-title":"Proc. the 1st IEEE\/ACM International Conference on AI Foundation Models and Software","author":"G Wang","year":"2024","unstructured":"Wang G, Li Y, Liu Y, Deng G, Li T, Xu G, Liu Y, Wang H, Wang K. MeTMaP: Metamorphic testing for detecting false vector matching problems in LLM augmented generation. In Proc. the 1st IEEE\/ACM International Conference on AI Foundation Models and Software, Apr. 2024, pp.12\u201323. DOI: https:\/\/doi.org\/10.1145\/3650105.3652297."},{"issue":"4","key":"5705_CR27","doi-asserted-by":"publisher","first-page":"194332","DOI":"10.1007\/s11704-024-3902-x","volume":"19","author":"L Zhang","year":"2025","unstructured":"Zhang L, Sun S, Yan J, Zhang J, Wu J, Liu J. Accelerating constraint-based neural network repairs by example prioritization and selection. Frontiers of Computer Science, 2025, 19(4): 194332. DOI: https:\/\/doi.org\/10.1007\/s11704-024-3902-x.","journal-title":"Frontiers of Computer Science"},{"key":"5705_CR28","doi-asserted-by":"publisher","first-page":"406","DOI":"10.1109\/QRS51102.2020.00059","volume-title":"Proc. the 20th IEEE International Conference on Software Quality, Reliability and Security","author":"Z Li","year":"2020","unstructured":"Li Z, Zhang L, Yan J, Zhang J, Zhang Z, Tse T H. PEACEPACT: Prioritizing examples to accelerate perturbation-based adversary generation for DNN classification testing. In Proc. the 20th IEEE International Conference on Software Quality, Reliability and Security, Dec. 2020, pp.406\u2013413. DOI: https:\/\/doi.org\/10.1109\/QRS51102.2020.00059."},{"key":"5705_CR29","doi-asserted-by":"publisher","first-page":"1097","DOI":"10.1145\/3209978.3210080","volume-title":"Proc. the 41st International ACM SIGIR Conference on Research and Development in Information Retrieval","author":"Y Zhu","year":"2018","unstructured":"Zhu Y, Lu S, Zheng L et al. Texygen: A benchmarking platform for text generation models. In Proc. the 41st International ACM SIGIR Conference on Research and Development in Information Retrieval, July 2018, pp. 1097\u20131100. DOI: https:\/\/doi.org\/10.1145\/3209978.3210080."}],"container-title":["Journal of Computer Science and Technology"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s11390-026-5705-z.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s11390-026-5705-z","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s11390-026-5705-z.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,9,4]],"date-time":"2026-09-04T02:43:00Z","timestamp":1788489780000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s11390-026-5705-z"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026,5]]},"references-count":29,"journal-issue":{"issue":"3","published-print":{"date-parts":[[2026,5]]}},"alternative-id":["5705"],"URL":"https:\/\/doi.org\/10.1007\/s11390-026-5705-z","relation":{},"ISSN":["1000-9000","1860-4749"],"issn-type":[{"value":"1000-9000","type":"print"},{"value":"1860-4749","type":"electronic"}],"subject":[],"published":{"date-parts":[[2026,5]]},"assertion":[{"value":"4 July 2025","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"14 May 2026","order":2,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"4 September 2026","order":3,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"Conflict of Interest\n                      The authors declare that they have no conflict of interest.","order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Ethics"}}]}}