{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,4]],"date-time":"2026-07-04T00:15:44Z","timestamp":1783124144453,"version":"3.54.6"},"reference-count":59,"publisher":"Elsevier BV","license":[{"start":{"date-parts":[[2026,10,1]],"date-time":"2026-10-01T00:00:00Z","timestamp":1790812800000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.elsevier.com\/tdm\/userlicense\/1.0\/"},{"start":{"date-parts":[[2026,10,1]],"date-time":"2026-10-01T00:00:00Z","timestamp":1790812800000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.elsevier.com\/legal\/tdmrep-license"},{"start":{"date-parts":[[2026,10,1]],"date-time":"2026-10-01T00:00:00Z","timestamp":1790812800000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-017"},{"start":{"date-parts":[[2026,10,1]],"date-time":"2026-10-01T00:00:00Z","timestamp":1790812800000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"},{"start":{"date-parts":[[2026,10,1]],"date-time":"2026-10-01T00:00:00Z","timestamp":1790812800000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-012"},{"start":{"date-parts":[[2026,10,1]],"date-time":"2026-10-01T00:00:00Z","timestamp":1790812800000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2026,10,1]],"date-time":"2026-10-01T00:00:00Z","timestamp":1790812800000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-004"}],"funder":[{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["61972373"],"award-info":[{"award-number":["61972373"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["elsevier.com","sciencedirect.com"],"crossmark-restriction":true},"short-container-title":["Expert Systems with Applications"],"published-print":{"date-parts":[[2026,10]]},"DOI":"10.1016\/j.eswa.2026.132644","type":"journal-article","created":{"date-parts":[[2026,5,19]],"date-time":"2026-05-19T15:18:26Z","timestamp":1779203906000},"page":"132644","update-policy":"https:\/\/doi.org\/10.1016\/elsevier_cm_policy","source":"Crossref","is-referenced-by-count":0,"special_numbering":"C","title":["HalluTrigger: Triggering input-conflicting hallucinations in large language models with semantic-guided metamorphic relations"],"prefix":"10.1016","volume":"328","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-6031-762X","authenticated-orcid":false,"given":"Chongyang","family":"Liu","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0009-5063-8209","authenticated-orcid":false,"given":"Xiaoning","family":"Ren","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-2979-7151","authenticated-orcid":false,"given":"Yinxing","family":"Xue","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"78","reference":[{"key":"10.1016\/j.eswa.2026.132644_bib0001","doi-asserted-by":"crossref","unstructured":"Alzantot, M., Sharma, Y., Elgohary, A., Ho, B.-J., Srivastava, M., & Chang, K.-W. (2018). Generating natural language adversarial examples. arXiv: 1804.07998,.","DOI":"10.18653\/v1\/D18-1316"},{"key":"10.1016\/j.eswa.2026.132644_bib0002","unstructured":"Anthropic (2023). Introducing claude. https:\/\/www.anthropic.com\/index\/introducing-claude."},{"key":"10.1016\/j.eswa.2026.132644_bib0003","unstructured":"Bang, Y., Cahyawijaya, S., Lee, N., Dai, W., Su, D., Wilie, B., Lovenia, H., Ji, Z., Yu, T., Chung, W. et al. (2023). A multitask, multilingual, multimodal evaluation of chatgpt on reasoning, hallucination, and interactivity. arXiv: 2302.04023."},{"key":"10.1016\/j.eswa.2026.132644_bib0004","series-title":"Proceedings of the 35th IEEE\/ACM International conference on automated software engineering","first-page":"1041","article-title":"Cats are not fish: Deep learning testing calls for out-of-distribution awareness","author":"Berend","year":"2020"},{"key":"10.1016\/j.eswa.2026.132644_bib0005","unstructured":"Bird, S., Loper, E., & Klein, E. (2001). NLTK. https:\/\/www.nltk.org\/."},{"key":"10.1016\/j.eswa.2026.132644_bib0006","series-title":"2013 13th International conference on quality software","first-page":"153","article-title":"On the correlation between the effectiveness of metamorphic relations and dissimilarities of test case executions","author":"Cao","year":"2013"},{"key":"10.1016\/j.eswa.2026.132644_bib0007","series-title":"2021 36th IEEE\/ACM International conference on automated software engineering (ASE)","first-page":"104","article-title":"Testing your question answering software via asking recursively","author":"Chen","year":"2021"},{"key":"10.1016\/j.eswa.2026.132644_bib0008","unstructured":"Chen, T. Y., Cheung, S. C., & Yiu, S. M. (2020). Metamorphic testing: A new approach for generating next test cases. arXiv: 2002.12543,."},{"key":"10.1016\/j.eswa.2026.132644_bib0009","series-title":"Proceedings of the 4th Ibero-American symposium on software engineering and knowledge engineering (JIISIC 2004)","first-page":"569","article-title":"Case studies on the selection of useful relations in metamorphic testing","author":"Chen","year":"2004"},{"key":"10.1016\/j.eswa.2026.132644_bib0010","unstructured":"Chiang, W.-L., Li, Z., Lin, Z., Sheng, Y., Wu, Z., Zhang, H., Zheng, L., Zhuang, S., Zhuang, Y., Gonzalez, J. E. et al. (2023). Vicuna: An open-source chatbot impressing GPT-4 with 90%* chatgpt quality. See https:\/\/vicuna.lmsys.org (accessed 14 April 2023), 2 (3), 6."},{"key":"10.1016\/j.eswa.2026.132644_bib0011","doi-asserted-by":"crossref","unstructured":"Dale, D., Voita, E., Barrault, L., & Costa-juss\u00e0, M. R. (2022). Detecting and mitigating hallucinations in machine translation: Model internal workings alone do well, sentence similarity even better. arXiv: 2212.08597,.","DOI":"10.18653\/v1\/2023.acl-long.3"},{"key":"10.1016\/j.eswa.2026.132644_bib0012","series-title":"Advances in neural information processing systems","article-title":"Flocks of stochastic parrots: Differentially private prompt learning for large language models","volume":"vol.36","author":"Duan","year":"2024"},{"key":"10.1016\/j.eswa.2026.132644_bib0013","doi-asserted-by":"crossref","first-page":"1066","DOI":"10.1162\/tacl_a_00506","article-title":"Evaluating attribution in dialogue systems: The BEGIN benchmark","volume":"10","author":"Dziri","year":"2022","journal-title":"Transactions of the Association for Computational Linguistics"},{"key":"10.1016\/j.eswa.2026.132644_bib0014","series-title":"2018\u202fIEEE Security and privacy workshops (SPW)","first-page":"50","article-title":"Black-box generation of adversarial text sequences to evade deep learning classifiers","author":"Gao","year":"2018"},{"key":"10.1016\/j.eswa.2026.132644_bib0015","doi-asserted-by":"crossref","unstructured":"Gao, T., Yao, X., & Chen, D. (2021). Simcse: Simple contrastive learning of sentence embeddings. arXiv: 2104.08821,.","DOI":"10.18653\/v1\/2021.emnlp-main.552"},{"key":"10.1016\/j.eswa.2026.132644_bib0016","doi-asserted-by":"crossref","unstructured":"Garg, S., & Ramakrishnan, G. (2020). Bae: BERT-based adversarial examples for text classification. arXiv: 2004.01970,.","DOI":"10.18653\/v1\/2020.emnlp-main.498"},{"key":"10.1016\/j.eswa.2026.132644_bib0017","unstructured":"Guerreiro, N. M., Voita, E., & Martins, A. F. T. (2022). Looking for a needle in a haystack: A comprehensive study of hallucinations in neural machine translation. arXiv: 2208.05309,."},{"key":"10.1016\/j.eswa.2026.132644_bib0018","unstructured":"Hendrycks, D., Burns, C., Basart, S., Zou, A., Mazeika, M., Song, D., & Steinhardt, J. (2020). Measuring massive multitask language understanding. arXiv: 2009.03300,."},{"key":"10.1016\/j.eswa.2026.132644_bib0019","unstructured":"Hu, E. J., Shen, Y., Wallis, P., Allen-Zhu, Z., Li, Y., Wang, S., Wang, L., & Chen, W. (2021). Lora: Low-rank adaptation of large language models. arXiv: 2106.09685,."},{"key":"10.1016\/j.eswa.2026.132644_bib0020","doi-asserted-by":"crossref","DOI":"10.1016\/j.inffus.2025.103376","article-title":"Syntactic paraphrase-based synthetic data generation for backdoor attacks against chinese language models","author":"Hu","year":"2025","journal-title":"Information Fusion"},{"key":"10.1016\/j.eswa.2026.132644_bib0021","series-title":"Proceedings of the 31st ACM SIGSOFT International symposium on software testing and analysis","first-page":"202","article-title":"Aeon: A method for automatic evaluation of nlp test cases","author":"Huang","year":"2022"},{"key":"10.1016\/j.eswa.2026.132644_bib0022","unstructured":"Huang, L., Yu, W., Ma, W., Zhong, W., Feng, Z., Wang, H., Chen, Q., Peng, W., Feng, X., Qin, B. et al. (2023). A survey on hallucination in large language models: Principles, taxonomy, challenges, and open questions. arXiv: 2311.05232,."},{"key":"10.1016\/j.eswa.2026.132644_bib0023","unstructured":"Jin, D., Jin, Z., Zhou, J. T., & Szolovits, P. (2019). Is BERT really robust? Natural language attack on text classification and entailment. arXiv: 1907.11932, 2, 10."},{"key":"10.1016\/j.eswa.2026.132644_bib0024","doi-asserted-by":"crossref","unstructured":"Khashabi, D., Min, S., Khot, T., Sabharwal, A., Tafjord, O., Clark, P., & Hajishirzi, H. (2020). Unifiedqa: Crossing format boundaries with a single qa system. arXiv: 2005.00700,.","DOI":"10.18653\/v1\/2020.findings-emnlp.171"},{"key":"10.1016\/j.eswa.2026.132644_bib0025","doi-asserted-by":"crossref","first-page":"317","DOI":"10.1162\/tacl_a_00023","article-title":"The narrativeqa reading comprehension challenge","volume":"6","author":"Ko\u010disk\u1ef3","year":"2018","journal-title":"Transactions of the Association for Computational Linguistics"},{"key":"10.1016\/j.eswa.2026.132644_bib0026","series-title":"International conference on learning representations","article-title":"Albert: A lite BERT for self-supervised learning of language representations","author":"Lan","year":"2020"},{"key":"10.1016\/j.eswa.2026.132644_bib0027","doi-asserted-by":"crossref","first-page":"34586","DOI":"10.52202\/068431-2506","article-title":"Factuality enhanced language models for open-ended text generation","volume":"35","author":"Lee","year":"2022","journal-title":"Advances in Neural Information Processing Systems"},{"key":"10.1016\/j.eswa.2026.132644_bib0028","doi-asserted-by":"crossref","unstructured":"Lewis, M., Liu, Y., Goyal, N., Ghazvininejad, M., Mohamed, A., Levy, O., Stoyanov, V., & Zettlemoyer, L. (2019). Bart: Denoising sequence-to-sequence pre-training for natural language generation, translation, and comprehension. arXiv: 1910.13461,.","DOI":"10.18653\/v1\/2020.acl-main.703"},{"key":"10.1016\/j.eswa.2026.132644_bib0029","doi-asserted-by":"crossref","unstructured":"Li, J., Ji, S., Du, T., Li, B., & Wang, T. (2018). Textbugger: Generating adversarial text against real-world applications. arXiv: 1812.05271,.","DOI":"10.14722\/ndss.2019.23138"},{"key":"10.1016\/j.eswa.2026.132644_bib0030","unstructured":"Li, N., Li, Y., Liu, Y., Shi, L., Wang, K., & Wang, H. (2024). Halluvault: A novel logic programming-aided metamorphic testing framework for detecting fact-conflicting hallucinations in large language models. arXiv: 2405.00648,."},{"key":"10.1016\/j.eswa.2026.132644_bib0031","unstructured":"Lin, S., Hilton, J., & Evans, O. (2021). Truthfulqa: Measuring how models mimic human falsehoods. arXiv: 2109.07958,."},{"key":"10.1016\/j.eswa.2026.132644_bib0032","series-title":"Proceedings of the 37th IEEE\/ACM international conference on automated software engineering","first-page":"1","article-title":"Qatest: A uniform fuzzing framework for question answering systems","author":"Liu","year":"2022"},{"key":"10.1016\/j.eswa.2026.132644_bib0033","unstructured":"LlamA, M. (2024). LLaMA 3. https:\/\/llama.meta.com\/llama3\/."},{"issue":"11","key":"10.1016\/j.eswa.2026.132644_bib0034","first-page":"2579","article-title":"Visualizing data using t-SNE","volume":"9","author":"Van der Maaten","year":"2008","journal-title":"Journal of Machine Learning Research"},{"key":"10.1016\/j.eswa.2026.132644_bib0035","doi-asserted-by":"crossref","unstructured":"McCoy, R. T., Pavlick, E., & Linzen, T. (2019). Right for the wrong reasons: Diagnosing syntactic heuristics in natural language inference. arXiv: 1902.01007,.","DOI":"10.18653\/v1\/P19-1334"},{"key":"10.1016\/j.eswa.2026.132644_bib0036","doi-asserted-by":"crossref","unstructured":"Min, S., Krishna, K., Lyu, X., Lewis, M., Yih, W.-t., Koh, P. W., Iyyer, M., Zettlemoyer, L., & Hajishirzi, H. (2023). Factscore: Fine-grained atomic evaluation of factual precision in long form text generation. arXiv: 2305.14251,.","DOI":"10.18653\/v1\/2023.emnlp-main.741"},{"key":"10.1016\/j.eswa.2026.132644_bib0037","doi-asserted-by":"crossref","unstructured":"Morris, J. X., Lifland, E., Yoo, J. Y., Grigsby, J., Jin, D., & Qi, Y. (2020). Textattack: A framework for adversarial attacks, data augmentation, and adversarial training in nlp. arXiv: 2005.05909,.","DOI":"10.18653\/v1\/2020.emnlp-demos.16"},{"key":"10.1016\/j.eswa.2026.132644_bib0038","series-title":"Advances in neural information processing systems","article-title":"Cross-domain transferability of adversarial perturbations","volume":"vol. 32","author":"Naseer","year":"2019"},{"key":"10.1016\/j.eswa.2026.132644_bib0039","unstructured":"OpenAI (2023). Gpt-4 technical report."},{"key":"10.1016\/j.eswa.2026.132644_bib0040","doi-asserted-by":"crossref","unstructured":"Rajpurkar, P., Jia, R., & Liang, P. (2018). Know what you don\u2019t know: Unanswerable questions for SQuAD. arXiv: 1806.03822,.","DOI":"10.18653\/v1\/P18-2124"},{"issue":"1","key":"10.1016\/j.eswa.2026.132644_bib0041","doi-asserted-by":"crossref","first-page":"26","DOI":"10.1109\/MAES.2007.327521","article-title":"The significance of letter position in word recognition","volume":"22","author":"Rawlinson","year":"2007","journal-title":"IEEE Aerospace and Electronic Systems Magazine"},{"key":"10.1016\/j.eswa.2026.132644_bib0042","series-title":"Proceedings IEEE international conference on software maintenance-1999 (ICSM\u201999).\u2019software maintenance for business change\u2019(cat. no. 99CB36360)","first-page":"179","article-title":"Test case prioritization: An empirical study","author":"Rothermel","year":"1999"},{"key":"10.1016\/j.eswa.2026.132644_bib0043","series-title":"Proceedings of the 37th IEEE\/ACM International conference on automated software engineering","first-page":"1","article-title":"Natural test generation for precise testing of question answering software","author":"Shen","year":"2022"},{"key":"10.1016\/j.eswa.2026.132644_bib0044","doi-asserted-by":"crossref","unstructured":"Wang, A., Singh, A., Michael, J., Hill, F., Levy, O., & Bowman, S. R. (2018). Glue: A multi-task benchmark and analysis platform for natural language understanding. arXiv: 1804.07461,.","DOI":"10.18653\/v1\/W18-5446"},{"key":"10.1016\/j.eswa.2026.132644_bib0045","doi-asserted-by":"crossref","unstructured":"Wang, S., Thompson, L., & Iyyer, M. (2021). Phrase-BERT: Improved phrase embeddings from bert with an application to corpus exploration. arXiv: 2109.06304,.","DOI":"10.18653\/v1\/2021.emnlp-main.846"},{"key":"10.1016\/j.eswa.2026.132644_bib0046","unstructured":"Williams, A., Nangia, N., & Bowman, S. R. (2017). A broad-coverage challenge corpus for sentence understanding through inference. arXiv: 1704.05426,."},{"key":"10.1016\/j.eswa.2026.132644_bib0047","series-title":"Asian conference on machine learning","first-page":"837","article-title":"Towards understanding and improving the transferability of adversarial examples in deep neural networks","author":"Wu","year":"2020"},{"key":"10.1016\/j.eswa.2026.132644_bib0048","series-title":"Proceedings of the 37th IEEE\/ACM International conference on automated software engineering","first-page":"1","article-title":"Boosting the revealing of detected violations in deep learning testing: A diversity-guided method","author":"Xie","year":"2022"},{"key":"10.1016\/j.eswa.2026.132644_bib0049","article-title":"Fuzzcode: Code large language model-based fuzz testing for industrial iot programs","author":"Yang","year":"2025","journal-title":"IEEE Internet of Things Journal"},{"key":"10.1016\/j.eswa.2026.132644_bib0050","unstructured":"Yu, J., Wang, X., Tu, S., Cao, S., Zhang-Li, D., Lv, X., Peng, H., Yao, Z., Zhang, X., Li, H. et al. (2023). Kola: Carefully benchmarking world knowledge of large language models. International Conference on Learning Representations,. 2024, 44594\u201344637."},{"key":"10.1016\/j.eswa.2026.132644_bib0051","series-title":"Advances in neural information processing systems","article-title":"Revisiting out-of-distribution robustness in NLP: Benchmarks, analysis, and LLMs evaluations","volume":"vol. 36","author":"Yuan","year":"2024"},{"key":"10.1016\/j.eswa.2026.132644_bib0052","series-title":"Proceedings of the 2021 Conference on empirical methods in natural language processing","first-page":"1612","article-title":"On the transferability of adversarial attacks against neural text classifier","author":"Yuan","year":"2021"},{"key":"10.1016\/j.eswa.2026.132644_bib0053","unstructured":"Zha, Y., Yang, Y., Li, R., & Hu, Z. (2023). Alignscore: Evaluating factual consistency with a unified alignment function. arXiv: 2305.16739,."},{"key":"10.1016\/j.eswa.2026.132644_bib0054","series-title":"Proceedings of the 2024 on ACM SIGSAC Conference on computer and communications security","first-page":"4450","article-title":"Badmerging: Backdoor attacks against model merging","author":"Zhang","year":"2024"},{"key":"10.1016\/j.eswa.2026.132644_bib0055","doi-asserted-by":"crossref","first-page":"39","DOI":"10.1162\/tacl_a_00632","article-title":"Benchmarking large language models for news summarization","volume":"12","author":"Zhang","year":"2024","journal-title":"Transactions of the Association for Computational Linguistics"},{"key":"10.1016\/j.eswa.2026.132644_bib0056","unstructured":"Zhang, Y., Li, Y., Cui, L., Cai, D., Liu, L., Fu, T., Huang, X., Zhao, E., Zhang, Y., Chen, Y. et al. (2023). Siren\u2019s song in the AI ocean: a survey on hallucination in large language models. arXiv: 2309.01219,."},{"key":"10.1016\/j.eswa.2026.132644_bib0057","doi-asserted-by":"crossref","DOI":"10.1016\/j.eswa.2024.125856","article-title":"Clean-label backdoor attack and defense: An examination of language model vulnerability","volume":"265","author":"Zhao","year":"2025","journal-title":"Expert Systems with Applications"},{"key":"10.1016\/j.eswa.2026.132644_bib0058","doi-asserted-by":"crossref","unstructured":"Zhong, M., Yin, D., Yu, T., Zaidi, A., Mutuma, M., Jha, R., Awadallah, A. H., Celikyilmaz, A., Liu, Y., Qiu, X. et al. (2021). Qmsum: A new benchmark for query-based multi-domain meeting summarization. arXiv: 2104.05938,.","DOI":"10.18653\/v1\/2021.naacl-main.472"},{"key":"10.1016\/j.eswa.2026.132644_bib0059","doi-asserted-by":"crossref","unstructured":"Zhu, K., Wang, J., Zhou, J., Wang, Z., Chen, H., Wang, Y., Yang, L., Ye, W., Gong, N. Z., Zhang, Y. et al. (2023). Promptbench: Towards evaluating the robustness of large language models on adversarial prompts. arXiv: 2306.04528,.","DOI":"10.1145\/3689217.3690621"}],"container-title":["Expert Systems with Applications"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/api.elsevier.com\/content\/article\/PII:S0957417426015575?httpAccept=text\/xml","content-type":"text\/xml","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/api.elsevier.com\/content\/article\/PII:S0957417426015575?httpAccept=text\/plain","content-type":"text\/plain","content-version":"vor","intended-application":"text-mining"}],"deposited":{"date-parts":[[2026,7,3]],"date-time":"2026-07-03T23:16:55Z","timestamp":1783120615000},"score":1,"resource":{"primary":{"URL":"https:\/\/linkinghub.elsevier.com\/retrieve\/pii\/S0957417426015575"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026,10]]},"references-count":59,"alternative-id":["S0957417426015575"],"URL":"https:\/\/doi.org\/10.1016\/j.eswa.2026.132644","relation":{},"ISSN":["0957-4174"],"issn-type":[{"value":"0957-4174","type":"print"}],"subject":[],"published":{"date-parts":[[2026,10]]},"assertion":[{"value":"Elsevier","name":"publisher","label":"This article is maintained by"},{"value":"HalluTrigger: Triggering input-conflicting hallucinations in large language models with semantic-guided metamorphic relations","name":"articletitle","label":"Article Title"},{"value":"Expert Systems with Applications","name":"journaltitle","label":"Journal Title"},{"value":"https:\/\/doi.org\/10.1016\/j.eswa.2026.132644","name":"articlelink","label":"CrossRef DOI link to publisher maintained version"},{"value":"article","name":"content_type","label":"Content Type"},{"value":"\u00a9 2026 Elsevier Ltd. All rights are reserved, including those for text and data mining, AI training, and similar technologies.","name":"copyright","label":"Copyright"}],"article-number":"132644"}}