{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,9]],"date-time":"2026-07-09T22:21:51Z","timestamp":1783635711193,"version":"3.55.0"},"reference-count":56,"publisher":"Elsevier BV","license":[{"start":{"date-parts":[[2026,8,1]],"date-time":"2026-08-01T00:00:00Z","timestamp":1785542400000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.elsevier.com\/tdm\/userlicense\/1.0\/"},{"start":{"date-parts":[[2026,8,1]],"date-time":"2026-08-01T00:00:00Z","timestamp":1785542400000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.elsevier.com\/legal\/tdmrep-license"},{"start":{"date-parts":[[2026,8,1]],"date-time":"2026-08-01T00:00:00Z","timestamp":1785542400000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-017"},{"start":{"date-parts":[[2026,8,1]],"date-time":"2026-08-01T00:00:00Z","timestamp":1785542400000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"},{"start":{"date-parts":[[2026,8,1]],"date-time":"2026-08-01T00:00:00Z","timestamp":1785542400000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-012"},{"start":{"date-parts":[[2026,8,1]],"date-time":"2026-08-01T00:00:00Z","timestamp":1785542400000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2026,8,1]],"date-time":"2026-08-01T00:00:00Z","timestamp":1785542400000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-004"}],"funder":[{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["62372021"],"award-info":[{"award-number":["62372021"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100002358","name":"Beihang University","doi-asserted-by":"publisher","award":["606"],"award-info":[{"award-number":["606"]}],"id":[{"id":"10.13039\/501100002358","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100011347","name":"State Key Laboratory of Software Development Environment","doi-asserted-by":"publisher","id":[{"id":"10.13039\/501100011347","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100002855","name":"Ministry of Science and Technology of the People's Republic of China","doi-asserted-by":"publisher","id":[{"id":"10.13039\/501100002855","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100012166","name":"National Key Research and Development Program of China","doi-asserted-by":"publisher","award":["2024YFB3311503"],"award-info":[{"award-number":["2024YFB3311503"]}],"id":[{"id":"10.13039\/501100012166","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["elsevier.com","sciencedirect.com"],"crossmark-restriction":true},"short-container-title":["Knowledge-Based Systems"],"published-print":{"date-parts":[[2026,8]]},"DOI":"10.1016\/j.knosys.2026.116324","type":"journal-article","created":{"date-parts":[[2026,6,8]],"date-time":"2026-06-08T05:43:20Z","timestamp":1780897400000},"page":"116324","update-policy":"https:\/\/doi.org\/10.1016\/elsevier_cm_policy","source":"Crossref","is-referenced-by-count":0,"special_numbering":"C","title":["LGMT: Logic-Grounded Metamorphic Testing for evaluating the reasoning reliability of LLMs"],"prefix":"10.1016","volume":"348","author":[{"given":"Zenghui","family":"Zhou","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Man","family":"Li","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0005-5674-3094","authenticated-orcid":false,"given":"Xiaoke","family":"Fang","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Xinyi","family":"Zhou","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Weibin","family":"Lin","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-7922-9067","authenticated-orcid":false,"given":"Zheng","family":"Zheng","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"78","reference":[{"key":"10.1016\/j.knosys.2026.116324_b1","series-title":"DeepSeek-V3 technical report","author":"DeepSeek","year":"2025"},{"key":"10.1016\/j.knosys.2026.116324_b2","first-page":"9459","article-title":"Retrieval-augmented generation for knowledge-intensive NLP tasks","volume":"vol. 33","author":"Lewis","year":"2020"},{"key":"10.1016\/j.knosys.2026.116324_b3","series-title":"A survey of large language model agents for question answering","author":"Yue","year":"2025"},{"key":"10.1016\/j.knosys.2026.116324_b4","doi-asserted-by":"crossref","first-page":"50117","DOI":"10.52202\/075280-2180","article-title":"ToolQA: A dataset for LLM question answering with external tools","volume":"36","author":"Zhuang","year":"2023","journal-title":"Adv. Neural Inf. Process. Syst."},{"key":"10.1016\/j.knosys.2026.116324_b5","series-title":"DeepSeek-coder-V2: Breaking the barrier of closed-source models in code intelligence","author":"DeepSeek-AI","year":"2024"},{"key":"10.1016\/j.knosys.2026.116324_b6","series-title":"SWE-bench: Can language models resolve real-world GitHub issues?","author":"Jimenez","year":"2024"},{"key":"10.1016\/j.knosys.2026.116324_b7","series-title":"Code llama: Open foundation models for code","author":"Rozi\u00e8re","year":"2024"},{"key":"10.1016\/j.knosys.2026.116324_b8","series-title":"Intelligent virtual assistants with LLM-based process automation","author":"Guan","year":"2023"},{"key":"10.1016\/j.knosys.2026.116324_b9","series-title":"Thinking assistants: LLM-based conversational assistants that help users think by asking rather than answering","author":"Park","year":"2024"},{"key":"10.1016\/j.knosys.2026.116324_b10","series-title":"Proceedings of the Conference on Empirical Methods in Natural Language Processing","first-page":"3800","article-title":"Conditional and modal reasoning in large language models","author":"Holliday","year":"2024"},{"key":"10.1016\/j.knosys.2026.116324_b11","series-title":"Towards LogiGLUE: A brief survey and a benchmark for analyzing logical reasoning capabilities of language models","author":"Luo","year":"2023"},{"key":"10.1016\/j.knosys.2026.116324_b12","series-title":"Proceedings of the International Conference on Computational Intelligence and Intelligent Systems","first-page":"1","article-title":"Are large language models good at fuzzy reasoning?","author":"Singh","year":"2024"},{"issue":"2","key":"10.1016\/j.knosys.2026.116324_b13","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1145\/3703155","article-title":"A survey on hallucination in large language models: Principles, taxonomy, challenges, and open questions","volume":"43","author":"Huang","year":"2025","journal-title":"ACM Trans. Inf. Syst."},{"key":"10.1016\/j.knosys.2026.116324_b14","doi-asserted-by":"crossref","first-page":"148231","DOI":"10.1109\/ACCESS.2025.3601206","article-title":"A systematic literature review of hallucinations in large language models","volume":"13","author":"Woesle","year":"2025","journal-title":"IEEE Access"},{"issue":"4","key":"10.1016\/j.knosys.2026.116324_b15","doi-asserted-by":"crossref","first-page":"1620","DOI":"10.1109\/TKDE.2025.3536008","article-title":"Are large language models really good logical reasoners? A comprehensive evaluation and beyond","volume":"37","author":"Xu","year":"2025","journal-title":"IEEE Trans. Knowl. Data Eng."},{"key":"10.1016\/j.knosys.2026.116324_b16","series-title":"Proceedings of the International Conference on Empirical Methods in Natural Language Processing","first-page":"2124","article-title":"LogicAsker: Evaluating and improving the logical reasoning ability of large language models","author":"Wan","year":"2024"},{"key":"10.1016\/j.knosys.2026.116324_b17","series-title":"The Thirteenth International Conference on Learning Representations","article-title":"Logical consistency of large language models in fact-checking","author":"Ghosh","year":"2025"},{"key":"10.1016\/j.knosys.2026.116324_b18","series-title":"Evaluating the logical reasoning abilities of large reasoning models","author":"Liu","year":"2025"},{"key":"10.1016\/j.knosys.2026.116324_b19","series-title":"Proceedings of the International Conference on Learning Representations","article-title":"Reclor: a reading comprehension dataset requiring logical reasoning","author":"Yu","year":"2020"},{"key":"10.1016\/j.knosys.2026.116324_b20","first-page":"1","article-title":"From system 1 to system 2: A survey of reasoning large language models","author":"Zhang","year":"2025","journal-title":"IEEE Trans. Pattern Anal. Mach. Intell."},{"key":"10.1016\/j.knosys.2026.116324_b21","series-title":"Proceedings of the Conference on Empirical Methods in Natural Language Processing","first-page":"22017","article-title":"FOLIO: natural language reasoning with first-order logic","author":"Han","year":"2024"},{"key":"10.1016\/j.knosys.2026.116324_b22","series-title":"Proceedings of the Annual Meeting of the Association for Computational Linguistics","first-page":"13679","article-title":"LogicBench: Towards systematic evaluation of logical reasoning ability of large language models","author":"Parmar","year":"2024"},{"key":"10.1016\/j.knosys.2026.116324_b23","series-title":"Proceedings of the International Joint Conference on Artificial Intelligence","first-page":"3622","article-title":"Logiqa: a challenge dataset for machine reading comprehension with logical reasoning","author":"Liu","year":"2021"},{"key":"10.1016\/j.knosys.2026.116324_b24","series-title":"Benchmarking benchmark leakage in large language models","author":"Xu","year":"2024"},{"key":"10.1016\/j.knosys.2026.116324_b25","series-title":"Proceeding of the International Conference on Learning Representations","article-title":"Large language models meet symbolic provers for logical reasoning evaluation","author":"Qi","year":"2024"},{"key":"10.1016\/j.knosys.2026.116324_b26","doi-asserted-by":"crossref","unstructured":"N. Dziri, X. Lu, M. Sclar, X.L. Li, L. Jiang, B.Y. Lin, et al., Faith and fate: limits of transformers on compositionality, in: Proceedings of the International Conference on Neural Information Processing Systems, 2023, pp. 70293\u201370332.","DOI":"10.52202\/075280-3081"},{"key":"10.1016\/j.knosys.2026.116324_b27","series-title":"The International Conference on Learning Representations","article-title":"Language models are greedy reasoners: a systematic formal analysis of chain-of-thought","author":"Saparov","year":"2022"},{"key":"10.1016\/j.knosys.2026.116324_b28","doi-asserted-by":"crossref","first-page":"1843","DOI":"10.1145\/3689776","article-title":"Drowzee: metamorphic testing for fact-conflicting hallucination detection in large language models","volume":"8","author":"Li","year":"2024","journal-title":"Proc. ACM Program. Lang."},{"issue":"1","key":"10.1016\/j.knosys.2026.116324_b29","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1145\/3143561","article-title":"Metamorphic testing: a review of challenges and opportunities","volume":"51","author":"Chen","year":"2019","journal-title":"ACM Comput. Surv."},{"key":"10.1016\/j.knosys.2026.116324_b30","doi-asserted-by":"crossref","first-page":"1432","DOI":"10.1145\/3715784","article-title":"Detecting and reducing the factual hallucinations of large language models with metamorphic testing","volume":"2","author":"Wu","year":"2025","journal-title":"Proc. ACM Softw. Eng."},{"key":"10.1016\/j.knosys.2026.116324_b31","doi-asserted-by":"crossref","first-page":"425","DOI":"10.1145\/3715735","article-title":"Hallucination detection in large language models with metamorphic relations","volume":"2","author":"Yang","year":"2025","journal-title":"Proc. ACM Softw. Eng."},{"key":"10.1016\/j.knosys.2026.116324_b32","series-title":"Metamorphic testing: A new approach for generating next test cases","first-page":"11","author":"Chen","year":"1998"},{"key":"10.1016\/j.knosys.2026.116324_b33","series-title":"Preceeding of the International Conference on Automated Software Engineering","first-page":"104","article-title":"Testing your question answering software via asking recursively","author":"Chen","year":"2021"},{"key":"10.1016\/j.knosys.2026.116324_b34","series-title":"Metamorphic testing of large language models for natural language processing","author":"Cho","year":"2025"},{"key":"10.1016\/j.knosys.2026.116324_b35","series-title":"Proceedings of the 2025 Conference on Empirical Methods in Natural Language Processing","first-page":"12429","article-title":"Towards advanced mathematical reasoning for LLMs via first-order logic theorem proving","author":"Cao","year":"2025"},{"key":"10.1016\/j.knosys.2026.116324_b36","series-title":"Proceeding of the Conference on Empirical Methods in Natural Language Processing","article-title":"Logic-LM: Empowering large language models with symbolic solvers for faithful logical reasoning","author":"Pan","year":"2023"},{"key":"10.1016\/j.knosys.2026.116324_b37","series-title":"Proceedings of the 2025 Conference on Empirical Methods in Natural Language Processing","first-page":"16889","article-title":"Do large language models excel in complex logical reasoning with formal language?","author":"Jiang","year":"2025"},{"key":"10.1016\/j.knosys.2026.116324_b38","series-title":"Proceedings of the Conference on Empirical Methods in Natural Language Processing","first-page":"5153","article-title":"LINC: a neurosymbolic approach for logical reasoning by combining language models with first-order logic provers","author":"Olausson","year":"2023"},{"key":"10.1016\/j.knosys.2026.116324_b39","series-title":"Proceedings of the Conference on Empirical Methods in Natural Language Processing","first-page":"3738","article-title":"Diagnosing the first-order logical reasoning ability through logicnli","author":"Tian","year":"2021"},{"issue":"4","key":"10.1016\/j.knosys.2026.116324_b40","first-page":"637","article-title":"Errors of measurement in statistics","volume":"10","author":"Cochran","year":"1968","journal-title":"Technometrics"},{"key":"10.1016\/j.knosys.2026.116324_b41","series-title":"Think you have solved question answering? Try ARC, the AI2 reasoning challenge","author":"Clark","year":"2018"},{"key":"10.1016\/j.knosys.2026.116324_b42","series-title":"Proceedings of the Conference on Empirical Methods in Natural Language Processing","first-page":"20856","article-title":"Multi-LogiEval: towards evaluating multi-step logical reasoning ability of large language models","author":"Patel","year":"2024"},{"key":"10.1016\/j.knosys.2026.116324_b43","series-title":"Proceedings of the International Joint Conference on Artificial Intelligence","first-page":"3882","article-title":"Transformers as soft reasoners over language","author":"Clark","year":"2021"},{"key":"10.1016\/j.knosys.2026.116324_b44","series-title":"Findings of the Association for Computational Linguistics: ACL-IJCNLP 2021","first-page":"3621","article-title":"ProofWriter: generating implications, proofs, and abductive statements over natural language","author":"Tafjord","year":"2021"},{"key":"10.1016\/j.knosys.2026.116324_b45","series-title":"Proceedings of the 2021 Conference on Empirical Methods in Natural Language Processing","first-page":"7358","article-title":"Explaining answers with entailment trees","author":"Dalvi","year":"2021"},{"key":"10.1016\/j.knosys.2026.116324_b46","series-title":"Proceedings of the Conference on Empirical Methods in Natural Language Processing and the International Joint Conference on Natural Language Processing","first-page":"4505","article-title":"CLUTRR: a diagnostic benchmark for inductive reasoning from text","author":"Sinha","year":"2019"},{"key":"10.1016\/j.knosys.2026.116324_b47","doi-asserted-by":"crossref","unstructured":"J. Wei, X. Wang, D. Schuurmans, M. Bosma, B. Ichter, F. Xia, et al., Chain-of-thought prompting elicits reasoning in large language models, in: Proceedings of the International Conference on Neural Information Processing Systems, ISBN: 978-1-7138-7108-8, 2022, pp. 24824\u201324837.","DOI":"10.52202\/068431-1800"},{"key":"10.1016\/j.knosys.2026.116324_b48","series-title":"Findings of the Association for Computational Linguistics: ACL 2023","first-page":"13003","article-title":"Challenging BIG-bench tasks and whether chain-of-thought can solve them","author":"Suzgun","year":"2023"},{"key":"10.1016\/j.knosys.2026.116324_b49","unstructured":"X. Wang, J. Wei, D. Schuurmans, Q.V. Le, E.H. Chi, S. Narang, et al., Self-Consistency Improves Chain of Thought Reasoning in Language Models, in: The Eleventh International Conference on Learning Representations, 2022."},{"key":"10.1016\/j.knosys.2026.116324_b50","series-title":"Least-to-most prompting enables complex reasoning in large language models","author":"Zhou","year":"2023"},{"key":"10.1016\/j.knosys.2026.116324_b51","series-title":"First Conference on Language Modeling","article-title":"Beyond accuracy: Evaluating the reasoning behavior of large language models - A survey","author":"Mondorf","year":"2024"},{"issue":"9","key":"10.1016\/j.knosys.2026.116324_b52","doi-asserted-by":"crossref","first-page":"805","DOI":"10.1109\/TSE.2016.2532875","article-title":"A survey on metamorphic testing","volume":"42","author":"Segura","year":"2016","journal-title":"IEEE Trans. Softw. Eng."},{"key":"10.1016\/j.knosys.2026.116324_b53","series-title":"Proceedings of the International Conference on Software Engineering & Knowledge Engineering","first-page":"867","article-title":"Properties of machine learning applications for use in metamorphic testing","author":"Murphy","year":"2008"},{"issue":"4","key":"10.1016\/j.knosys.2026.116324_b54","doi-asserted-by":"crossref","first-page":"544","DOI":"10.1016\/j.jss.2010.11.920","article-title":"Testing and validating machine learning classifiers by metamorphic testing","volume":"84","author":"Xie","year":"2011","journal-title":"J. Syst. Softw."},{"key":"10.1016\/j.knosys.2026.116324_b55","series-title":"MetaRAG: Metamorphic testing for hallucination detection in RAG systems","author":"Sok","year":"2025"},{"key":"10.1016\/j.knosys.2026.116324_b56","series-title":"Hallucination detection for LLM-based text-to-SQL generation via two-stage metamorphic testing","author":"Yang","year":"2025"}],"container-title":["Knowledge-Based Systems"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/api.elsevier.com\/content\/article\/PII:S0950705126010506?httpAccept=text\/xml","content-type":"text\/xml","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/api.elsevier.com\/content\/article\/PII:S0950705126010506?httpAccept=text\/plain","content-type":"text\/plain","content-version":"vor","intended-application":"text-mining"}],"deposited":{"date-parts":[[2026,7,9]],"date-time":"2026-07-09T22:03:41Z","timestamp":1783634621000},"score":1,"resource":{"primary":{"URL":"https:\/\/linkinghub.elsevier.com\/retrieve\/pii\/S0950705126010506"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026,8]]},"references-count":56,"alternative-id":["S0950705126010506"],"URL":"https:\/\/doi.org\/10.1016\/j.knosys.2026.116324","relation":{},"ISSN":["0950-7051"],"issn-type":[{"value":"0950-7051","type":"print"}],"subject":[],"published":{"date-parts":[[2026,8]]},"assertion":[{"value":"Elsevier","name":"publisher","label":"This article is maintained by"},{"value":"LGMT: Logic-Grounded Metamorphic Testing for evaluating the reasoning reliability of LLMs","name":"articletitle","label":"Article Title"},{"value":"Knowledge-Based Systems","name":"journaltitle","label":"Journal Title"},{"value":"https:\/\/doi.org\/10.1016\/j.knosys.2026.116324","name":"articlelink","label":"CrossRef DOI link to publisher maintained version"},{"value":"article","name":"content_type","label":"Content Type"},{"value":"\u00a9 2026 Elsevier B.V. All rights are reserved, including those for text and data mining, AI training, and similar technologies.","name":"copyright","label":"Copyright"}],"article-number":"116324"}}