{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,1,23]],"date-time":"2026-01-23T16:05:35Z","timestamp":1769184335011,"version":"3.49.0"},"reference-count":55,"publisher":"Institute of Electrical and Electronics Engineers (IEEE)","license":[{"start":{"date-parts":[[2026,1,1]],"date-time":"2026-01-01T00:00:00Z","timestamp":1767225600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/legalcode"}],"funder":[{"DOI":"10.13039\/100031478","name":"NextGenerationEU","doi-asserted-by":"publisher","award":["C039-21OT"],"award-info":[{"award-number":["C039-21OT"]}],"id":[{"id":"10.13039\/100031478","id-type":"DOI","asserted-by":"publisher"}]},{"name":"Ministerio para la transformaci?n digital y de la funci?n p?blica","award":["C039-21OT"],"award-info":[{"award-number":["C039-21OT"]}]},{"DOI":"10.13039\/501100005367","name":"Universidad Nacional de Educaci?n a Distancia","doi-asserted-by":"publisher","award":["C039-21OT"],"award-info":[{"award-number":["C039-21OT"]}],"id":[{"id":"10.13039\/501100005367","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["IEEE Access"],"published-print":{"date-parts":[[2026]]},"DOI":"10.1109\/access.2026.3651579","type":"journal-article","created":{"date-parts":[[2026,1,6]],"date-time":"2026-01-06T18:36:49Z","timestamp":1767724609000},"page":"9384-9393","source":"Crossref","is-referenced-by-count":0,"title":["On the Limits of LLM Reasoning: Evidence From Contamination, Translation, and Answer Modification in Multiple-Choice Benchmarks"],"prefix":"10.1109","volume":"14","author":[{"ORCID":"https:\/\/orcid.org\/0000-0001-8665-3018","authenticated-orcid":false,"given":"Eva S\u00e1nchez","family":"Salido","sequence":"first","affiliation":[{"name":"Research Group in NLP & IR, National Distance Education University, Madrid, Spain"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-5341-9337","authenticated-orcid":false,"given":"Julio","family":"Gonzalo","sequence":"additional","affiliation":[{"name":"Research Group in NLP & IR, National Distance Education University, Madrid, Spain"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-2455-6655","authenticated-orcid":false,"given":"Guillermo","family":"Marco","sequence":"additional","affiliation":[{"name":"Research Group in NLP & IR, National Distance Education University, Madrid, Spain"}]}],"member":"263","reference":[{"key":"ref1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2021.naacl-main.324"},{"key":"ref2","doi-asserted-by":"publisher","DOI":"10.1038\/s42256-020-00257-z"},{"key":"ref3","doi-asserted-by":"publisher","DOI":"10.1073\/pnas.2322420121"},{"key":"ref4","first-page":"47159","article-title":"The generative AI paradox: \u2018What it can create, it may not understand","volume-title":"Proc. 12th Int. Conf. Learn. Represent.","volume":"2024","author":"West"},{"key":"ref5","article-title":"GPT-4O system card","author":"Hurst","year":"2024","journal-title":"arXiv:2410.21276"},{"key":"ref6","article-title":"OpenAI O1 system card","author":"Jaech","year":"2024","journal-title":"arXiv:2412.16720"},{"key":"ref7","article-title":"GPQA: A graduate-level Google-proof Q&A benchmark","volume-title":"Proc. 1st Conf. Lang. Model.","author":"Rein"},{"key":"ref8","article-title":"Functional benchmarks for robust evaluation of reasoning performance, and the reasoning gap","author":"Srivastava","year":"2024","journal-title":"arXiv:2402.19450"},{"key":"ref9","first-page":"94743","article-title":"GSM-symbolic: Understanding the limitations of mathematical reasoning in large language models","volume-title":"Proc. 13th Int. Conf. Learn. Represent.","volume":"2025","author":"Mirzadeh"},{"key":"ref10","article-title":"MATH-perturb: Benchmarking LLMs\u2019 math reasoning abilities against hard perturbations","volume-title":"Proc. 42nd Int. Conf. Mach. Learn.","author":"Huang"},{"key":"ref11","article-title":"Adversarial GLUE: A multi-task benchmark for robustness evaluation of language models","volume-title":"Proc. Neural Inf. Process. Syst. Track Datasets Benchmarks","volume":"1","author":"Wang"},{"key":"ref12","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2023.findings-emnlp.722"},{"key":"ref13","doi-asserted-by":"publisher","DOI":"10.1093\/9780198945215.003.0140"},{"key":"ref14","doi-asserted-by":"publisher","DOI":"10.1038\/s42256-023-00729-y"},{"key":"ref15","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2023.emnlp-main.276"},{"key":"ref16","doi-asserted-by":"publisher","DOI":"10.1145\/3664194"},{"key":"ref17","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2023.findings-acl.67"},{"key":"ref18","article-title":"Measuring massive multitask language understanding","volume-title":"Proc. Int. Conf. Learn. Represent.","author":"Hendrycks"},{"key":"ref19","first-page":"6184","article-title":"Bilingual evaluation of language models on general knowledge in university entrance exams with minimal contamination","volume-title":"Proc. 31st Int. Conf. Comput. Linguistics","author":"Salido"},{"key":"ref20","volume-title":"Multilingual MMLU Benchmark Leaderboard","author":"Zhou","year":"2024"},{"key":"ref21","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2025.acl-short.94"},{"key":"ref22","article-title":"GPT-4 technical report","volume-title":"arXiv:2303.08774","author":"Achiam","year":"2023"},{"key":"ref23","first-page":"1877","article-title":"Language models are few-shot learners","volume-title":"Proc. 34th Int. Conf. Neural Inf. Process. Syst.","volume":"33","author":"Brown"},{"key":"ref24","volume-title":"OpenAI O3-Mini","year":"2025"},{"key":"ref25","volume-title":"The Claude 3 Model Family: Opus, Sonnet, Haiku","year":"2024"},{"key":"ref26","article-title":"LLaMA: Open and efficient foundation language models","author":"Touvron","year":"2023","journal-title":"arXiv:2302.13971"},{"key":"ref27","volume-title":"Introducing Meta Llama 3: The Most Capable Openly Available LLM to Date","year":"2024"},{"key":"ref28","article-title":"Gemma: Open models based on Gemini research and technology","author":"Team","year":"2024","journal-title":"arXiv:2403.08295"},{"key":"ref29","article-title":"Gemma 2: Improving open language models at a practical size","author":"Team","year":"2024","journal-title":"arXiv:2408.00118"},{"key":"ref30","article-title":"Mistral 7B","author":"Jiang","year":"2023","journal-title":"arXiv:2310.06825"},{"key":"ref31","article-title":"Mixtral of experts","author":"Jiang","year":"2024","journal-title":"arXiv:2401.04088"},{"key":"ref32","article-title":"DeepSeek-R1: Incentivizing reasoning capability in LLMs via reinforcement learning","author":"Guo","year":"2025","journal-title":"arXiv:2501.12948"},{"key":"ref33","doi-asserted-by":"publisher","DOI":"10.11613\/BM.2012.031"},{"key":"ref34","article-title":"Scaling laws for neural language models","author":"Kaplan","year":"2020","journal-title":"arXiv:2001.08361"},{"key":"ref35","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2024.findings-acl.716"},{"key":"ref36","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2022.findings-emnlp.59"},{"key":"ref37","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2023.findings-acl.806"},{"key":"ref38","first-page":"2873","article-title":"Generalization without systematicity: On the compositional skills of sequence-to-sequence recurrent networks","volume-title":"Proc. 35th Int. Conf. Mach. Learn.","volume":"80","author":"Lake"},{"key":"ref39","doi-asserted-by":"publisher","DOI":"10.1098\/rstb.2019.0307"},{"issue":"1","key":"ref40","first-page":"48","article-title":"On the robustness of ChatGPT: An adversarial and out-of-distribution perspective","volume":"47","author":"Wang","year":"2023","journal-title":"IEEE Data Eng. Bull."},{"key":"ref41","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2022.naacl-main.339"},{"key":"ref42","article-title":"LLMs still can\u2019t plan; can LRMs? A preliminary evaluation of OpenAI\u2019s o1 on planBench","author":"Valmeekam","year":"2024","journal-title":"arXiv:2409.13373"},{"key":"ref43","first-page":"5211","article-title":"LLMs meet Bloom\u2019s taxonomy: A cognitive view on large language model evaluations","volume-title":"Proc. 31st Int. Conf. Comput. Linguistics","author":"Huber"},{"key":"ref44","article-title":"Alice in wonderland: Simple tasks reveal severe generalization and basic reasoning deficits in state-of-the-art large language models","volume-title":"Proc. Workshop Sci. Methods Understand. Deep Learn. (NeurIPS)","author":"Nezhurina"},{"key":"ref45","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2024.findings-emnlp.212"},{"key":"ref46","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2025.acl-long.169"},{"key":"ref47","article-title":"Selection-inference: Exploiting large language models for interpretable logical reasoning","volume-title":"Proc. 11th Int. Conf. Learn. Represent.","author":"Creswell"},{"key":"ref48","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2025.acl-long.1163"},{"key":"ref49","doi-asserted-by":"publisher","DOI":"10.1109\/FLLM63129.2024.10852488"},{"key":"ref50","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2025.findings-acl.1031"},{"key":"ref51","first-page":"5852","article-title":"LLMs may perform MCQA by selecting the least incorrect option","volume-title":"Proc. 31st Int. Conf. Comput. Linguistics","author":"Wang"},{"key":"ref52","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2025.acl-long.75"},{"key":"ref53","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2024.acl-long.786"},{"key":"ref54","doi-asserted-by":"publisher","DOI":"10.1007\/s10489-025-06765-y"},{"key":"ref55","doi-asserted-by":"publisher","DOI":"10.1093\/pnasnexus\/pgae233"}],"container-title":["IEEE Access"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx8\/6287639\/11323511\/11333297.pdf?arnumber=11333297","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,1,22]],"date-time":"2026-01-22T21:03:13Z","timestamp":1769115793000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/11333297\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026]]},"references-count":55,"URL":"https:\/\/doi.org\/10.1109\/access.2026.3651579","relation":{},"ISSN":["2169-3536"],"issn-type":[{"value":"2169-3536","type":"electronic"}],"subject":[],"published":{"date-parts":[[2026]]}}}