{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,4]],"date-time":"2026-07-04T13:22:00Z","timestamp":1783171320322,"version":"3.54.6"},"reference-count":46,"publisher":"Elsevier BV","license":[{"start":{"date-parts":[[2026,10,1]],"date-time":"2026-10-01T00:00:00Z","timestamp":1790812800000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.elsevier.com\/tdm\/userlicense\/1.0\/"},{"start":{"date-parts":[[2026,10,1]],"date-time":"2026-10-01T00:00:00Z","timestamp":1790812800000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.elsevier.com\/legal\/tdmrep-license"},{"start":{"date-parts":[[2026,6,5]],"date-time":"2026-06-05T00:00:00Z","timestamp":1780617600000},"content-version":"vor","delay-in-days":0,"URL":"http:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"funder":[{"DOI":"10.13039\/100010009","name":"Edge Hill University","doi-asserted-by":"publisher","id":[{"id":"10.13039\/100010009","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100003451","name":"University of the Basque Country","doi-asserted-by":"publisher","award":["US24\/10"],"award-info":[{"award-number":["US24\/10"]}],"id":[{"id":"10.13039\/501100003451","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["elsevier.com","sciencedirect.com"],"crossmark-restriction":true},"short-container-title":["Information and Software Technology"],"published-print":{"date-parts":[[2026,10]]},"DOI":"10.1016\/j.infsof.2026.108215","type":"journal-article","created":{"date-parts":[[2026,6,5]],"date-time":"2026-06-05T16:20:02Z","timestamp":1780676402000},"page":"108215","update-policy":"https:\/\/doi.org\/10.1016\/elsevier_cm_policy","source":"Crossref","is-referenced-by-count":0,"special_numbering":"C","title":["An empirical investigation of Large Language Models for automated software compliance testing: Evidence from web accessibility"],"prefix":"10.1016","volume":"198","author":[{"ORCID":"https:\/\/orcid.org\/0000-0001-7730-0472","authenticated-orcid":false,"given":"Juan-Miguel","family":"L\u00f3pez-Gil","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-7935-3612","authenticated-orcid":false,"given":"Juanan","family":"Pereira","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"78","reference":[{"key":"10.1016\/j.infsof.2026.108215_b1","series-title":"Proceedings of the 10th International Cross-Disciplinary Conference on Web Accessibility","article-title":"Benchmarking web accessibility evaluation tools: Measuring the harm of sole reliance on automated tests","author":"Vigo","year":"2013"},{"issue":"2","key":"10.1016\/j.infsof.2026.108215_b2","doi-asserted-by":"crossref","first-page":"8:1","DOI":"10.1145\/2141943.2141946","article-title":"Is accessibility conformance an elusive property? A study of validity and reliability of WCAG 2.0","volume":"4","author":"Brajnik","year":"2012","journal-title":"ACM Trans. Access. Comput."},{"key":"10.1016\/j.infsof.2026.108215_b3","article-title":"A large-scale web accessibility analysis considering technology adoption","author":"Martins","year":"2023","journal-title":"Univers. Access Inf. Soc."},{"key":"10.1016\/j.infsof.2026.108215_b4","unstructured":"A. Kirkpatrick, J. O\u2019Connor, A. Campbell, M. Cooper, Web Content Accessibility Guidelines (WCAG) 2.1, Technical Report, 2023, World Wide Web Consortium."},{"issue":"3","key":"10.1016\/j.infsof.2026.108215_b5","doi-asserted-by":"crossref","first-page":"252","DOI":"10.1007\/s10209-004-0105-y","article-title":"Comparing accessibility evaluation tools: a method for tool effectiveness","volume":"3","author":"Brajnik","year":"2004","journal-title":"Univers. Access Inf. Soc."},{"key":"10.1016\/j.infsof.2026.108215_b6","series-title":"Web Accessibility: Web Standards and Regulatory Compliance","author":"Thatcher","year":"2006"},{"key":"10.1016\/j.infsof.2026.108215_b7","series-title":"Limitations of automated testing","author":"UK Intelligence Community Design System","year":"2022"},{"key":"10.1016\/j.infsof.2026.108215_b8","unstructured":"Deque Systems, The Automated Accessibility Coverage Report, Technical Report, 2021, Deque Systems."},{"key":"10.1016\/j.infsof.2026.108215_b9","series-title":"HCI International 2022 \u2013 Late Breaking Papers: HCI for Health, Well-Being, Universal Access and Healthy Aging","first-page":"474","article-title":"Comparative analysis of accessibility testing tools and their limitations in RIAs","volume":"vol. 13521","author":"Okafor","year":"2022"},{"issue":"1","key":"10.1016\/j.infsof.2026.108215_b10","doi-asserted-by":"crossref","first-page":"3:1","DOI":"10.1145\/3556979","article-title":"The transparency of automatic web accessibility evaluation tools: Design criteria, state of the art, and user perception","volume":"16","author":"Manca","year":"2023","journal-title":"ACM Trans. Access. Comput."},{"issue":"4","key":"10.1016\/j.infsof.2026.108215_b11","doi-asserted-by":"crossref","DOI":"10.1145\/3368620","article-title":"Accessibility information needs in the enterprise","volume":"12","author":"Snider","year":"2020","journal-title":"ACM Trans. Access. Comput."},{"key":"10.1016\/j.infsof.2026.108215_b12","series-title":"Findings of the Association for Computational Linguistics: EMNLP 2023","first-page":"2803","article-title":"Understanding HTML with large language models","author":"Gur","year":"2023"},{"key":"10.1016\/j.infsof.2026.108215_b13","series-title":"Proceedings of the 33rd International Symposium on Software Testing and Analysis (ISSTA 2024) and 38th European Conference on Object-Oriented Programming (ECOOP 2024)","article-title":"Software quality assurance in the era of large language models","author":"Zhang","year":"2024"},{"issue":"1","key":"10.1016\/j.infsof.2026.108215_b14","doi-asserted-by":"crossref","first-page":"837","DOI":"10.1007\/s10209-024-01108-z","article-title":"Turning manual web accessibility success criteria into automatic: an LLM-based approach","volume":"24","author":"L\u00f3pez-Gil","year":"2025","journal-title":"Univers. Access Inf. Soc."},{"issue":"9","key":"10.1016\/j.infsof.2026.108215_b15","first-page":"220:1","article-title":"Large language models for software engineering: A systematic literature review","volume":"33","author":"Hou","year":"2024","journal-title":"ACM Trans. Softw. Eng. Methodol."},{"key":"10.1016\/j.infsof.2026.108215_b16","series-title":"Web Accessibility: A Foundation for Research, Human-Computer Interaction Series","first-page":"79","article-title":"Web accessibility evaluation","author":"Abou-Zahra","year":"2008"},{"issue":"1","key":"10.1016\/j.infsof.2026.108215_b17","doi-asserted-by":"crossref","first-page":"40","DOI":"10.3390\/info11010040","article-title":"Comparing web accessibility evaluation tools and evaluating the accessibility of webpages: Proposed frameworks","volume":"11","author":"Alsaeedi","year":"2020","journal-title":"Information"},{"issue":"2","key":"10.1016\/j.infsof.2026.108215_b18","doi-asserted-by":"crossref","first-page":"137","DOI":"10.1016\/j.intcom.2011.01.001","article-title":"Automatic web accessibility metrics: Where we are and where we can go","volume":"23","author":"Vigo","year":"2011","journal-title":"Interact. Comput."},{"key":"10.1016\/j.infsof.2026.108215_b19","doi-asserted-by":"crossref","DOI":"10.1016\/j.infsof.2024.107468","article-title":"Effective test generation using pre-trained large language models and mutation testing","volume":"171","author":"Dakhel","year":"2024","journal-title":"Inf. Softw. Technol."},{"key":"10.1016\/j.infsof.2026.108215_b20","doi-asserted-by":"crossref","DOI":"10.1016\/j.jss.2021.111031","article-title":"A software engineering perspective on engineering machine learning systems: State of the art and challenges","volume":"180","author":"Giray","year":"2021","journal-title":"J. Syst. Softw."},{"issue":"6","key":"10.1016\/j.infsof.2026.108215_b21","doi-asserted-by":"crossref","DOI":"10.1016\/j.patter.2025.101260","article-title":"Unleashing the potential of prompt engineering for large language models","volume":"6","author":"Chen","year":"2025","journal-title":"Patterns"},{"key":"10.1016\/j.infsof.2026.108215_b22","doi-asserted-by":"crossref","DOI":"10.1016\/j.jss.2023.111934","article-title":"A survey on machine learning techniques applied to source code","volume":"209","author":"Sharma","year":"2024","journal-title":"J. Syst. Softw."},{"key":"10.1016\/j.infsof.2026.108215_b23","series-title":"Fifth IEEE International Workshop on Web Site Evolution, 2003. Theme: Architecture. Proceedings","first-page":"66","article-title":"Benchmark for testing the evaluation tools for web pages accessibility","author":"Kirchner","year":"2003"},{"key":"10.1016\/j.infsof.2026.108215_b24","series-title":"Universal Access in Human-Computer Interaction. Applications and Services","first-page":"645","article-title":"Is it possible to predict the manual web accessibility result using the automatic result?","volume":"vol. 5616","author":"Casado Mart\u00ednez","year":"2009"},{"key":"10.1016\/j.infsof.2026.108215_b25","series-title":"Computers Helping People with Special Needs","first-page":"329","article-title":"A multi-site collaborative sampling for web accessibility evaluation","author":"Yu","year":"2020"},{"key":"10.1016\/j.infsof.2026.108215_b26","article-title":"Large-scale study of web accessibility metrics","author":"Martins","year":"2022","journal-title":"Univers. Access Inf. Soc."},{"key":"10.1016\/j.infsof.2026.108215_b27","series-title":"Proceedings of the 16th International Conference on PErvasive Technologies Related To Assistive Environments","first-page":"707","article-title":"Fostering websites accessibility: A case study on the use of the large language models ChatGPT for automatic remediation","author":"Othman","year":"2023"},{"key":"10.1016\/j.infsof.2026.108215_b28","series-title":"Proceedings of the 2024 CHI Conference on Human Factors in Computing Systems","first-page":"1","article-title":"AXNav: Replaying accessibility tests from natural language","author":"Taeb","year":"2024"},{"key":"10.1016\/j.infsof.2026.108215_b29","series-title":"ACCESS: Prompt engineering for automated web accessibility violation corrections","author":"Huang","year":"2024"},{"key":"10.1016\/j.infsof.2026.108215_b30","series-title":"From code to compliance: Assessing ChatGPT\u2019s utility in designing an accessible webpage - a case study","author":"Ahmed","year":"2025"},{"key":"10.1016\/j.infsof.2026.108215_b31","series-title":"2024 IEEE 21st Consumer Communications & Networking Conference (CCNC)","article-title":"On the interaction with large language models for web accessibility: Implications and challenges","author":"Delnevo","year":"2024"},{"key":"10.1016\/j.infsof.2026.108215_b32","doi-asserted-by":"crossref","DOI":"10.1016\/j.infsof.2025.107821","article-title":"Towards accessible website design through artificial intelligence: A systematic literature review","volume":"186","author":"Vera-Amaro","year":"2025","journal-title":"Inf. Softw. Technol."},{"key":"10.1016\/j.infsof.2026.108215_b33","series-title":"Non-determinism of \u201cdeterministic\u201d LLM settings","author":"Atil","year":"2025"},{"key":"10.1016\/j.infsof.2026.108215_b34","doi-asserted-by":"crossref","DOI":"10.1016\/j.csi.2024.103942","article-title":"Evaluating large language models for software testing","volume":"93","author":"Li","year":"2025","journal-title":"Comput. Stand. Interfaces"},{"issue":"3\u20134","key":"10.1016\/j.infsof.2026.108215_b35","doi-asserted-by":"crossref","first-page":"591","DOI":"10.1093\/biomet\/52.3-4.591","article-title":"An analysis of variance test for normality (complete samples)","volume":"52","author":"Shapiro","year":"1965","journal-title":"Biometrika"},{"issue":"2","key":"10.1016\/j.infsof.2026.108215_b36","first-page":"65","article-title":"A simple sequentially rejective multiple test procedure","volume":"6","author":"Holm","year":"1979","journal-title":"Scand. J. Stat."},{"key":"10.1016\/j.infsof.2026.108215_b37","doi-asserted-by":"crossref","DOI":"10.1016\/j.jss.2022.111518","article-title":"Mapping the structure and evolution of software testing research over the past three decades","volume":"195","author":"Salahirad","year":"2023","journal-title":"J. Syst. Softw."},{"key":"10.1016\/j.infsof.2026.108215_b38","series-title":"The Twelfth International Conference on Learning Representations","article-title":"Time travel in LLMs: Tracing data contamination in large language models","author":"Golchin","year":"2024"},{"key":"10.1016\/j.infsof.2026.108215_b39","series-title":"30th USENIX Security Symposium (USENIX Security 21)","first-page":"2633","article-title":"Extracting training data from large language models","author":"Carlini","year":"2021"},{"key":"10.1016\/j.infsof.2026.108215_b40","doi-asserted-by":"crossref","first-page":"809","DOI":"10.1162\/TACL.a.20","article-title":"Data contamination quiz: A tool to detect and estimate contamination in large language models","volume":"13","author":"Golchin","year":"2025","journal-title":"Trans. Assoc. Comput. Linguist."},{"key":"10.1016\/j.infsof.2026.108215_b41","series-title":"WCAG 3 introduction","author":"W3C WAI","year":"2025"},{"key":"10.1016\/j.infsof.2026.108215_b42","doi-asserted-by":"crossref","DOI":"10.1016\/j.jss.2022.111539","article-title":"Automating feature model maintainability evaluation using machine learning techniques","volume":"195","author":"Silva","year":"2023","journal-title":"J. Syst. Softw."},{"key":"10.1016\/j.infsof.2026.108215_b43","first-page":"24824","article-title":"Chain-of-thought prompting elicits reasoning in large language models","volume":"vol. 35","author":"Wei","year":"2022"},{"key":"10.1016\/j.infsof.2026.108215_b44","series-title":"The Eleventh International Conference on Learning Representations","article-title":"Self-consistency improves chain of thought reasoning in language models","author":"Wang","year":"2023"},{"key":"10.1016\/j.infsof.2026.108215_b45","first-page":"9459","article-title":"Retrieval-augmented generation for knowledge-intensive NLP tasks","volume":"vol. 33","author":"Lewis","year":"2020"},{"key":"10.1016\/j.infsof.2026.108215_b46","doi-asserted-by":"crossref","DOI":"10.1016\/j.jss.2023.111907","article-title":"Software engineering practices for machine learning \u2014 adoption, effects, and team assessment","volume":"209","author":"Serban","year":"2024","journal-title":"J. Syst. Softw."}],"container-title":["Information and Software Technology"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/api.elsevier.com\/content\/article\/PII:S0950584926002041?httpAccept=text\/xml","content-type":"text\/xml","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/api.elsevier.com\/content\/article\/PII:S0950584926002041?httpAccept=text\/plain","content-type":"text\/plain","content-version":"vor","intended-application":"text-mining"}],"deposited":{"date-parts":[[2026,7,4]],"date-time":"2026-07-04T13:01:53Z","timestamp":1783170113000},"score":1,"resource":{"primary":{"URL":"https:\/\/linkinghub.elsevier.com\/retrieve\/pii\/S0950584926002041"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026,10]]},"references-count":46,"alternative-id":["S0950584926002041"],"URL":"https:\/\/doi.org\/10.1016\/j.infsof.2026.108215","relation":{},"ISSN":["0950-5849"],"issn-type":[{"value":"0950-5849","type":"print"}],"subject":[],"published":{"date-parts":[[2026,10]]},"assertion":[{"value":"Elsevier","name":"publisher","label":"This article is maintained by"},{"value":"An empirical investigation of Large Language Models for automated software compliance testing: Evidence from web accessibility","name":"articletitle","label":"Article Title"},{"value":"Information and Software Technology","name":"journaltitle","label":"Journal Title"},{"value":"https:\/\/doi.org\/10.1016\/j.infsof.2026.108215","name":"articlelink","label":"CrossRef DOI link to publisher maintained version"},{"value":"article","name":"content_type","label":"Content Type"},{"value":"\u00a9 2026 The Authors. Published by Elsevier B.V.","name":"copyright","label":"Copyright"}],"article-number":"108215"}}