{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,2]],"date-time":"2026-07-02T03:45:32Z","timestamp":1782963932747,"version":"3.54.5"},"reference-count":25,"publisher":"Elsevier BV","license":[{"start":{"date-parts":[[2026,9,1]],"date-time":"2026-09-01T00:00:00Z","timestamp":1788220800000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.elsevier.com\/tdm\/userlicense\/1.0\/"},{"start":{"date-parts":[[2026,9,1]],"date-time":"2026-09-01T00:00:00Z","timestamp":1788220800000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.elsevier.com\/legal\/tdmrep-license"},{"start":{"date-parts":[[2026,5,26]],"date-time":"2026-05-26T00:00:00Z","timestamp":1779753600000},"content-version":"vor","delay-in-days":0,"URL":"http:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"funder":[{"DOI":"10.13039\/501100008530","name":"European Regional Development Fund","doi-asserted-by":"publisher","id":[{"id":"10.13039\/501100008530","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100001871","name":"Fundacao para a Ciencia e a Tecnologia","doi-asserted-by":"publisher","id":[{"id":"10.13039\/501100001871","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["elsevier.com","sciencedirect.com"],"crossmark-restriction":true},"short-container-title":["Intelligent Systems with Applications"],"published-print":{"date-parts":[[2026,9]]},"DOI":"10.1016\/j.iswa.2026.200681","type":"journal-article","created":{"date-parts":[[2026,5,27]],"date-time":"2026-05-27T23:57:37Z","timestamp":1779926257000},"page":"200681","update-policy":"https:\/\/doi.org\/10.1016\/elsevier_cm_policy","source":"Crossref","is-referenced-by-count":0,"special_numbering":"C","title":["RAG-Augmented LLMs for Penetration Testing: A benchmarking study of open-source LLM models"],"prefix":"10.1016","volume":"31","author":[{"given":"Oumaima Ben","family":"Fadhel","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0002-1365-0208","authenticated-orcid":false,"given":"Rui","family":"Fernandes","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Oscar","family":"Ribeiro","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Monia","family":"Najjar","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Nuno","family":"Lopes","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"78","reference":[{"key":"10.1016\/j.iswa.2026.200681_bib0001","unstructured":"Achiam, J., Adler, S., Agarwal, S., Ahmad, L., Akkaya, I., Aleman, F.L., Almeida, D., Altenschmidt, J., Altman, S., Anadkat, S., et al., 2023. Gpt-4 technical report. arXiv preprint arXiv:2303.08774."},{"key":"10.1016\/j.iswa.2026.200681_bib0002","doi-asserted-by":"crossref","first-page":"3781","DOI":"10.1016\/j.procs.2024.09.178","article-title":"A survey on rag with llms","volume":"246","author":"Arslan","year":"2024","journal-title":"Procedia Computer Science"},{"issue":"2","key":"10.1016\/j.iswa.2026.200681_bib0003","article-title":"A multitask, multilingual, multimodal evaluation of chatgpt on reasoning, hallucination, and interactivity. Centre for Artificial Intelligence Re- search (CAiRE) The Hong Kong University of","volume":"80","author":"Bang","year":"2023","journal-title":"Science and Technology"},{"key":"10.1016\/j.iswa.2026.200681_bib0004","series-title":"33rd USENIX Security Symposium (USENIX Security 24), USENIX As- sociation, Philadelphia, PA","first-page":"847","article-title":"PentestGPT: Evaluating and harnessing large language models for automated penetration testing","author":"Deng","year":"2024"},{"key":"10.1016\/j.iswa.2026.200681_bib0005","series-title":"Proceedings of the 2019 conference of the North American chapter of the association for computational linguistics: Human language tech- nologies","first-page":"4171","article-title":"Bert: Pre- training of deep bidirectional transformers for language understanding","volume":"1","author":"Devlin","year":"2019"},{"key":"10.1016\/j.iswa.2026.200681_bib0006","series-title":"Authorea Preprints","article-title":"Prompt engineering for chatgpt: A quick guide to tech- niques, tips, and best practices","author":"Ekin","year":"2023"},{"key":"10.1016\/j.iswa.2026.200681_bib0007","doi-asserted-by":"crossref","first-page":"852","DOI":"10.1016\/j.procs.2026.03.059","article-title":"Design and implementation of a rag-enhanced llm chatbot for penetration testing tasks","volume":"278","author":"Fadhel","year":"2026","journal-title":"Procedia Computer Science"},{"key":"10.1016\/j.iswa.2026.200681_bib0008","author":"Fernandes"},{"key":"10.1016\/j.iswa.2026.200681_bib0009","doi-asserted-by":"crossref","unstructured":"Fernandes, R., Lopes, N., Gonc\u00b8alves, J., Cosgrove, J., 2025b. Au- tonomous pentesting using reinforcement learning: A systematic lit- erature review. URL: https:\/\/ssrn.com\/abstract=5208526, doi:10.2139\/ssrn.5208526. available at SSRN.","DOI":"10.2139\/ssrn.5208526"},{"key":"10.1016\/j.iswa.2026.200681_bib0010","series-title":"Advances in computational intelligence","first-page":"55","article-title":"Penetra- tion testing with ai: Case studies on llm and rl-based attack agents","author":"Fernandes","year":"2026"},{"key":"10.1016\/j.iswa.2026.200681_bib0011","unstructured":"Genesis, J., Keane, F., 2025. Integrating knowledge retrieval with generation: A comprehensive survey of rag models in nlp. Preprints URL: https:\/\/doi.org\/10.20944\/preprints202504.0351. v1, 10.20944\/preprints202504.0351.v1."},{"key":"10.1016\/j.iswa.2026.200681_bib0012","author":"Henke"},{"key":"10.1016\/j.iswa.2026.200681_bib0013","unstructured":"Jiang, A.Q., Sablayrolles, A., Roux, A., Mensch, A., Savary, B., Bam-ford, C., Chaplot, D.S., Casas, D.d.l., Hanna, E.B., Bressand, F., et al., 2024. Mixtral of experts. arXiv preprint arXiv:2401.04088."},{"key":"10.1016\/j.iswa.2026.200681_bib0014","first-page":"507","article-title":"Web application development with streamlit","volume":"498","author":"Khorasani","year":"2022","journal-title":"Software Development"},{"key":"10.1016\/j.iswa.2026.200681_bib0015","author":"Ma"},{"key":"10.1016\/j.iswa.2026.200681_bib0017","doi-asserted-by":"crossref","first-page":"27730","DOI":"10.52202\/068431-2011","article-title":"Training language models to follow instructions with human feedback","volume":"35","author":"Ouyang","year":"2022","journal-title":"Advances in neural information processing systems"},{"key":"10.1016\/j.iswa.2026.200681_bib0018","unstructured":"Pecan, The role of llms in ai innovation. URL: https:\/\/www.pecan.ai\/blog\/role-of-llm-ai-innovation\/ (Accessed on 02-5-2025)."},{"key":"10.1016\/j.iswa.2026.200681_bib0019","first-page":"1674","article-title":"Text recognition and detec- tion from images using pytesseract","volume":"13","author":"Saoji","year":"2021","journal-title":"Journal of Interdisciplinary Cycle Research"},{"key":"10.1016\/j.iswa.2026.200681_bib0020","first-page":"2","article-title":"Technical guide to information security testing and assessment","volume":"800","author":"Scarfone","year":"2008","journal-title":"NIST Special Publication"},{"key":"10.1016\/j.iswa.2026.200681_bib0021","series-title":"Proceedings of the 20th ACM Asia Conference on Computer and Communications Security","first-page":"375","article-title":"Pentestagent: Incorporating llm agents to automated penetration testing","author":"Shen","year":"2025"},{"key":"10.1016\/j.iswa.2026.200681_bib0022","unstructured":"Team, G., Anil, R., Borgeaud, S., Alayrac, J.B., Yu, J., Soricut, R., Schalkwyk, J., Dai, A.M., Hauth, A., Millican, K., et al., 2023. Gem- ini: A family of highly capable multimodal models. arXiv preprint arXiv:2312.11805."},{"key":"10.1016\/j.iswa.2026.200681_bib0023","series-title":"2024 IEEE International Conference on Cyber Security and Resilience (CSR)","first-page":"296","article-title":"Cybermetric: A benchmark dataset based on retrieval-augmented gen- eration for evaluating llms in cybersecurity knowledge","author":"Tihanyi","year":"2024"},{"key":"10.1016\/j.iswa.2026.200681_bib0024","unstructured":"Touvron, H., Lavril, T., Izacard, G., Martinet, X., Lachaux, M.A., Lacroix, T., Rozi`ere, B., Goyal, N., Hambro, E., Azhar, F., et al., 2023. Llama: Open and efficient foundation language models. arXiv preprint arXiv:2302.13971."},{"key":"10.1016\/j.iswa.2026.200681_bib0025","author":"Zhang"},{"key":"10.1016\/j.iswa.2026.200681_bib0026","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1186\/s42400-025-00361-w","article-title":"When llms meet cybersecurity: A systematic literature review","volume":"8","author":"Zhang","year":"2025","journal-title":"Cybersecurity"}],"container-title":["Intelligent Systems with Applications"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/api.elsevier.com\/content\/article\/PII:S2667305326000566?httpAccept=text\/xml","content-type":"text\/xml","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/api.elsevier.com\/content\/article\/PII:S2667305326000566?httpAccept=text\/plain","content-type":"text\/plain","content-version":"vor","intended-application":"text-mining"}],"deposited":{"date-parts":[[2026,7,2]],"date-time":"2026-07-02T02:52:50Z","timestamp":1782960770000},"score":1,"resource":{"primary":{"URL":"https:\/\/linkinghub.elsevier.com\/retrieve\/pii\/S2667305326000566"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026,9]]},"references-count":25,"alternative-id":["S2667305326000566"],"URL":"https:\/\/doi.org\/10.1016\/j.iswa.2026.200681","relation":{},"ISSN":["2667-3053"],"issn-type":[{"value":"2667-3053","type":"print"}],"subject":[],"published":{"date-parts":[[2026,9]]},"assertion":[{"value":"Elsevier","name":"publisher","label":"This article is maintained by"},{"value":"RAG-Augmented LLMs for Penetration Testing: A benchmarking study of open-source LLM models","name":"articletitle","label":"Article Title"},{"value":"Intelligent Systems with Applications","name":"journaltitle","label":"Journal Title"},{"value":"https:\/\/doi.org\/10.1016\/j.iswa.2026.200681","name":"articlelink","label":"CrossRef DOI link to publisher maintained version"},{"value":"article","name":"content_type","label":"Content Type"},{"value":"\u00a9 2026 The Authors. Published by Elsevier Ltd.","name":"copyright","label":"Copyright"}],"article-number":"200681"}}