{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,3,27]],"date-time":"2025-03-27T12:08:46Z","timestamp":1743077326103,"version":"3.40.3"},"publisher-location":"Cham","reference-count":30,"publisher":"Springer Nature Switzerland","isbn-type":[{"type":"print","value":"9783031790065"},{"type":"electronic","value":"9783031790072"}],"license":[{"start":{"date-parts":[[2025,1,1]],"date-time":"2025-01-01T00:00:00Z","timestamp":1735689600000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2025,1,1]],"date-time":"2025-01-01T00:00:00Z","timestamp":1735689600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2025]]},"DOI":"10.1007\/978-3-031-79007-2_12","type":"book-chapter","created":{"date-parts":[[2025,1,28]],"date-time":"2025-01-28T20:02:03Z","timestamp":1738094523000},"page":"219-238","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":0,"title":["Evaluating Large Language Models in\u00a0Cybersecurity Knowledge with\u00a0Cisco Certificates"],"prefix":"10.1007","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-2323-0533","authenticated-orcid":false,"given":"Gustav","family":"Keppler","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0009-0005-3857-2578","authenticated-orcid":false,"given":"Jeremy","family":"Kunz","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-3572-9083","authenticated-orcid":false,"given":"Veit","family":"Hagenmeyer","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-1137-1782","authenticated-orcid":false,"given":"Ghada","family":"Elbez","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2025,1,29]]},"reference":[{"key":"12_CR1","unstructured":"Current Exam List. https:\/\/www.cisco.com\/c\/en\/us\/training-events\/training-certifications\/exams\/current-list.html"},{"key":"12_CR2","unstructured":"Free Exam Prep By IT Professionals | ExamTopics. https:\/\/www.examtopics.com\/"},{"key":"12_CR3","unstructured":"Hello GPT-4o. https:\/\/openai.com\/index\/hello-gpt-4o\/"},{"key":"12_CR4","unstructured":"Introducing Claude 3.5 Sonnet. https:\/\/www.anthropic.com\/news\/claude-3-5-sonnet"},{"key":"12_CR5","unstructured":"The Llama 3 Herd of Models | Research - AI at Meta. https:\/\/ai.meta.com\/research\/publications\/the-llama-3-herd-of-models\/"},{"key":"12_CR6","doi-asserted-by":"publisher","unstructured":"Abdin, M., et al.: Phi-3 technical report: a highly capable language model locally on your phone. https:\/\/doi.org\/10.48550\/arXiv.2404.14219","DOI":"10.48550\/arXiv.2404.14219"},{"key":"12_CR7","doi-asserted-by":"publisher","unstructured":"Balepur, N., Ravichander, A., Rudinger, R.: Artifacts or abduction: how do LLMs answer multiple-choice questions without the question? https:\/\/doi.org\/10.48550\/arXiv.2402.12483","DOI":"10.48550\/arXiv.2402.12483"},{"key":"12_CR8","doi-asserted-by":"publisher","unstructured":"Bang, Y., et al.: A multitask, multilingual, multimodal evaluation of ChatGPT on reasoning, hallucination, and interactivity. https:\/\/doi.org\/10.48550\/arXiv.2302.04023","DOI":"10.48550\/arXiv.2302.04023"},{"key":"12_CR9","doi-asserted-by":"publisher","unstructured":"Carlini, N., et al.: Extracting training data from large language models. https:\/\/doi.org\/10.48550\/arXiv.2012.07805","DOI":"10.48550\/arXiv.2012.07805"},{"key":"12_CR10","doi-asserted-by":"publisher","unstructured":"Chen, X., et al.: PaLI-X: on scaling up a multilingual vision and language model. https:\/\/doi.org\/10.48550\/arXiv.2305.18565","DOI":"10.48550\/arXiv.2305.18565"},{"key":"12_CR11","doi-asserted-by":"publisher","unstructured":"Chiang, W.L., et al.: Chatbot arena: an open platform for evaluating LLMs by human preference. https:\/\/doi.org\/10.48550\/arXiv.2403.04132","DOI":"10.48550\/arXiv.2403.04132"},{"key":"12_CR12","doi-asserted-by":"publisher","unstructured":"Hendrycks, D., et al.: Measuring massive multitask language understanding. https:\/\/doi.org\/10.48550\/arXiv.2009.03300","DOI":"10.48550\/arXiv.2009.03300"},{"key":"12_CR13","doi-asserted-by":"publisher","unstructured":"Li, F., Liang, K., Lin, Z., Katsikas, S.K. (eds.): Security and Privacy in Communication Networks: 18th EAI International Conference, SecureComm 2022, Virtual Event, October 2022, Proceedings, Lecture Notes of the Institute for Computer Sciences, Social Informatics and Telecommunications Engineering, 1st edn., vol.\u00a0462. Springer (2023). https:\/\/doi.org\/10.1007\/978-3-031-25538-0","DOI":"10.1007\/978-3-031-25538-0"},{"key":"12_CR14","unstructured":"Li, G., Li, Y., Guannan, W., Yang, H., Yu, Y.: SecEval: a comprehensive benchmark for evaluating cybersecurity knowledge of foundation models. https:\/\/github.com\/XuanwuAI\/SecEval"},{"key":"12_CR15","doi-asserted-by":"publisher","unstructured":"Li, T., et al.: From crowdsourced data to high-quality benchmarks: arena-hard and BenchBuilder pipeline. https:\/\/doi.org\/10.48550\/arXiv.2406.11939","DOI":"10.48550\/arXiv.2406.11939"},{"key":"12_CR16","doi-asserted-by":"publisher","unstructured":"Li, N., et al.: The WMDP benchmark: measuring and reducing malicious use with unlearning. https:\/\/doi.org\/10.48550\/arXiv.2403.03218","DOI":"10.48550\/arXiv.2403.03218"},{"key":"12_CR17","unstructured":"Liang, P., et al.: Holistic evaluation of language models. https:\/\/openreview.net\/forum?id=iO4LZibEqW"},{"key":"12_CR18","doi-asserted-by":"publisher","unstructured":"Liu, Z.: SecQA: a concise question-answering dataset for evaluating large language models in computer security. https:\/\/doi.org\/10.48550\/arXiv.2312.15838","DOI":"10.48550\/arXiv.2312.15838"},{"key":"12_CR19","unstructured":"Motlagh, F.N., Hajizadeh, M., Majd, M., Najafi, P., Cheng, F., Meinel, C.: Large language models in cybersecurity: state-of-the-art. http:\/\/arxiv.org\/abs\/2402.00891"},{"key":"12_CR20","unstructured":"OpenAI: GPT-4 technical report. https:\/\/arxiv.org\/pdf\/2303.08774.pdf"},{"key":"12_CR21","doi-asserted-by":"publisher","unstructured":"Sainz, O., et al.: NLP evaluation in trouble: on the need to measure LLM data contamination for each benchmark. https:\/\/doi.org\/10.48550\/arXiv.2310.18018","DOI":"10.48550\/arXiv.2310.18018"},{"key":"12_CR22","unstructured":"Tann, W., Liu, Y., Sim, J.H., Seah, C.M., Chang, E.C.: Using large language models for cybersecurity capture-the-flag challenges and certification questions. https:\/\/arxiv.org\/pdf\/2308.10443.pdf"},{"key":"12_CR23","doi-asserted-by":"publisher","unstructured":"Tihanyi, N., Ferrag, M.A., Jain, R., Bisztray, T., Debbah, M.: CyberMetric: a benchmark dataset based on retrieval-augmented generation for evaluating LLMs in cybersecurity knowledge. https:\/\/doi.org\/10.48550\/arXiv.2402.07688","DOI":"10.48550\/arXiv.2402.07688"},{"key":"12_CR24","unstructured":"Vaswani, A., et al.: Attention is all you need. https:\/\/arxiv.org\/pdf\/1706.03762.pdf"},{"key":"12_CR25","doi-asserted-by":"publisher","unstructured":"Wang, Y., et al.: MMLU-Pro: a more robust and challenging multi-task language understanding benchmark. https:\/\/doi.org\/10.48550\/arXiv.2406.01574","DOI":"10.48550\/arXiv.2406.01574"},{"key":"12_CR26","doi-asserted-by":"publisher","unstructured":"Wei, J., et al.: Chain-of-thought prompting elicits reasoning in large language models. https:\/\/doi.org\/10.48550\/arXiv.2201.11903","DOI":"10.48550\/arXiv.2201.11903"},{"key":"12_CR27","doi-asserted-by":"publisher","unstructured":"Xia, C.S., Wei, Y., Zhang, L.: Practical program repair in the era of large pre-trained language models. https:\/\/doi.org\/10.48550\/arXiv.2210.14179","DOI":"10.48550\/arXiv.2210.14179"},{"key":"12_CR28","doi-asserted-by":"publisher","unstructured":"Xu, H., et al.: Large language models for cyber security: a systematic literature review. https:\/\/doi.org\/10.48550\/arXiv.2405.04760","DOI":"10.48550\/arXiv.2405.04760"},{"key":"12_CR29","doi-asserted-by":"publisher","unstructured":"Yue, X., et al.: MMMU: a massive multi-discipline multimodal understanding and reasoning benchmark for expert AGI. https:\/\/doi.org\/10.48550\/arXiv.2311.16502","DOI":"10.48550\/arXiv.2311.16502"},{"key":"12_CR30","doi-asserted-by":"publisher","unstructured":"Zhang, J., Bu, H., Wen, H., Chen, Y., Li, L., Zhu, H.: When LLMs meet cybersecurity: a systematic literature review. https:\/\/doi.org\/10.48550\/arXiv.2405.03644","DOI":"10.48550\/arXiv.2405.03644"}],"container-title":["Lecture Notes in Computer Science","Secure IT Systems"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-031-79007-2_12","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,1,28]],"date-time":"2025-01-28T20:02:11Z","timestamp":1738094531000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-3-031-79007-2_12"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025]]},"ISBN":["9783031790065","9783031790072"],"references-count":30,"URL":"https:\/\/doi.org\/10.1007\/978-3-031-79007-2_12","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"type":"print","value":"0302-9743"},{"type":"electronic","value":"1611-3349"}],"subject":[],"published":{"date-parts":[[2025]]},"assertion":[{"value":"29 January 2025","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"The authors have no competing interests to declare that are relevant to the content of this article.","order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Disclosure of Interests"}},{"value":"NordSec","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Nordic Conference on Secure IT Systems","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Karlstad","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Sweden","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2024","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"6 November 2024","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"7 November 2024","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"29","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"nordsec2024","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"https:\/\/nordsec2024.kau.se\/","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}}]}}