{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,5,13]],"date-time":"2026-05-13T17:21:29Z","timestamp":1778692889221,"version":"3.51.4"},"reference-count":49,"publisher":"IEEE","license":[{"start":{"date-parts":[[2023,10,2]],"date-time":"2023-10-02T00:00:00Z","timestamp":1696204800000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2023,10,2]],"date-time":"2023-10-02T00:00:00Z","timestamp":1696204800000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"}],"funder":[{"DOI":"10.13039\/501100000038","name":"Natural Sciences and Engineering Research Council of Canada","doi-asserted-by":"publisher","id":[{"id":"10.13039\/501100000038","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2023,10,2]]},"DOI":"10.1109\/cns59707.2023.10288677","type":"proceedings-article","created":{"date-parts":[[2023,10,27]],"date-time":"2023-10-27T17:47:13Z","timestamp":1698428833000},"page":"1-6","source":"Crossref","is-referenced-by-count":24,"title":["Towards Evaluation and Understanding of Large Language Models for Cyber Operation Automation"],"prefix":"10.1109","author":[{"given":"Madeena","family":"Sultana","sequence":"first","affiliation":[{"name":"Defence Research and Development Canada,Ottawa,Canada"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Adrian","family":"Taylor","sequence":"additional","affiliation":[{"name":"Defence Research and Development Canada,Ottawa,Canada"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Li","family":"Li","sequence":"additional","affiliation":[{"name":"Defence Research and Development Canada,Ottawa,Canada"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Suryadipta","family":"Majumdar","sequence":"additional","affiliation":[{"name":"Concordia University,Montreal,Canada"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"263","reference":[{"key":"ref1","article-title":"Challenges and applications of large language models","author":"Kaddour","year":"2023","journal-title":"arXiv preprint arXiv:2307.10169"},{"key":"ref2","article-title":"Spam-t5: Benchmarking large language models for few-shot email spam detection","author":"Labonne","year":"2023","journal-title":"arXiv preprint arXiv:2304.01238"},{"key":"ref3","article-title":"Cysecbert: A domain-adapted language model for the cybersecurity domain","author":"Bayer","year":"2022","journal-title":"arXiv preprint arXiv:2212.02974"},{"key":"ref4","doi-asserted-by":"publisher","DOI":"10.1109\/access.2024.3363469"},{"key":"ref5","volume-title":"Cyber operations","year":"2023"},{"key":"ref6","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/w18-5446"},{"key":"ref7","article-title":"Beyond the imitation game: Quantifying and extrapolating the capabilities of language models","author":"Srivastava","year":"2022","journal-title":"arXiv preprint arXiv:2206.04615"},{"key":"ref8","article-title":"Holistic evaluation of language models","author":"Liang","year":"2022","journal-title":"arXiv preprint arXiv:2211.09110"},{"key":"ref9","article-title":"SuperGLUE: A stickier benchmark for general-purpose language understanding systems","volume":"32","author":"Wang","year":"2019","journal-title":"Advances in Neural Information Processing Systems"},{"key":"ref10","first-page":"39","article-title":"SecureBERT: A domain-specific language model for cybersecurity","volume-title":"Intl. Conf. on Security and Privacy in Communication Systems","author":"Aghaei"},{"key":"ref11","article-title":"LegalBench: Prototyping a collaborative benchmark for legal reasoning","author":"Guha","year":"2022","journal-title":"arXiv preprint arXiv:2209.06120"},{"key":"ref12","article-title":"AI and the everything in the whole wide world benchmark","author":"Raji","year":"2021","journal-title":"arXiv preprint arXiv:2111.15366"},{"key":"ref13","article-title":"Improving transparency in AI language models: A holistic evaluation","author":"Bommasani","year":"2023","journal-title":"HAI Policy & Society"},{"key":"ref14","first-page":"8489","article-title":"Reduce, reuse, recycle: Compositional generation with energy-based diffusion models and mcmc","volume-title":"Intl. Conf. on Machine Learning","author":"Du"},{"key":"ref15","doi-asserted-by":"publisher","DOI":"10.2139\/ssrn.3936759"},{"key":"ref16","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/W19-5006"},{"key":"ref17","article-title":"ClimaBench: A benchmark dataset for climate change text understanding in english","author":"Laud","year":"2023","journal-title":"arXiv preprint arXiv:2301.04253"},{"key":"ref18","doi-asserted-by":"publisher","DOI":"10.1038\/s41586-023-06291-2"},{"key":"ref19","doi-asserted-by":"publisher","DOI":"10.1145\/3487351.3492723"},{"key":"ref20","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE48619.2023.00204"},{"key":"ref21","doi-asserted-by":"publisher","DOI":"10.1109\/IJCNN52387.2021.9534113"},{"key":"ref22","doi-asserted-by":"publisher","DOI":"10.1109\/TC.2023.3257518"},{"key":"ref23","article-title":"Tbdetector: Transformer-based detector for advanced persistent threats with provenance graph","author":"Wang","year":"2023","journal-title":"arXiv preprint arXiv:2304.02838"},{"key":"ref24","doi-asserted-by":"publisher","DOI":"10.1109\/ICDM.2009.19"},{"key":"ref25","volume-title":"A large collection of system log datasets towards AI-powered log analytics","year":"2023"},{"key":"ref26","article-title":"LogQA: Question answering in unstructured logs","author":"Huang","year":"2023","journal-title":"arXiv preprint arXiv:2303.11715"},{"key":"ref27","doi-asserted-by":"publisher","DOI":"10.1109\/BigData52589.2021.9671824"},{"key":"ref28","article-title":"CyNER: A python library for cybersecurity named entity recognition","author":"Alam","year":"2022","journal-title":"arXiv preprint arXiv:2204.05754"},{"key":"ref29","doi-asserted-by":"publisher","DOI":"10.1016\/j.cose.2022.102763"},{"key":"ref30","doi-asserted-by":"publisher","DOI":"10.1145\/3607199.3607208"},{"key":"ref31","doi-asserted-by":"publisher","DOI":"10.1109\/ICDE53745.2022.00287"},{"key":"ref32","article-title":"On the uses of large language models to interpret ambiguous cyberattack descriptions","author":"Fayyazi","year":"2023","journal-title":"arXiv preprint arXiv:2306.14062"},{"key":"ref33","doi-asserted-by":"publisher","DOI":"10.1109\/BigData55660.2022.10020622"},{"key":"ref34","doi-asserted-by":"publisher","DOI":"10.3390\/a15090314"},{"key":"ref35","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-37586-6_15"},{"key":"ref36","doi-asserted-by":"publisher","DOI":"10.1016\/j.eswa.2023.122564"},{"key":"ref37","doi-asserted-by":"publisher","DOI":"10.1109\/MILCOM52596.2021.9653028"},{"key":"ref38","doi-asserted-by":"publisher","DOI":"10.1016\/j.jss.2022.111577"},{"key":"ref39","doi-asserted-by":"publisher","DOI":"10.1145\/3458723"},{"key":"ref40","volume-title":"Cybersecurity framework","year":"2023"},{"key":"ref41","volume-title":"TTCP cage challenge 3","year":"2022"},{"key":"ref42","volume-title":"Proving grounds: Virtual hacking & pentesting practice labs","year":"2023"},{"key":"ref43","volume-title":"CybORG: A Gym for the Development of Autonomous Cyber Agents","year":"2021"},{"key":"ref44","article-title":"CyGIL: A cyber gym for training autonomous agents over emulated network systems","author":"Li","year":"2021","journal-title":"arXiv preprint arXiv:2109.03331"},{"key":"ref45","volume-title":"A realistic cyber defense dataset (cse-cic-ids2018)","year":"2023"},{"key":"ref46","doi-asserted-by":"publisher","DOI":"10.18653\/vl\/PI7-1143"},{"key":"ref47","first-page":"92","article-title":"Introducing the enron corpus","volume-title":"Conf. on Collaboration, Electronic messaging, Anti-Abuse and Spam","volume":"45","author":"Klimt"},{"key":"ref48","doi-asserted-by":"publisher","DOI":"10.1109\/TrustCom50675.2020.00252"},{"key":"ref49","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-86890-1_24"}],"event":{"name":"2023 IEEE Conference on Communications and Network Security (CNS)","location":"Orlando, FL, USA","start":{"date-parts":[[2023,10,2]]},"end":{"date-parts":[[2023,10,5]]}},"container-title":["2023 IEEE Conference on Communications and Network Security (CNS)"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx7\/10288629\/10288642\/10288677.pdf?arnumber=10288677","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,9,11]],"date-time":"2024-09-11T05:01:55Z","timestamp":1726030915000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/10288677\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023,10,2]]},"references-count":49,"URL":"https:\/\/doi.org\/10.1109\/cns59707.2023.10288677","relation":{},"subject":[],"published":{"date-parts":[[2023,10,2]]}}}