{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,3]],"date-time":"2026-07-03T18:23:47Z","timestamp":1783103027247,"version":"3.54.6"},"publisher-location":"Singapore","reference-count":53,"publisher":"Springer Nature Singapore","isbn-type":[{"value":"9789819533459","type":"print"},{"value":"9789819533466","type":"electronic"}],"license":[{"start":{"date-parts":[[2025,11,23]],"date-time":"2025-11-23T00:00:00Z","timestamp":1763856000000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2025,11,23]],"date-time":"2025-11-23T00:00:00Z","timestamp":1763856000000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2026]]},"DOI":"10.1007\/978-981-95-3346-6_35","type":"book-chapter","created":{"date-parts":[[2025,11,22]],"date-time":"2025-11-22T05:49:56Z","timestamp":1763790596000},"page":"456-470","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":0,"title":["Open Bilingual Benchmark and\u00a0Leaderboard for\u00a0Large Language Models in\u00a0Cybersecurity"],"prefix":"10.1007","author":[{"given":"Wanlong","family":"Yu","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Wei","family":"Wan","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Zhenxu","family":"Wang","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Feng","family":"Li","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Kang","family":"Wang","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Gang","family":"Hu","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2025,11,23]]},"reference":[{"key":"35_CR1","doi-asserted-by":"publisher","first-page":"221","DOI":"10.1016\/j.procs.2015.08.443","volume":"62","author":"AI Abubakar","year":"2015","unstructured":"Abubakar, A.I., Chiroma, H., et al.: A review of the advances in cyber security benchmark datasets for evaluating data-driven based intrusion detection systems. Proc. Comput. Sci. 62, 221\u2013227 (2015)","journal-title":"Proc. Comput. Sci."},{"key":"35_CR2","doi-asserted-by":"crossref","unstructured":"Aghaei, E., Niu, X., et\u00a0al.: Securebert: a domain-specific language model for cybersecurity. In: International Conference on Security and Privacy in Communication Systems, pp. 39\u201356. Springer (2022)","DOI":"10.1007\/978-3-031-25538-0_3"},{"key":"35_CR3","unstructured":"Alam, M.T., Bhusal, D., Park, Y., Rastogi, N.: Cyner: A python library for cybersecurity named entity recognition. arXiv preprint arXiv:2204.05754 (2022)"},{"key":"35_CR4","doi-asserted-by":"crossref","unstructured":"Alam, M.T., Bhusal, D., et\u00a0al.: Ctibench: A benchmark for evaluating LLMs in cyber threat intelligence. arXiv preprint arXiv:2406.07599 (2024)","DOI":"10.52202\/079017-1607"},{"issue":"2","key":"35_CR5","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1145\/3652594","volume":"27","author":"M Bayer","year":"2024","unstructured":"Bayer, M., et al.: Cysecbert: A domain-adapted language model for the cybersecurity domain. ACM Trans. Privacy Secur. 27(2), 1\u201320 (2024)","journal-title":"ACM Trans. Privacy Secur."},{"key":"35_CR6","unstructured":"Bhatt, M., Chennabasappa, S., et\u00a0al.: Cyberseceval 2: A wide-ranging cybersecurity evaluation suite for large language models. arXiv preprint arXiv:2404.13161 (2024)"},{"key":"35_CR7","doi-asserted-by":"crossref","unstructured":"Bhusal, D., Alam, M.T., et\u00a0al.: Secure: Benchmarking large language models for cybersecurity. arXiv preprint arXiv:2405.20441 (2024)","DOI":"10.1109\/ACSAC63791.2024.00019"},{"key":"35_CR8","unstructured":"Clouditera: Secgpt (2025). https:\/\/github.com\/Clouditera\/secgpt. Accessed 20 Apr 2025"},{"key":"35_CR9","unstructured":"Cypher07: Cybersecurity news article dataset (2023). https:\/\/github.com\/cypher-07\/Cybersecurity-News-Article-Dataset. Accessed 20 Apr 2025"},{"key":"35_CR10","unstructured":"ddzipp: Autoaudit (2025). https:\/\/github.com\/ddzipp\/AutoAudit. Accessed 20 Apr 2025"},{"key":"35_CR11","unstructured":"Fang, R., Bindu, R., et\u00a0al.: LLM agents can autonomously exploit one-day vulnerabilities. arXiv preprint arXiv:2404.0814413, 14 (2024)"},{"key":"35_CR12","doi-asserted-by":"crossref","unstructured":"Fei, Z., Shen, et\u00a0al.: Lawbench: Benchmarking legal knowledge of large language models. arXiv preprint arXiv:2309.16289 (2023)","DOI":"10.18653\/v1\/2024.emnlp-main.452"},{"key":"35_CR13","unstructured":"Ferrag, M., Battah, A., et\u00a0al.: Securefalcon the next cyber reasoning system for cyber security. arXiv preprint arXiv:2307.06616"},{"key":"35_CR14","unstructured":"Gao, L., Tow, J., Abbasi, B., et\u00a0al.: A framework for few-shot language model evaluation (2024)"},{"key":"35_CR15","doi-asserted-by":"crossref","unstructured":"Goutte, C., Gaussier, E.: A probabilistic interpretation of precision, recall and f-score, with implication for evaluation. In: European Conference on Information Retrieval, pp. 345\u2013359. Springer (2005)","DOI":"10.1007\/978-3-540-31865-1_25"},{"key":"35_CR16","unstructured":"Guo, H., Yang, J., et\u00a0al.: Owl: A large language model for it operations. arXiv preprint arXiv:2309.09298 (2023)"},{"key":"35_CR17","unstructured":"Hendrycks, D., Burns, C., et\u00a0al.: Measuring massive multitask language understanding. arXiv preprint arXiv:2009.03300 (2020)"},{"key":"35_CR18","unstructured":"Highnam, K., Arulkumaran, K., et\u00a0al.: Beth dataset: real cybersecurity data for unsupervised anomaly detection research. In: CEUR Workshop Proc, vol.\u00a03095, pp. 1\u201312 (2021)"},{"key":"35_CR19","unstructured":"Hu, G., Qin, et\u00a0al.: No language is an island: Unifying Chinese and English in financial large language models, instruction data, and benchmarks. arXiv preprint arXiv:2403.06249 (2024)"},{"key":"35_CR20","unstructured":"James4Ever0: Network security questions. https:\/\/huggingface.co\/datasets\/James4Ever0\/network_security_questions (2023). Accessed 20 Apr 2025"},{"key":"35_CR21","unstructured":"Jiajia, H., Haoran, et\u00a0al.: Auditwen: an open-source large language model for audit. In: Proceedings of the 23rd Chinese National Conference on Computational Linguistics, pp. 1351\u20131365 (2024)"},{"key":"35_CR22","unstructured":"Jing, P., Tang, M., et\u00a0al.: Secbench: a comprehensive multi-dimensional benchmarking dataset for LLMs in cybersecurity. arXiv preprint arXiv:2412.20787 (2024)"},{"issue":"4","key":"35_CR23","doi-asserted-by":"publisher","first-page":"212","DOI":"10.1207\/s15430421tip4104_2","volume":"41","author":"DR Krathwohl","year":"2002","unstructured":"Krathwohl, D.R.: A revision of bloom\u2019s taxonomy: an overview. Theor. Pract. 41(4), 212\u2013218 (2002)","journal-title":"Theor. Pract."},{"key":"35_CR24","unstructured":"Li, G., et\u00a0al.: Seceval: a comprehensive benchmark for evaluating cybersecurity knowledge of foundation models. https:\/\/github.com\/XuanwuAI\/SecEval (2023)"},{"key":"35_CR25","unstructured":"Li, Z., Dutta, S., Naik, M.: LLM-assisted static analysis for detecting security vulnerabilities. arXiv preprint arXiv:2405.17238 (2024)"},{"key":"35_CR26","unstructured":"Liang, P., Bommasani, R., et\u00a0al.: Holistic evaluation of language models. arXiv preprint arXiv:2211.09110 (2022)"},{"key":"35_CR27","unstructured":"Lin, C.Y.: Rouge: A package for automatic evaluation of summaries. In: Text summarization branches out, pp. 74\u201381 (2004)"},{"key":"35_CR28","unstructured":"Liu, Y., Pei, C., et\u00a0al.: Opseval: A comprehensive task-oriented aiops benchmark for large language models. arXiv preprint arXiv:2310.07637 (2023)"},{"key":"35_CR29","unstructured":"Liu, Z.: Secqa: a concise question-answering dataset for evaluating large language models in computer security. arXiv preprint arXiv:2312.15838 (2023)"},{"key":"35_CR30","unstructured":"Liu, Z., Shi, J., Buford, J.F.: Cyberbench: a multi-task benchmark for evaluating large language models in cybersecurity. In: AAAI 2024 Workshop on Artificial Intelligence for Cyber Security (2024)"},{"key":"35_CR31","unstructured":"Miao, Y., Bai, Y., et\u00a0al.: An empirical study of netops capability of pre-trained large language models. arXiv preprint arXiv:2309.05557 (2023)"},{"key":"35_CR32","doi-asserted-by":"crossref","unstructured":"Motlagh, F.N., Hajizadeh, M., et\u00a0al.: Large language models in cybersecurity: state-of-the-art. arXiv preprint arXiv:2402.00891 (2024)","DOI":"10.5220\/0013377600003899"},{"key":"35_CR33","unstructured":"N., F.: Malicious URLs (2023). https:\/\/github.com\/faizann24\/Using-machine-learning-to-detect-malicious-URLs. Accessed 20 Apr 2025"},{"key":"35_CR34","doi-asserted-by":"crossref","unstructured":"Ranade, P., Piplai, A., Joshi, A., Finin, T.: Cybert: contextualized embeddings for the cybersecurity domain. In: 2021 IEEE International Conference on Big Data, pp. 3334\u20133342. IEEE (2021)","DOI":"10.1109\/BigData52589.2021.9671824"},{"key":"35_CR35","unstructured":"Roziere, B., Gehring, J., et\u00a0al.: Code llama: Open foundation models for code. arXiv preprint arXiv:2308.12950 (2023)"},{"key":"35_CR36","doi-asserted-by":"crossref","unstructured":"Shestov, A., Levichev, R., et\u00a0al.: Finetuning large language models for vulnerability detection. IEEE Access (2025)","DOI":"10.1109\/ACCESS.2025.3546700"},{"key":"35_CR37","unstructured":"Silva, A., Fang, S., et\u00a0al.: Repairllama: efficient representations and fine-tuned adapters for program repair. arXiv preprint arXiv:2312.15698 (2023)"},{"key":"35_CR38","unstructured":"Spangler, I.: Csic 2010 web application attacks (2010). https:\/\/www.kaggle.com\/datasets\/ispangler\/csic-2010-web-application-attacks. Accessed 20 Apr 2025"},{"key":"35_CR39","doi-asserted-by":"crossref","unstructured":"Tihanyi, N., Ferrag, M.A., et\u00a0al.: Cybermetric: a benchmark dataset based on retrieval-augmented generation for evaluating LLMs in cybersecurity knowledge. In: 2024 IEEE International Conference on Cyber Security and Resilience (CSR), pp. 296\u2013302. IEEE (2024)","DOI":"10.1109\/CSR61664.2024.10679494"},{"key":"35_CR40","doi-asserted-by":"crossref","unstructured":"Ullah, S., Han, M., et\u00a0al.: LLMs cannot reliably identify and reason about security vulnerabilities (yet?): A comprehensive evaluation, framework, and benchmarks. In: 2024 IEEE Symposium on Security and Privacy (SP), pp. 862\u2013880 (2024)","DOI":"10.1109\/SP54263.2024.00210"},{"key":"35_CR41","doi-asserted-by":"crossref","unstructured":"Wang, A., Singh, A., et\u00a0al.: Glue: A multi-task benchmark and analysis platform for natural language understanding. arXiv preprint arXiv:1804.07461 (2018)","DOI":"10.18653\/v1\/W18-5446"},{"key":"35_CR42","doi-asserted-by":"crossref","unstructured":"Xie, Q., Han, et\u00a0al.: Pixiu: a comprehensive benchmark, instruction dataset and large language model for finance. In: 37th International Conference on Neural Information Processing Systems (2023)","DOI":"10.52202\/075280-1454"},{"key":"35_CR43","unstructured":"Xie, Q., Han, et\u00a0al.: The finben: an holistic financial benchmark for large language models. arXiv preprint arXiv:2402.12659 (2024)"},{"issue":"1","key":"35_CR44","doi-asserted-by":"publisher","first-page":"97","DOI":"10.1007\/s42533-021-00079-7","volume":"3","author":"M Xu","year":"2021","unstructured":"Xu, M., Lu, C.: China-us cyber-crisis management. China Int. Strat. Rev. 3(1), 97\u2013114 (2021)","journal-title":"China Int. Strat. Rev."},{"key":"35_CR45","unstructured":"Yemmy1000: Cybersec embedding dataset (2025). https:\/\/huggingface.co\/datasets\/Yemmy1000\/cybersec_embedding"},{"key":"35_CR46","unstructured":"ystemsrx: Cybersecurity-sharegpt-chinese. https:\/\/huggingface.co\/datasets\/ystemsrx\/Cybersecurity-ShareGPT-Chinese (2023). Accessed 20 Apr 2025"},{"key":"35_CR47","doi-asserted-by":"crossref","unstructured":"Zaboli, A., Choi, S.L., et\u00a0al.: Chatgpt and other large language models for cybersecurity of smart grid applications. In: 2024 IEEE Power & Energy Society General Meeting (PESGM), pp.\u00a01\u20135 (2024)","DOI":"10.1109\/PESGM51994.2024.10688863"},{"key":"35_CR48","unstructured":"Zeroshot: Cybersecurity corpus (2023). https:\/\/huggingface.co\/datasets\/zeroshot\/cybersecurity-corpus. Accessed 20 Apr 2025"},{"issue":"1","key":"35_CR49","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1186\/s42400-025-00361-w","volume":"8","author":"J Zhang","year":"2025","unstructured":"Zhang, J., Bu, H., et al.: When LLMs meet cybersecurity: a systematic literature review. Cybersecurity 8(1), 1\u201341 (2025)","journal-title":"Cybersecurity"},{"key":"35_CR50","unstructured":"Zhang, T., Kishore, et\u00a0al.: Bertscore: evaluating text generation with BERT. arXiv preprint arXiv:1904.09675 (2019)"},{"key":"35_CR51","doi-asserted-by":"crossref","unstructured":"Zhang, X., Xiang, R., et\u00a0al.: D\u00f3lares or dollars? unraveling the bilingual prowess of financial LLMs between Spanish and English. In: ACM SIGKDD Conference on Knowledge Discovery and Data Mining, pp. 6236\u20136246 (2024)","DOI":"10.1145\/3637528.3671554"},{"key":"35_CR52","doi-asserted-by":"crossref","unstructured":"Zhou, S., Alon, U., Agarwal, S., Neubig, G.: Codebertscore: evaluating code generation with pretrained models of code. arXiv preprint arXiv:2302.05527 (2023)","DOI":"10.18653\/v1\/2023.emnlp-main.859"},{"issue":"4","key":"35_CR53","doi-asserted-by":"publisher","first-page":"627","DOI":"10.1109\/TSUSC.2023.3240411","volume":"8","author":"Y Zhou","year":"2023","unstructured":"Zhou, Y., Ren, Y., et al.: Cdtier: a Chinese dataset of threat intelligence entity relationships. IEEE Trans. Sustain. Comput. 8(4), 627\u2013638 (2023)","journal-title":"IEEE Trans. Sustain. Comput."}],"container-title":["Lecture Notes in Computer Science","Natural Language Processing and Chinese Computing"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-981-95-3346-6_35","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,7,3]],"date-time":"2026-07-03T18:13:50Z","timestamp":1783102430000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-981-95-3346-6_35"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,11,23]]},"ISBN":["9789819533459","9789819533466"],"references-count":53,"URL":"https:\/\/doi.org\/10.1007\/978-981-95-3346-6_35","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"value":"0302-9743","type":"print"},{"value":"1611-3349","type":"electronic"}],"subject":[],"published":{"date-parts":[[2025,11,23]]},"assertion":[{"value":"23 November 2025","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"NLPCC","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"CCF International Conference on Natural Language Processing and Chinese Computing","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Urumqi","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"China","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2025","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"7 August 2025","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"9 August 2025","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"14","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"nlpcc2025","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"http:\/\/tcci.ccf.org.cn\/conference\/2025\/index.php","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}}]}}