{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,19]],"date-time":"2026-04-19T19:47:34Z","timestamp":1776628054353,"version":"3.51.2"},"reference-count":39,"publisher":"Springer Science and Business Media LLC","issue":"3","license":[{"start":{"date-parts":[[2024,11,19]],"date-time":"2024-11-19T00:00:00Z","timestamp":1731974400000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2024,11,19]],"date-time":"2024-11-19T00:00:00Z","timestamp":1731974400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Front. Comput. Sci."],"published-print":{"date-parts":[[2025,3]]},"DOI":"10.1007\/s11704-024-3380-1","type":"journal-article","created":{"date-parts":[[2024,11,19]],"date-time":"2024-11-19T09:53:46Z","timestamp":1732010026000},"update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":11,"title":["Behaviour-diverse automatic penetration testing: a coverage-based deep reinforcement learning approach"],"prefix":"10.1007","volume":"19","author":[{"given":"Yizhou","family":"Yang","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Longde","family":"Chen","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Sha","family":"Liu","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Lanning","family":"Wang","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Haohuan","family":"Fu","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Xin","family":"Liu","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Zuoning","family":"Chen","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2024,11,19]]},"reference":[{"key":"3380_CR1","volume-title":"Proceedings of the 32nd Annual Conference on Computer Security Applications","author":"A Applebaum","year":"2016","unstructured":"Applebaum A, Miller D, Strom B, Korban C, Wolf R. Intelligent, automated red team emulation. In: Proceedings of the 32nd Annual Conference on Computer Security Applications. 2016"},{"key":"3380_CR2","volume-title":"Proceedings of the 25th International Conference on Automated Planning and Scheduling","author":"J Hoffmann","year":"2015","unstructured":"Hoffmann J. Simulated penetration testing: from \u201cdijkstra\u201d to \u201cturing test++\u201d. In: Proceedings of the 25th International Conference on Automated Planning and Scheduling. 2015"},{"issue":"7676","key":"3380_CR3","doi-asserted-by":"publisher","first-page":"354","DOI":"10.1038\/nature24270","volume":"550","author":"D Silver","year":"2017","unstructured":"Silver D, Schrittwieser J, Simonyan K, Antonoglou I, Huang A, Guez A, Hubert T, Baker L, Lai M, Bolton A, Chen Y, Lillicrap T, Hui F, Sifre L, van den Driessche G, Graepel T, Hassabis D. Mastering the game of Go without human knowledge. Nature, 2017, 550(7676): 354\u2013359","journal-title":"Nature"},{"issue":"7782","key":"3380_CR4","doi-asserted-by":"publisher","first-page":"350","DOI":"10.1038\/s41586-019-1724-z","volume":"575","author":"O Vinyals","year":"2019","unstructured":"Vinyals O, Babuschkin I, Czarnecki W M, Mathieu M, Dudzik A, Chung J, Choi D H, Powell R, Ewalds T, Georgiev P, Oh J, Horgan D, Kroiss M, Danihelka I, Huang A, Sifre L, Cai T, Agapiou J P, Jaderberg M, Vezhnevets A S, Leblond R, Pohlen T, Dalibard V, Budden D, Sulsky Y, Molloy J, Paine T L, Gulcehre C, Wang Z, Pfaff T, Wu Y, Ring R, Yogatama D, W\u00fcnsch D, Mckinney K, Smith O, Schaul T, Lillicrap T, Kavukcuoglu K, Hassabis D, Apps C, Silver D. Grandmaster level in StarCraft II using multi-agent reinforcement learning. Nature, 2019, 575(7782): 350\u2013354","journal-title":"Nature"},{"key":"3380_CR5","volume-title":"Proceedings of the 34th International Conference on Neural Information Processing Systems","author":"M Laskin","year":"2020","unstructured":"Laskin M, Lee K, Stooke A, Pinto L, Abbeel P, Srinivas A. Reinforcement learning with augmented data. In: Proceedings of the 34th International Conference on Neural Information Processing Systems. 2020"},{"key":"3380_CR6","first-page":"2","volume-title":"Proceedings of 2020 IEEE European Symposium on Security and Privacy Workshops","author":"Z Hu","year":"2020","unstructured":"Hu Z, Beuran R, Tan Y. Automated penetration testing using deep reinforcement learning. In: Proceedings of 2020 IEEE European Symposium on Security and Privacy Workshops. 2020, 2\u201310"},{"issue":"19","key":"3380_CR7","doi-asserted-by":"publisher","first-page":"8823","DOI":"10.3390\/app11198823","volume":"11","author":"S Zhou","year":"2021","unstructured":"Zhou S, Liu J, Hou D, Zhong X, Zhang Y. Autonomous penetration testing based on improved deep Q-network. Applied Sciences, 2021, 11(19): 8823","journal-title":"Applied Sciences"},{"key":"3380_CR8","unstructured":"Tran K, Akella A, Standen M, Kim J, Bowman D, Richer T, Lin C T. Deep hierarchical reinforcement agents for automated penetration testing. 2021, arXiv preprint arXiv: 2109.06449"},{"key":"3380_CR9","unstructured":"Schwartz J, Kurniawati H. Autonomous penetration testing using reinforcement learning. 2019, arXiv preprint arXiv: 1905.05965"},{"key":"3380_CR10","unstructured":"Schwartz J, Kurniawatti H. NASim: network attack simulator. Networkattacksimulator.readthedocs.io\/, 2019"},{"key":"3380_CR11","unstructured":"Baillie C, Standen M, Schwartz J, Docking M, Bowman D, Kim J. CybORG: an autonomous cyber operations research gym. 2020, arXiv preprint arXiv: 2002.10667"},{"key":"3380_CR12","first-page":"241","volume-title":"Proceedings of the 28th International Conference on Automated Planning and Scheduling","author":"D Shmaryahu","year":"2018","unstructured":"Shmaryahu D, Shani G, Hoffmann J, Steinmetz M. Simulated penetration testing as contingent planning. In: Proceedings of the 28th International Conference on Automated Planning and Scheduling. 2018, 241\u2013249"},{"key":"3380_CR13","doi-asserted-by":"publisher","first-page":"100219","DOI":"10.1016\/j.cosrev.2019.100219","volume":"35","author":"H S Lallie","year":"2020","unstructured":"Lallie H S, Debattista K, Bal J. A review of attack graph and attack tree visual syntax in cyber security. Computer Science Review, 2020, 35: 100219","journal-title":"Computer Science Review"},{"issue":"2","key":"3380_CR14","doi-asserted-by":"publisher","first-page":"293","DOI":"10.1007\/s10207-021-00554-7","volume":"21","author":"L Erd\u0151di","year":"2022","unstructured":"Erd\u0151di L, Zennaro F M. The agent web model: modeling web hacking for reinforcement learning. International Journal of Information Security, 2022, 21(2): 293\u2013309","journal-title":"International Journal of Information Security"},{"key":"3380_CR15","first-page":"1","volume-title":"Proceedings of 2022 International Joint Conference on Neural Networks","author":"Y Li","year":"2022","unstructured":"Li Y, Yan J, Naili M. Deep reinforcement learning for penetration testing of cyber-physical attacks in the smart grid. In: Proceedings of 2022 International Joint Conference on Neural Networks. 2022, 1\u20139"},{"key":"3380_CR16","first-page":"1","volume-title":"Proceedings of 2021 IEEE Symposium Series on Computational Intelligence","author":"R Gangupantulu","year":"2021","unstructured":"Gangupantulu R, Cody T, Rahma A, Redino C, Clark R, Park P. Crown jewels analysis using reinforcement learning with attack graphs. In: Proceedings of 2021 IEEE Symposium Series on Computational Intelligence. 2021, 1\u20136"},{"issue":"1","key":"3380_CR17","doi-asserted-by":"publisher","first-page":"6","DOI":"10.3390\/info11010006","volume":"11","author":"M C Ghanem","year":"2019","unstructured":"Ghanem M C, Chen T M. Reinforcement learning for efficient network penetration testing. Information, 2019, 11(1): 6","journal-title":"Information"},{"issue":"3","key":"3380_CR18","doi-asserted-by":"publisher","first-page":"441","DOI":"10.1049\/ise2.12107","volume":"17","author":"F M Zennaro","year":"2023","unstructured":"Zennaro F M, Erd\u0151di L. Modelling penetration testing with reinforcement learning using capture-the-flag challenges: trade-offs between model-free learning and a priori knowledge. IET Information Security, 2023, 17(3): 441\u2013457","journal-title":"IET Information Security"},{"key":"3380_CR19","first-page":"2778","volume-title":"Proceedings of 2017 IEEE Conference on Computer Vision and Pattern Recognition Workshops","author":"D Pathak","year":"2017","unstructured":"Pathak D, Agrawal P, Efros A A, Darrell T. Curiosity-driven exploration by self-supervised prediction. In: Proceedings of 2017 IEEE Conference on Computer Vision and Pattern Recognition Workshops. 2017, 2778\u20132787"},{"key":"3380_CR20","first-page":"476","volume-title":"Proceedings of the 7th International Conference on Computer and Communication Systems","author":"P Wang","year":"2022","unstructured":"Wang P, Liu J, Zhong X, Yang G, Zhou S, Zhang Y. DUSC-DQN: an improved deep Q-network for intelligent penetration testing path design. In: Proceedings of the 7th International Conference on Computer and Communication Systems. 2022, 476\u2013480"},{"key":"3380_CR21","first-page":"3540","volume-title":"Proceedings of the 34th International Conference on Machine Learning","author":"A S Vezhnevets","year":"2017","unstructured":"Vezhnevets A S, Osindero S, Schaul T, Heess N, Jaderberg M, Silver D, Kavukcuoglu K. Feudal networks for hierarchical reinforcement learning. In: Proceedings of the 34th International Conference on Machine Learning. 2017, 3540\u20133549"},{"key":"3380_CR22","first-page":"1087","volume-title":"Proceedings of the 35th International Conference on Machine Learning","author":"W Czarnecki","year":"2018","unstructured":"Czarnecki W, Jayakumar S, Jaderberg M, Hasenclever L, Teh Y W, Heess N, Osindero S, Pascanu R. Mix & match agent curricula for reinforcement learning. In: Proceedings of the 35th International Conference on Machine Learning. 2018, 1087\u20131095"},{"key":"3380_CR23","first-page":"285","volume-title":"Proceedings of the 37th International Conference on Machine Learning","author":"G Farquhar","year":"2020","unstructured":"Farquhar G, Gustafson L, Lin Z, Whiteson S, Usunier N, Synnaeve G. Growing action spaces. In: Proceedings of the 37th International Conference on Machine Learning. 2020, 285"},{"key":"3380_CR24","doi-asserted-by":"publisher","first-page":"82","DOI":"10.1007\/3-540-44719-9_6","volume-title":"Proceedings of the 1st International Conference on Evolutionary Multi-Criterion Optimization","author":"T Murata","year":"2001","unstructured":"Murata T, Ishibuchi H, Gen M. Specification of genetic search directions in cellular multi-objective genetic algorithms. In: Proceedings of the 1st International Conference on Evolutionary Multi-Criterion Optimization. 2001, 82\u201395"},{"issue":"4","key":"3380_CR25","doi-asserted-by":"publisher","first-page":"577","DOI":"10.1109\/TEVC.2013.2281535","volume":"18","author":"K Deb","year":"2014","unstructured":"Deb K, Jain H. An evolutionary many-objective optimization algorithm using reference-point-based nondominated sorting approach, Part I: solving problems with box constraints. IEEE Transactions on Evolutionary Computation, 2014, 18(4): 577\u2013601","journal-title":"IEEE Transactions on Evolutionary Computation"},{"key":"3380_CR26","unstructured":"Hsu C H, Chang S H, Liang J H, Chou H P, Liu C H, Chang S C, Pan J Y, Chen Y T, Wei W, Juan D C. MONAS: multi-objective neural architecture search using reinforcement learning. 2018, arXiv preprint arXiv: 1806.10332"},{"key":"3380_CR27","unstructured":"Mossalam H, Assael Y M, Roijers D M, Whiteson S. Multi-objective deep reinforcement learning. 2016, arXiv preprint arXiv: 1610.02707"},{"issue":"6443","key":"3380_CR28","doi-asserted-by":"publisher","first-page":"859","DOI":"10.1126\/science.aau6249","volume":"364","author":"M Jaderberg","year":"2019","unstructured":"Jaderberg M, Czarnecki W M, Dunning I, Marris L, Lever G, Castaneda A G, Beattie C, Rabinowitz N C, Morcos A S, Ruderman A, Sonnerat N, Green T, Deason L, Leibo J Z, Silver D, Hassabis D, Kavukcuoglu K, Graepel T. Human-level performance in 3D multiplayer games with population-based reinforcement learning. Science, 2019, 364(6443): 859\u2013865","journal-title":"Science"},{"key":"3380_CR29","first-page":"3371","volume-title":"Proceedings of the 29th International Joint Conference on Artificial Intelligence","author":"R Shen","year":"2021","unstructured":"Shen R, Zheng Y, Hao J, Meng Z, Chen Y, Fan C, Liu Y. Generating behavior-diverse game AIs with evolutionary multi-objective deep reinforcement learning. In: Proceedings of the 29th International Joint Conference on Artificial Intelligence. 2021, 3371\u20133377"},{"key":"3380_CR30","volume-title":"Mitre att&ck: Design and philosophy","author":"B E Strom","year":"2018","unstructured":"Strom B E, Applebaum A, Miller D P, Nickels K C, Pennington A G, Thomas C B. Mitre att&ck: Design and philosophy. Mitre Product MP, 2018"},{"key":"3380_CR31","unstructured":"Schulman J, Wolski F, Dhariwal P, Radford A, Klimov O. Proximal policy optimization algorithms. 2017, arXiv preprint arXiv: 1707.06347"},{"key":"3380_CR32","volume-title":"Proceedings of the 4th International Conference on Learning Representations","author":"J Schulman","year":"2016","unstructured":"Schulman J, Moritz P, Levine S, Jordan M I, Abbeel P. High-dimensional continuous control using generalized advantage estimation. In: Proceedings of the 4th International Conference on Learning Representations. 2016"},{"key":"3380_CR33","volume-title":"Proceedings of the 7th International Conference on Learning Representations","author":"Y Burda","year":"2019","unstructured":"Burda Y, Edwards H, Storkey A J, Klimov O. Exploration by random network distillation. In: Proceedings of the 7th International Conference on Learning Representations. 2019"},{"key":"3380_CR34","volume-title":"Reinforcement Learning: An Introduction","author":"R S Sutton","year":"2018","unstructured":"Sutton R S, Barto A G. Reinforcement Learning: An Introduction. 2nd ed. Cambridge: MIT Press, 2018","edition":"2nd ed."},{"key":"3380_CR35","doi-asserted-by":"publisher","first-page":"67","DOI":"10.1613\/jair.3987","volume":"48","author":"D M Roijers","year":"2013","unstructured":"Roijers D M, Vamplew P, Whiteson S, Dazeley R. A survey of multi-objective sequential decision-making. Journal of Artificial Intelligence Research, 2013, 48: 67\u2013113","journal-title":"Journal of Artificial Intelligence Research"},{"issue":"2","key":"3380_CR36","doi-asserted-by":"publisher","first-page":"212","DOI":"10.1109\/TG.2021.3049539","volume":"14","author":"I Oh","year":"2022","unstructured":"Oh I, Rho S, Moon S, Son S, Lee H, Chung J. Creating pro-level AI for a real-time fighting game using deep reinforcement learning. IEEE Transactions on Games, 2022, 14(2): 212\u2013220","journal-title":"IEEE Transactions on Games"},{"key":"3380_CR37","first-page":"29304","volume-title":"Proceedings of the 35th International Conference on Neural Information Processing Systems","author":"R Agarwal","year":"2021","unstructured":"Agarwal R, Schwarzer M, Castro P S, Courville A C, Bellemare M. Deep reinforcement learning at the edge of the statistical precipice. In: Proceedings of the 35th International Conference on Neural Information Processing Systems. 2021, 29304\u201329320"},{"key":"3380_CR38","volume-title":"Cyber autonomy gym for experimentation challenge 1","author":"M Standen","year":"2021","unstructured":"Standen M, Bowman D, Hoang S, Richer T, Lucas M, Van Tassel R. Cyber autonomy gym for experimentation challenge 1, 2021"},{"issue":"2","key":"3380_CR39","first-page":"65","volume":"3","author":"F H Katz","year":"2018","unstructured":"Katz F H. Breadth vs. depth: best practices teaching cybersecurity in a small public university sharing models. The Cyber Defense Review, 2018, 3(2): 65\u201372","journal-title":"The Cyber Defense Review"}],"container-title":["Frontiers of Computer Science"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s11704-024-3380-1.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s11704-024-3380-1","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s11704-024-3380-1.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,4,19]],"date-time":"2026-04-19T19:26:12Z","timestamp":1776626772000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s11704-024-3380-1"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,11,19]]},"references-count":39,"journal-issue":{"issue":"3","published-print":{"date-parts":[[2025,3]]}},"alternative-id":["3380"],"URL":"https:\/\/doi.org\/10.1007\/s11704-024-3380-1","relation":{},"ISSN":["2095-2228","2095-2236"],"issn-type":[{"value":"2095-2228","type":"print"},{"value":"2095-2236","type":"electronic"}],"subject":[],"published":{"date-parts":[[2024,11,19]]},"assertion":[{"value":"7 May 2023","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"25 February 2024","order":2,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"19 November 2024","order":3,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"Competing interests\n                      The authors declare that they have no competing interests or financial conflicts to disclose.","order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Ethics"}}],"article-number":"193309"}}