{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,6,3]],"date-time":"2026-06-03T03:08:43Z","timestamp":1780456123146,"version":"3.54.1"},"reference-count":46,"publisher":"Springer Science and Business Media LLC","issue":"21","license":[{"start":{"date-parts":[[2024,4,26]],"date-time":"2024-04-26T00:00:00Z","timestamp":1714089600000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2024,4,26]],"date-time":"2024-04-26T00:00:00Z","timestamp":1714089600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"funder":[{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["62073177"],"award-info":[{"award-number":["62073177"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["61973175"],"award-info":[{"award-number":["61973175"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["62003351"],"award-info":[{"award-number":["62003351"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Neural Comput & Applic"],"published-print":{"date-parts":[[2024,7]]},"DOI":"10.1007\/s00521-024-09720-z","type":"journal-article","created":{"date-parts":[[2024,4,26]],"date-time":"2024-04-26T12:02:15Z","timestamp":1714132935000},"page":"13341-13356","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":10,"title":["Air combat maneuver decision based on deep reinforcement learning with auxiliary reward"],"prefix":"10.1007","volume":"36","author":[{"given":"Tingyu","family":"Zhang","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Yongshuai","family":"Wang","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Mingwei","family":"Sun","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Zengqiang","family":"Chen","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2024,4,26]]},"reference":[{"key":"9720_CR1","doi-asserted-by":"publisher","first-page":"5649","DOI":"10.1007\/s00521-021-06702-3","volume":"34","author":"MN Alpdemir","year":"2022","unstructured":"Alpdemir MN (2022) Tactical UAV path optimization under radar threat using deep reinforcement learning. Neural Comput Appl 34:5649\u20135664","journal-title":"Neural Comput Appl"},{"key":"9720_CR2","doi-asserted-by":"publisher","first-page":"63","DOI":"10.1016\/j.neucom.2020.06.040","volume":"412","author":"H Liu","year":"2020","unstructured":"Liu H, Meng Q, Peng F, Lewis FL (2020) Heterogeneous formation control of multiple UAVs with limited-input leader via reinforcement learning. Neurocomputing 412:63\u201371","journal-title":"Neurocomputing"},{"key":"9720_CR3","doi-asserted-by":"publisher","first-page":"128","DOI":"10.1007\/s12559-019-09683-7","volume":"12","author":"K Zhou","year":"2020","unstructured":"Zhou K, Wei R, Xu Z (2020) An air combat decision learning system based on a brain-like cognitive mechanism. Cogn Comput 12:128\u2013139","journal-title":"Cogn Comput"},{"key":"9720_CR4","doi-asserted-by":"publisher","first-page":"883","DOI":"10.1109\/TRO.2018.2839087","volume":"34","author":"A Trotta","year":"2018","unstructured":"Trotta A, Felice MD, Montori F, Chowdhury KR, Bononi L (2018) Joint coverage, connectivity, and charging strategies for distributed UAV networks. IEEE Trans Robot 34:883\u2013900","journal-title":"IEEE Trans Robot"},{"key":"9720_CR5","doi-asserted-by":"publisher","first-page":"19765","DOI":"10.1007\/s00521-023-08784-7","volume":"35","author":"Z Sun","year":"2023","unstructured":"Sun Z, Wu H, Shi Y, Yu X, Gao Y, Pei W, Yang Z, Piao H, Hou Y (2023) Multi-agent air combat with two-stage graph-attention communication. Neural Comput Appl 35:19765\u201319781","journal-title":"Neural Comput Appl"},{"key":"9720_CR6","doi-asserted-by":"publisher","first-page":"305","DOI":"10.1016\/j.ast.2017.11.014","volume":"72","author":"H Shin","year":"2018","unstructured":"Shin H, Lee J, Kim H, Hyunchul Shim D (2018) An autonomous aerial combat framework for two-on-two engagements based on basic fighter maneuvers. Aerosp Sci Technol 72:305\u2013315","journal-title":"Aerosp Sci Technol"},{"key":"9720_CR7","doi-asserted-by":"publisher","first-page":"101","DOI":"10.1016\/j.neucom.2014.09.077","volume":"151","author":"Lope J Maravall","year":"2015","unstructured":"Maravall Lope J, Fuentes JP (2015) Vision-based anticipatory controller for the autonomous navigation of an UAV using artificial neural networks. Neurocomputing 151:101\u2013107","journal-title":"Neurocomputing"},{"key":"9720_CR8","doi-asserted-by":"publisher","first-page":"346","DOI":"10.1016\/j.neucom.2020.04.020","volume":"402","author":"X Dai","year":"2020","unstructured":"Dai X, Mao Y, Huang T (2020) Automatic obstacle avoidance of quadrotor UAV via CNN-based learning. Neurocomputing 402:346\u2013358","journal-title":"Neurocomputing"},{"key":"9720_CR9","doi-asserted-by":"publisher","DOI":"10.1016\/j.ast.2019.105534","volume":"96","author":"M Wang","year":"2020","unstructured":"Wang M, Wang L, Yue T, Liu H (2020) Influence of unmanned combat aerial vehicle agility on short-range aerial combat effectiveness. Aerosp Sci Technol 96:105534","journal-title":"Aerosp Sci Technol"},{"key":"9720_CR10","doi-asserted-by":"crossref","unstructured":"Zhou K, Wei R, Xu Z, Zhang Q (2018) (2018) A brain like air combat learning system inspired by human learning mechanism. In: Proceedings of IEEE CSAA guidance, navigation and control conference (CGNCC). IEEE, Xiamen, pp 1\u20136","DOI":"10.1109\/GNCC42960.2018.9018975"},{"key":"9720_CR11","doi-asserted-by":"crossref","unstructured":"Wang X, Guo K, Chao T, Wang S (2022) Design of differential game guidance law for dual defense aircrafts. In: Proceedings of 2022 5th international symposium on autonomous systems (ISAS). IEEE, Hangzhou, pp 1\u20136","DOI":"10.1109\/ISAS55863.2022.9757303"},{"key":"9720_CR12","doi-asserted-by":"crossref","unstructured":"Weintraub IE, Pachter M, Garcia E (2020) (2020) An introduction to pursuit-evasion differential games. In: Proceedings of American control conference (ACC). IEEE, Denver, pp 1049\u20131066","DOI":"10.23919\/ACC45564.2020.9147205"},{"key":"9720_CR13","doi-asserted-by":"publisher","first-page":"11619","DOI":"10.1109\/TII.2023.3248075","volume":"19","author":"W Ruan","year":"2023","unstructured":"Ruan W, Sun Y, Deng Y, Duan H (2023) Hawk-pigeon game tactics for unmanned aerial vehicle swarm target defense. IEEE Trans Ind Inform 19:11619\u201311629","journal-title":"IEEE Trans Ind Inform"},{"key":"9720_CR14","doi-asserted-by":"publisher","first-page":"11624","DOI":"10.1109\/ACCESS.2019.2933022","volume":"8","author":"Y Ma","year":"2020","unstructured":"Ma Y, Wang G, Hu X, Luo H, Lei X (2020) Cooperative occupancy decision making of multi-UAV in beyond-visual-range air combat: a game theory approach. IEEE Access 8:11624\u201311634","journal-title":"IEEE Access"},{"key":"9720_CR15","doi-asserted-by":"crossref","unstructured":"Kang Y, Pu Z, Liu Z (2020) (2020) Air-to-air combat tactical decision method based on SIRMs fuzzy logic and improved genetic algorithm. In: Proceedings of international conference on guidance, navigation and control (ICGNC). Springer, Tianjin, pp 3699\u20133709","DOI":"10.1007\/978-981-15-8155-7_308"},{"key":"9720_CR16","doi-asserted-by":"publisher","DOI":"10.1016\/j.eswa.2022.117448","volume":"203","author":"JB Crumpacker","year":"2022","unstructured":"Crumpacker JB, Robbins MJ, Jenkins PR (2022) An approximate dynamic programming approach for solving an air combat maneuvering problem. Expert Syst Appl 203:117448","journal-title":"Expert Syst Appl"},{"key":"9720_CR17","doi-asserted-by":"crossref","unstructured":"Sharma R (2014) (2014) Fuzzy Q learning based UAV autopilot. In: Proceedings of innovative applications of computational intelligence on power, energy and controls with their impact on humanity (CIPECH). IEEE, Ghaziabad, pp 29\u201333","DOI":"10.1109\/CIPECH.2014.7019067"},{"key":"9720_CR18","doi-asserted-by":"publisher","first-page":"409","DOI":"10.1016\/j.neucom.2015.11.046","volume":"177","author":"Y Liu","year":"2016","unstructured":"Liu Y, Liu W, Obaid MA, Abbas IA (2016) Exponential stability of Markovian jumping Cohen\u2013Grossberg neural networks with mixed mode-dependent time-delays. Neurocomputing 177:409\u2013415","journal-title":"Neurocomputing"},{"key":"9720_CR19","doi-asserted-by":"publisher","first-page":"448","DOI":"10.1016\/j.jfranklin.2015.11.013","volume":"353","author":"B Du","year":"2016","unstructured":"Du B, Liu Y, Atiatallah Abbas I (2016) Existence and asymptotic behavior results of periodic solution for discrete-time neutral-type neural networks. J Frankl Inst 353:448\u2013461","journal-title":"J Frankl Inst"},{"key":"9720_CR20","doi-asserted-by":"publisher","first-page":"16791","DOI":"10.1007\/s00521-022-07947-2","volume":"35","author":"R Emuna","year":"2022","unstructured":"Emuna R, Duffney R, Borowsky A, Biess A (2022) Example-guided learning of stochastic human driving policies using deep reinforcement learning. Neural Comput Appl 35:16791\u201316804","journal-title":"Neural Comput Appl"},{"key":"9720_CR21","doi-asserted-by":"publisher","first-page":"288","DOI":"10.1016\/j.ins.2022.11.057","volume":"619","author":"F Kiani","year":"2023","unstructured":"Kiani F, Sara\u00e7 \u00d6F (2023) A novel intelligent traffic recovery model for emergency vehicles based on context-aware reinforcement learning. Inf Sci 619:288\u2013309","journal-title":"Inf Sci"},{"key":"9720_CR22","doi-asserted-by":"publisher","first-page":"1293","DOI":"10.3390\/smartcities5040066","volume":"5","author":"S Damadam","year":"2022","unstructured":"Damadam S, Zourbakhsh M, Javidan R, Faroughi A (2022) An intelligent IoT based traffic light management system: deep reinforcement learning. Smart Cities 5:1293\u20131311","journal-title":"Smart Cities"},{"key":"9720_CR23","doi-asserted-by":"publisher","first-page":"509","DOI":"10.1016\/j.ins.2022.11.062","volume":"619","author":"R Zhu","year":"2023","unstructured":"Zhu R, Li L, Wu S, Lv P, Li Y, Xu M (2023) Multi-agent broad reinforcement learning for intelligent traffic light control. Inf Sci 619:509\u2013525","journal-title":"Inf Sci"},{"key":"9720_CR24","doi-asserted-by":"publisher","DOI":"10.1016\/j.energy.2020.117591","volume":"201","author":"G Du","year":"2020","unstructured":"Du G, Zou Y, Zhang X, Liu T, Wu J, He D (2020) Deep reinforcement learning based energy management for a hybrid electric vehicle. Energy 201:117591","journal-title":"Energy"},{"key":"9720_CR25","doi-asserted-by":"publisher","DOI":"10.1016\/j.conengprac.2023.105475","volume":"134","author":"D Yang","year":"2023","unstructured":"Yang D, Karimi HR, Pawelczyk M (2023) A new intelligent fault diagnosis framework for rotating machinery based on deep transfer reinforcement learning. Control Eng Pract 134:105475","journal-title":"Control Eng Pract"},{"key":"9720_CR26","doi-asserted-by":"publisher","first-page":"5723","DOI":"10.1109\/TVT.2020.2982508","volume":"69","author":"Q Liu","year":"2020","unstructured":"Liu Q, Shi L, Sun L, Li J, Ding M, Shu FS (2020) Path planning for UAV-mounted mobile edge computing with deep reinforcement learning. IEEE Trans Veh Technol 69:5723\u20135728","journal-title":"IEEE Trans Veh Technol"},{"key":"9720_CR27","doi-asserted-by":"publisher","first-page":"294","DOI":"10.1109\/TIV.2019.2955905","volume":"5","author":"C-J Hoel","year":"2020","unstructured":"Hoel C-J, Driggs-Campbell K, Wolff K, Laine L, Kochenderfer MJ (2020) Combining planning and deep reinforcement learning in tactical decision making for autonomous driving. IEEE Trans Intell Veh 5:294\u2013305","journal-title":"IEEE Trans Intell Veh"},{"key":"9720_CR28","doi-asserted-by":"publisher","first-page":"108759","DOI":"10.1016\/j.automatica.2019.108759","volume":"113","author":"AS Leong","year":"2020","unstructured":"Leong AS, Ramaswamy A, Quevedo DE, Karl H (2020) Deep reinforcement learning for wireless sensor scheduling in cyber-physical system. Automatic 113:108759","journal-title":"Automatic"},{"key":"9720_CR29","doi-asserted-by":"crossref","unstructured":"Liessner R, Schmitt J, Dietermann A, B\u00e4ker B (2019) Hyperparameter optimization for deep reinforcement learning in vehicle energy management. In: Proceedings of 11th international conference on agents artificial intelligence SCITEPRESS\u2014science and technology publications, Prague, pp 134\u2013144","DOI":"10.5220\/0007364701340144"},{"key":"9720_CR30","doi-asserted-by":"crossref","unstructured":"Chen Y, Zhang J, Yang Q, Zhou Y, Shi G, Wu Y (2020) Design and verification of UAV maneuver decision Simulation system based on deep Q-learning network. In: Proceedings of 2020 16th international conference on control, automation, robotics and vision (ICARCV). IEEE, Shenzhen, pp 817\u2013823","DOI":"10.1109\/ICARCV50220.2020.9305467"},{"key":"9720_CR31","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1155\/2023\/3657814","volume":"2023","author":"Y Cao","year":"2023","unstructured":"Cao Y, Kou Y-X, Li Z-W, Xu A (2023) Autonomous maneuver decision of UCAV air combat based on double deep Q network algorithm and stochastic game theory. Int J Aerosp Eng 2023:1\u201320","journal-title":"Int J Aerosp Eng"},{"key":"9720_CR32","doi-asserted-by":"publisher","first-page":"66","DOI":"10.1016\/j.dt.2022.08.010","volume":"29","author":"J Zhang","year":"2023","unstructured":"Zhang J, Yu Y, Zheng L, Yang Q, Shi G, Wu Y (2023) Situational continuity-based air combat autonomous maneuvering decision-making. Def Technol 29:66\u201379","journal-title":"Def Technol"},{"key":"9720_CR33","doi-asserted-by":"crossref","unstructured":"Yang Q, Zhu Y, Zhang J, Qiao S, Liu J (2019) UAV air combat autonomous maneuver decision based on DDPG algorithm. In: 2019 IEEE 15th international conference on control automation. ICCA. IEEE, Edinburgh, pp 37\u201342","DOI":"10.1109\/ICCA.2019.8899703"},{"key":"9720_CR34","doi-asserted-by":"publisher","first-page":"1421","DOI":"10.23919\/JSEE.2021.000121","volume":"32","author":"J Zhang","year":"2021","unstructured":"Zhang J, Yang Q, Shi G (2021) UAV cooperative air combat maneuver decision based on multi-agent reinforcement learning. J Syst Eng Electron 32:1421\u20131438","journal-title":"J Syst Eng Electron"},{"key":"9720_CR35","doi-asserted-by":"publisher","first-page":"182","DOI":"10.1016\/j.comcom.2023.01.009","volume":"200","author":"Z Wang","year":"2023","unstructured":"Wang Z, Guo Y, Li N, Hu S, Wang M (2023) Autonomous collaborative combat strategy of unmanned system group in continuous dynamic environment based on PD-MADDPG. Comput Commun 200:182\u2013204","journal-title":"Comput Commun"},{"key":"9720_CR36","first-page":"1","volume":"2023","author":"L Li","year":"2023","unstructured":"Li L, Zhang X, Qian C et al (2023) Basic flight maneuver generation of fixed-wing plane based on proximal policy optimization. Neural Comput Appl 2023:1\u201317","journal-title":"Neural Comput Appl"},{"key":"9720_CR37","doi-asserted-by":"publisher","first-page":"172988142198954","DOI":"10.1177\/1729881421989546","volume":"18","author":"Z Wang","year":"2021","unstructured":"Wang Z, Li H, Wu Z, Wu H (2021) A pretrained proximal policy optimization algorithm with reward shaping for aircraft guidance to a moving destination in three-dimensional continuous space. Int J Adv Robot Syst 18:172988142198954","journal-title":"Int J Adv Robot Syst"},{"key":"9720_CR38","doi-asserted-by":"publisher","first-page":"563","DOI":"10.3390\/aerospace9100563","volume":"9","author":"X Liu","year":"2022","unstructured":"Liu X, Yin Y, Su Y, Ming R (2022) A multi-UCAV cooperative decision-making method based on an MAPPO algorithm for beyond-visual-range air combat. Aerospace 9:563\u2013582","journal-title":"Aerospace"},{"key":"9720_CR39","doi-asserted-by":"publisher","DOI":"10.1016\/j.ast.2022.107857","volume":"129","author":"J Xu","year":"2022","unstructured":"Xu J, Zhang J, Yang L, Liu C (2022) Autonomous decision-making for dogfights based on a tactical pursuit point approach. Aerosp Sci Technol 129:107857","journal-title":"Aerosp Sci Technol"},{"key":"9720_CR40","doi-asserted-by":"publisher","first-page":"1608","DOI":"10.1049\/cit2.12195","volume":"8","author":"B Li","year":"2023","unstructured":"Li B, Bai S, Liang S, Ma R, Neretin E, Huang J (2023) Manoeuvre decision-making of unmanned aerial vehicles in air combat based on an expert actor-based soft actor critic algorithm. CAAI Trans Intell Technol 8:1608\u20131619","journal-title":"CAAI Trans Intell Technol"},{"key":"9720_CR41","doi-asserted-by":"publisher","first-page":"64","DOI":"10.1049\/cit2.12109","volume":"8","author":"B Li","year":"2023","unstructured":"Li B, Huang J, Bai S, Gan Z, Liang S, Evgeny N, Yao S (2023) Autonomous air combat decision-making of UAV based on parallel self-play reinforcement learning. CAAI Trans Intell Technol 8:64\u201381","journal-title":"CAAI Trans Intell Technol"},{"key":"9720_CR42","doi-asserted-by":"publisher","first-page":"86","DOI":"10.21629\/JSEE.2018.01.09","volume":"29","author":"C Huang","year":"2018","unstructured":"Huang C, Dong K, Huang H, Tang S (2018) Autonomous air combat maneuver decision using Bayesian inference and moving horizon optimization. J Syst Eng Electron 29:86\u201397","journal-title":"J Syst Eng Electron"},{"key":"9720_CR43","doi-asserted-by":"publisher","first-page":"43","DOI":"10.1080\/14702436.2022.2102486","volume":"23","author":"J Johnson","year":"2023","unstructured":"Johnson J (2023) Automating the OODA loop in the age of intelligent machines: reaffirming the role of humans in command-and-control decision-making in the digital age. Def Stud 23:43\u201367","journal-title":"Def Stud"},{"key":"9720_CR44","doi-asserted-by":"publisher","first-page":"881","DOI":"10.1016\/j.cja.2017.03.004","volume":"30","author":"LX Wang","year":"2017","unstructured":"Wang LX, Guo YG, Zhang Q, Yue T (2017) Suggestion for aircraft flying qualities requirements of a short-range air combat mission. Chin J Aeronaut 30:881\u2013897","journal-title":"Chin J Aeronaut"},{"key":"9720_CR45","doi-asserted-by":"publisher","first-page":"658","DOI":"10.3390\/aerospace9110658","volume":"9","author":"Y Li","year":"2022","unstructured":"Li Y, Lyu Y, Shi J, Li W (2022) Autonomous maneuver decision of air combat based on simulated operation command and FRV-DDPG algorithm. Aerospace 9:658\u2013676","journal-title":"Aerospace"},{"key":"9720_CR46","doi-asserted-by":"crossref","unstructured":"Austin F, Carbone G, Falco M, Hinz H, Lewis M (1987) Automated maneuvering decisions for air-to-air combat. In: Guidance, navigation and control conference, pp 2393","DOI":"10.2514\/6.1987-2393"}],"container-title":["Neural Computing and Applications"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s00521-024-09720-z.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s00521-024-09720-z\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s00521-024-09720-z.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,7,12]],"date-time":"2024-07-12T10:15:47Z","timestamp":1720779347000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s00521-024-09720-z"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,4,26]]},"references-count":46,"journal-issue":{"issue":"21","published-print":{"date-parts":[[2024,7]]}},"alternative-id":["9720"],"URL":"https:\/\/doi.org\/10.1007\/s00521-024-09720-z","relation":{},"ISSN":["0941-0643","1433-3058"],"issn-type":[{"value":"0941-0643","type":"print"},{"value":"1433-3058","type":"electronic"}],"subject":[],"published":{"date-parts":[[2024,4,26]]},"assertion":[{"value":"12 August 2023","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"25 March 2024","order":2,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"26 April 2024","order":3,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Declarations"}},{"value":"The authors declare that they have no conflict of interest.","order":2,"name":"Ethics","group":{"name":"EthicsHeading","label":"Conflict of interest"}},{"value":"This study belongs to the improvement and application innovation of reinforcement learning algorithms, so it does not involve ethical issues.","order":3,"name":"Ethics","group":{"name":"EthicsHeading","label":"Ethical approval"}},{"value":"All authors are aware of this paper and agree to its submission.","order":4,"name":"Ethics","group":{"name":"EthicsHeading","label":"Informed consent"}}]}}