{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,5,15]],"date-time":"2026-05-15T06:47:23Z","timestamp":1778827643966,"version":"3.51.4"},"reference-count":112,"publisher":"Institute of Electrical and Electronics Engineers (IEEE)","issue":"2","license":[{"start":{"date-parts":[[2022,4,1]],"date-time":"2022-04-01T00:00:00Z","timestamp":1648771200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/ieeexplore.ieee.org\/Xplorehelp\/downloads\/license-information\/IEEE.html"},{"start":{"date-parts":[[2022,4,1]],"date-time":"2022-04-01T00:00:00Z","timestamp":1648771200000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2022,4,1]],"date-time":"2022-04-01T00:00:00Z","timestamp":1648771200000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"}],"funder":[{"DOI":"10.13039\/100008982","name":"Qatar National Research Fund","doi-asserted-by":"publisher","id":[{"id":"10.13039\/100008982","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100007458","name":"Qatar Foundation","doi-asserted-by":"publisher","id":[{"id":"10.13039\/100007458","id-type":"DOI","asserted-by":"publisher"}]},{"name":"National Priorities Research Program","award":["13S-0206-200273"],"award-info":[{"award-number":["13S-0206-200273"]}]}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["IEEE Trans. Artif. Intell."],"published-print":{"date-parts":[[2022,4]]},"DOI":"10.1109\/tai.2021.3111139","type":"journal-article","created":{"date-parts":[[2021,9,13]],"date-time":"2021-09-13T17:21:35Z","timestamp":1631553695000},"page":"90-109","source":"Crossref","is-referenced-by-count":128,"title":["Challenges and Countermeasures for Adversarial Attacks on Deep Reinforcement Learning"],"prefix":"10.1109","volume":"3","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-6384-7466","authenticated-orcid":false,"given":"Inaam","family":"Ilahi","sequence":"first","affiliation":[{"name":"Information Technology University, Lahore, Pakistan"}],"role":[{"role":"author","vocab":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-5015-8605","authenticated-orcid":false,"given":"Muhammad","family":"Usama","sequence":"additional","affiliation":[{"name":"Information Technology University, Lahore, Pakistan"}],"role":[{"role":"author","vocab":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-9466-2475","authenticated-orcid":false,"given":"Junaid","family":"Qadir","sequence":"additional","affiliation":[{"name":"Information Technology University, Lahore, Pakistan"}],"role":[{"role":"author","vocab":"crossref"}]},{"given":"Muhammad Umar","family":"Janjua","sequence":"additional","affiliation":[{"name":"Information Technology University, Lahore, Pakistan"}],"role":[{"role":"author","vocab":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-0903-1204","authenticated-orcid":false,"given":"Ala","family":"Al-Fuqaha","sequence":"additional","affiliation":[{"name":"Hamad Bin Khalifa University, Doha, Qatar"}],"role":[{"role":"author","vocab":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-9528-0863","authenticated-orcid":false,"given":"Dinh Thai","family":"Hoang","sequence":"additional","affiliation":[{"name":"University of Technology Sydney, Ultimo, NSW, Australia"}],"role":[{"role":"author","vocab":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-7442-7416","authenticated-orcid":false,"given":"Dusit","family":"Niyato","sequence":"additional","affiliation":[{"name":"Nanyang Technological University, Singapore"}],"role":[{"role":"author","vocab":"crossref"}]}],"member":"263","reference":[{"key":"ref1","volume-title":"Learning From Data","volume":"4","author":"Abu-Mostafa","year":"2012"},{"key":"ref2","doi-asserted-by":"publisher","DOI":"10.1109\/MSP.2017.2743240"},{"key":"ref3","doi-asserted-by":"publisher","DOI":"10.1109\/ICRA.2017.7989385"},{"key":"ref4","doi-asserted-by":"publisher","DOI":"10.1145\/3072959.3073602"},{"key":"ref5","doi-asserted-by":"publisher","DOI":"10.1109\/URAI.2018.8441797"},{"key":"ref6","first-page":"147","article-title":"Continuous state-space models for optimal sepsis treatment-a deep reinforcement learning approach","volume-title":"Proc. Mach. Learn. Healthcare","author":"Raghu","year":"2017"},{"key":"ref7","doi-asserted-by":"publisher","DOI":"10.1109\/TNNLS.2016.2522401"},{"key":"ref8","article-title":"Deep reinforcement learning solutions for energy microgrids management","volume-title":"Proc. Eur. Workshop Reinforcement Learn.","author":"Franois-Lavet","year":"2016"},{"key":"ref9","doi-asserted-by":"publisher","DOI":"10.1016\/j.scs.2016.12.012"},{"key":"ref10","doi-asserted-by":"publisher","DOI":"10.1109\/COMST.2019.2916583"},{"key":"ref11","doi-asserted-by":"publisher","DOI":"10.1038\/nature14236"},{"key":"ref12","doi-asserted-by":"publisher","DOI":"10.1038\/nature16961"},{"key":"ref13","doi-asserted-by":"publisher","DOI":"10.1126\/science.aar6404"},{"key":"ref14","article-title":"Dota 2 with large scale deep reinforcement learning","author":"Berner","year":"2019"},{"key":"ref15","doi-asserted-by":"publisher","DOI":"10.1109\/TITS.2021.3054625"},{"key":"ref16","doi-asserted-by":"publisher","DOI":"10.3905\/jfds.2020.1.030"},{"key":"ref17","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-62416-7_19"},{"key":"ref18","doi-asserted-by":"publisher","DOI":"10.1109\/ACCESS.2018.2807385"},{"key":"ref19","article-title":"The faults in our pi stars: Security issues and open challenges in deep reinforcement learning","author":"Behzadan","year":"2018"},{"key":"ref21","doi-asserted-by":"publisher","DOI":"10.1201\/9781351006620-6"},{"key":"ref22","article-title":"The space of transferable adversarial examples","author":"Tramr","year":"2017"},{"key":"ref23","article-title":"Query-efficient hard-label black-box attack: An optimization-based approach","author":"Cheng","year":"2018"},{"key":"ref24","doi-asserted-by":"publisher","DOI":"10.1145\/3128572.3140448"},{"key":"ref25","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v33i01.3301742"},{"key":"ref26","doi-asserted-by":"publisher","DOI":"10.1109\/TNNLS.2019.2933524"},{"key":"ref27","doi-asserted-by":"publisher","DOI":"10.1109\/COMST.2020.2975048"},{"key":"ref28","doi-asserted-by":"publisher","DOI":"10.1109\/TrustCom\/BigDataSE.2018.00079"},{"key":"ref29","doi-asserted-by":"publisher","DOI":"10.2200\/S00861ED1V01Y201806AIM039"},{"key":"ref30","doi-asserted-by":"publisher","DOI":"10.1109\/EuroSP.2016.36"},{"key":"ref31","article-title":"Adversarial attacks on neural network policies","volume-title":"Proc. Int. Conf. Learn. Represent. Workshop","author":"Huang","year":"2017"},{"key":"ref32","doi-asserted-by":"publisher","DOI":"10.24963\/ijcai.2017\/525"},{"key":"ref33","doi-asserted-by":"publisher","DOI":"10.1109\/SP.2017.49"},{"key":"ref34","article-title":"Sequential attacks on agents for long-term adversarial goals","author":"Tretschk","year":"2018"},{"key":"ref35","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v32i1.11672"},{"key":"ref36","first-page":"2040","article-title":"Robust deep reinforcement learning with adversarial attacks","volume-title":"Proc. 17th Int. Conf. Auton. Agents MultiAgent Syst. Int. Found. Auton. Agents Multiagent Syst.","author":"Pattanaik","year":"2018"},{"key":"ref37","article-title":"Delving into adversarial attacks on deep policies","volume-title":"Proc. Int. Conf. Learn. Represent. Workshop","author":"Kos","year":"2017"},{"key":"ref38","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v34i04.6047"},{"key":"ref39","article-title":"Targeted attacks on deep reinforcement learning agents through adversarial observations","volume-title":"Auton. Agents Multi-Agent Syst.","author":"Hussenot","year":"2020"},{"key":"ref40","doi-asserted-by":"publisher","DOI":"10.1145\/3320269.3384715"},{"key":"ref41","article-title":"Cooperative multi-agent reinforcement learning for low-level wireless communication","author":"Vrieze","year":"2018"},{"key":"ref42","first-page":"1151","article-title":"Multi-agent reinforcement learning for traffic light control","volume-title":"Proc. 17th Int. Conf. Mach. Learn.","author":"Wiering","year":"2000"},{"key":"ref43","article-title":"Safe, multi-agent, reinforcement learning for autonomous driving","author":"Shalev-Shwartz","year":"2016"},{"key":"ref44","doi-asserted-by":"publisher","DOI":"10.1109\/SPW50608.2020.00027"},{"key":"ref45","article-title":"Adversarial attacks on reinforcement learning based energy management systems of extended range electric delivery vehicles","author":"Wang","year":"2020"},{"key":"ref46","article-title":"Characterizing attacks on deep reinforcement learning","author":"Xiao","year":"2019"},{"key":"ref47","article-title":"Gradient band-based adversarial training for generalized attack immunity of A3C path finding","author":"Chen","year":"2018"},{"key":"ref48","doi-asserted-by":"publisher","DOI":"10.1109\/DSC.2018.00126"},{"key":"ref49","article-title":"Adversarial policies: Attacking deep reinforcement learning","volume-title":"Proc. Int. Conf. Learn. Represent.","author":"Gleave","year":"2020"},{"key":"ref50","doi-asserted-by":"publisher","DOI":"10.1109\/ICASSP40776.2020.9053342"},{"key":"ref51","doi-asserted-by":"publisher","DOI":"10.1109\/DAC18072.2020.9218663"},{"key":"ref52","article-title":"Sequential triggers for watermarking of deep reinforcement learning policies","author":"Behzadan","year":"2019"},{"key":"ref53","doi-asserted-by":"publisher","DOI":"10.1109\/MIC.2021.3049190"},{"key":"ref54","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-01554-1_9"},{"key":"ref55","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-32430-8_14"},{"key":"ref56","first-page":"7974","article-title":"Policy teaching via environment poisoning: Training-time adversarial attacks against reinforcement learning","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Rakhsha","year":"2020"},{"key":"ref57","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v34i04.5887"},{"key":"ref58","article-title":"Adversarial exploitation of policy imitation","author":"Behzadan","year":"2019"},{"key":"ref59","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v32i1.11796"},{"key":"ref60","doi-asserted-by":"publisher","DOI":"10.5244\/C.31.11"},{"key":"ref61","doi-asserted-by":"publisher","DOI":"10.1145\/3433210.3453090"},{"key":"ref62","article-title":"Sample efficient actor-critic with experience replay","volume-title":"Proc. Int. Conf. Learn. Represent.","author":"Wang","year":"2017"},{"key":"ref63","first-page":"5279","article-title":"Scalable trust-region method for deep reinforcement learning using Kronecker-factored approximation","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Wu","year":"2017"},{"key":"ref64","doi-asserted-by":"publisher","DOI":"10.1145\/3394486.3403089"},{"key":"ref65","doi-asserted-by":"publisher","DOI":"10.1145\/3450267.3450537"},{"key":"ref66","article-title":"Benchmarking safe exploration in deep reinforcement learning","author":"Ray","year":"2019"},{"key":"ref67","article-title":"Whatever does not kill deep reinforcement learning, makes it stronger","author":"Behzadan","year":"2017"},{"key":"ref68","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-99229-7_34"},{"key":"ref69","article-title":"Noisy networks for exploration","volume-title":"Proc. Int. Conf. Learn. Represent.","author":"Fortunato","year":"2018"},{"key":"ref70","article-title":"Analysis and improvement of adversarial training in DQN agents with adversarially-guided exploration (AGE)","author":"Behzadan","year":"2019"},{"key":"ref71","doi-asserted-by":"publisher","DOI":"10.23919\/ACC45564.2020.9147846"},{"key":"ref72","article-title":"Robust reinforcement learning using adversarial populations","author":"Vinitsky","year":"2020"},{"key":"ref73","first-page":"2817","article-title":"Robust adversarial reinforcement learning","volume-title":"Proc. 34th Int. Conf. Mach. Learn.","author":"Pinto","year":"2017"},{"key":"ref74","article-title":"Wasserstein robust reinforcement learning","author":"Abdullah","year":"2019"},{"key":"ref75","doi-asserted-by":"publisher","DOI":"10.1016\/j.geb.2016.06.004"},{"key":"ref76","doi-asserted-by":"publisher","DOI":"10.1109\/IROS.2018.8594037"},{"key":"ref77","doi-asserted-by":"publisher","DOI":"10.1109\/IROS.2017.8206245"},{"key":"ref78","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v34i04.6086"},{"key":"ref79","article-title":"Distributionally robust reinforcement learning","volume-title":"Proc. Int. Conf. Mach. Learn. Workshop","author":"Smirnova","year":"2019"},{"key":"ref80","article-title":"Action robust reinforcement learning and applications in continuous control","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Tessler","year":"2019"},{"key":"ref81","article-title":"Enhancing performance of reinforcement learning models in the presence of noisy rewards","author":"Kumar","year":"2019"},{"key":"ref82","article-title":"Online robustness training for deep reinforcement learning","author":"Fischer","year":"2019"},{"key":"ref83","doi-asserted-by":"publisher","DOI":"10.1109\/ICRA.2019.8794293"},{"key":"ref84","first-page":"4026","article-title":"Deep exploration via bootstrapped DQN","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Osband","year":"2016"},{"key":"ref85","doi-asserted-by":"publisher","DOI":"10.1109\/ICMLA51294.2020.00042"},{"key":"ref86","first-page":"1328","article-title":"Certified adversarial robustness for deep reinforcement learning","volume-title":"Proc. Conf. Robot Learn.","author":"Ltjens","year":"2020"},{"key":"ref87","first-page":"21024","article-title":"Robust deep reinforcement learning against adversarial perturbations on state observations","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Zhang","year":"2020"},{"key":"ref88","article-title":"Robust deep reinforcement learning through adversarial loss","author":"Oikarinen","year":"2020"},{"key":"ref89","article-title":"Robust reinforcement learning on state observations with learned optimal adversary","author":"Zhang","year":"2021"},{"key":"ref90","article-title":"Detecting adversarial attacks on neural network policies with visual foresight","author":"Lin","year":"2017"},{"key":"ref91","doi-asserted-by":"publisher","DOI":"10.1201\/9781351251389-8"},{"key":"ref92","first-page":"9916","article-title":"Online robust policy learning in the presence of unknown adversaries","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Havens","year":"2018"},{"key":"ref93","doi-asserted-by":"publisher","DOI":"10.1109\/DSC.2018.00125"},{"key":"ref94","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v33i01.33019939"},{"key":"ref95","doi-asserted-by":"publisher","DOI":"10.1109\/SP.2016.41"},{"key":"ref96","article-title":"Defensive distillation is not robust to adversarial examples","author":"Carlini","year":"2016"},{"key":"ref97","article-title":"Policy distillation","volume-title":"Proc. Int. Conf. Learn. Represent.","author":"Rusu","year":"2016"},{"key":"ref98","first-page":"1331","article-title":"Distilling policy distillation","volume-title":"Proc. Mach. Learn. Res.","author":"Czarnecki","year":"2019"},{"key":"ref99","article-title":"Defending adversarial attacks without adversarial attacks in deep reinforcement learning","author":"Qu","year":"2020"},{"key":"ref100","doi-asserted-by":"publisher","DOI":"10.1109\/MITS.2019.2898964"},{"key":"ref101","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-26250-1_25"},{"key":"ref102","article-title":"OpenAI gym","author":"Brockman","year":"2016"},{"key":"ref103","article-title":"TensorFlow: Large-scale machine learning on heterogeneous systems","author":"Abadi","year":"2015"},{"key":"ref104","article-title":"OpenAI Baselines","author":"Dhariwal","year":"2017"},{"key":"ref105","article-title":"Reinforcement learning coach","author":"Caspi","year":"2017"},{"key":"ref106","article-title":"Horizon: Facebooks open source applied reinforcement learning platform","author":"Gauci","year":"2019"},{"key":"ref108","doi-asserted-by":"publisher","DOI":"10.1145\/3345768.3355908"},{"key":"ref109","article-title":"Technical report on the CleverHans v2.1.0 adversarial examples library","author":"Papernot","year":"2018"},{"key":"ref110","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2017.17"},{"key":"ref111","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-57321-8_5"},{"key":"ref112","doi-asserted-by":"publisher","DOI":"10.1016\/j.knosys.2020.106685"},{"key":"ref113","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-01424-7_27"},{"key":"ref114","doi-asserted-by":"publisher","DOI":"10.1109\/TPAMI.2023.3292075"}],"container-title":["IEEE Transactions on Artificial Intelligence"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx7\/9078688\/9741091\/09536399.pdf?arnumber=9536399","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,8,23]],"date-time":"2025-08-23T01:10:06Z","timestamp":1755911406000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/9536399\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2022,4]]},"references-count":112,"journal-issue":{"issue":"2"},"URL":"https:\/\/doi.org\/10.1109\/tai.2021.3111139","relation":{},"ISSN":["2691-4581"],"issn-type":[{"value":"2691-4581","type":"electronic"}],"subject":[],"published":{"date-parts":[[2022,4]]}}}