{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,8,18]],"date-time":"2026-08-18T01:46:48Z","timestamp":1787017608677,"version":"build-2736575974"},"publisher-location":"New York, NY, USA","reference-count":42,"publisher":"ACM","content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2024,12,18]]},"DOI":"10.1145\/3719545.3759089","type":"proceedings-article","created":{"date-parts":[[2025,9,16]],"date-time":"2025-09-16T09:38:41Z","timestamp":1758015521000},"page":"71-80","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":1,"title":["A Comprehensive Evaluation Framework for Multi-Agent Reinforcement Learning"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0009-0002-7487-5945","authenticated-orcid":false,"given":"Zonglei","family":"Jing","sequence":"first","affiliation":[{"name":"Beihang University, Beijing, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0004-5774-0699","authenticated-orcid":false,"given":"Xiaojun","family":"Chang","sequence":"additional","affiliation":[{"name":"China Academy of Aerospace Science and Innovation, Beijing, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0006-0693-6098","authenticated-orcid":false,"given":"Mingxuan","family":"Zhu","sequence":"additional","affiliation":[{"name":"China Academy of Aerospace Science and Innovation, Beijing, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-7947-6743","authenticated-orcid":false,"given":"Simin","family":"Li","sequence":"additional","affiliation":[{"name":"Beihang University, Beijing, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-4224-1318","authenticated-orcid":false,"given":"Aishan","family":"Liu","sequence":"additional","affiliation":[{"name":"Beihang University, Beijing, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-4891-152X","authenticated-orcid":false,"given":"Xiaoqian","family":"Li","sequence":"additional","affiliation":[{"name":"Taishan University, Taian, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-8425-4195","authenticated-orcid":false,"given":"Xianglong","family":"Liu","sequence":"additional","affiliation":[{"name":"Beihang University, Beijing, China"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2025,9,16]]},"reference":[{"key":"e_1_3_3_1_2_2","first-page":"576","volume-title":"Proceedings of the 5th Conference on Robot Learning","author":"Batra Sumeet","year":"2022","unstructured":"Sumeet Batra, Zhehui Huang, Aleksei Petrenko, Tushar Kumar, Artem Molchanov, and Gaurav\u00a0S. Sukhatme. 2022. Decentralized Control of Quadrotor Swarms with End-to-end Deep Reinforcement Learning. In Proceedings of the 5th Conference on Robot Learning. PMLR, 576\u2013586."},{"key":"e_1_3_3_1_3_2","doi-asserted-by":"publisher","DOI":"10.1109\/SP.2017.49"},{"key":"e_1_3_3_1_4_2","unstructured":"Yuanpei Chen Tianhao Wu Shengjie Wang Xidong Feng Jiechuan Jiang Zongqing Lu Stephen McAleer Hao Dong Song-Chun Zhu and Yaodong Yang. 2022. Towards Human-Level Bimanual Dexterous Manipulation with Reinforcement Learning. Advances in Neural Information Processing Systems 35 (Dec. 2022) 5150\u20135163."},{"key":"e_1_3_3_1_5_2","doi-asserted-by":"publisher","unstructured":"Tianshu Chu Sandeep Chinchali and Sachin Katti. 2020. Multi-Agent Reinforcement Learning for Networked System Control. arxiv:https:\/\/arXiv.org\/abs\/2004.01339\u00a0[cs stat] 10.48550\/arXiv.2004.01339","DOI":"10.48550\/arXiv.2004.01339"},{"key":"e_1_3_3_1_6_2","unstructured":"Benjamin Ellis Skander Moalla Mikayel Samvelyan Mingfei Sun Anuj Mahajan Jakob\u00a0N. Foerster and Shimon Whiteson. 2022. SMACv2: An Improved Benchmark for Cooperative Multi-Agent Reinforcement Learning. arxiv:https:\/\/arXiv.org\/abs\/2212.07489\u00a0[cs]"},{"key":"e_1_3_3_1_7_2","doi-asserted-by":"publisher","unstructured":"Ian\u00a0J. Goodfellow Jonathon Shlens and Christian Szegedy. 2015. Explaining and Harnessing Adversarial Examples. arxiv:https:\/\/arXiv.org\/abs\/1412.6572\u00a0[cs stat] 10.48550\/arXiv.1412.6572","DOI":"10.48550\/arXiv.1412.6572"},{"key":"e_1_3_3_1_8_2","doi-asserted-by":"publisher","DOI":"10.14459\/2022MD1639974"},{"key":"e_1_3_3_1_9_2","doi-asserted-by":"publisher","unstructured":"Jun Guo Wei Bao Jiakai Wang Yuqing Ma Xinghai Gao Gang Xiao Aishan Liu Jian Dong Xianglong Liu and Wenjun Wu. 2023. A Comprehensive Evaluation Framework for Deep Model Robustness. Pattern Recognition 137 (May 2023) 109308. 10.1016\/j.patcog.2023.109308","DOI":"10.1016\/j.patcog.2023.109308"},{"key":"e_1_3_3_1_10_2","doi-asserted-by":"publisher","DOI":"10.1109\/CVPRW56347.2022.00022"},{"key":"e_1_3_3_1_11_2","doi-asserted-by":"publisher","unstructured":"Yi Han Benjamin I.\u00a0P. Rubinstein Tamas Abraham Tansu Alpcan Olivier De\u00a0Vel Sarah Erfani David Hubczenko Christopher Leckie and Paul Montague. 2018. Reinforcement Learning for Autonomous Defence in Software-Defined Networking. arxiv:https:\/\/arXiv.org\/abs\/1808.05770\u00a0[cs stat] 10.48550\/arXiv.1808.05770","DOI":"10.48550\/arXiv.1808.05770"},{"key":"e_1_3_3_1_12_2","unstructured":"Siyi Hu Yifan Zhong Minquan Gao Weixun Wang Hao Dong Zhihui Li Xiaodan Liang Xiaojun Chang and Yaodong Yang. 2022. MARLlib: Extending RLlib for Multi-agent Reinforcement Learning. arxiv:https:\/\/arXiv.org\/abs\/2210.13708\u00a0[cs]"},{"key":"e_1_3_3_1_13_2","doi-asserted-by":"publisher","unstructured":"Sandy Huang Nicolas Papernot Ian Goodfellow Yan Duan and Pieter Abbeel. 2017. Adversarial Attacks on Neural Network Policies. arxiv:https:\/\/arXiv.org\/abs\/1702.0228410.48550\/arXiv.1702.02284","DOI":"10.48550\/arXiv.1702.02284"},{"key":"e_1_3_3_1_14_2","doi-asserted-by":"publisher","unstructured":"Zonglei Jing Xiaoqian Li Peijun Ju and Huanshui Zhang. 2024. Optimal control and filtering for hierarchical decision problems with h_\u0131nfty constraint based on stackelberg strategy. IEEE Trans. Automat. Control 69 9 (Sept. 2024) 6238\u20136245. 10.1109\/TAC.2024.3374487","DOI":"10.1109\/TAC.2024.3374487"},{"key":"e_1_3_3_1_15_2","doi-asserted-by":"publisher","unstructured":"Karol Kurach Anton Raichuk Piotr Sta\u0144czyk Micha\u0142 Zaj\u0105c Olivier Bachem Lasse Espeholt Carlos Riquelme Damien Vincent Marcin Michalski Olivier Bousquet and Sylvain Gelly. 2020. Google Research Football: A Novel Reinforcement Learning Environment. arxiv:https:\/\/arXiv.org\/abs\/1907.11180\u00a0[cs stat] 10.48550\/arXiv.1907.11180","DOI":"10.48550\/arXiv.1907.11180"},{"key":"e_1_3_3_1_16_2","unstructured":"Jan Leike Miljan Martic Victoria Krakovna Pedro\u00a0A. Ortega Tom Everitt Andrew Lefrancq Laurent Orseau and Shane Legg. 2017. AI Safety Gridworlds. arxiv:https:\/\/arXiv.org\/abs\/1711.09883\u00a0[cs]"},{"key":"e_1_3_3_1_17_2","unstructured":"Quanyi Li Zhenghao Peng Lan Feng Qihang Zhang Zhenghai Xue and Bolei Zhou. 2022. MetaDrive: Composing Diverse Driving Scenarios for Generalizable Reinforcement Learning. arxiv:https:\/\/arXiv.org\/abs\/2109.12674\u00a0[cs]"},{"key":"e_1_3_3_1_18_2","unstructured":"Simin Li Jun Guo Jingqiao Xiu Pu Feng Xin Yu Aishan Liu Wenjun Wu and Xianglong Liu. 2023. Attacking cooperative multi-agent reinforcement learning by adversarial minority influence. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2302.03322 (2023)."},{"key":"e_1_3_3_1_19_2","unstructured":"Simin Li Jun Guo Jingqiao Xiu Ruixiao Xu Xin Yu Jiakai Wang Aishan Liu Yaodong Yang and Xianglong Liu. 2023. Byzantine robust cooperative multi-agent reinforcement learning as a bayesian game. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2305.12872 (2023)."},{"key":"e_1_3_3_1_20_2","unstructured":"Simin Li Ruixiao Xu Jun Guo Pu Feng Jiakai Wang Aishan Liu Yaodong Yang Xianglong Liu and Weifeng Lv. 2023. MIR2: Towards Provably Robust Multi-Agent Reinforcement Learning by Mutual Information Regularization. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2310.09833 (2023)."},{"key":"e_1_3_3_1_21_2","doi-asserted-by":"publisher","DOI":"10.23919\/CCC63176.2024.10662305"},{"key":"e_1_3_3_1_22_2","unstructured":"Eric Liang Richard Liaw Philipp Moritz Robert Nishihara Roy Fox Ken Goldberg Joseph\u00a0E. Gonzalez Michael\u00a0I. Jordan and Ion Stoica. 2018. RLlib: Abstractions for Distributed Reinforcement Learning. arxiv:https:\/\/arXiv.org\/abs\/1712.09381\u00a0[cs]"},{"key":"e_1_3_3_1_23_2","unstructured":"Ryan Lowe Yi Wu Aviv Tamar Jean Harb Pieter Abbeel and Igor Mordatch. 2020. Multi-Agent Actor-Critic for Mixed Cooperative-Competitive Environments. arxiv:https:\/\/arXiv.org\/abs\/1706.02275"},{"key":"e_1_3_3_1_24_2","doi-asserted-by":"publisher","unstructured":"Aleksander Madry Aleksandar Makelov Ludwig Schmidt Dimitris Tsipras and Adrian Vladu. 2019. Towards Deep Learning Models Resistant to Adversarial Attacks. arXiv:https:\/\/arXiv.org\/abs\/1706.06083 [cs stat] (Sept. 2019). arxiv:https:\/\/arXiv.org\/abs\/1706.06083\u00a0[cs stat] 10.48550\/ARXIV.1706.06083","DOI":"10.48550\/ARXIV.1706.06083"},{"key":"e_1_3_3_1_25_2","doi-asserted-by":"publisher","unstructured":"Volodymyr Mnih Koray Kavukcuoglu David Silver Alex Graves Ioannis Antonoglou Daan Wierstra and Martin Riedmiller. 2013. Playing Atari with Deep Reinforcement Learning. arxiv:https:\/\/arXiv.org\/abs\/1312.560210.48550\/arXiv.1312.5602","DOI":"10.48550\/arXiv.1312.5602"},{"key":"e_1_3_3_1_26_2","doi-asserted-by":"crossref","unstructured":"Igor Mordatch and Pieter Abbeel. 2018. Emergence of Grounded Compositional Language in Multi-Agent Populations. arxiv:https:\/\/arXiv.org\/abs\/1703.04908\u00a0[cs]","DOI":"10.1609\/aaai.v32i1.11492"},{"key":"e_1_3_3_1_27_2","doi-asserted-by":"publisher","unstructured":"Georgios Papoudakis Filippos Christianos Arrasy Rahman and Stefano\u00a0V. Albrecht. 2019. Dealing with Non-Stationarity in Multi-Agent Deep Reinforcement Learning. arxiv:https:\/\/arXiv.org\/abs\/1906.04737\u00a0[cs stat] 10.48550\/arXiv.1906.04737","DOI":"10.48550\/arXiv.1906.04737"},{"key":"e_1_3_3_1_28_2","doi-asserted-by":"publisher","unstructured":"Georgios Papoudakis Filippos Christianos Lukas Sch\u00e4fer and Stefano\u00a0V. Albrecht. 2020. Benchmarking Multi-Agent Deep Reinforcement Learning Algorithms in Cooperative Tasks. 10.48550\/arXiv.2006.07869","DOI":"10.48550\/arXiv.2006.07869"},{"key":"e_1_3_3_1_29_2","doi-asserted-by":"publisher","unstructured":"Bei Peng Tabish Rashid Christian A.\u00a0Schroeder De\u00a0Witt Pierre-Alexandre Kamienny Philip H.\u00a0S. Torr Wendelin B\u00f6hmer and Shimon Whiteson. 2021. FACMAC: Factored Multi-Agent Centralised Policy Gradients. arxiv:https:\/\/arXiv.org\/abs\/2003.06709\u00a0[cs stat] 10.48550\/arXiv.2003.06709","DOI":"10.48550\/arXiv.2003.06709"},{"key":"e_1_3_3_1_30_2","doi-asserted-by":"publisher","unstructured":"Tabish Rashid Mikayel Samvelyan Christian\u00a0Schroeder de Witt Gregory Farquhar Jakob Foerster and Shimon Whiteson. 2018. QMIX: Monotonic Value Function Factorisation for Deep Multi-Agent Reinforcement Learning. arxiv:https:\/\/arXiv.org\/abs\/1803.11485\u00a0[cs stat] 10.48550\/arXiv.1803.11485","DOI":"10.48550\/arXiv.1803.11485"},{"key":"e_1_3_3_1_31_2","unstructured":"Mikayel Samvelyan Tabish Rashid Christian\u00a0Schroeder De\u00a0Witt Gregory Farquhar Nantas Nardelli Tim G.\u00a0J. Rudner Chia-Man Hung Philip H.\u00a0S. Torr Jakob Foerster and Shimon Whiteson. 2019. The StarCraft Multi-Agent Challenge. arxiv:https:\/\/arXiv.org\/abs\/1902.04043\u00a0[cs stat]"},{"key":"e_1_3_3_1_32_2","unstructured":"Peter Sunehag Guy Lever Audrunas Gruslys Wojciech\u00a0Marian Czarnecki Vinicius Zambaldi Max Jaderberg Marc Lanctot Nicolas Sonnerat Joel\u00a0Z. Leibo Karl Tuyls and Thore Graepel. 2017. Value-Decomposition Networks For Cooperative Multi-Agent Learning. arxiv:https:\/\/arXiv.org\/abs\/1706.05296\u00a0[cs]"},{"key":"e_1_3_3_1_33_2","doi-asserted-by":"publisher","DOI":"10.5555\/3091529.3091572"},{"key":"e_1_3_3_1_34_2","unstructured":"Shiyu Tang Ruihao Gong Yan Wang Aishan Liu Jiakai Wang Xinyun Chen Fengwei Yu Xianglong Liu Dawn Song Alan Yuille Philip H.\u00a0S. Torr and Dacheng Tao. 2022. RobustART: Benchmarking Robustness on Architecture Design and Training Techniques. arxiv:https:\/\/arXiv.org\/abs\/2109.05211\u00a0[cs]"},{"key":"e_1_3_3_1_35_2","doi-asserted-by":"publisher","DOI":"10.1109\/IROS.2012.6386109"},{"key":"e_1_3_3_1_36_2","doi-asserted-by":"publisher","unstructured":"Jianhong Wang Wangkun Xu Yunjie Gu Wenbin Song and Tim\u00a0C. Green. 2022. Multi-Agent Reinforcement Learning for Active Voltage Control on Power Distribution Networks. arxiv:https:\/\/arXiv.org\/abs\/2110.14300\u00a0[cs] 10.48550\/arXiv.2110.14300","DOI":"10.48550\/arXiv.2110.14300"},{"key":"e_1_3_3_1_37_2","doi-asserted-by":"publisher","unstructured":"Yaodong Yang and Jun Wang. 2021. An Overview of Multi-Agent Reinforcement Learning from Game Theoretical Perspective. arxiv:https:\/\/arXiv.org\/abs\/2011.0058310.48550\/arXiv.2011.00583","DOI":"10.48550\/arXiv.2011.00583"},{"key":"e_1_3_3_1_38_2","doi-asserted-by":"publisher","unstructured":"Chao Yu Akash Velu Eugene Vinitsky Jiaxuan Gao Yu Wang Alexandre Bayen and Yi Wu. 2022. The Surprising Effectiveness of PPO in Cooperative Multi-Agent Games. arxiv:https:\/\/arXiv.org\/abs\/2103.01955\u00a0[cs] 10.48550\/arXiv.2103.01955","DOI":"10.48550\/arXiv.2103.01955"},{"key":"e_1_3_3_1_39_2","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v38i16.29709"},{"key":"e_1_3_3_1_40_2","doi-asserted-by":"publisher","DOI":"10.3233\/FAIA230609"},{"key":"e_1_3_3_1_41_2","doi-asserted-by":"publisher","DOI":"10.1109\/ICRA57147.2024.10611035"},{"key":"e_1_3_3_1_42_2","doi-asserted-by":"publisher","unstructured":"Yifan Zhong Jakub\u00a0Grudzien Kuba Siyi Hu Jiaming Ji and Yaodong Yang. 2023. Heterogeneous-Agent Reinforcement Learning. arxiv:https:\/\/arXiv.org\/abs\/2304.09870\u00a0[cs] 10.48550\/arXiv.2304.09870","DOI":"10.48550\/arXiv.2304.09870"},{"key":"e_1_3_3_1_43_2","unstructured":"Yihe Zhou Shunyu Liu Yunpeng Qing Kaixuan Chen Tongya Zheng Yanhao Huang Jie Song and Mingli Song. 2023. Is Centralized Training with Decentralized Execution Framework Centralized Enough for MARL? arxiv:https:\/\/arXiv.org\/abs\/2305.17352\u00a0[cs]"}],"event":{"name":"DAI '24: 6th International Conference on Distributed Artificial Intelligences","location":"Singapore Singapore","acronym":"DAI '24"},"container-title":["Proceedings of the 2024 Sixth International Conference on Distributed Artificial Intelligences"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3719545.3759089","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,9,17]],"date-time":"2025-09-17T13:12:50Z","timestamp":1758114770000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3719545.3759089"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,12,18]]},"references-count":42,"alternative-id":["10.1145\/3719545.3759089","10.1145\/3719545"],"URL":"https:\/\/doi.org\/10.1145\/3719545.3759089","relation":{},"subject":[],"published":{"date-parts":[[2024,12,18]]},"assertion":[{"value":"2025-09-16","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}