{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,11,3]],"date-time":"2025-11-03T09:23:44Z","timestamp":1762161824827},"reference-count":56,"publisher":"Springer Science and Business Media LLC","issue":"1","license":[{"start":{"date-parts":[[2023,11,16]],"date-time":"2023-11-16T00:00:00Z","timestamp":1700092800000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2023,11,16]],"date-time":"2023-11-16T00:00:00Z","timestamp":1700092800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["SN COMPUT. SCI."],"DOI":"10.1007\/s42979-023-02326-7","type":"journal-article","created":{"date-parts":[[2023,11,16]],"date-time":"2023-11-16T08:02:30Z","timestamp":1700121750000},"update-policy":"http:\/\/dx.doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":1,"title":["Bias Estimation Correction in Multi-Agent Reinforcement Learning for Mixed Cooperative-Competitive Environments"],"prefix":"10.1007","volume":"5","author":[{"given":"Tamal","family":"Sarkar","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Shobhanjana","family":"Kalita","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2023,11,16]]},"reference":[{"key":"2326_CR1","doi-asserted-by":"publisher","first-page":"237","DOI":"10.1613\/jair.301","volume":"4","author":"LP Kaelbling","year":"1996","unstructured":"Kaelbling LP, Littman ML, Moore AW. Reinforcement learning: a survey. J Artif Intell Res. 1996;4:237\u201385.","journal-title":"J Artif Intell Res"},{"key":"2326_CR2","doi-asserted-by":"publisher","first-page":"28573","DOI":"10.1109\/ACCESS.2018.2831228","volume":"6","author":"A Dorri","year":"2018","unstructured":"Dorri A, Kanhere SS, Jurdak R. Multi-agent systems: a survey. IEEE Access. 2018;6:28573\u201393.","journal-title":"IEEE Access"},{"key":"2326_CR3","doi-asserted-by":"publisher","first-page":"247","DOI":"10.1016\/j.neucom.2020.02.127","volume":"405","author":"V Varga","year":"2020","unstructured":"Varga V, L\u0151rincz A. Reducing human efforts in video segmentation annotation with reinforcement learning. Neurocomputing. 2020;405:247\u201358.","journal-title":"Neurocomputing"},{"key":"2326_CR4","doi-asserted-by":"publisher","DOI":"10.1109\/TNNLS.2021.3070584","author":"L Chen","year":"2021","unstructured":"Chen L, Hu B, Guan ZH, Zhao L, Shen X. Multiagent meta-reinforcement learning for adaptive multipath routing optimization. IEEE Trans Neural Netw Learn Syst. 2021. https:\/\/doi.org\/10.1109\/TNNLS.2021.3070584.","journal-title":"IEEE Trans Neural Netw Learn Syst"},{"key":"2326_CR5","doi-asserted-by":"publisher","first-page":"50","DOI":"10.1016\/j.neucom.2022.04.131","volume":"497","author":"W-H Chen","year":"2022","unstructured":"Chen W-H. Perspective view of autonomous control in unknown environment: dual control for exploitation and exploration vs reinforcement learning. Neurocomputing. 2022;497:50\u201363.","journal-title":"Neurocomputing"},{"key":"2326_CR6","doi-asserted-by":"publisher","first-page":"243","DOI":"10.1016\/j.neucom.2019.06.067","volume":"361","author":"D Ding","year":"2019","unstructured":"Ding D, Ding Z, Wei G, Han F. An improved reinforcement learning algorithm based on knowledge transfer and applications in autonomous vehicles. Neurocomputing. 2019;361:243\u201355.","journal-title":"Neurocomputing"},{"key":"2326_CR7","doi-asserted-by":"publisher","first-page":"285","DOI":"10.1016\/j.neucom.2021.09.044","volume":"466","author":"W Zhou","year":"2021","unstructured":"Zhou W, Liu Z, Li J, Xu X, Shen L. Multi-target tracking for unmanned aerial vehicle swarms using deep reinforcement learning. Neurocomputing. 2021;466:285\u201397.","journal-title":"Neurocomputing"},{"key":"2326_CR8","doi-asserted-by":"publisher","first-page":"118","DOI":"10.1016\/j.neucom.2022.06.102","volume":"503","author":"H Jiang","year":"2022","unstructured":"Jiang H, et al. itd3-cln: learn to navigate in dynamic scene through deep reinforcement learning. Neurocomputing. 2022;503:118\u201328.","journal-title":"Neurocomputing"},{"issue":"19","key":"2326_CR9","doi-asserted-by":"publisher","first-page":"6900","DOI":"10.3390\/app10196900","volume":"10","author":"R Martin","year":"2020","unstructured":"Martin R, et al. Smart grid for industry using multi-agent reinforcement learning. Appl Sci. 2020;10(19):6900.","journal-title":"Appl Sci"},{"key":"2326_CR10","doi-asserted-by":"publisher","first-page":"210626","DOI":"10.1109\/ACCESS.2020.3038863","volume":"8","author":"S Aladdin","year":"2020","unstructured":"Aladdin S, El-Tantawy S, Fouda MM, Eldien AST. Marla-sg: multi-agent reinforcement learning algorithm for efficient demand response in smart grid. IEEE Access. 2020;8:210626\u201339.","journal-title":"IEEE Access"},{"key":"2326_CR11","doi-asserted-by":"crossref","unstructured":"Li S, et al. Robust multi-agent reinforcement learning via minimax deep deterministic policy gradient, vol. 33. 2019. p. 4213\u201320.","DOI":"10.1609\/aaai.v33i01.33014213"},{"key":"2326_CR12","unstructured":"Littman ML. Friend-or-foe q-learning in general-sum games, vol. 1. 2001. p. 322\u20138."},{"key":"2326_CR13","doi-asserted-by":"crossref","unstructured":"Wang Y, De\u00a0Silva CW. Multi-robot box-pushing: single-agent q-learning vs. team q-learning. IEEE. 2006. p. 3694\u201399.","DOI":"10.1109\/IROS.2006.281729"},{"issue":"4","key":"2326_CR14","doi-asserted-by":"publisher","first-page":"1823","DOI":"10.1109\/TVT.2010.2043124","volume":"59","author":"A Galindo-Serrano","year":"2010","unstructured":"Galindo-Serrano A, Giupponi L. Distributed q-learning for aggregated interference control in cognitive radio networks. IEEE Trans Veh Technol. 2010;59(4):1823\u201334.","journal-title":"IEEE Trans Veh Technol"},{"issue":"Nov","key":"2326_CR15","first-page":"1039","volume":"4","author":"J Hu","year":"2003","unstructured":"Hu J, Wellman MP. Nash q-learning for general-sum stochastic games. J Mach Learn Res. 2003;4(Nov):1039\u201369.","journal-title":"J Mach Learn Res"},{"key":"2326_CR16","unstructured":"Ackermann J, Gabler V, Osa T, Sugiyama M. Reducing overestimation bias in multi-agent domains using double centralized critics. 2019. arXiv:1910.01465."},{"key":"2326_CR17","doi-asserted-by":"publisher","first-page":"206","DOI":"10.1016\/j.neucom.2020.05.097","volume":"411","author":"F Zhang","year":"2020","unstructured":"Zhang F, Li J, Li Z. A td3-based multi-agent deep reinforcement learning method in mixed cooperation-competition environment. Neurocomputing. 2020;411:206\u201315.","journal-title":"Neurocomputing"},{"issue":"3","key":"2326_CR18","doi-asserted-by":"publisher","first-page":"279","DOI":"10.1007\/BF00992698","volume":"8","author":"CJCH Watkins","year":"1992","unstructured":"Watkins CJCH, Dayan P. Q-learning. Mach Learn. 1992;8(3):279\u201392. https:\/\/doi.org\/10.1007\/BF00992698.","journal-title":"Mach Learn"},{"issue":"7540","key":"2326_CR19","doi-asserted-by":"publisher","first-page":"529","DOI":"10.1038\/nature14236","volume":"518","author":"V Mnih","year":"2015","unstructured":"Mnih V, et al. Human-level control through deep reinforcement learning. Nature. 2015;518(7540):529\u201333.","journal-title":"Nature"},{"key":"2326_CR20","doi-asserted-by":"crossref","unstructured":"Littman ML. Markov games as a framework for multi-agent reinforcement learning. Elsevier; 1994 p. 157\u201363.","DOI":"10.1016\/B978-1-55860-335-6.50027-1"},{"key":"2326_CR21","doi-asserted-by":"crossref","unstructured":"Foerster J, Farquhar G, Afouras T, Nardelli N, Whiteson S. Counterfactual multi-agent policy gradients, vol. 32. 2018.","DOI":"10.1609\/aaai.v32i1.11794"},{"key":"2326_CR22","unstructured":"Sunehag P, et al. Value-decomposition networks for cooperative multi-agent learning. 2017. arXiv:1706.05296."},{"key":"2326_CR23","unstructured":"Schroeder\u00a0de Witt C, et al. Multi-agent common knowledge reinforcement learning. Advances in Neural Information Processing Systems, vol. 32. 2019. p. 9927\u201339."},{"key":"2326_CR24","unstructured":"Foerster JN, et al. Learning with opponent-learning awareness. 2017. arXiv:1709.04326."},{"key":"2326_CR25","unstructured":"Foerster JN, et al. Learning with opponent-learning awareness. 2017. arXiv:1709.04326."},{"key":"2326_CR26","unstructured":"Lowe R, et al. Multi-agent actor-critic for mixed cooperative-competitive environments. 2017. p. 6382\u201393."},{"key":"2326_CR27","first-page":"10199","volume":"33","author":"T Rashid","year":"2020","unstructured":"Rashid T, Farquhar G, Peng B, Whiteson S. Weighted qmix: expanding monotonic value function factorisation for deep multi-agent reinforcement learning. Adv Neural Inf Process Syst. 2020;33:10199\u2013210.","journal-title":"Adv Neural Inf Process Syst"},{"key":"2326_CR28","doi-asserted-by":"publisher","first-page":"191","DOI":"10.1016\/j.ins.2022.10.042","volume":"615","author":"W Du","year":"2022","unstructured":"Du W, et al. Value function factorization with dynamic weighting for deep multi-agent reinforcement learning. Inf Sci. 2022;615:191\u2013208.","journal-title":"Inf Sci"},{"key":"2326_CR29","unstructured":"Peng B, et al. Facmac: factored multi-agent centralised policy gradients. Advances in Neural Information Processing Systems, vol. 34. 2021."},{"key":"2326_CR30","unstructured":"Lillicrap TP, et al. Continuous control with deep reinforcement learning. 2015. arXiv:1509.02971 ."},{"key":"2326_CR31","unstructured":"Fujimoto S, Hoof H, Meger D. Addressing function approximation error in actor-critic methods, PMLR. 2018. p. 1587\u201396."},{"key":"2326_CR32","unstructured":"Zhou M, Liu Z, Sui P, Li Y, Chung YY, Larochelle H, Ranzato M, Hadsell R, Balcan MF, Lin H, editors. Learning implicit credit assignment for cooperative multi-agent reinforcement learning. (eds Larochelle, H., Ranzato, M., Hadsell, R., Balcan, M.\u00a0F. & Lin, H.) Advances in Neural Information Processing Systems, vol.\u00a033. Curran Associates, Inc.; 2020. p. 11853\u201364. https:\/\/proceedings.neurips.cc\/paper\/2020\/file\/8977ecbb8cb82d77fb091c7a7f186163-Paper.pdf."},{"key":"2326_CR33","unstructured":"Du Y, et al. Wallach, H. et\u00a0al. (eds) Liir: Learning individual intrinsic reward in multi-agent reinforcement learning. (eds Wallach, H. et\u00a0al.) Advances in Neural Information Processing Systems, vol.\u00a032. Curran Associates, Inc.; 2019."},{"key":"2326_CR34","unstructured":"Foerster J, Assael IA, De Freitas N, Whiteson S. Learning to communicate with deep multi-agent reinforcement learning. Advances in neural information processing systems, vol. 29. 2016."},{"key":"2326_CR35","doi-asserted-by":"publisher","DOI":"10.1109\/TNNLS.2021.3056418","author":"X Xu","year":"2021","unstructured":"Xu X, Li R, Zhao Z, Zhang H. Stigmergic independent reinforcement learning for multiagent collaboration. IEEE Trans Neural Netw Learn Syst. 2021. https:\/\/doi.org\/10.1109\/TNNLS.2021.3056418.","journal-title":"IEEE Trans Neural Netw Learn Syst"},{"key":"2326_CR36","doi-asserted-by":"publisher","DOI":"10.1109\/TNNLS.2021.3121546","author":"G Hu","year":"2021","unstructured":"Hu G, Zhu Y, Zhao D, Zhao M, Hao J. Event-triggered communication network with limited-bandwidth constraint for multi-agent reinforcement learning. IEEE Trans Neural Netw Learn Syst. 2021. https:\/\/doi.org\/10.1109\/TNNLS.2021.3121546.","journal-title":"IEEE Trans Neural Netw Learn Syst"},{"key":"2326_CR37","doi-asserted-by":"publisher","DOI":"10.1109\/TNNLS.2022.3146858","author":"Z Pu","year":"2022","unstructured":"Pu Z, Wang H, Liu Z, Yi J, Wu S. Attention enhanced reinforcement learning for multi agent cooperation. IEEE Trans Neural Netw Learn Syst. 2022. https:\/\/doi.org\/10.1109\/TNNLS.2022.3146858.","journal-title":"IEEE Trans Neural Netw Learn Syst"},{"key":"2326_CR38","doi-asserted-by":"publisher","DOI":"10.1109\/TNNLS.2021.3054685","author":"X Yang","year":"2021","unstructured":"Yang X, Zhang H, Wang Z. Data-based optimal consensus control for multiagent systems with policy gradient reinforcement learning. IEEE Trans Neural Netw Learn Syst. 2021. https:\/\/doi.org\/10.1109\/TNNLS.2021.3054685.","journal-title":"IEEE Trans Neural Netw Learn Syst"},{"issue":"12","key":"2326_CR39","doi-asserted-by":"publisher","first-page":"3083","DOI":"10.1109\/TNNLS.2015.2403394","volume":"26","author":"C Yu","year":"2015","unstructured":"Yu C, Zhang M, Ren F, Tan G. Emotional multiagent reinforcement learning in spatial social dilemmas. IEEE Trans Neural Netw Learn Syst. 2015;26(12):3083\u201396. https:\/\/doi.org\/10.1109\/TNNLS.2015.2403394.","journal-title":"IEEE Trans Neural Netw Learn Syst"},{"key":"2326_CR40","unstructured":"He Q, Hou X. Reducing estimation bias via weighted delayed deep deterministic policy gradient. 2020. arXiv:2006.12622."},{"issue":"11","key":"2326_CR41","doi-asserted-by":"publisher","first-page":"4933","DOI":"10.1109\/TNNLS.2019.2959129","volume":"31","author":"D Wu","year":"2020","unstructured":"Wu D, Dong X, Shen J, Hoi SC. Reducing estimation bias via triplet-average deep deterministic policy gradient. IEEE Trans Neural Netw Learn Syst. 2020;31(11):4933\u201345.","journal-title":"IEEE Trans Neural Netw Learn Syst"},{"key":"2326_CR42","first-page":"10246","volume":"34","author":"Z Ren","year":"2021","unstructured":"Ren Z, et al. On the estimation bias in double q-learning. Adv Neural Inf Process Syst. 2021;34:10246\u201359.","journal-title":"Adv Neural Inf Process Syst"},{"key":"2326_CR43","doi-asserted-by":"crossref","unstructured":"Kuznetsov A, Grishin A, Tsypin A, Ashukha A, Vetrov D. Automating control of overestimation bias for continuous reinforcement learning. 2021. arXiv:2110.13523.","DOI":"10.21203\/rs.3.rs-2385186\/v1"},{"key":"2326_CR44","doi-asserted-by":"crossref","unstructured":"Saglam B, Duran E, Cicek DC, Mutlu FB, Kozat SS. Estimation error correction in deep reinforcement learning for deterministic actor-critic methods. IEEE; 2021. p. 137\u201344.","DOI":"10.1109\/ICTAI52525.2021.00027"},{"key":"2326_CR45","doi-asserted-by":"crossref","unstructured":"Saglam B, Mutlu FB, Cicek DC, Kozat, SS. Parameter-free reduction of the estimation bias in deep reinforcement learning for deterministic policy gradients. 2021. arXiv:2109.11788.","DOI":"10.1109\/ICTAI52525.2021.00027"},{"key":"2326_CR46","doi-asserted-by":"publisher","DOI":"10.1016\/j.patcog.2022.108872","volume":"131","author":"Y Li","year":"2022","unstructured":"Li Y, Wang Y, Gan Y, Tan X. Alleviating the estimation bias of deep deterministic policy gradient via co-regularization. Pattern Recogn. 2022;131: 108872.","journal-title":"Pattern Recogn"},{"key":"2326_CR47","doi-asserted-by":"crossref","unstructured":"He Q, Hou X. Wd3: taming the estimation bias in deep reinforcement learning. IEEE; 2020. p. 391\u201398.","DOI":"10.1109\/ICTAI50040.2020.00068"},{"key":"2326_CR48","doi-asserted-by":"crossref","unstructured":"Cicek DC, et al. Awd3: dynamic reduction of the estimation bias. IEEE; 2021. p. 775\u201379.","DOI":"10.1109\/ICTAI52525.2021.00123"},{"key":"2326_CR49","doi-asserted-by":"publisher","DOI":"10.1016\/j.engappai.2022.105329","volume":"116","author":"B Liu","year":"2022","unstructured":"Liu B, Xie Y, Feng L, Fu P. Correcting biased value estimation in mixing value-based multi-agent reinforcement learning by multiple choice learning. Eng Appl Artif Intell. 2022;116: 105329.","journal-title":"Eng Appl Artif Intell"},{"key":"2326_CR50","unstructured":"Kingma DP, Ba J. Adam: a method for stochastic optimization. 2014. arXiv:1412.6980."},{"key":"2326_CR51","doi-asserted-by":"crossref","unstructured":"Sarkar T, Kalita S. A weighted critic update approach to multi agent twin delayed deep deterministic algorithm. IEEE; 2021. p. 1\u20136.","DOI":"10.1109\/INDICON52576.2021.9691489"},{"key":"2326_CR52","doi-asserted-by":"crossref","unstructured":"Bengio Y, Louradour J, Collobert R, Weston J. Curriculum learning. 2009. p. 41\u20138.","DOI":"10.1145\/1553374.1553380"},{"issue":"10","key":"2326_CR53","doi-asserted-by":"publisher","first-page":"1345","DOI":"10.1109\/TKDE.2009.191","volume":"22","author":"S Pan","year":"2010","unstructured":"Pan S, Yang Q. A survey on transfer learning. IEEE Trans Knowl Discov Data Eng. 2010;22(10):1345\u201359.","journal-title":"IEEE Trans Knowl Discov Data Eng"},{"key":"2326_CR54","unstructured":"Samvelyan M, et al. The starcraft multi-agent challenge. 2019. arXiv:1902.04043."},{"issue":"22","key":"2326_CR55","doi-asserted-by":"publisher","first-page":"7749","DOI":"10.3390\/en14227749","volume":"14","author":"W Li","year":"2021","unstructured":"Li W, Tang M, Zhang X, Gao D, Wang J. Operation of distributed battery considering demand response using deep reinforcement learning in grid edge control. Energies. 2021;14(22):7749.","journal-title":"Energies"},{"key":"2326_CR56","doi-asserted-by":"crossref","unstructured":"Zhang Y, Zhuang Z, Gao F, Wang J, Han Z. Multi-agent deep reinforcement learning for secure uav communications. IEEE; 2020. p. 1\u20135.","DOI":"10.1109\/WCNC45663.2020.9120592"}],"container-title":["SN Computer Science"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s42979-023-02326-7.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s42979-023-02326-7\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s42979-023-02326-7.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2023,11,16]],"date-time":"2023-11-16T08:29:19Z","timestamp":1700123359000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s42979-023-02326-7"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023,11,16]]},"references-count":56,"journal-issue":{"issue":"1","published-online":{"date-parts":[[2024,1]]}},"alternative-id":["2326"],"URL":"https:\/\/doi.org\/10.1007\/s42979-023-02326-7","relation":{},"ISSN":["2661-8907"],"issn-type":[{"value":"2661-8907","type":"electronic"}],"subject":[],"published":{"date-parts":[[2023,11,16]]},"assertion":[{"value":"28 February 2023","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"14 September 2023","order":2,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"16 November 2023","order":3,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Declarations"}},{"value":"On behalf of all authors, the corresponding author states that there is no conflict of interest.","order":2,"name":"Ethics","group":{"name":"EthicsHeading","label":"Conflict of interest"}}],"article-number":"15"}}