{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,5,21]],"date-time":"2026-05-21T09:29:21Z","timestamp":1779355761687,"version":"3.51.4"},"reference-count":45,"publisher":"Springer Science and Business Media LLC","issue":"12","license":[{"start":{"date-parts":[[2023,9,20]],"date-time":"2023-09-20T00:00:00Z","timestamp":1695168000000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2023,9,20]],"date-time":"2023-09-20T00:00:00Z","timestamp":1695168000000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"funder":[{"DOI":"10.13039\/501100001809","name":"NSFC","doi-asserted-by":"crossref","award":["61921006"],"award-info":[{"award-number":["61921006"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"crossref"}]},{"DOI":"10.13039\/501100012166","name":"National Key R&D Program of China","doi-asserted-by":"crossref","award":["2020YFA0713900"],"award-info":[{"award-number":["2020YFA0713900"]}],"id":[{"id":"10.13039\/501100012166","id-type":"DOI","asserted-by":"crossref"}]}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Mach Learn"],"published-print":{"date-parts":[[2023,12]]},"DOI":"10.1007\/s10994-023-06365-2","type":"journal-article","created":{"date-parts":[[2023,9,20]],"date-time":"2023-09-20T17:01:45Z","timestamp":1695229305000},"page":"5163-5190","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":7,"title":["State-based episodic memory for multi-agent reinforcement learning"],"prefix":"10.1007","volume":"112","author":[{"given":"Xiao","family":"Ma","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Wu-Jun","family":"Li","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2023,9,20]]},"reference":[{"key":"6365_CR1","unstructured":"Amarjyoti, S. (2017). Deep reinforcement learning for robotic manipulation\u2014the state of the art. arXiv:1701.08878."},{"key":"6365_CR2","doi-asserted-by":"publisher","DOI":"10.1093\/acprof:oso\/9780195100273.001.0001","volume-title":"The hippocampus book","author":"P Andersen","year":"2006","unstructured":"Andersen, P., Morris, R., Amaral, D., et al. (2006). The hippocampus book. Oxford: Oxford University Press."},{"key":"6365_CR3","unstructured":"Badia AP, Piot B, Kapturowski S, et\u00a0al (2020). Agent57: Outperforming the atari human benchmark. In ICML."},{"issue":"9","key":"6365_CR4","doi-asserted-by":"publisher","first-page":"509","DOI":"10.1145\/361002.361007","volume":"18","author":"JL Bentley","year":"1975","unstructured":"Bentley, J. L. (1975). Multidimensional binary search trees used for associative searching. Communications of the ACM, 18(9), 509\u2013517.","journal-title":"Communications of the ACM"},{"key":"6365_CR5","unstructured":"Berner, C., Brockman, G., Chan, B., et\u00a0al. (2019). Dota 2 with large scale deep reinforcement learning. arXiv:1912.06680."},{"key":"6365_CR6","unstructured":"Blundell, C., Uria, B., Pritzel, A., et\u00a0al. (2016). Model-free episodic control. arXiv:1606.04460."},{"issue":"1","key":"6365_CR7","doi-asserted-by":"publisher","first-page":"427","DOI":"10.1109\/TII.2012.2219061","volume":"9","author":"Y Cao","year":"2012","unstructured":"Cao, Y., Yu, W., Ren, W., et al. (2012). An overview of recent progress in the study of distributed multi-agent coordination. IEEE Transactions on Industrial Informatics, 9(1), 427\u2013438.","journal-title":"IEEE Transactions on Industrial Informatics"},{"key":"6365_CR8","unstructured":"Duan, Y., Chen, X., Houthooft, R., et\u00a0al. (2016). Benchmarking deep reinforcement learning for continuous control. In ICML."},{"key":"6365_CR9","doi-asserted-by":"crossref","unstructured":"Foerster, J. N., Farquhar, G., Afouras, T., et\u00a0al. (2018). Counterfactual multi-agent policy gradients. In AAAI.","DOI":"10.1609\/aaai.v32i1.11794"},{"issue":"3","key":"6365_CR10","doi-asserted-by":"publisher","first-page":"111","DOI":"10.1016\/j.tics.2013.01.001","volume":"17","author":"O Hardt","year":"2013","unstructured":"Hardt, O., Nader, K., & Nadel, L. (2013). Decay happens: The role of active forgetting in memory. Trends in Cognitive Sciences, 17(3), 111\u2013120.","journal-title":"Trends in Cognitive Sciences"},{"issue":"6","key":"6365_CR11","doi-asserted-by":"publisher","first-page":"750","DOI":"10.1007\/s10458-019-09421-1","volume":"33","author":"P Hernandez-Leal","year":"2019","unstructured":"Hernandez-Leal, P., Kartal, B., & Taylor, M. E. (2019). A survey and critique of multiagent deep reinforcement learning. Autonomous Agents and Multi-Agent Systems, 33(6), 750\u2013797.","journal-title":"Autonomous Agents and Multi-Agent Systems"},{"issue":"6","key":"6365_CR12","doi-asserted-by":"publisher","first-page":"1185","DOI":"10.1162\/neco.1994.6.6.1185","volume":"6","author":"TS Jaakkola","year":"1994","unstructured":"Jaakkola, T. S., Jordan, M. I., & Singh, S. P. (1994). On the convergence of stochastic iterative dynamic programming algorithms. Neural Computing, 6(6), 1185\u20131201.","journal-title":"Neural Computing"},{"key":"6365_CR13","unstructured":"Jin, C., Liu, Q., Wang, Y., et\u00a0al. (2021). V-learning: A simple, efficient, decentralized algorithm for multiagent RL. arXiv:2110.14555."},{"issue":"189\u2013206","key":"6365_CR14","first-page":"1","volume":"26","author":"WB Johnson","year":"1984","unstructured":"Johnson, W. B., & Lindenstrauss, J. (1984). Extensions of Lipschitz mappings into a Hilbert space. Contemporary Mathematics, 26(189\u2013206), 1.","journal-title":"Contemporary Mathematics"},{"issue":"11","key":"6365_CR15","doi-asserted-by":"publisher","first-page":"1238","DOI":"10.1177\/0278364913495721","volume":"32","author":"J Kober","year":"2013","unstructured":"Kober, J., Bagnell, J. A., & Peters, J. (2013). Reinforcement learning in robotics: A survey. The International Journal of Robotics Research, 32(11), 1238\u20131274.","journal-title":"The International Journal of Robotics Research"},{"key":"6365_CR16","doi-asserted-by":"crossref","unstructured":"Kononenko, I., & Kukar, M. (2007). Machine learning and data mining. Horwood Publishing.","DOI":"10.1533\/9780857099440"},{"key":"6365_CR17","doi-asserted-by":"crossref","unstructured":"Lample, G., Chaplot, D. S. (2017). Playing FPS games with deep reinforcement learning. In AAAI.","DOI":"10.1609\/aaai.v31i1.10827"},{"key":"6365_CR18","unstructured":"Lengyel, M., & Dayan, P. (2007). Hippocampal contributions to control: The third way. In NeurIPS."},{"key":"6365_CR19","unstructured":"Lillicrap, T. P., Hunt, J. J., Pritzel, A., et\u00a0al. (2016). Continuous control with deep reinforcement learning. In ICLR."},{"key":"6365_CR20","doi-asserted-by":"crossref","unstructured":"Lin, Z., Zhao, T., Yang, G., et\u00a0al. (2018). Episodic memory deep q-networks. In IJCAI.","DOI":"10.24963\/ijcai.2018\/337"},{"key":"6365_CR21","unstructured":"Lowe, R., Wu, Y., Tamar, A., et\u00a0al. (2017). Multi-agent actor-critic for mixed cooperative-competitive environments. In NeurIPS."},{"key":"6365_CR22","unstructured":"Ma, X., & Li, W. (2021). State-based episodic memory for multi-agent reinforcement learning. arXiv:2110.09817."},{"key":"6365_CR23","unstructured":"Melo, F. S. (2001). Convergence of q-learning: A simple proof. Institute of Systems and Robotics, Technical Report, pp. 1\u20134."},{"issue":"7540","key":"6365_CR24","doi-asserted-by":"publisher","first-page":"529","DOI":"10.1038\/nature14236","volume":"518","author":"V Mnih","year":"2015","unstructured":"Mnih, V., Kavukcuoglu, K., Silver, D., et al. (2015). Human-level control through deep reinforcement learning. Nature, 518(7540), 529\u2013533.","journal-title":"Nature"},{"key":"6365_CR25","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-28929-8","volume-title":"A concise introduction to decentralized POMDPs","author":"FA Oliehoek","year":"2016","unstructured":"Oliehoek, F. A., & Amato, C. (2016). A concise introduction to decentralized POMDPs. Berlin: Springer."},{"key":"6365_CR26","doi-asserted-by":"publisher","first-page":"289","DOI":"10.1613\/jair.2447","volume":"32","author":"FA Oliehoek","year":"2008","unstructured":"Oliehoek, F. A., Spaan, M. T. J., & Vlassis, N. A. (2008). Optimal and approximate q-value functions for decentralized POMDPs. Journal of Artificial Intelligence Research, 32, 289\u2013353.","journal-title":"Journal of Artificial Intelligence Research"},{"issue":"1\u20132","key":"6365_CR27","doi-asserted-by":"publisher","first-page":"45","DOI":"10.1007\/s10994-006-9643-2","volume":"67","author":"R Powers","year":"2007","unstructured":"Powers, R., Shoham, Y., & Vu, T. (2007). A general criterion and an algorithmic framework for learning in multi-agent systems. Machine Learning, 67(1\u20132), 45\u201376.","journal-title":"Machine Learning"},{"key":"6365_CR28","unstructured":"Pritzel, A., Uria, B., Srinivasan, S., et\u00a0al. (2017). Neural episodic control. In ICML."},{"key":"6365_CR29","unstructured":"Rashid, T., Samvelyan, M., de\u00a0Witt, C. S., et\u00a0al. (2018). QMIX: Monotonic value function factorisation for deep multi-agent reinforcement learning. In ICML."},{"key":"6365_CR30","unstructured":"Rashid, T., Farquhar, G., Peng, B., et\u00a0al. (2020). Weighted QMIX: Expanding monotonic value function factorisation for deep multi-agent reinforcement learning. In NeurIPS."},{"key":"6365_CR31","unstructured":"Samvelyan, M., Rashid, T., de\u00a0Witt, C. S., et\u00a0al. (2019). The starcraft multi-agent challenge. In AAMAS."},{"key":"6365_CR32","unstructured":"Shalev-Shwartz, S., Shammah, S., Shashua, A. (2016). Safe, multi-agent, reinforcement learning for autonomous driving. arXiv:1610.03295."},{"issue":"7587","key":"6365_CR33","doi-asserted-by":"publisher","first-page":"484","DOI":"10.1038\/nature16961","volume":"529","author":"D Silver","year":"2016","unstructured":"Silver, D., Huang, A., Maddison, C. J., et al. (2016). Mastering the game of go with deep neural networks and tree search. Nature, 529(7587), 484\u2013489.","journal-title":"Nature"},{"key":"6365_CR34","unstructured":"Son, K., Kim, D., Kang, W. J., et\u00a0al. (2019). QTRAN: Learning to factorize with transformation for cooperative multi-agent reinforcement learning. In ICML."},{"issue":"3","key":"6365_CR35","doi-asserted-by":"publisher","first-page":"171","DOI":"10.1016\/j.nlm.2004.06.005","volume":"82","author":"LR Squire","year":"2004","unstructured":"Squire, L. R. (2004). Memory systems of the brain: A brief history and current perspective. Neurobiology of Learning and Memory, 82(3), 171\u2013177.","journal-title":"Neurobiology of Learning and Memory"},{"key":"6365_CR36","unstructured":"Sunehag, P., Lever, G., Gruslys, A., et\u00a0al. (2018). Value-decomposition networks for cooperative multi-agent learning based on team reward. In AAMAS."},{"key":"6365_CR37","doi-asserted-by":"crossref","unstructured":"Sutton, R. S., Barto, A. G. (1998). Reinforcement learning: An introduction. MIT Press.","DOI":"10.1109\/TNN.1998.712192"},{"key":"6365_CR38","doi-asserted-by":"crossref","unstructured":"Tan, M. (1993). Multi-agent reinforcement learning: Independent vs. cooperative agents. In ICML.","DOI":"10.1016\/B978-1-55860-307-3.50049-6"},{"key":"6365_CR39","unstructured":"Vinyals, O., Ewalds, T., Bartunov, S., et\u00a0al. (2017). Starcraft II: A new challenge for reinforcement learning. arXiv:1708.04782."},{"key":"6365_CR40","unstructured":"Wang, J., Ren, Z., Liu, T., et\u00a0al. (2020). QPLEX: Duplex dueling multi-agent q-learning. arXiv:2008.01062."},{"key":"6365_CR41","doi-asserted-by":"publisher","first-page":"279","DOI":"10.1007\/BF00992698","volume":"8","author":"CJCH Watkins","year":"1992","unstructured":"Watkins, C. J. C. H., & Dayan, P. (1992). Technical note q-learning. Machine Learning, 8, 279\u2013292.","journal-title":"Machine Learning"},{"key":"6365_CR42","unstructured":"Wiering, M. A. (2000). Multi-agent reinforcement learning for traffic light control. In ICML."},{"key":"6365_CR43","unstructured":"Yang, Y., Hao, J., Liao, B., et\u00a0al. (2020). Qatten: A general framework for cooperative multiagent reinforcement learning. arXiv:2002.03939."},{"key":"6365_CR44","unstructured":"Zheng, L., Chen, J., Wang, J., et\u00a0al. (2021). Episodic multi-agent reinforcement learning with curiosity-driven exploration. In NeurIPS."},{"key":"6365_CR45","unstructured":"Zhu, G., Lin, Z., Yang, G., et\u00a0al. (2020). Episodic reinforcement learning with associative memory. In ICLR."}],"container-title":["Machine Learning"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10994-023-06365-2.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s10994-023-06365-2\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10994-023-06365-2.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,9,20]],"date-time":"2024-09-20T00:02:32Z","timestamp":1726790552000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s10994-023-06365-2"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023,9,20]]},"references-count":45,"journal-issue":{"issue":"12","published-print":{"date-parts":[[2023,12]]}},"alternative-id":["6365"],"URL":"https:\/\/doi.org\/10.1007\/s10994-023-06365-2","relation":{},"ISSN":["0885-6125","1573-0565"],"issn-type":[{"value":"0885-6125","type":"print"},{"value":"1573-0565","type":"electronic"}],"subject":[],"published":{"date-parts":[[2023,9,20]]},"assertion":[{"value":"20 June 2022","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"15 March 2023","order":2,"name":"revised","label":"Revised","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"17 June 2023","order":3,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"20 September 2023","order":4,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Declarations"}},{"value":"All authors certify that they have no affiliations with or involvement in any organization or entity with any financial interest or non-financial interest in the subject matter or materials discussed in this manuscript.","order":2,"name":"Ethics","group":{"name":"EthicsHeading","label":"Conflict of interest"}},{"value":"This content has been made available to all.","name":"free","label":"Free to read"}]}}