{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,3]],"date-time":"2026-07-03T16:51:41Z","timestamp":1783097501040,"version":"3.54.6"},"publisher-location":"Cham","reference-count":30,"publisher":"Springer Nature Switzerland","isbn-type":[{"value":"9783031773662","type":"print"},{"value":"9783031773679","type":"electronic"}],"license":[{"start":{"date-parts":[[2024,11,15]],"date-time":"2024-11-15T00:00:00Z","timestamp":1731628800000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2024,11,15]],"date-time":"2024-11-15T00:00:00Z","timestamp":1731628800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2025]]},"DOI":"10.1007\/978-3-031-77367-9_20","type":"book-chapter","created":{"date-parts":[[2024,11,16]],"date-time":"2024-11-16T04:57:34Z","timestamp":1731733054000},"page":"267-283","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":7,"title":["Improving Reinforcement Learning-Based Autonomous Agents with\u00a0Causal Models"],"prefix":"10.1007","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-9572-3150","authenticated-orcid":false,"given":"Giovanni","family":"Briglia","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-9663-1071","authenticated-orcid":false,"given":"Marco","family":"Lippi","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-8921-8150","authenticated-orcid":false,"given":"Stefano","family":"Mariani","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-6837-8806","authenticated-orcid":false,"given":"Franco","family":"Zambonelli","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2024,11,15]]},"reference":[{"key":"20_CR1","unstructured":"Agarwal, R., Schwarzer, M., Castro, P.S., Courville, A.C., Bellemare, M.: Deep reinforcement learning at the edge of the statistical precipice. In: Advances in Neural Information Processing Systems 34, pp. 29304\u201329320 (2021)"},{"key":"20_CR2","unstructured":"Amin, S., Gomrokchi, M., Satija, H., van Hoof, H., Precup, D.: A survey of exploration methods in reinforcement learning (2021). https:\/\/arxiv.org\/abs\/2109.00157"},{"key":"20_CR3","unstructured":"Burda, Y., Edwards, H., Pathak, D., Storkey, A.J., Darrell, T., Efros, A.A.: Large-scale study of curiosity-driven learning (2018). http:\/\/arxiv.org\/abs\/1808.04355"},{"key":"20_CR4","series-title":"Lecture Notes in Computer Science (Lecture Notes in Artificial Intelligence)","doi-asserted-by":"publisher","first-page":"213","DOI":"10.1007\/978-3-030-89817-5_16","volume-title":"Advances in Computational Intelligence","author":"I Feliciano-Avelino","year":"2021","unstructured":"Feliciano-Avelino, I., M\u00e9ndez-Molina, A., Morales, E.F., Sucar, L.E.: Causal based action selection policy for reinforcement learning. In: Batyrshin, I., Gelbukh, A., Sidorov, G. (eds.) MICAI 2021. LNCS (LNAI), vol. 13067, pp. 213\u2013227. Springer, Cham (2021). https:\/\/doi.org\/10.1007\/978-3-030-89817-5_16"},{"key":"20_CR5","unstructured":"Gorsane, R., Mahjoub, O., de Kock, R.J., Dubb, R., Singh, S., Pretorius, A.: Towards a standardised performance evaluation protocol for cooperative MARL. In: Advances in Neural Information Processing Systems 35, pp. 5510\u20135521 (2022)"},{"key":"20_CR6","unstructured":"Hinton, G., Vinyals, O., Dean, J.: Distilling the knowledge in a neural network. arXiv preprint arXiv:1503.02531 (2015)"},{"issue":"5","key":"20_CR7","doi-asserted-by":"publisher","first-page":"1668","DOI":"10.4249\/scholarpedia.1668","volume":"2","author":"GE Hinton","year":"2007","unstructured":"Hinton, G.E.: Boltzmann machine. Scholarpedia 2(5), 1668 (2007)","journal-title":"Scholarpedia"},{"key":"20_CR8","unstructured":"Hu, X., et\u00a0al.: Causality-driven hierarchical structure discovery for reinforcement learning. In: Advances in Neural Information Processing Systems 35 (2022)"},{"key":"20_CR9","doi-asserted-by":"publisher","unstructured":"Huang, Q.: Model-based or model-free, a review of approaches in reinforcement learning. In: 2020 International Conference on Computing and Data Science (CDS), pp. 219\u2013221 (2020). https:\/\/doi.org\/10.1109\/CDS49703.2020.00051","DOI":"10.1109\/CDS49703.2020.00051"},{"key":"20_CR10","unstructured":"Janner, M., Fu, J., Zhang, M., Levine, S.: When to trust your model: model-based policy optimization. In: Advances in Neural Information Processing Systems 32. Curran Associates, Inc. (2019)"},{"key":"20_CR11","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1016\/j.inffus.2022.03.003","volume":"85","author":"P Ladosz","year":"2022","unstructured":"Ladosz, P., Weng, L., Kim, M., Oh, H.: Exploration in deep reinforcement learning: a survey. Inf. Fusion 85, 1\u201322 (2022). https:\/\/doi.org\/10.1016\/j.inffus.2022.03.003","journal-title":"Inf. Fusion"},{"key":"20_CR12","doi-asserted-by":"publisher","first-page":"163","DOI":"10.1007\/978-3-031-37616-0_14","volume-title":"Advances in Practical Applications of Agents, Multi-Agent Systems, and Cognitive Mimetics","author":"S Mariani","year":"2023","unstructured":"Mariani, S., Roseti, P., Zambonelli, F.: Multi-agent learning of causal networks in the Internet of Things. In: Mathieu, P., Dignum, F., Novais, P., De la Prieta, F. (eds.) PAAMS 2023. LNCS, vol. 13955, pp. 163\u2013174. Springer, Cham (2023). https:\/\/doi.org\/10.1007\/978-3-031-37616-0_14"},{"issue":"3","key":"20_CR13","first-page":"95","volume":"149","author":"A M\u00e9ndez-Molina","year":"2020","unstructured":"M\u00e9ndez-Molina, A., Feliciano-Avelino, I., Morales, E.F., Sucar, L.E.: Causal based Q-learning. Res. Comput. Sci. 149(3), 95\u2013104 (2020)","journal-title":"Res. Comput. Sci."},{"key":"20_CR14","doi-asserted-by":"publisher","first-page":"126462","DOI":"10.1109\/ACCESS.2023.3331728","volume":"11","author":"A M\u00e9ndez-Molina","year":"2023","unstructured":"M\u00e9ndez-Molina, A., Morales, E.F., Sucar, L.E.: CARL: a synergistic framework for causal reinforcement learning. IEEE Access 11, 126462\u2013126481 (2023)","journal-title":"IEEE Access"},{"issue":"7540","key":"20_CR15","doi-asserted-by":"publisher","first-page":"529","DOI":"10.1038\/nature14236","volume":"518","author":"V Mnih","year":"2015","unstructured":"Mnih, V., et al.: Human-level control through deep reinforcement learning. Nature 518(7540), 529\u2013533 (2015)","journal-title":"Nature"},{"key":"20_CR16","doi-asserted-by":"publisher","first-page":"157","DOI":"10.1007\/978-1-4899-1424-8_9","volume-title":"Mathematical Models for Handling Partial Knowledge in Artificial Intelligence","author":"J Pearl","year":"1995","unstructured":"Pearl, J.: From Bayesian networks to causal networks. In: Coletti, G., Dubois, D., Scozzafava, R. (eds.) Mathematical Models for Handling Partial Knowledge in Artificial Intelligence, pp. 157\u2013182. Springer, Boston (1995). https:\/\/doi.org\/10.1007\/978-1-4899-1424-8_9"},{"key":"20_CR17","doi-asserted-by":"crossref","unstructured":"Pearl, J.: Graphical models for probabilistic and causal reasoning. In: Computing Handbook, Third Edition: Computer Science and Software Engineering, pp. 44:1\u201344:24. CRC Press (2014)","DOI":"10.1201\/b16812-50"},{"issue":"3","key":"20_CR18","doi-asserted-by":"publisher","first-page":"54","DOI":"10.1145\/3241036","volume":"62","author":"J Pearl","year":"2019","unstructured":"Pearl, J.: The seven tools of causal inference, with reflections on machine learning. Commun. ACM 62(3), 54\u201360 (2019). https:\/\/doi.org\/10.1145\/3241036","journal-title":"Commun. ACM"},{"key":"20_CR19","volume-title":"Elements of Causal Inference: Foundations and Learning Algorithms","author":"J Peters","year":"2017","unstructured":"Peters, J., Janzing, D., Sch\u00f6lkopf, B.: Elements of Causal Inference: Foundations and Learning Algorithms. The MIT Press, Cambridge (2017)"},{"key":"20_CR20","unstructured":"Peters, J., Mooij, J.M., Janzing, D., Sch\u00f6lkopf, B.: Identifiability of causal graphs using functional models. In: Proceedings of the Twenty-Seventh Conference on Uncertainty in Artificial Intelligence (2011)"},{"key":"20_CR21","unstructured":"Seitzer, M., Sch\u00f6lkopf, B., Martius, G.: Causal influence detection for improving efficiency in reinforcement learning. In: Advances in Neural Information Processing Systems 34, pp. 22905\u201322918 (2021)"},{"key":"20_CR22","doi-asserted-by":"crossref","unstructured":"Singh, S., Barto, A.G., Chentanez, N.: Intrinsically motivated reinforcement learning. In: Advances in Neural Information Processing Systems 17 (2004)","DOI":"10.21236\/ADA440280"},{"issue":"3","key":"20_CR23","doi-asserted-by":"publisher","first-page":"287","DOI":"10.1023\/A:1007678930559","volume":"38","author":"S Singh","year":"2000","unstructured":"Singh, S., Jaakkola, T.S., Littman, M.L., Szepesv\u00e1ri, C.: Convergence results for single-step on-policy reinforcement-learning algorithms. Mach. Learn. 38(3), 287\u2013308 (2000). https:\/\/doi.org\/10.1023\/A:1007678930559","journal-title":"Mach. Learn."},{"key":"20_CR24","volume-title":"Reinforcement Learning: An Introduction","author":"RS Sutton","year":"2018","unstructured":"Sutton, R.S., Barto, A.G.: Reinforcement Learning: An Introduction. MIT Press, Cambridge (2018)"},{"issue":"3\u20134","key":"20_CR25","doi-asserted-by":"publisher","first-page":"285","DOI":"10.1093\/biomet\/25.3-4.285","volume":"25","author":"WR Thompson","year":"1933","unstructured":"Thompson, W.R.: On the likelihood that one unknown probability exceeds another in view of the evidence of two samples. Biometrika 25(3\u20134), 285\u2013294 (1933)","journal-title":"Biometrika"},{"key":"20_CR26","doi-asserted-by":"publisher","first-page":"279","DOI":"10.1007\/BF00992698","volume":"8","author":"CJ Watkins","year":"1992","unstructured":"Watkins, C.J., Dayan, P.: Q-learning. Mach. Learn. 8, 279\u2013292 (1992)","journal-title":"Mach. Learn."},{"key":"20_CR27","doi-asserted-by":"crossref","unstructured":"Whiteson, S., Tanner, B., Taylor, M.E., Stone, P.: Protecting against evaluation overfitting in empirical reinforcement learning. In: 2011 IEEE Symposium on Adaptive Dynamic Programming and Reinforcement Learning (ADPRL), pp. 120\u2013127. IEEE (2011)","DOI":"10.1109\/ADPRL.2011.5967363"},{"key":"20_CR28","unstructured":"Zeng, Y., Cai, R., Sun, F., Huang, L., Hao, Z.: A survey on causal reinforcement learning (2023). https:\/\/arxiv.org\/abs\/2302.05209"},{"key":"20_CR29","unstructured":"Zheng, X., Aragam, B., Ravikumar, P.K., Xing, E.P.: DAGs with no tears: continuous optimization for structure learning. In: Advances in Neural Information Processing Systems 31 (2018)"},{"key":"20_CR30","doi-asserted-by":"crossref","unstructured":"Zhu, W., Yu, C., Zhang, Q.: Causal deep reinforcement learning using observational data. arXiv preprint arXiv:2211.15355 (2022)","DOI":"10.24963\/ijcai.2023\/524"}],"container-title":["Lecture Notes in Computer Science","PRIMA 2024: Principles and Practice of Multi-Agent Systems"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-031-77367-9_20","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,11,16]],"date-time":"2024-11-16T05:08:24Z","timestamp":1731733704000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-3-031-77367-9_20"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,11,15]]},"ISBN":["9783031773662","9783031773679"],"references-count":30,"URL":"https:\/\/doi.org\/10.1007\/978-3-031-77367-9_20","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"value":"0302-9743","type":"print"},{"value":"1611-3349","type":"electronic"}],"subject":[],"published":{"date-parts":[[2024,11,15]]},"assertion":[{"value":"15 November 2024","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"The authors have no competing interests to declare that are relevant to the content of this article.","order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Disclosure of Interests"}},{"value":"PRIMA","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"International Conference on Principles and Practice of Multi-Agent Systems","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Kyoto","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Japan","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2024","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"18 November 2024","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"24 November 2024","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"25","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"prima2024","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}}]}}