{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,8,26]],"date-time":"2025-08-26T06:50:32Z","timestamp":1756191032486,"version":"3.40.3"},"publisher-location":"Cham","reference-count":31,"publisher":"Springer International Publishing","isbn-type":[{"type":"print","value":"9783030616151"},{"type":"electronic","value":"9783030616168"}],"license":[{"start":{"date-parts":[[2020,1,1]],"date-time":"2020-01-01T00:00:00Z","timestamp":1577836800000},"content-version":"tdm","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"},{"start":{"date-parts":[[2020,1,1]],"date-time":"2020-01-01T00:00:00Z","timestamp":1577836800000},"content-version":"vor","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2020]]},"DOI":"10.1007\/978-3-030-61616-8_33","type":"book-chapter","created":{"date-parts":[[2020,10,16]],"date-time":"2020-10-16T23:07:42Z","timestamp":1602889662000},"page":"408-419","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":10,"title":["Curious Hierarchical Actor-Critic Reinforcement Learning"],"prefix":"10.1007","author":[{"given":"Frank","family":"R\u00f6der","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Manfred","family":"Eppe","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Phuong D. H.","family":"Nguyen","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Stefan","family":"Wermter","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2020,10,14]]},"reference":[{"key":"33_CR1","unstructured":"Alet, F., Schneider, M.F., Lozano-Perez, T., Kaelbling, L.P.: Meta-learning curiosity algorithms. In: International Conference on Learning Representations (ICLR), p. online (2020)"},{"key":"33_CR2","unstructured":"Andrychowicz, M., et al.: Hindsight experience replay. In: Conference on Neural Information Processing Systems (NeurIPS), pp. 5048\u20135058. Curran Associates, Inc. (2017)"},{"key":"33_CR3","doi-asserted-by":"crossref","unstructured":"Bacon, P.L., Harb, J., Precup, D.: The option-critic architecture. In: Conference on Artificial Intelligence (AAAI), pp. 1726\u20131734. AAAI Press (2017)","DOI":"10.1609\/aaai.v31i1.10916"},{"key":"33_CR4","doi-asserted-by":"crossref","unstructured":"Botvinick, M., Weinstein, A.: Model-based hierarchical reinforcement learning and human action control. Philos. Trans. Roy. Soc. B: Biol. Sci. 369(1655) (2014)","DOI":"10.1098\/rstb.2013.0480"},{"key":"33_CR5","unstructured":"Burda, Y., Edwards, H., Pathak, D., Storkey, A., Darrell, T., Efros, A.A.: Large-scale study of curiosity-driven learning. In: International Conference on Learning Representations (ICLR), p. online (2019)"},{"key":"33_CR6","unstructured":"Burda, Y., Edwards, H., Storkey, A., Klimov, O.: Exploration by random network distillation. In: International Conference on Learning Representations (ICLR), p. online (2019)"},{"key":"33_CR7","doi-asserted-by":"publisher","first-page":"925","DOI":"10.3389\/fpsyg.2016.00925","volume":"7","author":"MV Butz","year":"2016","unstructured":"Butz, M.V.: Toward a unified sub-symbolic computational theory of cognition. Front. Psychol. 7, 925 (2016)","journal-title":"Front. Psychol."},{"key":"33_CR8","unstructured":"Colas, C., Fournier, P., Sigaud, O., Chetouani, M., Oudeyer, P.Y.: CURIOUS: intrinsically motivated modular multi-goal reinforcement learning. In: International Conference on Machine Learning (ICML), pp. 1331\u20131340 (2019)"},{"key":"33_CR9","doi-asserted-by":"crossref","unstructured":"Eppe, M., Nguyen, P.D.H., Wermter, S.: From semantics to execution: integrating action planning with reinforcement learning for robotic causal problem-solving. Front. Robot. AI 6 (2019)","DOI":"10.3389\/frobt.2019.00123"},{"key":"33_CR10","doi-asserted-by":"crossref","unstructured":"Forestier, S., Oudeyer, P.Y.: Modular active curiosity-driven discovery of tool use. In: IEEE International Conference on Intelligent Robots and Systems, pp. 3965\u20133972. IEEE (2016)","DOI":"10.1109\/IROS.2016.7759584"},{"issue":"1\u20132","key":"33_CR11","doi-asserted-by":"publisher","first-page":"137","DOI":"10.1007\/s00422-011-0424-z","volume":"104","author":"K Friston","year":"2011","unstructured":"Friston, K., Mattout, J., Kilner, J.: Action understanding and active inference. Biol. Cybern. 104(1\u20132), 137\u2013160 (2011)","journal-title":"Biol. Cybern."},{"issue":"12","key":"33_CR12","doi-asserted-by":"publisher","first-page":"758","DOI":"10.1038\/s41583-018-0078-0","volume":"19","author":"J Gottlieb","year":"2018","unstructured":"Gottlieb, J., Oudeyer, P.Y.: Towards a neuroscience of active sampling and curiosity. Nat. Rev. Neurosci. 19(12), 758\u2013770 (2018)","journal-title":"Nat. Rev. Neurosci."},{"key":"33_CR13","doi-asserted-by":"crossref","unstructured":"Hafez, M.B., Weber, C., Wermter, S.: Curiosity-driven exploration enhances motor skills of continuous actor-critic learner. In: IEEE International Conference on Development and Learning and Epigenetic Robotics (ICDL-EpiRob), pp. 39\u201346. IEEE (2017)","DOI":"10.1109\/DEVLRN.2017.8329785"},{"key":"33_CR14","doi-asserted-by":"publisher","first-page":"170","DOI":"10.1016\/j.artint.2015.05.002","volume":"247","author":"T Hester","year":"2017","unstructured":"Hester, T., Stone, P.: Intrinsically motivated model learning for developing curious robots. Artif. Intell. 247, 170\u201386 (2017)","journal-title":"Artif. Intell."},{"key":"33_CR15","unstructured":"Jaderberg, M., et al.: Reinforcement learning with unsupervised auxiliary tasks. In: International Conference on Learning Representations (ICLR), p. online (2017)"},{"key":"33_CR16","unstructured":"Jiang, Y., Gu, S.S., Murphy, K.P., Finn, C.: Language as an abstraction forhierarchical deep reinforcement learning. In: Neural Information Processing Systems (NeurIPS), pp. 9419\u20139431. Curran Associates, Inc. (2019)"},{"key":"33_CR17","unstructured":"Kingma, D.P., Ba, J.L.: Adam: a method for stochastic optimization. In: International Conference on Learning Representations (ICLR), p. online (2015)"},{"key":"33_CR18","unstructured":"Kulkarni, T.D., Narasimhan, K., Saeedi, A., Tenenbaum, J.B.: Hierarchical deep reinforcement learning: integrating temporal abstraction and intrinsic motivation. In: Conference on Neural Information Processing Systems (NeurIPS), pp. 3675\u20133683 (2016)"},{"key":"33_CR19","unstructured":"Levy, A., Konidaris, G., Platt, R., Saenko, K.: Learning multi-level hierarchies with hindsight. In: International Conference on Learning Representations (ICLR), p. online (2019)"},{"key":"33_CR20","unstructured":"Lillicrap, T.P., et al.: Continuous control with deep reinforcement learning. In: International Conference on Learning Representations (ICLR), p. online (2016)"},{"issue":"7540","key":"33_CR21","doi-asserted-by":"publisher","first-page":"529","DOI":"10.1038\/nature14236","volume":"518","author":"V Mnih","year":"2015","unstructured":"Mnih, V., et al.: Human-level control through deep reinforcement learning. Nature 518(7540), 529\u2013533 (2015)","journal-title":"Nature"},{"key":"33_CR22","unstructured":"Nachum, O., Gu, S.S., Lee, H., Levine, S.: Data-efficient hierarchical reinforcement learning. In: Conference on Neural Information Processing Systems (NeurIPS), pp. 3303\u20133313. Curran Associates, Inc. (2018)"},{"key":"33_CR23","doi-asserted-by":"crossref","unstructured":"Pathak, D., Agrawal, P., Efros, A.A., Darrell, T.: Curiosity-driven exploration by self-supervised prediction. In: International Conference on Machine Learning (ICML), pp. 2778\u20132787. PMLR (2017)","DOI":"10.1109\/CVPRW.2017.70"},{"key":"33_CR24","doi-asserted-by":"crossref","unstructured":"Pezzulo, G., Rigoli, F., Friston, K.J.: Hierarchical Active Inference: A Theory of Motivated Control (2018)","DOI":"10.1016\/j.tics.2018.01.009"},{"key":"33_CR25","doi-asserted-by":"crossref","unstructured":"Rohmer, E., Singh, S.P.N., Freese, M.: Coppeliasim (formerly v-rep): a versatile and scalable robot simulation framework. In: Proceedings of the International Conference on Intelligent Robots and Systems (IROS) (2013)","DOI":"10.1109\/IROS.2013.6696520"},{"key":"33_CR26","unstructured":"Schaul, T., Horgan, D., Gregor, K., Silver, D.: Universal value function approximators. In: International Conference on Machine Learning (ICML), vol. 37, pp. 1312\u20131320. PMLR (2015)"},{"key":"33_CR27","first-page":"39","volume":"3","author":"G Schillaci","year":"2016","unstructured":"Schillaci, G., Hafner, V.V., Lara, B.: Exploration behaviors, body representations, and simulation processes for the development of cognition in artificial agents. Front. Robot. AI 3, 39 (2016)","journal-title":"Front. Robot. AI"},{"issue":"3","key":"33_CR28","doi-asserted-by":"publisher","first-page":"230","DOI":"10.1109\/TAMD.2010.2056368","volume":"2","author":"J Schmidhuber","year":"2010","unstructured":"Schmidhuber, J.: Formal theory of creativity, fun, and intrinsic motivation (1990\u20132010). IEEE Trans. Auton. Mental Dev. 2(3), 230\u2013247 (2010)","journal-title":"IEEE Trans. Auton. Mental Dev."},{"key":"33_CR29","unstructured":"Silver, D., Lever, G., Hees, N., Degris, T., Wierstra, D., Riedmiller, M.: Deterministic policy gradient algorithms. In: International Conference on Machine Learning (ICML), vol. 32, pp. 387\u2013395 (2014)"},{"key":"33_CR30","unstructured":"Vezhnevets, A.S., et al.: FeUdal networks for hierarchical reinforcement learning. In: International Conference on Machine Learning (ICML), vol. 70, pp. 3540\u20133549. PMLR (2017)"},{"key":"33_CR31","unstructured":"Watters, N., Matthey, L., Bosnjak, M., Burgess, C.P., Lerchner, A.: COBRA: Data-Efficient Model-Based RL through Unsupervised Object Discovery and Curiosity-Driven Exploration (2019)"}],"container-title":["Lecture Notes in Computer Science","Artificial Neural Networks and Machine Learning \u2013 ICANN 2020"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-030-61616-8_33","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2022,11,23]],"date-time":"2022-11-23T18:28:45Z","timestamp":1669228125000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-3-030-61616-8_33"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2020]]},"ISBN":["9783030616151","9783030616168"],"references-count":31,"URL":"https:\/\/doi.org\/10.1007\/978-3-030-61616-8_33","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"type":"print","value":"0302-9743"},{"type":"electronic","value":"1611-3349"}],"subject":[],"published":{"date-parts":[[2020]]},"assertion":[{"value":"14 October 2020","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"ICANN","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"International Conference on Artificial Neural Networks","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Bratislava","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Slovakia","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2020","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"15 September 2020","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"18 September 2020","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"29","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"icann2020","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"https:\/\/e-nns.org\/icann2020\/","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Single-blind","order":1,"name":"type","label":"Type","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"OCS","order":2,"name":"conference_management_system","label":"Conference Management System","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"249","order":3,"name":"number_of_submissions_sent_for_review","label":"Number of Submissions Sent for Review","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"139","order":4,"name":"number_of_full_papers_accepted","label":"Number of Full Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"0","order":5,"name":"number_of_short_papers_accepted","label":"Number of Short Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"56% - The value is computed by the equation \"Number of Full Papers Accepted \/ Number of Submissions Sent for Review * 100\" and then rounded to a whole number.","order":6,"name":"acceptance_rate_of_full_papers","label":"Acceptance Rate of Full Papers","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"3","order":7,"name":"average_number_of_reviews_per_paper","label":"Average Number of Reviews per Paper","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"2.5","order":8,"name":"average_number_of_papers_per_reviewer","label":"Average Number of Papers per Reviewer","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"Yes","order":9,"name":"external_reviewers_involved","label":"External Reviewers Involved","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"*The conference was postponed to 2021 due to the COVID-19 pandemic.","order":10,"name":"additional_info_on_review_process","label":"Additional Info on Review Process","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}}]}}