{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,2,21]],"date-time":"2025-02-21T17:45:51Z","timestamp":1740159951280,"version":"3.37.3"},"reference-count":38,"publisher":"Springer Science and Business Media LLC","issue":"6","license":[{"start":{"date-parts":[[2023,1,4]],"date-time":"2023-01-04T00:00:00Z","timestamp":1672790400000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2023,1,4]],"date-time":"2023-01-04T00:00:00Z","timestamp":1672790400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Evolving Systems"],"published-print":{"date-parts":[[2023,12]]},"DOI":"10.1007\/s12530-022-09478-6","type":"journal-article","created":{"date-parts":[[2023,1,4]],"date-time":"2023-01-04T18:02:51Z","timestamp":1672855371000},"page":"957-980","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":0,"title":["Autonomous acquisition of arbitrarily complex skills using locality based graph theoretic features: a syntactic approach to hierarchical reinforcement learning"],"prefix":"10.1007","volume":"14","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-2759-4924","authenticated-orcid":false,"given":"Zeynep","family":"Kumralba\u015f","sequence":"first","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0002-9680-1325","authenticated-orcid":false,"given":"Semiha Hazel","family":"\u00c7avu\u015f","sequence":"additional","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0001-7680-3182","authenticated-orcid":false,"given":"Kutalm\u0131\u015f","family":"Co\u015fkun","sequence":"additional","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0001-6880-5153","authenticated-orcid":false,"given":"Borahan","family":"T\u00fcmer","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2023,1,4]]},"reference":[{"key":"9478_CR1","doi-asserted-by":"crossref","unstructured":"Aktunc R, Toroslu IH, Ozer M, et\u00a0al (2015) A dynamic modularity based community detection algorithm for large-scale networks: Dslm. In: Proceedings of the 2015 IEEE\/ACM international conference on advances in social networks analysis and mining 2015, pp 1177\u20131183","DOI":"10.1145\/2808797.2808822"},{"key":"9478_CR2","doi-asserted-by":"publisher","first-page":"P10.008","DOI":"10.1088\/1742-5468\/2008\/10\/P10008","volume":"2008","author":"VD Blondel","year":"2008","unstructured":"Blondel VD, Guillaume JL, Lambiotte R et al (2008) Fast unfolding of communities in large networks. J Stat Mech-Theory Exp 2008:P10.008","journal-title":"J Stat Mech-Theory Exp"},{"key":"9478_CR3","doi-asserted-by":"crossref","unstructured":"Bohlin L, Edler D, Lancichinetti A, et\u00a0al (2014) Community detection and visualization of networks with the map equation framework. In: Measuring scholarly impact. Springer, pp 3\u201334","DOI":"10.1007\/978-3-319-10377-8_1"},{"key":"9478_CR4","doi-asserted-by":"publisher","unstructured":"Cockcroft M, Mawjee S, James S, et\u00a0al (2020) Learning options from demonstration using skill segmentation. In: 2020 International SAUPEC\/RobMech\/PRASA Conference, pp 1\u20136. https:\/\/doi.org\/10.1109\/SAUPEC\/RobMech\/PRASA48453.2020.9040988","DOI":"10.1109\/SAUPEC\/RobMech\/PRASA48453.2020.9040988"},{"issue":"1","key":"9478_CR5","doi-asserted-by":"publisher","first-page":"15","DOI":"10.1007\/s13278-016-0325-1","volume":"6","author":"M Cordeiro","year":"2016","unstructured":"Cordeiro M, Sarmento RP, Gama J (2016) Dynamic community detection in evolving networks using locality modularity optimization. Soc Netw Anal Min 6(1):15","journal-title":"Soc Netw Anal Min"},{"key":"9478_CR6","doi-asserted-by":"publisher","first-page":"104","DOI":"10.1007\/s10994-016-5580-x","volume":"2016","author":"C Daniel","year":"2016","unstructured":"Daniel C, van Hoof H, Peters J et al (2016) Probabilistic inference for determining options in reinforcement learning. Mach Learn 2016:104. https:\/\/doi.org\/10.1007\/s10994-016-5580-x","journal-title":"Mach Learn"},{"key":"9478_CR7","unstructured":"Davoodabadi M, Beigy H (2011a) A new method for discovering subgoals and constructing options in reinforcement learning. In: Proceedings of the 5th Indian international conference on artificial intelligence, IICAI 2011 pp 441\u2013450"},{"key":"9478_CR8","unstructured":"Davoodabadi M, Beigy H (2011b) A new method for discovering subgoals and constructing options in reinforcement learning. In: IICAI, pp 441\u2013450"},{"issue":"105","key":"9478_CR9","first-page":"574","volume":"82","author":"MD Farahani","year":"2019","unstructured":"Farahani MD, Mozayani N (2019) Automatic construction and evaluation of macro-actions in reinforcement learning. Appl Soft Comput 82(105):574","journal-title":"Appl Soft Comput"},{"issue":"2","key":"9478_CR10","doi-asserted-by":"publisher","first-page":"298","DOI":"10.21136\/CMJ.1973.101168","volume":"23","author":"M Fiedler","year":"1973","unstructured":"Fiedler M (1973) Algebraic connectivity of graphs. Czechoslovak Math J 23(2):298\u2013305","journal-title":"Czechoslovak Math J"},{"key":"9478_CR11","doi-asserted-by":"crossref","unstructured":"Fu KS (1977) Introduction to syntactic pattern recognition. In: Syntactic pattern recognition, applications. Springer, p 1\u201330","DOI":"10.1007\/978-3-642-66438-0_1"},{"key":"9478_CR12","unstructured":"Ghafoorian M, Taghizadeh N, Beigy H (2013) Automatic abstraction in reinforcement learning using ant system algorithm. IN: AAAI Spring Symposium\u2014Technical Report, pp 9\u201314"},{"key":"9478_CR13","unstructured":"Jinnai Y, Park JW, Abel D, et\u00a0al (2019) Discovering options for exploration by minimizing cover time. In: International Conference on Machine Learning, PMLR, pp 3130\u20133139"},{"issue":"3","key":"9478_CR14","doi-asserted-by":"publisher","first-page":"227","DOI":"10.1007\/s12530-017-9193-9","volume":"9","author":"SJ Kazemitabar","year":"2018","unstructured":"Kazemitabar SJ, Taghizadeh N, Beigy H (2018) A graph-theoretic approach toward autonomous skill acquisition in reinforcement learning. Evol Syst 9(3):227\u2013244","journal-title":"Evol Syst"},{"issue":"3","key":"9478_CR15","doi-asserted-by":"publisher","first-page":"293","DOI":"10.1007\/BF00992699","volume":"8","author":"LJ Lin","year":"1992","unstructured":"Lin LJ (1992) Self-improving reactive agents based on reinforcement learning, planning and teaching. Mach Learn 8(3):293\u2013321","journal-title":"Mach Learn"},{"key":"9478_CR16","unstructured":"Machado MC, Bellemare MG, Bowling M (2017) A laplacian framework for option discovery in reinforcement learning. In: International conference on machine learning, PMLR, pp 2295\u20132304"},{"key":"9478_CR17","unstructured":"McGovern A, Barto AG (2001) Automatic discovery of subgoals in reinforcement learning using diverse density. In: Computer Science Department Faculty Publication Series, p 8"},{"issue":"2","key":"9478_CR18","doi-asserted-by":"publisher","DOI":"10.1103\/PhysRevE.69.026113","volume":"69","author":"ME Newman","year":"2004","unstructured":"Newman ME, Girvan M (2004) Finding and evaluating community structure in networks. Phys Rev E 69(2):026113","journal-title":"Phys Rev E"},{"issue":"036","key":"9478_CR19","doi-asserted-by":"publisher","first-page":"106","DOI":"10.1103\/PhysRevE.76.036106","volume":"76","author":"N Raghavan","year":"2007","unstructured":"Raghavan N, Albert R, Kumara S (2007) Near linear time algorithm to detect community structures in large-scale networks. Phys Rev E Stat Nonlinear Soft Matter Phys 76(036):106. https:\/\/doi.org\/10.1103\/PhysRevE.76.036106","journal-title":"Phys Rev E Stat Nonlinear Soft Matter Phys"},{"issue":"1\u20132","key":"9478_CR20","doi-asserted-by":"publisher","first-page":"20","DOI":"10.1016\/j.physd.2006.09.009","volume":"224","author":"J Reichardt","year":"2006","unstructured":"Reichardt J, Bornholdt S (2006) When are networks truly modular? Phys D: Nonlinear Phenom 224(1\u20132):20\u201326","journal-title":"Phys D: Nonlinear Phenom"},{"key":"9478_CR21","doi-asserted-by":"publisher","first-page":"447","DOI":"10.24507\/ijicic.18.02.447","volume":"2022","author":"GE Setyawan","year":"2022","unstructured":"Setyawan GE, Sawada H, Hartono P (2022) Combinations of micro-macro states and subgoals discovery in hierarchical reinforcement learning for path finding. Int J Innov Comput Inf Control 2022:447\u2013462. https:\/\/doi.org\/10.24507\/ijicic.18.02.447","journal-title":"Int J Innov Comput Inf Control"},{"issue":"2","key":"9478_CR22","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1007\/s10458-022-09571-9","volume":"36","author":"E Shafipour Yourdshahi","year":"2022","unstructured":"Shafipour Yourdshahi E, do Carmo-Alves MA, Varma A et al (2022) On-line estimators for ad-hoc task execution: learning types and parameters of teammates for effective teamwork. Autonomous Agents Multi-Agent Syst 36(2):1\u201349","journal-title":"Autonomous Agents Multi-Agent Syst"},{"key":"9478_CR23","doi-asserted-by":"publisher","first-page":"104","DOI":"10.1016\/j.patrec.2016.08.009","volume":"87","author":"F Shoeleh","year":"2017","unstructured":"Shoeleh F, Asadpour M (2017) Graph based skill acquisition and transfer learning for continuous reinforcement learning domains. Pattern Recogn Lett 87:104\u2013116","journal-title":"Pattern Recogn Lett"},{"key":"9478_CR24","unstructured":"Simsek O, Barreto AS (2008) Skill characterization based on betweenness. In: Advances in neural information processing systems, pp 1497\u20131504"},{"key":"9478_CR25","doi-asserted-by":"crossref","unstructured":"\u015eim\u015fek \u00d6, Barto AG (2004a) Using relative novelty to identify useful temporal abstractions in reinforcement learning. In: Proceedings of the twenty-first international conference on Machine learning, p\u00a095","DOI":"10.1145\/1015330.1015353"},{"key":"9478_CR26","doi-asserted-by":"crossref","unstructured":"\u015eim\u015fek \u00d6, Barto AG (2004b) Using relative novelty to identify useful temporal abstractions in reinforcement learning. In: Proceedings of the twenty-first international conference on Machine learning, p\u00a095","DOI":"10.1145\/1015330.1015353"},{"issue":"3","key":"9478_CR27","doi-asserted-by":"publisher","first-page":"315","DOI":"10.1080\/01969722.2019.1691851","volume":"51","author":"BG S\u00fcrmeli","year":"2020","unstructured":"S\u00fcrmeli BG, T\u00fcmer MB (2020) Multivariate time series clustering and its application in industrial systems. Cybern Syst 51(3):315\u2013334. https:\/\/doi.org\/10.1080\/01969722.2019.1691851","journal-title":"Cybern Syst"},{"key":"9478_CR28","doi-asserted-by":"crossref","unstructured":"Stolle M, Precup D (2002) Learning options in reinforcement learning. In: International symposium on abstraction, reformulation, and approximation. Springer, pp 212\u2013223","DOI":"10.1007\/3-540-45622-8_16"},{"key":"9478_CR29","volume-title":"Reinforcement learning: an introduction","author":"RS Sutton","year":"2018","unstructured":"Sutton RS, Barto AG (2018) Reinforcement learning: an introduction. MIT press, Cambridge"},{"key":"9478_CR30","unstructured":"Sutton RS, Precup D, Singh SP (1998) Intra-option learning about temporally abstract actions. In: ICML, pp 556\u2013564"},{"issue":"1\u20132","key":"9478_CR31","doi-asserted-by":"publisher","first-page":"181","DOI":"10.1016\/S0004-3702(99)00052-1","volume":"112","author":"RS Sutton","year":"1999","unstructured":"Sutton RS, Precup D, Singh S (1999) Between mdps and semi-mdps: a framework for temporal abstraction in reinforcement learning. Artif Intell 112(1\u20132):181\u2013211","journal-title":"Artif Intell"},{"issue":"1","key":"9478_CR32","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1038\/s41598-019-41695-z","volume":"9","author":"VA Traag","year":"2019","unstructured":"Traag VA, Waltman L, Van Eck NJ (2019) From louvain to leiden: guaranteeing well-connected communities. Sci Rep 9(1):1\u201312","journal-title":"Sci Rep"},{"issue":"6","key":"9478_CR33","doi-asserted-by":"publisher","first-page":"951","DOI":"10.1109\/TSMCB.2002.804365","volume":"33","author":"M Tumer","year":"2003","unstructured":"Tumer M, Belfore L, Ropella K (2003) A syntactic methodology for automatic diagnosis by analysis of continuous time measurements using hierarchical signal representations. IEEE Trans Syst Man Cybern Part B (Cybern) 33(6):951\u2013965. https:\/\/doi.org\/10.1109\/TSMCB.2002.804365","journal-title":"IEEE Trans Syst Man Cybern Part B (Cybern)"},{"issue":"11","key":"9478_CR34","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1140\/epjb\/e2013-40829-0","volume":"86","author":"L Waltman","year":"2013","unstructured":"Waltman L, Van Eck NJ (2013) A smart local moving algorithm for large-scale modularity-based community detection. Eur Phys J B 86(11):1\u201314","journal-title":"Eur Phys J B"},{"issue":"3","key":"9478_CR35","doi-asserted-by":"publisher","first-page":"279","DOI":"10.1007\/BF00992698","volume":"8","author":"CJ Watkins","year":"1992","unstructured":"Watkins CJ, Dayan P (1992) Q-learning. Mach Learn 8(3):279\u2013292","journal-title":"Mach Learn"},{"key":"9478_CR36","doi-asserted-by":"publisher","first-page":"156","DOI":"10.1155\/2018\/2085721","volume":"2018","author":"X Xu","year":"2018","unstructured":"Xu X, Yang M, Li G et al (2018) Constructing temporally extended actions through incremental community detection. Comput Intell Neurosci 2018:156","journal-title":"Comput Intell Neurosci"},{"issue":"108","key":"9478_CR37","first-page":"151","volume":"241","author":"X Zhu","year":"2022","unstructured":"Zhu X, Zhang R, Zhu W (2022) Mdmd options discovery for accelerating exploration in sparse-reward domains. Knowl-Based Syst 241(108):151","journal-title":"Knowl-Based Syst"},{"key":"9478_CR38","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1109\/TKDE.2019.2951419","volume":"2019","author":"D Zhuang","year":"2019","unstructured":"Zhuang D, Chang MJ, Li M (2019) Dynamo: dynamic community detection by incrementally maximizing modularity. IEEE Trans Knowl Data Eng 2019:1\u20131. https:\/\/doi.org\/10.1109\/TKDE.2019.2951419","journal-title":"IEEE Trans Knowl Data Eng"}],"container-title":["Evolving Systems"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s12530-022-09478-6.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s12530-022-09478-6\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s12530-022-09478-6.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2023,10,16]],"date-time":"2023-10-16T09:33:31Z","timestamp":1697448811000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s12530-022-09478-6"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023,1,4]]},"references-count":38,"journal-issue":{"issue":"6","published-print":{"date-parts":[[2023,12]]}},"alternative-id":["9478"],"URL":"https:\/\/doi.org\/10.1007\/s12530-022-09478-6","relation":{},"ISSN":["1868-6478","1868-6486"],"issn-type":[{"type":"print","value":"1868-6478"},{"type":"electronic","value":"1868-6486"}],"subject":[],"published":{"date-parts":[[2023,1,4]]},"assertion":[{"value":"18 July 2022","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"7 December 2022","order":2,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"4 January 2023","order":3,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Declarations"}},{"value":"The authors have no relevant financial or non-financial interests to disclose.","order":2,"name":"Ethics","group":{"name":"EthicsHeading","label":"Conflict of interest"}},{"value":"Not applicable","order":3,"name":"Ethics","group":{"name":"EthicsHeading","label":"Ethical standard"}},{"value":"Not applicable","order":4,"name":"Ethics","group":{"name":"EthicsHeading","label":"Consent to participate"}},{"value":"Not applicable","order":5,"name":"Ethics","group":{"name":"EthicsHeading","label":"Consent for publication"}},{"value":"The software used in this study is available from the corresponding author on reasonable request.","order":6,"name":"Ethics","group":{"name":"EthicsHeading","label":"Code availability"}}]}}