{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,26]],"date-time":"2026-04-26T20:08:41Z","timestamp":1777234121589,"version":"3.51.4"},"reference-count":44,"publisher":"Springer Science and Business Media LLC","issue":"1","license":[{"start":{"date-parts":[[2021,2,10]],"date-time":"2021-02-10T00:00:00Z","timestamp":1612915200000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2021,2,10]],"date-time":"2021-02-10T00:00:00Z","timestamp":1612915200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"funder":[{"DOI":"10.13039\/501100003448","name":"General Secretariat for Research and Technology","doi-asserted-by":"publisher","award":["National Matching Funds 2017-2018"],"award-info":[{"award-number":["National Matching Funds 2017-2018"]}],"id":[{"id":"10.13039\/501100003448","id-type":"DOI","asserted-by":"publisher"}]},{"name":"SESAR","award":["DART"],"award-info":[{"award-number":["DART"]}]}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Neural Comput & Applic"],"published-print":{"date-parts":[[2023,1]]},"DOI":"10.1007\/s00521-021-05748-7","type":"journal-article","created":{"date-parts":[[2021,2,10]],"date-time":"2021-02-10T19:54:06Z","timestamp":1612986846000},"page":"147-159","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":24,"title":["Hierarchical multiagent reinforcement learning schemes for air traffic management"],"prefix":"10.1007","volume":"35","author":[{"given":"Christos","family":"Spatharis","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Alevizos","family":"Bastas","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Theocharis","family":"Kravaris","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Konstantinos","family":"Blekas","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-5451-622X","authenticated-orcid":false,"given":"George A.","family":"Vouros","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Jose Manuel","family":"Cordero","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2021,2,10]]},"reference":[{"key":"5748_CR1","unstructured":"Abel D, Hershkowitz DE, Littman ML (2016) Near optimal behavior via approximate state abstraction. In: International conference on machine learning (ICML\u201816), vol 48, pp 2915\u20132923"},{"key":"5748_CR2","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1007\/s10458-010-9142-5","volume":"24","author":"AK Agogino","year":"2012","unstructured":"Agogino AK, Tumer K (2012) A multiagent approach to managing air traffic flow. Auton Agents Multiagent Syst 24:1\u201325","journal-title":"Auton Agents Multiagent Syst"},{"key":"5748_CR3","unstructured":"Andreas J, Klein D, Levine S (2017) Modular multitask reinforcement learning with policy sketches. In: 34th international conference on machine learning (ICML), pp 166\u20131751"},{"key":"5748_CR4","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-37583-5","volume-title":"Visual analytics of movement","author":"G Andrienko","year":"2013","unstructured":"Andrienko G, Andrienko N, Bak P, Keim D, Wrobel S (2013) Visual analytics of movement. Springer, Berlin"},{"key":"5748_CR5","doi-asserted-by":"crossref","unstructured":"Bacon P, Harb J, Precup D (2017) The option-critic architecture. In: Proceedings of the thirty-first AAAI conference on artificial intelligence, AAAI\u201917, pp 1726\u20131734. AAAI Press","DOI":"10.1609\/aaai.v31i1.10916"},{"key":"5748_CR6","unstructured":"Bai A, Srivastava S, Russell S (2016) Markovian state and action abstractions for MDPs via hierarchical MCTS. In: Proceedings of the twenty-fifth international joint conference on artificial intelligence, IJCAI\u201916, pp 3029\u20133037. AAAI Press"},{"key":"5748_CR7","doi-asserted-by":"crossref","unstructured":"Bazzan ALC, Wahle J, Kl\u00fcgl F (1999) Agents in traffic modelling\u2014from reactive to social behaviour. In: 23rd annual german conference on artificial intelligence, pp 303\u2013306","DOI":"10.1007\/3-540-48238-5_28"},{"key":"5748_CR8","doi-asserted-by":"crossref","unstructured":"Chen J, Wang Z, Tomizuka M (2018) Deep hierarchical reinforcement learning for autonomous driving with distinct behaviors. In: 2018 IEEE intelligent vehicles symposium (IV), pp 1239\u20131244","DOI":"10.1109\/IVS.2018.8500368"},{"key":"5748_CR9","unstructured":"Colby M, Tumer K (2013) Multiagent reinforcement learnng in a distributed sensor network with indirect feedback. In: International conference on autonomous agents and multi-agent systems (AAMAS\u201913), pp 941\u2013948"},{"key":"5748_CR10","unstructured":"Cook AJ, Tanner G (2015) European airline delay cost reference values. http:\/\/www.eurocontrol.int\/publications\/european-airline-delaycost-reference-values"},{"key":"5748_CR11","unstructured":"Dayan P, Hinton GE (1992) Feudal reinforcement learning. In: Advances in neural information processing systems, [NIPS Conference], vol 5, pp 271\u2013278"},{"key":"5748_CR12","unstructured":"Delvin S, Yliniemi L, Kudenko D, Tumer K (2014) Potential-based difference rewars for multiagent reinforcement learning. In: International conference on autonomous agents and multi-agent systems (AAMAS\u201914), pp 165\u2013172"},{"key":"5748_CR13","doi-asserted-by":"publisher","first-page":"227","DOI":"10.1613\/jair.639","volume":"13","author":"T Dietterich","year":"2000","unstructured":"Dietterich T (2000) Hierarchical reinforcement learning with the maxq value function decomposition. J Artif Intell Res 13:227\u2013303","journal-title":"J Artif Intell Res"},{"key":"5748_CR14","unstructured":"Frans K, Ho J, Chen X, Abbeel P, Schulman J (2017) Meta learning shared hierarchies. Technical Report. arXiv preprint arXiv:1710.09767"},{"key":"5748_CR15","unstructured":"Guestrin C, Lagoudakis M, Parr R (2002) Coordinated reinforcement learning. In: International conference on machine learning (ICML\u201802), pp 227\u2013234"},{"key":"5748_CR16","unstructured":"Jong N, Stone P (2005) State abstraction discovery from irrelevant state variables. In: International joint conference on artificial intelligence (IJCAI \u201905), pp 752\u2013757"},{"key":"5748_CR17","doi-asserted-by":"crossref","unstructured":"Karp R, Koutsoupias E, Papadimitriou C, Shenker S (2000) Optimization problems in congestion control. In: 16th Annual symposium on foundations of computer science, pp 66\u201374","DOI":"10.1109\/SFCS.2000.892066"},{"key":"5748_CR18","first-page":"1789","volume":"7","author":"JR Kok","year":"2006","unstructured":"Kok JR, Vlassis N (2006) Collaborative multiagent reinforcement learning by payoff propagation. J Mach Learn Res 7:1789\u20131828","journal-title":"J Mach Learn Res"},{"key":"5748_CR19","unstructured":"Konidaris G, Barto A (2009) Efficient skill learning using abstraction selection. In: International joint conference on artificial intelligence (IJCAI \u201909), pp 1107\u20131112"},{"key":"5748_CR20","unstructured":"Kravaris T, Spatharis C, Bastas A, Vouros GA, Blekas K, Andrienko G, Andrienko N, Garcia JM (2019) Resolving congestions in the air traffic management domain via multiagent reinforcement learning methods. Technical Report. arXiv preprint arXiv:1912.06860"},{"key":"5748_CR21","doi-asserted-by":"crossref","unstructured":"Kravaris T, Vouros G, Spatharis C, Blekas K, Chalkiadakis G (2017) Learning policies for resolving demand\u2013capacity imbalances during pre-tactical air traffic management. In: Multiagent system technologies\u201415th German conference (MATES\u201817), pp 238\u2013255","DOI":"10.1007\/978-3-319-64798-2_15"},{"key":"5748_CR22","unstructured":"Kulkarni T, Narasimhan K, Saeedi A, Tenenbaum J (2016) Hierarchical deep reinforcement learning: Integrating temporal abstraction and intrinsic motivation. In: Advances in Neural Information Processing Systems (NIPS\u201916), pp 3675\u20133683"},{"key":"5748_CR23","unstructured":"Li L, Walsh T, Littman M (2006) Towards a unified theory of state abstraction for MDPs. In: International symposium on artificial intelligence and mathematics (ISAIM\u201806)"},{"key":"5748_CR24","doi-asserted-by":"publisher","first-page":"485","DOI":"10.1007\/s10514-019-09871-2","volume":"44","author":"A Ma","year":"2020","unstructured":"Ma A, Ouimet M, Cort\u00e9s J (2020) Hierarchical reinforcement learning via dynamic subspace search for multi-agent planning. Auton Robot 44:485\u2013503. https:\/\/doi.org\/10.1007\/s10514-019-09871-2","journal-title":"Auton Robot"},{"key":"5748_CR25","doi-asserted-by":"crossref","unstructured":"Makar R, Mahadevan S, Ghavamzadeh M (2001) Hierarchical multi-agent reinforcement learning. In: Proceedings of the fifth international conference on autonomous agents, AGENTS\u201901, pp 246\u2013253","DOI":"10.1145\/375735.376302"},{"key":"5748_CR26","unstructured":"Malialis K, Delvin S, Kudenko D (2016) Resource abstraction for reinforcement learning in multiagent congestion problems. In: International conference on autonomous agents and multi-agent systems (AAMAS\u201916), pp 503\u2013511"},{"key":"5748_CR27","doi-asserted-by":"publisher","unstructured":"Mannor S, Menanche I, Hoze A, Klein U (2004) Dynamic abstraction in reinforcement learning via clustering. In: International conference on machine learning (ICML\u201804). https:\/\/doi.org\/10.1145\/1015330.1015355","DOI":"10.1145\/1015330.1015355"},{"key":"5748_CR28","unstructured":"McGovern A, Barto A (2001) Automatic discovery of subgoals in reinforcement learning using diverse density. In: Proceedings of the eighteenth international conference on machine learning (ICML\u201901), pp 361\u2013368"},{"key":"5748_CR29","unstructured":"Meyers C (2006) Network flow problems and congestion games: complexity and approximation results. Ph.D. thesis, MIT"},{"issue":"1","key":"5748_CR30","doi-asserted-by":"publisher","first-page":"56","DOI":"10.1016\/S0022-0531(03)00106-6","volume":"114","author":"I Milchtaich","year":"2004","unstructured":"Milchtaich I (2004) Social optimality and cooperation in nonatomic congestion games. J Econ Theory 114(1):56\u201387","journal-title":"J Econ Theory"},{"key":"5748_CR31","unstructured":"Nachum T, Gu SS, Lee H, Levine S (2018) Data-efficient hierarchical reinforcement learning. In: 32nd Conference on neural information processing systems (NeurIPS 2018), pp 3303\u20133313"},{"key":"5748_CR32","unstructured":"Parr R, Russell S (1998) Reinforcement learning with hierarchies of machines. In: Proceedings of the 1997 conference on advances in neural information processing systems (NIPS'97), vol 10, pp 1043\u20131049"},{"issue":"4","key":"5748_CR33","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1145\/3072959.3073602","volume":"36","author":"XB Peng","year":"2017","unstructured":"Peng XB, Berseth G, Yin K, van de Panne M (2017) Deeploco: dynamic locomotion skills using hierarchical deep reinforcement learning. ACM Trans Graph 36(4):1\u201313. https:\/\/doi.org\/10.1145\/3072959.3073602","journal-title":"ACM Trans Graph"},{"issue":"15","key":"5748_CR34","doi-asserted-by":"publisher","first-page":"1508","DOI":"10.1016\/j.dam.2011.01.019","volume":"159","author":"M Penn","year":"2011","unstructured":"Penn M, Polukarov M, Tennenholtz M (2011) Congestion games with failures. Discr Appl Math 159(15):1508\u20131525","journal-title":"Discr Appl Math"},{"key":"5748_CR35","unstructured":"Radulescu R, Vrancx P, Nowe A (2017) Analysing congestion problems in multi-agent reinforcement learning. In: Proceedings of the 16th conference on autonomous agents and multiagent systems (AAMAS\u201917), pp 1705\u20131707"},{"issue":"7","key":"5748_CR36","doi-asserted-by":"publisher","first-page":"e0180234","DOI":"10.1371\/journal.pone.0180234","volume":"12","author":"D Rasmussen","year":"2017","unstructured":"Rasmussen D, Voelker A, Eliasmith C (2017) A neural model of hierarchical reinforcement learning. PLoS One 12(7):e0180234. https:\/\/doi.org\/10.1371\/journal.pone.0180234","journal-title":"PLoS One"},{"key":"5748_CR37","unstructured":"Riemer M, Liu M, Tesauro G (2018) Learning abstract options. In: Proceedings of the 32nd international conference on neural information processing systems, NIPS\u201918, pp 10445-10455. Curran Associates Inc., Red Hook"},{"key":"5748_CR38","doi-asserted-by":"publisher","first-page":"65","DOI":"10.1007\/BF01737559","volume":"2","author":"RW Rosenthal","year":"1973","unstructured":"Rosenthal RW (1973) A class of games processing pure-strategy nash equilibria. Int J Game Theory 2:65\u201367","journal-title":"Int J Game Theory"},{"key":"5748_CR39","doi-asserted-by":"crossref","unstructured":"Spatharis C, Blekas K, Bastas A, Kravaris T, Vouros GA (2019) Collaborative multiagent reinforcement learning schemes for air traffic management. In: 10th international conference on information, intelligence, systems and applications (IISA), pp 1\u20138","DOI":"10.1109\/IISA.2019.8900719"},{"key":"5748_CR40","doi-asserted-by":"crossref","unstructured":"Spatharis C, Kravaris T, Vouros GA, Blekas K, Chalkadiakis G, Garcia JMC, Fern\u00e1ndez EC (2018) Multiagent reinforcement learning methods to resolve demand capacity balance problems. In: Hellenic A.I. conference (SETN 2018), pp 2:1\u20132:9","DOI":"10.1109\/DASC.2018.8569346"},{"key":"5748_CR41","doi-asserted-by":"crossref","unstructured":"Spatharis C, Kravaris T, Vouros GA, Blekas K, Cordero JMG (2018) Multiagent reinforcement learning methods for resolving demand\u2014capacity imbalances. In: Digital avionics systems conference (DASC\u201918)","DOI":"10.1109\/DASC.2018.8569346"},{"issue":"1\u20132","key":"5748_CR42","doi-asserted-by":"publisher","first-page":"181","DOI":"10.1016\/S0004-3702(99)00052-1","volume":"112","author":"R Sutton","year":"1999","unstructured":"Sutton R, Precup D, Singh S (1999) Between MDPs and semi-MDPs: a framework for temporal abstraction in reinforcement learning. Artif Intell 112(1\u20132):181\u2013211","journal-title":"Artif Intell"},{"key":"5748_CR43","doi-asserted-by":"crossref","unstructured":"Tessler C, Givony S, Zahavy T, Mankowitz DJ, Mannor S (2017) A deep hierarchical approach to lifelong learning in minecraft. In: Proceedings of the thirty-first AAAI conference on artificial intelligence (AAAI '17), pp 1553\u20131561","DOI":"10.1609\/aaai.v31i1.10744"},{"key":"5748_CR44","unstructured":"Tumer K, Welch Z, Agogino A (2008) Aligning social welfare and agent preferences to alleviate traffic congestion. In: Proceedings of the 7th international joint conference on autonomous agents and multiagent systems (AAMAS\u201908), vol 2, pp 655\u2013662"}],"container-title":["Neural Computing and Applications"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s00521-021-05748-7.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s00521-021-05748-7\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s00521-021-05748-7.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2023,1,7]],"date-time":"2023-01-07T06:08:18Z","timestamp":1673071698000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s00521-021-05748-7"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2021,2,10]]},"references-count":44,"journal-issue":{"issue":"1","published-print":{"date-parts":[[2023,1]]}},"alternative-id":["5748"],"URL":"https:\/\/doi.org\/10.1007\/s00521-021-05748-7","relation":{},"ISSN":["0941-0643","1433-3058"],"issn-type":[{"value":"0941-0643","type":"print"},{"value":"1433-3058","type":"electronic"}],"subject":[],"published":{"date-parts":[[2021,2,10]]},"assertion":[{"value":"30 January 2020","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"16 January 2021","order":2,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"10 February 2021","order":3,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Compliance with ethical standards"}},{"value":"The authors declare that they have no conflict of interest.","order":2,"name":"Ethics","group":{"name":"EthicsHeading","label":"Conflict of interest"}}]}}