{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,6,25]],"date-time":"2026-06-25T12:52:36Z","timestamp":1782391956576,"version":"3.54.5"},"reference-count":36,"publisher":"Springer Science and Business Media LLC","issue":"7","license":[{"start":{"date-parts":[[2026,6,25]],"date-time":"2026-06-25T00:00:00Z","timestamp":1782345600000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2026,6,25]],"date-time":"2026-06-25T00:00:00Z","timestamp":1782345600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"funder":[{"name":"Fu jian Provincial Natural Science Foundation of China","award":["2024J01157"],"award-info":[{"award-number":["2024J01157"]}]}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Computing"],"published-print":{"date-parts":[[2026,7]]},"DOI":"10.1007\/s00607-026-01708-y","type":"journal-article","created":{"date-parts":[[2026,6,25]],"date-time":"2026-06-25T12:16:04Z","timestamp":1782389764000},"update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":0,"title":["A theoretical framework for multi-agent hierarchical reinforcement learning in cloud resource orchestration with convergence guarantees"],"prefix":"10.1007","volume":"108","author":[{"given":"Chunmao","family":"Jiang","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Longmei","family":"Tang","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Ruyi","family":"Ye","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Hao","family":"Zhang","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2026,6,25]]},"reference":[{"issue":"4","key":"1708_CR1","doi-asserted-by":"publisher","first-page":"50","DOI":"10.1145\/1721654.1721672","volume":"53","author":"M Armbrust","year":"2010","unstructured":"Armbrust M, Fox A, Griffith R, Joseph AD, Katz R, Konwinski A, Lee G, Patterson D, Rabkin A, Stoica I, Zaharia M (2010) A view of cloud computing. Commun ACM 53(4):50\u201358","journal-title":"Commun ACM"},{"issue":"3","key":"1708_CR2","doi-asserted-by":"publisher","first-page":"567","DOI":"10.1007\/s10922-014-9307-7","volume":"23","author":"B Jennings","year":"2015","unstructured":"Jennings B, Stadler R (2015) Resource management in clouds: Survey and research challenges. J Netw Syst Manage 23(3):567\u2013619","journal-title":"J Netw Syst Manage"},{"issue":"4","key":"1708_CR3","first-page":"73","volume":"51","author":"C Qu","year":"2018","unstructured":"Qu C, Calheiros RN, Buyya R (2018) Auto-scaling web applications in clouds: a taxonomy and survey. ACM Comput Surv 51(4):73","journal-title":"ACM Comput Surv"},{"key":"1708_CR4","doi-asserted-by":"crossref","unstructured":"Mao H, Alizadeh M, Menache I, Kandula S (2016) Resource management with deep reinforcement learning. Proceedings of the 15th ACM Workshop on Hot Topics in Networks. ACM, pp 50\u201356","DOI":"10.1145\/3005745.3005750"},{"key":"1708_CR5","doi-asserted-by":"crossref","unstructured":"Xu Z, Wang Y, Tang J, Wang J, Gursoy MC (2017) A deep reinforcement learning based framework for power-efficient resource allocation in cloud rans. In: 2017 IEEE International Conference on Communications (ICC), 1\u20136. IEEE","DOI":"10.1109\/ICC.2017.7997286"},{"key":"1708_CR6","unstructured":"Gu Y, Liu Z, Dai S, Liu C, Wang Y, Wang S, Theodoropoulos G, Cheng L (2025) Deep reinforcement learning for job scheduling and resource management in cloud computing: An algorithm-level review. arXiv:2501.01007"},{"issue":"2","key":"1708_CR7","doi-asserted-by":"publisher","first-page":"1658","DOI":"10.1109\/TVT.2021.3055511","volume":"70","author":"C Xu","year":"2021","unstructured":"Xu C, Liu S, Zhang C, Huang Y, Lu Z, Yang L (2021) Multi-agent reinforcement learning based distributed transmission in collaborative cloud-edge systems. IEEE Trans Veh Technol 70(2):1658\u20131672","journal-title":"IEEE Trans Veh Technol"},{"issue":"6","key":"1708_CR8","doi-asserted-by":"publisher","first-page":"2391","DOI":"10.1016\/j.jksuci.2022.03.016","volume":"34","author":"A Belgacem","year":"2022","unstructured":"Belgacem A, Mahmoudi S, Kihl M (2022) Intelligent multi-agent reinforcement learning model for resources allocation in cloud computing. J King Saud Univ-Comput Inform Sci 34(6):2391\u20132404","journal-title":"J King Saud Univ-Comput Inform Sci"},{"key":"1708_CR9","doi-asserted-by":"crossref","unstructured":"We X, Farooq J, Chen J (2024) Multi-agent distributed decentralized dynamic resource orchestration in 5g edge-cloud networks. 2024 IEEE 13th International Conference on Cloud Networking (CloudNet). IEEE, pp 1\u20138","DOI":"10.1109\/CloudNet62863.2024.10815780"},{"key":"1708_CR10","unstructured":"Sankaran P, Lingishetty SK, Choudhury A (2025) Multi-agent reinforcement learning for autonomous cloud resource management.(ResearchGate publication)"},{"key":"1708_CR11","doi-asserted-by":"crossref","unstructured":"Brunnbauer A, Lemmel J, Babaiee Z, Neubauer S, Grosu R (2024) Scalable offline reinforcement learning for mean field games. arXiv: 2410.17898","DOI":"10.65109\/AEYG6154"},{"key":"1708_CR12","unstructured":"Ocello A, Mancini L, Labbi S, Tiapkin D, Belouchrani A, Moulines E (2025) Mean-field multi-agent learning: a trust region approach. Proceedings of the 2025 European Signal Processing Conference (EUSIPCO). pp 1932\u20131936"},{"key":"1708_CR13","doi-asserted-by":"crossref","unstructured":"Bacon P, Harb J, Precup D (2017) The option-critic architecture. Proceedings of the Thirty-First AAAI Conference on Artificial Intelligence. pp 1726\u20131734","DOI":"10.1609\/aaai.v31i1.10916"},{"issue":"1","key":"1708_CR14","first-page":"5","volume":"6","author":"S Mahadevan","year":"2001","unstructured":"Mahadevan S, Ghavamzadeh M, Makar R (2001) Hierarchical multi-agent reinforcement learning. Auton Agent Multi-Agent Syst 6(1):5\u201334","journal-title":"Auton Agent Multi-Agent Syst"},{"key":"1708_CR15","unstructured":"Tang H, Hao J, Lv T, Chen Y, Zhang Z, Jia H, Ren C, Zheng Y, Meng Z, Fan C (2018) Hierarchical deep multiagent reinforcement learning with temporal abstraction. arXiv: 1809.09332"},{"key":"1708_CR16","doi-asserted-by":"publisher","first-page":"359","DOI":"10.1016\/j.neunet.2023.02.037","volume":"162","author":"J Wang","year":"2023","unstructured":"Wang J, Yuan M, Li Y, Zhao Z (2023) Hierarchical attention master-slave for heterogeneous multi-agent reinforcement learning. Neural Netw 162:359\u2013368","journal-title":"Neural Netw"},{"key":"1708_CR17","doi-asserted-by":"publisher","first-page":"536","DOI":"10.1016\/j.jmsy.2025.03.019","volume":"80","author":"L Ding","year":"2025","unstructured":"Ding L, Guan Z, Luo D, Yue L (2025) Data-driven hierarchical multi-policy deep reinforcement learning framework for multi-objective multiplicity dynamic flexible job shop scheduling. J Manuf Syst 80:536\u2013562","journal-title":"J Manuf Syst"},{"issue":"18","key":"1708_CR18","doi-asserted-by":"publisher","first-page":"1231","DOI":"10.1049\/cmu2.12824","volume":"18","author":"S Wang","year":"2024","unstructured":"Wang S, Cao H, Yang L (2024) A hierarchical reinforcement learning approach for energy-aware service function chain dynamic deployment in iot. IET Commun 18(18):1231\u20131243","journal-title":"IET Commun"},{"key":"1708_CR19","doi-asserted-by":"publisher","unstructured":"Angiuli A, Fouque J-P, Lauri\u00e8re M, Zhang M (2024) Convergence of multi-scale reinforcement q-learning algorithms for mean field game and control problems. arXiv:2312.06659. https:\/\/doi.org\/10.48550\/arXiv.2312.06659","DOI":"10.48550\/arXiv.2312.06659"},{"key":"1708_CR20","doi-asserted-by":"publisher","unstructured":"Maheshwari C, Wu M, Sastry S (2024) Convergence of decentralized actor-critic algorithm in general-sum markov games. arXiv:2409.04613. https:\/\/doi.org\/10.48550\/arXiv.2409.04613","DOI":"10.48550\/arXiv.2409.04613"},{"issue":"178","key":"1708_CR21","first-page":"1","volume":"21","author":"T Rashid","year":"2020","unstructured":"Rashid T, Samvelyan M, De Witt CS, Farquhar G, Foerster J, Whiteson S (2020) Monotonic value function factorisation for deep multi-agent reinforcement learning. J Mach Learn Res 21(178):1\u201351","journal-title":"J Mach Learn Res"},{"key":"1708_CR22","unstructured":"Lowe R, Wu YI, Tamar A, Harb J, Pieter Abbeel O, Mordatch I (2017) Multi-agent actor-critic for mixed cooperative-competitive environments. Advances in neural information processing systems. p 30"},{"key":"1708_CR23","doi-asserted-by":"crossref","unstructured":"Liu N, Li Z, Xu J, Xu Z, Lin S, Qiu Q, Tang J, Wang Y (2017) A hierarchical framework of cloud resource allocation and power management using deep reinforcement learning. In: 2017 IEEE 37th International Conference on Distributed Computing Systems (ICDCS), 372\u2013382. IEEE","DOI":"10.1109\/ICDCS.2017.123"},{"key":"1708_CR24","doi-asserted-by":"publisher","first-page":"74429","DOI":"10.1109\/ACCESS.2018.2881964","volume":"6","author":"R Li","year":"2018","unstructured":"Li R, Zhao Z, Sun Q, Yang C, Chen X, Zhao M, Zhang H et al (2018) Deep reinforcement learning for resource management in network slicing. IEEE Access 6:74429\u201374441","journal-title":"IEEE Access"},{"issue":"4","key":"1708_CR25","doi-asserted-by":"publisher","first-page":"341","DOI":"10.1023\/A:1025696116075","volume":"13","author":"AG Barto","year":"2003","unstructured":"Barto AG, Mahadevan S (2003) Recent advances in hierarchical reinforcement learning. Discrete Event Dyn Syst 13(4):341\u2013379","journal-title":"Discrete Event Dyn Syst"},{"issue":"1\u20132","key":"1708_CR26","doi-asserted-by":"publisher","first-page":"181","DOI":"10.1016\/S0004-3702(99)00052-1","volume":"112","author":"RS Sutton","year":"1999","unstructured":"Sutton RS, Precup D, Singh S (1999) Between mdps and semi-mdps: A framework for temporal abstraction in reinforcement learning. Artif Intell 112(1\u20132):181\u2013211","journal-title":"Artif Intell"},{"key":"1708_CR27","unstructured":"Melo FS (2008) Convergence of q-learning: a simple proof. Institute Of Systems and Robotics. Tech Rep 1\u20134"},{"key":"1708_CR28","unstructured":"Zhang Z, Oh MS, Hairi F, Luo Z, Velasquez A, Liu J (2025) Finite-time global optimality convergence in deep neural actor-critic methods for decentralized multi-agent reinforcement learning. arXiv: 2505.18433"},{"key":"1708_CR29","doi-asserted-by":"crossref","unstructured":"Zhang C, Wei L, Fan J, Liu Z, Huang Y (2024) Lyapunov-guided multi-agent reinforcement learning for delay-sensitive wireless scheduling. arXiv:2411.01766","DOI":"10.1109\/GLOBECOM59602.2025.11431637"},{"key":"1708_CR30","doi-asserted-by":"crossref","unstructured":"Drazin PG (1992) Nonlinear Systems vol. 10. Cambridge University Press","DOI":"10.1017\/CBO9781139172455"},{"key":"1708_CR31","unstructured":"Kakade SM (2002) A natural policy gradient. Advances in Neural Information Processing Systems. pp 1531\u20131538"},{"key":"1708_CR32","unstructured":"Dong K, Wang Y, Chen X, Wang L (2019) Q-learning with ucb exploration is sample efficient for infinite-horizon mdp. arXiv:1901.09311"},{"key":"1708_CR33","doi-asserted-by":"crossref","unstructured":"Calheiros RN, Ranjan R, Beloglazov A, De Rose CA, Buyya R (2011) Cloudsim: a toolkit for modeling and simulation of cloud computing environments and evaluation of resource provisioning algorithms. Software Practice and experience 41(1):23\u201350","DOI":"10.1002\/spe.995"},{"key":"1708_CR34","unstructured":"Reiss C, Wilkes J, Hellerstein JL (2011) Google cluster-usage traces: format+ schema. In: Technical Report. Google Inc"},{"issue":"4","key":"1708_CR35","doi-asserted-by":"publisher","first-page":"559","DOI":"10.1007\/s10723-014-9314-7","volume":"12","author":"T Lorido-Botr\u00e1n","year":"2014","unstructured":"Lorido-Botr\u00e1n T, Miguel-Alonso J, Lozano JA (2014) A review of auto-scaling techniques for elastic applications in cloud environments. J Grid Comput 12(4):559\u2013592","journal-title":"J Grid Comput"},{"key":"1708_CR36","unstructured":"Kingma DP, Ba J (2014) Adam: A method for stochastic optimization. 1412(6) arXiv: 1412.6980"}],"container-title":["Computing"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s00607-026-01708-y.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s00607-026-01708-y","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s00607-026-01708-y.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,6,25]],"date-time":"2026-06-25T12:16:41Z","timestamp":1782389801000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s00607-026-01708-y"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026,6,25]]},"references-count":36,"journal-issue":{"issue":"7","published-print":{"date-parts":[[2026,7]]}},"alternative-id":["1708"],"URL":"https:\/\/doi.org\/10.1007\/s00607-026-01708-y","relation":{},"ISSN":["0010-485X","1436-5057"],"issn-type":[{"value":"0010-485X","type":"print"},{"value":"1436-5057","type":"electronic"}],"subject":[],"published":{"date-parts":[[2026,6,25]]},"assertion":[{"value":"9 October 2025","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"18 June 2026","order":2,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"25 June 2026","order":3,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Declarations"}},{"value":"The authors declare that they have no known competing financial interests or personal relationships that could have appeared to influence the work reported in this paper.","order":2,"name":"Ethics","group":{"name":"EthicsHeading","label":"Conflict of interest"}},{"value":"This study did not include any studies involving human participants or animals performed by any of the authors.","order":3,"name":"Ethics","group":{"name":"EthicsHeading","label":"Ethical approval"}}],"article-number":"109"}}