{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,3,27]],"date-time":"2025-03-27T13:33:40Z","timestamp":1743082420870,"version":"3.40.3"},"publisher-location":"Cham","reference-count":55,"publisher":"Springer International Publishing","isbn-type":[{"type":"print","value":"9783030779382"},{"type":"electronic","value":"9783030779399"}],"license":[{"start":{"date-parts":[[2021,1,1]],"date-time":"2021-01-01T00:00:00Z","timestamp":1609459200000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"},{"start":{"date-parts":[[2021,1,1]],"date-time":"2021-01-01T00:00:00Z","timestamp":1609459200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2021]]},"DOI":"10.1007\/978-3-030-77939-9_8","type":"book-chapter","created":{"date-parts":[[2021,10,1]],"date-time":"2021-10-01T19:26:52Z","timestamp":1633116412000},"page":"259-308","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":0,"title":["Deep Reinforcement Learning for Autonomous Mobile Networks in Micro-grids"],"prefix":"10.1007","author":[{"given":"Marco","family":"Miozzo","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Nicola","family":"Piovesan","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Dagnachew Azene","family":"Temesgene","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Paolo","family":"Dini","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2021,10,2]]},"reference":[{"unstructured":"3GPP (2017) TR 21.866.; ; E-U; Study on energy efficiency aspects of 3GPP standards v15","key":"8_CR1"},{"unstructured":"3GPP (2017) TS 38.801.; ; E-U; Study on new radio access technology: radio access architecture and interfaces v14","key":"8_CR2"},{"issue":"1","key":"8_CR3","doi-asserted-by":"publisher","first-page":"450","DOI":"10.1109\/JIOT.2017.2750180","volume":"5","author":"N Abbas","year":"2018","unstructured":"Abbas N, Zhang Y, Taherkordi A, Skeie T (2018) Mobile edge computing: a survey. IEEE Internet Things J 5(1):450\u2013465","journal-title":"IEEE Internet Things J"},{"unstructured":"Access EUTR (2010) Further advancements for E-UTRA physical layer aspects, 3GPP TS 36.814, V9. 0.0, Mar","key":"8_CR4"},{"issue":"1","key":"8_CR5","doi-asserted-by":"publisher","first-page":"117","DOI":"10.3390\/challe6010117","volume":"6","author":"AS Andrae","year":"2015","unstructured":"Andrae AS, Edler T (2015) On global electricity usage of communication technology: trends to 2030. MDPI Challenges 6(1):117\u2013157","journal-title":"MDPI Challenges"},{"unstructured":"Auer G, Blume O, Giannini V, Godor I, Imran M, Jading Y, Katranaras E, Olsson M, Sabella D, Skillermark P et al (2013) EARTH Deliverable D2.3: energy efficiency analysis of the reference systems, areas of improvements and target breakdown. Project Deliverable D2.3, www.ict-earth.eu","key":"8_CR6"},{"unstructured":"Bellman R (1957) Dynamic programming. Princeton University Press, Princeton, NJ","key":"8_CR7"},{"doi-asserted-by":"crossref","unstructured":"Bu\u015foniu L, Babu\u0161ka R, De Schutter B (2010) Multi-agent reinforcement learning: an overview. Springer, Berlin, pp 183\u2013221","key":"8_CR8","DOI":"10.1007\/978-3-642-14435-6_7"},{"doi-asserted-by":"crossref","unstructured":"Busoniu L, Babuska R, Schutter BD (2008) A comprehensive survey of multiagent reinforcement learning. IEEE Trans Syst Man Cybern Part C (Appl Rev 38(2):156\u2013172","key":"8_CR9","DOI":"10.1109\/TSMCC.2007.913919"},{"doi-asserted-by":"crossref","unstructured":"Chen Q, Zheng Z, Hu C, Wang D, Liu F (2019) Data-driven task allocation for multi-task transfer learning on the edge. In: 2019 IEEE 39th International conference on distributed computing systems (ICDCS), pp 1040\u20131050","key":"8_CR10","DOI":"10.1109\/ICDCS.2019.00107"},{"issue":"8","key":"8_CR11","doi-asserted-by":"publisher","first-page":"7457","DOI":"10.1109\/JIOT.2020.2984887","volume":"7","author":"S Deng","year":"2020","unstructured":"Deng S, Zhao H, Fang W, Yin J, Dustdar S, Zomaya AY (2020) Edge intelligence: the confluence of edge computing and artificial intelligence. IEEE Internet Things J 7(8):7457\u20137469","journal-title":"IEEE Internet Things J"},{"doi-asserted-by":"crossref","unstructured":"Desset C, Debaillie B, Giannini V, Fehske A, Auer G, Holtkamp H, Wajda W, Sabella D, Richter F, Gonzalez MJ et al (2012) Flexible power modeling of LTE base stations. In: 2012 IEEE Wireless communications and networking conference (WCNC). IEEE, 2858\u20132862","key":"8_CR12","DOI":"10.1109\/WCNC.2012.6214289"},{"unstructured":"Dulac-Arnold G, Evans R, van Hasselt H, Sunehag P, Lillicrap T, Hunt J, Mann T, Weber T, Degris T, Coppin B (2015) Deep reinforcement learning in large discrete action spaces. arXiv preprint arXiv:1512.07679","key":"8_CR13"},{"unstructured":"ETSI (2017) ES 203 208; Environmental Engineering (EE); Assessment of mobile network energy efficiency (v1.2.1)","key":"8_CR14"},{"unstructured":"Fudenberg D, Levine DK (1998) The theory of learning in games, vol 1. MIT Press Books, The MIT Press","key":"8_CR15"},{"doi-asserted-by":"crossref","unstructured":"Han F, Zhao S, Zhang L, Wu J (2016) Survey of strategies for switching off base stations in heterogeneous networks for greener 5G systems. IEEE Access 4:4959\u20134973","key":"8_CR16","DOI":"10.1109\/ACCESS.2016.2598813"},{"doi-asserted-by":"crossref","unstructured":"Hassan HAH, Nuaymi L, Pelov A (2013) Renewable energy in cellular networks: a survey. In: 2013 IEEE Online conference on green communications (OnlineGreenComm), pp 1\u20137","key":"8_CR17","DOI":"10.1109\/OnlineGreenCom.2013.6731020"},{"issue":"3","key":"8_CR18","doi-asserted-by":"publisher","first-page":"317","DOI":"10.1109\/T-VT.1980.23859","volume":"29","author":"M Hata","year":"1980","unstructured":"Hata M (1980) Empirical formula for propagation loss in land mobile radio services. IEEE Trans Veh Technol 29(3):317\u2013325","journal-title":"IEEE Trans Veh Technol"},{"key":"8_CR19","doi-asserted-by":"publisher","first-page":"64","DOI":"10.1016\/j.comcom.2014.02.008","volume":"50","author":"WV Heddeghem","year":"2014","unstructured":"Heddeghem WV, Lambert S, Lannoo B, Colle D, Pickavet M, Demeester P (2014) Trends in worldwide ICT electricity consumption from 2007 to 2012. Comput Commun 50:64\u201376","journal-title":"Comput Commun"},{"doi-asserted-by":"crossref","unstructured":"Kabalci Y (2016) A survey on smart metering and smart grid communication. Renew Sustain Energy Rev 57:302\u2013318","key":"8_CR20","DOI":"10.1016\/j.rser.2015.12.114"},{"doi-asserted-by":"crossref","unstructured":"Lara A, Kolasani A, Ramamurthy B (2014) Network innovation using OpenFlow: a survey. IEEE Commun Surv Tutorials 16(1):493\u2013512","key":"8_CR21","DOI":"10.1109\/SURV.2013.081313.00105"},{"doi-asserted-by":"crossref","unstructured":"Le TP, Vien NA, Chung T (2018) A deep hierarchical reinforcement learning algorithm in partially observable Markov decision processes. IEEE Access 6:49089\u201349102","key":"8_CR22","DOI":"10.1109\/ACCESS.2018.2854283"},{"doi-asserted-by":"crossref","unstructured":"Leccese F, Leonowicz Z (2012) Intelligent wireless street lighting system. In: 2012 11th International conference on environment and electrical engineering. IEEE, pp 958\u2013961","key":"8_CR23","DOI":"10.1109\/EEEIC.2012.6221515"},{"unstructured":"Lillicrap TP, Hunt JJ, Pritzel A, Heess N, Erez T, Tassa Y, Silver D, Wierstra D (2016) Continuous control with deep reinforcement learning. In: ICLR","key":"8_CR24"},{"unstructured":"Lin Y, Han S, Mao H, Wang Y, Dally WJ (2017) Deep gradient compression: reducing the communication bandwidth for distributed training","key":"8_CR25"},{"doi-asserted-by":"crossref","unstructured":"Lindemark B, Oberg G (2001) Solar power for radio base station (RBS) sites applications including system dimensioning, cell planning and operation. In: Twenty-third international telecommunications energy conference INTELEC 2001, pp 587\u2013590","key":"8_CR26","DOI":"10.1049\/cp:20010655"},{"issue":"4","key":"8_CR27","doi-asserted-by":"publisher","first-page":"2078","DOI":"10.1109\/COMST.2015.2439636","volume":"17","author":"D L\u00f3pez-P\u00e9rez","year":"2015","unstructured":"L\u00f3pez-P\u00e9rez D, Ding M, Claussen H, Jafari AH (2015) Towards 1 Gbps\/UE in cellular systems: understanding ultra-dense small cell deployments. IEEE Commun Surv Tutorials 17(4):2078\u20132101","journal-title":"IEEE Commun Surv Tutorials"},{"doi-asserted-by":"crossref","unstructured":"Marhon SA, Cameron CJF, Kremer SC (2013) Recurrent neural networks","key":"8_CR28","DOI":"10.1007\/978-3-642-36657-4_2"},{"doi-asserted-by":"crossref","unstructured":"Mezzavilla M, Miozzo M, Rossi M, Baldo N, Zorzi M (2012) A lightweight and accurate link abstraction model for the simulation of LTE networks in ns-3. In: Proceedings of the 15th ACM international conference on modeling, analysis and simulation of wireless and mobile systems, MSWiM\u201912. ACM, New York, pp 55\u201360","key":"8_CR29","DOI":"10.1145\/2387238.2387250"},{"doi-asserted-by":"crossref","unstructured":"Miozzo M, Zordan D, Dini P, Rossi M (2014) SolarStat: modeling photovoltaic sources through stochastic Markov processes. In: IEEE International energy conference (ENERGYCON), Cavtat, Croatia, pp 688\u2013695","key":"8_CR30","DOI":"10.1109\/ENERGYCON.2014.6850501"},{"unstructured":"Mnih V, Kavukcuoglu K, Silver D, Graves A, Antonoglou I, Wierstra D, Riedmiller M (2013) Playing Atari with deep reinforcement learning. In: NIPS Deep learning workshop","key":"8_CR31"},{"issue":"7540","key":"8_CR32","doi-asserted-by":"publisher","first-page":"529","DOI":"10.1038\/nature14236","volume":"518","author":"V Mnih","year":"2015","unstructured":"Mnih V, Kavukcuoglu K, Silver D, Rusu AA, Veness J, Bellemare MG, Graves A, Riedmiller M, Fidjeland AK, Ostrovski G, Petersen S, Beattie C, Sadik A, Antonoglou I, King H, Kumaran D, Wierstra D, Legg S, Hassabis D (2015) Human-level control through deep reinforcement learning. Nature 518(7540):529\u2013533","journal-title":"Nature"},{"doi-asserted-by":"crossref","unstructured":"M\u00fcllner R, Riener A (2011) An energy efficient pedestrian aware smart street lighting system. Int J Pervasive Comput Commun","key":"8_CR33","DOI":"10.1108\/17427371111146437"},{"unstructured":"NEC (n.d.) NFV C-RAN for efficient RAN resource allocation. Online white paper. Accessed on 16 Mar 2016","key":"8_CR34"},{"unstructured":"NREL, National Renewable Energy Laboratory (n.d.) Renewable resource data center. http:\/\/www.nrel.gov\/rredc\/","key":"8_CR35"},{"unstructured":"Park C, Lee J (2020) Mobile edge computing-enabled heterogeneous networks. IEEE Trans Wirel Commun 1","key":"8_CR36"},{"doi-asserted-by":"crossref","unstructured":"Piovesan N, Dini P (2017) Optimal direct load control of renewable powered small cells: a shortest path approach. Internet Technol Lett e7\u2013n\/a. e7","key":"8_CR37","DOI":"10.1002\/itl2.7"},{"doi-asserted-by":"crossref","unstructured":"Piovesan N, Dini P (2018) Unsupervised learning of representations from solar energy data. In: 2018 IEEE 29th Annual international symposium on personal, indoor and mobile radio communications (PIMRC), pp 1\u20136","key":"8_CR38","DOI":"10.1109\/PIMRC.2018.8580891"},{"doi-asserted-by":"crossref","unstructured":"Piovesan N, Lopez-Perez D, Miozzo M, Dini P (2020) Joint load control and energy sharing for renewable powered small base stations: a machine learning approach. IEEE Trans Green Commun Networking 1","key":"8_CR39","DOI":"10.1109\/TGCN.2020.3027063"},{"doi-asserted-by":"crossref","unstructured":"Piovesan N, Miozzo M, Dini P (2020) Modeling the environment in deep reinforcement learning: the case of energy harvesting base stations. In: ICASSP 2020-2020 IEEE International conference on acoustics, speech and signal processing (ICASSP). IEEE, pp 8996\u20139000","key":"8_CR40","DOI":"10.1109\/ICASSP40776.2020.9054646"},{"key":"8_CR41","doi-asserted-by":"publisher","first-page":"31140","DOI":"10.1109\/ACCESS.2019.2903499","volume":"7","author":"N Piovesan","year":"2019","unstructured":"Piovesan N, Temesgene DA, Miozzo M, Dini P (2019) Joint load control and energy sharing for autonomous operation of 5G mobile networks in micro-grids. IEEE Access 7:31140\u201331150","journal-title":"IEEE Access"},{"issue":"1","key":"8_CR42","doi-asserted-by":"publisher","first-page":"32","DOI":"10.1109\/MIC.2012.124","volume":"17","author":"G Piro","year":"2013","unstructured":"Piro G, Miozzo M, Forte G, Baldo N, Grieco LA, Boggia G, Dini P (2013) HetNets powered by renewable energy sources: sustainable next-generation cellular networks. IEEE Internet Comput 17(1):32\u201339","journal-title":"IEEE Internet Comput"},{"doi-asserted-by":"crossref","unstructured":"Sewak M (2019) Policy-based reinforcement learning approaches. Springer, Singapore, pp 127\u2013140","key":"8_CR43","DOI":"10.1007\/978-981-13-8285-7_10"},{"doi-asserted-by":"crossref","unstructured":"Sharma R, Biookaghazadeh S, Li B, Zhao M (2018) Are existing knowledge transfer techniques effective for deep learning with edge devices? In: 2018 IEEE International conference on edge computing (EDGE), pp 42\u201349","key":"8_CR44","DOI":"10.1109\/EDGE.2018.00013"},{"doi-asserted-by":"crossref","unstructured":"Sutton RS, Barto AG (1998) Reinforcement learning: an introduction. MIT Press, Cambridge","key":"8_CR45","DOI":"10.1016\/S1474-6670(17)38315-5"},{"unstructured":"Tang H, Gan S, Zhang C, Zhang T, Liu J (2018) Communication compression for decentralized training. In: Bengio S, Wallach H, Larochelle H, Grauman K, Cesa-Bianchi N, Garnett R (eds) Advances in neural information processing systems 31. Curran Associates Inc., pp 7652\u20137662","key":"8_CR46"},{"key":"8_CR47","doi-asserted-by":"publisher","first-page":"48","DOI":"10.1016\/j.comcom.2019.09.002","volume":"148","author":"DA Temesgene","year":"2019","unstructured":"Temesgene DA, Miozzo M, Dini P (2019) Dynamic control of functional splits for energy harvesting virtual small cells: a distributed reinforcement learning approach. Comput Commun 148:48\u201361","journal-title":"Comput Commun"},{"doi-asserted-by":"crossref","unstructured":"Temesgene DA, Miozzo M, Gunduz D, Dini P (2020) Distributed deep reinforcement learning for functional split control in energy harvesting virtualized small cells. IEEE Trans Sustain Comput 1","key":"8_CR48","DOI":"10.1109\/TSUSC.2020.3025139"},{"doi-asserted-by":"crossref","unstructured":"Temesgene DA, Piovesan N, Miozzo M, Dini P (2018) Optimal placement of baseband functions for energy harvesting virtual small cells. In: 2018 IEEE 88th Vehicular technology conference (VTC-Fall), pp 1\u20136","key":"8_CR49","DOI":"10.1109\/VTCFall.2018.8690929"},{"doi-asserted-by":"crossref","unstructured":"Ton DT, Smith MA (2012) The U.S. department of energy\u2019s microgrid initiative. Electr J 25(8):84\u201394","key":"8_CR50","DOI":"10.1016\/j.tej.2012.09.013"},{"doi-asserted-by":"crossref","unstructured":"Trinh HD, Bui N, Widmer J, Giupponi L, Dini P (2017) Analysis and modeling of mobile traffic using real traces. In: 2017 IEEE 28th Annual international symposium on personal, indoor, and mobile radio communications (PIMRC). IEEE, pp 1\u20136","key":"8_CR51","DOI":"10.1109\/PIMRC.2017.8292200"},{"issue":"1","key":"8_CR52","first-page":"3483","volume":"15","author":"K Van Moffaert","year":"2014","unstructured":"Van Moffaert K, Now\u00e9 A (2014) Multi-objective reinforcement learning using sets of pareto dominating policies. J Mach Learn Res 15(1):3483\u20133512","journal-title":"J Mach Learn Res"},{"issue":"2","key":"8_CR53","doi-asserted-by":"publisher","first-page":"1147","DOI":"10.1109\/TNET.2016.2623950","volume":"25","author":"F Xu","year":"2017","unstructured":"Xu F, Li Y, Wang H, Zhang P, Jin D (2017) Understanding mobile traffic patterns of large scale cellular towers in urban environment. IEEE\/ACM Trans Networking 25(2):1147\u20131161","journal-title":"IEEE\/ACM Trans Networking"},{"issue":"8","key":"8_CR54","doi-asserted-by":"publisher","first-page":"1738","DOI":"10.1109\/JPROC.2019.2918951","volume":"107","author":"Z Zhou","year":"2019","unstructured":"Zhou Z, Chen X, Li E, Zeng L, Luo K, Zhang J (2019) Edge intelligence: paving the last mile of artificial intelligence with edge computing. Proc IEEE 107(8):1738\u20131762","journal-title":"Proc IEEE"},{"doi-asserted-by":"crossref","unstructured":"Zordan D, Miozzo M, Dini P, Rossi M (2015) When telecommunications networks meet energy grids: cellular networks with energy harvesting and trading capabilities. IEEE Commun Mag 53(6):117\u2013123","key":"8_CR55","DOI":"10.1109\/MCOM.2015.7120026"}],"container-title":["Studies in Computational Intelligence","Deep Learning for Unmanned Systems"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-030-77939-9_8","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2021,10,1]],"date-time":"2021-10-01T19:38:27Z","timestamp":1633117107000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-3-030-77939-9_8"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2021]]},"ISBN":["9783030779382","9783030779399"],"references-count":55,"URL":"https:\/\/doi.org\/10.1007\/978-3-030-77939-9_8","relation":{},"ISSN":["1860-949X","1860-9503"],"issn-type":[{"type":"print","value":"1860-949X"},{"type":"electronic","value":"1860-9503"}],"subject":[],"published":{"date-parts":[[2021]]},"assertion":[{"value":"2 October 2021","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}}]}}