{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,6,13]],"date-time":"2026-06-13T09:32:47Z","timestamp":1781343167608,"version":"3.54.1"},"reference-count":44,"publisher":"Springer Science and Business Media LLC","issue":"1","license":[{"start":{"date-parts":[[2022,7,3]],"date-time":"2022-07-03T00:00:00Z","timestamp":1656806400000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2022,7,3]],"date-time":"2022-07-03T00:00:00Z","timestamp":1656806400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["J Supercomput"],"published-print":{"date-parts":[[2023,1]]},"DOI":"10.1007\/s11227-022-04643-9","type":"journal-article","created":{"date-parts":[[2022,7,3]],"date-time":"2022-07-03T13:02:16Z","timestamp":1656853336000},"page":"75-108","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":5,"title":["Q-learning-based algorithms for dynamic transmission control in IoT equipment"],"prefix":"10.1007","volume":"79","author":[{"given":"Hanieh","family":"Malekijou","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-0798-3981","authenticated-orcid":false,"given":"Vesal","family":"Hakami","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Nastooh Taheri","family":"Javan","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Amirhossein","family":"Malekijoo","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2022,7,3]]},"reference":[{"key":"4643_CR1","doi-asserted-by":"publisher","first-page":"6859","DOI":"10.1007\/s11227-018-2288-7","volume":"74","author":"D Lee","year":"2018","unstructured":"Lee D, Lee H (2018) IoT service classification and clustering for integration of IoT service platforms. J Supercomput 74:6859\u20136875","journal-title":"J Supercomput"},{"issue":"6","key":"4643_CR2","doi-asserted-by":"publisher","first-page":"79","DOI":"10.1109\/MCOM.2015.7120021","volume":"53","author":"Y He","year":"2015","unstructured":"He Y, Cheng X, Peng W, Stuber GL (2015) A survey of energy harvesting communications: models and offline optimal policies. IEEE Commun Mag 53(6):79\u201385. https:\/\/doi.org\/10.1109\/MCOM.2015.7120021","journal-title":"IEEE Commun Mag"},{"key":"4643_CR3","doi-asserted-by":"publisher","first-page":"220","DOI":"10.1109\/TCOMM.2011.112811.100349","volume":"60","author":"J Yang","year":"2012","unstructured":"Yang J, Ulukus S (2012) Optimal packet scheduling in an energy harvesting communication system. IEEE Trans Commun 60:220\u2013230","journal-title":"IEEE Trans Commun"},{"key":"4643_CR4","doi-asserted-by":"publisher","first-page":"4723","DOI":"10.1007\/s11276-020-02351-x","volume":"26","author":"DK Sah","year":"2020","unstructured":"Sah DK, Amgoth T (2020) A novel efficient clustering protocol for energy harvesting in wireless sensor networks. Wireless Netw 26:4723\u20134737","journal-title":"Wireless Netw"},{"key":"4643_CR5","doi-asserted-by":"publisher","first-page":"3620","DOI":"10.1109\/JSAC.2016.2612039","volume":"34","author":"D Shaviv","year":"2016","unstructured":"Shaviv D, Zgur AO (2016) Universally near optimal online power control for energy harvesting nodes. IEEE J Sel Areas Commun 34:3620\u20133631","journal-title":"IEEE J Sel Areas Commun"},{"key":"4643_CR6","doi-asserted-by":"publisher","first-page":"2975","DOI":"10.1109\/TWC.2018.2805336","volume":"17","author":"A Arafa","year":"2018","unstructured":"Arafa A, Baknina A, Ulukus S (2018) Online fixed fraction policies in energy harvesting communication systems. IEEE Trans Wireless Commun 17:2975\u20132986","journal-title":"IEEE Trans Wireless Commun"},{"issue":"5","key":"4643_CR7","doi-asserted-by":"publisher","first-page":"895","DOI":"10.1109\/JSTSP.2013.2258656","volume":"7","author":"A Aprem","year":"2013","unstructured":"Aprem A, Murthy CR, Mehta NB (2013) Transmit power control policies for energy harvesting sensors with retransmissions. IEEE J Sel Topics Signal Process 7(5):895\u2013906","journal-title":"IEEE J Sel Topics Signal Process"},{"key":"4643_CR8","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-79995-2","volume-title":"Stochastic network optimization with application to communication and queuing systems","author":"M Neely","year":"2010","unstructured":"Neely M (2010) Stochastic network optimization with application to communication and queuing systems. Morgan and Claypool"},{"key":"4643_CR9","doi-asserted-by":"publisher","first-page":"1409","DOI":"10.1109\/TSP.2020.2973125","volume":"68","author":"N Sharma","year":"2020","unstructured":"Sharma N, Mastronarde N, Chakareski J (2020) Accelerated structure-aware reinforcement learning for delay-sensitive energy harvesting wireless sensors. IEEE Trans Signal Process 68:1409\u20131424","journal-title":"IEEE Trans Signal Process"},{"key":"4643_CR10","doi-asserted-by":"crossref","unstructured":"Toorchi N, Chakareski J, and Mastronarde N. (2016) Fast and low- complexity reinforcement learning for delay-sensitive energy harvesting wireless visual sensing systems. IEEE International Conference on Image Processing (ICIP), 1804\u20131808.","DOI":"10.1109\/ICIP.2016.7532669"},{"key":"4643_CR11","doi-asserted-by":"publisher","DOI":"10.1145\/3520129","author":"S Shahhosseini","year":"2022","unstructured":"Shahhosseini S, Seo D, Kanduri A, Hu T, Lim S, Donyanavard B, Rahmani AM, Dutt N (2022) Online learning for orchestration of inference in multi-user end-edge-cloud networks. ACM Trans Embed Comput Syst. https:\/\/doi.org\/10.1145\/3520129","journal-title":"ACM Trans Embed Comput Syst"},{"key":"4643_CR12","doi-asserted-by":"publisher","first-page":"14625","DOI":"10.1007\/s11042-017-5051-9","volume":"77","author":"R Aslani","year":"2018","unstructured":"Aslani R, Hakami V, Dehghan M (2018) A token-based incentive mechanism for video streaming applications in peer- to-peer networks. Multim Tools Appl 77:14625\u201314653","journal-title":"Multim Tools Appl"},{"key":"4643_CR13","doi-asserted-by":"publisher","first-page":"560","DOI":"10.1109\/TMC.2017.2732979","volume":"17","author":"C Wang","year":"2018","unstructured":"Wang C, Li J, Yang Y, Ye F (2018) Combining solar energy harvesting with wireless charging for hybrid wireless sensor networks. IEEE Trans Mob Comput 17:560\u2013576","journal-title":"IEEE Trans Mob Comput"},{"key":"4643_CR14","doi-asserted-by":"publisher","unstructured":"Malekijoo A, Fadaeieslam MJ, Malekijou H, Homayounfar M, Alizadeh-Shabdiz F, Rawassizadeh R, (2021), FEDZIP: A Compression Framework for Communication-Efficient Federated Learning. https:\/\/doi.org\/10.48550\/arXiv.2102.01593","DOI":"10.48550\/arXiv.2102.01593"},{"key":"4643_CR15","doi-asserted-by":"publisher","first-page":"82","DOI":"10.1109\/WCL.2012.112012.120754","volume":"2","author":"KJ Prabuchandran","year":"2013","unstructured":"Prabuchandran KJ, Meena SK, Bhatnagar S (2013) Q-learning based energy management policies for a single sensor node with finite buffer. IEEE Wireless Commun Lett 2:82\u201385","journal-title":"IEEE Wireless Commun Lett"},{"key":"4643_CR16","doi-asserted-by":"publisher","first-page":"32","DOI":"10.1145\/1274858.1274870","volume":"6","author":"A Kansal","year":"2007","unstructured":"Kansal A, Jason H, Zahedi S, Srivastava M (2007) Power management in energy harvesting sensor networks. ACM Trans Embedd Comput Syst 6:32\u201344","journal-title":"ACM Trans Embedd Comput Syst"},{"key":"4643_CR17","doi-asserted-by":"crossref","unstructured":"Mastronarde N, Modares J, Wu C, and Chakareski J. (2016) Reinforcement learning for energy-efficient delay-sensitive csma\/ca scheduling. IEEE Global Communications Conference (GLOBECOM), 1\u20137.","DOI":"10.1109\/GLOCOM.2016.7842209"},{"key":"4643_CR18","doi-asserted-by":"publisher","first-page":"554","DOI":"10.1016\/j.comcom.2020.07.005","volume":"160","author":"V Hakami","year":"2020","unstructured":"Hakami V, Mostafavi SA, Javan NT, Rashidi Z (2020) An optimal policy for joint compression and transmission control in delay-constrained energy harvesting IoT devices. Comput Commun 160:554\u2013566. https:\/\/doi.org\/10.1016\/j.comcom.2020.07.005","journal-title":"Comput Commun"},{"key":"4643_CR19","doi-asserted-by":"crossref","unstructured":"Masadeh A, Wang Z, and Kamal AE. (2018) Reinforcement learning exploration algorithms for energy harvesting communications systems. IEEE International Conference on Communications (ICC), 1\u20136.","DOI":"10.1109\/ICC.2018.8422710"},{"issue":"8","key":"4643_CR20","doi-asserted-by":"publisher","first-page":"5106","DOI":"10.1109\/TCOMM.2021.3077948","volume":"69","author":"S Hu","year":"2021","unstructured":"Hu S, Chen W (2021) Joint lossy compression and power allocation in low latency wireless communications for IIoT: a cross-layer approach. IEEE Trans Commun 69(8):5106\u20135120. https:\/\/doi.org\/10.1109\/TCOMM.2021.3077948","journal-title":"IEEE Trans Commun"},{"key":"4643_CR21","doi-asserted-by":"publisher","first-page":"3959","DOI":"10.1007\/s00521-021-06656-6","volume":"34","author":"F Namjoonia","year":"2022","unstructured":"Namjoonia F, Sheikhi M, Hakami V (2022) Fast reinforcement learning algorithms for joint adaptive source coding and transmission control in IoT devices with renewable energy storage. Neural Comput Appl 34:3959\u20133979. https:\/\/doi.org\/10.1007\/s00521-021-06656-6","journal-title":"Neural Comput Appl"},{"issue":"2","key":"4643_CR22","first-page":"322","volume":"31","author":"LU Wenwei","year":"2021","unstructured":"Wenwei LU, Siliang G, Yihua Z (2021) Timely data delivery for energy-harvesting IoT devices. Chin J Electron 31(2):322\u2013336","journal-title":"Chin J Electron"},{"key":"4643_CR23","doi-asserted-by":"publisher","first-page":"547","DOI":"10.1109\/TWC.2009.070905","volume":"8","author":"J Lei","year":"2009","unstructured":"Lei J, Yates R, Greenstein L (2009) A generic model for optimizing single-hop transmission policy of replenishable sensors. IEEE Trans Wireless Commun 8:547\u2013551","journal-title":"IEEE Trans Wireless Commun"},{"key":"4643_CR24","doi-asserted-by":"publisher","first-page":"1872","DOI":"10.1109\/TWC.2013.030413.121120","volume":"12","author":"P Blasco","year":"2013","unstructured":"Blasco P, Gunduz D, Dohler M (2013) A learning theoretic approach to energy harvesting communication system optimization. IEEE Trans Wireless Commun 12:1872\u20131882","journal-title":"IEEE Trans Wireless Commun"},{"key":"4643_CR25","unstructured":"Putterman M. (2014) Markov decision processes.:discrete stochastic dynamic programming."},{"key":"4643_CR26","doi-asserted-by":"publisher","first-page":"258","DOI":"10.23919\/ICN.2020.0021","volume":"3","author":"Y Xiao","year":"2020","unstructured":"Xiao Y, Niu L, Ding Y, Liu S, Fan Y (2020) Reinforcement learning based energy-efficient internet-of-things video transmission. Intell Converg Netw 3:258\u2013270. https:\/\/doi.org\/10.23919\/ICN.2020.0021","journal-title":"Intell Converg Netw"},{"key":"4643_CR27","volume-title":"Reinforcement learning: an introduction","author":"R Sutton","year":"2018","unstructured":"Sutton R, Barto AG (2018) Reinforcement learning: an introduction. MIT Press"},{"key":"4643_CR28","doi-asserted-by":"publisher","first-page":"5996","DOI":"10.1007\/s11227-019-03069-0","volume":"76","author":"G Prakash","year":"2020","unstructured":"Prakash G, Krishnamoorthy R, Kalaivaani PT (2020) Resource key distribution and allocation based on sensor vehicle nodes for energy harvesting in vehicular ad hoc networks for transport application. J Supercomput 76:5996\u20136009","journal-title":"J Supercomput"},{"key":"4643_CR29","doi-asserted-by":"publisher","first-page":"2009","DOI":"10.1109\/JIOT.2018.2872440","volume":"6","author":"M Chu","year":"2019","unstructured":"Chu M, Li H, Liao X, Cui S (2019) Reinforcement learning-based multiaccess control and battery prediction with energy harvesting in IOT systems. IEEE Internet Things J 6:2009\u20132020","journal-title":"IEEE Internet Things J"},{"key":"4643_CR30","doi-asserted-by":"publisher","DOI":"10.1007\/s11227-021-04085-9","author":"H Teimourian","year":"2021","unstructured":"Teimourian H, Teimourian A, Dimililer K et al (2021) The potential of wind energy via an intelligent IoT-oriented assessment. J Supercomput. https:\/\/doi.org\/10.1007\/s11227-021-04085-9","journal-title":"J Supercomput"},{"issue":"5","key":"4643_CR31","doi-asserted-by":"publisher","first-page":"1135","DOI":"10.1109\/18.995554","volume":"48","author":"RA Berry","year":"2002","unstructured":"Berry RA, Gallager RG (2002) Communication over fading channels with delay constraints\u201d. IEEE Trans Inf Theory 48(5):1135\u20131149","journal-title":"IEEE Trans Inf Theory"},{"key":"4643_CR32","volume-title":"Constrained Markov decision processes","author":"E Altman","year":"1999","unstructured":"Altman E (1999) Constrained Markov decision processes. Routledge"},{"key":"4643_CR33","first-page":"871","volume":"43","author":"A Gosavi","year":"2014","unstructured":"Gosavi A (2014) \u201cVariance-penalized markov decision processes: dynamic programming and reinforcement learning techniques. Int J Gener Syst 43:871","journal-title":"Int J Gener Syst"},{"key":"4643_CR34","volume-title":"Nonlinear programming","author":"D Bertsekas","year":"1999","unstructured":"Bertsekas D (1999) Nonlinear programming. Athena Scientific"},{"key":"4643_CR35","doi-asserted-by":"publisher","first-page":"525","DOI":"10.1007\/BF02745577","volume":"22","author":"V Borkar","year":"1997","unstructured":"Borkar V, Konda V (1997) The actor-critic algorithm as multi-time-scale stochastic approximation. Sadhana 22:525\u2013543","journal-title":"Sadhana"},{"key":"4643_CR36","doi-asserted-by":"publisher","first-page":"1055","DOI":"10.1109\/TCOMM.2004.831354","volume":"52","author":"H Wang","year":"2004","unstructured":"Wang H, Mandayam NB (2004) A simple packet-transmission scheme for wireless data over fading channels. IEEE Trans Commun 52:1055\u20131059","journal-title":"IEEE Trans Commun"},{"key":"4643_CR37","volume-title":"Constrained markov decision processes","author":"E Altman","year":"1999","unstructured":"Altman E, Asingleutility I (1999) Constrained markov decision processes. Routledge"},{"key":"4643_CR38","volume-title":"Markov decision processes: discrete stochastic dynamic programming","author":"ML Puterman","year":"2014","unstructured":"Puterman ML (2014) Markov decision processes: discrete stochastic dynamic programming. Wiley"},{"key":"4643_CR39","doi-asserted-by":"publisher","first-page":"383","DOI":"10.1287\/opre.9.3.383","volume":"9","author":"JDC Little","year":"1961","unstructured":"Little JDC (1961) A proof for the queuing formula: L = (lambda) w. Oper Res 9:383\u2013387","journal-title":"Oper Res"},{"key":"4643_CR40","doi-asserted-by":"publisher","first-page":"145","DOI":"10.1145\/2492101.1555367","volume":"37","author":"AB Sharma","year":"2009","unstructured":"Sharma AB, Golubchik L, Govindan R, Neely MJ (2009) Dynamic data compression in multi-hop wireless networks. Sigmetrics Perform Eval Rev 37:145\u2013156","journal-title":"Sigmetrics Perform Eval Rev"},{"key":"4643_CR41","volume-title":"Machine Learning","author":"TM Mitchell","year":"1997","unstructured":"Mitchell TM (1997) Machine Learning, 1st edn. McGraw-Hill Inc.","edition":"1"},{"key":"4643_CR42","doi-asserted-by":"crossref","DOI":"10.1007\/978-1-4899-7491-4","volume-title":"Simulation-based optimization parametric optimization techniques and reinforcement learning","author":"A Gosavi","year":"2015","unstructured":"Gosavi A (2015) Simulation-based optimization parametric optimization techniques and reinforcement learning. Springer"},{"key":"4643_CR43","doi-asserted-by":"publisher","first-page":"4610","DOI":"10.1109\/TIT.2017.2773526","volume":"64","author":"P Sakulkar","year":"2018","unstructured":"Sakulkar P, Krishnamachari B (2018) Online learning schemes for power allocation in energy harvesting communications. IEEE Trans Inf Theory 64:4610\u20134628","journal-title":"IEEE Trans Inf Theory"},{"key":"4643_CR44","doi-asserted-by":"publisher","first-page":"1336","DOI":"10.1109\/TWC.2015.2489200","volume":"15","author":"D Zordan","year":"2016","unstructured":"Zordan D, Melodia T, Rossi M (2016) On the design of temporal compression strategies for energy harvesting sensor networks. IEEE Trans Wireless Commun 15:1336\u20131352","journal-title":"IEEE Trans Wireless Commun"}],"container-title":["The Journal of Supercomputing"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s11227-022-04643-9.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s11227-022-04643-9\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s11227-022-04643-9.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2023,2,10]],"date-time":"2023-02-10T17:32:54Z","timestamp":1676050374000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s11227-022-04643-9"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2022,7,3]]},"references-count":44,"journal-issue":{"issue":"1","published-print":{"date-parts":[[2023,1]]}},"alternative-id":["4643"],"URL":"https:\/\/doi.org\/10.1007\/s11227-022-04643-9","relation":{},"ISSN":["0920-8542","1573-0484"],"issn-type":[{"value":"0920-8542","type":"print"},{"value":"1573-0484","type":"electronic"}],"subject":[],"published":{"date-parts":[[2022,7,3]]},"assertion":[{"value":"1 June 2022","order":1,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"3 July 2022","order":2,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Declarations"}},{"value":"All authors declare that they have no conflict of interest that are relevant to the content of this article.","order":2,"name":"Ethics","group":{"name":"EthicsHeading","label":"Conflict of interest"}},{"value":"Data sharing is not applicable\u2014no new data generated.","order":3,"name":"Ethics","group":{"name":"EthicsHeading","label":"Data availability"}}]}}