{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,4,15]],"date-time":"2025-04-15T05:37:37Z","timestamp":1744695457956,"version":"3.37.3"},"reference-count":28,"publisher":"Springer Science and Business Media LLC","issue":"S1","license":[{"start":{"date-parts":[[2017,11,10]],"date-time":"2017-11-10T00:00:00Z","timestamp":1510272000000},"content-version":"tdm","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"}],"funder":[{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["61672371","61602334"],"award-info":[{"award-number":["61672371","61602334"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["61502329","61502323"],"award-info":[{"award-number":["61502329","61502323"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["61272005","61303108"],"award-info":[{"award-number":["61272005","61303108"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["61373094","61472262"],"award-info":[{"award-number":["61373094","61472262"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]},{"name":"Natural Science Foundation of Jiangsu","award":["BK20140283"],"award-info":[{"award-number":["BK20140283"]}]},{"name":"High School Natural Foundation of Jiangsu","award":["BK2012616","13KJB520020"],"award-info":[{"award-number":["BK2012616","13KJB520020"]}]},{"name":"Fundation of Ministry of Housing and Urban-Rural Development of the People\u2019s Republic of China","award":["2015-K1-047"],"award-info":[{"award-number":["2015-K1-047"]}]},{"name":"Key Laboratory of Symbolic Computation and Knowledge Engineering of Ministry of Education, Jilin University","award":["93K172014K04"],"award-info":[{"award-number":["93K172014K04"]}]},{"name":"Suzhou Industrial application of basic research program part","award":["SYG201422"],"award-info":[{"award-number":["SYG201422"]}]}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Cluster Comput"],"published-print":{"date-parts":[[2019,1]]},"DOI":"10.1007\/s10586-017-1303-8","type":"journal-article","created":{"date-parts":[[2017,11,10]],"date-time":"2017-11-10T10:54:03Z","timestamp":1510311243000},"page":"795-807","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":4,"title":["Residual Sarsa algorithm with function approximation"],"prefix":"10.1007","volume":"22","author":[{"given":"Fu","family":"Qiming","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Hu","family":"Wen","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Liu","family":"Quan","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Luo","family":"Heng","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Hu","family":"Lingyao","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Chen","family":"Jianping","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2017,11,10]]},"reference":[{"key":"1303_CR1","volume-title":"Reinforcement learning: an introduction","author":"RS Sutton","year":"1998","unstructured":"Sutton, R.S., Barto, A.G.: Reinforcement learning: an introduction. MIT press, Cambridge (1998)"},{"issue":"1","key":"1303_CR2","first-page":"66","volume":"32","author":"Q Liu","year":"2011","unstructured":"Liu, Q., Fu, Q.M., Gong, S.R., Fu, Y.C., Cui, Z.M.: Reinforcement learning algorithm based on minimum state method and average reward. J. Commun. 32(1), 66\u201371 (2011)","journal-title":"J. Commun."},{"key":"1303_CR3","first-page":"9","volume":"3","author":"RS Sutton","year":"1988","unstructured":"Sutton, R.S.: Learning to predict by the method of temporal differences. Mach. Learn. 3, 9\u201344 (1988)","journal-title":"Mach. Learn."},{"key":"1303_CR4","doi-asserted-by":"crossref","unstructured":"Go, C.K., Lao, B., Yoshimoto J., et al.: A reinforcement learning approach to the shepherding task using Sarsa. In: Proceedings of International Joint Conference on Neural Networks (IJCNN), Kuala Lumpur, Malaysia (2016)","DOI":"10.1109\/IJCNN.2016.7727694"},{"key":"1303_CR5","doi-asserted-by":"publisher","first-page":"321","DOI":"10.1016\/j.asoc.2015.09.003","volume":"38","author":"S Chettibi","year":"2016","unstructured":"Chettibi, S., Chikhi, S.: Dynamic fuzzy logic and reinforcement learning for adaptive energy efficient routing in mobile ad-hoc networks. Appl. Soft Comput. 38, 321\u2013328 (2016)","journal-title":"Appl. Soft Comput."},{"key":"1303_CR6","doi-asserted-by":"crossref","unstructured":"Ortiz, A., Al-Shatri, H., Li, X., et al.: Reinforcement learning for energy harvesting point-to-point communications. In: Proceedings of IEEE International Conference on Communications (ICC), Kuala Lumpur, Malaysia (2016)","DOI":"10.1109\/ICC.2016.7511405"},{"key":"1303_CR7","unstructured":"Saadatjou, F., Derhami, V., Majd, V.: Balance of exploration and exploitation in deterministic and stochastic environment in reinforcement learning. In: Proceedings of the 11th Annual Computer Society of Iran Computer Conference, Tehran, Iran (2006)"},{"issue":"3","key":"1303_CR8","doi-asserted-by":"publisher","first-page":"177","DOI":"10.1080\/10255810213482","volume":"4","author":"G Yen","year":"2002","unstructured":"Yen, G., Yang, F., Hickey, T.: Coordination of exploration and exploitation in a dynamic environment. Int. J. Smart Eng. Syst. Des. 4(3), 177\u2013182 (2002)","journal-title":"Int. J. Smart Eng. Syst. Des."},{"issue":"4","key":"1303_CR9","doi-asserted-by":"publisher","first-page":"578","DOI":"10.1016\/j.fss.2009.05.003","volume":"161","author":"V Derhami","year":"2010","unstructured":"Derhami, V., Majd, V.J., Ahmadabadi, M.N.: Exploration and exploitation balance management in fuzzy reinforcement learning. Fuzzy Sets Syst. 161(4), 578\u2013595 (2010)","journal-title":"Fuzzy Sets Syst."},{"key":"1303_CR10","doi-asserted-by":"crossref","unstructured":"You, S.H., Liu, Q., Fu, Q.M., et al.: A Bayesian Sarsa learning algorithm with bandit-based method. In: Proceedings of International Conference on Neural Information Processing (2015)","DOI":"10.1007\/978-3-319-26532-2_13"},{"issue":"8","key":"1303_CR11","first-page":"1469","volume":"41","author":"Q Liu","year":"2013","unstructured":"Liu, Q., Li, J., Fu, Q.M.: A multiple-goal Sarsa(\n                    \n                      \n                    \n                    $$\\lambda )$$\n                    \n                      \n                        \n                          \u03bb\n                          )\n                        \n                      \n                    \n                   algorithm based on lost reward of greatest mass. J. Electron. 41(8), 1469\u20131473 (2013)","journal-title":"J. Electron."},{"key":"1303_CR12","first-page":"77","volume":"1","author":"F Xiao","year":"2013","unstructured":"Xiao, F., Liu, Q., Fu, Q.M.: Gradient descent Sarsa(\n                    \n                      \n                    \n                    $$\\lambda )$$\n                    \n                      \n                        \n                          \u03bb\n                          )\n                        \n                      \n                    \n                   algorithm based on the adaptive potential function shaping reward mechanism. J. Commun. 1, 77\u201388 (2013)","journal-title":"J. Commun."},{"issue":"11","key":"1303_CR13","first-page":"2157","volume":"42","author":"QM Fu","year":"2014","unstructured":"Fu, Q.M., Liu, Q., You, S.H.: A novel fast Sarsa algorithm based on value function transfer. J. Electron. 42(11), 2157\u20132161 (2014)","journal-title":"J. Electron."},{"key":"1303_CR14","doi-asserted-by":"crossref","unstructured":"Zhu, H., Zhu, F., Fu, Y., et al.: A kernel-based Sarsa(\n                    \n                      \n                    \n                    $$\\lambda )$$\n                    \n                      \n                        \n                          \u03bb\n                          )\n                        \n                      \n                    \n                   algorithm with clustering-based sample sparsification. In: Proceedings of International Conference on Neural Information Processing, Kyoto, Japan (2016)","DOI":"10.1007\/978-3-319-46675-0_24"},{"issue":"1","key":"1303_CR15","doi-asserted-by":"publisher","first-page":"89","DOI":"10.1007\/s10994-007-5038-2","volume":"71","author":"A Antos","year":"2008","unstructured":"Antos, A., Szepesvari, C., Mounos, R.: Learning near-optimal polices with bellman-residual minimization based fitted policy iteration and a single sample path. Mach. Learn. 71(1), 89\u2013129 (2008)","journal-title":"Mach. Learn."},{"key":"1303_CR16","volume-title":"Reinforcement learning and dynamic programming using function approximators","author":"L Busoniu","year":"2010","unstructured":"Busoniu, L., Babuska, R., De Schutter, B., et al.: Reinforcement learning and dynamic programming using function approximators. CRC Press, New York (2010)"},{"key":"1303_CR17","doi-asserted-by":"crossref","unstructured":"Indyk, P., Motwani, R.: Approximate nearest neighbors: towards removing the curse of dimensionality. In: Proceedings of the 30th annual ACM symposium on Theory of computing, New York, USA (1998)","DOI":"10.1145\/276698.276876"},{"key":"1303_CR18","doi-asserted-by":"crossref","unstructured":"Geist, M., Pietquin, O.: Parametric value function approximation. In: Proceedings of the IEEE Symposium on Adaptive Dynamic Programming and Reinforcement Learning, Paris, France (2011)","DOI":"10.1109\/ADPRL.2011.5967355"},{"key":"1303_CR19","doi-asserted-by":"crossref","unstructured":"Akimoto, Y., Auger, A., Hansen N.: Comparison-based natural gradient optimization in high dimension. In: Proceedings of the Annual Conference on Genetic and Evolutionary Computation. Vancouver, Canada (2014)","DOI":"10.1145\/2576768.2598258"},{"key":"1303_CR20","doi-asserted-by":"crossref","unstructured":"Sutton, R.S., Maei, H.R., Szepesv\u00e1ri, C. et al.: A convergent O(n) Temporal-difference algorithm for Off-policy learning with linear function approximation. In: Proceedings of the Advances Neural Information Processing Systems, Vancouver, Canada (2009)","DOI":"10.1145\/1553374.1553501"},{"key":"1303_CR21","doi-asserted-by":"crossref","unstructured":"Sutton, R.S., Hamid, R.M., Precup, D.: Fast gradient-descent methods for temporal-difference learning with linear function approximation. In: Proceedings of the 26th International Conference on Machine Learning, New York, USA (2009)","DOI":"10.1145\/1553374.1553501"},{"key":"1303_CR22","unstructured":"Maei, H.R, Szepesvari, C., Bhatnagar, S. et al.: Toward off-policy learning control with function approixamtion. In: Proceedings of the 27th International Conference on Machine Learning, Haifa, Israel (2010)"},{"issue":"1\u20132","key":"1303_CR23","doi-asserted-by":"publisher","first-page":"205","DOI":"10.1007\/s10994-011-5251-x","volume":"84","author":"S Kalyanakrish","year":"2011","unstructured":"Kalyanakrish, S., Stone, P.: Characterizing reinforcement learning methods through parameterized learning problems. Mach. Learn. 84(1\u20132), 205\u2013247 (2011)","journal-title":"Mach. Learn."},{"key":"1303_CR24","doi-asserted-by":"crossref","unstructured":"Ja\u015bkowski, W., Szubert, M., Liskowski, P. et al.: High-dimensional function approximation for knowledge-free reinforcement learning: a case study in SZ-Tetris. In: Proceedings of the Annual Conference on Genetic and Evolutionary Computation, New York, USA (2015)","DOI":"10.1145\/2739480.2754783"},{"key":"1303_CR25","unstructured":"Van Seijen, H.: Effective multi-step temporal-difference learning for non-linear function approximation. arXiv preprint \n                    arXiv:1608.05151\n                    \n                   (2016)"},{"key":"1303_CR26","unstructured":"Veeriah, V., Van Seijen, H., Sutton, R.S.: Forward actor-Critic for nonlinear function approximation in reinforcement learning. In: Proceedings of the 16th Conference on Autonomous Agents and MultiAgent Systems, S\u00e3o Paulo, Brazil (2017)"},{"issue":"3","key":"1303_CR27","doi-asserted-by":"publisher","first-page":"287","DOI":"10.1023\/A:1007678930559","volume":"38","author":"S Singh","year":"2000","unstructured":"Singh, S., Jaakkola, T., Littman, M.L., et al.: Convergence results for single-step on-policy reinforcement-learning algorithms. Mach. Learn. 38(3), 287\u2013308 (2000)","journal-title":"Mach. Learn."},{"issue":"2","key":"1303_CR28","doi-asserted-by":"publisher","first-page":"357","DOI":"10.1109\/21.229449","volume":"23","author":"E Barnard","year":"1993","unstructured":"Barnard, E.: Temporal-difference methods and Markov models. IEEE Trans. Syst. Man Cybern. 23(2), 357\u2013365 (1993)","journal-title":"IEEE Trans. Syst. Man Cybern."}],"container-title":["Cluster Computing"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s10586-017-1303-8.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/article\/10.1007\/s10586-017-1303-8\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s10586-017-1303-8.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2019,9,22]],"date-time":"2019-09-22T04:38:46Z","timestamp":1569127126000},"score":1,"resource":{"primary":{"URL":"http:\/\/link.springer.com\/10.1007\/s10586-017-1303-8"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2017,11,10]]},"references-count":28,"journal-issue":{"issue":"S1","published-print":{"date-parts":[[2019,1]]}},"alternative-id":["1303"],"URL":"https:\/\/doi.org\/10.1007\/s10586-017-1303-8","relation":{},"ISSN":["1386-7857","1573-7543"],"issn-type":[{"type":"print","value":"1386-7857"},{"type":"electronic","value":"1573-7543"}],"subject":[],"published":{"date-parts":[[2017,11,10]]},"assertion":[{"value":"18 July 2017","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"17 October 2017","order":2,"name":"revised","label":"Revised","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"28 October 2017","order":3,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"10 November 2017","order":4,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}}]}}