{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,9,11]],"date-time":"2026-09-11T16:25:41Z","timestamp":1789143941303,"version":"build-2803163510"},"reference-count":37,"publisher":"Springer Science and Business Media LLC","issue":"11","license":[{"start":{"date-parts":[[2011,8,11]],"date-time":"2011-08-11T00:00:00Z","timestamp":1313020800000},"content-version":"tdm","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["Sci. China Inf. Sci."],"published-print":{"date-parts":[[2011,11]]},"DOI":"10.1007\/s11432-011-4332-6","type":"journal-article","created":{"date-parts":[[2011,8,11]],"date-time":"2011-08-11T02:09:37Z","timestamp":1313028577000},"page":"2279-2294","source":"Crossref","is-referenced-by-count":19,"title":["Hybrid MDP based integrated hierarchical Q-learning"],"prefix":"10.1007","volume":"54","author":[{"given":"ChunLin","family":"Chen","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"DaoYi","family":"Dong","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Han-Xiong","family":"Li","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Tzyh-Jong","family":"Tarn","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2011,8,11]]},"reference":[{"key":"4332_CR1","first-page":"133","volume-title":"Reinforcement Learning: An Introduction","author":"R. Sutton","year":"1998","unstructured":"Sutton R, Barto A G. Reinforcement Learning: An Introduction. Cambridge: MIT Press, 1998. 133\u2013156"},{"key":"4332_CR2","doi-asserted-by":"crossref","first-page":"2360","DOI":"10.1007\/s11432-009-0223-5","volume":"52","author":"Z. Y. Feng","year":"2009","unstructured":"Feng Z Y, Liang L T, Tan L, et al. Q-learning based heterogenous network self-optimization for reconfigurable network with CPC assistance. Sci China Ser F-Inf Sci, 2009, 52: 2360\u20132368","journal-title":"Sci China Ser F-Inf Sci"},{"key":"4332_CR3","doi-asserted-by":"crossref","first-page":"150","DOI":"10.1109\/TSMCB.2004.840124","volume":"35","author":"P. He","year":"2005","unstructured":"He P, Jagannathan S. Reinforcement learning-based output feedback control of nonlinear systems with input constraints. IEEE Trans Syst Man Cybern Part B-Cybern, 2005, 35: 150\u2013154","journal-title":"IEEE Trans Syst Man Cybern Part B-Cybern"},{"key":"4332_CR4","doi-asserted-by":"crossref","first-page":"111","DOI":"10.1016\/j.robot.2003.11.006","volume":"46","author":"T. Kondo","year":"2004","unstructured":"Kondo T, Ito K. A reinforcement learning with evolutionary state recruitment strategy for autonomous mobile robots control. Robot Auton Syst, 2004, 46: 111\u2013124","journal-title":"Robot Auton Syst"},{"key":"4332_CR5","doi-asserted-by":"crossref","first-page":"37","DOI":"10.1016\/S0921-8890(01)00113-0","volume":"36","author":"J. Morimoto","year":"2001","unstructured":"Morimoto J, Doya K. Acquisition of stand-up behavior by a real robot using hierarchical reinforcement learning. Robot Auton Syst, 2001, 36: 37\u201351","journal-title":"Robot Auton Syst"},{"key":"4332_CR6","first-page":"789","volume":"6","author":"C. Chen","year":"2010","unstructured":"Chen C, Dong D. Grey system based reactive navigation of mobile robots using reinforcement learning. Int J Innov Comp Inf Control, 2010, 6: 789\u2013800","journal-title":"Int J Innov Comp Inf Control"},{"key":"4332_CR7","doi-asserted-by":"crossref","first-page":"1954","DOI":"10.1007\/s11432-009-0198-2","volume":"52","author":"D. Z. Cheng","year":"2009","unstructured":"Cheng D Z. Advances in automation and control research in China. Sci China Ser F-Inf Sci, 2009, 52: 1954\u20131963","journal-title":"Sci China Ser F-Inf Sci"},{"key":"4332_CR8","doi-asserted-by":"crossref","first-page":"314","DOI":"10.1109\/3477.752807","volume":"29","author":"N. H. C. Yung","year":"1999","unstructured":"Yung N H C, Ye C. An intelligent mobile vehicle navigator based on fuzzy logic and reinforcement learning. IEEE Trans Syst Man Cybern Part B-Cybern, 1999, 29: 314\u2013321","journal-title":"IEEE Trans Syst Man Cybern Part B-Cybern"},{"key":"4332_CR9","doi-asserted-by":"crossref","first-page":"84","DOI":"10.1016\/j.robot.2005.09.015","volume":"54","author":"A. Montesanto","year":"2006","unstructured":"Montesanto A, Tascini G, Puliti P, et al. Navigation with memory in a partially observable environment. Robot Auton Syst, 2006, 54: 84\u201394","journal-title":"Robot Auton Syst"},{"key":"4332_CR10","first-page":"9","volume":"3","author":"R. Sutton","year":"1988","unstructured":"Sutton R. Learning to predict by the methods of temporal difference. Mach Learn, 1988, 3: 9\u201344","journal-title":"Mach Learn"},{"key":"4332_CR11","first-page":"279","volume":"8","author":"J. C. H. Watkins","year":"1992","unstructured":"Watkins J C H, Dayan P. Q-learning. Mach Learn, 1992, 8: 279\u2013292","journal-title":"Mach Learn"},{"key":"4332_CR12","volume-title":"Neuro-dynamic Programming","author":"D. P. Bertsekas","year":"1996","unstructured":"Bertsekas D P, Tsitsiklis J N. Neuro-dynamic Programming. Belmont: Athena Scientific, 1996. 36\u201351"},{"key":"4332_CR13","doi-asserted-by":"crossref","first-page":"399","DOI":"10.1007\/11750321_38","volume":"3959","author":"C. Chen","year":"2006","unstructured":"Chen C, Dong D, Chen Z. Grey reinforcement learning for incomplete information processing. Lect Notes Comput Sci, 2006, 3959: 399\u2013407","journal-title":"Lect Notes Comput Sci"},{"key":"4332_CR14","doi-asserted-by":"crossref","first-page":"1207","DOI":"10.1109\/TSMCB.2008.925743","volume":"38","author":"D. Dong","year":"2008","unstructured":"Dong D, Chen C, Li H, et al. Quantum reinforcement learning. IEEE Trans Syst Man Cybern Part B-Cybern, 2008, 38: 1207\u20131220","journal-title":"IEEE Trans Syst Man Cybern Part B-Cybern"},{"key":"4332_CR15","doi-asserted-by":"crossref","first-page":"957","DOI":"10.1109\/TSMCB.2008.926603","volume":"38","author":"D. Dong","year":"2008","unstructured":"Dong D, Chen C, Tarn T J, et al. Incoherent control of quantum systems with wavefunction controllable subspaces via quantum reinforcement learning. IEEE Trans Syst Man Cybern Part B-Cybern, 2008, 38: 957\u2013962","journal-title":"IEEE Trans Syst Man Cybern Part B-Cybern"},{"key":"4332_CR16","doi-asserted-by":"crossref","first-page":"1071","DOI":"10.1142\/S0219749906002419","volume":"4","author":"C. Chen","year":"2006","unstructured":"Chen C, Dong D, Chen Z. Quantum computation for action selection using reinforcement learning. Int J Quantum Inf, 2006, 4: 1071\u20131083","journal-title":"Int J Quantum Inf"},{"key":"4332_CR17","doi-asserted-by":"crossref","first-page":"1691","DOI":"10.1088\/0256-307X\/23\/7\/010","volume":"23","author":"D. Dong","year":"2006","unstructured":"Dong D, Chen C, Chen Z, et al. Quantum mechanics helps in learning for more intelligent robots. Chin Phys Lett, 2006, 23: 1691\u20131694","journal-title":"Chin Phys Lett"},{"key":"4332_CR18","doi-asserted-by":"crossref","first-page":"513","DOI":"10.1017\/S0263574705002596","volume":"24","author":"D. Dong","year":"2006","unstructured":"Dong D, Chen C, Zhang C, et al. Quantum robot: structure, algorithms and applications. Robotica, 2006, 24: 513\u2013521","journal-title":"Robotica"},{"key":"4332_CR19","first-page":"283","volume":"22","author":"P. Jing","year":"1996","unstructured":"Jing P, Ronald J W. Increment multi-step Q-learning. Mach Learn, 1996, 22: 283\u2013291","journal-title":"Mach Learn"},{"key":"4332_CR20","first-page":"159","volume":"22","author":"S. Mahadevan","year":"1996","unstructured":"Mahadevan S. Average reward reinforcement learning: Foundations, algorithms and empirical results. Mach Learn, 1996, 22: 159\u2013195","journal-title":"Mach Learn"},{"key":"4332_CR21","doi-asserted-by":"crossref","first-page":"241","DOI":"10.1016\/S0921-8890(03)00074-5","volume":"44","author":"P. Althaus","year":"2003","unstructured":"Althaus P, Christensen H I. Smooth task switching through behavior competition. Robot Auton Syst, 2003, 44: 241\u2013249","journal-title":"Robot Auton Syst"},{"key":"4332_CR22","doi-asserted-by":"crossref","unstructured":"Hallerdal M, Hallamy J. Behavior selection on a mobile robot using W-learning. In: Hallam B, Floreano D, Hallam J, et al., eds. Proceedings of the Seventh International Conference on the Simulation of Adaptive Behavior on from animals to animates, Edinburgh, UK, 2002. 93\u2013102","DOI":"10.7551\/mitpress\/3121.001.0001"},{"key":"4332_CR23","doi-asserted-by":"crossref","first-page":"219","DOI":"10.1177\/105971239700600202","volume":"6","author":"M. Wiering","year":"1997","unstructured":"Wiering M, Schmidhuber J. HQ-Learning. Adapt Behav, 1997, 6: 219\u2013246","journal-title":"Adapt Behav"},{"key":"4332_CR24","doi-asserted-by":"crossref","first-page":"41","DOI":"10.1023\/A:1022140919877","volume":"13","author":"A. G. Barto","year":"2003","unstructured":"Barto A G, Mahadevan S. Recent advances in hierarchical reinforcement learning. Discret Event Dyn Syst-Theory Appl, 2003, 13: 41\u201377","journal-title":"Discret Event Dyn Syst-Theory Appl"},{"key":"4332_CR25","first-page":"611","volume":"16","author":"C. Chen","year":"2005","unstructured":"Chen C, Chen Z. Reinforcement learning for mobile robot: From reaction to deliberation. J Syst Eng Electron, 2005, 16: 611\u2013617","journal-title":"J Syst Eng Electron"},{"key":"4332_CR26","doi-asserted-by":"crossref","first-page":"674","DOI":"10.1109\/9.580874","volume":"42","author":"J. N. Tsitsiklis","year":"1997","unstructured":"Tsitsiklis J N, VanRoy B. An analysis of temporal-difference learning with function approximation. IEEE Trans Autom Control, 1997, 42: 674\u2013690","journal-title":"IEEE Trans Autom Control"},{"key":"4332_CR27","first-page":"1057","volume":"12","author":"R. S. Sutton","year":"2000","unstructured":"Sutton R S, McAllester D, Singh S, et al. Policy gradient methods for reinforcement learning with function approximation. Adv Neural Inf Process Syst, 2000, 12: 1057\u20131063","journal-title":"Adv Neural Inf Process Syst"},{"key":"4332_CR28","doi-asserted-by":"crossref","first-page":"161","DOI":"10.1023\/A:1017928328829","volume":"49","author":"D. Ormoneit","year":"2002","unstructured":"Ormoneit D, Sen S. Kernel-based reinforcement learning. Mach Learn, 2002, 49: 161\u2013178","journal-title":"Mach Learn"},{"key":"4332_CR29","doi-asserted-by":"crossref","first-page":"181","DOI":"10.1016\/S0004-3702(99)00052-1","volume":"112","author":"R. Sutton","year":"1999","unstructured":"Sutton R, Precup D, Singh S. Between MDPs and semi-MDPs: A framework for temporal abstraction in reinforcement learning. Artif Intell, 1999, 112: 181\u2013211","journal-title":"Artif Intell"},{"key":"4332_CR30","first-page":"1043","volume":"10","author":"P. Parr","year":"1998","unstructured":"Parr P, Russell S. Reinforcement learning with hierarchies of machines. Adv Neural Inf Process Syst, 1998, 10: 1043\u20131049","journal-title":"Adv Neural Inf Process Syst"},{"key":"4332_CR31","doi-asserted-by":"crossref","first-page":"227","DOI":"10.1613\/jair.639","volume":"13","author":"T. G. Dietterich","year":"2000","unstructured":"Dietterich T G. Hierarchical reinforcement learning with the Maxq value function decomposition. J Artif Intell Res, 2000, 13: 227\u2013303","journal-title":"J Artif Intell Res"},{"key":"4332_CR32","first-page":"30","volume-title":"Hierarchical learning and planning in partially observable Markov decision processes","author":"G. Theocharous","year":"2002","unstructured":"Theocharous G. Hierarchical learning and planning in partially observable Markov decision processes. Dissertation for Doctoral Degree. East Lansing: Michigan State University, USA, 2002. 30\u201372"},{"key":"4332_CR33","doi-asserted-by":"crossref","first-page":"37","DOI":"10.1109\/MRA.2008.921541","volume":"15","author":"C. Chen","year":"2008","unstructured":"Chen C, Li H, Dong D. Hybrid control for autonomous mobile robot navigation-a hierarchical Q-learning algorithm. IEEE Robot Autom Mag, 2008, 15: 37\u201347","journal-title":"IEEE Robot Autom Mag"},{"key":"4332_CR34","first-page":"1","volume-title":"Qualitative Reasoning: Modeling and Simulation with Incomplete Knowledge","author":"B. Kuipers","year":"1994","unstructured":"Kuipers B. Qualitative Reasoning: Modeling and Simulation with Incomplete Knowledge. Cambridge: MIT Press, 1994. 1\u201327"},{"key":"4332_CR35","doi-asserted-by":"crossref","first-page":"215","DOI":"10.1016\/S0004-3702(97)00050-7","volume":"95","author":"D. Berleant","year":"1997","unstructured":"Berleant D, Kuipers B. Qualitative and quantitative simulation: Bridging the gap. Artif Intell, 1997, 95: 215\u2013255","journal-title":"Artif Intell"},{"key":"4332_CR36","doi-asserted-by":"crossref","first-page":"2140","DOI":"10.1109\/TSMCB.2004.832154","volume":"34","author":"M. Z. Guo","year":"2004","unstructured":"Guo M Z, Liu Y, Malec J. A new Q-learning algorithm based on the metropolis criterion. IEEE Trans Syst Man Cybern Part B-Cybern, 2004, 34: 2140\u20132143","journal-title":"IEEE Trans Syst Man Cybern Part B-Cybern"},{"key":"4332_CR37","unstructured":"Dong D, Chen C, Chu J, et al. Robust quantum-inspired reinforcement learning for robot navigation. IEEE-ASME Trans Mechatron, 2011, in press"}],"container-title":["Science China Information Sciences"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s11432-011-4332-6.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/article\/10.1007\/s11432-011-4332-6\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s11432-011-4332-6","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2020,6,22]],"date-time":"2020-06-22T05:57:30Z","timestamp":1592805450000},"score":1,"resource":{"primary":{"URL":"http:\/\/link.springer.com\/10.1007\/s11432-011-4332-6"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2011,8,11]]},"references-count":37,"journal-issue":{"issue":"11","published-print":{"date-parts":[[2011,11]]}},"alternative-id":["4332"],"URL":"https:\/\/doi.org\/10.1007\/s11432-011-4332-6","relation":{},"ISSN":["1674-733X","1869-1919"],"issn-type":[{"value":"1674-733X","type":"print"},{"value":"1869-1919","type":"electronic"}],"subject":[],"published":{"date-parts":[[2011,8,11]]}}}