{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,26]],"date-time":"2026-04-26T11:50:45Z","timestamp":1777204245188,"version":"3.51.4"},"reference-count":39,"publisher":"Springer Science and Business Media LLC","issue":"13","license":[{"start":{"date-parts":[[2022,3,16]],"date-time":"2022-03-16T00:00:00Z","timestamp":1647388800000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"},{"start":{"date-parts":[[2022,3,16]],"date-time":"2022-03-16T00:00:00Z","timestamp":1647388800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Appl Intell"],"published-print":{"date-parts":[[2022,10]]},"DOI":"10.1007\/s10489-022-03363-0","type":"journal-article","created":{"date-parts":[[2022,3,16]],"date-time":"2022-03-16T13:02:41Z","timestamp":1647435761000},"page":"15447-15461","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":5,"title":["Improving the exploration efficiency of DQNs via the confidence bound methods"],"prefix":"10.1007","volume":"52","author":[{"ORCID":"https:\/\/orcid.org\/0000-0001-9372-1453","authenticated-orcid":false,"given":"Yingpeng","family":"Wen","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Qinliang","family":"Su","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Minghua","family":"Shen","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Nong","family":"Xiao","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2022,3,16]]},"reference":[{"key":"3363_CR1","doi-asserted-by":"crossref","unstructured":"Auer P, Ortner R (2006) Logarithmic online regret bounds for undiscounted reinforcement learning. In: Conference on advances in neural information processing systems","DOI":"10.7551\/mitpress\/7503.003.0011"},{"key":"3363_CR2","doi-asserted-by":"crossref","unstructured":"Balaprakash P, Egele R, Salim M, Wild S, Vishwanath V, Xia F, Brettin T, Stevens R (2019) Scalable reinforcement-learning-based neural architecture search for cancer deep learning research. In: Proceedings of the international conference for high performance computing, networking, storage and analysis, pp 1\u201333","DOI":"10.1145\/3295500.3356202"},{"key":"3363_CR3","first-page":"17","volume-title":"Intrinsic Motivation and Reinforcement Learning","author":"GA Barto","year":"2013","unstructured":"Barto GA (2013) Intrinsic Motivation and Reinforcement Learning. Springer, Berlin, pp 17\u201347"},{"key":"3363_CR4","unstructured":"Bellemare M, Srinivasan S, Ostrovski G, Schaul T, Saxton D, Munos R (2016) Unifying count-based exploration and intrinsic motivation. In: Advances in neural information processing systems, pp 1471\u20131479"},{"key":"3363_CR5","unstructured":"Bellemare MG, Dabney W, Munos R (2017) A distributional perspective on reinforcement learning. In: Proceedings of the 34th international conference on machine learning, vol 70, pp 449\u2013458. JMLR.org"},{"issue":"1","key":"3363_CR6","doi-asserted-by":"publisher","first-page":"253","DOI":"10.1613\/jair.3912","volume":"47","author":"MG Bellemare","year":"2013","unstructured":"Bellemare MG, Naddaf Y, Veness J, Bowling M (2013) The arcade learning environment: an evaluation platform for general agents. J Artif Intell Res 47(1):253\u2013279","journal-title":"J Artif Intell Res"},{"key":"3363_CR7","unstructured":"Castro PS, Moitra S, Gelada C, Kumar S, Bellemare MG (2018) Dopamine: A research framework for deep reinforcement learning. arXiv:1812.06110"},{"key":"3363_CR8","unstructured":"Chapelle O, Li L (2011) An empirical evaluation of thompson sampling. In: Shawe-Taylor J, Zemel RS, Bartlett PL, Pereira F, Weinberger KQ (eds) advances in neural information processing systems. http:\/\/papers.nips.cc\/paper\/4321-an-empirical-evaluation-of-thompson-sampling.pdf, vol 24. Curran Associates Inc, pp 2249\u20132257"},{"key":"3363_CR9","unstructured":"Chen G, Peng Y, Zhang M (2018) Effective exploration for deep reinforcement learning via bootstrapped q-ensembles under tsallis entropy regularization. arXiv:1809.00403"},{"key":"3363_CR10","unstructured":"Chen RY, Sidor S, Abbeel P, Schulman J (2017) Ucb exploration via q-ensembles. arXiv:1706.01502"},{"key":"3363_CR11","doi-asserted-by":"crossref","unstructured":"Dabney W, Ostrovski G, Silver D, Munos R (2018) Implicit quantile networks for distributional reinforcement learning. In: International conference on machine learning. PMLR, pp 1096\u20131105","DOI":"10.1609\/aaai.v32i1.11791"},{"key":"3363_CR12","doi-asserted-by":"crossref","unstructured":"Diuk C, Cohen A, Littman ML (2008) An object-oriented representation for efficient reinforcement learning. In: International conference on machine learning","DOI":"10.1145\/1390156.1390187"},{"key":"3363_CR13","unstructured":"Hasselt HV, Guez A, Silver D (2015) Deep reinforcement learning with double q-learning Computer Science"},{"key":"3363_CR14","doi-asserted-by":"crossref","unstructured":"He J, Chen J, He X, Gao J, Ostendorf M (2016) Deep reinforcement learning with a natural language action space. In: Proceedings of the 54th annual meeting of the association for computational linguistics (Volume 1: Long Papers)","DOI":"10.18653\/v1\/P16-1153"},{"key":"3363_CR15","doi-asserted-by":"crossref","unstructured":"Hessel M, Modayil J, Van Hasselt H, Schaul T, Ostrovski G, Dabney W, Horgan D, Piot B, Azar M, Silver D (2018) Rainbow: Combining improvements in deep reinforcement learning. In: Thirty-second AAAI conference on artificial intelligence","DOI":"10.1609\/aaai.v32i1.11796"},{"key":"3363_CR16","unstructured":"Horgan D, Quan J, Budden D, Barth-Maron G, Hessel M, van Hasselt H, Silver D (2018) Distributed prioritized experience replay"},{"key":"3363_CR17","unstructured":"Jaksch T, Ortner R, Auer P (2008) Near-optimal regret bounds for reinforcement learning. In: Conference on neural information processing systems"},{"key":"3363_CR18","unstructured":"Jin C, Allen-Zhu Z, Bubeck S, Jordan MI (2018) Is q-learning provably efficient? In: Bengio S, Wallach H, Larochelle H, Grauman K, Cesa-Bianchi N, Garnett R (eds) Advances in neural information processing systems. http:\/\/papers.nips.cc\/paper\/7735-is-q-learning-provably-efficient.pdf, vol 31. Curran Associates Inc, pp 4863\u20134873"},{"issue":"3","key":"3363_CR19","doi-asserted-by":"publisher","first-page":"1466","DOI":"10.1109\/LRA.2018.2800085","volume":"3","author":"K Lee","year":"2018","unstructured":"Lee K, Choi S, Oh S (2018) Sparse markov decision processes with causal sparse tsallis entropy regularization for reinforcement learning. IEEE Robot Autom Lett 3(3):1466\u20131473","journal-title":"IEEE Robot Autom Lett"},{"issue":"6","key":"3363_CR20","first-page":"A187","volume":"8","author":"TP Lillicrap","year":"2015","unstructured":"Lillicrap TP, Hunt JJ, Pritzel A, Heess N, Erez T, Tassa Y, Silver D, Wierstra D (2015) Continuous control with deep reinforcement learning. Comput Sci 8(6):A187","journal-title":"Comput Sci"},{"key":"3363_CR21","doi-asserted-by":"crossref","unstructured":"Martin J, Sasikumar SN, Everitt T, Hutter M (2017) Count-based exploration in feature space for reinforcement learning. arXiv:1706.08090","DOI":"10.24963\/ijcai.2017\/344"},{"key":"3363_CR22","unstructured":"Obando-Ceron JS, Castro PS (2020) Revisiting rainbow: Promoting more insightful and inclusive deep reinforcement learning research. arXiv:2011.14826"},{"key":"3363_CR23","unstructured":"OpenAI (2019) Openai five defeats dota 2 world champions. https:\/\/openai.com\/blog\/openai-five-defeats-dota-2-world-champions\/"},{"key":"3363_CR24","first-page":"4026","volume":"29","author":"I Osband","year":"2016","unstructured":"Osband I, Blundell C, Pritzel A, Van Roy B (2016) Deep exploration via bootstrapped dqn. Adv Neural Inf Process Syst 29:4026\u20134034","journal-title":"Adv Neural Inf Process Syst"},{"key":"3363_CR25","unstructured":"Osband I, Russo D, Roy BV (2013) (more) efficient reinforcement learning via posterior sampling. Advances in Neural Information Processing Systems, 3003\u20133011"},{"key":"3363_CR26","unstructured":"Ostrovski G, Bellemare MG, van den Oord A, Munos R (2017) Count-based exploration with neural density models. In: Proceedings of the 34th international conference on machine learning, vol 70, pp 2721\u20132730. JMLR.org"},{"issue":"1","key":"3363_CR27","doi-asserted-by":"publisher","first-page":"55","DOI":"10.1007\/s10514-009-9120-4","volume":"27","author":"M Riedmiller","year":"2009","unstructured":"Riedmiller M, Gabel T, Hafner R, Lange S (2009) Reinforcement learning for robot soccer. Auton Robot 27(1):55\u201373","journal-title":"Auton Robot"},{"issue":"4","key":"3363_CR28","doi-asserted-by":"publisher","first-page":"A95","DOI":"10.1287\/moor.2014.0650","volume":"39","author":"D Russo","year":"2014","unstructured":"Russo D, Roy BV (2014) Learning to optimize via posterior sampling. Math Oper Res 39 (4):A95","journal-title":"Math Oper Res"},{"issue":"1","key":"3363_CR29","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1561\/2200000070","volume":"11","author":"DJ Russo","year":"2018","unstructured":"Russo DJ, Van Roy B, Kazerouni A, Osband I, Wen Z et al (2018) A tutorial on thompson sampling. Foundations and Trends\u00ae;, in Machine Learning 11(1):1\u201396","journal-title":"Foundations and Trends\u00ae;, in Machine Learning"},{"key":"3363_CR30","unstructured":"Schaul T, Quan J, Antonoglou I, Silver D (2015) Prioritized experience replay. arXiv:1511.05952"},{"key":"3363_CR31","unstructured":"Schulman J, Levine S, Moritz P, Jordan MI, Abbeel P (2015) Trust region policy optimization. In: ICML"},{"issue":"7587","key":"3363_CR32","doi-asserted-by":"publisher","first-page":"484","DOI":"10.1038\/nature16961","volume":"529","author":"D Silver","year":"2016","unstructured":"Silver D, Huang A, Maddison CJ, Guez A, Sifre L, Driessche GVD, Schrittwieser J, Antonoglou I, Panneershelvam V, Lanctot M (2016) Mastering the game of go with deep neural networks and tree search. Nature 529(7587):484\u2013489","journal-title":"Nature"},{"key":"3363_CR33","unstructured":"Strens M (2000) A bayesian framework for reinforcement learning. In: Seventeenth international conference on machine learning"},{"issue":"3","key":"3363_CR34","doi-asserted-by":"publisher","first-page":"58","DOI":"10.1145\/203330.203343","volume":"38","author":"G Tesauro","year":"1995","unstructured":"Tesauro G (1995) Temporal difference learning and td-gammon. Communications of the Acm 38 (3):58\u201368","journal-title":"Communications of the Acm"},{"issue":"3\/4","key":"3363_CR35","doi-asserted-by":"publisher","first-page":"285","DOI":"10.2307\/2332286","volume":"25","author":"WR Thompson","year":"1933","unstructured":"Thompson WR (1933) On the likelihood that one unknown probability exceeds another in view of the evidence of two samples. Biometrika 25(3\/4):285\u2013294","journal-title":"Biometrika"},{"issue":"7540","key":"3363_CR36","doi-asserted-by":"publisher","first-page":"529","DOI":"10.1038\/nature14236","volume":"518","author":"M Volodymyr","year":"2015","unstructured":"Volodymyr M, Koray K, David S, Rusu AA, Joel V, Bellemare MG, Alex G, Martin R, Fidjeland AK, Georg O (2015) Human-level control through deep reinforcement learning. Nature 518(7540):529","journal-title":"Nature"},{"key":"3363_CR37","unstructured":"Young K, Tian T (2019) Minatar: An atari-inspired testbed for thorough and reproducible reinforcement learning experiments. arXiv:1903.03176"},{"key":"3363_CR38","unstructured":"Zambaldi V, Raposo D, Santoro A, Bapst V, Li Y, Babuschkin I, Tuyls K, Reichert D, Lillicrap T, Lockhart E et al (2018) Relational deep reinforcement learning. arXiv:1806.01830"},{"key":"3363_CR39","unstructured":"Zoph B, Le QV (2016) Neural architecture search with reinforcement learning. arXiv:1611.01578"}],"container-title":["Applied Intelligence"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10489-022-03363-0.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s10489-022-03363-0\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10489-022-03363-0.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2023,11,18]],"date-time":"2023-11-18T22:20:01Z","timestamp":1700346001000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s10489-022-03363-0"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2022,3,16]]},"references-count":39,"journal-issue":{"issue":"13","published-print":{"date-parts":[[2022,10]]}},"alternative-id":["3363"],"URL":"https:\/\/doi.org\/10.1007\/s10489-022-03363-0","relation":{},"ISSN":["0924-669X","1573-7497"],"issn-type":[{"value":"0924-669X","type":"print"},{"value":"1573-7497","type":"electronic"}],"subject":[],"published":{"date-parts":[[2022,3,16]]},"assertion":[{"value":"7 February 2022","order":1,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"16 March 2022","order":2,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}}]}}