{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,3,25]],"date-time":"2025-03-25T14:31:41Z","timestamp":1742913101163,"version":"3.40.3"},"publisher-location":"Singapore","reference-count":31,"publisher":"Springer Nature Singapore","isbn-type":[{"type":"print","value":"9789819981311"},{"type":"electronic","value":"9789819981328"}],"license":[{"start":{"date-parts":[[2023,11,26]],"date-time":"2023-11-26T00:00:00Z","timestamp":1700956800000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2023,11,26]],"date-time":"2023-11-26T00:00:00Z","timestamp":1700956800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2024]]},"DOI":"10.1007\/978-981-99-8132-8_29","type":"book-chapter","created":{"date-parts":[[2023,11,25]],"date-time":"2023-11-25T10:02:23Z","timestamp":1700906543000},"page":"378-395","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":0,"title":["Bias Reduced Methods to\u00a0Q-learning"],"prefix":"10.1007","author":[{"given":"Patig\u00fcl","family":"Abliz","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Shi","family":"Ying","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2023,11,26]]},"reference":[{"key":"29_CR1","doi-asserted-by":"crossref","unstructured":"Szepesvt\u2019ari, C.: Reinforcement Learning Algorithms for MDPs. Wiley Encyclopedia of Operations Research and Management Science (2011)","DOI":"10.1002\/9780470400531.eorms0714"},{"key":"29_CR2","unstructured":"Sutton, R.S., Barto, A.G.: Reinforcement learning: an introduction (2018)"},{"issue":"3\u20134","key":"29_CR3","doi-asserted-by":"publisher","first-page":"279","DOI":"10.1007\/BF00992698","volume":"8","author":"CJ Watkins","year":"1992","unstructured":"Watkins, C.J., Dayan, P.: Q-learning. Mach. Learn. 8(3\u20134), 279\u2013292 (1992)","journal-title":"Mach. Learn."},{"issue":"3","key":"29_CR4","doi-asserted-by":"publisher","first-page":"185","DOI":"10.1007\/BF00993306","volume":"16","author":"JN Tsitsiklis","year":"1994","unstructured":"Tsitsiklis, J.N.: Asynchronous stochastic approximation and Q-learning. Mach. Learn. 16(3), 185\u2013202 (1994)","journal-title":"Mach. Learn."},{"issue":"6","key":"29_CR5","doi-asserted-by":"publisher","first-page":"1185","DOI":"10.1162\/neco.1994.6.6.1185","volume":"6","author":"T Jaakkola","year":"2014","unstructured":"Jaakkola, T., Jordan, M.I., Singh, S.P.: On the convergence of stochastic iterative dynamic programming algorithms. Neural Comput. 6(6), 1185\u20131201 (2014)","journal-title":"Neural Comput."},{"key":"29_CR6","doi-asserted-by":"publisher","unstructured":"Zhang, Y., Qu, G., Xu, P., Lin, Y., Chen, Z., Wierman, A.: Global convergence of localized policy iteration in networked multi-agent reinforcement learning. Proc. ACM Meas. Anal. Comput. Syst. 7(1), 13:1\u201313:51 (2023). https:\/\/doi.org\/10.1145\/3579443","DOI":"10.1145\/3579443"},{"key":"29_CR7","doi-asserted-by":"publisher","DOI":"10.1016\/j.knosys.2022.108489","volume":"243","author":"H Jahanshahi","year":"2022","unstructured":"Jahanshahi, H., et al.: A deep reinforcement learning approach for the meal delivery problem. Knowl.-Based Syst. 243, 108489 (2022)","journal-title":"Knowl.-Based Syst."},{"key":"29_CR8","doi-asserted-by":"publisher","unstructured":"de Moraes, L.D., et al.: Double deep reinforcement learning techniques for low dimensional sensing mapless navigation of terrestrial mobile robots 2023, CoRR abs\/2301.11173. arXiv:2301.11173. https:\/\/doi.org\/10.48550\/arXiv.2301.11173","DOI":"10.48550\/arXiv.2301.11173"},{"key":"29_CR9","doi-asserted-by":"publisher","first-page":"72628","DOI":"10.1109\/ACCESS.2022.3189021","volume":"10","author":"B Brandao","year":"2022","unstructured":"Brandao, B., de Lima, T.W., Soares, A., Melo, L.C., Mt\u2019aximo, M.R.O.A.: Multiagent reinforcement learning for strategic decision making and control in robotic soccer through self-play. IEEE Access 10, 72628\u201372642 (2022). https:\/\/doi.org\/10.1109\/ACCESS.2022.3189021","journal-title":"IEEE Access"},{"key":"29_CR10","doi-asserted-by":"publisher","first-page":"144","DOI":"10.1109\/ACCESS.2022.3232853","volume":"11","author":"N Sutisna","year":"2023","unstructured":"Sutisna, N., Ilmy, A.M.R., Syafalni, I., Mulyawan, R., Adiono, T.: FARANE-Q: fast parallel and pipeline Q-learning accelerator for configurable reinforcement learning SOC. IEEE Access 11, 144\u2013161 (2023). https:\/\/doi.org\/10.1109\/ACCESS.2022.3232853","journal-title":"IEEE Access"},{"key":"29_CR11","unstructured":"Thrun, S., Schwartz, A.: Issues in using function approximation for reinforcement learning. In: Proceedings of the Fourth Connectionist Models Summer School, Hillsdale, NJ, pp. 255\u2013263 (1993)"},{"issue":"3","key":"29_CR12","doi-asserted-by":"publisher","first-page":"311","DOI":"10.1287\/mnsc.1050.0451","volume":"52","author":"JE Smith","year":"2006","unstructured":"Smith, J.E., Winkler, R.L.: The optimizer\u2019s curse: skepticism and post-decision surprise in decision analysis. Manage. Sci. 52(3), 311\u2013322 (2006)","journal-title":"Manage. Sci."},{"key":"29_CR13","first-page":"2613","volume":"23","author":"H Hasselt","year":"2010","unstructured":"Hasselt, H.: Double Q-learning. Adv. Neural. Inf. Process. Syst. 23, 2613\u20132621 (2010)","journal-title":"Adv. Neural. Inf. Process. Syst."},{"key":"29_CR14","unstructured":"Devraj, A.M., Meyn, S.P.: Zap Q-learning. In: Guyon, I., et al. (eds.) Advances in Neural Information Processing Systems 30: Annual Conference on Neural Information Processing Systems, 4\u20139 December 2017, Long Beach, CA, USA, pp. 2235\u20132244 (2017). https:\/\/proceedings.neurips.cc\/paper\/2017\/hash\/4671aeaf49c792689533b00664a5c3ef-Abstract.html"},{"key":"29_CR15","unstructured":"Cini, A., D\u2019Eramo, C., Peters, J., Alippi, C.: Deep reinforcement learning with weighted Q-learning, CoRR abs\/2003.09280 (2020). arXiv:2003.09280"},{"key":"29_CR16","doi-asserted-by":"publisher","unstructured":"Lale, S., Shi, Y., Qu, G., Azizzadenesheli, K., Wierman, A., Anandkumar, A.: KCRL: krasovskii-constrained reinforcement learning with guaranteed stability in nonlinear dynamical systems, CoRR abs\/2206.01704 (2022). arXiv:2206.01704. https:\/\/doi.org\/10.48550\/arXiv.2206.01704","DOI":"10.48550\/arXiv.2206.01704"},{"key":"29_CR17","unstructured":"Fujimoto, S., Hoof, H., Meger, D.: Addressing function approximation error in actor-critic methods. In: International Conference on Machine Learning, PMLR, pp. 1587\u20131596 (2018)"},{"key":"29_CR18","unstructured":"Lan, Q., Pan, Y., Fyshe, A., White, M.: Maxmin Q-learning: controlling the estimation bias of Q-learning. In: 8th International Conference on Learning Representations, ICLR 2020, Addis Ababa, Ethiopia, 26\u201330 April 2020. OpenReview.net (2020). https:\/\/openreview.net\/forum?id=Bkg0u3Etwr"},{"key":"29_CR19","doi-asserted-by":"crossref","unstructured":"Wei, W., Zhang, Y., Liang, J., Li, L., Li, Y.: Controlling underestimation bias in reinforcement learning via quasi-median operation. In: Proceedings of the 36th Conference on Innovative Applications of Artificial Intelligence, AAAI 2022, Virtual, 2\u20139 February 2002 (2002)","DOI":"10.1609\/aaai.v36i8.20840"},{"key":"29_CR20","unstructured":"Chen, X., Wang, C., Zhou, Z., Ross, K.W.: Randomized ensembled double Q-learning: learning fast without a model. In: 9th International Conference on Learning Representations, ICLR 2021, Virtual Event, Austria, 3\u20137 May 2021. OpenReview.net (2021). https:\/\/openreview.net\/forum?id=AY8zfZm0tDd"},{"key":"29_CR21","unstructured":"Anschel, O., Baram, N., Shimkin, N.: Averaged-DQN: variance reduction and stabilization for deep reinforcement learning. In: International Conference on Machine Learning, pp. 176\u2013185. PMLR (2017)"},{"key":"29_CR22","first-page":"24778","volume":"34","author":"H Wang","year":"2021","unstructured":"Wang, H., Lin, S., Zhang, J.: Adaptive ensemble Q-learning: minimizing estimation bias via error feedback. Adv. Neural. Inf. Process. Syst. 34, 24778\u201324790 (2021)","journal-title":"Adv. Neural. Inf. Process. Syst."},{"key":"29_CR23","series-title":"LNCS","doi-asserted-by":"publisher","first-page":"343","DOI":"10.1007\/978-3-031-19907-3_33","volume-title":"Artificial General Intelligence","author":"D Papadimitriou","year":"2023","unstructured":"Papadimitriou, D.: Monte Carlo bias correction in Q-learning. In: Goertzel, B., Ikl\u00e9, M., Potapov, A., Ponomaryov, D. (eds.) AGI 2022. LNCS, vol. 13539, pp. 343\u2013352. Springer, Cham (2023). https:\/\/doi.org\/10.1007\/978-3-031-19907-3_33"},{"key":"29_CR24","doi-asserted-by":"crossref","unstructured":"Tan, T., Hong, X., Feng, L.: Q-learning with heterogeneous update strategy. Available at SSRN 4341963 (2023)","DOI":"10.2139\/ssrn.4341963"},{"key":"29_CR25","unstructured":"Van Hasselt, H.: Estimating the maximum expected value: an analysis of (nested) cross validation and the maximum sample average, arXiv preprint arXiv:1302.7175 (2013)"},{"issue":"Sep","key":"29_CR26","first-page":"1089","volume":"5","author":"Y Bengio","year":"2004","unstructured":"Bengio, Y., Grandvalet, Y.: No unbiased estimator of the variance of k-fold cross-validation. J. Mach. Learn. Res. 5(Sep), 1089\u20131105 (2004)","journal-title":"J. Mach. Learn. Res."},{"key":"29_CR27","doi-asserted-by":"crossref","unstructured":"Berrar, D.: Cross-validation (2019)","DOI":"10.1016\/B978-0-12-809633-8.20349-X"},{"key":"29_CR28","unstructured":"Brockman, G., et al.: Openai gym, arXiv preprint arXiv:1606.01540 (2016)"},{"key":"29_CR29","unstructured":"Tasfi, N.: Pygame learning environment, GitHub repository (2016)"},{"key":"29_CR30","unstructured":"Lan, Q.: A pytorch reinforcement learning framework for exploring new ideas (2019). https:\/\/github.com\/qlan3\/Explorer"},{"key":"29_CR31","series-title":"Lecture Notes in Computer Science (Lecture Notes in Artificial Intelligence)","doi-asserted-by":"publisher","first-page":"26","DOI":"10.1007\/3-540-44914-0_2","volume-title":"Abstraction, Reformulation, and Approximation","author":"TG Dietterich","year":"2000","unstructured":"Dietterich, T.G.: An overview of MAXQ hierarchical reinforcement learning. In: Choueiry, B.Y., Walsh, T. (eds.) SARA 2000. LNCS (LNAI), vol. 1864, pp. 26\u201344. Springer, Heidelberg (2000). https:\/\/doi.org\/10.1007\/3-540-44914-0_2"}],"container-title":["Communications in Computer and Information Science","Neural Information Processing"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-981-99-8132-8_29","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,11,3]],"date-time":"2024-11-03T09:34:21Z","timestamp":1730626461000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-981-99-8132-8_29"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023,11,26]]},"ISBN":["9789819981311","9789819981328"],"references-count":31,"URL":"https:\/\/doi.org\/10.1007\/978-981-99-8132-8_29","relation":{},"ISSN":["1865-0929","1865-0937"],"issn-type":[{"type":"print","value":"1865-0929"},{"type":"electronic","value":"1865-0937"}],"subject":[],"published":{"date-parts":[[2023,11,26]]},"assertion":[{"value":"26 November 2023","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"ICONIP","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"International Conference on Neural Information Processing","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Changsha","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"China","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2023","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"20 November 2023","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"23 November 2023","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"30","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"iconip2023","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"http:\/\/iconip2023.org\/","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Single-blind","order":1,"name":"type","label":"Type","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"EasyChair","order":2,"name":"conference_management_system","label":"Conference Management System","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"1274","order":3,"name":"number_of_submissions_sent_for_review","label":"Number of Submissions Sent for Review","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"650","order":4,"name":"number_of_full_papers_accepted","label":"Number of Full Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"0","order":5,"name":"number_of_short_papers_accepted","label":"Number of Short Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"51% - The value is computed by the equation \"Number of Full Papers Accepted \/ Number of Submissions Sent for Review * 100\" and then rounded to a whole number.","order":6,"name":"acceptance_rate_of_full_papers","label":"Acceptance Rate of Full Papers","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"4.14","order":7,"name":"average_number_of_reviews_per_paper","label":"Average Number of Reviews per Paper","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"2.46","order":8,"name":"average_number_of_papers_per_reviewer","label":"Average Number of Papers per Reviewer","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"Yes","order":9,"name":"external_reviewers_involved","label":"External Reviewers Involved","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}}]}}