{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,3,6]],"date-time":"2026-03-06T18:54:43Z","timestamp":1772823283556,"version":"3.50.1"},"publisher-location":"Cham","reference-count":41,"publisher":"Springer International Publishing","isbn-type":[{"value":"9783030914141","type":"print"},{"value":"9783030914158","type":"electronic"}],"license":[{"start":{"date-parts":[[2021,1,1]],"date-time":"2021-01-01T00:00:00Z","timestamp":1609459200000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"},{"start":{"date-parts":[[2021,1,1]],"date-time":"2021-01-01T00:00:00Z","timestamp":1609459200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2021]]},"DOI":"10.1007\/978-3-030-91415-8_10","type":"book-chapter","created":{"date-parts":[[2021,11,17]],"date-time":"2021-11-17T18:04:03Z","timestamp":1637172243000},"page":"105-117","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":5,"title":["Combining Model-Based and Model-Free Reinforcement Learning Policies for More Efficient Sepsis Treatment"],"prefix":"10.1007","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-3255-1467","authenticated-orcid":false,"given":"Xiangyu","family":"Liu","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-4371-3663","authenticated-orcid":false,"given":"Chao","family":"Yu","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Qikai","family":"Huang","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Luhao","family":"Wang","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Jianfeng","family":"Wu","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Xiangdong","family":"Guan","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2021,11,18]]},"reference":[{"issue":"1","key":"10_CR1","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1186\/s13613-016-0231-8","volume":"7","author":"L Byrne","year":"2017","unstructured":"Byrne, L., Van Haren, F.: Fluid resuscitation in human sepsis: time to rewrite history? Ann. Intensive Care 7(1), 1\u20138 (2017). https:\/\/doi.org\/10.1186\/s13613-016-0231-8","journal-title":"Ann. Intensive Care"},{"issue":"4","key":"10_CR2","doi-asserted-by":"publisher","first-page":"367","DOI":"10.1016\/S0167-9473(01)00065-2","volume":"38","author":"J Friedman","year":"2002","unstructured":"Friedman, J.: Stochastic gradient boosting. Comput. Stat. Data Anal. 38(4), 367\u2013378 (2002)","journal-title":"Comput. Stat. Data Anal."},{"key":"10_CR3","unstructured":"Futoma, J., et al.: Learning to treat sepsis with multi-output Gaussian process deep recurrent Q-networks (2018)"},{"key":"10_CR4","doi-asserted-by":"publisher","first-page":"i1585","DOI":"10.1136\/bmj.i1585","volume":"353","author":"J Gotts","year":"2016","unstructured":"Gotts, J., Matthay, M.: Sepsis: pathophysiology and clinical management. BMJ 353, i1585 (2016). https:\/\/doi.org\/10.1136\/bmj.i1585","journal-title":"BMJ"},{"key":"10_CR5","doi-asserted-by":"crossref","unstructured":"Hanna, J., Stone, P., Niekum, S.: Bootstrapping with models: confidence intervals for off-policy evaluation. In: Proceedings of the AAAI Conference on Artificial Intelligence, pp. 538\u2013546 (2017)","DOI":"10.1609\/aaai.v31i1.11123"},{"issue":"4","key":"10_CR6","doi-asserted-by":"publisher","first-page":"985","DOI":"10.1093\/biomet\/asm076","volume":"94","author":"M Henmi","year":"2007","unstructured":"Henmi, M., Yoshida, R., Eguchi, S.: Importance sampling via the estimated sampler. Biometrika 94(4), 985\u2013991 (2007)","journal-title":"Biometrika"},{"issue":"1","key":"10_CR7","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1038\/sdata.2016.35","volume":"3","author":"A Johnson","year":"2016","unstructured":"Johnson, A., Pollard, T., Shen, L., Li Wei, L., Feng, M., Ghassemi, M., et al.: MIMIC-III, a freely accessible critical care database. Sci. Data 3(1), 1\u20139 (2016)","journal-title":"Sci. Data"},{"issue":"11","key":"10_CR8","doi-asserted-by":"publisher","first-page":"1716","DOI":"10.1038\/s41591-018-0213-5","volume":"24","author":"M Komorowski","year":"2018","unstructured":"Komorowski, M., Celi, L.A., Badawi, O., Gordon, A., Faisal, A.: The artificial intelligence clinician learns optimal treatment strategies for sepsis in intensive care. Nat. Med. 24(11), 1716\u20131720 (2018)","journal-title":"Nat. Med."},{"key":"10_CR9","unstructured":"Komorowski, M., Gordon, A., Celi, L., Faisal, A.: A Markov decision process to suggest optimal treatment of severe infections in intensive care. In: Neural Information Processing Systems Workshop on Machine Learning for Health (2016)"},{"key":"10_CR10","unstructured":"Li, L., Komorowski, M., Faisal, A.: The actor search tree critic (ASTC) for off-policy POMDP learning in medical decision making. arXiv preprint arXiv:1805.11548 (2018)"},{"issue":"7553","key":"10_CR11","doi-asserted-by":"publisher","first-page":"445","DOI":"10.1038\/nature14540","volume":"521","author":"M Littman","year":"2015","unstructured":"Littman, M.: Reinforcement learning improves behaviour from evaluative feedback. Nature 521(7553), 445\u2013451 (2015)","journal-title":"Nature"},{"issue":"5","key":"10_CR12","doi-asserted-by":"publisher","first-page":"561","DOI":"10.1111\/aas.12479","volume":"59","author":"P Marik","year":"2015","unstructured":"Marik, P.: The demise of early goal-directed therapy for severe sepsis and septic shock. Acta Anaesthesiol. Scand. 59(5), 561\u2013567 (2015)","journal-title":"Acta Anaesthesiol. Scand."},{"issue":"3","key":"10_CR13","doi-asserted-by":"publisher","first-page":"339","DOI":"10.1093\/bja\/aev349","volume":"116","author":"P Marik","year":"2016","unstructured":"Marik, P., Bellomo, R.: A rational approach to fluid therapy in sepsis. BJA Br. J. Anaesthesia 116(3), 339\u2013349 (2016)","journal-title":"BJA Br. J. Anaesthesia"},{"issue":"7540","key":"10_CR14","doi-asserted-by":"publisher","first-page":"529","DOI":"10.1038\/nature14236","volume":"518","author":"V Mnih","year":"2015","unstructured":"Mnih, V., Kavukcuoglu, K., Silver, D., Rusu, A., Veness, J., Bellemare, M., et al.: Human-level control through deep reinforcement learning. Nature 518(7540), 529\u2013533 (2015)","journal-title":"Nature"},{"key":"10_CR15","doi-asserted-by":"publisher","unstructured":"Nahler, G.: Last value carried forward (LVCF). In: Dictionary of Pharmaceutical Medicine, pp. 105\u2013105. Springer, Vienna (2009). https:\/\/doi.org\/10.1007\/978-3-211-89836-9_773","DOI":"10.1007\/978-3-211-89836-9_773"},{"key":"10_CR16","doi-asserted-by":"crossref","unstructured":"Pal, C.V., Leon, F.: Brief survey of model-based reinforcement learning techniques. In: 2020 24th International Conference on System Theory, Control and Computing, pp. 92\u201397. IEEE (2020)","DOI":"10.1109\/ICSTCC50638.2020.9259716"},{"key":"10_CR17","unstructured":"Peng, X., Ding, Y., Wihl, D., Gottesman, O., Komorowski, M., Lehman, L.W., Ross, A., et al.: Improving sepsis treatment strategies by combining deep and kernel-based reinforcement learning. In: AMIA Annual Symposium Proceedings, vol. 2018, p. 887 (2018)"},{"key":"10_CR18","unstructured":"Pong, V., Gu, S., Dalal, M., Levine, S.: Temporal difference models: model-free deep RL for model-based control. arXiv preprint arXiv:1802.09081 (2018)"},{"key":"10_CR19","unstructured":"Raghu, A., Komorowski, M., Ahmed, I., Celi, L.A., Szolovits, P., Ghassemi, M.: Deep reinforcement learning for sepsis treatment. arXiv preprint arXiv:1711.09602 (2017)"},{"key":"10_CR20","unstructured":"Raghu, A., Komorowski, M., Celi, L.A., Szolovits, P., Ghassemi, M.: Continuous state-space models for optimal sepsis treatment: a deep reinforcement learning approach. In: Machine Learning for Healthcare Conference, pp. 147\u2013163 (2017)"},{"key":"10_CR21","unstructured":"Raghu, A., Komorowski, M., Singh, S.: Model-based reinforcement learning for sepsis treatment. arXiv preprint arXiv:1811.09602 (2018)"},{"issue":"3","key":"10_CR22","doi-asserted-by":"publisher","first-page":"304","DOI":"10.1007\/s00134-017-4683-6","volume":"43","author":"A Rhodes","year":"2017","unstructured":"Rhodes, A., Evans, L., Alhazzani, W., Levy, M., Antonelli, M., Ferrer, R., et al.: Surviving sepsis campaign: international guidelines for management of sepsis and septic shock: 2016. Intensive Care Med. 43(3), 304\u2013377 (2017)","journal-title":"Intensive Care Med."},{"key":"10_CR23","doi-asserted-by":"publisher","first-page":"102003","DOI":"10.1016\/j.artmed.2020.102003","volume":"112","author":"L Roggeveen","year":"2021","unstructured":"Roggeveen, L., El Hassouni, A., Ahrendt, J., Guo, T., Fleuren, L., Thoral, P., et al.: Transatlantic transferability of a new reinforcement learning model for optimizing haemodynamic treatment for critically ill patients with sepsis. Artif. Intell. Med. 112, 102003 (2021)","journal-title":"Artif. Intell. Med."},{"key":"10_CR24","unstructured":"Schaul, T., Quan, J., Antonoglou, I., Silver, D.: Prioritized experience replay. arXiv preprint arXiv:1511.05952 (2015)"},{"key":"10_CR25","unstructured":"Schulman, J., Wolski, F., Dhariwal, P., Radford, A., Klimov, O.: Proximal policy optimization algorithms. arXiv preprint arXiv:1707.06347 (2017)"},{"issue":"8","key":"10_CR26","doi-asserted-by":"publisher","first-page":"801","DOI":"10.1001\/jama.2016.0287","volume":"315","author":"M Singer","year":"2016","unstructured":"Singer, M., Deutschman, C., Seymour, C.W., Shankar Hari, M., Annane, D., Bauer, M., et al.: The third international consensus definitions for sepsis and septic shock (Sepsis-3). JAMA 315(8), 801\u2013810 (2016)","journal-title":"JAMA"},{"issue":"1","key":"10_CR27","first-page":"9","volume":"3","author":"R Sutton","year":"1988","unstructured":"Sutton, R.: Learning to predict by the methods of temporal differences. Mach. Learn. 3(1), 9\u201344 (1988)","journal-title":"Mach. Learn."},{"key":"10_CR28","unstructured":"Sutton, R.: Generalization in reinforcement learning: successful examples using sparse coarse coding. In: Advances in Neural Information Processing Systems, pp. 1038\u20131044 (1996)"},{"key":"10_CR29","volume-title":"Reinforcement Learning: An Introduction","author":"R Sutton","year":"2018","unstructured":"Sutton, R., Barto, A.: Reinforcement Learning: An Introduction. MIT Press, Cambridge (2018)"},{"key":"10_CR30","unstructured":"Sutton, R., McAllester, D., Singh, S., Mansour, Y.: Policy gradient methods for reinforcement learning with function approximation. In: Advances in Neural Information Processing Systems, pp. 1057\u20131063 (2000)"},{"key":"10_CR31","doi-asserted-by":"crossref","unstructured":"Thomas, P., Theocharous, G., Ghavamzadeh, M.: High-confidence off-policy evaluation. In: Proceedings of the AAAI Conference on Artificial Intelligence, vol. 29, pp. 3000\u20133006 (2015)","DOI":"10.1609\/aaai.v29i1.9541"},{"key":"10_CR32","unstructured":"Thomas, P., Theocharous, G., Ghavamzadeh, M.: High confidence policy improvement. In: International Conference on Machine Learning, pp. 2380\u20132388 (2015)"},{"key":"10_CR33","unstructured":"Utomo, C.P., Li, X., Chen, W.: Treatment recommendation in critical care: a scalable and interpretable approach in partially observable health states. In: 39th International Conference on Information Systems, pp. 1\u20139 (2018)"},{"key":"10_CR34","doi-asserted-by":"crossref","unstructured":"Van Hasselt, H., Guez, A., Silver, D.: Deep reinforcement learning with double q-learning. In: Proceedings of the AAAI Conference on Artificial Intelligence, vol. 30, pp. 2094\u20132100 (2016)","DOI":"10.1609\/aaai.v30i1.10295"},{"issue":"10","key":"10_CR35","doi-asserted-by":"publisher","first-page":"2158","DOI":"10.1097\/CCM.0000000000000520","volume":"42","author":"J Waechter","year":"2014","unstructured":"Waechter, J., Kumar, A., Lapinsky, S., Marshall, J., Dodek, P., Arabi, Y., et al.: Interaction between fluids and vasoactive agents on mortality in septic shock: a multicenter, observational study. Crit. Care Med. 42(10), 2158\u20132168 (2014)","journal-title":"Crit. Care Med."},{"key":"10_CR36","unstructured":"Wang, T., et al.: Benchmarking model-based reinforcement learning. arXiv preprint arXiv:1907.02057 (2019)"},{"key":"10_CR37","unstructured":"Wang, Z., Schaul, T., Hessel, M., Hasselt, H., Lanctot, M., Freitas, N.: Dueling network architectures for deep reinforcement learning. In: International Conference on Machine Learning, pp. 1995\u20132003 (2016)"},{"key":"10_CR38","unstructured":"Watkins, C.J.C.H.: Learning from delayed rewards. King\u2019s College, Cambridge United Kingdom (1989)"},{"key":"10_CR39","unstructured":"Yu, C., Liu, J., Nemati, S.: Reinforcement learning in healthcare: a survey. arXiv preprint arXiv:1908.08796 (2019)"},{"key":"10_CR40","doi-asserted-by":"crossref","unstructured":"Yu, C., Ren, G., Liu, J.: Deep inverse reinforcement learning for sepsis treatment. In: 2019 IEEE International Conference on Healthcare Informatics, pp. 1\u20133. IEEE (2019)","DOI":"10.1109\/ICHI.2019.8904645"},{"key":"10_CR41","unstructured":"Zaheer, M., Reddi, S., Sachan, D., Kale, S., Kumar, S.: Adaptive methods for nonconvex optimization. In: Advances in Neural Information Processing Systems, vol. 31, pp. 9815\u20139825 (2018)"}],"container-title":["Lecture Notes in Computer Science","Bioinformatics Research and Applications"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-030-91415-8_10","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2023,1,15]],"date-time":"2023-01-15T18:14:38Z","timestamp":1673806478000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-3-030-91415-8_10"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2021]]},"ISBN":["9783030914141","9783030914158"],"references-count":41,"URL":"https:\/\/doi.org\/10.1007\/978-3-030-91415-8_10","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"value":"0302-9743","type":"print"},{"value":"1611-3349","type":"electronic"}],"subject":[],"published":{"date-parts":[[2021]]},"assertion":[{"value":"18 November 2021","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"ISBRA","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"International Symposium on Bioinformatics Research and Applications","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Shenzhen","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"China","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2021","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"26 November 2021","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"28 November 2021","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"17","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"isbra2021","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"https:\/\/alan.cs.gsu.edu\/isbra21\/?q=node\/1","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Single-blind","order":1,"name":"type","label":"Type","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"EasyChair","order":2,"name":"conference_management_system","label":"Conference Management System","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"135","order":3,"name":"number_of_submissions_sent_for_review","label":"Number of Submissions Sent for Review","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"51","order":4,"name":"number_of_full_papers_accepted","label":"Number of Full Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"0","order":5,"name":"number_of_short_papers_accepted","label":"Number of Short Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"38% - The value is computed by the equation \"Number of Full Papers Accepted \/ Number of Submissions Sent for Review * 100\" and then rounded to a whole number.","order":6,"name":"acceptance_rate_of_full_papers","label":"Acceptance Rate of Full Papers","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"2.97","order":7,"name":"average_number_of_reviews_per_paper","label":"Average Number of Reviews per Paper","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"2.95","order":8,"name":"average_number_of_papers_per_reviewer","label":"Average Number of Papers per Reviewer","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"Yes","order":9,"name":"external_reviewers_involved","label":"External Reviewers Involved","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}}]}}