{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,3,27]],"date-time":"2026-03-27T02:05:03Z","timestamp":1774577103035,"version":"3.50.1"},"publisher-location":"Cham","reference-count":35,"publisher":"Springer International Publishing","isbn-type":[{"value":"9783030494346","type":"print"},{"value":"9783030494353","type":"electronic"}],"license":[{"start":{"date-parts":[[2020,1,1]],"date-time":"2020-01-01T00:00:00Z","timestamp":1577836800000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2020,1,1]],"date-time":"2020-01-01T00:00:00Z","timestamp":1577836800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2020]]},"DOI":"10.1007\/978-3-030-49435-3_11","type":"book-chapter","created":{"date-parts":[[2020,6,2]],"date-time":"2020-06-02T20:11:27Z","timestamp":1591128687000},"page":"169-184","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":42,"title":["Online Reinforcement Learning for Self-adaptive Information Systems"],"prefix":"10.1007","author":[{"ORCID":"https:\/\/orcid.org\/0000-0001-5662-1306","authenticated-orcid":false,"given":"Alexander","family":"Palm","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-4808-8297","authenticated-orcid":false,"given":"Andreas","family":"Metzger","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-2199-5257","authenticated-orcid":false,"given":"Klaus","family":"Pohl","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2020,6,3]]},"reference":[{"key":"11_CR1","doi-asserted-by":"crossref","unstructured":"Amoui, M., Salehie, M., Mirarab, S., Tahvildari, L.: Adaptive action selection in autonomic software using reinforcement learning. In: 4th International Conference on Autonomic and Autonomous Systems (ICAS 2008), pp. 175\u2013181. IEEE (2008)","DOI":"10.1109\/ICAS.2008.35"},{"key":"11_CR2","doi-asserted-by":"crossref","unstructured":"Arabnejad, H., Pahl, C., Jamshidi, P., Estrada, G.: A comparison of reinforcement learning techniques for fuzzy cloud auto-scaling. In: 17th International Symposium on Cluster, Cloud and Grid Computing (CCGRID 2017), pp. 64\u201373. ACM (2017)","DOI":"10.1109\/CCGRID.2017.15"},{"key":"11_CR3","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"421","DOI":"10.1007\/978-3-642-25535-9_28","volume-title":"Service-Oriented Computing","author":"R Aschoff","year":"2011","unstructured":"Aschoff, R., Zisman, A.: QoS-driven proactive adaptation of service composition. In: Kappel, G., Maamar, Z., Motahari-Nezhad, H.R. (eds.) ICSOC 2011. LNCS, vol. 7084, pp. 421\u2013435. Springer, Heidelberg (2011). https:\/\/doi.org\/10.1007\/978-3-642-25535-9_28"},{"issue":"12","key":"11_CR4","doi-asserted-by":"publisher","first-page":"1656","DOI":"10.1002\/cpe.2864","volume":"25","author":"E Barrett","year":"2013","unstructured":"Barrett, E., Howley, E., Duggan, J.: Applying reinforcement learning towards automating resource allocation and application scalability in the cloud. Concurrency Comput. Pract. Exp. 25(12), 1656\u20131674 (2013)","journal-title":"Concurrency Comput. Pract. Exp."},{"issue":"4","key":"11_CR5","doi-asserted-by":"publisher","first-page":"681","DOI":"10.1109\/TPDS.2012.174","volume":"24","author":"X Bu","year":"2013","unstructured":"Bu, X., Rao, J., Xu, C.: Coordinated self-configuration of virtual machines and appliances using a model-free learning approach. IEEE Trans. Parallel Distrib. Syst. 24(4), 681\u2013690 (2013)","journal-title":"IEEE Trans. Parallel Distrib. Syst."},{"key":"11_CR6","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"53","DOI":"10.1007\/978-3-319-44482-6_4","volume-title":"Service-Oriented and Cloud Computing","author":"M Caporuscio","year":"2016","unstructured":"Caporuscio, M., D\u2019Angelo, M., Grassi, V., Mirandola, R.: Reinforcement learning techniques for decentralized self-adaptive service assembly. In: Aiello, M., Johnsen, E.B., Dustdar, S., Georgievski, I. (eds.) ESOCC 2016. LNCS, vol. 9846, pp. 53\u201368. Springer, Cham (2016). https:\/\/doi.org\/10.1007\/978-3-319-44482-6_4"},{"issue":"5","key":"11_CR7","doi-asserted-by":"publisher","first-page":"453","DOI":"10.1109\/TSE.2016.2608826","volume":"43","author":"T Chen","year":"2017","unstructured":"Chen, T., Bahsoon, R.: Self-adaptive and online QoS modeling for cloud-based software services. IEEE Trans. Software Eng. 43(5), 453\u2013475 (2017)","journal-title":"IEEE Trans. Software Eng."},{"key":"11_CR8","doi-asserted-by":"crossref","unstructured":"D\u2019Ippolito, N., Braberman, V.A., Kramer, J., Magee, J., Sykes, D., Uchitel, S.: Hope for the best, prepare for the worst: multi-tier control for adaptive systems. In: 36th International Conference on Software Engineering (ICSE 2014), pp. 688\u2013699. ACM (2014)","DOI":"10.1145\/2568225.2568264"},{"key":"11_CR9","unstructured":"Dulac-Arnold, G., Evans, R., Sunehag, P., Coppin, B.: Reinforcement learning in large discrete action spaces. CoRR abs\/1512.07679 (2015)"},{"key":"11_CR10","unstructured":"Dutreilh, X., Kirgizov, S., Melekhova, O., Malenfant, J., Rivierre, N., Truck, I.: Using reinforcement learning for autonomic resource allocation in clouds: towards a fully automated workflow. In: 7th International Conference on Autonomic and Autonomous Systems (ICAS 2011), pp. 67\u201374 (2011)"},{"issue":"3","key":"11_CR11","doi-asserted-by":"publisher","first-page":"16:1","DOI":"10.1145\/3092691","volume":"12","author":"RVR Filho","year":"2017","unstructured":"Filho, R.V.R., Porter, B.: Defining emergent software using continuous self-assembly, perception, and learning. TAAS 12(3), 16:1\u201316:25 (2017)","journal-title":"TAAS"},{"key":"11_CR12","doi-asserted-by":"publisher","first-page":"42","DOI":"10.1016\/j.jss.2016.01.026","volume":"115","author":"JM Franco","year":"2016","unstructured":"Franco, J.M., Correia, F., Barbosa, R., Rela, M.Z., Schmerl, B.R., Garlan, D.: Improving self-adaptation planning through software architecture-based stochastic modeling. J. Syst. Softw. 115, 42\u201360 (2016)","journal-title":"J. Syst. Softw."},{"issue":"1","key":"11_CR13","doi-asserted-by":"publisher","first-page":"127","DOI":"10.1016\/j.datak.2010.09.002","volume":"70","author":"Z Huang","year":"2011","unstructured":"Huang, Z., van der Aalst, W.M.P., Lu, X., Duan, H.: Reinforcement learning based resource allocation in business process management. Data Knowl. Eng. 70(1), 127\u2013145 (2011)","journal-title":"Data Knowl. Eng."},{"issue":"3","key":"11_CR14","doi-asserted-by":"publisher","first-page":"15:1","DOI":"10.1145\/2724719","volume":"10","author":"DG de la Iglesia","year":"2015","unstructured":"de la Iglesia, D.G., Weyns, D.: MAPE-K formal templates to rigorously design behaviors for self-adaptive systems. TAAS 10(3), 15:1\u201315:31 (2015)","journal-title":"TAAS"},{"key":"11_CR15","doi-asserted-by":"crossref","unstructured":"Jamshidi, P., Camara, J., Schmerl, B., K\u00e4stner, C., Garlan, D.: Machine learning meets quantitative planning: Enabling self-adaptation in autonomous robots. In: 14th Symposium on Software Engineering for Adaptive and Self-Managing Systems (SEAMS 2019). ACM (2019)","DOI":"10.1109\/SEAMS.2019.00015"},{"issue":"1","key":"11_CR16","doi-asserted-by":"publisher","first-page":"41","DOI":"10.1109\/MC.2003.1160055","volume":"36","author":"JO Kephart","year":"2003","unstructured":"Kephart, J.O., Chess, D.M.: The vision of autonomic computing. IEEE Comput. 36(1), 41\u201350 (2003)","journal-title":"IEEE Comput."},{"key":"11_CR17","doi-asserted-by":"crossref","unstructured":"Klein, C., Maggio, M., Arz\u00e9n, K.E., Hern\u00e1ndez-Rodriguez, F.: Brownout: building more robust cloud applications. In: 36th International Confernce on Software Engineering (ICSE 2014), pp. 700\u2013711. ACM (2014)","DOI":"10.1145\/2568225.2568227"},{"key":"11_CR18","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1007\/978-3-642-35813-5_1","volume-title":"Software Engineering for Self-Adaptive Systems II","author":"R de Lemos","year":"2013","unstructured":"de Lemos, R., et al.: Software Engineering for self-adaptive systems: a second research roadmap. In: de Lemos, R., Giese, H., M\u00fcller, H.A., Shaw, M. (eds.) Software Engineering for Self-Adaptive Systems II. LNCS, vol. 7475, pp. 1\u201332. Springer, Heidelberg (2013). https:\/\/doi.org\/10.1007\/978-3-642-35813-5_1"},{"issue":"4","key":"11_CR19","doi-asserted-by":"publisher","first-page":"559","DOI":"10.1007\/s10723-014-9314-7","volume":"12","author":"T Lorido-Botran","year":"2014","unstructured":"Lorido-Botran, T., Miguel-Alonso, J., Lozano, J.A.: A review of auto-scaling techniques for elastic applications in cloud environments. J. Grid Comput. 12(4), 559\u2013592 (2014)","journal-title":"J. Grid Comput."},{"issue":"1","key":"11_CR20","doi-asserted-by":"publisher","first-page":"169","DOI":"10.1109\/TPDS.2017.2744627","volume":"29","author":"Z\u00c1 Mann","year":"2018","unstructured":"Mann, Z.\u00c1.: Resource optimization across the cloud stack. IEEE Trans. Parallel Distrib. Syst. 29(1), 169\u2013182 (2018)","journal-title":"IEEE Trans. Parallel Distrib. Syst."},{"key":"11_CR21","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"547","DOI":"10.1007\/978-3-030-21290-2_34","volume-title":"Advanced Information Systems Engineering","author":"A Metzger","year":"2019","unstructured":"Metzger, A., Neubauer, A., Bohn, P., Pohl, K.: Proactive process adaptation using deep learning ensembles. In: Giorgini, P., Weber, B. (eds.) CAiSE 2019. LNCS, vol. 11483, pp. 547\u2013562. Springer, Cham (2019). https:\/\/doi.org\/10.1007\/978-3-030-21290-2_34"},{"key":"11_CR22","doi-asserted-by":"crossref","unstructured":"Moustafa, A., Zhang, M.: Learning efficient compositions for QoS-aware service provisioning. In: International Conference Conference on Web Services (ICWS 2014), pp. 185\u2013192. IEEE Computer Society (2014)","DOI":"10.1109\/ICWS.2014.37"},{"key":"11_CR23","unstructured":"Nachum, O., Norouzi, M., Xu, K., Schuurmans, D.: Bridging the gap between value and policy based reinforcement learning. In: Advances in Neural Information Processing Systems (NIPS 2017), vol. 12, pp. 2772\u20132782 (2017)"},{"key":"11_CR24","doi-asserted-by":"crossref","unstructured":"Ramirez, A.J., Jensen, A.C., Cheng, B.H.C.: A taxonomy of uncertainty for dynamically adaptive systems. In: 7th International Symposium on Software Engineering for Adaptive and Self-Managing Systems (SEAMS 2012), pp. 99\u2013108 (2012)","DOI":"10.1109\/SEAMS.2012.6224396"},{"issue":"2","key":"11_CR25","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1145\/1516533.1516538","volume":"4","author":"M Salehie","year":"2009","unstructured":"Salehie, M., Tahvildari, L.: Self-adaptive software: landscape and research challenges. TAAS 4(2), 1\u201342 (2009)","journal-title":"TAAS"},{"key":"11_CR26","unstructured":"Schulman, J., Wolski, F., Dhariwal, P., Radford, A., Klimov, O.: Proximal policy optimization algorithms. CoRR abs\/1707.06347 (2017)"},{"key":"11_CR27","series-title":"Lecture Notes in Business Information Processing","doi-asserted-by":"publisher","first-page":"203","DOI":"10.1007\/978-3-030-24854-3_13","volume-title":"Business Modeling and Software Design","author":"J Silvander","year":"2019","unstructured":"Silvander, J.: Business process optimization with reinforcement learning. In: Shishkov, B. (ed.) BMSD 2019. LNBIP, vol. 356, pp. 203\u2013212. Springer, Cham (2019). https:\/\/doi.org\/10.1007\/978-3-030-24854-3_13"},{"key":"11_CR28","volume-title":"Reinforcement Learning: An Introduction","author":"RS Sutton","year":"2018","unstructured":"Sutton, R.S., Barto, A.G.: Reinforcement Learning: An Introduction. MIT Press, Cambridge (2018)"},{"key":"11_CR29","unstructured":"Sutton, R.S., McAllester, D.A., Singh, S.P., Mansour, Y.: Policy gradient methods for reinforcement learning with function approximation. In: Advances in Neural Information Processing Systems 12 (NIPS 1999), pp. 1057\u20131063 (2000)"},{"key":"11_CR30","series-title":"Lecture Notes in Business Information Processing","doi-asserted-by":"publisher","first-page":"91","DOI":"10.1007\/978-3-319-98651-7_6","volume-title":"Business Process Management Forum","author":"I Teinemaa","year":"2018","unstructured":"Teinemaa, I., Tax, N., de Leoni, M., Dumas, M., Maggi, F.M.: Alarm-based prescriptive process monitoring. In: Weske, M., Montali, M., Weber, I., vom Brocke, J. (eds.) BPM 2018. LNBIP, vol. 329, pp. 91\u2013107. Springer, Cham (2018). https:\/\/doi.org\/10.1007\/978-3-319-98651-7_6"},{"issue":"3","key":"11_CR31","doi-asserted-by":"publisher","first-page":"287","DOI":"10.1007\/s10586-007-0035-6","volume":"10","author":"G Tesauro","year":"2007","unstructured":"Tesauro, G., Jong, N.K., Das, R., Bennani, M.N.: On the use of hybrid reinforcement learning for autonomic resource allocation. Cluster Comput. 10(3), 287\u2013299 (2007)","journal-title":"Cluster Comput."},{"issue":"2","key":"11_CR32","first-page":"8:1","volume":"12","author":"H Wang","year":"2017","unstructured":"Wang, H., et al.: Integrating reinforcement learning with multi-agent techniques for adaptive service composition. TAAS 12(2), 8:1\u20138:42 (2017)","journal-title":"TAAS"},{"key":"11_CR33","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"383","DOI":"10.1007\/978-3-319-69035-3_27","volume-title":"Service-Oriented Computing","author":"H Wang","year":"2017","unstructured":"Wang, H., Gu, M., Yu, Q., Fei, H., Li, J., Tao, Y.: Large-scale and adaptive service composition using deep reinforcement learning. In: Maximilien, M., Vallecillo, A., Wang, J., Oriol, M. (eds.) ICSOC 2017. LNCS, vol. 10601, pp. 383\u2013391. Springer, Cham (2017). https:\/\/doi.org\/10.1007\/978-3-319-69035-3_27"},{"issue":"2","key":"11_CR34","doi-asserted-by":"publisher","first-page":"95","DOI":"10.1016\/j.jpdc.2011.10.003","volume":"72","author":"C Xu","year":"2012","unstructured":"Xu, C., Rao, J., Bu, X.: URL: a unified reinforcement learning approach for autonomic cloud management. J. Parallel Distrib. Comput. 72(2), 95\u2013105 (2012)","journal-title":"J. Parallel Distrib. Comput."},{"key":"11_CR35","doi-asserted-by":"crossref","unstructured":"Zhao, T., Zhang, W., Zhao, H., Jin, Z.: A reinforcement learning-based framework for the generation and evolution of adaptation rules. In: International Conference on Autonomic Computing (ICAC 2017), pp. 103\u2013112. IEEE Computer Society (2017)","DOI":"10.1109\/ICAC.2017.47"}],"container-title":["Lecture Notes in Computer Science","Advanced Information Systems Engineering"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-030-49435-3_11","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,1]],"date-time":"2025-06-01T22:02:42Z","timestamp":1748815362000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-3-030-49435-3_11"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2020]]},"ISBN":["9783030494346","9783030494353"],"references-count":35,"URL":"https:\/\/doi.org\/10.1007\/978-3-030-49435-3_11","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"value":"0302-9743","type":"print"},{"value":"1611-3349","type":"electronic"}],"subject":[],"published":{"date-parts":[[2020]]},"assertion":[{"value":"3 June 2020","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"CAiSE","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"International Conference on Advanced Information Systems Engineering","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Grenoble","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"France","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2020","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"8 June 2020","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"12 June 2020","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"32","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"caise2020","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"http:\/\/caise20.imag.fr\/","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Single-blind","order":1,"name":"type","label":"Type","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"EasyChair","order":2,"name":"conference_management_system","label":"Conference Management System","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"185","order":3,"name":"number_of_submissions_sent_for_review","label":"Number of Submissions Sent for Review","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"33","order":4,"name":"number_of_full_papers_accepted","label":"Number of Full Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"0","order":5,"name":"number_of_short_papers_accepted","label":"Number of Short Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"18% - The value is computed by the equation \"Number of Full Papers Accepted \/ Number of Submissions Sent for Review * 100\" and then rounded to a whole number.","order":6,"name":"acceptance_rate_of_full_papers","label":"Acceptance Rate of Full Papers","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"3","order":7,"name":"average_number_of_reviews_per_paper","label":"Average Number of Reviews per Paper","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"5","order":8,"name":"average_number_of_papers_per_reviewer","label":"Average Number of Papers per Reviewer","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"Yes","order":9,"name":"external_reviewers_involved","label":"External Reviewers Involved","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"In addition, 11 papers were published from the CAiSE 2020 workshops. The conference was held virtually due to the COVID-19 pandemic.","order":10,"name":"additional_info_on_review_process","label":"Additional Info on Review Process","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"This content has been made available to all.","name":"free","label":"Free to read"}]}}