{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,3,28]],"date-time":"2025-03-28T02:50:55Z","timestamp":1743130255181,"version":"3.40.3"},"publisher-location":"Cham","reference-count":33,"publisher":"Springer Nature Switzerland","isbn-type":[{"type":"print","value":"9783031663253"},{"type":"electronic","value":"9783031663260"}],"license":[{"start":{"date-parts":[[2024,1,1]],"date-time":"2024-01-01T00:00:00Z","timestamp":1704067200000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2024,1,1]],"date-time":"2024-01-01T00:00:00Z","timestamp":1704067200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2024]]},"DOI":"10.1007\/978-3-031-66326-0_16","type":"book-chapter","created":{"date-parts":[[2024,7,29]],"date-time":"2024-07-29T12:59:46Z","timestamp":1722257986000},"page":"257-271","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":0,"title":["Architecture for\u00a0Digital Twin-Based Reinforcement Learning Optimization of\u00a0Cyber-Physical Systems"],"prefix":"10.1007","author":[{"ORCID":"https:\/\/orcid.org\/0009-0006-0305-7848","authenticated-orcid":false,"given":"Elias","family":"Modrakowski","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Niklas","family":"Braun","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0009-0007-1656-5670","authenticated-orcid":false,"given":"Mehrnoush","family":"Hajnorouzi","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-0828-9385","authenticated-orcid":false,"given":"Andreas","family":"Eich","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Narges","family":"Javaheri","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Richard","family":"Doornbos","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0009-0006-0344-6253","authenticated-orcid":false,"given":"Sebastian","family":"Moritz","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Jan-Willem","family":"Bikker","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Rutger","family":"van Beek","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2024,7,30]]},"reference":[{"key":"16_CR1","doi-asserted-by":"publisher","unstructured":"Abdolmaleki, A., Springenberg, J.T., Tassa, Y., Munos, R., Heess, N., Riedmiller, M.: Maximum a posteriori policy optimisation (2018). https:\/\/doi.org\/10.48550\/arXiv.1806.06920","DOI":"10.48550\/arXiv.1806.06920"},{"key":"16_CR2","doi-asserted-by":"publisher","DOI":"10.1016\/j.aei.2020.101225","volume":"47","author":"S Aheleroff","year":"2021","unstructured":"Aheleroff, S., Xu, X., Zhong, R.Y., Lu, Y.: Digital twin as a service (DTAAS) in industry 4.0: An architecture reference model. Adv. Eng. Inform. 47, 101225 (2021). https:\/\/doi.org\/10.1016\/j.aei.2020.101225","journal-title":"Adv. Eng. Inform."},{"issue":"5","key":"16_CR3","doi-asserted-by":"publisher","first-page":"429","DOI":"10.1080\/0951192X.2020.1747642","volume":"33","author":"K Alexopoulos","year":"2020","unstructured":"Alexopoulos, K., Nikolakis, N., Chryssolouris, G.: Digital twin-driven supervised machine learning for the development of artificial intelligence applications in manufacturing. Int. J. Comput. Integr. Manuf. 33(5), 429\u2013439 (2020). https:\/\/doi.org\/10.1080\/0951192X.2020.1747642","journal-title":"Int. J. Comput. Integr. Manuf."},{"key":"16_CR4","doi-asserted-by":"publisher","unstructured":"Amiranashvili, A., Argus, M., Hermann, L., Burgard, W., Brox, T.: Pre-training of deep RL agents for improved learning under domain randomization. eprint arXiv:2104.14386 (2021). https:\/\/doi.org\/10.48550\/arXiv.2104.14386","DOI":"10.48550\/arXiv.2104.14386"},{"key":"16_CR5","doi-asserted-by":"publisher","first-page":"167653","DOI":"10.1109\/ACCESS.2019.2953499","volume":"7","author":"BR Barricelli","year":"2019","unstructured":"Barricelli, B.R., Casiraghi, E., Fogli, D.: A survey on digital twin: definitions, characteristics, applications, and design implications. IEEE Access 7, 167653\u2013167671 (2019). https:\/\/doi.org\/10.1109\/ACCESS.2019.2953499","journal-title":"IEEE Access"},{"key":"16_CR6","doi-asserted-by":"publisher","first-page":"164","DOI":"10.1007\/978-3-030-34843-4_14","volume-title":"Complex Syst. Des. Manag.","author":"B Brankovic","year":"2020","unstructured":"Brankovic, B., Binder, C., Draxler, D., Neureiter, C., Lastro, G.: Towards a cross-domain modeling approach in system-of-systems architectures. In: Boy, G.A., Guegan, A., Krob, D., Vion, V. (eds.) CSDM 2019, pp. 164\u2013175. Springer, Cham (2020). https:\/\/doi.org\/10.1007\/978-3-030-34843-4_14"},{"key":"16_CR7","unstructured":"Cobbe, K., Klimov, O., Hesse, C., Kim, T., Schulman, J.: Quantifying generalization in reinforcement learning. In: 36th International Conference on Machine Learning, vol. PMLR 97. PMLR, Long Beach, USA (2019)"},{"issue":"7897","key":"16_CR8","doi-asserted-by":"publisher","first-page":"414","DOI":"10.1038\/s41586-021-04301-9","volume":"602","author":"J Degrave","year":"2022","unstructured":"Degrave, J., et al.: Magnetic control of tokamak plasmas through deep reinforcement learning. Nature 602(7897), 414\u2013419 (2022). https:\/\/doi.org\/10.1038\/s41586-021-04301-9","journal-title":"Nature"},{"issue":"9","key":"16_CR9","doi-asserted-by":"publisher","first-page":"2419","DOI":"10.1007\/s10994-021-05961-4","volume":"110","author":"G Dulac-Arnold","year":"2021","unstructured":"Dulac-Arnold, G., et al.: Challenges of real-world reinforcement learning: definitions, benchmarks and analysis. Mach. Learn. 110(9), 2419\u20132468 (2021)","journal-title":"Mach. Learn."},{"key":"16_CR10","doi-asserted-by":"publisher","first-page":"50335","DOI":"10.1109\/ACCESS.2022.3172964","volume":"10","author":"E Ferko","year":"2022","unstructured":"Ferko, E., Bucaioni, A., Behnam, M.: Architecting digital twins. IEEE Access 10, 50335\u201350350 (2022). https:\/\/doi.org\/10.1109\/ACCESS.2022.3172964","journal-title":"IEEE Access"},{"key":"16_CR11","doi-asserted-by":"crossref","unstructured":"Gan, X., Zuo, Y., Zhang, A., Li, S., Tao, F.: Digital twin-enabled adaptive scheduling strategy based on deep reinforcement learning. Sci. China Technol. Sci. 1\u201315 (2023)","DOI":"10.1007\/s11431-022-2413-5"},{"key":"16_CR12","doi-asserted-by":"publisher","unstructured":"Grieves, M., Vickers, J., (None): Digital twin: mitigating unpredictable, undesirable emergent behavior in complex systems. In: Kahlen, F.J., Flumerfelt, S., Alves, A. (eds.) Transdisciplinary Perspectives on Complex Systems, pp. 85\u2013113. Springer, Cham (2016). https:\/\/doi.org\/10.1007\/978-3-319-38756-7","DOI":"10.1007\/978-3-319-38756-7"},{"key":"16_CR13","unstructured":"Haj-Ali, A., Ahmed, N.K., Willke, T., Gonzalez, J., Asanovic, K., Stoica, I.: A view on deep reinforcement learning in system optimization"},{"key":"16_CR14","unstructured":"Hankel, M., Rexroth, B.: Das Referenzarchitekturmodell Industrie 4.0 (RAMI 4.0)"},{"key":"16_CR15","unstructured":"IEEE: IEEE standard for modeling and simulation (m &s) high level architecture (HLA): Framework and rules (2010). https:\/\/standards.ieee.org\/ieee\/1516\/3744\/"},{"issue":"3","key":"16_CR16","doi-asserted-by":"publisher","first-page":"56","DOI":"10.3390\/asi5030056","volume":"5","author":"S Jamil","year":"2022","unstructured":"Jamil, S., Rahman, M.: Fawad: a comprehensive survey of digital twins and federated learning for industrial internet of things (IIOT), internet of vehicles (IOV) and internet of drones (IOD). Appl. Syst. Innov. 5(3), 56 (2022)","journal-title":"Appl. Syst. Innov."},{"key":"16_CR17","doi-asserted-by":"publisher","first-page":"36","DOI":"10.1016\/j.cirpj.2020.02.002","volume":"29","author":"D Jones","year":"2020","unstructured":"Jones, D., Snider, C., Nassehi, A., Yon, J., Hicks, B.: Characterising the digital twin: a systematic literature review. CIRP J. Manuf. Sci. Technol. 29, 36\u201352 (2020). https:\/\/doi.org\/10.1016\/j.cirpj.2020.02.002","journal-title":"CIRP J. Manuf. Sci. Technol."},{"issue":"12","key":"16_CR18","doi-asserted-by":"publisher","first-page":"1077","DOI":"10.1038\/s42256-022-00573-6","volume":"4","author":"H Ju","year":"2022","unstructured":"Ju, H., Juan, R., Gomez, R., Nakamura, K., Li, G.: Transferring policy of deep reinforcement learning from simulation to reality for robotics. Nature Mach. Intell. 4(12), 1077\u20131087 (2022). https:\/\/doi.org\/10.1038\/s42256-022-00573-6","journal-title":"Nature Mach. Intell."},{"key":"16_CR19","unstructured":"Julian, R., Swanson, B., Sukhatme, G.S., Levine, S., Finn, C., Hausman, K.: Never stop learning: the effectiveness of fine-tuning in robotic reinforcement learning. arXiv preprint arXiv:2004.10190 (2020)"},{"key":"16_CR20","doi-asserted-by":"publisher","unstructured":"Kairouz, P., et al. (eds.): Advances and Open Problems in Federated Learning, Foundation and Trends in Machine Learning, vol.\u00a014. Now Publishers Inc. (2021). https:\/\/doi.org\/10.1561\/2200000083","DOI":"10.1561\/2200000083"},{"key":"16_CR21","doi-asserted-by":"publisher","first-page":"106","DOI":"10.1016\/j.cag.2021.01.011","volume":"95","author":"M Matulis","year":"2021","unstructured":"Matulis, M., Harvey, C.: A robot arm digital twin utilising reinforcement learning. Comput. Graph. 95, 106\u2013114 (2021). https:\/\/doi.org\/10.1016\/j.cag.2021.01.011","journal-title":"Comput. Graph."},{"key":"16_CR22","unstructured":"Menzel, T., Bagschik, G., Maurer, M.: 2018 IEEE Intelligent Vehicles Symposium (IV): 26\u201330 June 2018, Piscataway, NJ. IEEE (2018)"},{"key":"16_CR23","unstructured":"Modelica Association Project FMI: Functional mock-up interface for model exchange and co-simulation (2019)"},{"key":"16_CR24","doi-asserted-by":"publisher","unstructured":"Neurohr, C., Westhofen, L., Henning, T., de\u00a0Graaff, T., M\u00f6hlmann, E., B\u00f6de, E.: Fundamental considerations around scenario-based testing for automated driving. In: IEEE Intelligent Vehicles Symposium Proceedings, pp. 121\u2013127 (2020). https:\/\/doi.org\/10.1109\/IV47402.2020.9304823","DOI":"10.1109\/IV47402.2020.9304823"},{"key":"16_CR25","doi-asserted-by":"crossref","unstructured":"Osi\u0144ski, B., et al.: Simulation-based reinforcement learning for real-world autonomous driving. In: 2020 IEEE International Conference on Robotics and Automation (ICRA), pp. 6411\u20136418. IEEE (2020)","DOI":"10.1109\/ICRA40945.2020.9196730"},{"key":"16_CR26","doi-asserted-by":"publisher","unstructured":"Pohl, K., Broy, M., Daembkes, H., H\u00f6nninger, H. (eds.): Advanced Model-Based Engineering of Embedded Systems. Springer, Cham (2016). https:\/\/doi.org\/10.1007\/978-3-319-48003-9","DOI":"10.1007\/978-3-319-48003-9"},{"issue":"13","key":"16_CR27","doi-asserted-by":"publisher","first-page":"11141","DOI":"10.1109\/JIOT.2021.3127873","volume":"9","author":"G Shen","year":"2021","unstructured":"Shen, G., et al.: Deep reinforcement learning for flocking motion of multi-UAV systems: learn from a digital twin. IEEE Internet Things J. 9(13), 11141\u201311153 (2021)","journal-title":"IEEE Internet Things J."},{"key":"16_CR28","doi-asserted-by":"publisher","first-page":"8354","DOI":"10.1109\/TVT.2023.3245120","volume":"72","author":"G Shen","year":"2023","unstructured":"Shen, G., Lei, L., Zhang, X., Li, Z., Cai, S., Zhang, L.: Multi-UAV cooperative search based on reinforcement learning with a digital twin driven training framework. IEEE Trans. Veh. Technol. 72, 8354\u20138368 (2023)","journal-title":"IEEE Trans. Veh. Technol."},{"key":"16_CR29","doi-asserted-by":"publisher","unstructured":"Stark, R., Damerau, T.: Digital twin. In: Chatti, S., Tolio, T. (eds.) CIRP Encyclopedia of Production Engineering, pp.\u00a01\u20138. Springer Berlin Heidelberg, Heidelberg (2019). https:\/\/doi.org\/10.1007\/978-3-642-35950-7_16870-1","DOI":"10.1007\/978-3-642-35950-7_16870-1"},{"issue":"8","key":"16_CR30","doi-asserted-by":"publisher","first-page":"5605","DOI":"10.1109\/TII.2020.3034674","volume":"17","author":"W Sun","year":"2021","unstructured":"Sun, W., Lei, S., Wang, L., Liu, Z., Zhang, Y.: Adaptive federated learning and digital twin for industrial internet of things. IEEE Trans. Industr. Inf. 17(8), 5605\u20135614 (2021). https:\/\/doi.org\/10.1109\/TII.2020.3034674","journal-title":"IEEE Trans. Industr. Inf."},{"key":"16_CR31","unstructured":"Sutton, R.S., Barto, A.: Reinforcement Learning: An Introduction. Adaptive Computation and Machine Learning, 2nd edn. The MIT Press, Cambridge, Massachusetts and London, England (2018)"},{"key":"16_CR32","doi-asserted-by":"publisher","unstructured":"Wagg, D.J., Worden, K., Barthorpe, R.J., Gardner, P.: Digital twins: state-of-the-art and future directions for modeling and simulation in engineering dynamics applications. ASCE - ASME J. Risk Uncertainty Eng. Syst. 6(3) (2020). https:\/\/doi.org\/10.1115\/1.4046739","DOI":"10.1115\/1.4046739"},{"key":"16_CR33","doi-asserted-by":"publisher","unstructured":"Zhang, Z., Zahng, D., Qiu, R.C.: Deep reinforcement learning for power system: an overview. CSEE J. Power Energy Syst. 6(1) (2020). https:\/\/doi.org\/10.17775\/CSEEJPES.2019.00920","DOI":"10.17775\/CSEEJPES.2019.00920"}],"container-title":["Lecture Notes in Computer Science","Software Architecture. ECSA 2023 Tracks, Workshops, and Doctoral Symposium"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-031-66326-0_16","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,7,29]],"date-time":"2024-07-29T13:03:38Z","timestamp":1722258218000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-3-031-66326-0_16"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024]]},"ISBN":["9783031663253","9783031663260"],"references-count":33,"URL":"https:\/\/doi.org\/10.1007\/978-3-031-66326-0_16","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"type":"print","value":"0302-9743"},{"type":"electronic","value":"1611-3349"}],"subject":[],"published":{"date-parts":[[2024]]},"assertion":[{"value":"30 July 2024","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"ECSA","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"European Conference on Software Architecture","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Istanbul","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"T\u00fcrkiye","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2023","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"18 September 2023","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"22 September 2023","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"ecsa2023","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Double-blind","order":1,"name":"type","label":"Type","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"EasyChair","order":2,"name":"conference_management_system","label":"Conference Management System","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"71","order":3,"name":"number_of_submissions_sent_for_review","label":"Number of Submissions Sent for Review","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"16","order":4,"name":"number_of_full_papers_accepted","label":"Number of Full Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"9","order":5,"name":"number_of_short_papers_accepted","label":"Number of Short Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"23% - The value is computed by the equation \"Number of Full Papers Accepted \/ Number of Submissions Sent for Review * 100\" and then rounded to a whole number.","order":6,"name":"acceptance_rate_of_full_papers","label":"Acceptance Rate of Full Papers","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"3","order":7,"name":"average_number_of_reviews_per_paper","label":"Average Number of Reviews per Paper","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"3","order":8,"name":"average_number_of_papers_per_reviewer","label":"Average Number of Papers per Reviewer","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"Yes","order":9,"name":"external_reviewers_involved","label":"External Reviewers Involved","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}}]}}