{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,8,19]],"date-time":"2026-08-19T15:28:16Z","timestamp":1787153296346,"version":"build-2736575974"},"publisher-location":"Cham","reference-count":38,"publisher":"Springer Nature Switzerland","isbn-type":[{"value":"9783032193421","type":"print"},{"value":"9783032193438","type":"electronic"}],"license":[{"start":{"date-parts":[[2026,1,1]],"date-time":"2026-01-01T00:00:00Z","timestamp":1767225600000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2026,1,1]],"date-time":"2026-01-01T00:00:00Z","timestamp":1767225600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2026]]},"DOI":"10.1007\/978-3-032-19343-8_7","type":"book-chapter","created":{"date-parts":[[2026,4,15]],"date-time":"2026-04-15T23:02:34Z","timestamp":1776294154000},"page":"103-118","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":0,"title":["A Benchmark Study of\u00a0Deep Reinforcement Learning Algorithms for\u00a0the\u00a0Container Stowage Planning Problem"],"prefix":"10.1007","author":[{"given":"Yunqi","family":"Huang","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Nishith","family":"Chennakeshava","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Alexis","family":"Carras","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Vladislav","family":"Neverov","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Wei","family":"Liu","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Aske","family":"Plaat","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Yingjie","family":"Fan","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2026,4,1]]},"reference":[{"key":"7_CR1","doi-asserted-by":"publisher","unstructured":"Ambrosino, D., Sciomachen, A., Tanfani, E.: Stowing a containership: the master bay plan problem. Transp. Res. Part A Policy Pract. 38(2), 81\u201399 (2004). https:\/\/doi.org\/10.1016\/j.tra.2003.09.002, https:\/\/www.sciencedirect.com\/science\/article\/pii\/S0965856403000892","DOI":"10.1016\/j.tra.2003.09.002"},{"issue":"1\u20134","key":"7_CR2","doi-asserted-by":"publisher","first-page":"271","DOI":"10.1016\/0360-8352(93)90273-Z","volume":"25","author":"M Avriel","year":"1993","unstructured":"Avriel, M., Penn, M.: Exact and approximate solutions of the container ship stowage problem. Comput. Ind. Eng. 25(1\u20134), 271\u2013274 (1993)","journal-title":"Comput. Ind. Eng."},{"key":"7_CR3","doi-asserted-by":"publisher","first-page":"495","DOI":"10.1016\/j.asoc.2018.01.006","volume":"65","author":"AT Azevedo","year":"2018","unstructured":"Azevedo, A.T., de Salles Neto, L.L., Chaves, A.A., Moretti, A.C.: Solving the 3D stowage planning problem integrated with the quay crane scheduling problem by representation by rules and genetic algorithm. Appl. Soft Comput. 65, 495\u2013516 (2018)","journal-title":"Appl. Soft Comput."},{"issue":"3731","key":"7_CR4","doi-asserted-by":"publisher","first-page":"34","DOI":"10.1126\/science.153.3731.34","volume":"153","author":"R Bellman","year":"1966","unstructured":"Bellman, R.: Dynamic programming. Science 153(3731), 34\u201337 (1966)","journal-title":"Science"},{"key":"7_CR5","unstructured":"Brockman, G., et al.: OpenAI gym (2016)"},{"issue":"10","key":"7_CR6","doi-asserted-by":"publisher","first-page":"1832","DOI":"10.3390\/jmse12101832","volume":"12","author":"J Cho","year":"2024","unstructured":"Cho, J., Ku, N.: Developing a container ship loading-planning program using reinforcement learning. J. Mar. Sci. Eng. 12(10), 1832 (2024)","journal-title":"J. Mar. Sci. Eng."},{"key":"7_CR7","unstructured":"Cruz-Reyes, L., et al.: Lower and upper bounds for the master bay planning problem. Int. J Comb. Optim. Probl. Inf. 6(1), 42\u201352 (2015)"},{"key":"7_CR8","doi-asserted-by":"crossref","unstructured":"Dabney, W., Rowland, M., Bellemare, M., Munos, R.: Distributional reinforcement learning with quantile regression. In: Proceedings of the AAAI Conference on Artificial Intelligence, vol.\u00a032 (2018)","DOI":"10.1609\/aaai.v32i1.11791"},{"key":"7_CR9","doi-asserted-by":"publisher","unstructured":"He, J., Zhang, L., Deng, Y., Yu, H., Huang, M., Tan, C.: An allocation approach for external truck tasks appointment in automated container terminal. Adv. Eng. Inf. 55, 101864 (2023). https:\/\/doi.org\/10.1016\/j.aei.2022.101864, https:\/\/www.sciencedirect.com\/science\/article\/pii\/S1474034622003226","DOI":"10.1016\/j.aei.2022.101864"},{"key":"7_CR10","doi-asserted-by":"crossref","unstructured":"Henderson, P., Islam, R., Bachman, P., Pineau, J., Precup, D., Meger, D.: Deep reinforcement learning that matters. In: Proceedings of the AAAI Conference on Artificial Intelligence, vol.\u00a032 (2018)","DOI":"10.1609\/aaai.v32i1.11694"},{"issue":"18","key":"7_CR11","doi-asserted-by":"publisher","first-page":"2236","DOI":"10.3390\/math9182236","volume":"9","author":"HP Hsu","year":"2021","unstructured":"Hsu, H.P., Wang, C.N., Fu, H.P., Dang, T.T.: Joint scheduling of yard crane, yard truck, and quay crane for container terminal considering vessel stowage plan: an integrated simulation-based optimization approach. Mathematics 9(18), 2236 (2021)","journal-title":"Mathematics"},{"key":"7_CR12","unstructured":"Huang, S., Onta\u00f1\u00f3n, S.: A closer look at invalid action masking in policy gradient algorithms. arXiv preprint arXiv:2006.14171 (2020)"},{"key":"7_CR13","doi-asserted-by":"publisher","first-page":"23881","DOI":"10.52202\/068431-1734","volume":"35","author":"V Jayawardana","year":"2022","unstructured":"Jayawardana, V., Tang, C., Li, S., Suo, D., Wu, C.: The impact of task underspecification in evaluating deep reinforcement learning. Adv. Neural. Inf. Process. Syst. 35, 23881\u201323893 (2022)","journal-title":"Adv. Neural. Inf. Process. Syst."},{"key":"7_CR14","unstructured":"Jensen, R.M., Pacino, D., Ajspur, M.L., Vesterdal, C.: Container vessel stowage planning. Weilbach (2018)"},{"key":"7_CR15","doi-asserted-by":"crossref","unstructured":"Jiang, T., Zeng, B., Wang, Y., Yan, W.: A new heuristic reinforcement learning for container relocation problem. J. Phys. Conf. Ser. 1873, 012050 (2021)","DOI":"10.1088\/1742-6596\/1873\/1\/012050"},{"issue":"1","key":"7_CR16","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1007\/s10696-020-09385-5","volume":"33","author":"D Kizilay","year":"2021","unstructured":"Kizilay, D., Eliiyi, D.T.: A comprehensive review of quay crane scheduling, yard operations and integrations thereof in container terminals. Flex. Serv. Manuf. J. 33(1), 1\u201342 (2021)","journal-title":"Flex. Serv. Manuf. J."},{"key":"7_CR17","unstructured":"Mnih, V., et al.: Asynchronous methods for deep reinforcement learning. In: International Conference on Machine Learning, pp. 1928\u20131937. PMLR (2016)"},{"key":"7_CR18","unstructured":"Mnih, V., et al.: Playing Atari with deep reinforcement learning. arXiv preprint arXiv:1312.5602 (2013)"},{"key":"7_CR19","doi-asserted-by":"crossref","unstructured":"Padakandla, S.: A survey of reinforcement learning algorithms for dynamically varying environments. ACM Comput. Surv. (CSUR) 54(6), 1\u201325 (2021)","DOI":"10.1145\/3459991"},{"key":"7_CR20","unstructured":"Raffin, A., Hill, A., Gleave, A., Kanervisto, A., Ernestus, M., Dormann, N.: Stable-Baselines3: reliable reinforcement learning implementations. J. Mach. Learn. Res. 22(268), 1\u20138 (2021). http:\/\/jmlr.org\/papers\/v22\/20-1364.html"},{"key":"7_CR21","doi-asserted-by":"publisher","unstructured":"Reda, D., Tao, T., van\u00a0de Panne, M.: Learning to locomote: understanding how environment design matters for deep reinforcement learning. In: Motion, Interaction and Games, MIG \u201920, October 2020. ACM (2020). https:\/\/doi.org\/10.1145\/3424636.3426907","DOI":"10.1145\/3424636.3426907"},{"key":"7_CR22","unstructured":"Schulman, J., Levine, S., Abbeel, P., Jordan, M., Moritz, P.: Trust region policy optimization. In: International Conference on Machine Learning, pp. 1889\u20131897. PMLR (2015)"},{"key":"7_CR23","unstructured":"Schulman, J., Wolski, F., Dhariwal, P., Radford, A., Klimov, O.: Proximal policy optimization algorithms. arXiv preprint arXiv:1707.06347 (2017)"},{"issue":"s3","key":"7_CR24","doi-asserted-by":"publisher","first-page":"102","DOI":"10.1515\/pomr-2017-0111","volume":"24","author":"Y Shen","year":"2017","unstructured":"Shen, Y., Zhao, N., Xia, M., Du, X.: A deep q-learning network for ship stowage planning problem. Polish Maritime Res. 24(s3), 102\u2013109 (2017)","journal-title":"Polish Maritime Res."},{"key":"7_CR25","volume-title":"Reinforcement Learning: An Introduction","author":"RS Sutton","year":"2018","unstructured":"Sutton, R.S., Barto, A.G.: Reinforcement Learning: An Introduction. A Bradford Book, Cambridge (2018)"},{"key":"7_CR26","unstructured":"Terry, J., et al.: PettingZoo: gym for multi-agent reinforcement learning. Adv. Neural. Inf. Process. Syst. 34, 15032\u201315043 (2021)"},{"key":"7_CR27","unstructured":"United Nations Conference on Trade and Development: Review of Maritime Transport 2024, 2024 edn. United Nations (2024). https:\/\/www.un-ilibrary.org\/content\/books\/9789211065923"},{"key":"7_CR28","doi-asserted-by":"crossref","unstructured":"van Twiller, J., Grbic, D., Jensen, R.M.: Towards a deep reinforcement learning model of master bay stowage planning. In: International Conference on Computational Logistics, pp. 105\u2013121. Springer (2023)","DOI":"10.1007\/978-3-031-43612-3_6"},{"key":"7_CR29","doi-asserted-by":"publisher","unstructured":"van Twiller, J., Sivertsen, A., Pacino, D., Jensen, R.M.: Literature survey on the container stowage planning problem. Eur. J. Oper. Res. 317(3), 841\u2013857 (2024). https:\/\/doi.org\/10.1016\/j.ejor.2023.12.018, https:\/\/www.sciencedirect.com\/science\/article\/pii\/S0377221723009517","DOI":"10.1016\/j.ejor.2023.12.018"},{"key":"7_CR30","unstructured":"Wei, L., Wei, F., Schmitz, S., Kunal, K.: Optimization of container relocation problem via reinforcement learning. Logist. J. Proc. 2021(17) (2021)"},{"key":"7_CR31","doi-asserted-by":"publisher","first-page":"229","DOI":"10.1023\/A:1022672621406","volume":"8","author":"RJ Williams","year":"1992","unstructured":"Williams, R.J.: Simple statistical gradient-following algorithms for connectionist reinforcement learning. Mach. Learn. 8, 229\u2013256 (1992)","journal-title":"Mach. Learn."},{"issue":"11","key":"7_CR32","doi-asserted-by":"publisher","first-page":"1248","DOI":"10.1057\/palgrave.jors.2601022","volume":"51","author":"ID Wilson","year":"2000","unstructured":"Wilson, I.D., Roach, P.A.: Container stowage planning: a methodology for generating computerised solutions. J. Oper. Res. Soc. 51(11), 1248\u20131255 (2000)","journal-title":"J. Oper. Res. Soc."},{"key":"7_CR33","doi-asserted-by":"crossref","unstructured":"Xia, M., Li, Y., Shen, Y., Zhao, N.: Loading sequencing problem in container terminal with deep q-learning. J. Coast. Res. 103(SI), 817\u2013821 (2020)","DOI":"10.2112\/SI103-169.1"},{"key":"7_CR34","doi-asserted-by":"publisher","first-page":"540","DOI":"10.2112\/SI83-090.1","volume":"83","author":"N Zhao","year":"2018","unstructured":"Zhao, N., Guo, Y., Xiang, T., Xia, M., Shen, Y., Mi, C.: Container ship stowage based on Monte Carlo tree search. J. Coast. Res. 83, 540\u2013547 (2018)","journal-title":"J. Coast. Res."},{"key":"7_CR35","doi-asserted-by":"crossref","unstructured":"Zhao, W., Queralta, J.P., Westerlund, T.: Sim-to-real transfer in deep reinforcement learning for robotics: a survey. In: 2020 IEEE Symposium Series on Computational Intelligence (SSCI), pp. 737\u2013744. IEEE (2020)","DOI":"10.1109\/SSCI47803.2020.9308468"},{"key":"7_CR36","doi-asserted-by":"crossref","unstructured":"Zheng, K., Lu, Z., Sun, X.: An effective heuristic for the integrated scheduling problem of automated container handling system using twin 40\u2019cranes. In: 2010 Second International Conference on Computer Modeling and Simulation, vol.\u00a01, pp. 406\u2013410. IEEE (2010)","DOI":"10.1109\/ICCMS.2010.290"},{"key":"7_CR37","doi-asserted-by":"publisher","first-page":"106318","DOI":"10.1016\/j.ocecoaman.2022.106318","volume":"230","author":"C Zhou","year":"2022","unstructured":"Zhou, C., Zhu, S., Bell, M.G., Lee, L.H., Chew, E.P.: Emerging technology and management research in the container terminals: trends and the Covid-19 pandemic impacts. Ocean Coast. Manage. 230, 106318 (2022)","journal-title":"Ocean Coast. Manage."},{"issue":"1","key":"7_CR38","first-page":"4382745","volume":"2020","author":"H Zhu","year":"2020","unstructured":"Zhu, H., Ji, M., Guo, W.: Integer linear programming models for the containership stowage problem. Math. Probl. Eng. 2020(1), 4382745 (2020)","journal-title":"Math. Probl. Eng."}],"container-title":["Lecture Notes in Logistics","Dynamics in Logistics"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-032-19343-8_7","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,8,19]],"date-time":"2026-08-19T15:05:05Z","timestamp":1787151905000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-3-032-19343-8_7"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026]]},"ISBN":["9783032193421","9783032193438"],"references-count":38,"URL":"https:\/\/doi.org\/10.1007\/978-3-032-19343-8_7","relation":{},"ISSN":["2194-8917","2194-8925"],"issn-type":[{"value":"2194-8917","type":"print"},{"value":"2194-8925","type":"electronic"}],"subject":[],"published":{"date-parts":[[2026]]},"assertion":[{"value":"1 April 2026","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"LDIC","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"International Conference on Dynamics in Logistics","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Bremen","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Germany","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2026","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"25 February 2026","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"27 February 2026","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"10","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"ldic2026","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"https:\/\/www.ldic-conference.org\/program\/","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}}]}}