{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2024,9,11]],"date-time":"2024-09-11T19:46:24Z","timestamp":1726083984489},"publisher-location":"Cham","reference-count":40,"publisher":"Springer International Publishing","isbn-type":[{"type":"print","value":"9783030399573"},{"type":"electronic","value":"9783030399580"}],"license":[{"start":{"date-parts":[[2020,1,1]],"date-time":"2020-01-01T00:00:00Z","timestamp":1577836800000},"content-version":"tdm","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2020]]},"DOI":"10.1007\/978-3-030-39958-0_17","type":"book-chapter","created":{"date-parts":[[2020,5,7]],"date-time":"2020-05-07T13:29:08Z","timestamp":1588858148000},"page":"345-366","update-policy":"http:\/\/dx.doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":2,"title":["Evolving a Dota 2 Hero Bot with a Probabilistic Shared Memory Model"],"prefix":"10.1007","author":[{"given":"Robert J.","family":"Smith","sequence":"first","affiliation":[]},{"given":"Malcolm I.","family":"Heywood","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2020,5,8]]},"reference":[{"key":"17_CR1","first-page":"19","volume":"9597","author":"A Agapitos","year":"2016","unstructured":"Agapitos, A., Brabazon, A., O\u2019Neill, M.: Genetic programming with memory for financial trading. In: EvoApplications, LNCS, vol. 9597, pp. 19\u201334 (2016)","journal-title":"LNCS"},{"key":"17_CR2","doi-asserted-by":"publisher","first-page":"204","DOI":"10.1109\/72.80232","volume":"15","author":"SVB Aiyer","year":"1990","unstructured":"Aiyer, S.V.B., Niranjan, N., Fallside, F.: A theoretical investigation into the performance of the Hopfield model. IEEE Transactions on Neural Networks 15, 204\u2013215 (1990)","journal-title":"IEEE Transactions on Neural Networks"},{"key":"17_CR3","first-page":"161","volume":"1598","author":"B Andersson","year":"1999","unstructured":"Andersson, B., Nordin, P., Nordahl, M.: Reactive and memory-based genetic programming for robot control. In: European Conference on Genetic Programming, LNCS, vol. 1598, pp. 161\u2013172 (1999)","journal-title":"LNCS"},{"key":"17_CR4","unstructured":"Andre, D.: Evolution of mapmaking ability: Strategies for the evolution of learning, planning, and memory using genetic programming. In: IEEE World Congress on Computational Intelligence, pp. 250\u2013255 (1994)"},{"key":"17_CR5","unstructured":"Babicki, S., Arndt, D., Marcu, A., Liang, Y., Grant, J.R., Maciejewski, A., Wishart, D.S.: Heatmapper: web-enabled heat mapping for all. Nucleic Acids Research (2016). http:\/\/www.heatmapper.ca\/"},{"key":"17_CR6","unstructured":"Brameier, M., Banzhaf, W.: Linear Genetic Programming. Springer (2007)"},{"key":"17_CR7","unstructured":"Brave, S.: The evolution of memory and mental models using genetic programming. In: Proceedings of the Annual Conference on Genetic Programming (1996)"},{"key":"17_CR8","doi-asserted-by":"publisher","first-page":"179","DOI":"10.1207\/s15516709cog1402_1","volume":"14","author":"JL Elman","year":"1990","unstructured":"Elman, J.L.: Finding structure in time. Cognitive Science 14, 179\u2013211 (1990)","journal-title":"Cognitive Science"},{"key":"17_CR9","unstructured":"Graves, A., Wayne, G., Danihelka, I.: Neural turing machines. CoRR abs\/1410.5401 (2014)"},{"issue":"7626","key":"17_CR10","doi-asserted-by":"publisher","first-page":"471","DOI":"10.1038\/nature20101","volume":"538","author":"A Graves","year":"2016","unstructured":"Graves, A., Wayne, G., Reynolds, M., Harley, T., Danihelka, I., Grabska-Barwinska, A., Colmenarejo, S.G., Grefenstette, E., Ramalho, T., Agapiou, J., Badia, A.P., Hermann, K.M., Zwols, Y., Ostrovski, G., Cain, A., King, H., Summerfield, C., Blunsom, P., Kavukcuoglu, K., Hassabis, D.: Hybrid computing using a neural network with dynamic external memory. Nature 538(7626), 471\u2013476 (2016)","journal-title":"Nature"},{"issue":"10","key":"17_CR11","doi-asserted-by":"publisher","first-page":"2222","DOI":"10.1109\/TNNLS.2016.2582924","volume":"28","author":"K Greff","year":"2017","unstructured":"Greff, K., Srivastava, R.K., Koutn\u00edk, J., Steunebrink, B.R., Schmidhuber, J.: LSTM: A search space odyssey. IEEE Transactions on Neural Networks and Learning Systems 28(10), 2222\u20132231 (2017)","journal-title":"IEEE Transactions on Neural Networks and Learning Systems"},{"key":"17_CR12","doi-asserted-by":"crossref","unstructured":"Greve, R.B., Jacobsen, E.J., Risi, S.: Evolving neural turing machines for reward-based learning. In: ACM Genetic and Evolutionary Computation Conference, pp. 117\u2013124 (2016)","DOI":"10.1145\/2908812.2908930"},{"key":"17_CR13","unstructured":"Grossberg, S.: Content-addressable memory storage by neural networks: A general model and global Liapunov method. In: E.L. Schwartz (ed.) Computational Neuroscience, pp. 56\u201365. MIT Press (1990)"},{"key":"17_CR14","first-page":"529","volume":"7835","author":"F Haddadi","year":"2013","unstructured":"Haddadi, F., Kayacik, H.G., Zincir-Heywood, A.N., Heywood, M.I.: Malicious automatically generated domain name detection using stateful-SBB. In: EvoApplications, LNCS, vol. 7835, pp. 529\u2013539 (2013)","journal-title":"LNCS"},{"issue":"8","key":"17_CR15","doi-asserted-by":"publisher","first-page":"1735","DOI":"10.1162\/neco.1997.9.8.1735","volume":"9","author":"S Hochreiter","year":"1997","unstructured":"Hochreiter, S., Schmidhuber, J.: Long short-term memory. Neural Computation 9(8), 1735\u20131780 (1997)","journal-title":"Neural Computation"},{"key":"17_CR16","doi-asserted-by":"crossref","unstructured":"Huelsbergen, L.: Toward simulated evolution of machine language iteration. In: Proceedings of the Annual Conference on Genetic Programming, pp. 315\u2013320 (1996)","DOI":"10.7551\/mitpress\/3242.003.0045"},{"key":"17_CR17","doi-asserted-by":"publisher","first-page":"859","DOI":"10.1126\/science.aau6249","volume":"364","author":"M Jaderberg","year":"2019","unstructured":"Jaderberg, M., Czarnecki, W.M., Dunning, I., Marris, L., Lever, G., Casta\u00f1eda, A.G., Beattie, C., Rabinowitz, N.C., Morcos, A.S., Ruderman, A., Sonnerat, N., Green, T., Deason, L., Leibo, J.Z., Silver, D., Hassabis, D., Kavukcuoglu, K., Graepel, T.: Human-level performance in 3D multiplayer games with population-based reinforcement learning. Science 364, 859\u2013865 (2019)","journal-title":"Science"},{"key":"17_CR18","doi-asserted-by":"crossref","unstructured":"Kelly, S., Banzhaf, W.: Temporal memory sharing in visual reinforcement learning. In: W. Banzhaf, E. Goodman, L. Sheneman, L. Trujillo, B. Worzel (eds.) Genetic Programming Theory and Practice, vol. XVII. Springer (2020)","DOI":"10.1007\/978-3-030-04735-1"},{"key":"17_CR19","first-page":"64","volume":"10196","author":"S Kelly","year":"2017","unstructured":"Kelly, S., Heywood, M.I.: Emergent tangled graph representations for Atari game playing agents. In: European Conference on Genetic Programming, LNCS, vol. 10196, pp. 64\u201379 (2017)","journal-title":"LNCS"},{"key":"17_CR20","doi-asserted-by":"crossref","unstructured":"Kelly, S., Heywood, M.I.: Multi-task learning in Atari video games with emergent tangled program graphs. In: ACM Genetic and Evolutionary Computation Conference, pp. 195\u2013202 (2017)","DOI":"10.1145\/3071178.3071303"},{"issue":"3","key":"17_CR21","doi-asserted-by":"publisher","first-page":"347","DOI":"10.1162\/evco_a_00232","volume":"26","author":"S Kelly","year":"2018","unstructured":"Kelly, S., Heywood, M.I.: Emergent solutions to high-dimensional multitask reinforcement learning. Evolutionary Computation 26(3), 347\u2013380 (2018)","journal-title":"Evolutionary Computation"},{"key":"17_CR22","doi-asserted-by":"crossref","unstructured":"Kelly, S., Smith, R.J., Heywood, M.I.: Emergent policy discovery for visual reinforcement learning through tangled program graphs: A tutorial. In: W. Banzhaf, L. Spector, L. Sheneman (eds.) Genetic Programming Theory and Practice, vol. XVI, chap. 3, pp. 37\u201357. Springer (2019)","DOI":"10.1007\/978-3-030-04735-1_3"},{"key":"17_CR23","doi-asserted-by":"crossref","unstructured":"Langdon, W.B.: Genetic Programming and Data Structures. Kluwer Academic (1998)","DOI":"10.1007\/978-1-4615-5731-9"},{"key":"17_CR24","doi-asserted-by":"crossref","unstructured":"Lichodzijewski, P., Heywood, M.I.: Symbiosis, complexification and simplicity under GP. In: Proceedings of the ACM Genetic and Evolutionary Computation Conference, pp. 853\u2013860 (2010)","DOI":"10.1145\/1830483.1830640"},{"key":"17_CR25","doi-asserted-by":"publisher","first-page":"523","DOI":"10.1613\/jair.5699","volume":"61","author":"MC Machado","year":"2018","unstructured":"Machado, M.C., Bellemare, M.G., Talvitie, E., Veness, J., Hausknecht, M., Bowling, M.: Revisiting the arcade learning environment: evaluation protocols and open problems for general agents. Journal of Artificial Intelligence Research 61, 523\u2013562 (2018)","journal-title":"Journal of Artificial Intelligence Research"},{"key":"17_CR26","doi-asserted-by":"crossref","unstructured":"Merrild, J., Rasmussen, M.A., Risi, S.: Hyperntm: Evolving scalable neural turing machines through hyperneat. In: EvoApplications, pp. 750\u2013766 (2018)","DOI":"10.1007\/978-3-319-77538-8_50"},{"issue":"7540","key":"17_CR27","doi-asserted-by":"publisher","first-page":"529","DOI":"10.1038\/nature14236","volume":"518","author":"V Mnih","year":"2015","unstructured":"Mnih, V., Kavukcuoglu, K., Silver, D., Rusu, A.A., Veness, J., Bellemare, M.G., Graves, A., Riedmiller, M., Fidjeland, A.K., Ostrovski, G., Petersen, S., Beattie, C., Sadik, A., Antonoglou, I., King, H., Kumaran, D., Wierstra, D., Legg, S., Hassabis, D.: Human-level control through deep reinforcement learning. Nature 518(7540), 529\u2013533 (2015)","journal-title":"Nature"},{"key":"17_CR28","unstructured":"Nordin, P.: A compiling genetic programming system that directly manipulates the machine code. In: K.E. Kinnear (ed.) Advances in Genetic Programming, pp. 311\u2013332. MIT Press (1994)"},{"key":"17_CR29","doi-asserted-by":"crossref","unstructured":"Poli, R., McPhee, N.F., Citi, L., Crane, E.: Memory with memory in genetic programming. Journal of Artificial Evolution and Applications (2009)","DOI":"10.1155\/2009\/570606"},{"key":"17_CR30","unstructured":"Salimans, T., Ho, J., Chen, X., Sutskever, I.: Evolution strategies as a scalable alternative to reinforcement learning. CoRR abs\/1703.03864 (2016)"},{"key":"17_CR31","doi-asserted-by":"crossref","unstructured":"Sapienza, A., Peng, H., Ferrara, E.: Performance dynamics and success in online games. In: IEEE International Conference on Data Mining Workshops, pp. 902\u2013909 (2017)","DOI":"10.1109\/ICDMW.2017.124"},{"key":"17_CR32","first-page":"135","volume":"10781","author":"RJ Smith","year":"2018","unstructured":"Smith, R.J., Heywood, M.I.: Scaling tangled program graphs to visual reinforcement learning in ViZDoom. In: European Conference on Genetic Programming, LNCS, vol. 10781, pp. 135\u2013150 (2018)","journal-title":"LNCS"},{"key":"17_CR33","doi-asserted-by":"crossref","unstructured":"Smith, R.J., Heywood, M.I.: Evolving Dota 2 Shadow Fiend bots using genetic programming with external memory. In: Proceedings of the ACM Genetic and Evolutionary Computation Conference (2019)","DOI":"10.1145\/3321707.3321866"},{"key":"17_CR34","first-page":"162","volume":"11451","author":"RJ Smith","year":"2019","unstructured":"Smith, R.J., Heywood, M.I.: A model of external memory for navigation in partially observable visual reinforcement learning tasks. In: European Conference on Genetic Programming, LNCS, vol. 11451, pp. 162\u2013177 (2019)","journal-title":"LNCS"},{"key":"17_CR35","doi-asserted-by":"crossref","unstructured":"Spector, L., Luke, S.: Cultural transmission of information in genetic programming. In: Annual Conference on Genetic Programming, pp. 209\u2013214 (1996)","DOI":"10.7551\/mitpress\/3242.003.0028"},{"key":"17_CR36","unstructured":"Such, F.P., Madhavan, V., Conti, E., Lehman, J., Stanley, K.O., Clune, J.: Deep neuroevolution: Genetic algorithms are a competitive alternative for training deep neural networks for reinforcement learning. CoRR abs\/1712.06567 (2018)"},{"key":"17_CR37","unstructured":"Teller, A.: The evolution of mental models. In: K.E. Kinnear (ed.) Advances in Genetic Programming, pp. 199\u2013220. MIT Press (1994)"},{"key":"17_CR38","unstructured":"Teller, A.: Turing completeness in the language of genetic programming with indexed memory. In: IEEE Congress on Evolutionary Computation, pp. 136\u2013141 (1994)"},{"key":"17_CR39","unstructured":"Wayne, G., Hung, C.C., Amos, D., Mirza, M., Ahuja, A., Grabska-Barwi\u0144ska, A., Rae, J., Mirowski, P., Leibo, J.Z., Santoro, A., Gemici, M., Reynolds, M., Harley, T., Abramson, J., Mohamed, S., Rezende, D., Saxton, D., Cain, A., Hillier, C., Silver, D., Kavukcuoglu, K., Botvinick, M., Hasssbis, D., Lillicrap, T.: Unsupervised predictive memory in a goal-directed agent. CoRR abs\/1803.10760 (2018)"},{"key":"17_CR40","doi-asserted-by":"crossref","unstructured":"Wydmuch, M., Kempka, M., Ja\u015bkowski, W.: ViZDoom competitions: Playing doom from pixels. IEEE Transactions on Games to appear (2019)","DOI":"10.1109\/TG.2018.2877047"}],"container-title":["Genetic and Evolutionary Computation","Genetic Programming Theory and Practice XVII"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-030-39958-0_17","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,8,5]],"date-time":"2024-08-05T13:03:59Z","timestamp":1722863039000},"score":1,"resource":{"primary":{"URL":"http:\/\/link.springer.com\/10.1007\/978-3-030-39958-0_17"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2020]]},"ISBN":["9783030399573","9783030399580"],"references-count":40,"URL":"https:\/\/doi.org\/10.1007\/978-3-030-39958-0_17","relation":{},"ISSN":["1932-0167","1932-0175"],"issn-type":[{"type":"print","value":"1932-0167"},{"type":"electronic","value":"1932-0175"}],"subject":[],"published":{"date-parts":[[2020]]},"assertion":[{"value":"8 May 2020","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}}]}}