{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,11,1]],"date-time":"2025-11-01T15:35:33Z","timestamp":1762011333152,"version":"build-2065373602"},"publisher-location":"Cham","reference-count":40,"publisher":"Springer International Publishing","isbn-type":[{"type":"print","value":"9783030047344"},{"type":"electronic","value":"9783030047351"}],"license":[{"start":{"date-parts":[[2019,1,1]],"date-time":"2019-01-01T00:00:00Z","timestamp":1546300800000},"content-version":"tdm","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2019]]},"DOI":"10.1007\/978-3-030-04735-1_3","type":"book-chapter","created":{"date-parts":[[2019,1,24]],"date-time":"2019-01-24T15:57:37Z","timestamp":1548345457000},"page":"37-57","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":12,"title":["Emergent Policy Discovery for Visual Reinforcement Learning Through Tangled Program Graphs: A Tutorial"],"prefix":"10.1007","author":[{"given":"Stephen","family":"Kelly","sequence":"first","affiliation":[]},{"given":"Robert J.","family":"Smith","sequence":"additional","affiliation":[]},{"given":"Malcolm I.","family":"Heywood","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2019,1,24]]},"reference":[{"key":"3_CR1","doi-asserted-by":"crossref","unstructured":"Atkinson, T., Plump, D., Stepney, S.: Evolving graphs by graph programming. In: European Conference on Genetic Programming, Lecture Notes in Computer Science, vol. 10781, pp. 35\u201351. Springer (2018)","DOI":"10.1007\/978-3-319-77553-1_3"},{"key":"3_CR2","doi-asserted-by":"crossref","unstructured":"Banzhaf, W.: Artificial regulatory networks and genetic programming. In: R. Riolo, B. Worzel (eds.) Genetic Programming Theory and Practice, pp. 43\u201362. Springer (2003)","DOI":"10.1007\/978-1-4419-8983-3_4"},{"key":"3_CR3","doi-asserted-by":"publisher","first-page":"253","DOI":"10.1613\/jair.3912","volume":"47","author":"MG Bellemare","year":"2013","unstructured":"Bellemare, M.G., Naddaf, Y., Veness, J., Bowling, M.: The arcade learning environment: An evaluation platform for general agents. Journal of Artificial Intelligence Research 47, 253\u2013279 (2013)","journal-title":"Journal of Artificial Intelligence Research"},{"key":"3_CR4","doi-asserted-by":"publisher","first-page":"381","DOI":"10.1023\/A:1012978805372","volume":"2","author":"M Brameier","year":"2001","unstructured":"Brameier, M., Banzhaf, W.: Evolving teams of predictors with linear genetic programming. Genetic Programming and Evolvable Machines 2, 381\u2013407 (2001)","journal-title":"Genetic Programming and Evolvable Machines"},{"key":"3_CR5","unstructured":"Brameier, M., Banzhaf, W.: Linear Genetic Programming, Springer (2007)"},{"key":"3_CR6","first-page":"97","volume":"GECCO-2012","author":"JA Doucette","year":"2012","unstructured":"Doucette, J.A., Lichodzijewski, P., Heywood, M.I.: Hierarchical task decomposition through symbiosis in reinforcement learning. In: ACM Genetic and Evolutionary Computation Conference (GECCO-2012), pp. 97\u2013104 (2012)","journal-title":"In: ACM Genetic and Evolutionary Computation Conference"},{"key":"3_CR7","doi-asserted-by":"publisher","first-page":"71","DOI":"10.1007\/s10710-011-9151-4","volume":"13","author":"JA Doucette","year":"2012","unstructured":"Doucette, J.A., McIntyre, A.R., Lichodzijewski, P., Heywood, M.I.: Symbiotic coevolutionary genetic programming: a benchmarking study under large attribute spaces. Genetic Programming and Evolvable Machines 13, 71\u2013101 (2012)","journal-title":"Genetic Programming and Evolvable Machines"},{"key":"3_CR8","unstructured":"Fogal, L., Owens, A., Walsh, M.: Artificial intelligence through a simulation of evolution. In: Proceedings of the Cybernetic Sciences Symposium, pp. 131\u2013155 (1965)"},{"key":"3_CR9","doi-asserted-by":"publisher","first-page":"355","DOI":"10.1109\/TCIAIG.2013.2294713","volume":"6","author":"M Hausknecht","year":"2014","unstructured":"Hausknecht, M., Lehman, J., Miikkulainen, R., Stone, P.: A neuroevolution approach to general Atari game playing. IEEE Transactions on Computational Intelligence and AI in Games 6, 355\u2013366 (2014)","journal-title":"IEEE Transactions on Computational Intelligence and AI in Games"},{"key":"3_CR10","doi-asserted-by":"crossref","unstructured":"Jia, B., Ebner, M.: Evolving game state features from raw pixels. In: European Conference on Genetic Programming, Lecture Notes in Computer Science, vol. 10196, pp. 52\u201363. Springer (2017)","DOI":"10.1007\/978-3-319-55696-3_4"},{"key":"3_CR11","doi-asserted-by":"crossref","unstructured":"Kelly, S., Heywood, M.I.: On diversity, teaming, and hierarchical policies: Observations from the keepaway soccer task. In: European Conference on Genetic Programming 2014, Lecture Notes in Computer Science, vol. 8599, pp. 75\u201386. Springer (2014)","DOI":"10.1007\/978-3-662-44303-3_7"},{"key":"3_CR12","doi-asserted-by":"crossref","unstructured":"Kelly, S., Heywood, M.I.: Emergent tangled graph representations for Atari game playing agents. In: European Conference on Genetic Programming 2017, Lecture Notes in Computer Science, vol. 10196, pp. 64\u201379. Springer (2017)","DOI":"10.1007\/978-3-319-55696-3_5"},{"key":"3_CR13","first-page":"195","volume":"GECCO-2017","author":"S Kelly","year":"2017","unstructured":"Kelly, S., Heywood, M.I.: Multi-task learning in Atari video games with emergent tangled program graphs. In: ACM Genetic and Evolutionary Computation Conference (GECCO-2017), pp. 195\u2013202 (2017)","journal-title":"In: ACM Genetic and Evolutionary Computation Conference"},{"key":"3_CR14","doi-asserted-by":"publisher","first-page":"195","DOI":"10.1109\/TCIAIG.2017.2766980","volume":"10","author":"S Kelly","year":"2018","unstructured":"Kelly, S., Heywood, M.I.: Discovering agent behaviors through code reuse: Examples from Half-Field Offense and Ms. Pac-Man. IEEE Transactions on Games 10, 195\u2013208 (2018)","journal-title":"Pac-Man. IEEE Transactions on Games"},{"issue":"3","key":"3_CR15","doi-asserted-by":"publisher","first-page":"347","DOI":"10.1162\/evco_a_00232","volume":"26","author":"Stephen Kelly","year":"2018","unstructured":"Kelly, S., Heywood, M.I.: Emergent solutions to high-dimensional multi-task reinforcement learning. Evolutionary Computation 26(3) (2018)","journal-title":"Evolutionary Computation"},{"key":"3_CR16","doi-asserted-by":"crossref","unstructured":"Kelly, S., Lichodzijewski, P., Heywood, M.I.: On run time libraries and hierarchical symbiosis. In: IEEE Congress on Evolutionary Computation, pp. 1\u20138 (2012)","DOI":"10.1109\/CEC.2012.6252966"},{"key":"3_CR17","doi-asserted-by":"crossref","unstructured":"Kempka, M., Wydmuch, M., Runc, G., Toczek, J., Ja\u015bkowski, W.: ViZDoom: A Doom-based AI research platform for visual reinforcement learning. In: IEEE Conference on Computational Intelligence and Games, pp. 1\u20138 (2016)","DOI":"10.1109\/CIG.2016.7860433"},{"key":"3_CR18","doi-asserted-by":"publisher","first-page":"123","DOI":"10.1016\/j.swevo.2017.09.008","volume":"39","author":"S Khanchi","year":"2018","unstructured":"Khanchi, S., Vahdat, A., Heywood, M.I., Zincir-Heywood, A.N.: On botnet detection with genetic programming under streaming data label budgets and class imbalance. Swarm and Evolutionary Computation 39, 123\u2013140 (2018)","journal-title":"Swarm and Evolutionary Computation"},{"key":"3_CR19","doi-asserted-by":"publisher","first-page":"331","DOI":"10.1007\/s10710-008-9067-9","volume":"9","author":"P Lichodzijewski","year":"2008","unstructured":"Lichodzijewski, P., Heywood, M.I.: Coevolutionary bid-based genetic programming for problem decomposition in classification. Genetic Programming and Evolvable Machines 9, 331\u2013365 (2008)","journal-title":"Genetic Programming and Evolvable Machines"},{"key":"3_CR20","first-page":"363","volume":"GECCO-2008","author":"P Lichodzijewski","year":"2008","unstructured":"Lichodzijewski, P., Heywood, M.I.: Managing team-based problem solving with symbiotic bid-based genetic programming. In: ACM Genetic and Evolutionary Computation Conference (GECCO-2008), pp. 363\u2013370 (2008)","journal-title":"In: ACM Genetic and Evolutionary Computation Conference"},{"key":"3_CR21","first-page":"853","volume":"GECCO-2010","author":"P Lichodzijewski","year":"2010","unstructured":"Lichodzijewski, P., Heywood, M.I.: Symbiosis, complexification and simplicity under GP. In: Proceedings of the ACM Genetic and Evolutionary Computation Conference (GECCO-2010), pp. 853\u2013860 (2010)","journal-title":"In: Proceedings of the ACM Genetic and Evolutionary Computation Conference"},{"key":"3_CR22","doi-asserted-by":"crossref","unstructured":"Lichodzijewski, P., Heywood, M.I.: The Rubik\u2019s Cube and GP temporal sequence learning. In: R. Riolo, T. McConaghy, E. Vladislavleva (eds.) Genetic Programming Theory and Practice VIII, pp. 35\u201354. Springer (2011)","DOI":"10.1007\/978-1-4419-7747-2_3"},{"key":"3_CR23","doi-asserted-by":"publisher","first-page":"369","DOI":"10.1162\/evco.2007.15.3.369","volume":"15","author":"S Mabu","year":"2007","unstructured":"Mabu, S., Hirasawa, K., Hu, J.: A graph-based evolutionary algorithm: Genetic network programming and its extension using reinforcement learning. Evolutionary Computation 15, 369\u2013398 (2007)","journal-title":"Evolutionary Computation"},{"key":"3_CR24","doi-asserted-by":"publisher","first-page":"523","DOI":"10.1613\/jair.5699","volume":"61","author":"MC Machado","year":"2018","unstructured":"Machado, M.C., Bellemare, M.G., Talvitie, E., Veness, J., Hausknecht, M.J., Bowling, M.: Revisiting the arcade learning environment: Evaluation protocols and open problems for general agents. Journal of Artificial Intelligence Research 61, 523\u2013562 (2018)","journal-title":"Journal of Artificial Intelligence Research"},{"key":"3_CR25","unstructured":"Metzen, J.H., Edgington, M., Kassahun, Y., Kirchner, F.: Analysis of an evolutionary reinforcement learning method in multiagent domain. In: ACM International Conference on Autonomous Agents and Multiagent Systems, pp. 291\u2013298 (2008)"},{"key":"3_CR26","unstructured":"Miikkulainen, R., Liang, J.Z., Meyerson, E., Rawal, A., Fink, D., Francon, O., Raju, B., Shahrzad, H., Navruzyan, A., Duffy, N., Hodjat, B.: Evolving deep neural networks. CoRR abs\/1703.00548 (2017)"},{"key":"3_CR27","doi-asserted-by":"crossref","unstructured":"Miller, J.F., Thomson, P.: Cartesian genetic programming. In: European Conference on Genetic Programming 2000, Lecture Notes in Computer Science, vol. 1802, pp. 121\u2013132. Springer (2000)","DOI":"10.1007\/978-3-540-46239-2_9"},{"key":"3_CR28","doi-asserted-by":"publisher","first-page":"529","DOI":"10.1038\/nature14236","volume":"518","author":"V Mnih","year":"2015","unstructured":"Mnih, V., Kavukcuoglu, K., Silver, D., Rusu, A.A., Veness, J., Bellemare, M.G., Graves, A., Riedmiller, M., Fidjeland, A.K., Ostrovski, G., Petersen, S., Beattie, C., Sadik, A., Antonoglou, I., King, H., Kumaran, D., Wierstra, D., Legg, S., Hassabis, D.: Human-level control through deep reinforcement learning. Nature 518, 529\u2013533 (2015)","journal-title":"Nature"},{"key":"3_CR29","unstructured":"Salimans, T., Ho, J., Chen, X., Sutskever, I.: Evolution strategies as a scalable alternative to reinforcement learning. CoRR abs\/1703.03864 (2017)"},{"key":"3_CR30","first-page":"1009","volume":"GECCO-2017","author":"RJ Smith","year":"2017","unstructured":"Smith, R.J., Heywood, M.I.: Coevolving deep hierarchies of programs to solve complex tasks. In: ACM Genetic and Evolutionary Computation Conference (GECCO-2017), pp. 1009\u20131016 (2017)","journal-title":"In: ACM Genetic and Evolutionary Computation Conference"},{"key":"3_CR31","doi-asserted-by":"crossref","unstructured":"Smith, R.J., Heywood, M.I.: Scaling tangled program graphs to visual reinforcement learning in ViZDoom. In: European Conference on Genetic Programming 2018, Lecture Notes in Computer Science, vol. 10781, pp. 135\u2013150. Springer (2018)","DOI":"10.1007\/978-3-319-77553-1_9"},{"key":"3_CR32","doi-asserted-by":"crossref","unstructured":"Spector, L., McPhee, N.F.: Expressive genetic programming: concepts and applications. In: ACM Genetic and Evolutionary Computation Conference (Tutorial) (2016)","DOI":"10.1145\/2908961.2926988"},{"issue":"2","key":"3_CR33","doi-asserted-by":"publisher","first-page":"99","DOI":"10.1162\/106365602320169811","volume":"10","author":"Kenneth O. Stanley","year":"2002","unstructured":"Stanley, K.O., Miikkulainen, R.: Evolving neural networks through augmenting topologies. Evolutionary Computation 10 (2002)","journal-title":"Evolutionary Computation"},{"key":"3_CR34","unstructured":"Such, F.P., Madhavan, V., Conti, E., Lehman, J., Stanley, K.O., Clune, J.: Deep neuroevolution: Genetic algorithms are a competitive alternative for training deep neural networks for reinforcement learning. CoRR abs\/1712.06567 (2018)"},{"key":"3_CR35","volume-title":"Pado: A new learning architecture for object recognition","author":"A Teller","year":"1996","unstructured":"Teller, A., Veloso, M.: Pado: A new learning architecture for object recognition. In: Symbolic visual learning. Oxford University Press (1996)"},{"key":"3_CR36","first-page":"1708","volume":"GECCO-2007","author":"R Thomason","year":"2007","unstructured":"Thomason, R., Soule, T.: Novel ways of improving cooperation and performance in ensemble classifiers. In: ACM Genetic and Evolutionary Computation Conference (GECCO-2007), pp. 1708\u20131715 (2007)","journal-title":"In: ACM Genetic and Evolutionary Computation Conference"},{"key":"3_CR37","doi-asserted-by":"publisher","first-page":"135","DOI":"10.1007\/s12065-014-0115-5","volume":"7","author":"AJ Turner","year":"2014","unstructured":"Turner, A.J., Miller, J.F.: Neuroevolution: Evolving heterogeneous artificial neural networks. Evolutionary Intelligence 7, 135\u2013154 (2014)","journal-title":"Evolutionary Intelligence"},{"key":"3_CR38","doi-asserted-by":"crossref","unstructured":"Vahdat, A., Morgan, J., McIntyre, A.R., Heywood, M.I., Zincir-Heywood, A.N.: Evolving GP classifiers for streaming data tasks with concept change and label budgets: A benchmarking study. In: A.H. Gandomi, A.H. Alavi, C. Ryan (eds.) Handbook of Genetic Programming Applications, pp. 451\u2013480. Springer (2015)","DOI":"10.1007\/978-3-319-20883-1_18"},{"key":"3_CR39","first-page":"229","volume":"GECCO-2018","author":"DG Wilson","year":"2018","unstructured":"Wilson, D.G., Cussat-Blanc, S., Luga, H., Miller, J.F.: Evolving simple programs for playing Atari games. In: ACM Genetic and Evolutionary Computation Conference (GECCO-2018), pp. 229\u2013236 (2018)","journal-title":"In: ACM Genetic and Evolutionary Computation Conference"},{"key":"3_CR40","first-page":"1403","volume":"GECCO-2011","author":"SX Wu","year":"2011","unstructured":"Wu, S.X., Banzhaf, W.: Rethinking multilevel selection in genetic programming. In: ACM Genetic and Evolutionary Computation Conference (GECCO-2011), pp. 1403\u20131410 (2011)","journal-title":"In: ACM Genetic and Evolutionary Computation Conference"}],"container-title":["Genetic and Evolutionary Computation","Genetic Programming Theory and Practice XVI"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-030-04735-1_3","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2019,5,18]],"date-time":"2019-05-18T22:19:28Z","timestamp":1558217968000},"score":1,"resource":{"primary":{"URL":"http:\/\/link.springer.com\/10.1007\/978-3-030-04735-1_3"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2019]]},"ISBN":["9783030047344","9783030047351"],"references-count":40,"URL":"https:\/\/doi.org\/10.1007\/978-3-030-04735-1_3","relation":{},"ISSN":["1932-0167","1932-0175"],"issn-type":[{"type":"print","value":"1932-0167"},{"type":"electronic","value":"1932-0175"}],"subject":[],"published":{"date-parts":[[2019]]},"assertion":[{"value":"24 January 2019","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}}]}}