{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,28]],"date-time":"2026-04-28T07:13:36Z","timestamp":1777360416153,"version":"3.51.4"},"publisher-location":"New York, NY, USA","reference-count":68,"publisher":"ACM","license":[{"start":{"date-parts":[[2012,7,7]],"date-time":"2012-07-07T00:00:00Z","timestamp":1341619200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2012,7,7]]},"DOI":"10.1145\/2330163.2330273","type":"proceedings-article","created":{"date-parts":[[2012,7,10]],"date-time":"2012-07-10T16:41:12Z","timestamp":1341938472000},"page":"791-798","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":216,"title":["Genetic programming needs better benchmarks"],"prefix":"10.1145","author":[{"given":"James","family":"McDermott","sequence":"first","affiliation":[{"name":"Massachusetts Institute of Technology, Cambridge, MA, USA"}]},{"given":"David R.","family":"White","sequence":"additional","affiliation":[{"name":"University of Glasgow, Glasgow, United Kingdom"}]},{"given":"Sean","family":"Luke","sequence":"additional","affiliation":[{"name":"George Mason University, Fairfax, VA, USA"}]},{"given":"Luca","family":"Manzoni","sequence":"additional","affiliation":[{"name":"Universita degli Studi di Milano - Bicocca, Milan, Italy"}]},{"given":"Mauro","family":"Castelli","sequence":"additional","affiliation":[{"name":"Universita degli Studi di Milano - Bicocca, Milan, Italy"}]},{"given":"Leonardo","family":"Vanneschi","sequence":"additional","affiliation":[{"name":"Universidade Nova de Lisboa, Lisbon, Portugal"}]},{"given":"Wojciech","family":"Jaskowski","sequence":"additional","affiliation":[{"name":"Poznan University of Technology, Poznan, Poland"}]},{"given":"Krzysztof","family":"Krawiec","sequence":"additional","affiliation":[{"name":"Poznan University of Technology, Poznan, Poland"}]},{"given":"Robin","family":"Harper","sequence":"additional","affiliation":[{"name":"University of Sydney, Sydney, Australia"}]},{"given":"Kenneth","family":"De Jong","sequence":"additional","affiliation":[{"name":"George Mason University, Fairfax, VA, USA"}]},{"given":"Una-May","family":"O'Reilly","sequence":"additional","affiliation":[{"name":"Massachusetts Institute of Technology, Cambridge, MA, USA"}]}],"member":"320","published-online":{"date-parts":[[2012,7,7]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"publisher","DOI":"10.5555\/1763756.1763786"},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.5555\/1761486.1761488"},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10710-005-2990-0"},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.1109\/CEC.2011.5949748"},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1109\/ITW.2010.5593376"},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-01129-0_27"},{"key":"e_1_3_2_1_7_1","volume-title":"Proc","author":"Christensen S.","year":"2002","unstructured":"S. Christensen and F. Oppacher . An Analysis of Koza's Computational Effort Statistic for Genetic Programming . In Proc . EuroGP. Springer-Verlag , 2002 . S. Christensen and F. Oppacher. An Analysis of Koza's Computational Effort Statistic for Genetic Programming. In Proc. EuroGP. Springer-Verlag, 2002."},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.1109\/CEC.2006.1688330"},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.5555\/2008402.2008430"},{"key":"e_1_3_2_1_10_1","volume-title":"Proc. GECCO.","author":"Daida J. M.","year":"1999","unstructured":"J. M. Daida , Challenges with Verification, Repeatability, and Meaningful Comparison in Genetic Programming: Gibson's Magic . In Proc. GECCO. 1999 . J. M. Daida, et al. Challenges with Verification, Repeatability, and Meaningful Comparison in Genetic Programming: Gibson's Magic. In Proc. GECCO. 1999."},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.1023\/A:1011504414730"},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1109\/CEC.2006.1688631"},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.1080\/09528130903010295"},{"key":"e_1_3_2_1_15_1","volume-title":"Proc. IEEE Computational Intelligence and Games.","author":"Ebner M.","year":"2009","unstructured":"M. Ebner and T. Tiede . Evolving Driving Controllers using Genetic Programming . In Proc. IEEE Computational Intelligence and Games. 2009 . M. Ebner and T. Tiede. Evolving Driving Controllers using Genetic Programming. In Proc. IEEE Computational Intelligence and Games. 2009."},{"key":"e_1_3_2_1_16_1","volume-title":"Late Breaking Papers at GECCO.","author":"Feldt R.","year":"2000","unstructured":"R. Feldt , GP-Beagle: A benchmarking problem repository for the genetic programming community . In Late Breaking Papers at GECCO. 2000 . R. Feldt, et al. GP-Beagle: A benchmarking problem repository for the genetic programming community. In Late Breaking Papers at GECCO. 2000."},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-12239-2_17"},{"key":"e_1_3_2_1_18_1","volume-title":"Proc. GECCO.","author":"Gathercole C.","year":"1996","unstructured":"C. Gathercole and P. Ross . An Adverse Interaction between Crossover and Restricted Tree Depth in Genetic Programming . In Proc. GECCO. 1996 . C. Gathercole and P. Ross. An Adverse Interaction between Crossover and Restricted Tree Depth in Genetic Programming. In Proc. GECCO. 1996."},{"key":"e_1_3_2_1_19_1","volume-title":"http:\/\/tech.groups.yahoo.com\/group\/genetic\\textunderscoreprogramming\/message\/5410","author":"Mailing List Discussion Genetic Programming","year":"2012","unstructured":"Genetic Programming Mailing List Discussion . http:\/\/tech.groups.yahoo.com\/group\/genetic\\textunderscoreprogramming\/message\/5410 , 2012 . {Online: accessed 27-Jan-2012}. Genetic Programming Mailing List Discussion. http:\/\/tech.groups.yahoo.com\/group\/genetic\\textunderscoreprogramming\/message\/5410, 2012. {Online: accessed 27-Jan-2012}."},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.5555\/646806.706294"},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-540-31989-4_19"},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.dss.2006.04.004"},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10710-010-9114-1"},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-540-31989-4_11"},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.1145\/1143997.1144141"},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.1007\/11729976_7"},{"key":"e_1_3_2_1_27_1","volume-title":"Proc. Second Nordic Workshop on Genetic Algorithms","author":"J\u00e4ske H.","year":"1996","unstructured":"H. J\u00e4ske . Prediction of Sunspots by GP . In Proc. Second Nordic Workshop on Genetic Algorithms . Vaasa, Finland , 1996 . H. J\u00e4ske. Prediction of Sunspots by GP. In Proc. Second Nordic Workshop on Genetic Algorithms. Vaasa, Finland, 1996."},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"publisher","DOI":"10.5555\/1762668.1762676"},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICNN.1993.298674"},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-1-4614-1770-5_8"},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"publisher","DOI":"10.1109\/IJCNN.1992.227324"},{"key":"e_1_3_2_1_32_1","volume-title":"Genetic Programming: On the Programming of Computers by Means of Natural Selection","author":"Koza J. R.","year":"1992","unstructured":"J. R. Koza . Genetic Programming: On the Programming of Computers by Means of Natural Selection . MIT Press , 1992 . J. R. Koza. Genetic Programming: On the Programming of Computers by Means of Natural Selection. MIT Press, 1992."},{"key":"e_1_3_2_1_33_1","volume-title":"Genetic Programming II: Automatic Discovery of Reusable Programs","author":"Koza J. R.","year":"1994","unstructured":"J. R. Koza . Genetic Programming II: Automatic Discovery of Reusable Programs . MIT Press , 1994 . J. R. Koza. Genetic Programming II: Automatic Discovery of Reusable Programs. MIT Press, 1994."},{"key":"e_1_3_2_1_34_1","doi-asserted-by":"publisher","DOI":"10.1007\/s11047-007-9038-8"},{"issue":"4","key":"e_1_3_2_1_35_1","first-page":"285","volume":"15","author":"Langdon W. B.","year":"2005","unstructured":"W. B. Langdon and W. Banzhaf . Repeated Sequences in Linear Genetic Programming Genomes. Complex Systems , 15 ( 4 ): 285 -- 306 , 2005 . W. B. Langdon and W. Banzhaf. Repeated Sequences in Linear Genetic Programming Genomes. Complex Systems, 15(4):285--306, 2005.","journal-title":"Repeated Sequences in Linear Genetic Programming Genomes. Complex Systems"},{"key":"e_1_3_2_1_36_1","volume-title":"Proc. GECCO.","author":"Langdon W. B.","year":"1997","unstructured":"W. B. Langdon and R. Poli . An Analysis of the MAX problem in Genetic Programming . In Proc. GECCO. 1997 . W. B. Langdon and R. Poli. An Analysis of the MAX problem in Genetic Programming. In Proc. GECCO. 1997."},{"key":"e_1_3_2_1_37_1","doi-asserted-by":"publisher","DOI":"10.1145\/1569901.1570162"},{"key":"e_1_3_2_1_38_1","volume-title":"Automated DNA Motif Discovery. arXiv.org","author":"Langdon W. B.","year":"2010","unstructured":"W. B. Langdon , O. Sanchez Graillet , and A. P. Harrison . Automated DNA Motif Discovery. arXiv.org , 2010 . W. B. Langdon, O. Sanchez Graillet, and A. P. Harrison. Automated DNA Motif Discovery. arXiv.org, 2010."},{"key":"e_1_3_2_1_39_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-12148-7_14"},{"key":"e_1_3_2_1_40_1","volume-title":"http:\/\/algoval.essex.ac.uk:8080\/othello\/html\/Othello.html","author":"Lucas S.","year":"2012","unstructured":"S. Lucas . Othello Competition . http:\/\/algoval.essex.ac.uk:8080\/othello\/html\/Othello.html , 2012 . {Online; accessed 27-Jan-2012}. S. Lucas. Othello Competition. http:\/\/algoval.essex.ac.uk:8080\/othello\/html\/Othello.html, 2012. {Online; accessed 27-Jan-2012}."},{"key":"e_1_3_2_1_41_1","unstructured":"S. Lucas. The Physical Travelling Salesperson Problem. http:\/\/algoval.essex.ac.uk\/ptsp\/ptsp.html 2012. {Online: accessed 27-Jan-2012}.  S. Lucas. The Physical Travelling Salesperson Problem. http:\/\/algoval.essex.ac.uk\/ptsp\/ptsp.html 2012. {Online: accessed 27-Jan-2012}."},{"key":"e_1_3_2_1_42_1","unstructured":"S. Luke. ECJ 20: An Evolutionary Computation Library in Java. http:\/\/cs.gmu.edu\/~eclab\/projects\/ecj\/ 2012. {Online; accessed 27-Jan-2012}.  S. Luke. ECJ 20: An Evolutionary Computation Library in Java. http:\/\/cs.gmu.edu\/~eclab\/projects\/ecj\/ 2012. {Online; accessed 27-Jan-2012}."},{"key":"e_1_3_2_1_43_1","volume-title":"Proc. GECCO.","author":"Luke S.","year":"2002","unstructured":"S. Luke and L. Panait . Is The Perfect The Enemy Of The Good? In Proc. GECCO. 2002 . S. Luke and L. Panait. Is The Perfect The Enemy Of The Good? In Proc. GECCO. 2002."},{"key":"e_1_3_2_1_44_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-1-4614-1770-5_13"},{"key":"e_1_3_2_1_45_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10710-010-9121-2"},{"key":"e_1_3_2_1_46_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-12148-7_17"},{"key":"e_1_3_2_1_47_1","doi-asserted-by":"publisher","DOI":"10.5555\/1762668.1762684"},{"key":"e_1_3_2_1_48_1","doi-asserted-by":"publisher","DOI":"10.1109\/CEC.2008.4631308"},{"key":"e_1_3_2_1_49_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10710-010-9113-2"},{"key":"e_1_3_2_1_50_1","doi-asserted-by":"publisher","DOI":"10.1162\/evco.1997.5.4.401"},{"key":"e_1_3_2_1_51_1","volume-title":"Late Breaking Papers at GECCO.","author":"Paterson N.","year":"2000","unstructured":"N. Paterson and M. Livesey . Performance Comparison in Genetic Programming . In Late Breaking Papers at GECCO. 2000 . N. Paterson and M. Livesey. Performance Comparison in Genetic Programming. In Late Breaking Papers at GECCO. 2000."},{"key":"e_1_3_2_1_52_1","first-page":"299","volume-title":"Advances in Genetic Programming 2","author":"Punch B.","year":"1996","unstructured":"B. Punch , D. Zongker , and E. Goodman . The Royal Tree Problem, a Benchmark for Single and Multiple Population Genetic Programming . In Advances in Genetic Programming 2 , pp. 299 -- 316 . MIT Press , 1996 . B. Punch, D. Zongker, and E. Goodman. The Royal Tree Problem, a Benchmark for Single and Multiple Population Genetic Programming. In Advances in Genetic Programming 2, pp. 299--316. MIT Press, 1996."},{"key":"e_1_3_2_1_53_1","first-page":"153","volume-title":"Proc. EP","author":"Salomon R.","year":"1996","unstructured":"R. Salomon . Performance degradation of genetic algorithms under coordinate rotation . In Proc. EP , pp. 153 -- 161 . 1996 . R. Salomon. Performance degradation of genetic algorithms under coordinate rotation. In Proc. EP, pp. 153--161. 1996."},{"key":"e_1_3_2_1_55_1","doi-asserted-by":"publisher","DOI":"10.1109\/MCI.2006.329693"},{"key":"e_1_3_2_1_56_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10710-008-9075-9"},{"key":"e_1_3_2_1_57_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-13214-8_22"},{"key":"e_1_3_2_1_58_1","doi-asserted-by":"publisher","DOI":"10.1109\/TSMCC.2007.897326"},{"key":"e_1_3_2_1_59_1","doi-asserted-by":"publisher","DOI":"10.5555\/1719293.1719326"},{"key":"e_1_3_2_1_60_1","doi-asserted-by":"publisher","DOI":"10.1162\/1063656054088549"},{"key":"e_1_3_2_1_62_1","doi-asserted-by":"publisher","DOI":"10.1145\/2001576.2001773"},{"key":"e_1_3_2_1_63_1","doi-asserted-by":"publisher","DOI":"10.1145\/1569901.1570222"},{"key":"e_1_3_2_1_64_1","doi-asserted-by":"publisher","DOI":"10.1109\/TEVC.2008.926486"},{"key":"e_1_3_2_1_65_1","doi-asserted-by":"publisher","DOI":"10.1109\/TEVC.2006.882430"},{"key":"e_1_3_2_1_66_1","doi-asserted-by":"publisher","DOI":"10.5555\/1763756.1763776"},{"key":"e_1_3_2_1_67_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10710-009-9093-2"},{"key":"e_1_3_2_1_68_1","doi-asserted-by":"publisher","DOI":"10.1145\/1276958.1277294"},{"key":"e_1_3_2_1_69_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE.2009.5070536"},{"key":"e_1_3_2_1_70_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10710-009-9087-0"},{"key":"e_1_3_2_1_71_1","doi-asserted-by":"publisher","DOI":"10.1023\/A:1012926821302"}],"event":{"name":"GECCO '12: Genetic and Evolutionary Computation Conference","location":"Philadelphia Pennsylvania USA","acronym":"GECCO '12","sponsor":["SIGEVO ACM Special Interest Group on Genetic and Evolutionary Computation"]},"container-title":["Proceedings of the 14th annual conference on Genetic and evolutionary computation"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/2330163.2330273","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/2330163.2330273","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,18]],"date-time":"2025-06-18T09:20:55Z","timestamp":1750238455000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/2330163.2330273"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2012,7,7]]},"references-count":68,"alternative-id":["10.1145\/2330163.2330273","10.1145\/2330163"],"URL":"https:\/\/doi.org\/10.1145\/2330163.2330273","relation":{},"subject":[],"published":{"date-parts":[[2012,7,7]]},"assertion":[{"value":"2012-07-07","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}