{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,10]],"date-time":"2026-04-10T16:28:26Z","timestamp":1775838506924,"version":"3.50.1"},"reference-count":27,"publisher":"Springer Science and Business Media LLC","issue":"1","license":[{"start":{"date-parts":[[2017,12,1]],"date-time":"2017-12-01T00:00:00Z","timestamp":1512086400000},"content-version":"tdm","delay-in-days":0,"URL":"http:\/\/creativecommons.org\/licenses\/by\/4.0\/"},{"start":{"date-parts":[[2017,12,11]],"date-time":"2017-12-11T00:00:00Z","timestamp":1512950400000},"content-version":"vor","delay-in-days":10,"URL":"http:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"funder":[{"DOI":"10.13039\/100000002","name":"National Institutes of Health","doi-asserted-by":"crossref","award":["AI116794"],"award-info":[{"award-number":["AI116794"]}],"id":[{"id":"10.13039\/100000002","id-type":"DOI","asserted-by":"crossref"}]},{"DOI":"10.13039\/100000002","name":"National Institutes of Health","doi-asserted-by":"crossref","award":["DK112217"],"award-info":[{"award-number":["DK112217"]}],"id":[{"id":"10.13039\/100000002","id-type":"DOI","asserted-by":"crossref"}]},{"DOI":"10.13039\/100000002","name":"National Institutes of Health","doi-asserted-by":"crossref","award":["ES013508"],"award-info":[{"award-number":["ES013508"]}],"id":[{"id":"10.13039\/100000002","id-type":"DOI","asserted-by":"crossref"}]},{"DOI":"10.13039\/100000002","name":"National Institutes of Health","doi-asserted-by":"crossref","award":["EY022300"],"award-info":[{"award-number":["EY022300"]}],"id":[{"id":"10.13039\/100000002","id-type":"DOI","asserted-by":"crossref"}]},{"DOI":"10.13039\/100000002","name":"National Institutes of Health","doi-asserted-by":"crossref","award":["HL134015"],"award-info":[{"award-number":["HL134015"]}],"id":[{"id":"10.13039\/100000002","id-type":"DOI","asserted-by":"crossref"}]},{"DOI":"10.13039\/100000002","name":"National Institutes of Health","doi-asserted-by":"crossref","award":["LM009012"],"award-info":[{"award-number":["LM009012"]}],"id":[{"id":"10.13039\/100000002","id-type":"DOI","asserted-by":"crossref"}]},{"DOI":"10.13039\/100000002","name":"National Institutes of Health","doi-asserted-by":"crossref","award":["LM010098"],"award-info":[{"award-number":["LM010098"]}],"id":[{"id":"10.13039\/100000002","id-type":"DOI","asserted-by":"crossref"}]},{"DOI":"10.13039\/100000002","name":"National Institutes of Health","doi-asserted-by":"crossref","award":["LM011360"],"award-info":[{"award-number":["LM011360"]}],"id":[{"id":"10.13039\/100000002","id-type":"DOI","asserted-by":"crossref"}]},{"DOI":"10.13039\/100000002","name":"National Institutes of Health","doi-asserted-by":"crossref","award":["TR001263"],"award-info":[{"award-number":["TR001263"]}],"id":[{"id":"10.13039\/100000002","id-type":"DOI","asserted-by":"crossref"}]}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["BioData Mining"],"published-print":{"date-parts":[[2017,12]]},"DOI":"10.1186\/s13040-017-0154-4","type":"journal-article","created":{"date-parts":[[2017,12,11]],"date-time":"2017-12-11T12:42:15Z","timestamp":1512996135000},"update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":249,"title":["PMLB: a large benchmark suite for machine learning evaluation and comparison"],"prefix":"10.1186","volume":"10","author":[{"given":"Randal S.","family":"Olson","sequence":"first","affiliation":[]},{"given":"William","family":"La Cava","sequence":"additional","affiliation":[]},{"given":"Patryk","family":"Orzechowski","sequence":"additional","affiliation":[]},{"given":"Ryan J.","family":"Urbanowicz","sequence":"additional","affiliation":[]},{"given":"Jason H.","family":"Moore","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2017,12,11]]},"reference":[{"key":"154_CR1","doi-asserted-by":"publisher","DOI":"10.1007\/978-0-387-84858-7","volume-title":"The elements of statistical learning: data mining, inference, and prediction","author":"TJ Hastie","year":"2009","unstructured":"Hastie TJ, Tibshirani RJ, Friedman JH. The elements of statistical learning: data mining, inference, and prediction. New York: Springer; 2009."},{"key":"154_CR2","volume-title":"Proceedings of the 23rd International Conference on Machine Learning","author":"R Caruana","year":"2006","unstructured":"Caruana R, Niculescu-Mizil A. An empirical comparison of supervised learning algorithms. In: Proceedings of the 23rd International Conference on Machine Learning. Pittsburgh: ACM: 2006. p. 161\u20138. \n                    https:\/\/dl.acm.org\/citation.cfm?id=1143865\n                    \n                  ."},{"issue":"1","key":"154_CR3","doi-asserted-by":"publisher","first-page":"16","DOI":"10.1186\/1756-0381-5-16","volume":"5","author":"RJ Urbanowicz","year":"2012","unstructured":"Urbanowicz RJ, Kiralis J, Sinnott-Armstrong NA, Heberling T, Fisher JM, Moore JH. Gametes: a fast, direct algorithm for generating pure, strict, epistatic models with random architectures. BioData Min. 2012; 5(1):16.","journal-title":"BioData Min"},{"issue":"1","key":"154_CR4","doi-asserted-by":"publisher","first-page":"15","DOI":"10.1186\/1756-0381-5-15","volume":"5","author":"RJ Urbanowicz","year":"2012","unstructured":"Urbanowicz RJ, Kiralis J, Fisher JM, Moore JH. Predicting the difficulty of pure, strict, epistatic models: metrics for simulated model selection. BioData Min. 2012; 5(1):15.","journal-title":"BioData Min"},{"key":"154_CR5","doi-asserted-by":"publisher","first-page":"506","DOI":"10.1145\/792538.792543","volume":"50","author":"A Blum","year":"2003","unstructured":"Blum A, Kalai A, Wasserman H. Noise-tolerant Learning, the Parity Problem, and the Statistical Query Model. J ACM. 2003; 50:506\u201319. doi:\n                    10.1145\/792538.792543\n                    \n                  .","journal-title":"J ACM"},{"key":"154_CR6","volume-title":"Genetic programming: on the programming of computers by means of natural selection","author":"JR Koza","year":"1992","unstructured":"Koza JR. Genetic programming: on the programming of computers by means of natural selection. Cambridge: MIT Press; 1992."},{"issue":"3","key":"154_CR7","doi-asserted-by":"publisher","first-page":"339","DOI":"10.1007\/s10710-010-9113-2","volume":"11","author":"M O\u2019Neill","year":"2010","unstructured":"O\u2019Neill M, Vanneschi L, Gustafson S, Banzhaf W. Open issues in genetic programming. Genet Program Evolvable Mach. 2010; 11(3):339\u201363.","journal-title":"Genet Program Evolvable Mach"},{"key":"154_CR8","volume-title":"Proceedings of the 14th Annual Conference on Genetic and Evolutionary Computation. GECCO \u201912","author":"J McDermott","year":"2012","unstructured":"McDermott J, White DR, Luke S, Manzoni L, Castelli M, Vanneschi L, Jaskowski W, Krawiec K, Harper R, De Jong K, O\u2019Reilly UM. Genetic programming needs better benchmarks. In: Proceedings of the 14th Annual Conference on Genetic and Evolutionary Computation. GECCO \u201912. New York: ACM: 2012. p. 791\u20138."},{"issue":"1","key":"154_CR9","doi-asserted-by":"publisher","first-page":"3","DOI":"10.1007\/s10710-012-9177-2","volume":"14","author":"DR White","year":"2013","unstructured":"White DR, McDermott J, Castelli M, Manzoni L, Goldman BW, Kronberger G, Ja\u015bkowski W, O\u2019Reilly UM, Luke S. Better gp benchmarks: community survey results and proposals. Genet Program Evolvable Mach. 2013; 14(1):3\u201329.","journal-title":"Genet Program Evolvable Mach"},{"key":"154_CR10","doi-asserted-by":"publisher","first-page":"323","DOI":"10.1016\/j.neunet.2012.02.016","volume":"32","author":"J Stallkamp","year":"2012","unstructured":"Stallkamp J, Schlipsing M, Salmen J, Igel C. Man vs. computer: Benchmarking machine learning algorithms for traffic sign recognition. Neural Netw. 2012; 32:323\u201332.","journal-title":"Neural Netw"},{"issue":"2","key":"154_CR11","doi-asserted-by":"publisher","first-page":"252","DOI":"10.1016\/j.jtbi.2005.11.036","volume":"241","author":"JH Moore","year":"2006","unstructured":"Moore JH, Gilbert JC, Tsai CT, Chiang FT, Holden T, Barney N, White BC. A flexible computational framework for detecting, characterizing, and interpreting statistical patterns of epistasis in genetic studies of human disease susceptibility. J Theor Biol. 2006; 241(2):252\u201361.","journal-title":"J Theor Biol"},{"issue":"1","key":"154_CR12","doi-asserted-by":"publisher","first-page":"14","DOI":"10.1186\/s13040-016-0093-5","volume":"9","author":"J Li","year":"2016","unstructured":"Li J, Malley JD, Andrew AS, Karagas MR, Moore JH. Detecting gene-gene interactions using a permutation-based random forest method. BioData Min. 2016; 9(1):14.","journal-title":"BioData Min"},{"key":"154_CR13","doi-asserted-by":"publisher","first-page":"237","DOI":"10.1016\/j.ins.2013.08.059","volume":"261","author":"N Maci\u00e0","year":"2014","unstructured":"Maci\u00e0 N, Bernad\u00f3-Mansilla E. Towards UCI+: a mindful repository design. Inf Sci. 2014; 261:237\u201362.","journal-title":"Inf Sci"},{"key":"154_CR14","unstructured":"Lichman M. UCI machine learning repository. 2013. \n                    http:\/\/archive.ics.uci.edu\/ml\n                    \n                  . Accessed 21 Nov 2017."},{"key":"154_CR15","unstructured":"Goldbloom A. Kaggle: your homr for data science. \n                    http:\/\/www.kaggle.com\n                    \n                  . Accessed 15 Jan 2017."},{"issue":"2","key":"154_CR16","doi-asserted-by":"publisher","first-page":"49","DOI":"10.1145\/2641190.2641198","volume":"15","author":"J Vanschoren","year":"2014","unstructured":"Vanschoren J, Van Rijn JN, Bischl B, Torgo L. Openml: networked science in machine learning. ACM SIGKDD Explor Newsl. 2014; 15(2):49\u201360.","journal-title":"ACM SIGKDD Explor Newsl"},{"key":"154_CR17","unstructured":"Segal MR. Machine learning benchmarks and random forest regression: eScholarship Repository. University of California; 2004. \n                    http:\/\/repositories.cdlib.org\/cbmb\/bench_rf_regn\n                    \n                  ."},{"key":"154_CR18","doi-asserted-by":"crossref","unstructured":"Mu\u00f1oz Acosta MA, Villanova L, Baatar D, Smith-Miles K. Instance spaces for machine learning classification. Mach Learn. 2017. preprint. \n                    https:\/\/www.researchgate.net\/publication\/315835025_Instance_Spaces_for_Machine_Learning_Classification\n                    \n                  .","DOI":"10.1007\/s10994-017-5629-5"},{"key":"154_CR19","doi-asserted-by":"crossref","unstructured":"Reiss A, Stricker D. Creating and benchmarking a new dataset for physical activity monitoring. In: Proceedings of the 5th International Conference on PErvasive Technologies Related to Assistive Environments. Heraklion: ACM: 2012. p. 40. \n                    https:\/\/dl.acm.org\/citation.cfm?id=2413148\n                    \n                  .","DOI":"10.1145\/2413097.2413148"},{"key":"154_CR20","doi-asserted-by":"publisher","first-page":"292","DOI":"10.1016\/j.engappai.2016.07.004","volume":"55","author":"W La Cava","year":"2016","unstructured":"La Cava W, Danai K, Spector L. Inference of compact nonlinear dynamic models by epigenetic local search. Eng Appl Artif Intell. 2016; 55:292\u2013306.","journal-title":"Eng Appl Artif Intell"},{"issue":"2-3","key":"154_CR21","first-page":"255","volume":"17","author":"J Alcal\u00e1","year":"2010","unstructured":"Alcal\u00e1 J, Fern\u00e1ndez A, Luengo J, Derrac J, Garc\u00eda S, S\u00e1nchez L, Herrera F. Keel data-mining software tool: data set repository, integration of algorithms and experimental analysis framework. J Mult-Valued Log Soft Comput. 2010; 17(2-3):255\u201387.","journal-title":"J Mult-Valued Log Soft Comput"},{"key":"154_CR22","first-page":"2825","volume":"12","author":"F Pedregosa","year":"2011","unstructured":"Pedregosa F, Varoquaux G, Gramfort A, Michel V, Thirion B, Grisel O, Blondel M, Prettenhofer P, Weiss R, Dubourg V, Vanderplas J, Passos A, Cournapeau D, Brucher M, Perrot M, Duchesnay E. Scikit-learn: machine learning in Python. J Mach Learn Res. 2011; 12:2825\u201330.","journal-title":"J Mach Learn Res"},{"key":"154_CR23","unstructured":"Reif M. A comprehensive dataset for evaluating approaches of various meta-learning tasks. In: First International Conference on Pattern Recognition and Methods (ICPRAM). 2012. \n                    https:\/\/www.dfki.de\/web\/forschung\/publikationen\/renameFileForDownload?filename=ICPRAM_2012_59_CR.pdf&file_id=uploads_1357\n                    \n                  ."},{"key":"154_CR24","unstructured":"Pandas: Python data analysis library. \n                    http:\/\/pandas.pydata.org\/\n                    \n                  . Accessed 21 Nov 2017."},{"issue":"4","key":"154_CR25","doi-asserted-by":"publisher","first-page":"306","DOI":"10.1002\/gepi.20211","volume":"31","author":"DR Velez","year":"2007","unstructured":"Velez DR, et al. A balanced accuracy function for epistasis modeling in imbalanced datasets using multifactor dimensionality reduction. Genet Epidemiol. 2007; 31(4):306\u201315.","journal-title":"Genet Epidemiol"},{"issue":"2-3","key":"154_CR26","doi-asserted-by":"publisher","first-page":"89","DOI":"10.1007\/s12065-015-0128-8","volume":"8","author":"RJ Urbanowicz","year":"2015","unstructured":"Urbanowicz RJ, Moore JH. Exstracs 2.0: description and evaluation of a scalable learning classifier system. Evol Intel. 2015; 8(2-3):89\u2013116.","journal-title":"Evol Intel"},{"issue":"4","key":"154_CR27","doi-asserted-by":"publisher","first-page":"703","DOI":"10.1101\/gr.648603","volume":"13","author":"Y Kluger","year":"2003","unstructured":"Kluger Y, Basri R, Chang JT, Gerstein M. Spectral biclustering of microarray data: coclustering genes and conditions. Genome Res. 2003; 13(4):703\u201316.","journal-title":"Genome Res"}],"container-title":["BioData Mining"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1186\/s13040-017-0154-4.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/article\/10.1186\/s13040-017-0154-4\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1186\/s13040-017-0154-4.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2020,5,14]],"date-time":"2020-05-14T12:57:23Z","timestamp":1589461043000},"score":1,"resource":{"primary":{"URL":"https:\/\/biodatamining.biomedcentral.com\/articles\/10.1186\/s13040-017-0154-4"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2017,12]]},"references-count":27,"journal-issue":{"issue":"1","published-print":{"date-parts":[[2017,12]]}},"alternative-id":["154"],"URL":"https:\/\/doi.org\/10.1186\/s13040-017-0154-4","relation":{},"ISSN":["1756-0381"],"issn-type":[{"value":"1756-0381","type":"electronic"}],"subject":[],"published":{"date-parts":[[2017,12]]},"assertion":[{"value":"9 March 2017","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"7 November 2017","order":2,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"11 December 2017","order":3,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"Not applicable. All data used in this study was publicly available online and does not contain private information about any particular individual.","order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Ethics approval and consent to participate"}},{"value":"Not applicable","order":2,"name":"Ethics","group":{"name":"EthicsHeading","label":"Consent for publication"}},{"value":"The authors declare that they have no competing interests.","order":3,"name":"Ethics","group":{"name":"EthicsHeading","label":"Competing interests"}},{"value":"Springer Nature remains neutral with regard to jurisdictional claims in published maps and institutional affiliations.","order":4,"name":"Ethics","group":{"name":"EthicsHeading","label":"Publisher\u2019s Note"}}],"article-number":"36"}}