{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,23]],"date-time":"2026-07-23T05:05:07Z","timestamp":1784783107102,"version":"3.55.0"},"reference-count":12,"publisher":"Oxford University Press (OUP)","issue":"3","license":[{"start":{"date-parts":[[2021,10,22]],"date-time":"2021-10-22T00:00:00Z","timestamp":1634860800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"funder":[{"DOI":"10.13039\/100000002","name":"National Institutes of Health","doi-asserted-by":"publisher","award":["R01-AI116794"],"award-info":[{"award-number":["R01-AI116794"]}],"id":[{"id":"10.13039\/100000002","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000002","name":"National Institutes of Health","doi-asserted-by":"publisher","award":["R01-LM010098"],"award-info":[{"award-number":["R01-LM010098"]}],"id":[{"id":"10.13039\/100000002","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000002","name":"National Institutes of Health","doi-asserted-by":"publisher","award":["R01-LM012601"],"award-info":[{"award-number":["R01-LM012601"]}],"id":[{"id":"10.13039\/100000002","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000002","name":"National Institutes of Health","doi-asserted-by":"publisher","award":["T32-ES019851"],"award-info":[{"award-number":["T32-ES019851"]}],"id":[{"id":"10.13039\/100000002","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000002","name":"National Institutes of Health","doi-asserted-by":"publisher","award":["K99-LM012926"],"award-info":[{"award-number":["K99-LM012926"]}],"id":[{"id":"10.13039\/100000002","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000002","name":"National Institutes of Health","doi-asserted-by":"publisher","award":["K99-LM012646"],"award-info":[{"award-number":["K99-LM012646"]}],"id":[{"id":"10.13039\/100000002","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2022,1,12]]},"abstract":"<jats:title>Abstract<\/jats:title>\n               <jats:sec>\n                  <jats:title>Motivation<\/jats:title>\n                  <jats:p>Novel machine learning and statistical modeling studies rely on standardized comparisons to existing methods using well-studied benchmark datasets. Few tools exist that provide rapid access to many of these datasets through a standardized, user-friendly interface that integrates well with popular data science workflows.<\/jats:p>\n               <\/jats:sec>\n               <jats:sec>\n                  <jats:title>Results<\/jats:title>\n                  <jats:p>This release of PMLB (Penn Machine Learning Benchmarks) provides the largest collection of diverse, public benchmark datasets for evaluating new machine learning and data science methods aggregated in one location. v1.0 introduces a number of critical improvements developed following discussions with the open-source community.<\/jats:p>\n               <\/jats:sec>\n               <jats:sec>\n                  <jats:title>Availability and implementation<\/jats:title>\n                  <jats:p>PMLB is available at https:\/\/github.com\/EpistasisLab\/pmlb. Python and R interfaces for PMLB can be installed through the Python Package Index and Comprehensive R Archive Network, respectively.<\/jats:p>\n               <\/jats:sec>","DOI":"10.1093\/bioinformatics\/btab727","type":"journal-article","created":{"date-parts":[[2021,10,19]],"date-time":"2021-10-19T13:23:38Z","timestamp":1634649818000},"page":"878-880","source":"Crossref","is-referenced-by-count":63,"title":["PMLB v1.0: an open-source dataset collection for benchmarking machine learning methods"],"prefix":"10.1093","volume":"38","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-7999-4399","authenticated-orcid":false,"given":"Joseph D","family":"Romano","sequence":"first","affiliation":[{"name":"Institute for Biomedical Informatics, University of Pennsylvania , Philadelphia, PA 19104, USA"},{"name":"Center of Excellence in Environmental Toxicology, University of Pennsylvania , Philadelphia, PA 19104, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Trang T","family":"Le","sequence":"additional","affiliation":[{"name":"Institute for Biomedical Informatics, University of Pennsylvania , Philadelphia, PA 19104, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-1332-2960","authenticated-orcid":false,"given":"William","family":"La Cava","sequence":"additional","affiliation":[{"name":"Institute for Biomedical Informatics, University of Pennsylvania , Philadelphia, PA 19104, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"John T","family":"Gregg","sequence":"additional","affiliation":[{"name":"Institute for Biomedical Informatics, University of Pennsylvania , Philadelphia, PA 19104, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Daniel J","family":"Goldberg","sequence":"additional","affiliation":[{"name":"Department of Computer Science & Engineering, Washington University in St. Louis , St. Louis, MO 63130, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Praneel","family":"Chakraborty","sequence":"additional","affiliation":[{"name":"School of Arts and Sciences, University of Pennsylvania , Philadelphia, PA 19104, USA"},{"name":"Wharton School, University of Pennsylvania , Philadelphia, PA 19104, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Natasha L","family":"Ray","sequence":"additional","affiliation":[{"name":"Princeton Day School , Princeton, NJ 08540, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-3012-7446","authenticated-orcid":false,"given":"Daniel","family":"Himmelstein","sequence":"additional","affiliation":[{"name":"Related Sciences , Denver, CO 80220, USA"},{"name":"Department of Systems Pharmacology & Translational Therapeutics, University of Pennsylvania , Philadelphia, PA 19104, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Weixuan","family":"Fu","sequence":"additional","affiliation":[{"name":"Institute for Biomedical Informatics, University of Pennsylvania , Philadelphia, PA 19104, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-5015-1099","authenticated-orcid":false,"given":"Jason H","family":"Moore","sequence":"additional","affiliation":[{"name":"Institute for Biomedical Informatics, University of Pennsylvania , Philadelphia, PA 19104, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"286","published-online":{"date-parts":[[2021,10,22]]},"reference":[{"key":"2023020108482484500_btab727-B1","first-page":"161","author":"Caruana","year":"2006"},{"key":"2023020108482484500_btab727-B2","first-page":"57","article-title":"Limits on learning machine accuracy imposed by data quality","author":"Cortes","year":"1995"},{"key":"2023020108482484500_btab727-B3"},{"key":"2023020108482484500_btab727-B4","volume-title":"The Elements of Statistical Learning","author":"Friedman","year":"2001"},{"key":"2023020108482484500_btab727-B5","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1038\/s41467-019-09406-4","article-title":"Systematic benchmarking of omics computational tools","volume":"10","author":"Mangul","year":"2019","journal-title":"Nat. Commun"},{"key":"2023020108482484500_btab727-B6","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1186\/s13059-020-01988-3","article-title":"Benchmarking of computational error-correction methods for next-generation sequencing data","volume":"21","author":"Mitchell","year":"2020","journal-title":"Genome Biol"},{"key":"2023020108482484500_btab727-B7","doi-asserted-by":"crossref","first-page":"83","DOI":"10.1186\/2193-1801-3-83","article-title":"Benchmarking network for clinical and humanistic outcomes in diabetes (bench-d) study: protocol, tools, and population","volume":"3","author":"Nicolucci","year":"2014","journal-title":"Springerplus"},{"key":"2023020108482484500_btab727-B8","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1186\/s13040-017-0154-4","article-title":"Pmlb: a large benchmark suite for machine learning evaluation and comparison","volume":"10","author":"Olson","year":"2017","journal-title":"BioData Min"},{"key":"2023020108482484500_btab727-B9","first-page":"2825","article-title":"Scikit-learn: machine learning in Python","volume":"12","author":"Pedregosa","year":"2011","journal-title":"J. Mach. Learn. Res"},{"key":"2023020108482484500_btab727-B10","first-page":"263","article-title":"Foundations of json schema","author":"Pezoa","year":"2016"},{"key":"2023020108482484500_btab727-B11","doi-asserted-by":"crossref","first-page":"323","DOI":"10.1016\/j.neunet.2012.02.016","article-title":"Man vs. computer: benchmarking machine learning algorithms for traffic sign recognition","volume":"32","author":"Stallkamp","year":"2012","journal-title":"Neural Netw"},{"key":"2023020108482484500_btab727-B12","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1186\/s13059-019-1738-8","article-title":"Essential guidelines for computational method benchmarking","volume":"20","author":"Weber","year":"2019","journal-title":"Genome Biol"}],"container-title":["Bioinformatics"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/academic.oup.com\/bioinformatics\/advance-article-pdf\/doi\/10.1093\/bioinformatics\/btab727\/41048448\/btab727.pdf","content-type":"application\/pdf","content-version":"am","intended-application":"syndication"},{"URL":"https:\/\/academic.oup.com\/bioinformatics\/article-pdf\/38\/3\/878\/49007845\/btab727.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"syndication"},{"URL":"https:\/\/academic.oup.com\/bioinformatics\/article-pdf\/38\/3\/878\/49007845\/btab727.pdf","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2023,2,1]],"date-time":"2023-02-01T20:06:16Z","timestamp":1675281976000},"score":1,"resource":{"primary":{"URL":"https:\/\/academic.oup.com\/bioinformatics\/article\/38\/3\/878\/6408434"}},"subtitle":[],"editor":[{"given":"Janet","family":"Kelso","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"editor"}]}],"short-title":[],"issued":{"date-parts":[[2021,10,22]]},"references-count":12,"journal-issue":{"issue":"3","published-print":{"date-parts":[[2022,1,12]]}},"URL":"https:\/\/doi.org\/10.1093\/bioinformatics\/btab727","relation":{},"ISSN":["1367-4803","1367-4811"],"issn-type":[{"value":"1367-4803","type":"print"},{"value":"1367-4811","type":"electronic"}],"subject":[],"published-other":{"date-parts":[[2022,2,1]]},"published":{"date-parts":[[2021,10,22]]}}}