{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,28]],"date-time":"2026-10-28T01:34:00Z","timestamp":1793151240573,"version":"4.5.0"},"reference-count":67,"publisher":"Frontiers Media SA","license":[{"start":{"date-parts":[[2022,9,27]],"date-time":"2022-09-27T00:00:00Z","timestamp":1664236800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"content-domain":{"domain":["frontiersin.org"],"crossmark-restriction":true},"short-container-title":["Front. Big Data"],"DOI":"10.3389\/fdata.2022.979465","type":"journal-article","created":{"date-parts":[[2022,9,27]],"date-time":"2022-09-27T08:06:00Z","timestamp":1664265960000},"update-policy":"https:\/\/doi.org\/10.3389\/crossmark-policy","source":"Crossref","is-referenced-by-count":13,"title":["The ABC recommendations for validation of supervised machine learning results in biomedical sciences"],"prefix":"10.3389","volume":"5","author":[{"given":"Davide","family":"Chicco","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Giuseppe","family":"Jurman","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"1965","published-online":{"date-parts":[[2022,9,27]]},"reference":[{"key":"B1","doi-asserted-by":"publisher","first-page":"btab852","DOI":"10.1093\/bioinformatics\/btab852","article-title":"geoCancerPrognosticDatasetsRetriever, a bioinformatics tool to easily identify cancer prognostic datasets on Gene Expression Omnibus (GEO)","volume":"2021","author":"Alameer","year":"2022","journal-title":"Bioinformatics"},{"key":"B2","doi-asserted-by":"publisher","first-page":"505","DOI":"10.1038\/s41557-021-00716-z","article-title":"Best practices in machine learning for chemistry","volume":"13","author":"Artrith","year":"2021","journal-title":"Nat. Chem."},{"key":"B3","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1186\/s41512-021-00114-6","article-title":"Graphical calibration curves and the integrated calibration index (ICI) for competing risk models","volume":"6","author":"Austin","year":"2022","journal-title":"Diagn. Progn. Res."},{"key":"B4","doi-asserted-by":"publisher","first-page":"6","DOI":"10.1038\/s41562-017-0189-z","article-title":"Redefine statistical significance","volume":"2","author":"Benjamin","year":"2018","journal-title":"Nat. Hum. Behav."},{"key":"B5","doi-asserted-by":"publisher","first-page":"1104","DOI":"10.1016\/j.compbiomed.2005.09.002","article-title":"Machine learning in bioinformatics: a brief survey and recommendations for practitioners","volume":"36","author":"Bhaskar","year":"2006","journal-title":"Comput. Biol. Med."},{"key":"B6","unstructured":"BionetworksS.\n          Breast Cancer Prognosis DREAM Education Challenge2021"},{"key":"B7","doi-asserted-by":"publisher","first-page":"l737","DOI":"10.1136\/bmj.l737","article-title":"Guide to presenting clinical prediction models for use in clinical settings","volume":"365","author":"Bonnett","year":"2019","journal-title":"BMJ"},{"key":"B8","doi-asserted-by":"publisher","first-page":"R23","DOI":"10.1186\/gb-2003-4-2-p1","article-title":"The GRID: the general repository for interaction datasets","volume":"4","author":"Breitkreutz","year":"2003","journal-title":"Genome Biol."},{"key":"B9","first-page":"167","article-title":"AI slipping on tiles: data leakage in digital pathology,","volume-title":"Proceedings of ICPR 2021 \u2013 The 25th International Conference on Pattern Recognition. ICPR International Workshops and Challenges","author":"Bussola","year":"2021"},{"key":"B10","doi-asserted-by":"publisher","first-page":"104510","DOI":"10.1016\/j.ijmedinf.2021.104510","article-title":"The need to separate the wheat from the chaff in medical informatics: introducing a comprehensive checklist for the (self)-assessment of medical AI studies","volume":"153","author":"Cabitza","year":"2021","journal-title":"Int. J. Med. Inform."},{"key":"B11","doi-asserted-by":"publisher","first-page":"271","DOI":"10.1038\/s41398-019-0607-2","article-title":"Recommendations and future directions for supervised machine learning in psychiatry","volume":"9","author":"Cearns","year":"2019","journal-title":"Transl. Psychiatry"},{"key":"B12","doi-asserted-by":"publisher","first-page":"e17238","DOI":"10.1371\/journal.pone.0017238","article-title":"Removing batch effects in analysis of expression microarray data: an evaluation of six batch adjustment methods","volume":"6","author":"Chen","year":"2011","journal-title":"PLoS ONE"},{"key":"B13","doi-asserted-by":"publisher","first-page":"35","DOI":"10.1186\/s13040-017-0155-3","article-title":"Ten quick tips for machine learning in computational biology","volume":"10","author":"Chicco","year":"2017","journal-title":"BioData Min."},{"key":"B14","first-page":"1","article-title":"A survey on publicly available open datasets of electronic health records (EHRs) of patients with neuroblastoma","author":"Chicco","year":"","journal-title":"Data Sci. J"},{"key":"B15","doi-asserted-by":"publisher","first-page":"6","DOI":"10.1186\/s12864-019-6413-7","article-title":"The advantages of the Matthews correlation coefficient (MCC) over F1 score and accuracy in binary classification evaluation","volume":"21","author":"Chicco","year":"2020","journal-title":"BMC Genomics"},{"key":"B16","doi-asserted-by":"publisher","first-page":"876814","DOI":"10.3389\/frobt.2022.876814","article-title":"An invitation to greater use of Matthews correlation coefficient in robotics and artificial intelligence","volume":"9","author":"Chicco","year":"2022","journal-title":"Front. Robot. AI"},{"key":"B17","doi-asserted-by":"publisher","first-page":"47112","DOI":"10.1109\/ACCESS.2021.3068614","article-title":"The benefits of the Matthews correlation coefficient (MCC) over the diagnostic odds ratio (DOR) in binary classification assessment","volume":"9","author":"Chicco","year":"2021","journal-title":"IEEE Access."},{"key":"B18","doi-asserted-by":"publisher","first-page":"13","DOI":"10.1186\/s13040-021-00244-z","article-title":"The Matthews correlation coefficient (MCC) is more reliable than balanced accuracy, bookmaker informedness, and markedness in two-class confusion matrix evaluation","volume":"14","author":"Chicco","year":"","journal-title":"BioData Min."},{"key":"B19","doi-asserted-by":"publisher","first-page":"e623","DOI":"10.7717\/peerj-cs.623","article-title":"The coefficient of determination R-squared is more informative than SMAPE, MAE, MAPE, MSE and RMSE in regression analysis evaluation","volume":"7","author":"Chicco","year":"","journal-title":"PeerJ Comput. Sci."},{"key":"B20","doi-asserted-by":"publisher","first-page":"78368","DOI":"10.1109\/ACCESS.2021.3084050","article-title":"The Matthews correlation coefficient (MCC) is more informative than Cohens Kappa and Brier score in binary classification assessment","volume":"9","author":"Chicco","year":"","journal-title":"IEEE Access."},{"key":"B21","doi-asserted-by":"publisher","first-page":"1045","DOI":"10.1007\/s10278-013-9622-7","article-title":"The Cancer Imaging Archive (TCIA): maintaining and operating a public information repository","volume":"26","author":"Clark","year":"2013","journal-title":"J. Digit. Imaging"},{"key":"B22","doi-asserted-by":"publisher","first-page":"1692","DOI":"10.1177\/0962280213497434","article-title":"Assessing calibration of prognostic risk scores","volume":"25","author":"Crowson","year":"2016","journal-title":"Stat. Methods Med. Res."},{"key":"B23","doi-asserted-by":"publisher","DOI":"10.48550\/arXiv.2011.03395","article-title":"Underspecification presents challenges for credibility in modern machine learning","author":"D'Amour","year":"2020","journal-title":"arXiv Preprint arXiv:2011.03395"},{"key":"B24","doi-asserted-by":"publisher","first-page":"326","DOI":"10.1145\/212094.212114","article-title":"Overfitting and undercomputing in machine learning","volume":"27","author":"Dietterich","year":"1995","journal-title":"ACM Comput. Surveys"},{"key":"B25","doi-asserted-by":"publisher","first-page":"78","DOI":"10.1145\/2347736.2347755","article-title":"A few useful things to know about machine learning","volume":"55","author":"Domingos","year":"2012","journal-title":"Commun. ACM"},{"key":"B26","unstructured":"Overhead Geopose Challenge2022"},{"key":"B27","doi-asserted-by":"publisher","first-page":"207","DOI":"10.1093\/nar\/30.1.207","article-title":"Gene Expression Omnibus: NCBI gene expression and hybridization array data repository","volume":"30","author":"Edgar","year":"2002","journal-title":"Nucl. Acids Res."},{"key":"B28","doi-asserted-by":"crossref","first-page":"569","DOI":"10.1007\/978-1-4612-4380-9_41","article-title":"Bootstrap methods: another look at the jackknife,","volume-title":"Breakthroughs in Statistics","author":"Efron","year":"1992"},{"key":"B29","doi-asserted-by":"crossref","DOI":"10.1201\/9780429246593","volume-title":"An Introduction to the Bootstrap","author":"Efron","year":"1994"},{"key":"B30","doi-asserted-by":"crossref","first-page":"3","DOI":"10.1007\/978-3-030-05318-5_1","article-title":"Hyperparameter optimization,","volume-title":"Automated Machine Learning","author":"Feurer","year":"2019"},{"key":"B31","doi-asserted-by":"publisher","first-page":"283","DOI":"10.1056\/NEJMc2104626","article-title":"The clinician and dataset shift in artificial intelligence","volume":"385","author":"Finlayson","year":"2021","journal-title":"N. Engl. J. Med."},{"key":"B32","doi-asserted-by":"publisher","first-page":"D330","DOI":"10.1093\/nar\/gky1055","article-title":"The Gene Ontology resource: 20 years and still GOing strong","volume":"47","year":"2019","journal-title":"Nucl. Acids Res."},{"key":"B33","doi-asserted-by":"publisher","first-page":"e124","DOI":"10.1371\/journal.pmed.0020124","article-title":"Why most published research findings are false","volume":"2","author":"Ioannidis","year":"2005","journal-title":"PLOS Med."},{"key":"B34","doi-asserted-by":"publisher","first-page":"52","DOI":"10.1145\/846183.846195","article-title":"Data snooping, dredging and fishing: the dark side of data mining a SIGKDD99 panel report","volume":"1","author":"Jensen","year":"2000","journal-title":"ACM SIGKDD Explor. Newsl."},{"key":"B35","unstructured":"Kaggle.com \u2013 Find Open Datasets2022"},{"key":"B36","doi-asserted-by":"publisher","first-page":"e51","DOI":"10.1016\/S2589-7500(20)30240-5","article-title":"A global review of publicly available datasets for ophthalmological imaging: barriers to access, usability, and generalisability","volume":"3","author":"Khan","year":"2021","journal-title":"Lancet Digit. Health"},{"key":"B37","doi-asserted-by":"publisher","first-page":"690","DOI":"10.1038\/s41598-018-36873-4","article-title":"Stratification of amyotrophic lateral sclerosis patients: a crowdsourcing approach","volume":"9","author":"Kueffner","year":"2019","journal-title":"Sci. Reports"},{"key":"B38","doi-asserted-by":"publisher","first-page":"50","DOI":"10.1109\/TCBB.2007.70267","article-title":"Data-fusion in clustering microarray data: balancing discovery and interpretability","volume":"7","author":"Kustra","year":"2008","journal-title":"IEEE\/ACM Trans. Comput. Biol. Bioinform."},{"key":"B39","doi-asserted-by":"publisher","first-page":"86","DOI":"10.1093\/bib\/bbk007","article-title":"Machine learning in bioinformatics","volume":"7","author":"Larra\u00f1aga","year":"2006","journal-title":"Brief. Bioinform."},{"key":"B40","doi-asserted-by":"publisher","first-page":"413","DOI":"10.1016\/j.jacr.2020.09.060","article-title":"Regulatory frameworks for development and evaluation of artificial intelligence\u2013based diagnostic imaging algorithms: summary and recommendations","volume":"18","author":"Larson","year":"2021","journal-title":"J. Amer. Coll. Radiol."},{"key":"B41","doi-asserted-by":"publisher","first-page":"e1009803","DOI":"10.1371\/journal.pcbi.1009803","article-title":"Ten quick tips for deep learning in biology","volume":"18","author":"Lee","year":"2022","journal-title":"PLoS Comput. Biol."},{"key":"B42","doi-asserted-by":"publisher","first-page":"e48175","DOI":"10.7554\/eLife.48175.005","article-title":"Science forum: ten common statistical mistakes to watch out for when writing or reviewing a manuscript","volume":"8","author":"Makin","year":"2019","journal-title":"eLife"},{"key":"B43","doi-asserted-by":"publisher","first-page":"827","DOI":"10.1038\/nbt.1665","article-title":"The MicroArray quality control (MAQC)-II study of common practices for the development and validation of microarray-based predictive models","volume":"28","year":"2010","journal-title":"Nat. Biotechnol."},{"key":"B44","doi-asserted-by":"publisher","first-page":"442","DOI":"10.1016\/0005-2795(75)90109-9","article-title":"Comparison of the predicted and observed secondary structure of T4 phage lysozyme","volume":"405","author":"Matthews","year":"1975","journal-title":"Biochim. Biophys. Acta Prot. Struct."},{"key":"B45","doi-asserted-by":"publisher","first-page":"n2281","DOI":"10.1136\/bmj.n2281","article-title":"Risk of bias in studies on prediction models developed using supervised machine learning techniques: systematic review","volume":"375","author":"Navarro","year":"2021","journal-title":"BMJ"},{"key":"B46","doi-asserted-by":"publisher","first-page":"140","DOI":"10.1080\/00031305.1990.10475704","article-title":"Data splitting","volume":"44","author":"Picard","year":"1990","journal-title":"Amer. Stat."},{"key":"B47","doi-asserted-by":"publisher","first-page":"4230","DOI":"10.1002\/sim.9025","article-title":"Minimum sample size for external validation of a clinical prediction model with a binary outcome","volume":"40","author":"Riley","year":"2021","journal-title":"Stat. Med."},{"key":"B48","doi-asserted-by":"publisher","first-page":"i3140","DOI":"10.1136\/bmj.i3140","article-title":"External validation of clinical prediction models using big datasets from e-health records or IPD meta-analysis: opportunities and challenges","volume":"353","author":"Riley","year":"2016","journal-title":"BMJ"},{"key":"B49","unstructured":"DREAM Challenges Publications2022"},{"key":"B50","doi-asserted-by":"publisher","first-page":"903","DOI":"10.1038\/nbt.2957","article-title":"A comprehensive assessment of RNA-seq accuracy, reproducibility and information content by the sequencing quality control consortium","volume":"32","year":"2014","journal-title":"Nat. Biotechnol."},{"key":"B51","unstructured":"SewellM.\n          Data Snooping2021"},{"key":"B52","doi-asserted-by":"publisher","first-page":"106","DOI":"10.1002\/ehf2.13073","article-title":"Machine learning vs. conventional statistical models for predicting heart failure readmission and mortality","volume":"8","author":"Shin","year":"2021","journal-title":"ESC Heart Fail."},{"key":"B53","doi-asserted-by":"publisher","first-page":"078816","DOI":"10.1101\/078816","article-title":"I tried a bunch of things: the dangers of unexpected overfitting in classification","volume":"2016","author":"Skocik","year":"2016","journal-title":"bioRxiv"},{"key":"B54","unstructured":"SmithM. K.\n          Data snooping2021"},{"key":"B55","doi-asserted-by":"publisher","first-page":"e006556","DOI":"10.1161\/CIRCOUTCOMES.120.006556","article-title":"Recommendations for reporting machine learning analyses in clinical research","volume":"13","author":"Stevens","year":"2020","journal-title":"Circ. Cardiovasc. Qual. Outcomes"},{"key":"B56","doi-asserted-by":"publisher","first-page":"1925","DOI":"10.1093\/eurheartj\/ehu207","article-title":"Towards better clinical prediction models: seven steps for development and an ABCD for validation","volume":"35","author":"Steyerberg","year":"2014","journal-title":"Eur. Heart J."},{"key":"B57","doi-asserted-by":"publisher","first-page":"e116","DOI":"10.1371\/journal.pcbi.0030116","article-title":"Machine learning and its applications to biology","volume":"3","author":"Tarca","year":"2007","journal-title":"PLoS Comput. Biol."},{"key":"B58","unstructured":"Machine Learning Repository1987"},{"key":"B59","doi-asserted-by":"publisher","first-page":"219","DOI":"10.1016\/j.jclinepi.2021.05.018","article-title":"Methodology over metrics: current scientific standards are a disservice to patients and society","volume":"138","author":"Van Calster","year":"2021","journal-title":"J. Clin. Epidemiol."},{"key":"B60","doi-asserted-by":"publisher","first-page":"3837","DOI":"10.5555\/2946645.3007063","article-title":"Are random forests truly the best classifiers?","volume":"17","author":"Wainberg","year":"2016","journal-title":"J. Mach. Learn. Res."},{"key":"B61","doi-asserted-by":"publisher","first-page":"1122","DOI":"10.1038\/s41592-021-01205-4","article-title":"DOME: recommendations for supervised machine learning validation in biology","volume":"5","author":"Walsh","year":"2021","journal-title":"Nat. Methods"},{"key":"B62","doi-asserted-by":"publisher","first-page":"169","DOI":"10.1038\/s41576-021-00434-9","article-title":"Navigating the pitfalls of applying machine learning in genomics","volume":"23","author":"Whalen","year":"2021","journal-title":"Nat. Rev. Genet."},{"key":"B63","doi-asserted-by":"publisher","first-page":"1097","DOI":"10.1111\/1468-0262.00152","article-title":"A reality check for data snooping","volume":"68","author":"White","year":"2000","journal-title":"Econometrica"},{"key":"B64","doi-asserted-by":"publisher","first-page":"51","DOI":"10.7326\/M18-1376","article-title":"PROBAST: a tool to assess the risk of bias and applicability of prediction model studies","volume":"170","author":"Wolff","year":"2019","journal-title":"Ann. Intern. Med."},{"key":"B65","first-page":"557","article-title":"Correlation and causation","author":"Wright","year":"1921","journal-title":"J. Agric. Res."},{"key":"B66","first-page":"78","article-title":"Analysis of k-fold cross-validation over hold-out validation on colossal datasets for quality classification,","volume-title":"Proceedings of IACC 2016\u2014the 6th International Conference on Advanced Computing","author":"Yadav","year":"2016"},{"key":"B67","doi-asserted-by":"publisher","first-page":"133","DOI":"10.1186\/s13059-015-0694-1","article-title":"Comparison of RNA-seq and microarray-based models for clinical endpoint prediction","volume":"16","author":"Zhang","year":"2015","journal-title":"Genome Biol."}],"container-title":["Frontiers in Big Data"],"original-title":[],"link":[{"URL":"https:\/\/www.frontiersin.org\/articles\/10.3389\/fdata.2022.979465\/full","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2022,9,27]],"date-time":"2022-09-27T08:06:10Z","timestamp":1664265970000},"score":1,"resource":{"primary":{"URL":"https:\/\/www.frontiersin.org\/articles\/10.3389\/fdata.2022.979465\/full"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2022,9,27]]},"references-count":67,"alternative-id":["10.3389\/fdata.2022.979465"],"URL":"https:\/\/doi.org\/10.3389\/fdata.2022.979465","relation":{},"ISSN":["2624-909X"],"issn-type":[{"value":"2624-909X","type":"electronic"}],"subject":[],"published":{"date-parts":[[2022,9,27]]},"article-number":"979465"}}