{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,6,2]],"date-time":"2026-06-02T16:36:03Z","timestamp":1780418163831,"version":"3.54.1"},"update-to":[{"DOI":"10.1371\/journal.pcbi.1007419","type":"new_version","label":"New version","source":"publisher","updated":{"date-parts":[[2019,11,14]],"date-time":"2019-11-14T00:00:00Z","timestamp":1573689600000}}],"reference-count":32,"publisher":"Public Library of Science (PLoS)","issue":"11","license":[{"start":{"date-parts":[[2019,11,4]],"date-time":"2019-11-04T00:00:00Z","timestamp":1572825600000},"content-version":"vor","delay-in-days":0,"URL":"http:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"funder":[{"DOI":"10.13039\/501100004756","name":"Emil Aaltonen Foundation","doi-asserted-by":"crossref","id":[{"id":"10.13039\/501100004756","id-type":"DOI","asserted-by":"crossref"}]},{"DOI":"10.13039\/100007797","name":"University of Helsinki","doi-asserted-by":"crossref","id":[{"id":"10.13039\/100007797","id-type":"DOI","asserted-by":"crossref"}]},{"name":"Helsinki Institute of Life Sciences, University of Helsinki"}],"content-domain":{"domain":["www.ploscompbiol.org"],"crossmark-restriction":false},"short-container-title":["PLoS Comput Biol"],"DOI":"10.1371\/journal.pcbi.1007419","type":"journal-article","created":{"date-parts":[[2019,11,4]],"date-time":"2019-11-04T13:34:50Z","timestamp":1572874490000},"page":"e1007419","update-policy":"https:\/\/doi.org\/10.1371\/journal.pcbi.corrections_policy","source":"Crossref","is-referenced-by-count":14,"title":["Novel comparison of evaluation metrics for gene ontology classifiers reveals drastic performance differences"],"prefix":"10.1371","volume":"15","author":[{"given":"Ilya","family":"Plyusnin","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Liisa","family":"Holm","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-4764-9790","authenticated-orcid":true,"given":"Petri","family":"T\u00f6r\u00f6nen","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"340","published-online":{"date-parts":[[2019,11,4]]},"reference":[{"issue":"3","key":"pcbi.1007419.ref001","doi-asserted-by":"crossref","first-page":"225","DOI":"10.1093\/bib\/bbl004","article-title":"Automated protein function prediction\u2014the genomic challenge","volume":"7","author":"I Friedberg","year":"2006","journal-title":"Briefings in bioinformatics"},{"issue":"3","key":"pcbi.1007419.ref002","doi-asserted-by":"crossref","first-page":"221","DOI":"10.1038\/nmeth.2340","article-title":"A large-scale evaluation of computational protein function prediction","volume":"10","author":"P Radivojac","year":"2013","journal-title":"Nature methods"},{"issue":"1","key":"pcbi.1007419.ref003","doi-asserted-by":"crossref","first-page":"184","DOI":"10.1186\/s13059-016-1037-6","article-title":"An expanded evaluation of protein function prediction methods shows an improvement in accuracy","volume":"17","author":"Y Jiang","year":"2016","journal-title":"Genome biology"},{"issue":"1","key":"pcbi.1007419.ref004","doi-asserted-by":"crossref","first-page":"S2","DOI":"10.1186\/gb-2008-9-s1-s2","article-title":"A critical assessment of Mus musculus gene function prediction using integrated genomic evidence","volume":"9","author":"L Pe\u00f1a-Castillo","year":"2008","journal-title":"Genome biology"},{"key":"pcbi.1007419.ref005","article-title":"The CAFA challenge reports improved protein function prediction and new functional annotations for hundreds of genes through experimental screens","author":"N Zhou","year":"2019","journal-title":"bioRxiv"},{"issue":"10","key":"pcbi.1007419.ref006","doi-asserted-by":"crossref","first-page":"1544","DOI":"10.1093\/bioinformatics\/btu851","article-title":"PANNZER: high-throughput functional annotation of uncharacterized proteins in an error-prone environment","volume":"31","author":"P Koskinen","year":"2015","journal-title":"Bioinformatics"},{"issue":"W1","key":"pcbi.1007419.ref007","doi-asserted-by":"crossref","first-page":"W84","DOI":"10.1093\/nar\/gky350","article-title":"PANNZER2: a rapid functional annotation web server","volume":"46","author":"P T\u00f6r\u00f6nen","year":"2018","journal-title":"Nucleic Acids Research"},{"issue":"1","key":"pcbi.1007419.ref008","doi-asserted-by":"crossref","first-page":"27","DOI":"10.1016\/j.patrec.2008.08.010","article-title":"An experimental comparison of performance measures for classification","volume":"30","author":"C Ferri","year":"2009","journal-title":"Pattern Recognition Letters"},{"issue":"3","key":"pcbi.1007419.ref009","doi-asserted-by":"crossref","first-page":"400","DOI":"10.1111\/j.1751-5823.2012.00183.x","article-title":"Assessing the performance of classification methods","volume":"80","author":"DJ Hand","year":"2012","journal-title":"International Statistical Review"},{"key":"pcbi.1007419.ref010","doi-asserted-by":"crossref","DOI":"10.1017\/CBO9780511921803","volume-title":"Evaluating learning algorithms: a classification perspective","author":"N Japkowicz","year":"2011"},{"key":"pcbi.1007419.ref011","doi-asserted-by":"crossref","first-page":"S15","DOI":"10.1186\/1471-2105-14-S3-S15","article-title":"Characterizing the state of the art in the computational assignment of gene function: lessons from the first critical assessment of functional annotation (CAFA)","volume":"vol. 14","author":"J Gillis","year":"2013","journal-title":"BMC bioinformatics"},{"issue":"1","key":"pcbi.1007419.ref012","doi-asserted-by":"crossref","first-page":"41","DOI":"10.1186\/s13742-015-0082-5","article-title":"A close look at protein function prediction evaluation protocols","volume":"4","author":"I Kahanda","year":"2015","journal-title":"GigaScience"},{"issue":"1","key":"pcbi.1007419.ref013","doi-asserted-by":"crossref","first-page":"25","DOI":"10.1038\/75556","article-title":"Gene Ontology: tool for the unification of biology","volume":"25","author":"M Ashburner","year":"2000","journal-title":"Nature genetics"},{"key":"pcbi.1007419.ref014","doi-asserted-by":"crossref","unstructured":"Eisner R, Poulin B, Szafron D, Lu P, Greiner R. Improving Protein Function Prediction using the Hierarchical Structure of the Gene Ontology. In: 2005 IEEE Symposium on Computational Intelligence in Bioinformatics and Computational Biology. Institute of Electrical and Electronics Engineers (IEEE); 2005. Available from: https:\/\/doi.org\/10.1109%2Fcibcb.2005.1594940.","DOI":"10.1109\/CIBCB.2005.1594940"},{"issue":"19","key":"pcbi.1007419.ref015","doi-asserted-by":"crossref","first-page":"2996","DOI":"10.1093\/bioinformatics\/btw366","article-title":"NegGOA: negative GO annotations selection using ontology structure","volume":"32","author":"G Fu","year":"2016","journal-title":"Bioinformatics"},{"issue":"12","key":"pcbi.1007419.ref016","doi-asserted-by":"crossref","first-page":"1665","DOI":"10.1016\/j.patrec.2010.05.017","article-title":"Two information-theoretic tools to assess the performance of multi-class classifiers","volume":"31","author":"FJ Valverde-Albacete","year":"2010","journal-title":"Pattern Recognition Letters"},{"issue":"1","key":"pcbi.1007419.ref017","doi-asserted-by":"crossref","first-page":"e84217","DOI":"10.1371\/journal.pone.0084217","article-title":"100% classification accuracy considered harmful: The normalized information transfer factor explains the accuracy paradox","volume":"9","author":"FJ Valverde-Albacete","year":"2014","journal-title":"PloS one"},{"issue":"1","key":"pcbi.1007419.ref018","doi-asserted-by":"crossref","first-page":"537","DOI":"10.1038\/msb.2011.70","article-title":"The self-assessment trap: can we all be better than average?","volume":"7","author":"R Norel","year":"2011","journal-title":"Molecular systems biology"},{"issue":"7","key":"pcbi.1007419.ref019","doi-asserted-by":"crossref","first-page":"e1000443","DOI":"10.1371\/journal.pcbi.1000443","article-title":"Semantic similarity in biomedical ontologies","volume":"5","author":"C Pesquita","year":"2009","journal-title":"PLoS comput biol"},{"key":"pcbi.1007419.ref020","unstructured":"Pesquita C, Faria D, Bastos H, Falcao A, Couto F. Evaluating GO-based semantic similarity measures. In: Proc. 10th Annual Bio-Ontologies Meeting. vol. 37; 2007. p. 38."},{"issue":"7","key":"pcbi.1007419.ref021","doi-asserted-by":"crossref","first-page":"2086","DOI":"10.1002\/prot.23029","article-title":"Analysis of protein function and its prediction from amino acid sequence","volume":"79","author":"WT Clark","year":"2011","journal-title":"Proteins: Structure, Function, and Bioinformatics"},{"key":"pcbi.1007419.ref022","unstructured":"Lin D, et al. An information-theoretic definition of similarity. In: Icml. vol. 98. Citeseer; 1998. p. 296\u2013304."},{"key":"pcbi.1007419.ref023","doi-asserted-by":"crossref","first-page":"95","DOI":"10.1613\/jair.514","article-title":"Semantic similarity in a taxonomy: An information-based measure and its application to problems of ambiguity in natural language","volume":"11","author":"P Resnik","year":"1999","journal-title":"J Artif Intell Res(JAIR)"},{"issue":"13","key":"pcbi.1007419.ref024","doi-asserted-by":"crossref","first-page":"i53","DOI":"10.1093\/bioinformatics\/btt228","article-title":"Information-theoretic evaluation of predicted ontological annotations","volume":"29","author":"WT Clark","year":"2013","journal-title":"Bioinformatics"},{"issue":"6","key":"pcbi.1007419.ref025","doi-asserted-by":"crossref","first-page":"822","DOI":"10.1093\/bioinformatics\/btq037","article-title":"Small-sample precision of ROC-related estimates","volume":"26","author":"B Hanczar","year":"2010","journal-title":"Bioinformatics"},{"issue":"13","key":"pcbi.1007419.ref026","doi-asserted-by":"crossref","first-page":"i53","DOI":"10.1093\/bioinformatics\/btt228","article-title":"Information-theoretic evaluation of predicted ontological annotations","volume":"29","author":"WT Clark","year":"2013","journal-title":"Bioinformatics"},{"key":"pcbi.1007419.ref027","doi-asserted-by":"crossref","unstructured":"Boyd K, Eng KH, Page CD. Area under the precision-recall curve: point estimates and confidence intervals. In: Joint European conference on machine learning and knowledge discovery in databases. Springer; 2013. p. 451\u2013466.","DOI":"10.1007\/978-3-642-40994-3_29"},{"issue":"11","key":"pcbi.1007419.ref028","doi-asserted-by":"crossref","first-page":"1251","DOI":"10.1038\/nbt1346","article-title":"The OBO Foundry: coordinated evolution of ontologies to support biomedical data integration","volume":"25","author":"B Smith","year":"2007","journal-title":"Nature biotechnology"},{"key":"pcbi.1007419.ref029","doi-asserted-by":"crossref","DOI":"10.7551\/mitpress\/7287.001.0001","volume-title":"WordNet: An Electronic Lexical Database","author":"C Fellbaum","year":"1998"},{"issue":"8","key":"pcbi.1007419.ref030","doi-asserted-by":"crossref","first-page":"e124","DOI":"10.1371\/journal.pmed.0020124","article-title":"Why most published research findings are false","volume":"2","author":"JP Ioannidis","year":"2005","journal-title":"PLoS medicine"},{"issue":"10","key":"pcbi.1007419.ref031","doi-asserted-by":"crossref","first-page":"e1001747","DOI":"10.1371\/journal.pmed.1001747","article-title":"How to make more published research true","volume":"11","author":"JP Ioannidis","year":"2014","journal-title":"PLoS medicine"},{"issue":"3","key":"pcbi.1007419.ref032","doi-asserted-by":"crossref","first-page":"437","DOI":"10.1093\/bioinformatics\/btp648","article-title":"Over-optimism in bioinformatics research","volume":"26","author":"AL Boulesteix","year":"2010","journal-title":"Bioinformatics"}],"updated-by":[{"DOI":"10.1371\/journal.pcbi.1010249","type":"correction","label":"Correction","source":"publisher","updated":{"date-parts":[[2022,6,9]],"date-time":"2022-06-09T00:00:00Z","timestamp":1654732800000}},{"DOI":"10.1371\/journal.pcbi.1007419","type":"new_version","label":"New version","source":"publisher","updated":{"date-parts":[[2019,11,14]],"date-time":"2019-11-14T00:00:00Z","timestamp":1573689600000}}],"container-title":["PLOS Computational Biology"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/dx.plos.org\/10.1371\/journal.pcbi.1007419","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2019,11,14]],"date-time":"2019-11-14T13:57:31Z","timestamp":1573739851000},"score":1,"resource":{"primary":{"URL":"https:\/\/dx.plos.org\/10.1371\/journal.pcbi.1007419"}},"subtitle":[],"editor":[{"given":"Predrag","family":"Radivojac","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"editor"}]}],"short-title":[],"issued":{"date-parts":[[2019,11,4]]},"references-count":32,"journal-issue":{"issue":"11","published-online":{"date-parts":[[2019,11,4]]}},"URL":"https:\/\/doi.org\/10.1371\/journal.pcbi.1007419","relation":{"has-preprint":[{"id-type":"doi","id":"10.1101\/427096","asserted-by":"object"}]},"ISSN":["1553-7358"],"issn-type":[{"value":"1553-7358","type":"electronic"}],"subject":[],"published":{"date-parts":[[2019,11,4]]}}}