{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,6,3]],"date-time":"2026-06-03T00:10:23Z","timestamp":1780445423476,"version":"3.54.1"},"reference-count":59,"publisher":"Oxford University Press (OUP)","license":[{"start":{"date-parts":[[2018,12,20]],"date-time":"2018-12-20T00:00:00Z","timestamp":1545264000000},"content-version":"vor","delay-in-days":353,"URL":"http:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"funder":[{"DOI":"10.13039\/100000001","name":"National Science Foundation","doi-asserted-by":"publisher","award":["DBI-1062542"],"award-info":[{"award-number":["DBI-1062542"]}],"id":[{"id":"10.13039\/100000001","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000001","name":"National Science Foundation","doi-asserted-by":"publisher","award":["EF-0849982"],"award-info":[{"award-number":["EF-0849982"]}],"id":[{"id":"10.13039\/100000001","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000001","name":"National Science Foundation","doi-asserted-by":"publisher","award":["DBI-1147266"],"award-info":[{"award-number":["DBI-1147266"]}],"id":[{"id":"10.13039\/100000001","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2018,1,1]]},"DOI":"10.1093\/database\/bay110","type":"journal-article","created":{"date-parts":[[2018,9,25]],"date-time":"2018-09-25T23:22:22Z","timestamp":1537917742000},"source":"Crossref","is-referenced-by-count":19,"title":["Annotation of phenotypes using ontologies: a gold standard for the training and evaluation of natural language processing systems"],"prefix":"10.1093","volume":"2018","author":[{"given":"Wasila","family":"Dahdul","sequence":"first","affiliation":[{"name":"University of South Dakota, Vermillion, SD, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Prashanti","family":"Manda","sequence":"additional","affiliation":[{"name":"University of North Carolina at Greensboro, Greensboro, NC, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Hong","family":"Cui","sequence":"additional","affiliation":[{"name":"University of Arizona, Tucson, AZ, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"James P","family":"Balhoff","sequence":"additional","affiliation":[{"name":"University of North Carolina at Chapel Hill, Chapel Hill, NC, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"T Alexander","family":"Dececchi","sequence":"additional","affiliation":[{"name":"University of South Dakota, Vermillion, SD, USA"},{"name":"Current affiliation: University of Pittsburgh at Johnstown, Johnstown, PA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Nizar","family":"Ibrahim","sequence":"additional","affiliation":[{"name":"University of Chicago, Chicago, IL, USA"},{"name":"Current affiliation: University of Detroit Mercy, Detroit, MI, USA & University of Portsmouth, Portsmouth, UK"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Hilmar","family":"Lapp","sequence":"additional","affiliation":[{"name":"Duke University, Durham, NC, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Todd","family":"Vision","sequence":"additional","affiliation":[{"name":"University of North Carolina at Chapel Hill, Chapel Hill, NC, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Paula M","family":"Mabee","sequence":"additional","affiliation":[{"name":"University of South Dakota, Vermillion, SD, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"286","published-online":{"date-parts":[[2018,12,14]]},"reference":[{"key":"key\n\t\t\t\t2019012510110436100_ref1","doi-asserted-by":"crossref","first-page":"1251","DOI":"10.1038\/nbt1346","article-title":"The OBO Foundry: coordinated evolution of ontologies to support biomedical data integration","volume":"25","author":"Smith","year":"2007","journal-title":"Nat. Biotechnol."},{"key":"key\n\t\t\t\t2019012510110436100_ref2","first-page":"313","article-title":"Data Extraction, Transformation, and Dissemination through ZFIN","volume":"104","author":"Howe,","year":"2011","journal-title":"The Zebrafish: Genetics, Genomics and Informatics"},{"key":"key\n\t\t\t\t2019012510110436100_ref3","doi-asserted-by":"crossref","first-page":"D822","DOI":"10.1093\/nar\/gkq1077","article-title":"ZFIN: enhancements and updates to the Zebrafish Model Organism Database","volume":"39","author":"Bradford,","year":"2011","journal-title":"Nucleic Acids Res."},{"key":"key\n\t\t\t\t2019012510110436100_ref4","first-page":"D761","article-title":"Xenbase: a","volume":"36","author":"Bowes,","year":"2008","journal-title":"Nucleic Acids Res."},{"key":"key\n\t\t\t\t2019012510110436100_ref5","doi-asserted-by":"crossref","first-page":"D712","DOI":"10.1093\/nar\/gkn886","article-title":"The Mouse Genome Database genotypes::phenotypes","volume":"37","author":"Blake,","year":"2009","journal-title":"Nucleic Acids Res."},{"key":"key\n\t\t\t\t2019012510110436100_ref6","volume-title":"Representing phenotypes in OWL. In: Proceedings of the OWLED Workshop on OWL: Experience and Directions","author":"Mungall,","year":"2007"},{"key":"key\n\t\t\t\t2019012510110436100_ref7","doi-asserted-by":"crossref","first-page":"R2","DOI":"10.1186\/gb-2010-11-1-r2","article-title":"Integrating phenotype ontologies across multiple species","volume":"11","author":"Mungall,","year":"2010","journal-title":"Genome Biol."},{"key":"key\n\t\t\t\t2019012510110436100_ref8","doi-asserted-by":"crossref","DOI":"10.1371\/journal.pbio.1002033","article-title":"Finding our way through phenotypes","volume":"13","author":"Deans,","year":"2015","journal-title":"PLoS Biol."},{"key":"key\n\t\t\t\t2019012510110436100_ref9","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1186\/2041-1480-3-S2-S5","article-title":"Towards improving phenotype representation in OWL","volume":"3","author":"Loebe","year":"2012","journal-title":"J. Biomed. Semantics"},{"issue":"3","key":"key\n\t\t\t\t2019012510110436100_ref10","doi-asserted-by":"crossref","first-page":"301","DOI":"10.1111\/j.1096-0031.2009.00286.x","article-title":"The linguistic problem of morphology: structure versus homology and the standardization of morphological data","volume":"26","author":"Vogt","year":"2010","journal-title":"Cladistics"},{"key":"key\n\t\t\t\t2019012510110436100_ref11","doi-asserted-by":"crossref","first-page":"200","DOI":"10.1111\/cla.12195","article-title":"Towards a semantic approach to numerical tree inference in phylogenetics","volume":"34","author":"Vogt","year":"2018","journal-title":"Cladistics"},{"key":"key\n\t\t\t\t2019012510110436100_ref12","doi-asserted-by":"crossref","first-page":"45","DOI":"10.1186\/2041-1480-5-45","article-title":"Annotation of phenotypic diversity: decoupling data curation and ontology curation using Phenex","volume":"5","author":"Balhoff","year":"2014","journal-title":"J. Biomed. Semantics"},{"key":"key\n\t\t\t\t2019012510110436100_ref13","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1002\/pra2.2015.145052010020","article-title":"Charaparser+EQ: performance evaluation without gold standard","volume":"52","author":"Cui","year":"2015","journal-title":"Proceedings of the Association for Information Science and Technology"},{"key":"key\n\t\t\t\t2019012510110436100_ref14","doi-asserted-by":"crossref","first-page":"345","DOI":"10.1016\/j.tree.2007.03.013","article-title":"Phenotype ontologies: the bridge between genomics and evolution","volume":"22","author":"Mabee","year":"2007","journal-title":"Trends Ecol. Evol"},{"key":"key\n\t\t\t\t2019012510110436100_ref15","doi-asserted-by":"crossref","first-page":"1253","DOI":"10.1093\/bioinformatics\/bts125","article-title":"Harmonization of gene\/protein annotations: towards a gold standard MEDLINE","volume":"28","author":"Campos","year":"2012","journal-title":"Bioinformatics"},{"key":"key\n\t\t\t\t2019012510110436100_ref16","first-page":"481","article-title":"Using silver and semi-gold standard corpora to compare open named entity recognisers","author":"Groza","year":"2013","journal-title":"In: 2013 IEEE International Conference on Bioinformatics and Biomedicine (BIBM)"},{"key":"key\n\t\t\t\t2019012510110436100_ref17","doi-asserted-by":"crossref","first-page":"59","DOI":"10.1186\/1471-2105-15-59","article-title":"Large-scale biomedical concept recognition: an evaluation of current automatic annotators and their parameters","volume":"15","author":"Funk","year":"2014","journal-title":"BMC Bioinformatics"},{"key":"key\n\t\t\t\t2019012510110436100_ref18","doi-asserted-by":"crossref","first-page":"300","DOI":"10.1111\/j.1439-0426.2012.01985.x","article-title":"500,000 fish phenotypes: the new informatics landscape for evolutionary and developmental biology of the vertebrate skeleton","volume":"28","author":"Mabee","year":"2012","journal-title":"J. Appl. Ichthyol."},{"key":"key\n\t\t\t\t2019012510110436100_ref19","doi-asserted-by":"crossref","first-page":"161","DOI":"10.1186\/1471-2105-13-161","article-title":"Concept annotation in the CRAFT corpus","volume":"13","author":"Bada","year":"2012","journal-title":"BMC Bioinformatics"},{"key":"key\n\t\t\t\t2019012510110436100_ref20","doi-asserted-by":"crossref","DOI":"10.1371\/journal.pcbi.1000443","article-title":"Semantic similarity in biomedical ontologies","volume":"5","author":"Pesquita","year":"2009","journal-title":"PLoS Comput. Biol."},{"key":"key\n\t\t\t\t2019012510110436100_ref21","article-title":"Gold-standard ontology-based annotation of concepts in biomedical text in the craft corpus: updates and extensions","volume":"1747","author":"Bada","year":"2016","journal-title":"In: ICBO\/BioCreative, CEUR Workshop Proceedings"},{"key":"key\n\t\t\t\t2019012510110436100_ref22","doi-asserted-by":"crossref","first-page":"i180","DOI":"10.1093\/bioinformatics\/btg1023","article-title":"Genia corpus\u2014a semantically annotated corpus for bio-textmining","volume":"19","author":"Kim","year":"2003","journal-title":"Bioinformatics"},{"key":"key\n\t\t\t\t2019012510110436100_ref23","doi-asserted-by":"crossref","first-page":"S2","DOI":"10.1186\/1471-2105-12-S8-S2","article-title":"The gene normalization task in BioCreative III","volume":"12","author":"Lu","year":"2011","journal-title":"BMC Bioinformatics"},{"key":"key\n\t\t\t\t2019012510110436100_ref24","doi-asserted-by":"crossref","first-page":"948","DOI":"10.1093\/jamia\/ocv037","article-title":"A multilingual gold-standard corpus for biomedical concept recognition: the Mantra GSC","volume":"22","author":"Kors","year":"2015","journal-title":"J. Am. Med. Inform. Assoc."},{"key":"key\n\t\t\t\t2019012510110436100_ref25","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1371\/journal.pone.0116040","article-title":"Generation of silver standard concept annotations from biomedical texts with special relevance to phenotypes","volume":"10","author":"Oellrich","year":"2015","journal-title":"PLoS One"},{"key":"key\n\t\t\t\t2019012510110436100_ref26","doi-asserted-by":"crossref","first-page":"163","DOI":"10.1142\/S0219720010004562","article-title":"CALBC silver standard corpus","volume":"8","author":"Rebholz-Schuhmann","year":"2010","journal-title":"J. Bioinform. Comput. Biol."},{"key":"key\n\t\t\t\t2019012510110436100_ref27","doi-asserted-by":"crossref","first-page":"326","DOI":"10.1186\/1471-2105-10-326","article-title":"Text mining and manual curation of chemical-gene-disease networks for the Comparative Toxicogenomics Database (CTD)","volume":"10","author":"Wiegers","year":"2009","journal-title":"BMC Bioinformatics"},{"key":"key\n\t\t\t\t2019012510110436100_ref28","doi-asserted-by":"crossref","first-page":"329","DOI":"10.1186\/1471-2105-12-329","article-title":"Development of a classification scheme for disease-related enzyme information","volume":"12","author":"S\u00f6hngen","year":"2011","journal-title":"BMC Bioinformatics"},{"key":"key\n\t\t\t\t2019012510110436100_ref29","doi-asserted-by":"crossref","first-page":"S17","DOI":"10.1186\/1471-2105-6-S1-S17","article-title":"An evaluation of GO annotation retrieval for BioCreAtIvE and GOA","volume":"6","author":"Camon","year":"2005","journal-title":"BMC Bioinformatics"},{"key":"key\n\t\t\t\t2019012510110436100_ref30","first-page":"241","article-title":"Early sharks and primitive gnathostome interrelationships. In:","volume-title":"Major Events in Early Vertebrate Evolution","author":"Coates","year":"2001"},{"key":"key\n\t\t\t\t2019012510110436100_ref31","doi-asserted-by":"crossref","first-page":"530","DOI":"10.1080\/10635150590950326","article-title":"Integration of morphological data sets for phylogenetic analysis of Amniota: the importance of integumentary characters and increased taxonomic sampling","volume":"54","author":"Hill","year":"2005","journal-title":"Syst. Biol."},{"key":"key\n\t\t\t\t2019012510110436100_ref32","doi-asserted-by":"crossref","first-page":"135","DOI":"10.4202\/app.2010.0101","article-title":"A new salamander from the late Paleocene\u2013early Eocene of Ukraine","volume":"57","author":"Skutschas","year":"2012","journal-title":"Acta Palaeontol. Pol."},{"key":"key\n\t\t\t\t2019012510110436100_ref33","article-title":"Podargiform affinities of the enigmatic","volume":"6","author":"Nesbitt","year":"2011","journal-title":"PLoS One"},{"key":"key\n\t\t\t\t2019012510110436100_ref34","first-page":"1","volume-title":"A morphological phylogenetic analysis of Middle American cichlids with special emphasis on the section Nandopsis sensu Regan. Museum of Zoology, University of Michigan","author":"Chakrabarty","year":"2007"},{"key":"key\n\t\t\t\t2019012510110436100_ref35","doi-asserted-by":"crossref","first-page":"662","DOI":"10.1126\/science.1229237","article-title":"The placental mammal ancestor and the post-K-Pg radiation of placentals","volume":"339","author":"O\u2019Leary","year":"2013","journal-title":"Science"},{"key":"key\n\t\t\t\t2019012510110436100_ref36","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1206\/310.1","article-title":"Phylogeny and systematics of","volume":"310","author":"Conrad","year":"2008","journal-title":"Bull. Am. Mus. Nat. Hist."},{"key":"key\n\t\t\t\t2019012510110436100_ref37","doi-asserted-by":"crossref","first-page":"e10500","DOI":"10.1371\/journal.pone.0010500","article-title":"Phenex: ontological annotation of phenotypic diversity","volume":"5","author":"Balhoff","year":"2010","journal-title":"PLoS One"},{"key":"key\n\t\t\t\t2019012510110436100_ref38","doi-asserted-by":"crossref","first-page":"R5","DOI":"10.1186\/gb-2012-13-1-r5","article-title":"Uberon, an integrative multi-species anatomy ontology","volume":"13","author":"Mungall","year":"2012","journal-title":"Genome Biol."},{"key":"key\n\t\t\t\t2019012510110436100_ref39","doi-asserted-by":"crossref","first-page":"21","DOI":"10.1186\/2041-1480-5-21","article-title":"Unification of multi-species vertebrate anatomy ontologies for comparative biology in Uberon","volume":"5","author":"Haendel","year":"2014","journal-title":"J. Biomed. Semantics"},{"key":"key\n\t\t\t\t2019012510110436100_ref40","doi-asserted-by":"crossref","first-page":"545","DOI":"10.1002\/cfg.430","article-title":"Ontologies for the description of mouse phenotypes","volume":"5","author":"Gkoutos","year":"2004","journal-title":"Comp. Funct. Genomics"},{"key":"key\n\t\t\t\t2019012510110436100_ref41","doi-asserted-by":"crossref","first-page":"R8","DOI":"10.1186\/gb-2004-6-1-r8","article-title":"Using ontologies to describe mouse phenotypes","volume":"6","author":"Gkoutos","year":"2004","journal-title":"Genome Biol."},{"key":"key\n\t\t\t\t2019012510110436100_ref42","doi-asserted-by":"crossref","first-page":"34","DOI":"10.1186\/2041-1480-5-34","article-title":"Nose to tail, roots to shoots: spatial descriptors for phenotypic diversity in the Biological Spatial Ontology","volume":"5","author":"Dahdul","year":"2014","journal-title":"J. Biomed. Semantics"},{"key":"key\n\t\t\t\t2019012510110436100_ref43","doi-asserted-by":"crossref","DOI":"10.1371\/journal.pone.0010708","article-title":"Evolutionary characters, phenotypes and ontologies: curating data from the systematic biology literature","volume":"5","author":"Dahdul","year":"2010","journal-title":"PLoS One"},{"key":"key\n\t\t\t\t2019012510110436100_ref44","article-title":"Phenoscape guide to character annotation","author":"Dahdul","year":"2018"},{"key":"key\n\t\t\t\t2019012510110436100_ref45","doi-asserted-by":"crossref","first-page":"W541","DOI":"10.1093\/nar\/gkr469","article-title":"BioPortal: enhanced functionality via new web services from the National Center for Biomedical Ontology to access and use ontologies in software applications","volume":"39","author":"Whetzel","year":"2011","journal-title":"Nucleic Acids Res."},{"key":"key\n\t\t\t\t2019012510110436100_ref46","doi-asserted-by":"crossref","first-page":"327","DOI":"10.1186\/1471-2105-9-327","article-title":"Gene ontology term overlap as a measure of gene functional similarity","volume":"9","author":"Meeta","year":"2008","journal-title":"BMC Bioinformatics"},{"key":"key\n\t\t\t\t2019012510110436100_ref47","doi-asserted-by":"crossref","first-page":"95","DOI":"10.1613\/jair.514","article-title":"Semantic similarity in a taxonomy: an information-based measure and its application to problems of ambiguity in natural language","volume":"11","author":"Resnik","year":"1999","journal-title":"J. Artif. Intell. Res."},{"key":"key\n\t\t\t\t2019012510110436100_ref48","unstructured":"Euzenat,J. (2007) Semantic precision and recall for ontology alignment evaluation. In: Proceedings of the 20th International Joint Conference on Artificial Intelligence (IJCAI\u201907), Hyderabad, India, 348\u2013353."},{"key":"key\n\t\t\t\t2019012510110436100_ref49","unstructured":"Bada,M., Baumgartner,W.A. Jr, Funk,C. et al. (2014) Semantic precision and recall for concept annotation of text. In: Proceedings of Bio-Ontologies, Boston, Massachusetts, 30\u201337."},{"key":"key\n\t\t\t\t2019012510110436100_ref50","doi-asserted-by":"crossref","first-page":"77","DOI":"10.1111\/j.1745-459X.2003.tb00374.x","article-title":"Using Anderson\u2019s statistic to compare distributions of consumer preference rankings","volume":"18","author":"Brockhoff","year":"2003","journal-title":"J. Sens. Stud."},{"key":"key\n\t\t\t\t2019012510110436100_ref51","doi-asserted-by":"crossref","first-page":"675","DOI":"10.1093\/sysbio\/sys025","article-title":"Nexml_ rich, extensible, and verifiable representation of comparative data and metadata","volume":"61","author":"Vos","year":"2012","journal-title":"Syst. Biol."},{"key":"key\n\t\t\t\t2019012510110436100_ref52","author":"Balhoff","year":"2017"},{"key":"key\n\t\t\t\t2019012510110436100_ref53","first-page":"bav040","article-title":"Moving the mountain: analysis of the effort required to transform comparative anatomy into computable anatomy","volume-title":"Database (Oxford)","author":"Dahdul","year":"2015"},{"key":"key\n\t\t\t\t2019012510110436100_ref54","doi-asserted-by":"crossref","DOI":"10.1371\/journal.pbio.2002846","article-title":"Biocuration: distilling data into knowledge","volume":"16","author":"International Society for Biocuration","year":"2018","journal-title":"PLoS Biol."},{"key":"key\n\t\t\t\t2019012510110436100_ref55","doi-asserted-by":"crossref","first-page":"936","DOI":"10.1093\/sysbio\/syv031","article-title":"Toward synthesizing our knowledge of morphology: using ontologies and machine reasoning to extract presence\/absence evolutionary phenotypes across studies","volume":"64","author":"Dececchi","year":"2015","journal-title":"Syst. Biol."},{"key":"key\n\t\t\t\t2019012510110436100_ref56","doi-asserted-by":"crossref","first-page":"D865","DOI":"10.1093\/nar\/gkw1039","article-title":"The human phenotype ontology in 2017","volume":"45","author":"K\u00f6hler","year":"2017","journal-title":"Nucleic Acids Res."},{"key":"key\n\t\t\t\t2019012510110436100_ref57","doi-asserted-by":"crossref","first-page":"1275","DOI":"10.1016\/j.artint.2009.06.003","article-title":"Representing ontologies using description logics, description graphs, and rules","volume":"173","author":"Motik","year":"2009","journal-title":"Artif. Intell."},{"key":"key\n\t\t\t\t2019012510110436100_ref58","doi-asserted-by":"crossref","first-page":"bas056","DOI":"10.1093\/database\/bas056","article-title":"An overview of the BioCreative 2012 Workshop Track III: interactive text mining task","volume":"2013","author":"Arighi","year":"2013","journal-title":"Database (Oxford)"},{"key":"key\n\t\t\t\t2019012510110436100_ref59","doi-asserted-by":"crossref","DOI":"10.1371\/journal.pbio.1000247","article-title":"Linking human diseases to animal models using ontology-based phenotype annotation","volume":"7","author":"Washington","year":"2009","journal-title":"PLoS Biol"}],"container-title":["Database"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/academic.oup.com\/database\/article-pdf\/doi\/10.1093\/database\/bay110\/27329249\/bay110.pdf","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2019,10,24]],"date-time":"2019-10-24T17:43:47Z","timestamp":1571939027000},"score":1,"resource":{"primary":{"URL":"https:\/\/academic.oup.com\/database\/article\/doi\/10.1093\/database\/bay110\/5255130"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2018,1,1]]},"references-count":59,"URL":"https:\/\/doi.org\/10.1093\/database\/bay110","relation":{"has-preprint":[{"id-type":"doi","id":"10.1101\/322156","asserted-by":"object"}]},"ISSN":["1758-0463"],"issn-type":[{"value":"1758-0463","type":"electronic"}],"subject":[],"published-other":{"date-parts":[[2018]]},"published":{"date-parts":[[2018,1,1]]}}}