{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,5,6]],"date-time":"2026-05-06T03:18:02Z","timestamp":1778037482138,"version":"3.51.4"},"reference-count":48,"publisher":"Springer Science and Business Media LLC","issue":"S4","license":[{"start":{"date-parts":[[2019,8,1]],"date-time":"2019-08-01T00:00:00Z","timestamp":1564617600000},"content-version":"tdm","delay-in-days":0,"URL":"http:\/\/creativecommons.org\/licenses\/by\/4.0\/"},{"start":{"date-parts":[[2019,8,8]],"date-time":"2019-08-08T00:00:00Z","timestamp":1565222400000},"content-version":"vor","delay-in-days":7,"URL":"http:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["BMC Med Inform Decis Mak"],"published-print":{"date-parts":[[2019,8]]},"DOI":"10.1186\/s12911-019-0859-z","type":"journal-article","created":{"date-parts":[[2019,8,8]],"date-time":"2019-08-08T00:02:32Z","timestamp":1565222552000},"update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":21,"title":["Architecture and usability of OntoKeeper, an ontology evaluation tool"],"prefix":"10.1186","volume":"19","author":[{"given":"Muhammad","family":"Amith","sequence":"first","affiliation":[]},{"given":"Frank","family":"Manion","sequence":"additional","affiliation":[]},{"given":"Chen","family":"Liang","sequence":"additional","affiliation":[]},{"given":"Marcelline","family":"Harris","sequence":"additional","affiliation":[]},{"given":"Dennis","family":"Wang","sequence":"additional","affiliation":[]},{"given":"Yongqun","family":"He","sequence":"additional","affiliation":[]},{"given":"Cui","family":"Tao","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2019,8,8]]},"reference":[{"key":"859_CR1","unstructured":"Abele A, McCrae J. The Linking Open Data cloud diagram. 2017. http:\/\/lod-cloud.net . Accessed 10 June 2019."},{"key":"859_CR2","volume-title":"International Workshop on Biomedical Data Mining, Modeling, and Semantic Integration: A Promising Approach to Solving Unmet Medical Needs, BDM2I 2015-co-located with the 14th International Semantic Web Conference, ISWC 2015","author":"M Amith","year":"2015","unstructured":"Amith M, Tao C. A web application towards semiotic-based evaluation of biomedical ontologies. In: International Workshop on Biomedical Data Mining, Modeling, and Semantic Integration: A Promising Approach to Solving Unmet Medical Needs, BDM2I 2015-co-located with the 14th International Semantic Web Conference, ISWC 2015. Bethlehem: CEUR-WS: 2015."},{"key":"859_CR3","unstructured":"Brank J, Grobelnik M, Mladeni\u0107 D. A survey of ontology evaluation techniques. In: Proceedings of the Conference on Data Mining and Data Warehouses (SiKDD 2005). Ljubljana: 2005."},{"key":"859_CR4","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1016\/j.jbi.2018.02.010","volume":"80","author":"M Amith","year":"2018","unstructured":"Amith M, He Z, Bian J, Lossio-Ventura JA, Tao C. Assessing the practice of biomedical ontology evaluation: Gaps and opportunities. J Biomed Inform. 2018; 80:1\u201313.","journal-title":"J Biomed Inform"},{"key":"859_CR5","volume-title":"International Conference On Semantic Computing","author":"S T artir","year":"2007","unstructured":"T artir S, Arpinar IB. Ontology evaluation and ranking using ontoqa. In: International Conference On Semantic Computing. Irvine: IEEE: 2007. p. 185\u201392."},{"key":"859_CR6","volume-title":"IEEE ICDM Workshop on Knowledge Acquisition from Distributed, Autonomous, Semantically Heterogeneous Data and Knowledge Sources","author":"S Tartir","year":"2005","unstructured":"Tartir S, Arpinar IB, Moore M, Sheth AP, Aleman-Meza B. Ontoqa: Metric-based ontology quality analysis. In: IEEE ICDM Workshop on Knowledge Acquisition from Distributed, Autonomous, Semantically Heterogeneous Data and Knowledge Sources. New Orleans: IEEE: 2005."},{"key":"859_CR7","volume-title":"International Conference On Advanced Language Processing and Web Information Technology","author":"X Jianliang","year":"2008","unstructured":"Jianliang X, Xiaowei M. A web-based ontology evaluation system. In: International Conference On Advanced Language Processing and Web Information Technology. Liaoning: IEEE: 2008. p. 104\u20137."},{"key":"859_CR8","volume-title":"2011 International Conference On Process Automation, Control and Computing","author":"T Aruna","year":"2011","unstructured":"Aruna T, Saranya K, Bhandari C. A survey on ontology evaluation tools. In: 2011 International Conference On Process Automation, Control and Computing. Coimbatore: IEEE: 2011. p. 1\u20135."},{"issue":"3","key":"859_CR9","doi-asserted-by":"publisher","first-page":"254","DOI":"10.23919\/TST.2017.7914198","volume":"22","author":"H Zhu","year":"2017","unstructured":"Zhu H, Liu D, Bayley I, Aldea A, Yang Y, Chen Y. Quality model and metrics of ontology for semantic descriptions of web services. Tsinghua Sci Technol. 2017; 22(3):254\u201372.","journal-title":"Tsinghua Sci Technol"},{"key":"859_CR10","volume-title":"2016 Symposium On Service-Oriented System Engineering","author":"D Liu","year":"2016","unstructured":"Liu D, Yang Y, Chen Y, Zhu H, Bayley I, Aldea A. Evaluating the ontological semantic description of web services generated from algebraic specifications. In: 2016 Symposium On Service-Oriented System Engineering. Irvine: IEEE: 2016. p. 211\u201320."},{"key":"859_CR11","volume-title":"2014 16th International Symposium On Symbolic and Numeric Algorithms for Scientific Computing","author":"A Groza","year":"2014","unstructured":"Groza A, Dragoste I, Sincai I, Jimborean I, Moraru V. An ontology selection and ranking system based on the analytic hierarchy process. In: 2014 16th International Symposium On Symbolic and Numeric Algorithms for Scientific Computing. Timisoara: IEEE: 2014. p. 293\u2013300."},{"key":"859_CR12","volume-title":"2014 International Conference On Communication Problem-Solving","author":"I Jimborean","year":"2014","unstructured":"Jimborean I, Groza A. Ranking ontologies in the ontology building competition boc 2014. In: 2014 International Conference On Communication Problem-Solving. Beijing: IEEE: 2014. p. 75\u201382."},{"issue":"1","key":"859_CR13","doi-asserted-by":"publisher","first-page":"84","DOI":"10.1016\/j.datak.2004.11.010","volume":"55","author":"A Burton-Jones","year":"2005","unstructured":"Burton-Jones A, Storey VC, Sugumaran V, Ahluwalia P. A semiotic metrics suite for assessing the quality of ontologies. Data Knowl Eng. 2005; 55(1):84\u2013102.","journal-title":"Data Knowl Eng"},{"issue":"1","key":"859_CR14","first-page":"83","volume":"1","author":"TL Saaty","year":"2008","unstructured":"Saaty TL. Decision making with the analytic hierarchy process. Int J Serv Sci. 2008; 1(1):83\u201398.","journal-title":"Int J Serv Sci"},{"key":"859_CR15","volume-title":"International Semantic Web Conference","author":"H Alani","year":"2006","unstructured":"Alani H, Brewster C, Shadbolt N. Ranking ontologies with aktiverank. In: International Semantic Web Conference. Berlin: Springer: 2006. p. 1\u201315."},{"key":"859_CR16","volume-title":"23rd International Conference on Data Engineering","author":"W Yu","year":"2007","unstructured":"Yu W, Li Q, Chen J, Cao J. Os_rank: structure analysis for ontology ranking. In: 23rd International Conference on Data Engineering. Istanbul: IEEE: 2007. p. 339\u201346."},{"issue":"1","key":"859_CR17","doi-asserted-by":"crossref","first-page":"11","DOI":"10.3233\/SW-2011-0025","volume":"2","author":"M Horridge","year":"2011","unstructured":"Horridge M, Bechhofer S. The owl api: A java api for owl ontologies. Semantic Web. 2011; 2(1):11\u201321.","journal-title":"Semantic Web"},{"key":"859_CR18","unstructured":"Apache Software Foundation. Apache Jena. https:\/\/jena.apache.org . Accessed 10 June 2019."},{"key":"859_CR19","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-540-24750-0_13","volume-title":"Ontology evaluation","author":"A G\u00f3mez-P\u00e9rez","year":"2004","unstructured":"G\u00f3mez-P\u00e9rez A. Ontology evaluation. Berlin Heidelberg: Springer; 2004, pp. 251\u201373."},{"issue":"1-2","key":"859_CR20","doi-asserted-by":"publisher","first-page":"161","DOI":"10.1016\/S0169-023X(97)00056-6","volume":"25","author":"R Studer","year":"1998","unstructured":"Studer R, Benjamins VR, Fensel D. Knowledge engineering: principles and methods. Data Knowl Eng. 1998; 25(1-2):161\u201397.","journal-title":"Data Knowl Eng"},{"key":"859_CR21","unstructured":"Cyganiak R, Wood D, Lanthaler M. Rdf 1.1 concepts and abstract syntax. W3c Recomm. 2014;25(02). https:\/\/www.w3.org\/TR\/rdf11-concepts\/ . Accessed 8 July 2019."},{"key":"859_CR22","unstructured":"Consortium WWW, et al.OWL 2 Web Ontology Language Document Overview (Second Edition).2012. https:\/\/www.w3.org\/TR\/owl2-overview\/ . Accessed 4 July 2019."},{"key":"859_CR23","volume-title":"Course in General Linguistics","author":"F De Saussure","year":"2011","unstructured":"De Saussure F. Course in General Linguistics. New York: Columbia University Press; 2011."},{"key":"859_CR24","doi-asserted-by":"crossref","DOI":"10.4324\/9780203996157","volume-title":"The Pursuit of Signs","author":"J Culler","year":"2005","unstructured":"Culler J. The Pursuit of Signs. London: Routledge; 2005."},{"key":"859_CR25","volume-title":"Foundations of the Theory of Signs, vol 1","author":"C Morris","year":"1938","unstructured":"Morris C. Foundations of the Theory of Signs, vol 1. Chicago: University of Chicago Press; 1938."},{"key":"859_CR26","volume-title":"Enacting Research Methods in Information Systems","author":"R Price","year":"2016","unstructured":"Price R, Shanks G. A semiotic information quality framework: development and comparative analysis. In: Enacting Research Methods in Information Systems. Cham: Palgrave Macmillan: 2016. p. 219\u201350."},{"key":"859_CR27","volume-title":"Proceedings of the Winter Simulation Conference","author":"A Tolk","year":"2012","unstructured":"Tolk A, Diallo SY, Padilla JJ. Semiotics, entropy, and interoperability of simulation systems: mathematical foundations of m&s standardization. In: Proceedings of the Winter Simulation Conference. Berlin: 2012 Winter Simulation Conference: 2012. p. 243."},{"key":"859_CR28","volume-title":"Language Resources and Evaluation","author":"RQ Dividino","year":"2008","unstructured":"Dividino RQ, Romanelli M, Sonntag D, et al.Semiotic-based ontology evaluation tool (s-ontoeval). In: Language Resources and Evaluation. Marrakech: Morocco: 2008."},{"key":"859_CR29","unstructured":"Gangemi A, Catenacci C, Ciaramita M, Lehmann J. Qood grid: A metaontology-based framework for ontology evaluation and selection. In: 4th International EON Workshop located at the 15th International World Wide Web Conference. Edinburgh, United Kingdom: 2006."},{"issue":"1","key":"859_CR30","doi-asserted-by":"publisher","first-page":"15","DOI":"10.1080\/014492900118768","volume":"19","author":"R Stamper","year":"2000","unstructured":"Stamper R, Liu K, Hafkamp M, Ades Y. Understanding the roles of signs and norms in organizations-a semiotic approach to information systems design. Behav Inf Technol. 2000; 19(1):15\u201327.","journal-title":"Behav Inf Technol"},{"key":"859_CR31","first-page":"838","volume":"245","author":"M Amith","year":"2017","unstructured":"Amith M, Manion F, Harris M, Zhang Y, Xu H, Tao C. Expressing biomedical ontologies in natural language for expert evaluation. Stud Health Technol Inform. 2017; 245:838\u201342.","journal-title":"Stud Health Technol Inform"},{"key":"859_CR32","unstructured":"Finlayson MA. Java libraries for accessing the princeton wordnet: Comparison and evaluation. In: Proceedings of the 7th Global Wordnet Conference, Tartu, Estonia, vol 137: 2014."},{"issue":"11","key":"859_CR33","doi-asserted-by":"publisher","first-page":"39","DOI":"10.1145\/219717.219748","volume":"38","author":"GA Miller","year":"1995","unstructured":"Miller GA. WordNet: a lexical database for English. Commun ACM. 1995; 38(11):39\u201341.","journal-title":"Commun ACM"},{"issue":"1","key":"859_CR34","doi-asserted-by":"publisher","first-page":"17","DOI":"10.1186\/s13326-017-0124-2","volume":"8","author":"M Amith","year":"2017","unstructured":"Amith M, Tao C. Modulated evaluation metrics for drug-based ontologies. J Biomed Semant. 2017; 8(1):17. https:\/\/doi.org\/10.1186\/s13326-017-0124-2 .","journal-title":"J Biomed Semant"},{"key":"859_CR35","volume-title":"International Semantic Web Conference","author":"L B\u00fchmann","year":"2015","unstructured":"B\u00fchmann L, Usbeck R, Ngomo A-CN. ASSESS\u2014Automatic Self-Assessment Using Linked Data. In: International Semantic Web Conference. Berlin: Springer: 2015. p. 76\u201389."},{"issue":"194","key":"859_CR36","first-page":"4","volume":"189","author":"J Brooke","year":"1996","unstructured":"Brooke J. SUS-A quick and dirty usability scale. Usability Eval Ind. 1996; 189(194):4\u20137.","journal-title":"Usability Eval Ind"},{"issue":"2","key":"859_CR37","first-page":"29","volume":"8","author":"J Brooke","year":"2013","unstructured":"Brooke J. SUS: a retrospective. J Usability Stud. 2013; 8(2):29\u201340.","journal-title":"J Usability Stud"},{"key":"859_CR38","unstructured":"U.S. Department of Health & Human Services. System Usability Scale (SUS). 2019. https:\/\/www.usability.gov\/how-to-and-tools\/methods\/system-usability-scale.html . Accessed 23 Apr 2019."},{"key":"859_CR39","volume-title":"Quantifying the User Experience: Practical Statistics for User Research","author":"J Sauro","year":"2016","unstructured":"Sauro J, Lewis JR. Quantifying the User Experience: Practical Statistics for User Research. Cambridge: Morgan Kaufmann; 2016."},{"issue":"5","key":"859_CR40","doi-asserted-by":"publisher","first-page":"17","DOI":"10.1109\/MITP.2005.122","volume":"7","author":"S Liu","year":"2005","unstructured":"Liu S, Ma W, Moore R, Ganesan V, Nelson S. Rxnorm: prescription for electronic drug information exchange. IT Prof. 2005; 7(5):17\u201323.","journal-title":"IT Prof"},{"key":"859_CR41","unstructured":"Veterans Health Administration National Drug File. Natl Cent Biomed Ontol BioPortal. 2015. http:\/\/bioportal.bioontology.org\/ontologies\/VANDF . Accessed 10 Apr 2017."},{"key":"859_CR42","unstructured":"The Drug Ontology. Natl Cent Biomed Ontol BioPortal. 2015. http:\/\/bioportal.bioontology.org\/ontologies\/DRON . Accessed 10 Apr 2017."},{"key":"859_CR43","unstructured":"The Drug-Drug Interactions Ontology. Natl Center Biomed Ontol BioPortal. 2016. http:\/\/bioportal.bioontology.org\/ontologies\/DINTO . Accessed 10 Apr 2017."},{"key":"859_CR44","unstructured":"Vaccine Ontology. Natl Center Biomed Ontol BioPortal. 2016. http:\/\/bioportal.bioontology.org\/ontologies\/VO . Accessed 10 Apr 2017."},{"key":"859_CR45","unstructured":"Pharmacovigilance Ontology. Natl Center Biomed Ontol Bioportal. 2011. http:\/\/bioportal.bioontology.org\/ontologies\/PVONTO . Accessed 10 Apr 2017."},{"key":"859_CR46","unstructured":"Introduction to Ontologies with Protege. Department of Computer Science and Software Engineering - Cal Poly, San Luis Obispo. 2011. https:\/\/wiki.csc.calpoly.edu\/OntologyTutorial\/wiki\/IntroductionToOntologiesWithProtege . Accessed 26 Nov 2016."},{"key":"859_CR47","unstructured":"Time Event Ontology. School of Biomedical Informatics, Ontology Research Group, University of Texas Health Science Center. 2016. https:\/\/sbmi.uth.edu\/ontology\/project\/time-event-ontology.htm . Accessed 10 Apr 2019."},{"key":"859_CR48","unstructured":"Lin Y, Harris MR, Manion FJ, et al.Development of a BFO-based Informed Consent Ontology (ICO). Houston: Proceedings of the 5th International Conference on Biomedical Ontology; 2014. http:\/\/ceurws.org\/Vol-1327\/icbo2014_paper_54.pdf . Accessed 28 Apr 2016."}],"container-title":["BMC Medical Informatics and Decision Making"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1186\/s12911-019-0859-z.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/article\/10.1186\/s12911-019-0859-z\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1186\/s12911-019-0859-z.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2022,9,25]],"date-time":"2022-09-25T08:43:34Z","timestamp":1664095414000},"score":1,"resource":{"primary":{"URL":"https:\/\/bmcmedinformdecismak.biomedcentral.com\/articles\/10.1186\/s12911-019-0859-z"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2019,8]]},"references-count":48,"journal-issue":{"issue":"S4","published-print":{"date-parts":[[2019,8]]}},"alternative-id":["859"],"URL":"https:\/\/doi.org\/10.1186\/s12911-019-0859-z","relation":{},"ISSN":["1472-6947"],"issn-type":[{"value":"1472-6947","type":"electronic"}],"subject":[],"published":{"date-parts":[[2019,8]]},"assertion":[{"value":"8 August 2019","order":1,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"Not applicable.","order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Ethics approval and consent to participate"}},{"value":"Not applicable.","order":2,"name":"Ethics","group":{"name":"EthicsHeading","label":"Consent for publication"}},{"value":"The authors declare that they have no competing interests.","order":3,"name":"Ethics","group":{"name":"EthicsHeading","label":"Competing interests"}}],"article-number":"152"}}