{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,1,17]],"date-time":"2026-01-17T10:20:12Z","timestamp":1768645212729,"version":"3.49.0"},"reference-count":21,"publisher":"Oxford University Press (OUP)","issue":"11","license":[{"start":{"date-parts":[[2020,3,10]],"date-time":"2020-03-10T00:00:00Z","timestamp":1583798400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/academic.oup.com\/journals\/pages\/open_access\/funder_policies\/chorus\/standard_publication_model"}],"funder":[{"DOI":"10.13039\/100000001","name":"National Science Foundation","doi-asserted-by":"publisher","award":["DMS-1521092"],"award-info":[{"award-number":["DMS-1521092"]}],"id":[{"id":"10.13039\/100000001","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2020,6,1]]},"abstract":"<jats:title>Abstract<\/jats:title>\n               <jats:sec>\n                  <jats:title>Motivation<\/jats:title>\n                  <jats:p>Cluster analysis is widely used to identify interesting subgroups in biomedical data. Since true class labels are unknown in the unsupervised setting, it is challenging to validate any cluster obtained computationally, an important problem barely addressed by the research community.<\/jats:p>\n               <\/jats:sec>\n               <jats:sec>\n                  <jats:title>Results<\/jats:title>\n                  <jats:p>We have developed a toolkit called covering point set (CPS) analysis to quantify uncertainty at the levels of individual clusters and overall partitions. Functions have been developed to effectively visualize the inherent variation in any cluster for data of high dimension, and provide more comprehensive view on potentially interesting subgroups in the data. Applying to three usage scenarios for biomedical data, we demonstrate that CPS analysis is more effective for evaluating uncertainty of clusters comparing to state-of-the-art measurements. We also showcase how to use CPS analysis to select data generation technologies or visualization methods.<\/jats:p>\n               <\/jats:sec>\n               <jats:sec>\n                  <jats:title>Availability and implementation<\/jats:title>\n                  <jats:p>The method is implemented in an R package called OTclust, available on CRAN.<\/jats:p>\n               <\/jats:sec>\n               <jats:sec>\n                  <jats:title>Contact<\/jats:title>\n                  <jats:p>lzz46@psu.edu or jiali@psu.edu<\/jats:p>\n               <\/jats:sec>\n               <jats:sec>\n                  <jats:title>Supplementary information<\/jats:title>\n                  <jats:p>Supplementary data are available at Bioinformatics online.<\/jats:p>\n               <\/jats:sec>","DOI":"10.1093\/bioinformatics\/btaa165","type":"journal-article","created":{"date-parts":[[2020,3,6]],"date-time":"2020-03-06T12:26:22Z","timestamp":1583497582000},"page":"3516-3521","source":"Crossref","is-referenced-by-count":10,"title":["CPS analysis: self-contained validation of biomedical data clustering"],"prefix":"10.1093","volume":"36","author":[{"given":"Lixiang","family":"Zhang","sequence":"first","affiliation":[{"name":"Department of Statistics , The Pennsylvania State University, University Park, PA 16802, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Lin","family":"Lin","sequence":"additional","affiliation":[{"name":"Department of Statistics , The Pennsylvania State University, University Park, PA 16802, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Jia","family":"Li","sequence":"additional","affiliation":[{"name":"Department of Statistics , The Pennsylvania State University, University Park, PA 16802, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"286","published-online":{"date-parts":[[2020,3,10]]},"reference":[{"key":"2023062312020818000_btaa165-B1","doi-asserted-by":"crossref","first-page":"44","DOI":"10.1007\/978-3-642-23780-5_13","volume-title":"Joint European Conference on Machine Learning and Knowledge Discovery in Databases","author":"Almeida","year":"2011"},{"key":"2023062312020818000_btaa165-B2","doi-asserted-by":"crossref","first-page":"95","DOI":"10.1080\/01969727408546059","article-title":"Well-separated clusters and optimal fuzzy partitions","volume":"4","author":"Dunn","year":"1974","journal-title":"J. Cybern"},{"key":"2023062312020818000_btaa165-B3","doi-asserted-by":"crossref","first-page":"776","DOI":"10.1126\/science.1247651","article-title":"Massively parallel single-cell RNA-seq for marker-free decomposition of tissues into cell types","volume":"343","author":"Jaitin","year":"2014","journal-title":"Science"},{"key":"2023062312020818000_btaa165-B4","doi-asserted-by":"crossref","first-page":"483","DOI":"10.1038\/nmeth.4236","article-title":"Sc3: consensus clustering of single-cell RNA-seq data","volume":"14","author":"Kiselev","year":"2017","journal-title":"Nat. Methods"},{"key":"2023062312020818000_btaa165-B5","doi-asserted-by":"crossref","first-page":"1365","DOI":"10.1089\/cmb.2018.0027","article-title":"Structure-aware principal component analysis for single-cell RNA-seq data","volume":"25","author":"Lall","year":"2018","journal-title":"J. Comput. Biol"},{"key":"2023062312020818000_btaa165-B6","doi-asserted-by":"crossref","first-page":"359","DOI":"10.1002\/sam.11418","article-title":"Optimal transport, mean partition, and uncertainty assessment in cluster analysis","volume":"12","author":"Li","year":"2019","journal-title":"Stat. Anal. Data Min"},{"key":"2023062312020818000_btaa165-B7","author":"Malika","year":"2015"},{"key":"2023062312020818000_btaa165-B8","author":"McInnes","year":"2018"},{"key":"2023062312020818000_btaa165-B9","doi-asserted-by":"crossref","first-page":"873","DOI":"10.1016\/j.jmva.2006.11.013","article-title":"Comparing clusterings \u2013 an information based distance","volume":"98","author":"Meil\u0103","year":"2007","journal-title":"J. Multivar. Anal"},{"key":"2023062312020818000_btaa165-B10","doi-asserted-by":"crossref","first-page":"1053","DOI":"10.1038\/nbt.2967","article-title":"Low-coverage single-cell mrna sequencing reveals cellular heterogeneity and activated signaling pathways in developing cerebral cortex","volume":"32","author":"Pollen","year":"2014","journal-title":"Nat. Biotechnol"},{"key":"2023062312020818000_btaa165-B11","doi-asserted-by":"crossref","first-page":"846","DOI":"10.1080\/01621459.1971.10482356","article-title":"Objective criteria for the evaluation of clustering methods","volume":"66","author":"Rand","year":"1971","journal-title":"J. Am. Stat. Assoc"},{"key":"2023062312020818000_btaa165-B12","doi-asserted-by":"crossref","first-page":"53","DOI":"10.1016\/0377-0427(87)90125-7","article-title":"Silhouettes: a graphical aid to the interpretation and validation of cluster analysis","volume":"20","author":"Rousseeuw","year":"1987","journal-title":"J. Comput. Appl. Math"},{"key":"2023062312020818000_btaa165-B13","doi-asserted-by":"crossref","first-page":"236","DOI":"10.1038\/nature12172","article-title":"Single-cell transcriptomics reveals bimodality in expression and splicing in immune cells","volume":"498","author":"Shalek","year":"2013","journal-title":"Nature"},{"key":"2023062312020818000_btaa165-B14","doi-asserted-by":"crossref","first-page":"618","DOI":"10.1038\/nrg3542","article-title":"Single-cell sequencing-based technologies will revolutionize whole-organism science","volume":"14","author":"Shapiro","year":"2013","journal-title":"Nat. Rev. Genet"},{"key":"2023062312020818000_btaa165-B15","first-page":"267","volume-title":"Psychometrika","author":"Thorndike","year":"1953"},{"key":"2023062312020818000_btaa165-B16","doi-asserted-by":"crossref","first-page":"411","DOI":"10.1111\/1467-9868.00293","article-title":"Estimating the number of clusters in a data set via the gap statistic","volume":"63","author":"Tibshirani","year":"2001","journal-title":"J. R. Stat. Soc. B"},{"key":"2023062312020818000_btaa165-B17","doi-asserted-by":"crossref","first-page":"371","DOI":"10.1038\/nature13173","article-title":"Reconstructing lineage hierarchies of the distal lung epithelium using single-cell RNA-seq","volume":"509","author":"Treutlein","year":"2014","journal-title":"Nature"},{"key":"2023062312020818000_btaa165-B18","first-page":"2579","article-title":"Visualizing data using t-SNE","volume":"9","author":"van der Maaten","year":"2008","journal-title":"J. Mach. Learn. Res"},{"key":"2023062312020818000_btaa165-B19","first-page":"235","article-title":"Clustering stability: an overview","volume":"2","author":"von Luxburg","year":"2010","journal-title":"Found. Trends Mach. Learn"},{"key":"2023062312020818000_btaa165-B20","doi-asserted-by":"crossref","first-page":"1131","DOI":"10.1038\/nsmb.2660","article-title":"Single-cell RNA-seq profiling of human preimplantation embryos and embryonic stem cells","volume":"20","author":"Yan","year":"2013","journal-title":"Nat. Struct. Mol. Biol"},{"key":"2023062312020818000_btaa165-B21","first-page":"1028","author":"Zhou","year":"2005"}],"container-title":["Bioinformatics"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/academic.oup.com\/bioinformatics\/advance-article-pdf\/doi\/10.1093\/bioinformatics\/btaa165\/33049090\/btaa165.pdf","content-type":"application\/pdf","content-version":"am","intended-application":"syndication"},{"URL":"https:\/\/academic.oup.com\/bioinformatics\/article-pdf\/36\/11\/3516\/50670623\/bioinformatics_36_11_3516.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"syndication"},{"URL":"https:\/\/academic.oup.com\/bioinformatics\/article-pdf\/36\/11\/3516\/50670623\/bioinformatics_36_11_3516.pdf","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2023,6,24]],"date-time":"2023-06-24T21:25:08Z","timestamp":1687641908000},"score":1,"resource":{"primary":{"URL":"https:\/\/academic.oup.com\/bioinformatics\/article\/36\/11\/3516\/5802459"}},"subtitle":[],"editor":[{"given":"Jonathan","family":"Wren","sequence":"additional","affiliation":[],"role":[{"role":"editor","vocabulary":"crossref"}]}],"short-title":[],"issued":{"date-parts":[[2020,3,10]]},"references-count":21,"journal-issue":{"issue":"11","published-print":{"date-parts":[[2020,6,1]]}},"URL":"https:\/\/doi.org\/10.1093\/bioinformatics\/btaa165","relation":{},"ISSN":["1367-4803","1367-4811"],"issn-type":[{"value":"1367-4803","type":"print"},{"value":"1367-4811","type":"electronic"}],"subject":[],"published-other":{"date-parts":[[2020,6]]},"published":{"date-parts":[[2020,3,10]]}}}