{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,7]],"date-time":"2026-04-07T20:42:59Z","timestamp":1775594579225,"version":"3.50.1"},"reference-count":32,"publisher":"Springer Science and Business Media LLC","issue":"1","license":[{"start":{"date-parts":[[2011,3,17]],"date-time":"2011-03-17T00:00:00Z","timestamp":1300320000000},"content-version":"tdm","delay-in-days":0,"URL":"http:\/\/creativecommons.org\/licenses\/by\/2.0"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["BMC Bioinformatics"],"published-print":{"date-parts":[[2011,12]]},"DOI":"10.1186\/1471-2105-12-77","type":"journal-article","created":{"date-parts":[[2011,3,17]],"date-time":"2011-03-17T19:37:41Z","timestamp":1300390661000},"source":"Crossref","is-referenced-by-count":11194,"title":["pROC: an open-source package for R and S+ to analyze and compare ROC curves"],"prefix":"10.1186","volume":"12","author":[{"given":"Xavier","family":"Robin","sequence":"first","affiliation":[]},{"given":"Natacha","family":"Turck","sequence":"additional","affiliation":[]},{"given":"Alexandre","family":"Hainard","sequence":"additional","affiliation":[]},{"given":"Natalia","family":"Tiberti","sequence":"additional","affiliation":[]},{"given":"Fr\u00e9d\u00e9rique","family":"Lisacek","sequence":"additional","affiliation":[]},{"given":"Jean-Charles","family":"Sanchez","sequence":"additional","affiliation":[]},{"given":"Markus","family":"M\u00fcller","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2011,3,17]]},"reference":[{"key":"4448_CR1","doi-asserted-by":"publisher","first-page":"990","DOI":"10.1126\/science.182.4116.990","volume":"182","author":"JA Swets","year":"1973","unstructured":"Swets JA: The Relative Operating Characteristic in Psychology. Science 1973, 182: 990\u20131000. 10.1126\/science.182.4116.990","journal-title":"Science"},{"key":"4448_CR2","doi-asserted-by":"crossref","DOI":"10.1093\/oso\/9780198509844.001.0001","volume-title":"The statistical evaluation of medical tests for classification and prediction","author":"MS Pepe","year":"2003","unstructured":"Pepe MS: The statistical evaluation of medical tests for classification and prediction. Oxford: Oxford University Press; 2003."},{"key":"4448_CR3","doi-asserted-by":"publisher","first-page":"198","DOI":"10.1093\/bib\/bbm064","volume":"9","author":"P Sonego","year":"2008","unstructured":"Sonego P, Kocsor A, Pongor S: ROC analysis: applications to the classification of biological sequences and 3D structures. Brief Bioinform 2008, 9: 198\u2013209. 10.1093\/bib\/bbm064","journal-title":"Brief Bioinform"},{"key":"4448_CR4","doi-asserted-by":"publisher","first-page":"861","DOI":"10.1016\/j.patrec.2005.10.010","volume":"27","author":"T Fawcett","year":"2006","unstructured":"Fawcett T: An introduction to ROC analysis. Pattern Recogn Lett 2006, 27: 861\u2013874. 10.1016\/j.patrec.2005.10.010","journal-title":"Pattern Recogn Lett"},{"key":"4448_CR5","doi-asserted-by":"publisher","first-page":"822","DOI":"10.1093\/bioinformatics\/btq037","volume":"26","author":"B Hanczar","year":"2010","unstructured":"Hanczar B, Hua J, Sima C, Weinstein J, Bittner M, Dougherty ER: Small-sample precision of ROC-related estimates. Bioinformatics 2010, 26: 822\u2013830. 10.1093\/bioinformatics\/btq037","journal-title":"Bioinformatics"},{"key":"4448_CR6","doi-asserted-by":"publisher","first-page":"675","DOI":"10.1586\/epr.09.83","volume":"6","author":"X Robin","year":"2009","unstructured":"Robin X, Turck N, Hainard A, Lisacek F, Sanchez JC, M\u00fcller M: Bioinformatics for protein biomarker panel classification: What is needed to bring biomarker panels into in vitro diagnostics? Expert Rev Proteomics 2009, 6: 675\u2013689. 10.1586\/epr.09.83","journal-title":"Expert Rev Proteomics"},{"key":"4448_CR7","doi-asserted-by":"publisher","first-page":"190","DOI":"10.1177\/0272989X8900900307","volume":"9","author":"DK McClish","year":"1989","unstructured":"McClish DK: Analyzing a Portion of the ROC Curve. Med Decis Making 1989, 9: 190\u2013195. 10.1177\/0272989X8900900307","journal-title":"Med Decis Making"},{"key":"4448_CR8","doi-asserted-by":"publisher","first-page":"745","DOI":"10.1148\/radiology.201.3.8939225","volume":"201","author":"Y Jiang","year":"1996","unstructured":"Jiang Y, Metz CE, Nishikawa RM: A receiver operating characteristic partial area index for highly sensitive diagnostic tests. Radiology 1996, 201: 745\u2013750.","journal-title":"Radiology"},{"key":"4448_CR9","doi-asserted-by":"crossref","first-page":"121","DOI":"10.1177\/070674370705200210","volume":"52","author":"DL Streiner","year":"2007","unstructured":"Streiner DL, Cairney J: What's under the ROC? An introduction to receiver operating characteristics curves. Canadian Journal of Psychiatry Revue Canadienne De Psychiatrie 2007, 52: 121\u2013128.","journal-title":"Canadian Journal of Psychiatry Revue Canadienne De Psychiatrie"},{"key":"4448_CR10","doi-asserted-by":"publisher","first-page":"433","DOI":"10.1373\/49.3.433","volume":"49","author":"C Stephan","year":"2003","unstructured":"Stephan C, Wesseling S, Schink T, Jung K: Comparison of Eight Computer Programs for Receiver-Operating Characteristic Analysis. Clin Chem 2003, 49: 433\u2013439. 10.1373\/49.3.433","journal-title":"Clin Chem"},{"key":"4448_CR11","volume-title":"R: A Language and Environment for Statistical Computing","author":"R Development Core Team","year":"2010","unstructured":"R Development Core Team: R: A Language and Environment for Statistical Computing. Vienna, Austria: R Foundation for Statistical Computing; 2010."},{"key":"4448_CR12","doi-asserted-by":"publisher","first-page":"3940","DOI":"10.1093\/bioinformatics\/bti623","volume":"21","author":"T Sing","year":"2005","unstructured":"Sing T, Sander O, Beerenwinkel N, Lengauer T: ROCR: visualizing classifier performance in R. Bioinformatics 2005, 21: 3940\u20133941. 10.1093\/bioinformatics\/bti623","journal-title":"Bioinformatics"},{"key":"4448_CR13","unstructured":"NCAR: verification: Forecast verification utilities v. 1.31.[ http:\/\/CRAN.R-project.org\/package=verification ]"},{"key":"4448_CR14","unstructured":"Carey V, Redestig H: ROC: utilities for ROC, with uarray focus, v. 1.24.0.[ http:\/\/www.bioconductor.org ]"},{"key":"4448_CR15","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1177\/1536867X0900900101","volume":"9","author":"M Pepe","year":"2009","unstructured":"Pepe M, Longton G, Janes H: Estimation and Comparison of Receiver Operating Characteristic Curves. The Stata journal 2009, 9: 1.","journal-title":"The Stata journal"},{"key":"4448_CR16","doi-asserted-by":"publisher","first-page":"839","DOI":"10.1148\/radiology.148.3.6878708","volume":"148","author":"JA Hanley","year":"1983","unstructured":"Hanley JA, McNeil BJ: A method of comparing the areas under receiver operating characteristic curves derived from the same cases. Radiology 1983, 148: 839\u2013843.","journal-title":"Radiology"},{"key":"4448_CR17","doi-asserted-by":"publisher","first-page":"837","DOI":"10.2307\/2531595","volume":"44","author":"ER DeLong","year":"1988","unstructured":"DeLong ER, DeLong DM, Clarke-Pearson DL: Comparing the Areas under Two or More Correlated Receiver Operating Characteristic Curves: A Nonparametric Approach. Biometrics 1988, 44: 837\u2013845. 10.2307\/2531595","journal-title":"Biometrics"},{"key":"4448_CR18","doi-asserted-by":"publisher","first-page":"2873","DOI":"10.1002\/sim.2149","volume":"24","author":"AI Bandos","year":"2005","unstructured":"Bandos AI, Rockette HE, Gur D: A permutation test sensitive to differences in areas for comparing ROC curves from a paired design. Stat Med 2005, 24: 2873\u20132893. 10.1002\/sim.2149","journal-title":"Stat Med"},{"key":"4448_CR19","doi-asserted-by":"publisher","first-page":"364","DOI":"10.1093\/biostatistics\/kxm036","volume":"9","author":"TM Braun","year":"2008","unstructured":"Braun TM, Alonzo TA: A modified sign test for comparing paired ROC curves. Biostat 2008, 9: 364\u2013372. 10.1093\/biostatistics\/kxm036","journal-title":"Biostat"},{"key":"4448_CR20","doi-asserted-by":"publisher","first-page":"835","DOI":"10.1093\/biomet\/83.4.835","volume":"83","author":"ES Venkatraman","year":"1996","unstructured":"Venkatraman ES, Begg CB: A distribution-free procedure for comparing receiver operating characteristic curves from a paired experiment. Biometrika 1996, 83: 835\u2013848. 10.1093\/biomet\/83.4.835","journal-title":"Biometrika"},{"key":"4448_CR21","doi-asserted-by":"publisher","first-page":"414","DOI":"10.1016\/j.acra.2005.12.012","volume":"13","author":"AI Bandos","year":"2006","unstructured":"Bandos AI, Rockette HE, Gur D: A Permutation Test for Comparing ROC Curves in Multireader Studies: A Multi-reader ROC, Permutation Test. Acad Radiol 2006, 13: 414\u2013420. 10.1016\/j.acra.2005.12.012","journal-title":"Acad Radiol"},{"key":"4448_CR22","doi-asserted-by":"publisher","first-page":"1985","DOI":"10.1080\/03610928808829727","volume":"17","author":"A Moise","year":"1988","unstructured":"Moise A, Clement B, Raissis M: A test for crossing receiver operating characteristic (roc) curves. Communications in Statistics - Theory and Methods 1988, 17: 1985\u20132003. 10.1080\/03610928808829727","journal-title":"Communications in Statistics - Theory and Methods"},{"key":"4448_CR23","doi-asserted-by":"publisher","first-page":"1134","DOI":"10.1111\/j.0006-341X.2000.01134.x","volume":"56","author":"ES Venkatraman","year":"2000","unstructured":"Venkatraman ES: A Permutation Test to Compare Receiver Operating Characteristic Curves. Biometrics 2000, 56: 1134\u20131138. 10.1111\/j.0006-341X.2000.01134.x","journal-title":"Biometrics"},{"key":"4448_CR24","doi-asserted-by":"publisher","first-page":"499","DOI":"10.1002\/sim.4780130513","volume":"13","author":"G Campbell","year":"1994","unstructured":"Campbell G: Advances in statistical methodology for the evaluation of diagnostic and laboratory tests. Stat Med 1994, 13: 499\u2013508. 10.1002\/sim.4780130513","journal-title":"Stat Med"},{"key":"4448_CR25","unstructured":"Wickham H: plyr: Tools for splitting, applying and combining data v. 1.4.[ http:\/\/CRAN.R-project.org\/package=plyr ]"},{"key":"4448_CR26","doi-asserted-by":"publisher","first-page":"1141","DOI":"10.1002\/(SICI)1097-0258(20000515)19:9<1141::AID-SIM479>3.0.CO;2-F","volume":"19","author":"J Carpenter","year":"2000","unstructured":"Carpenter J, Bithell J: Bootstrap confidence intervals: when, which, what? A practical guide for medical statisticians. Stat Med 2000, 19: 1141\u20131164. 10.1002\/(SICI)1097-0258(20000515)19:9<1141::AID-SIM479>3.0.CO;2-F","journal-title":"Stat Med"},{"key":"4448_CR27","doi-asserted-by":"publisher","first-page":"1033","DOI":"10.1002\/(SICI)1097-0258(19980515)17:9<1033::AID-SIM784>3.0.CO;2-Z","volume":"17","author":"CE Metz","year":"1998","unstructured":"Metz CE, Herman BA, Shen JH: Maximum likelihood estimation of receiver operating characteristic (ROC) curves from continuously-distributed data. Stat Med 1998, 17: 1033\u20131053. 10.1002\/(SICI)1097-0258(19980515)17:9<1033::AID-SIM784>3.0.CO;2-Z","journal-title":"Stat Med"},{"key":"4448_CR28","doi-asserted-by":"publisher","first-page":"197","DOI":"10.1177\/0272989X8800800308","volume":"8","author":"JA Hanley","year":"1988","unstructured":"Hanley JA: The robustness of the \"binormal\" assumptions used in fitting ROC curves. Med Decis Making 1988, 8: 197\u2013203. 10.1177\/0272989X8800800308","journal-title":"Med Decis Making"},{"key":"4448_CR29","doi-asserted-by":"publisher","first-page":"2143","DOI":"10.1002\/(SICI)1097-0258(19971015)16:19<2143::AID-SIM655>3.0.CO;2-3","volume":"16","author":"KH Zou","year":"1997","unstructured":"Zou KH, Hall WJ, Shapiro DE: Smooth non-parametric receiver operating characteristic (ROC) curves for continuous diagnostic tests. Stat Med 1997, 16: 2143\u20132156. 10.1002\/(SICI)1097-0258(19971015)16:19<2143::AID-SIM655>3.0.CO;2-3","journal-title":"Stat Med"},{"key":"4448_CR30","doi-asserted-by":"publisher","DOI":"10.1007\/978-0-387-21706-2","volume-title":"Modern Applied Statistics with S","author":"WN Venables","year":"2002","unstructured":"Venables WN, Ripley BD: Modern Applied Statistics with S. Fourth edition. New York: Springer; 2002.","edition":"Fourth"},{"key":"4448_CR31","doi-asserted-by":"publisher","first-page":"107","DOI":"10.1007\/s00134-009-1641-y","volume":"36","author":"N Turck","year":"2010","unstructured":"Turck N, Vutskits L, Sanchez-Pena P, Robin X, Hainard A, Gex-Fabry M, Fouda C, Bassem H, Mueller M, Lisacek F, et al.: A multiparameter panel method for outcome prediction following aneurysmal subarachnoid hemorrhage. Intensive Care Med 2010, 36: 107\u2013115. 10.1007\/s00134-009-1641-y","journal-title":"Intensive Care Med"},{"key":"4448_CR32","doi-asserted-by":"publisher","DOI":"10.1007\/b137845","volume-title":"Statistical methods in bioinformatics","author":"WJ Ewens","year":"2005","unstructured":"Ewens WJ, Grant GR: Statistics (i): An Introduction to Statistical Inference. In Statistical methods in bioinformatics. New York: Springer-Verlag; 2005."}],"container-title":["BMC Bioinformatics"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1186\/1471-2105-12-77.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/article\/10.1186\/1471-2105-12-77\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1186\/1471-2105-12-77.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,4,5]],"date-time":"2024-04-05T08:15:37Z","timestamp":1712304937000},"score":1,"resource":{"primary":{"URL":"https:\/\/bmcbioinformatics.biomedcentral.com\/articles\/10.1186\/1471-2105-12-77"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2011,3,17]]},"references-count":32,"journal-issue":{"issue":"1","published-print":{"date-parts":[[2011,12]]}},"alternative-id":["4448"],"URL":"https:\/\/doi.org\/10.1186\/1471-2105-12-77","relation":{},"ISSN":["1471-2105"],"issn-type":[{"value":"1471-2105","type":"electronic"}],"subject":[],"published":{"date-parts":[[2011,3,17]]},"article-number":"77"}}