{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,9,27]],"date-time":"2026-09-27T03:26:26Z","timestamp":1790479586331,"version":"4.1.0"},"reference-count":43,"publisher":"Public Library of Science (PLoS)","issue":"10","license":[{"start":{"date-parts":[[2014,10,29]],"date-time":"2014-10-29T00:00:00Z","timestamp":1414540800000},"content-version":"unspecified","delay-in-days":0,"URL":"http:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"content-domain":{"domain":["www.plosone.org"],"crossmark-restriction":false},"short-container-title":["PLoS ONE"],"DOI":"10.1371\/journal.pone.0107633","type":"journal-article","created":{"date-parts":[[2014,10,29]],"date-time":"2014-10-29T17:52:14Z","timestamp":1414605134000},"page":"e107633","update-policy":"https:\/\/doi.org\/10.1371\/journal.pone.corrections_policy","source":"Crossref","is-referenced-by-count":14,"title":["Exploration of Analysis Methods for Diagnostic Imaging Tests: Problems with ROC AUC and Confidence Scores in CT Colonography"],"prefix":"10.1371","volume":"9","author":[{"given":"Susan","family":"Mallett","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Steve","family":"Halligan","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Gary S.","family":"Collins","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Doug G.","family":"Altman","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"340","published-online":{"date-parts":[[2014,10,29]]},"reference":[{"key":"ref1","doi-asserted-by":"crossref","first-page":"e3999","DOI":"10.1136\/bmj.e3999","article-title":"Interpreting diagnostic accuracy studies for patient care","volume":"345","author":"S Mallett","year":"2012","journal-title":"BMJ"},{"key":"ref2","doi-asserted-by":"crossref","first-page":"469","DOI":"10.1148\/radiol.10100354","article-title":"Incremental benefit of computer-aided detection when used as a second and concurrent reader of CT colonographic data: multiobserver study","volume":"258","author":"S Halligan","year":"2011","journal-title":"Radiology"},{"key":"ref3","doi-asserted-by":"crossref","first-page":"447","DOI":"10.1177\/0272989X9701700410","article-title":"Application of treatment thresholds to diagnostic-test evaluation: an alternative to the comparison of areas under receiver operating characteristic curves","volume":"17","author":"KG Moons","year":"1997","journal-title":"Med Decis Making"},{"key":"ref4","doi-asserted-by":"crossref","first-page":"89","DOI":"10.1016\/S0010-4825(99)00025-6","article-title":"An improved measure for comparing diagnostic tests","volume":"30","author":"NM Adams","year":"2000","journal-title":"Comput Biol Med"},{"key":"ref5","doi-asserted-by":"crossref","first-page":"53","DOI":"10.1186\/1472-6947-8-53","article-title":"Extensions to decision curve analysis, a novel method for evaluating diagnostic tests, prediction models and molecular markers","volume":"8","author":"AJ Vickers","year":"2008","journal-title":"BMC Med Inform Decis Mak"},{"key":"ref6","unstructured":"Staff DGfIaF (2012) Clinical Performance Assessment: Considerations for Computer-Assisted Detection Devices Applied to Radiology Images and Radiology Device Data - Premarket Approval (PMA) and Premarket Notification [510(k)] Submissions. In: Center for Devices and Radiological Health FaDA, U.S. Government, editor."},{"key":"ref7","doi-asserted-by":"crossref","first-page":"1508","DOI":"10.1016\/j.acra.2012.09.012","article-title":"Multi-reader ROC studies with split-plot designs: a comparison of statistical methods","volume":"19","author":"NA Obuchowski","year":"2012","journal-title":"Acad Radiol"},{"key":"ref8","article-title":"Patients&apos; and healthcare professionals&apos; values regarding true- and false-positive diagnosis during colorectal cancer screening with CT colonography: A discrete choice experiment","author":"D Boone","year":"2013","journal-title":"PLOS ONE"},{"key":"ref9","doi-asserted-by":"crossref","first-page":"1635","DOI":"10.1136\/bmj.320.7250.1635","article-title":"US women&apos;s attitudes to false positive mammography results and detection of ductal carcinoma in situ: cross sectional survey","volume":"320","author":"LM Schwartz","year":"2000","journal-title":"BMJ"},{"key":"ref10","doi-asserted-by":"crossref","first-page":"95","DOI":"10.1177\/0272989X9101100204","article-title":"The area under the ROC curve and its competitors","volume":"11","author":"J Hilden","year":"1991","journal-title":"Med Decis Making"},{"key":"ref11","doi-asserted-by":"crossref","first-page":"873","DOI":"10.1148\/radiology.218.3.r01mr29873","article-title":"Comparison of full-field digital mammography with screen-film mammography for cancer detection: results of 4,945 paired examinations","volume":"218","author":"JM Lewin","year":"2001","journal-title":"Radiology"},{"key":"ref12","doi-asserted-by":"crossref","first-page":"1690","DOI":"10.1053\/j.gastro.2006.09.051","article-title":"Computed tomographic colonography: assessment of radiologist performance with and without computer-aided detection","volume":"131","author":"S Halligan","year":"2006","journal-title":"Gastroenterology"},{"key":"ref13","doi-asserted-by":"crossref","first-page":"380","DOI":"10.1016\/S1076-6332(97)80121-3","article-title":"The \u201cproper\u201d binormal model: parametric receiver operating characteristic curve estimation with degenerate data","volume":"4","author":"X Pan","year":"1997","journal-title":"Acad Radiol"},{"key":"ref14","doi-asserted-by":"crossref","first-page":"720","DOI":"10.1097\/00004424-198609000-00009","article-title":"ROC methodology in radiologic imaging","volume":"21","author":"CE Metz","year":"1986","journal-title":"Invest Radiol"},{"key":"ref15","first-page":"980","article-title":"Multireader, multicase receiver operating characteristic analysis: an empirical comparison of five methods","volume":"11","author":"NA Obuchowski","year":"2004","journal-title":"Acad Radiol"},{"key":"ref16","doi-asserted-by":"crossref","first-page":"1452","DOI":"10.1016\/j.acra.2012.09.003","article-title":"Teaching receiver operating characteristic analysis: an interactive laboratory exercise","volume":"19","author":"J Eng","year":"2012","journal-title":"Acad Radiol"},{"key":"ref17","doi-asserted-by":"crossref","first-page":"637","DOI":"10.1148\/radiol.2303020921","article-title":"T2-weighted MR imaging in the assessment of cirrhotic liver","volume":"230","author":"HK Hussain","year":"2004","journal-title":"Radiology"},{"key":"ref18","doi-asserted-by":"crossref","first-page":"177","DOI":"10.1148\/radiol.2453062042","article-title":"Peripheral zone prostate cancer: accuracy of different interpretative approaches with MR and MR spectroscopic imaging","volume":"246","author":"AC Westphalen","year":"2008","journal-title":"Radiology"},{"key":"ref19","doi-asserted-by":"crossref","first-page":"140","DOI":"10.1148\/radiol.2451061116","article-title":"Computer-aided detection of colorectal polyps: can it improve sensitivity of less-experienced readers? Preliminary findings","volume":"245","author":"ME Baker","year":"2007","journal-title":"Radiology"},{"key":"ref20","doi-asserted-by":"crossref","first-page":"148","DOI":"10.1148\/radiol.2453062161","article-title":"CT colonography with computer-aided detection as a second reader: observer performance study","volume":"246","author":"N Petrick","year":"2008","journal-title":"Radiology"},{"key":"ref21","doi-asserted-by":"crossref","first-page":"567","DOI":"10.2307\/2533958","article-title":"Nonparametric analysis of clustered ROC curve data","volume":"53","author":"NA Obuchowski","year":"1997","journal-title":"Biometrics"},{"key":"ref22","doi-asserted-by":"crossref","first-page":"1469","DOI":"10.2214\/ajr.182.6.1821469","article-title":"Preoperative MRI of rectal cancer with and without rectal water filling: an intraindividual comparison","volume":"182","author":"MJ Kim","year":"2004","journal-title":"AJR Am J Roentgenol"},{"key":"ref23","doi-asserted-by":"crossref","first-page":"289","DOI":"10.1016\/j.acra.2005.11.006","article-title":"Using a human visual system model to optimize soft-copy mammography display: influence of veiling glare","volume":"13","author":"EA Krupinski","year":"2006","journal-title":"Acad Radiol"},{"key":"ref24","doi-asserted-by":"crossref","unstructured":"Krzanowski WJ, Hand DJ (2009) <sub>ROC curves for continuous data<\/sub>: CRC\/Chapman and Hall.","DOI":"10.1201\/9781439800225"},{"key":"ref25","doi-asserted-by":"crossref","first-page":"211","DOI":"10.1007\/BF03168117","article-title":"Some methodological questions concerning receiver operating characteristic (ROC) analysis as a method for assessing image quality in radiology","volume":"3","author":"MB Harrington","year":"1990","journal-title":"J Digit Imaging"},{"key":"ref26","doi-asserted-by":"crossref","first-page":"1261","DOI":"10.1029\/WR017i005p01261","article-title":"The mathematical structure of rainfall representations: A review of the stochastic rainfall models","volume":"17","author":"E Waymire","year":"1981","journal-title":"Water resources research"},{"key":"ref27","doi-asserted-by":"crossref","first-page":"181","DOI":"10.1037\/0033-2909.99.2.181","article-title":"Form of empirical ROCs in discrimination and diagnostic tasks: implications for theory and measurement of performance","volume":"99","author":"JA Swets","year":"1986","journal-title":"Psychol Bull"},{"key":"ref28","doi-asserted-by":"crossref","first-page":"197","DOI":"10.1177\/0272989X8800800308","article-title":"The robustness of the \u201cbinormal\u201d assumptions used in fitting ROC curves","volume":"8","author":"JA Hanley","year":"1988","journal-title":"Med Decis Making"},{"key":"ref29","doi-asserted-by":"crossref","first-page":"871","DOI":"10.1016\/j.acra.2007.03.014","article-title":"\u201cBinary\u201d and \u201cnon-binary\u201d detection tasks: are current performance measures optimal?","volume":"14","author":"D Gur","year":"2007","journal-title":"Acad Radiol"},{"key":"ref30","doi-asserted-by":"crossref","first-page":"420","DOI":"10.1016\/S1076-6332(00)80382-7","article-title":"A contaminated binormal model for ROC data: Part I. Some interesting examples of binormal degeneracy","volume":"7","author":"DD Dorfman","year":"2000","journal-title":"Acad Radiol"},{"key":"ref31","doi-asserted-by":"crossref","first-page":"234","DOI":"10.1097\/00004424-198903000-00012","article-title":"Some practical issues of experimental design and data analysis in radiological ROC studies","volume":"24","author":"CE Metz","year":"1989","journal-title":"Invest Radiol"},{"key":"ref32","doi-asserted-by":"crossref","first-page":"2191","DOI":"10.1002\/(SICI)1097-0258(19990915\/30)18:17\/18<2191::AID-SIM249>3.0.CO;2-M","article-title":"Modelling ordered categorical data: recent advances and future challenges","volume":"18","author":"A Agresti","year":"1999","journal-title":"Stat Med"},{"key":"ref33","doi-asserted-by":"crossref","first-page":"645","DOI":"10.1118\/1.2830376","article-title":"Anniversary paper: evaluation of medical imaging systems","volume":"35","author":"EA Krupinski","year":"2008","journal-title":"Med Phys"},{"key":"ref34","doi-asserted-by":"crossref","first-page":"12","DOI":"10.1148\/radiol.2471071321","article-title":"Comparing areas under receiver operating characteristic curves: potential impact of the \u201cLast\u201d experimentally measured operating point","volume":"247","author":"D Gur","year":"2008","journal-title":"Radiology"},{"key":"ref35","doi-asserted-by":"crossref","first-page":"84","DOI":"10.1016\/j.acra.2005.09.086","article-title":"Comparison of independent double readings and computer-aided diagnosis (CAD) for the diagnosis of breast calcifications","volume":"13","author":"Y Jiang","year":"2006","journal-title":"Acad Radiol"},{"key":"ref36","doi-asserted-by":"crossref","first-page":"157","DOI":"10.1002\/sim.2929","article-title":"Evaluating the added predictive ability of a new marker: from area under the ROC curve to reclassification and beyond","volume":"27","author":"MJ Pencina","year":"2008","journal-title":"Stat Med"},{"key":"ref37","doi-asserted-by":"crossref","first-page":"173","DOI":"10.1002\/sim.2991","article-title":"Comments on \u2018Evaluating the added predictive ability of a new marker: From area under the ROC curve to reclassification and beyond\u2019 by M. J. Pencina et al., Statistics in Medicine (DOI: 10.1002\/sim.2929)","volume":"27","author":"MS Pepe","year":"2008","journal-title":"Stat Med"},{"key":"ref38","doi-asserted-by":"crossref","first-page":"364","DOI":"10.2214\/ajr.184.2.01840364","article-title":"ROC analysis","volume":"184","author":"NA Obuchowski","year":"2005","journal-title":"AJR Am J Roentgenol"},{"key":"ref39","doi-asserted-by":"crossref","first-page":"102","DOI":"10.1177\/0272989X9101100205","article-title":"Signal detectability: the use of ROC curves and their analyses","volume":"11","author":"RM Centor","year":"1991","journal-title":"Med Decis Making"},{"key":"ref40","doi-asserted-by":"crossref","first-page":"341","DOI":"10.1016\/S1076-6332(00)80008-2","article-title":"Components-of-variance models and multiple-bootstrap experiments: an alternative method for random-effects, receiver operating characteristic analysis","volume":"7","author":"SV Beiden","year":"2000","journal-title":"Acad Radiol"},{"key":"ref41","doi-asserted-by":"crossref","first-page":"723","DOI":"10.1097\/00004424-199209000-00015","article-title":"Receiver operating characteristic rating analysis. Generalization to the population of readers and patients with the jackknife method","volume":"27","author":"DD Dorfman","year":"1992","journal-title":"Invest Radiol"},{"key":"ref42","doi-asserted-by":"crossref","first-page":"414","DOI":"10.1016\/j.acra.2005.12.012","article-title":"A permutation test for comparing ROC curves in multireader studies a multi-reader ROC, permutation test","volume":"13","author":"AI Bandos","year":"2006","journal-title":"Acad Radiol"},{"key":"ref43","doi-asserted-by":"crossref","first-page":"353","DOI":"10.1016\/j.acra.2005.11.030","article-title":"One-shot estimate of MRMC variance: AUC","volume":"13","author":"BD Gallas","year":"2006","journal-title":"Acad Radiol"}],"container-title":["PLoS ONE"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/dx.plos.org\/10.1371\/journal.pone.0107633","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2020,5,9]],"date-time":"2020-05-09T09:16:52Z","timestamp":1589015812000},"score":1,"resource":{"primary":{"URL":"https:\/\/dx.plos.org\/10.1371\/journal.pone.0107633"}},"subtitle":[],"editor":[{"given":"Jason","family":"Mulvenna","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"editor"}]}],"short-title":[],"issued":{"date-parts":[[2014,10,29]]},"references-count":43,"journal-issue":{"issue":"10","published-online":{"date-parts":[[2014,10,29]]}},"URL":"https:\/\/doi.org\/10.1371\/journal.pone.0107633","relation":{},"ISSN":["1932-6203"],"issn-type":[{"value":"1932-6203","type":"electronic"}],"subject":[],"published":{"date-parts":[[2014,10,29]]}}}