{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,5,19]],"date-time":"2026-05-19T09:23:25Z","timestamp":1779182605468,"version":"3.51.4"},"reference-count":63,"publisher":"Springer Science and Business Media LLC","issue":"1","license":[{"start":{"date-parts":[[2012,9,18]],"date-time":"2012-09-18T00:00:00Z","timestamp":1347926400000},"content-version":"tdm","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["Knowl Inf Syst"],"published-print":{"date-parts":[[2013,4]]},"DOI":"10.1007\/s10115-012-0558-x","type":"journal-article","created":{"date-parts":[[2012,9,17]],"date-time":"2012-09-17T12:57:53Z","timestamp":1347886673000},"page":"131-152","source":"Crossref","is-referenced-by-count":18,"title":["On measuring the performance of binary classifiers"],"prefix":"10.1007","volume":"35","author":[{"given":"Charles","family":"Parker","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2012,9,18]]},"reference":[{"key":"558_CR1","unstructured":"Aeberhard S, Coomans D, de Vel O (1992) Comparison of classifiers in high dimensional settings. Technical Report 92\u201302, Department of Computer Science and Department of Mathematics and Statistics, James Cook University, North Queensland"},{"issue":"4","key":"558_CR2","doi-asserted-by":"crossref","first-page":"1828","DOI":"10.1016\/j.csda.2010.11.018","volume":"55","author":"A Airola","year":"2011","unstructured":"Airola A, Pahikkala T, Waegeman W, De Baets B, Salakoski T (2011) An experimental comparison of cross-validation techniques for estimating the area under the ROC curve. Comput Stat Data Anal 55(4):1828\u20131844. doi: 10.1016\/j.csda.2010.11.018","journal-title":"Comput Stat Data Anal"},{"key":"558_CR3","doi-asserted-by":"crossref","unstructured":"Ait Elhadj A, Boughanem M, Mezghiche M, Souam F (2011) Using structural similarity for clustering XML documents. Knowl Inf Syst. doi: 10.1007\/s10115-011-0421-5","DOI":"10.1007\/s10115-011-0421-5"},{"key":"558_CR4","unstructured":"Alimoglu F, Alpaydi E (1996) Methods of combining multiple classifiers based on different representations for pen-based handwriting recognition. In: Proceedings of the fifth Turkish artificial intelligence and artificial neural networks symposium (TAINN \u201996), Istanbul, Turkey"},{"key":"558_CR5","unstructured":"Aslam JA, Yilmaz E (2005) A geometric interpretation and analysis of r-precision. In: Proceedings of the 14th ACM international conference on information and knowledge management, pp 664\u2013671"},{"key":"558_CR6","doi-asserted-by":"crossref","first-page":"874","DOI":"10.1016\/j.engappai.2007.09.009","volume":"21","author":"A Ben-David","year":"2008","unstructured":"Ben-David A (2008) About the relationship between ROC curves and Cohen\u2019s kappa. Eng Appl Artif Intell 21:874\u2013882","journal-title":"Eng Appl Artif Intell"},{"key":"558_CR7","doi-asserted-by":"crossref","unstructured":"Beran T, Hecker K, Coderre S, Wright B, Woloschuk W McLaughlin K (2011) Ego identity status of medical students in clerkship. Can Med Educ J 2(1):e4\u2013e10. http:\/\/www.cmej.ca\/cmej\/index.php\/cmej\/article\/view\/123","DOI":"10.36834\/cmej.36558"},{"key":"558_CR8","first-page":"993","volume":"3","author":"DM Blei","year":"2003","unstructured":"Blei DM, Ng AY, Jordan MI (2003) Latent dirichlet allocation. J Mach Learn Res 3:993\u20131022","journal-title":"J Mach Learn Res"},{"key":"558_CR9","volume-title":"Classification and regression trees","author":"L Breiman","year":"1984","unstructured":"Breiman L, Friedman JH, Olshen RA, Stone CJ (1984) Classification and regression trees. Wadsworth International Group, Belmont, CA"},{"key":"558_CR10","doi-asserted-by":"crossref","unstructured":"Bulling A, Ward JA, Gellersen H, Tr\u00f6ster G (2011) Eye movement analysis for activity recognition using electrooculography. IEEE Trans Pattern Anal Mach Intell 33(4):741\u2013753. http:\/\/doi.ieeecomputersociety.org\/10.1109\/TPAMI.2010.86","DOI":"10.1109\/TPAMI.2010.86"},{"key":"558_CR11","doi-asserted-by":"crossref","unstructured":"Cai W, Lee J-G, Zalis ME, Yoshida H (2011) Mosaic decomposition: An electronic cleansing method for inhomogeneously tagged regions in noncathartic CT colonography. IEEE Trans. Med. Imaging 30(3): 559\u2013574","DOI":"10.1109\/TMI.2010.2087389"},{"key":"558_CR12","doi-asserted-by":"crossref","unstructured":"Chazdon RL, Chao A, Cplwel RK, Shang-Yi L, Norden N, Letche SG, Clark DB, Finegan B, Arroyo JP (2011) A novel statistical method for classifying habitat generalists and specialists. J Ecol 92(6):1332\u20131343. http:\/\/www.esajournals.org\/doi\/pdf\/10.1890\/10-1345.1","DOI":"10.1890\/10-1345.1"},{"key":"558_CR13","unstructured":"Chen B-C, Guo J, Tseng BL, Yang J (2011) User reputation in a comment rating environment. In: Proceedings of the 17th ACM SIGKDD international conference on knowledge discovery and data mining, pp 159\u2013167"},{"key":"558_CR14","doi-asserted-by":"crossref","unstructured":"Chen F, Dai J, Wang B, Sahu S, Naphade M, Lu C-T (2011) Activity analysis based on low sample rate smart meters. In: Proceedings of the 17th ACM SIGKDD international conference on Knowledge discovery and data mining, KDD \u201911, ACM, New York, NY, USA, pp 240\u2013248. http:\/\/doi.acm.org\/10.1145\/2020408.2020450","DOI":"10.1145\/2020408.2020450"},{"issue":"1","key":"558_CR15","doi-asserted-by":"crossref","first-page":"37","DOI":"10.1177\/001316446002000104","volume":"20","author":"J Cohen","year":"1960","unstructured":"Cohen J (1960) A coefficient of agreement for nominal scales. Educ Psychol Meas 20(1):37\u201346","journal-title":"Educ Psychol Meas"},{"key":"558_CR16","doi-asserted-by":"crossref","unstructured":"Costa G, Manco G, Ortale R, Ritacco E (2011) From global to local and viceversa: uses of associative rule learning for classification in imprecise environments. Knowl Inf Syst 1\u201333. doi: 10.1007\/s10115-011-0458-5","DOI":"10.1007\/s10115-011-0458-5"},{"key":"558_CR17","doi-asserted-by":"crossref","unstructured":"Cotton CV, Ellis DPW, Loui AC (2011) Soundtrack classification by transient events. In: ICASSP, pp 473\u2013476","DOI":"10.1109\/ICASSP.2011.5946443"},{"key":"558_CR18","volume-title":"Mathematical methods of statistics","author":"H Cramer","year":"1946","unstructured":"Cramer H (1946) Mathematical methods of statistics. Princeton University Press, Princeton, NJ"},{"key":"558_CR19","first-page":"1","volume":"7","author":"J Demsar","year":"2006","unstructured":"Demsar J (2006) Statistical comparisons of classifiers over multiple data sets. J Mach Learn Res 7:1\u201330","journal-title":"J Mach Learn Res"},{"key":"558_CR20","doi-asserted-by":"crossref","first-page":"201","DOI":"10.1007\/s10588-005-5377-0","volume":"11","author":"J Diesner","year":"2005","unstructured":"Diesner J, Frantz TL, Carley KM (2005) Communication networks from the enron email corpus \u201cit\u2019s always about the people. enron is no different\u201d. Comput Math Organ Theory 11:201\u2013228","journal-title":"Comput Math Organ Theory"},{"key":"558_CR21","doi-asserted-by":"crossref","first-page":"614","DOI":"10.1111\/1541-0420.00071","volume":"59","author":"LE Dodd","year":"2003","unstructured":"Dodd LE, Pepe MS (2003) Partial AUC estimation and regression. Biometrics 59:614\u2013623","journal-title":"Biometrics"},{"issue":"1","key":"558_CR22","doi-asserted-by":"crossref","first-page":"95","DOI":"10.1007\/s10994-006-8199-5","volume":"65","author":"C Drummond","year":"2006","unstructured":"Drummond C, Holte RC (2006) Cost curves: an improved method for visualizing classifier performance. Mach Learn 65(1):95\u2013130","journal-title":"Mach Learn"},{"key":"558_CR23","unstructured":"Elisseeff A, Weston J (2005) A kernel method for multi-labelled classification. In: Annual ACM conference on research and development in information retrieval, pp 274\u2013281"},{"key":"558_CR24","unstructured":"Flach P, Hernandez-Orallo J, Ferri C (2011) A coherent interpretation of AUC as a measure of aggregated classification performance. In: Proceedings of the 28th international conference on machine learning (ICML-11). New York, NY, USA, pp 657\u2013664"},{"key":"558_CR25","first-page":"1289","volume":"3","author":"G Forman","year":"2003","unstructured":"Forman G (2003) An extensive empirical study of feature selection metrics for text classification. J Mach Learn Res 3:1289\u20131305","journal-title":"J Mach Learn Res"},{"key":"558_CR26","unstructured":"Frank A, Asuncion A (2010) UCI machine learning repository. http:\/\/archive.ics.uci.edu\/ml"},{"issue":"1","key":"558_CR27","first-page":"279","volume":"38","author":"Z Ghahramani","year":"2003","unstructured":"Ghahramani Z, Kim H-C (2003) Bayesian classifier combination. Biomed Environ Sens 38(1):279\u2013294","journal-title":"Biomed Environ Sens"},{"key":"558_CR28","doi-asserted-by":"crossref","first-page":"103","DOI":"10.1007\/s10994-009-5119-5","volume":"77","author":"DJ Hand","year":"2009","unstructured":"Hand DJ (2009) Measuring classifier performance: a coherent alternative to the area under the ROC curve. Mach Learn 77:103\u2013123","journal-title":"Mach Learn"},{"key":"558_CR29","doi-asserted-by":"crossref","first-page":"171","DOI":"10.1023\/A:1010920819831","volume":"45","author":"DJ Hand","year":"2001","unstructured":"Hand DJ, Till RJ (2001) A simple generalisation of the area under the ROC curve for multiple class classification problems. Mach Learn 45:171\u2013186","journal-title":"Mach Learn"},{"key":"558_CR30","doi-asserted-by":"crossref","unstructured":"Ji S, Yuan L, Li Y-X, Zhou Z-H, Kumar S, Ye J (2009) Drosophila gene expression pattern annotation using sparse features and term-term interactions. In: Proceedings of the 15th ACM SIGKDD international conference on knowledge discovery and data mining, pp 407\u2013416","DOI":"10.1145\/1557019.1557068"},{"key":"558_CR31","unstructured":"Kaymak U, Ben-David A, Potharst R (2010) AUK: a simple alternative to the AUC, Research Paper ERS-2010-024-LIS, Erasmus Research Institute of Management (ERIM). http:\/\/ideas.repec.org\/p\/dgr\/eureri\/1765019678.html"},{"key":"558_CR32","doi-asserted-by":"crossref","unstructured":"Klement W, Flach PA, Japkowicz N, Matwin S (2011) Smooth receiver operating characteristics (smROC) curves. In: ECML\/PKDD (2), pp 193\u2013208","DOI":"10.1007\/978-3-642-23783-6_13"},{"key":"558_CR33","unstructured":"Lavesson N, Axelsson S (2011) Similarity assessment for removal of noisy end user license agreements. Knowl Inf Syst 32:1\u201323"},{"key":"558_CR34","unstructured":"Li S, Lin C-Y, Song Y-I, Li Z (2011) Comparable entity mining from comparative questions. IEEE Trans Knowl Data Eng 99:650\u2013658"},{"key":"558_CR35","unstructured":"Ling CX, Huang J, Zhang H (2003) AUC: a statistically consistent and more discriminating measure than accuracy. In: IJCAI, pp 519\u2013526"},{"issue":"3","key":"558_CR36","doi-asserted-by":"crossref","first-page":"190","DOI":"10.1177\/0272989X8900900307","volume":"9","author":"D McClish","year":"1989","unstructured":"McClish D (1989) Analyzing a portion of the ROC curve. Med Decis Mak 9(3):190\u2013195","journal-title":"Med Decis Mak"},{"key":"558_CR37","volume-title":"Handbook of biological statistics","author":"JH McDonald","year":"2009","unstructured":"McDonald JH (2009) Handbook of biological statistics, 2nd edn. Sparky House Publishing, Baltimore, MD","edition":"2"},{"issue":"1","key":"558_CR38","doi-asserted-by":"crossref","first-page":"88","DOI":"10.1109\/TMM.2011.2168948","volume":"14","author":"M Merler","year":"2012","unstructured":"Merler M, Huang B, Xie L, Hua G, Natsev A (2012) Semantic model vectors for complex video event recognition. IEEE Trans Multimed 14(1):88\u2013101","journal-title":"IEEE Trans Multimed"},{"issue":"7","key":"558_CR39","doi-asserted-by":"crossref","first-page":"3430","DOI":"10.1109\/TSP.2011.2141668","volume":"59","author":"J Moragues","year":"2011","unstructured":"Moragues J, Vergara L, Gos\u00e1lbez J (2011) Generalized matched subspace filter for nonindependent noise based on ICA. IEEE Trans Signal Process 59(7):3430\u20133434","journal-title":"IEEE Trans Signal Process"},{"key":"558_CR40","doi-asserted-by":"crossref","unstructured":"Mowery D, Wiebe J, Visweswaran S, Harkema H, Chapman WW (2011) Building an automated SOAP classifier for emergency department reports. J Biomed Inform 45:71\u201381","DOI":"10.1016\/j.jbi.2011.08.020"},{"key":"558_CR41","doi-asserted-by":"crossref","unstructured":"Naik PK, Nitin N, Janmeja A, Puri S, Chawla K, Bhasin M, Jain K (2011) B-MIPT: a case tool for biomedical image processing and their classification using nearest neighbor and genetic algorithm. In: International conference on intelligent systems, modelling and simulation, pp 107\u2013112. doi: 10.1109\/ISMS.2011.26","DOI":"10.1109\/ISMS.2011.26"},{"key":"558_CR42","unstructured":"Park LAF (2011) Bootstrap confidence intervals for mean average precision. In: Proceedings of the fourth ASEARC conference, pp 51\u201354. http:\/\/staff.scem.uws.edu.au\/~lapark\/publications\/bootstrap-ap.pdf"},{"key":"558_CR43","doi-asserted-by":"crossref","unstructured":"Parker C (2010) An empirical study of feature extraction methods for audio classification. In: ICPR \u201910: the twentieth international conference on pattern recognition. Istanbul, Turkey","DOI":"10.1109\/ICPR.2010.1111"},{"key":"558_CR44","doi-asserted-by":"crossref","unstructured":"Parker C (2011) An analysis of performance measures for binary classification. In: The international conference on data mining. Vancouver, Canada","DOI":"10.1109\/ICDM.2011.21"},{"key":"558_CR45","unstructured":"Provost F, Fawcett T, Kohavi R (1997) The case against accuracy estimation for comparing induction algorithms. In: Proceedings of the fifteenth international conference on machine learning, pp 445\u2013453"},{"issue":"3","key":"558_CR46","doi-asserted-by":"crossref","first-page":"203","DOI":"10.1023\/A:1007601015854","volume":"42","author":"FJ Provost","year":"2001","unstructured":"Provost FJ, Fawcett T (2001) Robust classification for imprecise environments. Mach Learn 42(3):203\u2013231","journal-title":"Mach Learn"},{"issue":"6","key":"558_CR47","doi-asserted-by":"crossref","first-page":"1598","DOI":"10.1109\/TBME.2012.2190512","volume":"59","author":"P Ramachandran","year":"2012","unstructured":"Ramachandran P, Lu W-S, Antoniou A (2012) Filter-based methodology for the location of hot spots in proteins and exons in DNA. IEEE Trans Biomed Eng 59(6):1598\u20131609","journal-title":"IEEE Trans Biomed Eng"},{"key":"558_CR48","doi-asserted-by":"crossref","unstructured":"Renals S, Rohwer R (1989) Phoneme classification experiments using radial basis functions. In: International joint conference on neural networks, Washington","DOI":"10.1109\/IJCNN.1989.118620"},{"key":"558_CR49","volume-title":"Information retrieval","author":"CJV Rijsbergen","year":"1979","unstructured":"Rijsbergen CJV (1979) Information retrieval. Butterworth-Heinemann, Newton, MA"},{"key":"558_CR50","doi-asserted-by":"crossref","unstructured":"Robertson S (2012) On smoothing average precision. In: ECIR, pp 158\u2013169","DOI":"10.1007\/978-3-642-28997-2_14"},{"key":"558_CR51","volume-title":"Introduction to modern information retrieval","author":"G Salton","year":"1986","unstructured":"Salton G, McGill MJ (1986) Introduction to modern information retrieval. McGraw-Hill, New York, NY"},{"key":"558_CR52","doi-asserted-by":"crossref","first-page":"135","DOI":"10.1023\/A:1007649029923","volume":"39","author":"RE Schapire","year":"2000","unstructured":"Schapire RE, Singer Y (2000) BoosTexter: a boosting-based system for text categorization. Mach Learn 39:135\u2013168","journal-title":"Mach Learn"},{"key":"558_CR53","unstructured":"Trohidis K, Tsoumakas G, Kalliris G, Vlahavas I (2008) Multilabel classification of music into emotions. In: Proceedings of the 2008 international conference on music information retrieval (ISMIR 2008). Philadelphia, PA, USA, pp 325\u2013330"},{"key":"558_CR54","doi-asserted-by":"crossref","first-page":"467","DOI":"10.1109\/TASL.2007.913750","volume":"16","author":"D Turnbull","year":"2008","unstructured":"Turnbull D, Barrington L, Torres D, Lanckriet G (2008) Semantic annotation and retrieval of music and sound effects. IEEE Trans Audio Speech Lang Process 16:467\u2013476","journal-title":"IEEE Trans Audio Speech Lang Process"},{"key":"558_CR55","doi-asserted-by":"crossref","unstructured":"Uddin M, Maskrey V, Holland R (2011) A study to validate a self-reported version of the ONS drug dependence questionnaire. J Subst Use 16(4):273\u2013281. https:\/\/ueaeprints.uea.ac.uk\/14543\/","DOI":"10.3109\/14659890903490781"},{"key":"558_CR56","unstructured":"Valentini G, Dietterich TG (2003) Low bias bagged support vector machines. In: International conference on machine learning. Morgan Kaufmann, Burlington, MA, pp 752\u2013759"},{"key":"558_CR57","unstructured":"van der Maaten LJP, Postma EO, van den Herik HJ (2009) Dimensionality reduction: a comparative review. Technical Report TiCC-TR 2009\u2013005, Tilburg University"},{"issue":"1","key":"558_CR58","doi-asserted-by":"crossref","first-page":"67","DOI":"10.1007\/s11634-011-0094-7","volume":"6","author":"MJ Warrens","year":"2012","unstructured":"Warrens MJ (2012) Cohen\u2019s linearly weighted kappa is a weighted average. Adv Data Anal Classif 6(1):67\u201379","journal-title":"Adv Data Anal Classif"},{"issue":"1","key":"558_CR59","first-page":"1","volume":"1","author":"C wei Hsu","year":"2010","unstructured":"wei Hsu C, chung Chang C, jen Lin C (2010) A practical guide to support vector classification. Bioinformatics 1(1):1\u201316","journal-title":"Bioinformatics"},{"key":"558_CR60","volume-title":"Data mining: practical machine learning tools and techniques, second edition Morgan Kaufmann series in data management systems","author":"IH Witten","year":"2005","unstructured":"Witten IH, Frank E (2005) Data mining: practical machine learning tools and techniques, second edition Morgan Kaufmann series in data management systems, 2nd edn. Morgan Kaufmann, Burlington, MA","edition":"2"},{"issue":"1","key":"558_CR61","doi-asserted-by":"crossref","first-page":"276","DOI":"10.1109\/TASL.2011.2160853","volume":"20","author":"L Xie","year":"2012","unstructured":"Xie L, Zheng L, Liu Z, Zhang Y (2012) Laplacian eigenmaps for automatic story segmentation of broadcast news. IEEE Trans Audio Speech Lang Process 20(1):276\u2013289","journal-title":"IEEE Trans Audio Speech Lang Process"},{"key":"558_CR62","doi-asserted-by":"crossref","unstructured":"Ye M, Shou D, Lee W-C, Yin P, Janowicz K (2011) On the semantic annotation of places in location-based social networks. In: Proceedings of the 17th ACM SIGKDD international conference on knowledge discovery and data mining, KDD \u201911, ACM, New York, NY, USA, pp 520\u2013528. http:\/\/doi.acm.org\/10.1145\/2020408.2020491","DOI":"10.1145\/2020408.2020491"},{"key":"558_CR63","doi-asserted-by":"crossref","first-page":"173","DOI":"10.1007\/s10115-007-0101-7","volume":"16","author":"E Yilmaz","year":"2008","unstructured":"Yilmaz E, Aslam JA (2008) Estimating average precision when judgments are incomplete. Knowl Inf Syst 16:173\u2013211","journal-title":"Knowl Inf Syst"}],"container-title":["Knowledge and Information Systems"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s10115-012-0558-x.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/article\/10.1007\/s10115-012-0558-x\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s10115-012-0558-x","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2022,1,29]],"date-time":"2022-01-29T07:47:41Z","timestamp":1643442461000},"score":1,"resource":{"primary":{"URL":"http:\/\/link.springer.com\/10.1007\/s10115-012-0558-x"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2012,9,18]]},"references-count":63,"journal-issue":{"issue":"1","published-print":{"date-parts":[[2013,4]]}},"alternative-id":["558"],"URL":"https:\/\/doi.org\/10.1007\/s10115-012-0558-x","relation":{},"ISSN":["0219-1377","0219-3116"],"issn-type":[{"value":"0219-1377","type":"print"},{"value":"0219-3116","type":"electronic"}],"subject":[],"published":{"date-parts":[[2012,9,18]]}}}