{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,21]],"date-time":"2026-07-21T23:19:23Z","timestamp":1784675963922,"version":"3.55.0"},"reference-count":40,"publisher":"Springer Science and Business Media LLC","issue":"21","license":[{"start":{"date-parts":[[2021,8,22]],"date-time":"2021-08-22T00:00:00Z","timestamp":1629590400000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"},{"start":{"date-parts":[[2021,8,22]],"date-time":"2021-08-22T00:00:00Z","timestamp":1629590400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Neural Comput & Applic"],"published-print":{"date-parts":[[2021,11]]},"DOI":"10.1007\/s00521-021-06103-6","type":"journal-article","created":{"date-parts":[[2021,8,22]],"date-time":"2021-08-22T11:02:40Z","timestamp":1629630160000},"page":"14623-14650","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":48,"title":["BenchMetrics: a systematic benchmarking method for binary classification performance metrics"],"prefix":"10.1007","volume":"33","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-9337-097X","authenticated-orcid":false,"given":"G\u00fcrol","family":"Canbek","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-7387-8621","authenticated-orcid":false,"given":"Tugba","family":"Taskaya Temizel","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-0805-5818","authenticated-orcid":false,"given":"Seref","family":"Sagiroglu","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2021,8,22]]},"reference":[{"key":"6103_CR1","doi-asserted-by":"publisher","first-page":"216","DOI":"10.1016\/j.patcog.2019.02.023","volume":"91","author":"A Luque","year":"2019","unstructured":"Luque A, Carrasco A, Mart\u00edn A, de las Heras A (2019) The impact of class imbalance in classification performance metrics based on the binary confusion matrix. Pattern Recognit 91:216\u2013231. https:\/\/doi.org\/10.1016\/j.patcog.2019.02.023","journal-title":"Pattern Recognit"},{"key":"6103_CR2","doi-asserted-by":"publisher","first-page":"611","DOI":"10.3171\/2018.5.SPINE18543","volume":"29","author":"VE Staartjes","year":"2018","unstructured":"Staartjes VE, Schr\u00f6der ML (2018) Letter to the Editor. Class imbalance in machine learning for neurosurgical outcome prediction: are our models valid? J Neurosurg Spine 29:611\u2013612. https:\/\/doi.org\/10.3171\/2018.5.SPINE18543","journal-title":"J Neurosurg Spine"},{"key":"6103_CR3","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1002\/minf.201700127","volume":"37","author":"JB Brown","year":"2018","unstructured":"Brown JB (2018) Classifiers and their metrics quantified. Mol Inform 37:1\u201311. https:\/\/doi.org\/10.1002\/minf.201700127","journal-title":"Mol Inform"},{"key":"6103_CR4","unstructured":"Sokolova M (2006) Assessing invariance properties of evaluation measures. Proc Work Test Deployable Learn Decis Syst 19th Neural Inf Process Syst Conf (NIPS 2006) 1\u20136"},{"key":"6103_CR5","doi-asserted-by":"crossref","unstructured":"Ranawana R, Palade V (2006) Optimized precision\u2014a new measure for classifier performance evaluation. In: 2006 IEEE International Conference on Evolutionary Computation. IEEE, Vancouver, BC, Canada, pp 2254\u20132261","DOI":"10.1109\/CEC.2006.1688586"},{"key":"6103_CR6","doi-asserted-by":"publisher","first-page":"617","DOI":"10.1109\/ICPR.2010.156","volume":"2006","author":"V Garcia","year":"2010","unstructured":"Garcia V, Mollineda RA, Sanchez JS (2010) Theoretical analysis of a performance measure for imbalanced data. IEEE Int Conf Pattern Recognit 2006:617\u2013620. https:\/\/doi.org\/10.1109\/ICPR.2010.156","journal-title":"IEEE Int Conf Pattern Recognit"},{"key":"6103_CR7","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1371\/journal.pone.0177678","volume":"12","author":"S Boughorbel","year":"2017","unstructured":"Boughorbel S, Jarray F, El-Anbari M (2017) Optimal classifier for imbalanced data using Matthews correlation coefficient metric. PLoS ONE 12:1\u201317. https:\/\/doi.org\/10.1371\/journal.pone.0177678","journal-title":"PLoS ONE"},{"key":"6103_CR8","doi-asserted-by":"publisher","first-page":"27","DOI":"10.1016\/j.patrec.2008.08.010","volume":"30","author":"C Ferri","year":"2009","unstructured":"Ferri C, Hern\u00e1ndez-Orallo J, Modroiu R (2009) An experimental comparison of performance measures for classification. Pattern Recognit Lett 30:27\u201338. https:\/\/doi.org\/10.1016\/j.patrec.2008.08.010","journal-title":"Pattern Recognit Lett"},{"key":"6103_CR9","doi-asserted-by":"crossref","unstructured":"Seliya N, Khoshgoftaar TM, Van Hulse J (2009) Aggregating performance metrics for classifier evaluation. In: IEEE International Conference on Information Reuse and Integration, IRI. pp 35\u201340","DOI":"10.1109\/IRI.2009.5211611"},{"key":"6103_CR10","doi-asserted-by":"publisher","first-page":"20","DOI":"10.4018\/ijmcmc.2014100102","volume":"6","author":"Y Liu","year":"2016","unstructured":"Liu Y, Zhou Y, Wen S, Tang C (2016) A strategy on selecting performance metrics for classifier evaluation. Int J Mob Comput Multimed Commun 6:20\u201335. https:\/\/doi.org\/10.4018\/ijmcmc.2014100102","journal-title":"Int J Mob Comput Multimed Commun"},{"key":"6103_CR11","doi-asserted-by":"publisher","first-page":"242","DOI":"10.1016\/j.ins.2018.06.020","volume":"462","author":"D Brzezinski","year":"2018","unstructured":"Brzezinski D, Stefanowski J, Susmaga R, Szcz\u0229ch I (2018) Visual-based analysis of classification measures and their properties for class imbalanced problems. Inf Sci (Ny) 462:242\u2013261. https:\/\/doi.org\/10.1016\/j.ins.2018.06.020","journal-title":"Inf Sci (Ny)"},{"key":"6103_CR12","unstructured":"Hu B-G, Dong W-M (2014) A study on cost behaviors of binary classification measures in class-imbalanced problems. Comput Res Repos abs\/1403.7"},{"key":"6103_CR13","doi-asserted-by":"publisher","first-page":"427","DOI":"10.1016\/j.ipm.2009.03.002","volume":"45","author":"M Sokolova","year":"2009","unstructured":"Sokolova M, Lapalme G (2009) A systematic analysis of performance measures for classification tasks. Inf Process Manag 45:427\u2013437. https:\/\/doi.org\/10.1016\/j.ipm.2009.03.002","journal-title":"Inf Process Manag"},{"key":"6103_CR14","doi-asserted-by":"publisher","first-page":"299","DOI":"10.1109\/TKDE.2005.50","volume":"17","author":"J Huang","year":"2005","unstructured":"Huang J, Ling CX (2005) Using AUC and accuracy in evaluating learning algorithms. IEEE Trans Knowl Data Eng 17:299\u2013310. https:\/\/doi.org\/10.1109\/TKDE.2005.50","journal-title":"IEEE Trans Knowl Data Eng"},{"key":"6103_CR15","doi-asserted-by":"publisher","first-page":"189","DOI":"10.1007\/BF01617722","volume":"11","author":"A Forbes","year":"1995","unstructured":"Forbes A (1995) Classification-algorithm evaluation: five performance measures based on confusion matrices. J Clin Monit Comput 11:189\u2013206. https:\/\/doi.org\/10.1007\/BF01617722","journal-title":"J Clin Monit Comput"},{"key":"6103_CR16","doi-asserted-by":"publisher","first-page":"359","DOI":"10.1016\/j.ipm.2018.01.002","volume":"54","author":"RB Pereira","year":"2018","unstructured":"Pereira RB, Plastino A, Zadrozny B, Merschmann LHC (2018) Correlation analysis of performance measures for multi-label classification. Inf Process Manag 54:359\u2013369. https:\/\/doi.org\/10.1016\/j.ipm.2018.01.002","journal-title":"Inf Process Manag"},{"key":"6103_CR17","doi-asserted-by":"publisher","first-page":"1","DOI":"10.3389\/fncom.2014.00043","volume":"8","author":"S Straube","year":"2014","unstructured":"Straube S, Krell MM (2014) How to evaluate an agent\u2019s behavior to infrequent events? Reliable performance estimation insensitive to class distribution. Front Comput Neurosci 8:1\u20136. https:\/\/doi.org\/10.3389\/fncom.2014.00043","journal-title":"Front Comput Neurosci"},{"key":"6103_CR18","doi-asserted-by":"publisher","first-page":"1","DOI":"10.5121\/ijdkp.2015.5201","volume":"5","author":"M Hossin","year":"2015","unstructured":"Hossin M, Sulaiman MN (2015) A review on evaluation metrics for data classification evaluations. Int J Data Min Knowl Manag Process 5:1\u201311. https:\/\/doi.org\/10.5121\/ijdkp.2015.5201","journal-title":"Int J Data Min Knowl Manag Process"},{"key":"6103_CR19","doi-asserted-by":"publisher","DOI":"10.1016\/j.aci.2018.08.003","author":"A Tharwat","year":"2020","unstructured":"Tharwat A (2020) Classification assessment methods. Appl Comput Informatics Informatics ahead-of-p:1\u201313. https:\/\/doi.org\/10.1016\/j.aci.2018.08.003","journal-title":"Appl Comput Informatics Informatics ahead-of-p:1\u201313"},{"key":"6103_CR20","doi-asserted-by":"publisher","DOI":"10.1186\/s12864-019-6413-7","author":"D Chicco","year":"2020","unstructured":"Chicco D, Jurman G (2020) The advantages of the Matthews correlation coefficient (MCC) over F1 score and accuracy in binary classification evaluation. BMC Genomics. https:\/\/doi.org\/10.1186\/s12864-019-6413-7","journal-title":"BMC Genomics"},{"key":"6103_CR21","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1109\/TNNLS.2019.2899061","volume":"31","author":"D Brzezinski","year":"2020","unstructured":"Brzezinski D, Stefanowski J, Susmaga R, Szczech I (2020) On the dynamics of classification measures for imbalanced and streaming data. IEEE Trans Neural Networks Learn Syst 31:1\u201311. https:\/\/doi.org\/10.1109\/TNNLS.2019.2899061","journal-title":"IEEE Trans Neural Networks Learn Syst"},{"key":"6103_CR22","doi-asserted-by":"publisher","first-page":"412","DOI":"10.1093\/bioinformatics\/16.5.412","volume":"16","author":"P Baldi","year":"2000","unstructured":"Baldi P, Brunak S, Chauvin Y et al (2000) Assessing the accuracy of prediction algorithms for classification: an overview. Bioinformatics 16:412\u2013424. https:\/\/doi.org\/10.1093\/bioinformatics\/16.5.412","journal-title":"Bioinformatics"},{"key":"6103_CR23","doi-asserted-by":"publisher","first-page":"1169","DOI":"10.1016\/S1874-1029(11)60289-9","volume":"38","author":"B-G Hu","year":"2012","unstructured":"Hu B-G, He R, Yuan X-T (2012) Information-theoretic measures for objective evaluation of classifications. Acta Autom Sin 38:1169\u20131182. https:\/\/doi.org\/10.1016\/S1874-1029(11)60289-9","journal-title":"Acta Autom Sin"},{"key":"6103_CR24","doi-asserted-by":"publisher","first-page":"861","DOI":"10.1016\/j.patrec.2005.10.010","volume":"27","author":"T Fawcett","year":"2006","unstructured":"Fawcett T (2006) An introduction to ROC analysis. Pattern Recognit Lett 27:861\u2013874. https:\/\/doi.org\/10.1016\/j.patrec.2005.10.010","journal-title":"Pattern Recognit Lett"},{"key":"6103_CR25","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1371\/journal.pone.0084217","volume":"9","author":"FJ Valverde-Albacete","year":"2014","unstructured":"Valverde-Albacete FJ, Pel\u00e1ez-Moreno C (2014) 100% classification accuracy considered harmful: the normalized information transfer factor explains the accuracy paradox. PLoS ONE 9:1\u201310. https:\/\/doi.org\/10.1371\/journal.pone.0084217","journal-title":"PLoS ONE"},{"key":"6103_CR26","unstructured":"Shepperd M (2013) Assessing the predictive performance of machine learners in software defect prediction function. In: The 24th CREST Open Workshop (COW), on Machine Learning and Search Based Software Engineering (ML&SBSE). Centre for Research on Evolution, Search and Testing (CREST), London, pp 1\u201316"},{"key":"6103_CR27","unstructured":"Schr\u00f6der G, Thiele M, Lehner W (2011) Setting goals and choosing metrics for recommender system evaluations. In: UCERSTI 2 Workshop at the 5th ACM Conference on Recommender Systems. Chicago, Illinois, pp. 1\u20138"},{"key":"6103_CR28","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1371\/journal.pone.0222916","volume":"14","author":"R Delgado","year":"2019","unstructured":"Delgado R, Tibau XA (2019) Why Cohen\u2019s kappa should be avoided as performance measure in classification. PLoS ONE 14:1\u201326. https:\/\/doi.org\/10.1371\/journal.pone.0222916","journal-title":"PLoS ONE"},{"key":"6103_CR29","doi-asserted-by":"publisher","DOI":"10.1007\/s00521-020-05134-9","author":"J Ma","year":"2020","unstructured":"Ma J, Zhou S (2020) Metric learning-guided k nearest neighbor multilabel classifier. Neural Comput Appl. https:\/\/doi.org\/10.1007\/s00521-020-05134-9","journal-title":"Neural Comput Appl"},{"key":"6103_CR30","doi-asserted-by":"crossref","unstructured":"Fatourechi M, Ward RK, Mason SG, et al (2008) Comparison of evaluation metrics in classification applications with imbalanced datasets. In: 7th International Conference on Machine Learning and Applications (ICMLA). pp. 777\u2013782","DOI":"10.1109\/ICMLA.2008.34"},{"key":"6103_CR31","doi-asserted-by":"crossref","unstructured":"Seliya N, Khoshgoftaar TM, Van Hulse J (2009) A study on the relationships of classifier performance metrics. In: 21st IEEE International Conference on Tools with Artificial Intelligence, ICTAI. pp. 59\u201366","DOI":"10.1109\/ICTAI.2009.25"},{"key":"6103_CR32","doi-asserted-by":"crossref","unstructured":"Joshi MV (2002) On evaluating performance of classifiers for rare classes. In: Proceedings IEEE International Conference on Data Mining. IEEE, pp. 641\u2013644","DOI":"10.1109\/ICDM.2002.1184018"},{"key":"6103_CR33","doi-asserted-by":"crossref","unstructured":"Caruana R, Niculescu-Mizil A (2004) Data mining in metric space: an empirical analysis of supervised learning performance criteria. Proc 10th ACM SIGKDD Int Conf Knowl Discov Data Min 69\u201378. https:\/\/doi.org\/1-58113-888-1\/04\/0008","DOI":"10.1145\/1014052.1014063"},{"key":"6103_CR34","unstructured":"Huang J, Ling CX (2007) Constructing new and better evaluation measures for machine learning. IJCAI Int Jt Conf Artif Intell 859\u2013864"},{"key":"6103_CR35","doi-asserted-by":"publisher","DOI":"10.1017\/CBO9780511921803","volume-title":"Evaluating learning algorithms: a classification perspective","author":"N Japkowicz","year":"2011","unstructured":"Japkowicz N, Shah M (2011) Evaluating learning algorithms: a classification perspective. Cambridge University Press, Cambridge"},{"key":"6103_CR36","doi-asserted-by":"publisher","first-page":"1","DOI":"10.3390\/SYM12061013","volume":"12","author":"JE Contreras-Reyes","year":"2020","unstructured":"Contreras-Reyes JE (2020) An asymptotic test for bimodality using the Kullback-Leibler divergence. Symmetry (Basel) 12:1\u201313. https:\/\/doi.org\/10.3390\/SYM12061013","journal-title":"Symmetry (Basel)"},{"key":"6103_CR37","doi-asserted-by":"publisher","first-page":"827","DOI":"10.1038\/nbt.1665","volume":"28","author":"L Shi","year":"2010","unstructured":"Shi L, Campbell G, Jones WD et al (2010) The Microarray Quality Control (MAQC)-II study of common practices for the development and validation of microarray-based predictive models. Nat Biotechnol 28:827\u2013838. https:\/\/doi.org\/10.1038\/nbt.1665","journal-title":"Nat Biotechnol"},{"key":"6103_CR38","doi-asserted-by":"publisher","first-page":"6995","DOI":"10.1007\/s00521-018-3539-5","volume":"31","author":"A Rohani","year":"2019","unstructured":"Rohani A, Mamarabadi M (2019) Free alignment classification of dikarya fungi using some machine learning methods. Neural Comput Appl 31:6995\u20137016. https:\/\/doi.org\/10.1007\/s00521-018-3539-5","journal-title":"Neural Comput Appl"},{"key":"6103_CR39","doi-asserted-by":"publisher","first-page":"1163","DOI":"10.1007\/s00521-012-1324-4","volume":"24","author":"AT Azar","year":"2014","unstructured":"Azar AT, El-Said SA (2014) Performance analysis of support vector machines classifiers in breast cancer mammography recognition. Neural Comput Appl 24:1163\u20131177. https:\/\/doi.org\/10.1007\/s00521-012-1324-4","journal-title":"Neural Comput Appl"},{"key":"6103_CR40","doi-asserted-by":"crossref","unstructured":"Canbek G, Sagiroglu S, Taskaya Temizel T, Baykal N (2017) Binary classification performance measures\/metrics: A comprehensive visualized roadmap to gain new insights. In: 2017 International Conference on Computer Science and Engineering (UBMK). IEEE, Antalya, Turkey, pp. 821\u2013826","DOI":"10.1109\/UBMK.2017.8093539"}],"container-title":["Neural Computing and Applications"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s00521-021-06103-6.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s00521-021-06103-6\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s00521-021-06103-6.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2021,10,24]],"date-time":"2021-10-24T06:21:02Z","timestamp":1635056462000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s00521-021-06103-6"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2021,8,22]]},"references-count":40,"journal-issue":{"issue":"21","published-print":{"date-parts":[[2021,11]]}},"alternative-id":["6103"],"URL":"https:\/\/doi.org\/10.1007\/s00521-021-06103-6","relation":{},"ISSN":["0941-0643","1433-3058"],"issn-type":[{"value":"0941-0643","type":"print"},{"value":"1433-3058","type":"electronic"}],"subject":[],"published":{"date-parts":[[2021,8,22]]},"assertion":[{"value":"16 September 2020","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"5 May 2021","order":2,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"22 August 2021","order":3,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Declarations"}},{"value":"The authors have no relevant financial or non-financial interests to disclose.","order":2,"name":"Ethics","group":{"name":"EthicsHeading","label":"Conflicts of interest"}}]}}