{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,3]],"date-time":"2026-04-03T22:51:12Z","timestamp":1775256672093,"version":"3.50.1"},"reference-count":85,"publisher":"Springer Science and Business Media LLC","issue":"1","license":[{"start":{"date-parts":[[2022,10,16]],"date-time":"2022-10-16T00:00:00Z","timestamp":1665878400000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2022,10,16]],"date-time":"2022-10-16T00:00:00Z","timestamp":1665878400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["SN COMPUT. SCI."],"DOI":"10.1007\/s42979-022-01409-1","type":"journal-article","created":{"date-parts":[[2022,10,16]],"date-time":"2022-10-16T04:02:11Z","timestamp":1665892931000},"update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":46,"title":["PToPI: A Comprehensive Review, Analysis, and Knowledge Representation of Binary Classification Performance Measures\/Metrics"],"prefix":"10.1007","volume":"4","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-9337-097X","authenticated-orcid":false,"given":"G\u00fcrol","family":"Canbek","sequence":"first","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0001-7387-8621","authenticated-orcid":false,"given":"Tugba","family":"Taskaya Temizel","sequence":"additional","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0003-0805-5818","authenticated-orcid":false,"given":"Seref","family":"Sagiroglu","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2022,10,16]]},"reference":[{"key":"1409_CR1","volume-title":"Making information retrieval pay","author":"CN Mooers","year":"1951","unstructured":"Mooers CN. Making information retrieval pay. Boston: Boston Portland State University; 1951."},{"key":"1409_CR2","volume-title":"Factors affecting the performance of indexing systems","author":"C Cleverdon","year":"1966","unstructured":"Cleverdon C, Mills J, Keen M. Factors affecting the performance of indexing systems, vol. I. Cranfield: Cranfield University; 1966."},{"key":"1409_CR3","doi-asserted-by":"publisher","DOI":"10.1016\/j.aci.2018.08.003","author":"A Tharwat","year":"2020","unstructured":"Tharwat A. Classification assessment methods. Appl Comput Informa. 2020. https:\/\/doi.org\/10.1016\/j.aci.2018.08.003 (ahead-of-p).","journal-title":"Appl Comput Informa"},{"key":"1409_CR4","volume-title":"Factors affecting the performance of indexing systems","author":"C Cleverdon","year":"1966","unstructured":"Cleverdon C, Keen M. Factors affecting the performance of indexing systems, vol. II. Cranfield: Cranfield University; 1966."},{"key":"1409_CR5","volume-title":"Principles of numerical taxonomy","author":"RR Sokal","year":"1963","unstructured":"Sokal RR, Sneath PHA. Principles of numerical taxonomy. San Francisco: W. H. Freeman and Company; 1963."},{"key":"1409_CR6","first-page":"223","volume":"44","author":"P Jaccard","year":"1908","unstructured":"Jaccard P. Nouvelles recherches sur la distribution florale. Bull la Soci\u00e9t\u00e9 Vaudoise Des Sci Nat. 1908;44:223\u201370.","journal-title":"Bull la Soci\u00e9t\u00e9 Vaudoise Des Sci Nat"},{"key":"1409_CR7","doi-asserted-by":"publisher","DOI":"10.1017\/CBO9780511921803","volume-title":"Evaluating learning algorithms: a classification perspective","author":"N Japkowicz","year":"2011","unstructured":"Japkowicz N, Shah M. Evaluating learning algorithms: a classification perspective. Cambridge: Cambridge University Press; 2011."},{"key":"1409_CR8","first-page":"37","volume":"2","author":"DMW Powers","year":"2011","unstructured":"Powers DMW. Evaluation: From precision, recall and F-factor to ROC, informedness, markedness & correlation. J Mach Learn Technol. 2011;2:37\u201363.","journal-title":"J Mach Learn Technol"},{"key":"1409_CR9","doi-asserted-by":"publisher","DOI":"10.3390\/sym11010047","author":"A Luque","year":"2019","unstructured":"Luque A, Carrasco A, Mart\u00edn A, Lama JR. Exploring symmetry of binary classification performance metrics. Symmetry (Basel). 2019. https:\/\/doi.org\/10.3390\/sym11010047.","journal-title":"Symmetry (Basel)"},{"key":"1409_CR10","doi-asserted-by":"publisher","first-page":"1015","DOI":"10.1007\/11941439_114","volume":"4304","author":"M Sokolova","year":"2006","unstructured":"Sokolova M, Japkowicz N, Szpakowicz S. Beyond accuracy, F-score and ROC: a family of discriminant measures for performance evaluation. Adv Artif Intell. 2006;4304:1015\u201321. https:\/\/doi.org\/10.1007\/11941439_114.","journal-title":"Adv Artif Intell"},{"key":"1409_CR11","doi-asserted-by":"publisher","DOI":"10.1016\/j.cosrev.2020.100358","volume":"39","author":"A Razgallah","year":"2021","unstructured":"Razgallah A, Khoury R, Hall\u00e9 S, Khanmohammadi K. A survey of malware detection in Android apps: recommendations and perspectives for future research. Comput Sci Rev. 2021;39: 100358. https:\/\/doi.org\/10.1016\/j.cosrev.2020.100358.","journal-title":"Comput Sci Rev"},{"key":"1409_CR12","doi-asserted-by":"publisher","DOI":"10.1016\/j.cosrev.2021.100365","volume":"39","author":"V Sihag","year":"2021","unstructured":"Sihag V, Vardhan M, Singh P. A survey of Android application and malware hardening. Comput Sci Rev. 2021;39: 100365. https:\/\/doi.org\/10.1016\/j.cosrev.2021.100365.","journal-title":"Comput Sci Rev"},{"key":"1409_CR13","doi-asserted-by":"publisher","first-page":"1","DOI":"10.3389\/fncom.2014.00043","volume":"8","author":"S Straube","year":"2014","unstructured":"Straube S, Krell MM. How to evaluate an agent\u2019s behavior to infrequent events? Reliable performance estimation insensitive to class distribution. Front Comput Neurosci. 2014;8:1\u20136. https:\/\/doi.org\/10.3389\/fncom.2014.00043.","journal-title":"Front Comput Neurosci"},{"key":"1409_CR14","doi-asserted-by":"publisher","first-page":"216","DOI":"10.1016\/j.patcog.2019.02.023","volume":"91","author":"A Luque","year":"2019","unstructured":"Luque A, Carrasco A, Mart\u00edn A, de las Heras A. The impact of class imbalance in classification performance metrics based on the binary confusion matrix. Pattern Recognit. 2019;91:216\u201331. https:\/\/doi.org\/10.1016\/j.patcog.2019.02.023.","journal-title":"Pattern Recognit"},{"key":"1409_CR15","doi-asserted-by":"publisher","first-page":"242","DOI":"10.1016\/j.ins.2018.06.020","volume":"462","author":"D Brzezinski","year":"2018","unstructured":"Brzezinski D, Stefanowski J, Susmaga R, Szcz\u0229ch I. Visual-based analysis of classification measures and their properties for class imbalanced problems. Inf Sci (NY). 2018;462:242\u201361. https:\/\/doi.org\/10.1016\/j.ins.2018.06.020.","journal-title":"Inf Sci (NY)"},{"key":"1409_CR16","doi-asserted-by":"publisher","DOI":"10.1016\/j.patcog.2020.107197","volume":"102","author":"SS Mullick","year":"2020","unstructured":"Mullick SS, Datta S, Dhekane SG, Das S. Appropriateness of performance indices for imbalanced data classification: an analysis. Pattern Recognit. 2020;102: 107197. https:\/\/doi.org\/10.1016\/j.patcog.2020.107197.","journal-title":"Pattern Recognit"},{"key":"1409_CR17","doi-asserted-by":"publisher","first-page":"687","DOI":"10.1142\/S0218001409007326","volume":"23","author":"Y Sun","year":"2009","unstructured":"Sun Y, Wong AKC, Kamel MS. Classification of imbalanced data: a review. Int J Pattern Recognit Artif Intell. 2009;23:687\u2013719. https:\/\/doi.org\/10.1142\/S0218001409007326.","journal-title":"Int J Pattern Recognit Artif Intell"},{"key":"1409_CR18","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1371\/journal.pone.0084217","volume":"9","author":"FJ Valverde-Albacete","year":"2014","unstructured":"Valverde-Albacete FJ, Pel\u00e1ez-Moreno C. 100% classification accuracy considered harmful: the normalized information transfer factor explains the accuracy paradox. PLoS\u00a0One. 2014;9:1\u201310. https:\/\/doi.org\/10.1371\/journal.pone.0084217.","journal-title":"PLoS\u00a0One"},{"key":"1409_CR19","doi-asserted-by":"publisher","first-page":"1145","DOI":"10.1016\/S0031-3203(96)00142-2","volume":"30","author":"AP Bradley","year":"1997","unstructured":"Bradley AP. The use of the area under the ROC curve in the evaluation of machine learning algorithms. Pattern Recognit. 1997;30:1145\u201359.","journal-title":"Pattern Recognit"},{"key":"1409_CR20","doi-asserted-by":"publisher","DOI":"10.1186\/s12864-019-6413-7","author":"D Chicco","year":"2020","unstructured":"Chicco D, Jurman G. The advantages of the Matthews correlation coefficient (MCC) over F1 score and accuracy in binary classification evaluation. BMC Genom. 2020. https:\/\/doi.org\/10.1186\/s12864-019-6413-7.","journal-title":"BMC Genom"},{"key":"1409_CR21","unstructured":"Hu B-G, Dong W-M (2014) A study on cost behaviors of binary classification measures in class-imbalanced problems. Comput Res Repos abs\/1403.7"},{"key":"1409_CR22","first-page":"21","volume":"6","author":"V Labatut","year":"2011","unstructured":"Labatut V, Cherifi H. Evaluation of performance measures for classifiers comparison. Ubiquitous Comput Commun J. 2011;6:21\u201334.","journal-title":"Ubiquitous Comput Commun J"},{"key":"1409_CR23","doi-asserted-by":"publisher","first-page":"206","DOI":"10.1109\/TKDE.2011.207","volume":"25","author":"S Wang","year":"2013","unstructured":"Wang S, Yao X. Relationships between diversity of classification ensembles and single-class performance measures. IEEE Trans Knowl Data Eng. 2013;25:206\u201319. https:\/\/doi.org\/10.1109\/TKDE.2011.207.","journal-title":"IEEE Trans Knowl Data Eng"},{"key":"1409_CR24","doi-asserted-by":"publisher","first-page":"427","DOI":"10.1016\/j.ipm.2009.03.002","volume":"45","author":"M Sokolova","year":"2009","unstructured":"Sokolova M, Lapalme G. A systematic analysis of performance measures for classification tasks. Inf Process Manag. 2009;45:427\u201337. https:\/\/doi.org\/10.1016\/j.ipm.2009.03.002.","journal-title":"Inf Process Manag"},{"key":"1409_CR25","first-page":"43","volume":"8","author":"C Seung-Seok","year":"2010","unstructured":"Seung-Seok C, Sung-Hyuk C, Tappert CC. A survey of binary similarity and distance measures. J Syst Cybern Inform. 2010;8:43\u20138.","journal-title":"J Syst Cybern Inform"},{"key":"1409_CR26","volume-title":"Similarity coefficients for binary data: properties of coefficients, coefficient matrices, multi-way metrics and multivariate coefficient","author":"MJ Warrens","year":"2008","unstructured":"Warrens MJ. Similarity coefficients for binary data: properties of coefficients, coefficient matrices, multi-way metrics and multivariate coefficient. Leiden: Leiden University; 2008."},{"key":"1409_CR27","unstructured":"Yan B, Koyejo O, Zhong K, Ravikumar P (2018) Binary classification with karmic, threshold-quasi-concave metrics. In: Proceedings of the35th international conference on machine learning (ICML), Stockholm, Sweden, pp 5527\u20135536"},{"key":"1409_CR28","doi-asserted-by":"publisher","first-page":"189","DOI":"10.1007\/BF01617722","volume":"11","author":"A Forbes","year":"1995","unstructured":"Forbes A. Classification-algorithm evaluation: five performance measures based on confusion matrices. J Clin Monit Comput. 1995;11:189\u2013206. https:\/\/doi.org\/10.1007\/BF01617722.","journal-title":"J Clin Monit Comput"},{"key":"1409_CR29","doi-asserted-by":"publisher","first-page":"299","DOI":"10.1109\/TKDE.2005.50","volume":"17","author":"J Huang","year":"2005","unstructured":"Huang J, Ling CX. Using AUC and accuracy in evaluating learning algorithms. IEEE Trans Knowl Data Eng. 2005;17:299\u2013310. https:\/\/doi.org\/10.1109\/TKDE.2005.50.","journal-title":"IEEE Trans Knowl Data Eng"},{"key":"1409_CR30","doi-asserted-by":"publisher","first-page":"14623","DOI":"10.1007\/s00521-021-06103-6","volume":"33","author":"G Canbek","year":"2021","unstructured":"Canbek G, Taskaya Temizel T, Sagiroglu S. BenchMetrics: a systematic benchmarking method for binary-classification performance metrics. Neural Comput Appl. 2021;33:14623\u201350. https:\/\/doi.org\/10.1007\/s00521-021-06103-6.","journal-title":"Neural Comput Appl"},{"key":"1409_CR31","doi-asserted-by":"publisher","first-page":"359","DOI":"10.1016\/j.ipm.2018.01.002","volume":"54","author":"RB Pereira","year":"2018","unstructured":"Pereira RB, Plastino A, Zadrozny B, Merschmann LHC. Correlation analysis of performance measures for multi-label classification. Inf Process Manag. 2018;54:359\u201369. https:\/\/doi.org\/10.1016\/j.ipm.2018.01.002.","journal-title":"Inf Process Manag"},{"key":"1409_CR32","volume-title":"Binary and multiclass classification","author":"B Kolo","year":"2011","unstructured":"Kolo B. Binary and multiclass classification. Weatherford: Weatherford Press; 2011."},{"key":"1409_CR33","doi-asserted-by":"publisher","first-page":"1103","DOI":"10.1016\/j.ipm.2017.04.004","volume":"53","author":"M Kocher","year":"2017","unstructured":"Kocher M, Savoy J. Distance measures in author profiling. Inf Process Manag. 2017;53:1103\u201319. https:\/\/doi.org\/10.1016\/j.ipm.2017.04.004.","journal-title":"Inf Process Manag"},{"key":"1409_CR34","first-page":"122","volume-title":"Mycology in sustainable development: expanding concepts, vanishing borders","author":"RE Tulloss","year":"1997","unstructured":"Tulloss RE. Assessment of similarity indices for undesirable properties and a new tripartite similarity index based on cost functions. In: Mycology in sustainable development: expanding concepts, vanishing borders. Boone: Parkway Publishers; 1997. p. 122\u201343."},{"key":"1409_CR35","unstructured":"Koyejo OO, Natarajan N, Ravikumar PK, Dhillon IS (2014) Consistent binary classification with generalized performance metrics. In: Advances in neural information processing systems 27: annual conference on neural information processing systems 2014, December 8\u201313 2014, Montreal, Quebec, Canada. ACM, Montreal, Canada, pp 2744\u20132752"},{"key":"1409_CR36","doi-asserted-by":"publisher","first-page":"645","DOI":"10.1515\/amcs-2015-0047","volume":"25","author":"M Paradowski","year":"2015","unstructured":"Paradowski M. On the order equivalence relation of binary association measures. Int J Appl Math Comput Sci. 2015;25:645\u201357. https:\/\/doi.org\/10.1515\/amcs-2015-0047.","journal-title":"Int J Appl Math Comput Sci"},{"key":"1409_CR37","doi-asserted-by":"publisher","first-page":"791","DOI":"10.1016\/j.ipm.2015.03.005","volume":"51","author":"T Kenter","year":"2015","unstructured":"Kenter T, Balog K, De Rijke M. Evaluating document filtering systems over time. Inf Process Manag. 2015;51:791\u2013808. https:\/\/doi.org\/10.1016\/j.ipm.2015.03.005.","journal-title":"Inf Process Manag"},{"key":"1409_CR38","doi-asserted-by":"publisher","first-page":"60","DOI":"10.1016\/j.ins.2016.12.005","volume":"382\u2013383","author":"M Carbonero-Ruz","year":"2017","unstructured":"Carbonero-Ruz M, Mart\u00ednez-Estudillo FJ, Fern\u00e1ndez-Navarro F, et al. A two dimensional accuracy-based measure for classification performance. Inf Sci (NY). 2017;382\u2013383:60\u201380. https:\/\/doi.org\/10.1016\/j.ins.2016.12.005.","journal-title":"Inf Sci (NY)"},{"key":"1409_CR39","doi-asserted-by":"publisher","first-page":"1","DOI":"10.5121\/ijdkp.2015.5201","volume":"5","author":"M Hossin","year":"2015","unstructured":"Hossin M, Sulaiman MN. A review on evaluation metrics for data classification evaluations. Int J Data Min Knowl Manag Process. 2015;5:1\u201311. https:\/\/doi.org\/10.5121\/ijdkp.2015.5201.","journal-title":"Int J Data Min Knowl Manag Process"},{"key":"1409_CR40","unstructured":"Welty C, Paritosh P, Aroyo L (2020) Metrology for AI: from benchmarks to instruments. In: The 34th AAAI conference on artificial intelligence (evaluating evaluation of AI systems workshop, Meta-Eval 2020). New York, NY"},{"key":"1409_CR41","doi-asserted-by":"crossref","unstructured":"Canbek G, Sagiroglu S, Temizel TT, Baykal N (2017) Binary classification performance measures\/metrics: a comprehensive visualized roadmap to gain new insights. In: 2017 International conference on computer science and engineering (UBMK). IEEE, Antalya, Turkey, pp 821\u2013826","DOI":"10.1109\/UBMK.2017.8093539"},{"key":"1409_CR42","doi-asserted-by":"publisher","first-page":"1257","DOI":"10.1038\/ki.2009.92","volume":"75","author":"KJ van Stralen","year":"2009","unstructured":"van Stralen KJ, Stel VS, Reitsma JB, et al. Diagnostic methods I: sensitivity, specificity, and other measures of accuracy. Kidney Int. 2009;75:1257\u201363. https:\/\/doi.org\/10.1038\/ki.2009.92.","journal-title":"Kidney Int"},{"key":"1409_CR43","volume-title":"Statistical methods in the atmospheric sciences","author":"DS Wilks","year":"2006","unstructured":"Wilks DS. Statistical methods in the atmospheric sciences. 2nd ed. New York: Elsevier; 2006.","edition":"2"},{"key":"1409_CR44","doi-asserted-by":"publisher","first-page":"412","DOI":"10.1093\/bioinformatics\/16.5.412","volume":"16","author":"P Baldi","year":"2000","unstructured":"Baldi P, Brunak S, Chauvin Y, et al. Assessing the accuracy of prediction algorithms for classification: an overview. Bioinformatics. 2000;16:412\u201324. https:\/\/doi.org\/10.1093\/bioinformatics\/16.5.412.","journal-title":"Bioinformatics"},{"key":"1409_CR45","doi-asserted-by":"publisher","first-page":"27","DOI":"10.1016\/j.patrec.2008.08.010","volume":"30","author":"C Ferri","year":"2009","unstructured":"Ferri C, Hern\u00e1ndez-Orallo J, Modroiu R. An experimental comparison of performance measures for classification. Pattern Recognit Lett. 2009;30:27\u201338. https:\/\/doi.org\/10.1016\/j.patrec.2008.08.010.","journal-title":"Pattern Recognit Lett"},{"key":"1409_CR46","doi-asserted-by":"publisher","first-page":"25","DOI":"10.1049\/iet-ifs.2013.0095","volume":"8","author":"SY Yerima","year":"2014","unstructured":"Yerima SY, Sezer S, McWilliams G. Analysis of Bayesian classification-based approaches for Android malware detection. IET Inf Secur. 2014;8:25\u201336. https:\/\/doi.org\/10.1049\/iet-ifs.2013.0095.","journal-title":"IET Inf Secur"},{"key":"1409_CR47","doi-asserted-by":"publisher","first-page":"545","DOI":"10.1016\/j.ipm.2012.10.001","volume":"49","author":"B Hj\u00f8rland","year":"2013","unstructured":"Hj\u00f8rland B. Facet analysis: the logical approach to knowledge organization. Inf Process Manag. 2013;49:545\u201357. https:\/\/doi.org\/10.1016\/j.ipm.2012.10.001.","journal-title":"Inf Process Manag"},{"key":"1409_CR48","doi-asserted-by":"publisher","first-page":"9","DOI":"10.5771\/0943-7444-2011-1-9","volume":"38","author":"B Hj\u00f8rland","year":"2011","unstructured":"Hj\u00f8rland B, Scerri E, Dupr\u00e9 J. Forum: the philosophy of classification. Knowl Organ. 2011;38:9\u201324.","journal-title":"Knowl Organ"},{"key":"1409_CR49","doi-asserted-by":"publisher","DOI":"10.1007\/978-1-4614-7822-5","volume-title":"Concepts, ontologies, and knowledge representation","author":"G Jakus","year":"2013","unstructured":"Jakus G, Milutinovi\u0107 V, Omerovi\u0107 S, Toma\u017ei\u010d S. Concepts, ontologies, and knowledge representation. New York: Springer; 2013."},{"key":"1409_CR50","unstructured":"Huang M, Brian\u00e7on A (2018) Cerebri AI periodic table of data science. In: Cerebri. https:\/\/www.cerebriai.com\/periodic-table. Accessed 15 Aug 2019"},{"key":"1409_CR51","doi-asserted-by":"publisher","first-page":"455","DOI":"10.1007\/s11634-016-0274-6","volume":"12","author":"G Govaert","year":"2018","unstructured":"Govaert G, Nadif M. Mutual information, phi-squared and model-based co-clustering for contingency tables. Adv Data Anal Classif. 2018;12:455\u201388. https:\/\/doi.org\/10.1007\/s11634-016-0274-6.","journal-title":"Adv Data Anal Classif"},{"key":"1409_CR52","doi-asserted-by":"publisher","first-page":"1169","DOI":"10.1016\/S1874-1029(11)60289-9","volume":"38","author":"B-G Hu","year":"2012","unstructured":"Hu B-G, He R, Yuan X-T. Information-theoretic measures for objective evaluation of classifications. Acta Autom Sin. 2012;38:1169\u201382. https:\/\/doi.org\/10.1016\/S1874-1029(11)60289-9.","journal-title":"Acta Autom Sin"},{"key":"1409_CR53","doi-asserted-by":"publisher","first-page":"83","DOI":"10.1093\/bib\/bbr008","volume":"13","author":"D Berrar","year":"2012","unstructured":"Berrar D, Flach P. Caveats and pitfalls of ROC analysis in clinical microarray research (and how to avoid them). Brief Bioinform. 2012;13:83\u201397. https:\/\/doi.org\/10.1093\/bib\/bbr008.","journal-title":"Brief Bioinform"},{"key":"1409_CR54","doi-asserted-by":"publisher","first-page":"195","DOI":"10.1007\/s11634-014-0167-5","volume":"8","author":"T Voigt","year":"2014","unstructured":"Voigt T, Fried R, Backes M, Rhode W. Threshold optimization for classification in imbalanced data in a problem of gamma-ray astronomy. Adv Data Anal Classif. 2014;8:195\u2013216. https:\/\/doi.org\/10.1007\/s11634-014-0167-5.","journal-title":"Adv Data Anal Classif"},{"key":"1409_CR55","doi-asserted-by":"publisher","first-page":"546","DOI":"10.1016\/B978-0-12-809633-8.20351-8","volume":"1","author":"D Berrar","year":"2018","unstructured":"Berrar D. Performance measures for binary classification. Encycl Bioinform Comput Biol ABC Bioinform. 2018;1:546\u201360. https:\/\/doi.org\/10.1016\/B978-0-12-809633-8.20351-8.","journal-title":"Encycl Bioinform Comput Biol ABC Bioinform"},{"key":"1409_CR56","volume-title":"Forecast verification: a practitioner\u2019s guide in atmospheric science","author":"IT Jolliffe","year":"2012","unstructured":"Jolliffe IT, Stephenson DB. Forecast verification: a practitioner\u2019s guide in atmospheric science. 2nd ed. Hoboken: Wiley; 2012.","edition":"2"},{"key":"1409_CR57","first-page":"1947","volume-title":"Expert systems: the technology of knowledge management and decision making for the 21st century","author":"E Ikonen","year":"2001","unstructured":"Ikonen E, Kortela U, Najim K. Distributed logic processors in process identification. In: Leondes CT, editor. Expert systems: the technology of knowledge management and decision making for the 21st century. New York: Academic Press; 2001. p. 1947."},{"key":"1409_CR58","doi-asserted-by":"publisher","first-page":"1173","DOI":"10.1142\/S0218001411009093","volume":"25","author":"JS Cardoso","year":"2011","unstructured":"Cardoso JS, Sousa R. Measuring the performance of ordinal classification. Int J Pattern Recognit Artif Intell. 2011;25:1173\u201395. https:\/\/doi.org\/10.1142\/S0218001411009093.","journal-title":"Int J Pattern Recognit Artif Intell"},{"key":"1409_CR59","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1007\/s42979-020-00239-3","volume":"1","author":"S Hirose","year":"2020","unstructured":"Hirose S, Kozu T, Jin Y, Miyamura Y. Hierarchical relevance determination based on information criterion minimization. SN Comput Sci. 2020;1:1\u201319. https:\/\/doi.org\/10.1007\/s42979-020-00239-3.","journal-title":"SN Comput Sci"},{"key":"1409_CR60","doi-asserted-by":"publisher","first-page":"659","DOI":"10.1080\/0952813X.2019.1592235","volume":"31","author":"RJ Chin","year":"2019","unstructured":"Chin RJ, Lai SH, Ibrahim S, et al. Rheological wall slip velocity prediction model based on artificial neural network. J Exp Theor Artif Intell. 2019;31:659\u201376. https:\/\/doi.org\/10.1080\/0952813X.2019.1592235.","journal-title":"J Exp Theor Artif Intell"},{"key":"1409_CR61","doi-asserted-by":"crossref","unstructured":"Caruana R, Niculescu-Mizil A (2004) Data mining in metric space: an empirical analysis of supervised learning performance criteria. In: Proceedings of 10th ACM SIGKDD international conference on knowledge discovery and data mining, pp 69\u201378. 1-58113-888-1\/04\/0008","DOI":"10.1145\/1014052.1014063"},{"key":"1409_CR62","doi-asserted-by":"crossref","unstructured":"Ranawana R, Palade V (2006) Optimized precision - a new measure for classifier performance evaluation. In: 2006 IEEE international conference on evolutionary computation. IEEE, Vancouver, BC, Canada, pp 2254\u20132261","DOI":"10.1109\/CEC.2006.1688586"},{"key":"1409_CR63","doi-asserted-by":"publisher","first-page":"617","DOI":"10.1109\/ICPR.2010.156","volume":"1","author":"V Garcia","year":"2006","unstructured":"Garcia V, Mollineda RA, Sanchez JS. Theoretical analysis of a performance measure for imbalanced data. IEEE Int Conf Pattern Recognit. 2006;1:617\u201320. https:\/\/doi.org\/10.1109\/ICPR.2010.156.","journal-title":"IEEE Int Conf Pattern Recognit"},{"key":"1409_CR64","doi-asserted-by":"publisher","first-page":"669","DOI":"10.1016\/j.ijforecast.2015.12.003","volume":"32","author":"S Kim","year":"2016","unstructured":"Kim S, Kim H. A new metric of absolute percentage error for intermittent demand forecasts. Int J Forecast. 2016;32:669\u201379. https:\/\/doi.org\/10.1016\/j.ijforecast.2015.12.003.","journal-title":"Int J Forecast"},{"key":"1409_CR65","doi-asserted-by":"crossref","unstructured":"Texel PP (2013) Measure, metric, and indicator: an object-oriented approach for consistent terminology. In: Proceedings of IEEE Southeastcon. IEEE, Jacksonville, FL","DOI":"10.1109\/SECON.2013.6567438"},{"key":"1409_CR66","first-page":"262","volume":"2","author":"L Olsina","year":"2004","unstructured":"Olsina L, de los Angeles Mart\u00edn M,. Ontology for software metrics and indicators: Building process and decisions taken. J Web Eng. 2004;2:262\u201381.","journal-title":"J Web Eng"},{"key":"1409_CR67","doi-asserted-by":"publisher","first-page":"631","DOI":"10.1016\/j.infsof.2005.07.001","volume":"48","author":"F Garc\u00eda","year":"2006","unstructured":"Garc\u00eda F, Bertoa MF, Calero C, et al. Towards a consistent terminology for software measurement. Inf Softw Technol. 2006;48:631\u201344. https:\/\/doi.org\/10.1016\/j.infsof.2005.07.001.","journal-title":"Inf Softw Technol"},{"key":"1409_CR68","unstructured":"Zammito F (2019) What\u2019s considered a good log loss in machine learning? https:\/\/medium.com\/@fzammito\/whats-considered-a-good-log-loss-in-machine-learning-a529d400632d. Accessed 15 Jul 2020"},{"key":"1409_CR69","doi-asserted-by":"publisher","first-page":"989","DOI":"10.1136\/bmj.316.7136.989","volume":"316","author":"HTO Davies","year":"1998","unstructured":"Davies HTO, Crombie IK, Tavakoli M. When can odds ratios mislead? BMJ. 1998;316:989\u201391. https:\/\/doi.org\/10.1136\/bmj.316.7136.989.","journal-title":"BMJ"},{"key":"1409_CR70","doi-asserted-by":"publisher","first-page":"165","DOI":"10.1007\/s00038-008-7068-3","volume":"53","author":"CO Schmidt","year":"2008","unstructured":"Schmidt CO, Kohlmann T. When to use the odds ratio or the relative risk? Int J Public Health. 2008;53:165\u20137. https:\/\/doi.org\/10.1007\/s00038-008-7068-3.","journal-title":"Int J Public Health"},{"key":"1409_CR71","doi-asserted-by":"publisher","first-page":"1129","DOI":"10.1016\/S0895-4356(03)00177-X","volume":"56","author":"AS Glas","year":"2003","unstructured":"Glas AS, Lijmer JG, Prins MH, et al. The diagnostic odds ratio: a single indicator of test performance. J Clin Epidemiol. 2003;56:1129\u201335. https:\/\/doi.org\/10.1016\/S0895-4356(03)00177-X.","journal-title":"J Clin Epidemiol"},{"key":"1409_CR72","doi-asserted-by":"publisher","first-page":"653","DOI":"10.1002\/rth2.12148","volume":"2","author":"B Siegerink","year":"2018","unstructured":"Siegerink B, Rohmann JL. Impact of your results: beyond the relative risk. Res Pract Thromb Haemost. 2018;2:653\u20137. https:\/\/doi.org\/10.1002\/rth2.12148.","journal-title":"Res Pract Thromb Haemost"},{"key":"1409_CR73","unstructured":"Press WH (2008) Classifier performance: ROC, precision-recall, and all that. In: Computational statistics with application to bioinformatics. The University of Texas at Austin, Austin"},{"key":"1409_CR74","volume-title":"An introduction to information retrieval, online edition","author":"CD Manning","year":"2009","unstructured":"Manning CD, Raghavan P, Sch\u00fctze H. An introduction to information retrieval, online edition. Cambridge: Cambridge University Press; 2009."},{"key":"1409_CR75","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1016\/j.ijmedinf.2017.01.001","volume":"100","author":"FR Lucini","year":"2017","unstructured":"Lucini FR, S. Fogliatto F, Giovani GJ, et al. Text mining approach to predict hospital admissions using early medical records from the emergency department. Int J Med Inform. 2017;100:1\u20138. https:\/\/doi.org\/10.1016\/j.ijmedinf.2017.01.001.","journal-title":"Int J Med Inform"},{"key":"1409_CR76","doi-asserted-by":"publisher","first-page":"157","DOI":"10.1016\/j.future.2017.10.016","volume":"80","author":"SAR Shah","year":"2018","unstructured":"Shah SAR, Issac B. Performance comparison of intrusion detection systems and application of machine learning to Snort system. Futur Gener Comput Syst. 2018;80:157\u201370. https:\/\/doi.org\/10.1016\/j.future.2017.10.016.","journal-title":"Futur Gener Comput Syst"},{"key":"1409_CR77","doi-asserted-by":"publisher","first-page":"67","DOI":"10.1016\/j.inffus.2018.08.002","volume":"48","author":"H Faris","year":"2019","unstructured":"Faris H, Al-Zoubi AM, Heidari AA, et al. An intelligent system for spam detection and identification of the most relevant features based on evolutionary random weight networks. Inf Fusion. 2019;48:67\u201383. https:\/\/doi.org\/10.1016\/j.inffus.2018.08.002.","journal-title":"Inf Fusion"},{"key":"1409_CR78","doi-asserted-by":"publisher","DOI":"10.1016\/j.eswa.2020.113661","author":"MM Ahamad","year":"2020","unstructured":"Ahamad MM, Aktar S, Rashed-Al-Mahfuz M, et al. A machine learning model to identify early stage symptoms of SARS-Cov-2 infected patients. Expert Syst Appl. 2020. https:\/\/doi.org\/10.1016\/j.eswa.2020.113661.","journal-title":"Expert Syst Appl"},{"key":"1409_CR79","doi-asserted-by":"publisher","first-page":"874","DOI":"10.1016\/j.engappai.2007.09.009","volume":"21","author":"A Ben-David","year":"2008","unstructured":"Ben-David A. About the relationship between ROC curves and Cohen\u2019s kappa. Eng Appl Artif Intell. 2008;21:874\u201382. https:\/\/doi.org\/10.1016\/j.engappai.2007.09.009.","journal-title":"Eng Appl Artif Intell"},{"key":"1409_CR80","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1002\/minf.201700127","volume":"37","author":"JB Brown","year":"2018","unstructured":"Brown JB. Classifiers and their metrics quantified. Mol Inform. 2018;37:1\u201311. https:\/\/doi.org\/10.1002\/minf.201700127.","journal-title":"Mol Inform"},{"key":"1409_CR81","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1109\/TNNLS.2019.2899061","volume":"31","author":"D Brzezinski","year":"2020","unstructured":"Brzezinski D, Stefanowski J, Susmaga R, Szczech I. On the dynamics of classification measures for imbalanced and streaming data. IEEE Trans Neural Netw Learn Syst. 2020;31:1\u201311. https:\/\/doi.org\/10.1109\/TNNLS.2019.2899061.","journal-title":"IEEE Trans Neural Netw Learn Syst"},{"key":"1409_CR82","doi-asserted-by":"crossref","unstructured":"Abdualgalil B, Abraham S (2020) Applications of machine learning algorithms and performance comparison: a review. In: International conference on emerging trends in information technology and engineering, ic-ETITE 2020. pp 1\u20136","DOI":"10.1109\/ic-ETITE47903.2020.490"},{"key":"1409_CR83","doi-asserted-by":"publisher","first-page":"683","DOI":"10.1007\/s11634-017-0295-9","volume":"12","author":"JM Vivo","year":"2018","unstructured":"Vivo JM, Franco M, Vicari D. Rethinking an ROC partial area index for evaluating the classification performance at a high specificity range. Adv Data Anal Classif. 2018;12:683\u2013704. https:\/\/doi.org\/10.1007\/s11634-017-0295-9.","journal-title":"Adv Data Anal Classif"},{"key":"1409_CR84","doi-asserted-by":"publisher","first-page":"1601","DOI":"10.1109\/TKDE.2011.59","volume":"23","author":"RC Prati","year":"2011","unstructured":"Prati RC, Batista GEAPA, Monard MC. A survey on graphical methods for classification predictive performance evaluation. IEEE Trans Knowl Data Eng. 2011;23:1601\u201318. https:\/\/doi.org\/10.1109\/TKDE.2011.59.","journal-title":"IEEE Trans Knowl Data Eng"},{"key":"1409_CR85","doi-asserted-by":"publisher","first-page":"45","DOI":"10.28945\/4184","volume":"14","author":"A Botchkarev","year":"2019","unstructured":"Botchkarev A. A new typology design of performance metrics to measure errors in machine learning regression algorithms. Interdiscip J Inf Knowl Manag. 2019;14:45\u201379. https:\/\/doi.org\/10.28945\/4184.","journal-title":"Interdiscip J Inf Knowl Manag"}],"container-title":["SN Computer Science"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s42979-022-01409-1.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s42979-022-01409-1\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s42979-022-01409-1.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,4,9]],"date-time":"2025-04-09T00:11:20Z","timestamp":1744157480000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s42979-022-01409-1"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2022,10,16]]},"references-count":85,"journal-issue":{"issue":"1","published-online":{"date-parts":[[2023,1]]}},"alternative-id":["1409"],"URL":"https:\/\/doi.org\/10.1007\/s42979-022-01409-1","relation":{},"ISSN":["2661-8907"],"issn-type":[{"value":"2661-8907","type":"electronic"}],"subject":[],"published":{"date-parts":[[2022,10,16]]},"assertion":[{"value":"29 October 2021","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"13 September 2022","order":2,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"16 October 2022","order":3,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Declarations"}},{"value":"The authors declare that they have no conflict of interest.","order":2,"name":"Ethics","group":{"name":"EthicsHeading","label":"Conflict of interest"}}],"article-number":"13"}}