{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,6,18]],"date-time":"2026-06-18T11:18:47Z","timestamp":1781781527216,"version":"3.54.5"},"reference-count":104,"publisher":"Institute of Electrical and Electronics Engineers (IEEE)","license":[{"start":{"date-parts":[[2021,1,1]],"date-time":"2021-01-01T00:00:00Z","timestamp":1609459200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by-nc-nd\/4.0\/"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["IEEE Access"],"published-print":{"date-parts":[[2021]]},"DOI":"10.1109\/access.2021.3084050","type":"journal-article","created":{"date-parts":[[2021,5,26]],"date-time":"2021-05-26T22:06:05Z","timestamp":1622066765000},"page":"78368-78381","source":"Crossref","is-referenced-by-count":384,"title":["The Matthews Correlation Coefficient (MCC) is More Informative Than Cohen\u2019s Kappa and Brier Score in Binary Classification Assessment"],"prefix":"10.1109","volume":"9","author":[{"ORCID":"https:\/\/orcid.org\/0000-0001-9655-7142","authenticated-orcid":false,"given":"Davide","family":"Chicco","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-7302-640X","authenticated-orcid":false,"given":"Matthijs J.","family":"Warrens","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-2705-5728","authenticated-orcid":false,"given":"Giuseppe","family":"Jurman","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"263","reference":[{"key":"ref39","doi-asserted-by":"publisher","DOI":"10.1037\/1082-989X.9.3.386"},{"key":"ref38","doi-asserted-by":"publisher","DOI":"10.1371\/journal.pone.0222916"},{"key":"ref33","doi-asserted-by":"publisher","DOI":"10.1016\/j.stamet.2011.06.002"},{"key":"ref32","doi-asserted-by":"publisher","DOI":"10.2307\/3315487"},{"key":"ref31","doi-asserted-by":"publisher","DOI":"10.4172\/2161-0487.1000197"},{"key":"ref30","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1155\/2014\/231909","article-title":"New interpretations of Cohen&#x2019;s kappa","volume":"2014","author":"warrens","year":"2014","journal-title":"J Math"},{"key":"ref37","doi-asserted-by":"publisher","DOI":"10.1007\/s00357-008-9024-6"},{"key":"ref36","first-page":"226","article-title":"Cohen&#x2019;s kappa or McNemar&#x2019;s test? A comparison of binary repeated measurements","volume":"171","author":"krummenauer","year":"1999","journal-title":"RoFo Fortschritte Auf Dem Gebiete der Rontgenstrahlen und der Nuklearmedizin"},{"key":"ref35","first-page":"345","article-title":"A comparison of Cohen&#x2019;s kappa and agreement coefficients by Corrado Gini","volume":"16","author":"warrens","year":"2013","journal-title":"Int J Res Rev Appl Sci"},{"key":"ref34","doi-asserted-by":"publisher","DOI":"10.1016\/j.stamet.2012.05.004"},{"key":"ref28","doi-asserted-by":"publisher","DOI":"10.1016\/j.stamet.2010.09.004"},{"key":"ref27","doi-asserted-by":"publisher","DOI":"10.1016\/j.stamet.2010.05.003"},{"key":"ref29","doi-asserted-by":"publisher","DOI":"10.1177\/001316446002000104"},{"key":"ref20","doi-asserted-by":"publisher","DOI":"10.4135\/9781483395418.n9"},{"key":"ref22","doi-asserted-by":"publisher","DOI":"10.1207\/s15327906mbr1402_9"},{"key":"ref21","doi-asserted-by":"publisher","DOI":"10.1109\/TGRS.2013.2263510"},{"key":"ref24","first-page":"41","article-title":"The modification of the phi-coefficient reducing its dependence on the marginal distributions","volume":"2","author":"zysno","year":"1997","journal-title":"Methods of Psychological Research Online"},{"key":"ref23","doi-asserted-by":"publisher","DOI":"10.1016\/j.compbiolchem.2004.09.006"},{"key":"ref101","doi-asserted-by":"publisher","DOI":"10.1080\/01621459.1983.10478008"},{"key":"ref26","doi-asserted-by":"publisher","DOI":"10.1007\/s10742-011-0077-3"},{"key":"ref100","doi-asserted-by":"publisher","DOI":"10.1038\/s41598-020-61588-w"},{"key":"ref25","doi-asserted-by":"publisher","DOI":"10.1093\/ptj\/85.3.257"},{"key":"ref50","doi-asserted-by":"publisher","DOI":"10.1016\/j.jclinepi.2004.02.021"},{"key":"ref51","first-page":"129","article-title":"Can one use Cohen&#x2019;s kappa to examine disagreement?","volume":"1","author":"von eye","year":"2005","journal-title":"Methodology European J of Research Methods for the Behavioral and Social Sci"},{"key":"ref59","doi-asserted-by":"publisher","DOI":"10.1097\/NMD.0000000000000598"},{"key":"ref58","doi-asserted-by":"publisher","DOI":"10.2307\/2531434"},{"key":"ref57","first-page":"1","article-title":"On marginal dependencies of the $2\\times2$ kappa","volume":"2014","author":"warrens","year":"2014","journal-title":"Advanced Statistics"},{"key":"ref56","doi-asserted-by":"publisher","DOI":"10.1007\/s00357-010-9060-x"},{"key":"ref55","doi-asserted-by":"publisher","DOI":"10.1016\/0895-4356(93)90018-V"},{"key":"ref54","doi-asserted-by":"publisher","DOI":"10.1016\/0895-4356(90)90159-M"},{"key":"ref53","doi-asserted-by":"publisher","DOI":"10.1016\/0895-4356(90)90158-L"},{"key":"ref52","doi-asserted-by":"publisher","DOI":"10.1037\/a0037489"},{"key":"ref40","doi-asserted-by":"publisher","DOI":"10.1007\/s00357-008-9023-7"},{"key":"ref4","doi-asserted-by":"publisher","DOI":"10.1186\/s13040-021-00244-z"},{"key":"ref3","doi-asserted-by":"publisher","DOI":"10.1186\/s12864-019-6413-7"},{"key":"ref6","doi-asserted-by":"publisher","DOI":"10.1093\/bioinformatics\/16.5.412"},{"key":"ref5","doi-asserted-by":"publisher","DOI":"10.1109\/ACCESS.2021.3068614"},{"key":"ref8","year":"2020","journal-title":"Clog Loss Advance Alzheimer&#x2019;s Research With Stall Catchers"},{"key":"ref49","first-page":"93","article-title":"Interobserver agreement: Cohen&#x2019;s kappa coefficient does not necessarily reflect the percentage of patients with congruent classifications","volume":"35","author":"steinijans","year":"1997","journal-title":"Journal of Clinical Pharmacy and Therapeutics"},{"key":"ref7","year":"2019","journal-title":"Featured Prediction Competition VSB Power Line Fault Detection"},{"key":"ref9","doi-asserted-by":"publisher","DOI":"10.12688\/f1000research.7118.1"},{"key":"ref46","doi-asserted-by":"publisher","DOI":"10.1002\/sim.4780100512"},{"key":"ref45","doi-asserted-by":"publisher","DOI":"10.1037\/1082-989X.1.2.150"},{"key":"ref48","doi-asserted-by":"publisher","DOI":"10.1177\/0008068320060301"},{"key":"ref47","doi-asserted-by":"publisher","DOI":"10.1177\/0962280214552881"},{"key":"ref42","doi-asserted-by":"publisher","DOI":"10.1002\/9781118445112.stat00365"},{"key":"ref41","doi-asserted-by":"publisher","DOI":"10.1016\/j.engappai.2007.09.009"},{"key":"ref44","doi-asserted-by":"publisher","DOI":"10.1177\/001316449205200105"},{"key":"ref43","doi-asserted-by":"publisher","DOI":"10.1080\/03610920802618426"},{"key":"ref73","first-page":"2813","article-title":"A unified view of performance metrics: Translating threshold choice into expected classification loss","volume":"13","author":"hern\u00e1ndez-orallo","year":"2012","journal-title":"J Mach Learn Res"},{"key":"ref72","author":"tetlock","year":"2016","journal-title":"Superforecasting The Art and Science of Prediction"},{"key":"ref71","doi-asserted-by":"publisher","DOI":"10.1080\/01431161.2015.1011794"},{"key":"ref70","doi-asserted-by":"publisher","DOI":"10.1080\/01431161.2011.552923"},{"key":"ref76","doi-asserted-by":"publisher","DOI":"10.1175\/1520-0493(1986)114<2671:ANDOTB>2.0.CO;2"},{"key":"ref77","doi-asserted-by":"publisher","DOI":"10.1016\/S0169-2607(01)00157-2"},{"key":"ref74","doi-asserted-by":"publisher","DOI":"10.1175\/1520-0493(1950)078<0001:VOFEIT>2.0.CO;2"},{"key":"ref75","doi-asserted-by":"crossref","first-page":"26","DOI":"10.1080\/00031305.1985.10479382","article-title":"Separating the Brier score into calibration and refinement components: A graphical exposition","volume":"39","author":"blattenberger","year":"1985","journal-title":"Amer Statistician"},{"key":"ref78","article-title":"The problem with the Brier score","author":"jewson","year":"2004","journal-title":"arXiv physics\/0401046"},{"key":"ref79","doi-asserted-by":"publisher","DOI":"10.1002\/bimj.200610301"},{"key":"ref60","doi-asserted-by":"publisher","DOI":"10.1037\/h0026256"},{"key":"ref62","doi-asserted-by":"crossref","first-page":"223","DOI":"10.2466\/pr0.1989.65.1.223","article-title":"Note on Cohen&#x2019;s kappa","volume":"65","author":"kv\u00e5lseth","year":"1989","journal-title":"Psychol Rep"},{"key":"ref61","doi-asserted-by":"publisher","DOI":"10.1177\/001316447303300309"},{"key":"ref63","doi-asserted-by":"publisher","DOI":"10.1055\/s-2007-976535"},{"key":"ref64","doi-asserted-by":"publisher","DOI":"10.1177\/0013164488484007"},{"key":"ref65","doi-asserted-by":"publisher","DOI":"10.1177\/0013164405285548"},{"key":"ref66","doi-asserted-by":"publisher","DOI":"10.1093\/inthealth\/ihu010"},{"key":"ref67","first-page":"261","article-title":"The modified Cohen&#x2019;s kappa: Calculating interrater agreement for segmentation and annotation","author":"holle","year":"2013","journal-title":"Understanding Body Movement"},{"key":"ref68","doi-asserted-by":"publisher","DOI":"10.1109\/FUZZY.2010.5584447"},{"key":"ref2","doi-asserted-by":"publisher","DOI":"10.1371\/journal.pone.0041882"},{"key":"ref69","doi-asserted-by":"publisher","DOI":"10.1109\/TGRS.2005.843569"},{"key":"ref1","doi-asserted-by":"publisher","DOI":"10.1016\/0005-2795(75)90109-9"},{"key":"ref95","first-page":"585","article-title":"Brier curves: A new cost-based visualisation of classifier performance","author":"hern\u00e1ndez-orallo","year":"2011","journal-title":"Proc 28th Int Conf Mach Learn (ICML)"},{"key":"ref94","doi-asserted-by":"crossref","first-page":"185","DOI":"10.1017\/S1930297500007099","article-title":"Weighted Brier score decompositions for topically heterogenous forecasting tournaments","volume":"13","author":"merkle","year":"2018","journal-title":"Judgment and Decision Making"},{"key":"ref93","doi-asserted-by":"publisher","DOI":"10.1002\/qj.2228"},{"key":"ref92","doi-asserted-by":"publisher","DOI":"10.1002\/qj.1924"},{"key":"ref91","doi-asserted-by":"publisher","DOI":"10.1002\/qj.641"},{"key":"ref104","doi-asserted-by":"publisher","DOI":"10.1109\/IVS.2018.8500497"},{"key":"ref90","year":"2021","journal-title":"Machine Learning Repository"},{"key":"ref103","doi-asserted-by":"publisher","DOI":"10.1186\/s12880-015-0068-x"},{"key":"ref102","doi-asserted-by":"publisher","DOI":"10.1371\/journal.pone.0240215"},{"key":"ref98","article-title":"The MCC-F1 curve: A performance evaluation technique for binary classification","author":"cao","year":"2020","journal-title":"arXiv 2006 11278"},{"key":"ref99","doi-asserted-by":"publisher","DOI":"10.1061\/(ASCE)0733-9364(1994)120:2(288)"},{"key":"ref96","doi-asserted-by":"publisher","DOI":"10.2190\/CS.14.2.d"},{"key":"ref97","doi-asserted-by":"publisher","DOI":"10.1186\/s41512-017-0020-3"},{"key":"ref10","year":"2020","journal-title":"DREAM Challenges"},{"key":"ref11","doi-asserted-by":"publisher","DOI":"10.1038\/nbt.1665"},{"key":"ref12","doi-asserted-by":"publisher","DOI":"10.1038\/nbt.2957"},{"key":"ref13","doi-asserted-by":"publisher","DOI":"10.1371\/journal.pone.0177678"},{"key":"ref14","doi-asserted-by":"publisher","DOI":"10.1016\/j.patrec.2020.03.030"},{"key":"ref15","article-title":"A critical analysis of metrics used for measuring progress in artificial intelligence","author":"blagec","year":"2020","journal-title":"arXiv 2008 02577"},{"key":"ref16","doi-asserted-by":"publisher","DOI":"10.1186\/s13040-017-0155-3"},{"key":"ref82","doi-asserted-by":"publisher","DOI":"10.1175\/2007WAF2006116.1"},{"key":"ref17","year":"2020","journal-title":"Matthews Correlation Coefficient is the Best Classification Metric You&#x2019;ve Never Heard of"},{"key":"ref81","doi-asserted-by":"publisher","DOI":"10.1002\/met.21"},{"key":"ref18","year":"2020","journal-title":"You need to know about the Matthews Correlation Coefficient"},{"key":"ref84","doi-asserted-by":"publisher","DOI":"10.1175\/2007WAF2007049.1"},{"key":"ref19","volume":"43","author":"cram\u00e9r","year":"1999","journal-title":"Mathematical Methods of Statistics"},{"key":"ref83","doi-asserted-by":"publisher","DOI":"10.1175\/WAF1034.1"},{"key":"ref80","doi-asserted-by":"publisher","DOI":"10.1175\/MWR3442.1"},{"key":"ref89","doi-asserted-by":"publisher","DOI":"10.1145\/1830483.1830668"},{"key":"ref85","doi-asserted-by":"publisher","DOI":"10.1002\/qj.709"},{"key":"ref86","doi-asserted-by":"publisher","DOI":"10.1016\/j.jclinepi.2009.11.009"},{"key":"ref87","doi-asserted-by":"publisher","DOI":"10.1002\/sim.4780050506"},{"key":"ref88","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-540-89208-3_405"}],"container-title":["IEEE Access"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx7\/6287639\/9312710\/09440903.pdf?arnumber=9440903","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,8,31]],"date-time":"2024-08-31T16:29:29Z","timestamp":1725121769000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/9440903\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2021]]},"references-count":104,"URL":"https:\/\/doi.org\/10.1109\/access.2021.3084050","relation":{},"ISSN":["2169-3536"],"issn-type":[{"value":"2169-3536","type":"electronic"}],"subject":[],"published":{"date-parts":[[2021]]}}}