{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,10,3]],"date-time":"2025-10-03T13:13:29Z","timestamp":1759497209402,"version":"3.37.3"},"reference-count":41,"publisher":"Institute of Electrical and Electronics Engineers (IEEE)","issue":"2","license":[{"start":{"date-parts":[[2023,2,1]],"date-time":"2023-02-01T00:00:00Z","timestamp":1675209600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/legalcode"},{"start":{"date-parts":[[2023,2,1]],"date-time":"2023-02-01T00:00:00Z","timestamp":1675209600000},"content-version":"am","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/legalcode"}],"funder":[{"DOI":"10.13039\/100000104","name":"NASA","doi-asserted-by":"publisher","award":["NNH14ZDA001N","80NSSC20K1352"],"award-info":[{"award-number":["NNH14ZDA001N","80NSSC20K1352"]}],"id":[{"id":"10.13039\/100000104","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100008982","name":"National Science Foundation","doi-asserted-by":"publisher","award":["AC1443061","AC1931555"],"award-info":[{"award-number":["AC1443061","AC1931555"]}],"id":[{"id":"10.13039\/501100008982","id-type":"DOI","asserted-by":"publisher"}]},{"name":"Division of Advanced Cyber"},{"DOI":"10.13039\/100000164","name":"Division of Astronomical Sciences","doi-asserted-by":"publisher","id":[{"id":"10.13039\/100000164","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000159","name":"Division of Atmospheric and Geospace Sciences","doi-asserted-by":"publisher","id":[{"id":"10.13039\/100000159","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["IEEE Trans. Pattern Anal. Mach. Intell."],"published-print":{"date-parts":[[2023,2,1]]},"DOI":"10.1109\/tpami.2022.3167007","type":"journal-article","created":{"date-parts":[[2022,4,13]],"date-time":"2022-04-13T19:36:32Z","timestamp":1649878592000},"page":"1501-1513","source":"Crossref","is-referenced-by-count":3,"title":["Contingency Space: A Semimetric Space for Classification Evaluation"],"prefix":"10.1109","volume":"45","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-1631-5336","authenticated-orcid":false,"given":"Azim","family":"Ahmadzadeh","sequence":"first","affiliation":[{"name":"Department of Computer Science, Georgia State University, Atlanta, GA, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-1837-8365","authenticated-orcid":false,"given":"Dustin J.","family":"Kempton","sequence":"additional","affiliation":[{"name":"Department of Computer Science, Georgia State University, Atlanta, GA, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Petrus C.","family":"Martens","sequence":"additional","affiliation":[{"name":"Department of Physics and Astronomy, Georgia State University, Atlanta, GA, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Rafal A.","family":"Angryk","sequence":"additional","affiliation":[{"name":"Department of Computer Science, Georgia State University, Atlanta, GA, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"263","reference":[{"volume-title":"On the Theory of Contingency and its Relation to Association and Normal Correlation; On the General Theory of Skew Correlation and Non-Linear Regression","year":"1904","author":"Pearson","key":"ref1"},{"issue":"3","key":"ref2","article-title":"Tornado predictions","volume":"1","author":"Finley","year":"1884","journal-title":"Amer. Meteorological J. A Monthly Rev. Meteorol. Allied Branches Study (1884\u20131896)"},{"key":"ref3","doi-asserted-by":"publisher","DOI":"10.1175\/1520-0434(1996)011<0003:TFAASE>2.0.CO;2"},{"article-title":"The national artificial intelligence research and development strategic plan","year":"2016","author":"Science","key":"ref4"},{"year":"2019","key":"ref5","article-title":"The National Artificial Intelligence Research and Development Strategic Plan: 2019 Update, A Report by the Select Committee on Artificial Intelligence of the National Sciences & Technology Council"},{"issue":"4","key":"ref6","first-page":"35","article-title":"How evaluation guides AI research: The message still counts more than the medium","volume-title":"AI Mag.","volume":"9","author":"Cohen","year":"1988"},{"article-title":"Machine learning that matters","volume-title":"Proc. 29th Int. Conf. Mach. Learn.","author":"Wagstaff","key":"ref7"},{"key":"ref8","doi-asserted-by":"publisher","DOI":"10.1145\/3317287.3328534"},{"article-title":"Too many AI researchers think real-world problems are not relevant","year":"2020","author":"Kerner","key":"ref9"},{"article-title":"Deep learning: A critical appraisal","year":"2018","author":"Marcus","key":"ref10"},{"article-title":"The field of natural language processing is chasing the wrong goal","year":"2020","author":"Dunietz","key":"ref11"},{"key":"ref12","doi-asserted-by":"publisher","DOI":"10.3115\/1072017.1072026"},{"article-title":"On the Relationship Between the Frequency of Rain and Various Meteorological Parameters,","year":"1965","author":"Hanssen","key":"ref13"},{"key":"ref14","doi-asserted-by":"publisher","DOI":"10.1002\/1097-0142(1950)3:1<32::AID-CNCR2820030106>3.0.CO;2-3"},{"issue":"2","key":"ref15","doi-asserted-by":"crossref","first-page":"171","DOI":"10.1023\/A:1010920819831","article-title":"A simple generalisation of the area under the ROC curve for multiple class classification problems","volume":"45","author":"Hand","year":"2001","journal-title":"Mach. Learn."},{"issue":"5","key":"ref16","article-title":"Finley\u2019s tornado predictions","volume":"1","author":"Gilbert","year":"1884","journal-title":"Amer. MeteoroLog. J. A Monthly Rev. Meteorol. Allied Branches Study (1884\u20131896)"},{"key":"ref17","first-page":"122","article-title":"Association ratios","volume":"7","author":"Doolittle","year":"1888","journal-title":"Bull. Philos. Soc. Washington"},{"key":"ref18","doi-asserted-by":"publisher","DOI":"10.1029\/2007SW000337"},{"volume-title":"Metric Spaces","year":"2006","author":"O\u2019Searcoid","key":"ref19"},{"key":"ref20","doi-asserted-by":"publisher","DOI":"10.1126\/science.182.4116.990"},{"key":"ref21","doi-asserted-by":"publisher","DOI":"10.1109\/TIT.1954.1057460"},{"key":"ref22","doi-asserted-by":"publisher","DOI":"10.1109\/MCSE.2007.55"},{"key":"ref23","first-page":"1087","article-title":"A quantification of distance bias between evaluation metrics in classification","volume-title":"Proc. 17th Int. Conf. Mach. Learn.","author":"Vilalta"},{"issue":"3","key":"ref24","doi-asserted-by":"crossref","first-page":"203","DOI":"10.1023\/A:1007601015854","article-title":"Robust classification for imprecise environments","volume":"42","author":"Provost","year":"2001","journal-title":"Mach. Learn."},{"issue":"2","key":"ref25","article-title":"Toward reliable benchmarking of solar flare forecasting methods","volume-title":"The AstroPhys. J.","volume":"747","author":"Bloomfield","year":"2012"},{"key":"ref26","doi-asserted-by":"publisher","DOI":"10.1038\/s41597-020-0548-x"},{"key":"ref27","doi-asserted-by":"crossref","DOI":"10.3847\/1538-4365\/abec88","article-title":"How to train your flare prediction model: Revisiting robust sampling of rare events","volume-title":"CoRR","author":"Ahmadzadeh","year":"2021"},{"article-title":"The MNIST database of handwritten digits","year":"1998","author":"LeCun","key":"ref28"},{"key":"ref29","doi-asserted-by":"publisher","DOI":"10.1109\/5.726791"},{"key":"ref30","doi-asserted-by":"publisher","DOI":"10.5555\/3454287.3455008"},{"article-title":"Sulla determinazione emp\u00edrica di uma legge di distribuzione","year":"1933","author":"Kolmogorov-Smirnov","key":"ref31"},{"key":"ref32","doi-asserted-by":"publisher","DOI":"10.1214\/aoms\/1177730256"},{"key":"ref33","doi-asserted-by":"publisher","DOI":"10.1145\/1014052.1014063"},{"volume-title":"Calculus: Single and multivariable","year":"2020","author":"Hughes-Hallett","key":"ref34"},{"key":"ref35","doi-asserted-by":"publisher","DOI":"10.1002\/9781119960003"},{"issue":"2","key":"ref36","article-title":"A review on evaluation metrics for data classification evaluations","volume":"5","author":"Hossin","year":"2015","journal-title":"Int. J. Data Mining Knowl. Manage. Process"},{"key":"ref37","doi-asserted-by":"publisher","DOI":"10.1007\/3-540-44503-X_27"},{"key":"ref38","doi-asserted-by":"publisher","DOI":"10.1177\/0272989X9901900110"},{"article-title":"Note on the location of optimal classifiers in N-dimensional ROC space","year":"1999","author":"Srinivasan","key":"ref39"},{"key":"ref40","first-page":"416","article-title":"Improving accuracy and cost of two-class and multi-class probabilistic classifiers using ROC curves","volume-title":"Proc. 20th Int. Conf.Mach. Learn.","author":"Lachiche"},{"key":"ref41","doi-asserted-by":"publisher","DOI":"10.1109\/34.990132"}],"container-title":["IEEE Transactions on Pattern Analysis and Machine Intelligence"],"original-title":[],"link":[{"URL":"https:\/\/ieeexplore.ieee.org\/ielam\/34\/10008914\/9756878-aam.pdf","content-type":"application\/pdf","content-version":"am","intended-application":"syndication"},{"URL":"http:\/\/xplorestaging.ieee.org\/ielx7\/34\/10008914\/09756878.pdf?arnumber=9756878","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,1,22]],"date-time":"2024-01-22T21:26:48Z","timestamp":1705958808000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/9756878\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023,2,1]]},"references-count":41,"journal-issue":{"issue":"2"},"URL":"https:\/\/doi.org\/10.1109\/tpami.2022.3167007","relation":{},"ISSN":["0162-8828","2160-9292","1939-3539"],"issn-type":[{"type":"print","value":"0162-8828"},{"type":"electronic","value":"2160-9292"},{"type":"electronic","value":"1939-3539"}],"subject":[],"published":{"date-parts":[[2023,2,1]]}}}