{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,11,27]],"date-time":"2025-11-27T19:10:15Z","timestamp":1764270615047,"version":"3.46.0"},"reference-count":148,"publisher":"Institute of Electrical and Electronics Engineers (IEEE)","issue":"6","license":[{"start":{"date-parts":[[2025,12,1]],"date-time":"2025-12-01T00:00:00Z","timestamp":1764547200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/ieeexplore.ieee.org\/Xplorehelp\/downloads\/license-information\/IEEE.html"},{"start":{"date-parts":[[2025,12,1]],"date-time":"2025-12-01T00:00:00Z","timestamp":1764547200000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2025,12,1]],"date-time":"2025-12-01T00:00:00Z","timestamp":1764547200000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"}],"funder":[{"name":"Portuguese Recovery and Resilience Plan","award":["C645008882-00000055"],"award-info":[{"award-number":["C645008882-00000055"]}]}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["IEEE Trans. Emerg. Top. Comput. Intell."],"published-print":{"date-parts":[[2025,12]]},"DOI":"10.1109\/tetci.2025.3577486","type":"journal-article","created":{"date-parts":[[2025,6,18]],"date-time":"2025-06-18T13:41:04Z","timestamp":1750254064000},"page":"3754-3772","source":"Crossref","is-referenced-by-count":0,"title":["Towards a Comprehensive Taxonomy of Pointwise Reliability Assessment Techniques for Machine Learning Models"],"prefix":"10.1109","volume":"9","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-9300-4179","authenticated-orcid":false,"given":"Jo\u00e3o","family":"Loureiro","sequence":"first","affiliation":[{"name":"Centre for Informatics and Systems of the University of Coimbra, University of Coimbra, Coimbra, Portugal"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-2045-0741","authenticated-orcid":false,"given":"Lorena","family":"Petrella","sequence":"additional","affiliation":[{"name":"Centre for Informatics and Systems of the University of Coimbra, University of Coimbra, Coimbra, Portugal"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-0937-4044","authenticated-orcid":false,"given":"Paulo","family":"Gil","sequence":"additional","affiliation":[{"name":"Center of Technology and Systems of the Nova School of Science and Technology, Caparica, Lisbon, Portugal"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-3497-7368","authenticated-orcid":false,"given":"Teresa","family":"Rocha","sequence":"additional","affiliation":[{"name":"Polytechnic Institute of Coimbra, Coimbra, Portugal"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-1021-1173","authenticated-orcid":false,"given":"Sim\u00e3o","family":"Paredes","sequence":"additional","affiliation":[{"name":"Polytechnic Institute of Coimbra, Coimbra, Portugal"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-4622-474X","authenticated-orcid":false,"given":"Jorge","family":"Henriques","sequence":"additional","affiliation":[{"name":"Centre for Informatics and Systems of the University of Coimbra, University of Coimbra, Coimbra, Portugal"}]}],"member":"263","reference":[{"article-title":"Gathering strength, gathering storms: The one hundred year study on artificial intelligence (AI100) 2021 study panel report","year":"2021","author":"Littman","key":"ref1"},{"key":"ref2","doi-asserted-by":"publisher","DOI":"10.1109\/RBME.2020.3013489"},{"key":"ref3","doi-asserted-by":"publisher","DOI":"10.1080\/13600834.2019.1573501"},{"key":"ref4","doi-asserted-by":"publisher","DOI":"10.1609\/aimag.v38i3.2741"},{"key":"ref5","doi-asserted-by":"publisher","DOI":"10.1093\/idpl\/ipx005"},{"key":"ref6","doi-asserted-by":"publisher","DOI":"10.1109\/DSAA.2018.00018"},{"key":"ref7","doi-asserted-by":"publisher","DOI":"10.1186\/s13054-019-2564-9"},{"key":"ref8","doi-asserted-by":"publisher","DOI":"10.1007\/s00521-019-04051-w"},{"key":"ref9","doi-asserted-by":"publisher","DOI":"10.1038\/s41524-019-0248-2"},{"key":"ref10","first-page":"23631","article-title":"Mitigating neural network overconfidence with logit normalization","volume-title":"Proc. 39th Int. Conf. Mach. Learn.","author":"Wei","year":"2022"},{"key":"ref11","first-page":"13678","article-title":"A closer look at the robustness of contrastive language-image pre-training (CLIP)","volume-title":"Proc. 37th Int. Conf. Neural Inf. Process. Syst.","volume":"36","author":"Tu","year":"2024"},{"key":"ref12","doi-asserted-by":"publisher","DOI":"10.1109\/ACCESS.2018.2807385"},{"key":"ref13","doi-asserted-by":"publisher","DOI":"10.1145\/3442188.3445923"},{"key":"ref14","doi-asserted-by":"publisher","DOI":"10.1016\/S0893-6080(99)00080-5"},{"key":"ref15","doi-asserted-by":"publisher","DOI":"10.1007\/BF00116900"},{"key":"ref16","doi-asserted-by":"publisher","DOI":"10.1016\/j.ijhcs.2020.102493"},{"key":"ref17","doi-asserted-by":"publisher","DOI":"10.1016\/j.jbi.2022.103996"},{"key":"ref18","doi-asserted-by":"publisher","DOI":"10.1007\/3-540-36755-1_19"},{"key":"ref19","doi-asserted-by":"publisher","DOI":"10.1016\/j.datak.2008.08.001"},{"key":"ref20","doi-asserted-by":"publisher","DOI":"10.1016\/S0933-3657(03)00043-5"},{"key":"ref21","doi-asserted-by":"publisher","DOI":"10.3233\/IDA-2009-0371"},{"key":"ref22","doi-asserted-by":"publisher","DOI":"10.1371\/journal.pone.0048723"},{"key":"ref23","first-page":"722","article-title":"Transduction with confidence and credibility","volume-title":"Proc. 16th Int. Joint Conf. Artif. Intell.","author":"Saunders","year":"1999"},{"key":"ref24","doi-asserted-by":"publisher","DOI":"10.2307\/2987588"},{"key":"ref25","first-page":"3459","article-title":"Evaluating model calibration in classification","volume-title":"Proc. 22nd Int. Conf. Artif. Intell. Statist.","author":"Vaicenavicius","year":"2019"},{"issue":"3","key":"ref26","first-page":"61","article-title":"Probabilistic outputs for support vector machines and comparisons to regularized likelihood methods","volume":"10","author":"Platt","year":"1999","journal-title":"Adv. Large Margin Classifiers"},{"key":"ref27","first-page":"16","article-title":"Smooth isotonic regression: A new method to calibrate predictive models","volume-title":"Proc. AMIA Summits Transl. Sci.","author":"Jiang","year":"2011"},{"key":"ref28","first-page":"623","article-title":"Beta calibration: A well-founded and easily implemented improvement on logistic calibration for binary classifiers","volume-title":"Proc. Artif. Intell. Statist.","author":"Kull","year":"2017"},{"key":"ref29","doi-asserted-by":"publisher","DOI":"10.1111\/j.1467-9868.2007.00587.x"},{"key":"ref30","first-page":"2796","article-title":"Accurate uncertainties for deep learning using calibrated regression","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Kuleshov","year":"2018"},{"key":"ref31","first-page":"5897","article-title":"Distribution calibration for regression","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Song","year":"2019"},{"key":"ref32","doi-asserted-by":"publisher","DOI":"10.1175\/1520-0434(1993)008<0281:WIAGFA>2.0.CO;2"},{"key":"ref33","doi-asserted-by":"publisher","DOI":"10.1007\/s10994-021-05994-9"},{"key":"ref34","doi-asserted-by":"publisher","DOI":"10.1126\/science.aaw4399"},{"key":"ref35","doi-asserted-by":"publisher","DOI":"10.1002\/jae.1192"},{"key":"ref36","doi-asserted-by":"publisher","DOI":"10.1088\/2632-2153\/ab7e1a"},{"key":"ref37","first-page":"20827","article-title":"Out-of-distribution detection with deep nearest neighbors","volume-title":"Proc. 39th Int. Conf. Mach. Learn.","volume":"162","author":"Sun","year":"2022"},{"key":"ref38","first-page":"1617","article-title":"Mandoline: Model evaluation under distribution shift","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Chen","year":"2021"},{"key":"ref39","first-page":"1454","article-title":"Feed two birds with one scone: Exploiting wild data for both out-of-distribution generalization and detection","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Bai","year":"2023"},{"issue":"8","key":"ref40","first-page":"1823","article-title":"Classification with a reject option using a hinge loss","volume":"9","author":"Bartlett","year":"2008","journal-title":"J. Mach. Learn. Res."},{"key":"ref41","first-page":"2151","article-title":"SelectiveNet: A deep neural network with an integrated reject option","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Geifman","year":"2019"},{"key":"ref42","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-11261-4_10"},{"key":"ref43","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-540-69073-3_24"},{"key":"ref44","doi-asserted-by":"publisher","DOI":"10.1109\/TPAMI.2012.256"},{"key":"ref45","doi-asserted-by":"publisher","DOI":"10.1109\/TPAMI.2020.2981604"},{"key":"ref46","doi-asserted-by":"publisher","DOI":"10.3390\/electronics8080832"},{"key":"ref47","article-title":"Tutorial: Safe and reliable machine learning","volume-title":"ACM Conf. Fairness, Accountability, Transparency","author":"Saria","year":"2019"},{"key":"ref48","doi-asserted-by":"publisher","DOI":"10.1109\/72.963764"},{"key":"ref49","doi-asserted-by":"publisher","DOI":"10.1038\/s41746-020-00367-3"},{"key":"ref50","doi-asserted-by":"publisher","DOI":"10.1016\/j.strusafe.2008.06.020"},{"key":"ref51","doi-asserted-by":"publisher","DOI":"10.1016\/j.ins.2013.07.030"},{"key":"ref52","doi-asserted-by":"publisher","DOI":"10.1007\/s10994-021-05946-3"},{"volume-title":"Probabilistic Machine Learning: An Introduction","year":"2022","author":"Murphy","key":"ref53"},{"key":"ref54","first-page":"371","article-title":"A tutorial on conformal prediction","volume":"9","author":"Shafer","year":"2008","journal-title":"J. Mach. Learn. Res."},{"key":"ref55","first-page":"867","article-title":"Properties of machine learning applications for use in metamorphic testing","volume-title":"Proc. 20th Int. Conf. Softw. Eng. Knowl. Eng.","volume":"8","author":"Murphy","year":"2008"},{"key":"ref56","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-31753-3_7"},{"key":"ref57","doi-asserted-by":"publisher","DOI":"10.1561\/2200000101"},{"key":"ref58","doi-asserted-by":"publisher","DOI":"10.1186\/1471-2105-10-24"},{"key":"ref59","doi-asserted-by":"publisher","DOI":"10.1016\/j.jss.2010.11.920"},{"key":"ref60","doi-asserted-by":"publisher","DOI":"10.3390\/electronics11030396"},{"key":"ref61","doi-asserted-by":"publisher","DOI":"10.1145\/3236386.3241340"},{"key":"ref62","doi-asserted-by":"publisher","DOI":"10.1007\/s10115-005-0203-z"},{"key":"ref63","first-page":"444","article-title":"Machine-learning applications of algorithmic randomness","volume-title":"Proc. 16th Int. Conf. Mach. Learn.","author":"Vovk","year":"1999"},{"key":"ref64","doi-asserted-by":"publisher","DOI":"10.1007\/3-540-44795-4_31"},{"key":"ref65","doi-asserted-by":"publisher","DOI":"10.1007\/3-540-36755-1_32"},{"key":"ref66","first-page":"176","article-title":"Practical confidence and prediction intervals","volume":"9","author":"Heskes","year":"1996","journal-title":"in Proc. Adv. Neural Inf. Process. Syst."},{"key":"ref67","doi-asserted-by":"publisher","DOI":"10.3390\/molecules17054791"},{"key":"ref68","doi-asserted-by":"publisher","DOI":"10.1177\/026119290503300209"},{"key":"ref69","doi-asserted-by":"publisher","DOI":"10.1177\/026119290503300508"},{"key":"ref70","doi-asserted-by":"publisher","DOI":"10.1016\/0169-7439(87)80084-9"},{"key":"ref71","doi-asserted-by":"publisher","DOI":"10.1007\/978-1-4612-1098-6_3"},{"article-title":"The characterisation of (quantitative) structure-activity relationships: Preliminary guidance","year":"2005","author":"Worth","key":"ref72"},{"key":"ref73","doi-asserted-by":"publisher","DOI":"10.1021\/ci020058s"},{"key":"ref74","doi-asserted-by":"publisher","DOI":"10.1145\/3551385"},{"key":"ref75","doi-asserted-by":"publisher","DOI":"10.1109\/TSSC.1970.300339"},{"key":"ref76","doi-asserted-by":"publisher","DOI":"10.1039\/9781788016841-00063"},{"key":"ref77","doi-asserted-by":"publisher","DOI":"10.1093\/bioinformatics\/btn349"},{"key":"ref78","doi-asserted-by":"publisher","DOI":"10.1109\/ICIAP.2003.1234113"},{"key":"ref79","article-title":"Reject option paradigm for the reduction of support vectors","volume-title":"Proc. Eur. Symp. Artif. Neural Netw.","author":"Sousa","year":"2014"},{"key":"ref80","doi-asserted-by":"publisher","DOI":"10.1016\/0098-1354(92)80035-8"},{"key":"ref81","first-page":"2902","article-title":"Addressing failure prediction by learning model confidence","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","volume":"32","author":"Corbire","year":"2019"},{"key":"ref82","doi-asserted-by":"publisher","DOI":"10.1109\/72.410358"},{"key":"ref83","doi-asserted-by":"publisher","DOI":"10.1007\/s00521-015-1822-2"},{"key":"ref84","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v33i01.33013614"},{"article-title":"Uncertainty estimation for classification and risk prediction on medical tabular data","year":"2020","author":"Meijerink","key":"ref85"},{"key":"ref86","first-page":"1022","article-title":"Can you trust this prediction? auditing pointwise reliability after learning","volume-title":"Proc. 22nd Int. Conf. Artif. Intell. Statist.","author":"Schulam","year":"2019"},{"key":"ref87","doi-asserted-by":"publisher","DOI":"10.1049\/cp.2013.2636"},{"key":"ref88","doi-asserted-by":"publisher","DOI":"10.1109\/TENCON.2003.1273228"},{"key":"ref89","doi-asserted-by":"publisher","DOI":"10.1016\/S0169-7439(99)00047-7"},{"key":"ref90","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-540-74048-3_4"},{"key":"ref91","doi-asserted-by":"publisher","DOI":"10.1016\/j.ins.2007.05.027"},{"key":"ref92","doi-asserted-by":"publisher","DOI":"10.1023\/A:1020499411651"},{"key":"ref93","doi-asserted-by":"publisher","DOI":"10.3127\/ajis.v22i0.1538"},{"key":"ref94","doi-asserted-by":"publisher","DOI":"10.1080\/03610918.2017.1367806"},{"key":"ref95","doi-asserted-by":"publisher","DOI":"10.7815\/ijorcs.21.2011.011"},{"key":"ref96","first-page":"490","article-title":"Fast similarity search in the presence of noise, scaling, and translation in time-series databases","volume-title":"Proc. 21th Int. Conf. Very Large Data Bases","author":"Lin","year":"1995"},{"key":"ref97","doi-asserted-by":"publisher","DOI":"10.1016\/0098-3004(84)90020-7"},{"key":"ref98","doi-asserted-by":"publisher","DOI":"10.1007\/s00357-008-9004-x"},{"key":"ref99","doi-asserted-by":"publisher","DOI":"10.1016\/j.procs.2015.06.090"},{"key":"ref100","doi-asserted-by":"publisher","DOI":"10.1145\/3068335"},{"key":"ref101","doi-asserted-by":"publisher","DOI":"10.1145\/304181.304187"},{"key":"ref102","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-662-44848-9_35"},{"key":"ref103","doi-asserted-by":"publisher","DOI":"10.1109\/ISIT44484.2020.9174111"},{"key":"ref104","doi-asserted-by":"publisher","DOI":"10.1109\/TIT.1970.1054406"},{"key":"ref105","doi-asserted-by":"publisher","DOI":"10.1109\/ICOSP.2014.7015230"},{"key":"ref106","doi-asserted-by":"publisher","DOI":"10.1021\/ci049782w"},{"key":"ref107","doi-asserted-by":"publisher","DOI":"10.1186\/1758-2946-5-27"},{"issue":"6","key":"ref108","first-page":"1429","article-title":"A new distance-weighted K-nearest neighbor classifier","volume":"9","author":"Gou","year":"2012","journal-title":"J. Inf. Comput. Sci"},{"key":"ref109","doi-asserted-by":"publisher","DOI":"10.1109\/ICRA.2015.7139702"},{"key":"ref110","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2019.00300"},{"key":"ref111","article-title":"A reliable machine learning approach applied to single-cell classification in acute myeloid leukemia","volume-title":"AMIA Annu. Symp. Proc.","volume":"2020","author":"Nicora","year":"2020"},{"key":"ref112","doi-asserted-by":"publisher","DOI":"10.1016\/S0031-3203(02)00119-X"},{"key":"ref113","doi-asserted-by":"publisher","DOI":"10.1016\/0020-0190(72)90045-2"},{"key":"ref114","doi-asserted-by":"publisher","DOI":"10.1145\/321556.321564"},{"key":"ref115","doi-asserted-by":"publisher","DOI":"10.1007\/BF02712873"},{"key":"ref116","doi-asserted-by":"publisher","DOI":"10.1016\/0020-0190(73)90020-3"},{"key":"ref117","doi-asserted-by":"publisher","DOI":"10.1145\/359423.359430"},{"key":"ref118","doi-asserted-by":"publisher","DOI":"10.5220\/0006542603140321"},{"key":"ref119","first-page":"1321","article-title":"On calibration of modern neural networks","volume-title":"Proc. 34th Int. Conf. Mach. Learn.","volume":"70","author":"Guo","year":"2017"},{"key":"ref120","doi-asserted-by":"publisher","DOI":"10.1109\/ACCESS.2020.2996495"},{"key":"ref121","doi-asserted-by":"publisher","DOI":"10.1016\/j.inffus.2023.101978"},{"key":"ref122","doi-asserted-by":"publisher","DOI":"10.1016\/j.media.2024.103223"},{"key":"ref123","doi-asserted-by":"publisher","DOI":"10.1016\/j.compbiomed.2023.107758"},{"key":"ref124","first-page":"9690","article-title":"Uncertainty estimation using a single deep deterministic neural network","volume-title":"Proc. 37th Int. Conf. Mach. Learn.","volume":"119","author":"Amersfoort","year":"2020"},{"key":"ref125","first-page":"3183","article-title":"Evidential deep learning to quantify classification uncertainty","volume-title":"Proc. 32nd Int. Conf. Neural Inf. Process. Syst.","volume":"31","author":"Sensoy","year":"2018"},{"key":"ref126","first-page":"7047","article-title":"Predictive uncertainty estimation via prior networks","volume-title":"Proc. 32nd Int. Conf. Neural Inf. Process. Syst.","volume":"31","author":"Malinin","year":"2018"},{"key":"ref127","doi-asserted-by":"publisher","DOI":"10.1038\/s41586-024-07421-0"},{"key":"ref128","doi-asserted-by":"publisher","DOI":"10.1038\/s41524-022-00794-8"},{"key":"ref129","doi-asserted-by":"publisher","DOI":"10.14358\/PERS.82.3.189"},{"key":"ref130","first-page":"7419","article-title":"Pointwise uncertainty quantification for sparse variational Gaussian process regression with a brownian motion prior","volume-title":"Proc. 37th Int. Conf. Neural Inf. Process. Syst.","volume":"36","author":"Travis","year":"2024"},{"key":"ref131","doi-asserted-by":"publisher","DOI":"10.1016\/j.ress.2023.109750"},{"key":"ref132","doi-asserted-by":"publisher","DOI":"10.1016\/j.engappai.2024.107871"},{"key":"ref133","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v38i19.30084"},{"key":"ref134","doi-asserted-by":"publisher","DOI":"10.1007\/BF00058655"},{"key":"ref135","doi-asserted-by":"publisher","DOI":"10.1109\/IJCNN.1999.831133"},{"key":"ref136","doi-asserted-by":"publisher","DOI":"10.1186\/s13321-016-0182-y"},{"key":"ref137","first-page":"542","volume-title":"Cross-Validation","author":"Daniel","year":"2019"},{"key":"ref138","doi-asserted-by":"publisher","DOI":"10.1016\/j.patcog.2015.03.009"},{"key":"ref139","doi-asserted-by":"publisher","DOI":"10.1287\/ijoc.2020.1019"},{"key":"ref140","doi-asserted-by":"publisher","DOI":"10.1038\/s41746-019-0209-7"},{"key":"ref141","doi-asserted-by":"publisher","DOI":"10.1001\/archinte.163.19.2345"},{"key":"ref142","doi-asserted-by":"publisher","DOI":"10.1016\/j.patcog.2021.108507"},{"key":"ref143","doi-asserted-by":"publisher","DOI":"10.1093\/comjnl\/bxl065"},{"key":"ref144","doi-asserted-by":"publisher","DOI":"10.1007\/b106715"},{"key":"ref145","doi-asserted-by":"publisher","DOI":"10.1613\/jair.3198"},{"key":"ref146","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2016.2532875"},{"key":"ref147","doi-asserted-by":"publisher","DOI":"10.1029\/2020WR029471"},{"key":"ref148","doi-asserted-by":"publisher","DOI":"10.1016\/j.jss.2021.111062"}],"container-title":["IEEE Transactions on Emerging Topics in Computational Intelligence"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx8\/7433297\/11267152\/11039680.pdf?arnumber=11039680","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,11,27]],"date-time":"2025-11-27T18:59:53Z","timestamp":1764269993000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/11039680\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,12]]},"references-count":148,"journal-issue":{"issue":"6"},"URL":"https:\/\/doi.org\/10.1109\/tetci.2025.3577486","relation":{},"ISSN":["2471-285X"],"issn-type":[{"type":"electronic","value":"2471-285X"}],"subject":[],"published":{"date-parts":[[2025,12]]}}}