{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,1]],"date-time":"2026-04-01T18:29:57Z","timestamp":1775068197153,"version":"3.50.1"},"reference-count":91,"publisher":"Institute of Electrical and Electronics Engineers (IEEE)","license":[{"start":{"date-parts":[[2023,1,1]],"date-time":"2023-01-01T00:00:00Z","timestamp":1672531200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/ieeexplore.ieee.org\/Xplorehelp\/downloads\/license-information\/IEEE.html"},{"start":{"date-parts":[[2023,1,1]],"date-time":"2023-01-01T00:00:00Z","timestamp":1672531200000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2023,1,1]],"date-time":"2023-01-01T00:00:00Z","timestamp":1672531200000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["IEEE Trans. Pattern Anal. Mach. Intell."],"published-print":{"date-parts":[[2023]]},"DOI":"10.1109\/tpami.2023.3291676","type":"journal-article","created":{"date-parts":[[2023,7,3]],"date-time":"2023-07-03T18:09:15Z","timestamp":1688407755000},"page":"1-18","source":"Crossref","is-referenced-by-count":4,"title":["Towards More Reliable Confidence Estimation"],"prefix":"10.1109","author":[{"ORCID":"https:\/\/orcid.org\/0000-0001-5054-3394","authenticated-orcid":false,"given":"Haoxuan","family":"Qu","sequence":"first","affiliation":[{"name":"Singapore University of Technology and Design (SUTD), Singapore"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-6082-6002","authenticated-orcid":false,"given":"Lin Geng","family":"Foo","sequence":"additional","affiliation":[{"name":"Singapore University of Technology and Design (SUTD), Singapore"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Yanchao","family":"Li","sequence":"additional","affiliation":[{"name":"Singapore University of Technology and Design (SUTD), Singapore"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Jun","family":"Liu","sequence":"additional","affiliation":[{"name":"Singapore University of Technology and Design (SUTD), Singapore"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"263","reference":[{"key":"ref1","first-page":"1097","article-title":"Imagenet classification with deep convolutional neural networks","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","author":"Krizhevsky"},{"key":"ref2","doi-asserted-by":"publisher","DOI":"10.1109\/TPAMI.2016.2644615"},{"key":"ref3","article-title":"From big to small: Multi-scale local planar guidance for monocular depth estimation","author":"Lee","year":"2019"},{"key":"ref4","article-title":"Concrete problems in ai safety","author":"Amodei","year":"2016"},{"key":"ref5","doi-asserted-by":"publisher","DOI":"10.1561\/9781680836899"},{"key":"ref6","doi-asserted-by":"publisher","DOI":"10.1038\/s42256-019-0055-y"},{"key":"ref7","doi-asserted-by":"publisher","DOI":"10.1109\/IJCNN.2009.5178592"},{"key":"ref8","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-99229-7_39"},{"key":"ref9","doi-asserted-by":"publisher","DOI":"10.1016\/j.asoc.2013.11.009"},{"key":"ref10","first-page":"5541","article-title":"To trust or not to trust a classifier","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","author":"Jiang"},{"key":"ref11","article-title":"Addressing failure prediction by learning model confidence","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","author":"Corbi\u00e8re"},{"key":"ref12","first-page":"21533","article-title":"Learning to predict trustworthiness with steep slope loss","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","author":"Luo"},{"key":"ref13","article-title":"Identifying incorrect classifications with balanced uncertainty","author":"Li","year":"2021"},{"key":"ref14","article-title":"SLURP: Side learning uncertainty for regression problems","volume-title":"Proc. 32nd Brit. Mach. Vis. Conf.","author":"Yu"},{"key":"ref15","doi-asserted-by":"publisher","DOI":"10.1109\/5.726791"},{"key":"ref16","article-title":"Very deep convolutional networks for large-scale image recognition","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Simonyan"},{"key":"ref17","doi-asserted-by":"publisher","DOI":"10.1109\/ACPR.2015.7486599"},{"key":"ref18","article-title":"Learning multiple layers of features from tiny images","author":"Krizhevsky","year":"2009"},{"key":"ref19","first-page":"15 288","article-title":"Calibrating deep neural networks using focal loss","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","author":"Mukhoti"},{"key":"ref20","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR46437.2021.00999"},{"key":"ref21","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v34i07.6846"},{"key":"ref22","first-page":"1126","article-title":"Model-agnostic meta-learning for fast adaptation of deep networks","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Finn"},{"key":"ref23","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v32i1.11596"},{"key":"ref24","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR42600.2020.00620"},{"key":"ref25","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR46437.2021.00216"},{"key":"ref26","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR46437.2021.00875"},{"key":"ref27","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-19812-0_22"},{"key":"ref28","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-19772-7_23"},{"key":"ref29","article-title":"A baseline for detecting misclassified and out-of-distribution examples in neural networks","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Hendrycks"},{"key":"ref30","doi-asserted-by":"publisher","DOI":"10.5555\/3045390.3045502"},{"key":"ref31","first-page":"7034","article-title":"Confidence-aware learning for deep neural networks","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Moon"},{"key":"ref32","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v36i7.20773"},{"key":"ref33","doi-asserted-by":"publisher","DOI":"10.1109\/ICASSP39728.2021.9414153"},{"key":"ref34","first-page":"14980","article-title":"Detecting errors and estimating accuracy on unlabeled data with self-training ensembles","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","author":"Chen"},{"key":"ref35","first-page":"1321","article-title":"On calibration of modern neural networks","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Guo"},{"key":"ref36","first-page":"18 237","article-title":"Improving model calibration with accuracy versus uncertainty optimization","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","author":"Krishnan"},{"key":"ref37","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR42600.2020.00276"},{"key":"ref38","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2019.00018"},{"key":"ref39","first-page":"4163","article-title":"Learning loss for test-time augmentation","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","author":"Kim"},{"key":"ref40","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV48922.2021.01541"},{"key":"ref41","doi-asserted-by":"publisher","DOI":"10.1109\/IROS45743.2020.9340963"},{"key":"ref42","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-87199-4_57"},{"key":"ref43","doi-asserted-by":"publisher","DOI":"10.1109\/IVS.2018.8500495"},{"key":"ref44","article-title":"Quantifying point-prediction uncertainty in neural networks via residual estimation with an I\/O kernel","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Qiu"},{"key":"ref45","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v36i9.21243"},{"key":"ref46","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-20044-1_35"},{"key":"ref47","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV48922.2021.00817"},{"key":"ref48","article-title":"AlterSGD: Finding flat minima for continual learning by alternative training","author":"Huang","year":"2021"},{"key":"ref49","first-page":"4604","article-title":"Do we need zero training loss after achieving zero training error?","volume-title":"Proc. 37th Proc. Int. Conf. Mach. Learn.","author":"Ishida"},{"key":"ref50","first-page":"10","article-title":"Sharp-MAML: Sharpness-aware model-agnostic meta learning","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Abbas"},{"key":"ref51","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2022.acl-long.386"},{"key":"ref52","article-title":"On large-batch training for deep learning: Generalization gap and sharp minima","author":"Keskar","year":"2017","journal-title":"Proc. Int. Conf. Learn. Representations"},{"key":"ref53","article-title":"Computing nonvacuous generalization bounds for deep (stochastic) neural networks with many more parameters than training data","author":"Dziugaite","year":"2017","journal-title":"Proc. 33rd Ann. Conf. Uncertainty Artif. Intell. (UAI)"},{"key":"ref54","article-title":"Fantastic generalization measures and where to find them","author":"Jiang","year":"2020","journal-title":"Proc.Int. Conf. Learn. Representations"},{"key":"ref55","first-page":"7344","article-title":"Exactly computing the local Lipschitz constant of ReLU networks","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","author":"Jordan"},{"key":"ref56","article-title":"Generalized out-of-distribution detection: A survey","author":"Yang","year":"2021"},{"key":"ref57","article-title":"Enhancing the reliability of out-of-distribution image detection in neural networks","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Liang"},{"key":"ref58","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR42600.2020.01096"},{"key":"ref59","first-page":"7167","article-title":"A simple unified framework for detecting out-of-distribution samples and adversarial attacks","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","author":"Lee"},{"key":"ref60","first-page":"8491","article-title":"Detecting out-of-distribution examples with gram matrices","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Sastry"},{"key":"ref61","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2017.324"},{"key":"ref62","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-19812-0_23"},{"key":"ref63","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2017.167"},{"key":"ref64","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR46437.2021.01220"},{"key":"ref65","doi-asserted-by":"publisher","DOI":"10.1177\/0278364913491297"},{"key":"ref66","doi-asserted-by":"publisher","DOI":"10.1109\/3DV.2017.00012"},{"key":"ref67","first-page":"2366","article-title":"Depth map prediction from a single image using a multi-scale deep network","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","author":"Eigen"},{"key":"ref68","doi-asserted-by":"publisher","DOI":"10.5555\/3295222.3295309"},{"key":"ref69","first-page":"6402","article-title":"Simple and scalable predictive uncertainty estimation using deep ensembles","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","author":"Lakshminarayanan"},{"key":"ref70","doi-asserted-by":"publisher","DOI":"10.1117\/12.2228523"},{"issue":"1","key":"ref71","first-page":"1929","article-title":"Dropout: A simple way to prevent neural networks from overfitting","volume":"15","author":"Srivastava","year":"2014","journal-title":"J. Mach. Learn. Res."},{"key":"ref72","article-title":"mixup: Beyond empirical risk minimization","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Zhang"},{"key":"ref73","doi-asserted-by":"publisher","DOI":"10.48550\/ARXIV.1604.01685"},{"key":"ref74","first-page":"26 982","article-title":"Penalizing gradient norm for efficiently improving generalization in deep learning","volume-title":"Proc. 39th Proc. Int. Conf. Mach. Learn.","author":"Zhao"},{"key":"ref75","article-title":"Can we use gradient norm as a measure of generalization error for model selection in practice?","author":"An","year":"2021"},{"key":"ref76","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-01261-8_42"},{"key":"ref77","first-page":"20 827","article-title":"Out-of-distribution detection with deep nearest neighbors","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Sun"},{"key":"ref78","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2019.00821"},{"key":"ref79","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2009.5206848"},{"key":"ref80","article-title":"MNIST-C: A robustness benchmark for computer vision","author":"Mu","year":"2019"},{"key":"ref81","article-title":"Benchmarking neural network robustness to common corruptions and perturbations","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Hendrycks"},{"key":"ref82","article-title":"Reading digits in natural images with unsupervised feature learning","author":"Netzer","year":"2011"},{"key":"ref83","first-page":"1396","article-title":"Failing loudly: An empirical study of methods for detecting dataset shift","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","author":"Rabanser"},{"key":"ref84","article-title":"LSUN: Construction of a large-scale image dataset using deep learning with humans in the loop","author":"Yu","year":"2015"},{"issue":"7","key":"ref85","article-title":"Tiny ImageNet visual recognition challenge","volume":"7","author":"Le","year":"2015","journal-title":"CS 231N"},{"key":"ref86","article-title":"Benchmarking robustness in object detection: Autonomous driving when winter is coming","author":"Michaelis","year":"2019"},{"key":"ref87","doi-asserted-by":"publisher","DOI":"10.1016\/j.patrec.2008.04.005"},{"key":"ref88","first-page":"8759","article-title":"Scaling out-of-distribution detection for real-world settings","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Hendrycks"},{"key":"ref89","article-title":"Sharpness-aware minimization for efficiently improving generalization","author":"Foret","year":"2021","journal-title":"Proc. Conf. Learn. Representations"},{"key":"ref90","first-page":"2232","article-title":"An investigation into neural net optimization via hessian eigenvalue density","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Ghorbani"},{"key":"ref91","article-title":"In-between\u2019uncertainty in Bayesian neural networks","author":"Foong","year":"2019"}],"container-title":["IEEE Transactions on Pattern Analysis and Machine Intelligence"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx7\/34\/4359286\/10172026.pdf?arnumber=10172026","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,3,1]],"date-time":"2024-03-01T13:06:36Z","timestamp":1709298396000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/10172026\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023]]},"references-count":91,"URL":"https:\/\/doi.org\/10.1109\/tpami.2023.3291676","relation":{},"ISSN":["0162-8828","2160-9292","1939-3539"],"issn-type":[{"value":"0162-8828","type":"print"},{"value":"2160-9292","type":"electronic"},{"value":"1939-3539","type":"electronic"}],"subject":[],"published":{"date-parts":[[2023]]}}}