{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,31]],"date-time":"2026-07-31T22:40:28Z","timestamp":1785537628925,"version":"3.56.0"},"reference-count":88,"publisher":"Institute of Electrical and Electronics Engineers (IEEE)","issue":"4","license":[{"start":{"date-parts":[[2024,4,1]],"date-time":"2024-04-01T00:00:00Z","timestamp":1711929600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/ieeexplore.ieee.org\/Xplorehelp\/downloads\/license-information\/IEEE.html"},{"start":{"date-parts":[[2024,4,1]],"date-time":"2024-04-01T00:00:00Z","timestamp":1711929600000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2024,4,1]],"date-time":"2024-04-01T00:00:00Z","timestamp":1711929600000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"}],"funder":[{"name":"Wu Tsai Neuroscience: Translate program"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["IEEE Trans. Artif. Intell."],"published-print":{"date-parts":[[2024,4]]},"DOI":"10.1109\/tai.2023.3272876","type":"journal-article","created":{"date-parts":[[2023,5,3]],"date-time":"2023-05-03T14:55:59Z","timestamp":1683125759000},"page":"1473-1485","source":"Crossref","is-referenced-by-count":12,"title":["An Explainable and Actionable Mistrust Scoring Framework for Model Monitoring"],"prefix":"10.1109","volume":"5","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-3624-9169","authenticated-orcid":false,"given":"Nandita","family":"Bhaskhar","sequence":"first","affiliation":[{"name":"Department of Electrical Engineering, Stanford University, Stanford, CA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-5057-4369","authenticated-orcid":false,"given":"Daniel L.","family":"Rubin","sequence":"additional","affiliation":[{"name":"Departments of Biomedical Data Science, Radiology, and Medicine (Biomedical Informatics), Stanford University, Stanford, CA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Christopher","family":"Lee-Messer","sequence":"additional","affiliation":[{"name":"Department of Neurology, Stanford University, Stanford, CA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"263","reference":[{"issue":"7639","key":"ref1","first-page":"115","article-title":"Dermatologist-level classification of skin cancer with deep neural networks","volume-title":"Nature","volume":"542","author":"Esteva","year":"2017"},{"key":"ref2","doi-asserted-by":"publisher","DOI":"10.1148\/radiol.2019182716"},{"key":"ref3","first-page":"1097","article-title":"ImageNet classification with deep convolutional neural networks","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","author":"Krizhevsky","year":"2012"},{"key":"ref4","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2015.7298640"},{"key":"ref5","article-title":"Explaining and harnessing adversarial examples","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Goodfellow","year":"2015"},{"key":"ref6","first-page":"1321","article-title":"On calibration of modern neural networks","volume-title":"Proc. 34th Int. Conf. Mach. Learn.","volume":"70","author":"Guo","year":"2017"},{"key":"ref7","article-title":"Concrete problems in AI safety","author":"Amodei","year":"2016"},{"key":"ref8","first-page":"21464","article-title":"Energy-based out-of-distribution detection","volume-title":"Proc. 34th Int. Conf. Neural Inf. Process. Syst.","author":"Liu","year":"2020"},{"key":"ref9","doi-asserted-by":"publisher","DOI":"10.1145\/3233231"},{"key":"ref10","doi-asserted-by":"publisher","DOI":"10.1145\/3479864"},{"key":"ref11","first-page":"4391","article-title":"Label-free explainability for unsupervised models","volume-title":"Proc. 39th Int. Conf. Mach. Learn.","author":"Crabb","year":"2022"},{"key":"ref12","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/N16-3020"},{"key":"ref13","first-page":"4768","article-title":"A unified approach to interpreting model predictions","volume-title":"Proc. 31st Int. Conf. Neural Inf. Process. Syst.","author":"Lundberg","year":"2017"},{"key":"ref14","first-page":"1613","article-title":"Weight uncertainty in neural networks","volume-title":"Proc. 32nd Int. Conf. Mach. Learn.","volume":"37","author":"Blundell","year":"2015"},{"key":"ref15","article-title":"A baseline for detecting misclassified and out-of-distribution examples in neural networks","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Hendrycks","year":"2017"},{"key":"ref16","first-page":"7047","article-title":"Predictive uncertainty estimation via prior networks","volume-title":"Proc. 32nd Int. Conf. Neural Inf. Process. Syst.","author":"Malinin","year":"2018"},{"key":"ref17","first-page":"6405","article-title":"Simple and scalable predictive uncertainty estimation using deep ensembles","volume-title":"Proc. 31st Int. Conf. Neural Inf. Process. Syst.","author":"Lakshminarayanan","year":"2017"},{"key":"ref18","doi-asserted-by":"publisher","DOI":"10.5555\/3045390.3045502"},{"key":"ref19","first-page":"21281","article-title":"Uncertainty quantification and explainable artificial intelligence","volume-title":"Proc. EGU Gen. Assem. Conf. Abstr., SAO\/NASA Astrophys. Data Syst.","author":"de Castro","year":"2020"},{"key":"ref20","first-page":"3145","article-title":"Learning important features through propagating activation differences","volume-title":"Proc. 34th Int. Conf. Mach. Learn.","volume":"70","author":"Shrikumar","year":"2017"},{"key":"ref21","article-title":"Deep k-nearest neighbors: Towards confident, interpretable and robust deep learning","author":"Papernot","year":"2018"},{"key":"ref22","doi-asserted-by":"publisher","DOI":"10.1145\/335191.335388"},{"key":"ref23","first-page":"582","article-title":"Support vector method for novelty detection","volume-title":"Proc. 12th Int. Conf. Neural Inf. Process. Syst.","author":"Schlkopf","year":"1999"},{"key":"ref24","doi-asserted-by":"publisher","DOI":"10.1109\/ICDM.2008.17"},{"key":"ref25","first-page":"5546","article-title":"To trust or not to trust a classifier","volume-title":"Proc. 32nd Int. Conf. Neural Inf. Process. Syst.","author":"Jiang","year":"2018"},{"key":"ref26","first-page":"1396","article-title":"Failing loudly: An empirical study of methods for detecting dataset shift","volume-title":"Proc. 33rd Int. Conf. Neural Inf. Process. Syst.","author":"Rabanser","year":"2019"},{"key":"ref27","doi-asserted-by":"publisher","DOI":"10.2307\/2987588.JSTOR:2987588"},{"key":"ref28","doi-asserted-by":"publisher","DOI":"10.2307\/2287720"},{"key":"ref29","article-title":"A variational Dirichlet framework for out-of-distribution detection","author":"Chen","year":"2019"},{"key":"ref30","doi-asserted-by":"publisher","DOI":"10.5555\/2986459.2986721"},{"key":"ref31","doi-asserted-by":"crossref","DOI":"10.1007\/978-1-4612-0745-0","volume-title":"Bayesian Learning for Neural Networks","author":"Neal","year":"1996"},{"key":"ref32","first-page":"681","article-title":"Bayesian learning via stochastic gradient Langevin dynamics","volume-title":"Proc. 28th Int. Conf. Int. Conf. Mach. Learn.","author":"Welling","year":"2011"},{"key":"ref33","first-page":"4907","article-title":"Bayesian uncertainty estimation for batch normalized deep networks","volume-title":"Proc. 35th Int. Conf. Mach. Learn.","volume":"80","author":"Teye","year":"2018"},{"key":"ref34","article-title":"Deep autoencoding Gaussian mixture model for unsupervised anomaly detection","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Zong","year":"2018"},{"key":"ref35","first-page":"6823","article-title":"Generative probabilistic novelty detection with adversarial autoencoders","volume-title":"Proc. 32nd Int. Conf. Neural Inf. Process. Syst.","author":"Pidhorskyi","year":"2018"},{"key":"ref36","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-59050-9_12"},{"key":"ref37","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-10925-7_1"},{"key":"ref38","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2019.00301"},{"key":"ref39","article-title":"Enhancing the reliability of out-of-distribution image detection in neural networks","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Liang","year":"2018"},{"key":"ref40","first-page":"7167","article-title":"A simple unified framework for detecting out-of-distribution samples and adversarial attacks","volume-title":"Proc. 32nd Int. Conf. Neural Inf. Process. Syst.","author":"Lee","year":"2018"},{"key":"ref41","first-page":"11839","article-title":"CSI: Novelty detection via contrastive learning on distributionally shifted instances","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","author":"Tack","year":"2020"},{"key":"ref42","article-title":"Contrastive training for improved out-of-distribution detection","author":"Winkens","year":"2020"},{"key":"ref43","article-title":"SSD: A unified framework for self-supervised outlier detection","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Sehwag","year":"2021"},{"key":"ref44","article-title":"Deep anomaly detection with outlier exposure","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Hendrycks","year":"2019"},{"key":"ref45","article-title":"Deep semi-supervised anomaly detection","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Ruff","year":"2020"},{"key":"ref46","article-title":"Learning confidence for out-of-distribution detection in neural networks","author":"DeVries","year":"2018"},{"key":"ref47","first-page":"15663","article-title":"Using self-supervised learning can improve model robustness and uncertainty","volume-title":"Proc. 33rd Int. Conf. Neural Inf. Process. Syst.","author":"Hendrycks","year":"2019"},{"key":"ref48","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v34i04.5966"},{"key":"ref49","first-page":"7386","article-title":"Out-of-distribution detection using multiple semantic label representations","volume-title":"Proc. 32nd Int. Conf. Neural Inf. Process. Syst.","author":"Shalev","year":"2018"},{"key":"ref50","article-title":"Input complexity and out-of-distribution detection with likelihood-based generative models","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Serr","year":"2020"},{"key":"ref51","first-page":"20685","article-title":"Likelihood regret: An out-of-distribution detection score for variational auto-encoder","volume-title":"Proc. 34th Conf. Neural Inf. Process. Syst.","author":"Xiao","year":"2020"},{"key":"ref52","article-title":"WAIC, but why? Generative ensembles for robust anomaly detection","author":"Choi","year":"2019"},{"key":"ref53","first-page":"3608","article-title":"Implicit generation and modeling with energy based models","volume-title":"Proc. 33rd Int. Conf. Neural Inf. Process. Syst.","author":"Du","year":"2019"},{"key":"ref54","article-title":"Your classifier is secretly an energy based model and you should treat it like one","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Grathwohl","year":"2020"},{"key":"ref55","first-page":"14707","article-title":"Likelihood ratios for out-of-distribution detection","volume-title":"Proc. 33rd Int. Conf. Neural Inf. Process. Syst.","author":"Ren","year":"2019"},{"key":"ref56","article-title":"Do deep generative models know what they dont know","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Nalisnick","year":"2019"},{"key":"ref57","doi-asserted-by":"publisher","DOI":"10.1007\/s10115-016-0987-z"},{"key":"ref58","doi-asserted-by":"publisher","DOI":"10.1016\/B978-012088469-8.50019-X"},{"key":"ref59","first-page":"584","article-title":"The unscented particle filter","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","author":"van der Merwe","year":"2001"},{"key":"ref60","doi-asserted-by":"publisher","DOI":"10.1115\/1.3662552"},{"key":"ref61","doi-asserted-by":"publisher","DOI":"10.5555\/2888116.2888212"},{"key":"ref62","doi-asserted-by":"publisher","DOI":"10.1016\/j.patcog.2011.06.019"},{"key":"ref63","doi-asserted-by":"publisher","DOI":"10.1145\/2523813"},{"key":"ref64","doi-asserted-by":"publisher","DOI":"10.1214\/aoms\/1177730491.MR0022058.Zbl0041.26103"},{"key":"ref65","article-title":"Speech commands: A dataset for limited-vocabulary speech recognition","author":"Warden","year":"2018"},{"key":"ref66","article-title":"Free spoken digits dataset","author":"Jackson","year":"2017"},{"key":"ref67","doi-asserted-by":"publisher","DOI":"10.1109\/icassp.2017.7952190"},{"key":"ref68","doi-asserted-by":"publisher","DOI":"10.1038\/s41746-020-0264-0"},{"key":"ref69","doi-asserted-by":"publisher","DOI":"10.3389\/fninf.2018.00083"},{"key":"ref70","doi-asserted-by":"publisher","DOI":"10.3389\/fnins.2016.00196"},{"key":"ref71","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2016.90"},{"key":"ref72","article-title":"Learning multiple layers of features from tiny images","author":"Krizhevsky","year":"2009"},{"key":"ref73","doi-asserted-by":"publisher","DOI":"10.2118\/18761-MS"},{"key":"ref74","doi-asserted-by":"publisher","DOI":"10.1109\/5.726791"},{"key":"ref75","article-title":"Fashion-MNIST: A novel image dataset for benchmarking machine learning algorithms","author":"Xiao","year":"2017"},{"key":"ref76","doi-asserted-by":"publisher","DOI":"10.1109\/ijcnn.2017.7966217"},{"key":"ref77","article-title":"Deep learning for classical Japanese literature","author":"Tarin","year":"2018"},{"key":"ref78","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v34i04.5712"},{"key":"ref79","first-page":"281","article-title":"Some methods for classification and analysis of multivariate observations","volume-title":"Proc. 5th Berkeley Symp. Math. Statist. Probability","author":"MacQueen","year":"1967"},{"key":"ref80","article-title":"Scattering transforms in Python","author":"Andreux","year":"2018"},{"key":"ref81","article-title":"Deep inside convolutional networks: Visualising image classification models and saliency maps","author":"Simonyan","year":"2013"},{"key":"ref82","first-page":"3319","article-title":"Axiomatic attribution for deep networks","volume-title":"Proc. 34th Int. Conf. Mach. Learn.","volume":"70","author":"Sundararajan","year":"2017"},{"key":"ref83","first-page":"4768","article-title":"A unified approach to interpreting model predictions","volume-title":"Proc. 31st Int. Conf. Neural Inf. Process.","author":"Lundberg","year":"2017"},{"key":"ref84","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-10590-1_53"},{"key":"ref85","article-title":"Investigating the influence of noise and distractors on the interpretation of neural networks","author":"Kindermans","year":"2016"},{"key":"ref86","doi-asserted-by":"publisher","DOI":"10.21105\/joss.00861"},{"key":"ref87","first-page":"566","article-title":"HMM-based audio keyword generation","volume-title":"Proc. Adv. Multimedia Inf. Process.","author":"Xu","year":"2005"},{"key":"ref88","article-title":"Captum: A unified and generic model interpretability library for PyTorch","author":"Kokhlikyan","year":"2020"}],"container-title":["IEEE Transactions on Artificial Intelligence"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx7\/9078688\/10495741\/10114955.pdf?arnumber=10114955","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,8,23]],"date-time":"2025-08-23T01:09:14Z","timestamp":1755911354000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/10114955\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,4]]},"references-count":88,"journal-issue":{"issue":"4"},"URL":"https:\/\/doi.org\/10.1109\/tai.2023.3272876","relation":{},"ISSN":["2691-4581"],"issn-type":[{"value":"2691-4581","type":"electronic"}],"subject":[],"published":{"date-parts":[[2024,4]]}}}