{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,24]],"date-time":"2026-10-24T03:43:59Z","timestamp":1792813439854,"version":"4.5.0"},"reference-count":47,"publisher":"Elsevier BV","license":[{"start":{"date-parts":[[2022,1,1]],"date-time":"2022-01-01T00:00:00Z","timestamp":1640995200000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.elsevier.com\/tdm\/userlicense\/1.0\/"},{"start":{"date-parts":[[2022,1,1]],"date-time":"2022-01-01T00:00:00Z","timestamp":1640995200000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.elsevier.com\/legal\/tdmrep-license"},{"start":{"date-parts":[[2021,10,19]],"date-time":"2021-10-19T00:00:00Z","timestamp":1634601600000},"content-version":"vor","delay-in-days":0,"URL":"http:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"funder":[{"DOI":"10.13039\/501100005036","name":"Universitetet i Bergen","doi-asserted-by":"publisher","id":[{"id":"10.13039\/501100005036","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["elsevier.com","sciencedirect.com"],"crossmark-restriction":true},"short-container-title":["Neural Networks"],"published-print":{"date-parts":[[2022,1]]},"DOI":"10.1016\/j.neunet.2021.10.014","type":"journal-article","created":{"date-parts":[[2021,10,25]],"date-time":"2021-10-25T11:57:44Z","timestamp":1635163064000},"page":"164-176","update-policy":"https:\/\/doi.org\/10.1016\/elsevier_cm_policy","source":"Crossref","is-referenced-by-count":21,"special_numbering":"C","title":["Epistemic uncertainty quantification in deep learning classification by the Delta method"],"prefix":"10.1016","volume":"145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-4036-2690","authenticated-orcid":false,"given":"Geir K.","family":"Nilsen","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-7087-3059","authenticated-orcid":false,"given":"Antonella Z.","family":"Munthe-Kaas","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Hans J.","family":"Skaug","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-2860-8272","authenticated-orcid":false,"given":"Morten","family":"Brun","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"78","reference":[{"key":"10.1016\/j.neunet.2021.10.014_b1","series-title":"Negative eigenvalues of the hessian in deep neural networks","author":"Alain","year":"2019"},{"issue":"2","key":"10.1016\/j.neunet.2021.10.014_b2","doi-asserted-by":"crossref","first-page":"223","DOI":"10.1137\/16M1080173","article-title":"Optimization methods for large-scale machine learning","volume":"60","author":"Bottou","year":"2018","journal-title":"SIAM Review"},{"key":"10.1016\/j.neunet.2021.10.014_b3","series-title":"Online principal component analysis in high dimension: Which algorithm to choose?","author":"Cardot","year":"2015"},{"issue":"1","key":"10.1016\/j.neunet.2021.10.014_b4","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1214\/aos\/1176344552","article-title":"Bootstrap methods: Another look at the jackknife","volume":"7","author":"Efron","year":"1979","journal-title":"Annals of Statistics"},{"issue":"4","key":"10.1016\/j.neunet.2021.10.014_b5","doi-asserted-by":"crossref","first-page":"299","DOI":"10.1198\/000313006X152207","article-title":"On the so-called \u201cHuber Sandwich Estimator\u201d and \u2018Robust Standard Errors\u201d","volume":"60","author":"Freedman","year":"2006","journal-title":"The American Statistician"},{"key":"10.1016\/j.neunet.2021.10.014_b6","series-title":"Dropout as a approximation: Representing model uncertainty in deep learning","author":"Gal","year":"2016"},{"key":"10.1016\/j.neunet.2021.10.014_b7","series-title":"An investigation into neural net optimization via hessian eigenvalue density","author":"Ghorbani","year":"2019"},{"key":"10.1016\/j.neunet.2021.10.014_b8","series-title":"Deep learning","author":"Goodfellow","year":"2016"},{"key":"10.1016\/j.neunet.2021.10.014_b9","series-title":"Presented at the ICML 2019 workshop on theoretical physics for deep learning","article-title":"The Deep Learning Limit: are negative neural network eigenvalues just noise?","author":"Granziol","year":"2019"},{"key":"10.1016\/j.neunet.2021.10.014_b10","series-title":"Lecture 2: Taylor approximations","author":"Grosse","year":"2020"},{"key":"10.1016\/j.neunet.2021.10.014_b11","doi-asserted-by":"crossref","unstructured":"He, K., Zhang, X., Ren, S., & Sun, J. (2015). Delving deep into rectifiers: Surpassing human-level performance on imagenet classification. In Proceedings of the IEEE international conference on computer vision (pp. 1026\u20131034).","DOI":"10.1109\/ICCV.2015.123"},{"key":"10.1016\/j.neunet.2021.10.014_b12","doi-asserted-by":"crossref","unstructured":"He, K., Zhang, X., Ren, S., & Sun, J. (2016). Deep residual learning for image recognition. In Proceedings of the IEEE conference on computer vision and pattern recognition (pp. 770\u2013778).","DOI":"10.1109\/CVPR.2016.90"},{"issue":"2","key":"10.1016\/j.neunet.2021.10.014_b13","doi-asserted-by":"crossref","first-page":"124","DOI":"10.1080\/00031305.2012.687494","article-title":"Who invented the delta method?","volume":"66","author":"Hoef","year":"2012","journal-title":"The American Statistician"},{"key":"10.1016\/j.neunet.2021.10.014_b14","series-title":"Aleatoric and epistemic uncertainty in machine learning: An introduction to concepts and methods","author":"H\u00fcllermeier","year":"2020"},{"key":"10.1016\/j.neunet.2021.10.014_b15","series-title":"International conference on machine learning","first-page":"448","article-title":"Batch normalization: Accelerating deep network training by reducing internal covariate shift","author":"Ioffe","year":"2015"},{"key":"10.1016\/j.neunet.2021.10.014_b16","series-title":"Pathological spectra of the Fisher information metric and its variants in deep neural networks","author":"Karakida","year":"2019"},{"key":"10.1016\/j.neunet.2021.10.014_b17","series-title":"What uncertainties do we need in Bayesian deep learning for computer vision?","author":"Kendall","year":"2017"},{"issue":"9","key":"10.1016\/j.neunet.2021.10.014_b18","doi-asserted-by":"crossref","DOI":"10.1109\/TNN.2011.2162110","article-title":"A comprehensive review of neural network-based prediction intervals and new advances","volume":"22","author":"Khosravi","year":"2011","journal-title":"IEEE Transactions on Neural Networks"},{"key":"10.1016\/j.neunet.2021.10.014_b19","unstructured":"Kingma, D. P., & Ba, J. L. (2014). Adam: A method for stochastic optimization. In Proc. 3rd int. conf. learn. representations."},{"key":"10.1016\/j.neunet.2021.10.014_b20","series-title":"Advances in neural information processing systems (NIPS 1992)","article-title":"Automatic learning rate maximization by on-line estimation of the Hessian\u2019s eigenvectors","author":"LeCun","year":"1993"},{"key":"10.1016\/j.neunet.2021.10.014_b21","series-title":"Theory of point estimation","first-page":"125","author":"Lehmann","year":"1998"},{"issue":"8","key":"10.1016\/j.neunet.2021.10.014_b22","article-title":"Sequential karhunen\u2013loeve basis extraction and its application to images","volume":"9","author":"Levy","year":"2000","journal-title":"IEEE Transactions on Image Processing"},{"issue":"1","key":"10.1016\/j.neunet.2021.10.014_b23","doi-asserted-by":"crossref","first-page":"34","DOI":"10.1137\/130934283","article-title":"Approximating spectral densities of large matrices","volume":"58","author":"Lin","year":"2016","journal-title":"SIAM Review"},{"key":"10.1016\/j.neunet.2021.10.014_b24","doi-asserted-by":"crossref","first-page":"60","DOI":"10.1016\/j.media.2017.07.005","article-title":"A survey on deep learning in medical image analysis","volume":"42","author":"Litjens","year":"2017","journal-title":"Medical Image Analysis"},{"key":"10.1016\/j.neunet.2021.10.014_b25","series-title":"A general framework for uncertainty estimation in deep learning","author":"Loquercio","year":"2020"},{"issue":"3","key":"10.1016\/j.neunet.2021.10.014_b26","doi-asserted-by":"crossref","first-page":"448","DOI":"10.1162\/neco.1992.4.3.448","article-title":"A practical Bayesian framework for backpropagation networks","volume":"4","author":"MacKay","year":"1992","journal-title":"Neural Computation"},{"key":"10.1016\/j.neunet.2021.10.014_b27","series-title":"New insights and perspectives on the natural gradient method","author":"Martens","year":"2020"},{"key":"10.1016\/j.neunet.2021.10.014_b28","series-title":"Deep learning is singular, and that\u2019s good","author":"Murfet","year":"2020"},{"key":"10.1016\/j.neunet.2021.10.014_b29","series-title":"Deterministic implementations for reproducibility in deep reinforcement learning","author":"Nagarajan","year":"2019"},{"key":"10.1016\/j.neunet.2021.10.014_b30","series-title":"Handbook of econometrics","author":"Newey","year":"1994"},{"key":"10.1016\/j.neunet.2021.10.014_b31","series-title":"Efficient computation of Hessian matrices in TensorFlow","author":"Nilsen","year":"2019"},{"key":"10.1016\/j.neunet.2021.10.014_b32","series-title":"A comparison of the delta method and the bootstrap in deep learning classification","author":"Nilsen","year":"2021"},{"key":"10.1016\/j.neunet.2021.10.014_b33","series-title":"NIPS workshop on Bayesian deep learning","article-title":"Risk versus uncertainty in deep learning: Bayes, bootstrap and the dangers of dropout","author":"Osband","year":"2016"},{"key":"10.1016\/j.neunet.2021.10.014_b34","series-title":"Conference on neural information processing systems (NIPS)","article-title":"Deep exploration via bootstrapped DQN","author":"Osband","year":"2016"},{"issue":"1","key":"10.1016\/j.neunet.2021.10.014_b35","doi-asserted-by":"crossref","first-page":"147","DOI":"10.1162\/neco.1994.6.1.147","article-title":"Fast exact multiplication by the hessian","volume":"6","author":"Pearlmutter","year":"1994","journal-title":"Neural Computation"},{"key":"10.1016\/j.neunet.2021.10.014_b36","unstructured":"pyDeepDelta: A TensorFlow Module Implementing the Delta Method in Deep Learning Classification, https:\/\/github.com\/gknilsen\/pydeepdelta.git."},{"key":"10.1016\/j.neunet.2021.10.014_b37","series-title":"Eigenvalues of the hessian in deep learning: Singularity and beyond","author":"Sagun","year":"2017"},{"key":"10.1016\/j.neunet.2021.10.014_b38","series-title":"Empirical analysis of the hessian of over-parametrized neural networks","author":"Sagun","year":"2018"},{"key":"10.1016\/j.neunet.2021.10.014_b39","series-title":"Can you trust this prediction? Auditing pointwise reliability after learning","author":"Schulam","year":"2019"},{"key":"10.1016\/j.neunet.2021.10.014_b40","unstructured":"scikit-learn, https:\/\/scikit-learn.org\/."},{"key":"10.1016\/j.neunet.2021.10.014_b41","unstructured":"SciPy, http:\/\/www.scipy.org."},{"key":"10.1016\/j.neunet.2021.10.014_b42","series-title":"Learning from noisy labels with deep neural networks: A survey","author":"Song","year":"2020"},{"key":"10.1016\/j.neunet.2021.10.014_b43","series-title":"Numerical linear algebra","first-page":"243","author":"Trefethen","year":"1997"},{"key":"10.1016\/j.neunet.2021.10.014_b44","series-title":"Proceedings of the 2007 IEEE symposium on foundations of computational intelligence","article-title":"Almost all learning machines are singular","author":"Watanabe","year":"2007"},{"issue":"4","key":"10.1016\/j.neunet.2021.10.014_b45","doi-asserted-by":"crossref","first-page":"1445","DOI":"10.1109\/TPAMI.2020.2975798","article-title":"Deep multi-view enhancement hashing for image retrieval","volume":"43","author":"Yan","year":"2020","journal-title":"IEEE Transactions on Pattern Analysis and Machine Intelligence"},{"key":"10.1016\/j.neunet.2021.10.014_b46","series-title":"2020 IEEE international conference on big data (Big data)","first-page":"581","article-title":"Pyhessian: Neural networks through the lens of the hessian","author":"Yao","year":"2020"},{"key":"10.1016\/j.neunet.2021.10.014_b47","series-title":"2017 IEEE international conference on data mining workshops (ICDMW)","article-title":"Deep and confident prediction for time series at uber","author":"Zhu","year":"2017"}],"container-title":["Neural Networks"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/api.elsevier.com\/content\/article\/PII:S0893608021004056?httpAccept=text\/xml","content-type":"text\/xml","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/api.elsevier.com\/content\/article\/PII:S0893608021004056?httpAccept=text\/plain","content-type":"text\/plain","content-version":"vor","intended-application":"text-mining"}],"deposited":{"date-parts":[[2025,10,12]],"date-time":"2025-10-12T01:49:03Z","timestamp":1760233743000},"score":1,"resource":{"primary":{"URL":"https:\/\/linkinghub.elsevier.com\/retrieve\/pii\/S0893608021004056"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2022,1]]},"references-count":47,"alternative-id":["S0893608021004056"],"URL":"https:\/\/doi.org\/10.1016\/j.neunet.2021.10.014","relation":{},"ISSN":["0893-6080"],"issn-type":[{"value":"0893-6080","type":"print"}],"subject":[],"published":{"date-parts":[[2022,1]]},"assertion":[{"value":"Elsevier","name":"publisher","label":"This article is maintained by"},{"value":"Epistemic uncertainty quantification in deep learning classification by the Delta method","name":"articletitle","label":"Article Title"},{"value":"Neural Networks","name":"journaltitle","label":"Journal Title"},{"value":"https:\/\/doi.org\/10.1016\/j.neunet.2021.10.014","name":"articlelink","label":"CrossRef DOI link to publisher maintained version"},{"value":"article","name":"content_type","label":"Content Type"},{"value":"\u00a9 2021 The Author(s). Published by Elsevier Ltd.","name":"copyright","label":"Copyright"}]}}