{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,14]],"date-time":"2026-10-14T16:10:30Z","timestamp":1791994230328,"version":"4.4.0"},"reference-count":69,"publisher":"Oxford University Press (OUP)","issue":"4","license":[{"start":{"date-parts":[[2025,1,29]],"date-time":"2025-01-29T00:00:00Z","timestamp":1738108800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"funder":[{"DOI":"10.13039\/501100001459","name":"Ministry of Education, Singapore","doi-asserted-by":"publisher","award":["MOE-T2EP20222-0002"],"award-info":[{"award-number":["MOE-T2EP20222-0002"]}],"id":[{"id":"10.13039\/501100001459","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2025,9,15]]},"abstract":"<jats:title>Abstract<\/jats:title>\n                  <jats:p>Natural gradients can improve convergence in stochastic variational inference significantly but inverting the Fisher information matrix is daunting in high dimensions. Moreover, in Gaussian variational approximation, natural gradient updates of the precision matrix do not ensure positive definiteness. To tackle this issue, we derive analytic natural gradient updates of the Cholesky factor of the covariance or precision matrix and consider sparsity constraints representing different posterior correlation structures. Stochastic normalized natural gradient ascent with momentum is proposed for implementation in generalized linear mixed models and deep neural networks.<\/jats:p>","DOI":"10.1093\/jrsssb\/qkaf001","type":"journal-article","created":{"date-parts":[[2025,1,10]],"date-time":"2025-01-10T07:17:08Z","timestamp":1736493428000},"page":"930-956","source":"Crossref","is-referenced-by-count":5,"title":["Analytic natural gradient updates for Cholesky factor in Gaussian variational approximation"],"prefix":"10.1093","volume":"87","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-5002-847X","authenticated-orcid":false,"given":"Linda S L","family":"Tan","sequence":"first","affiliation":[{"name":"Department of Statistics and Data Science, National University of Singapore , Singapore 117546 ,","place":["Singapore"]}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"286","published-online":{"date-parts":[[2025,1,29]]},"reference":[{"key":"2025120808264658600_qkaf001-B1","volume-title":"An introduction to categorical data analysis","author":"Agresti","year":"2018","edition":"3rd ed"},{"issue":"2","key":"2025120808264658600_qkaf001-B2","doi-asserted-by":"publisher","first-page":"251","DOI":"10.1162\/089976698300017746","article-title":"Natural gradient works efficiently in learning","volume":"10","author":"Amari","year":"1998","journal-title":"Neural Computation"},{"key":"2025120808264658600_qkaf001-B3","doi-asserted-by":"crossref","DOI":"10.1007\/978-4-431-55978-8","volume-title":"Information geometry and its applications","author":"Amari","year":"2016"},{"key":"2025120808264658600_qkaf001-B4","first-page":"21","author":"Attias","year":"1999"},{"key":"2025120808264658600_qkaf001-B5","first-page":"404","author":"Balles","year":"2018"},{"issue":"1","key":"2025120808264658600_qkaf001-B6","doi-asserted-by":"publisher","first-page":"65","DOI":"10.1137\/141000671","article-title":"Julia: A fresh approach to numerical computing","volume":"59","author":"Bezanson","year":"2017","journal-title":"SIAM Review"},{"key":"2025120808264658600_qkaf001-B7","doi-asserted-by":"publisher","first-page":"203","DOI":"10.1007\/BF03014720","article-title":"Transformations des signaux al\u00e9atoires a travers les syst\u00e8mes non lin\u00e9aires sans m\u00e9moire","volume":"19","author":"Bonnet","year":"1964","journal-title":"Annales des T\u00e9l\u00e9communications"},{"issue":"2","key":"2025120808264658600_qkaf001-B8","doi-asserted-by":"publisher","first-page":"223","DOI":"10.1137\/16M1080173","article-title":"Optimization methods for large-scale machine learning","volume":"60","author":"Bottou","year":"2018","journal-title":"SIAM Review"},{"key":"2025120808264658600_qkaf001-B9","first-page":"2260","author":"Cutkosky","year":"2020"},{"issue":"5","key":"2025120808264658600_qkaf001-B10","doi-asserted-by":"publisher","first-page":"S57","DOI":"10.1016\/S0190-9622(98)70486-4","article-title":"Twelve weeks of continuous oral therapy for toenail onychomycosis caused by dermatophytes: A double-blind comparative trial of terbinafine 250 mg\/day versus itraconazole 200 mg\/day","volume":"38","author":"De Backer","year":"1998","journal-title":"Journal of the American Academy of Dermatology"},{"key":"2025120808264658600_qkaf001-B11","article-title":"A simple convergence proof of Adam and Adagrad","author":"D\u00e9fossez","year":"2022","journal-title":"Transactions on Machine Learning Research"},{"key":"2025120808264658600_qkaf001-B12","article-title":"Are you using test log-likelihood correctly?","author":"Deshpande","year":"2024","journal-title":"Transactions on Machine Learning Research"},{"key":"2025120808264658600_qkaf001-B13","first-page":"503","volume-title":"5th International Conference on Learning Representations","author":"Dinh","year":"2017"},{"key":"2025120808264658600_qkaf001-B14","first-page":"2587","volume-title":"Proceedings of the 37th International Conference on Machine Learning","author":"Domke","year":"2020"},{"key":"2025120808264658600_qkaf001-B15","first-page":"2121","article-title":"Adaptive subgradient methods for online learning and stochastic optimization","volume":"12","author":"Duchi","year":"2011","journal-title":"Journal of Machine Learning Research"},{"key":"2025120808264658600_qkaf001-B16","first-page":"2348","volume-title":"Advances in Neural Information Processing Systems","author":"Graves","year":"2011"},{"key":"2025120808264658600_qkaf001-B17","first-page":"723","article-title":"A kernel two-sample test","volume":"13","author":"Gretton","year":"2012","journal-title":"Journal of Machine Learning Research"},{"key":"2025120808264658600_qkaf001-B18","author":"Han","year":"2016"},{"key":"2025120808264658600_qkaf001-B19","author":"Hazan","year":"2015"},{"key":"2025120808264658600_qkaf001-B20","first-page":"1303","article-title":"Stochastic variational inference","volume":"14","author":"Hoffman","year":"2013","journal-title":"Journal of Machine Learning Research"},{"issue":"7","key":"2025120808264658600_qkaf001-B21","doi-asserted-by":"publisher","first-page":"605","DOI":"10.1001\/jama.280.7.605","article-title":"Randomized trial of estrogen plus progestin for secondary prevention of coronary heart disease in postmenopausal women","volume":"280","author":"Hulley","year":"1998","journal-title":"JAMA"},{"key":"2025120808264658600_qkaf001-B22","author":"Khan","year":"2017"},{"key":"2025120808264658600_qkaf001-B23","author":"Khan","year":"2018"},{"key":"2025120808264658600_qkaf001-B24","first-page":"3402","volume-title":"Advances in neural information processing systems","author":"Khan","year":"2015"},{"key":"2025120808264658600_qkaf001-B25","first-page":"44615","volume-title":"Proceedings of Advances in Neural Information Processing Systems","author":"Kim","year":"2023"},{"key":"2025120808264658600_qkaf001-B26","volume-title":"3rd International Conference on Learning Representations","author":"Kingma","year":"2015"},{"key":"2025120808264658600_qkaf001-B27","volume-title":"2nd International Conference on Learning Representations","author":"Kingma","year":"2014"},{"key":"2025120808264658600_qkaf001-B28","first-page":"1","article-title":"Automatic differentiation variational inference","volume":"18","author":"Kucukelbir","year":"2017","journal-title":"Journal of Machine Learning Research"},{"key":"2025120808264658600_qkaf001-B29","doi-asserted-by":"crossref","DOI":"10.1090\/gsm\/181","volume-title":"A first course in sobolev spaces","author":"Leoni","year":"2017","edition":"2nd ed"},{"key":"2025120808264658600_qkaf001-B30","author":"Lin","year":"2019"},{"key":"2025120808264658600_qkaf001-B31","author":"Lin","year":"2019"},{"key":"2025120808264658600_qkaf001-B32","author":"Lin","year":"2021"},{"key":"2025120808264658600_qkaf001-B33","author":"Lin","year":"2020"},{"issue":"4","key":"2025120808264658600_qkaf001-B34","doi-asserted-by":"publisher","first-page":"422","DOI":"10.1137\/0601049","article-title":"The elimination matrix: Some lemmas and applications","volume":"1","author":"Magnus","year":"1980","journal-title":"SIAM Journal on Algebraic Discrete Methods"},{"key":"2025120808264658600_qkaf001-B35","first-page":"1","article-title":"New insights and perspectives on the natural gradient method","volume":"21","author":"Martens","year":"2020","journal-title":"Journal of Machine Learning Research"},{"key":"2025120808264658600_qkaf001-B36","first-page":"2408","volume-title":"International Conference on Machine Learning","author":"Martens","year":"2015"},{"key":"2025120808264658600_qkaf001-B37","doi-asserted-by":"publisher","first-page":"Article 107012","DOI":"10.1016\/j.csda.2020.107012","article-title":"Variational inference for high dimensional structured factor copulas","volume":"151","author":"Nguyen","year":"2020","journal-title":"Computational Statistics & Data Analysis"},{"issue":"4","key":"2025120808264658600_qkaf001-B38","doi-asserted-by":"publisher","first-page":"971","DOI":"10.1007\/s11222-017-9773-3","article-title":"Variational Bayes with synthetic likelihood","volume":"28","author":"Ong","year":"2018","journal-title":"Statistics and Computing"},{"issue":"3","key":"2025120808264658600_qkaf001-B39","doi-asserted-by":"publisher","first-page":"786","DOI":"10.1162\/neco.2008.08-07-592","article-title":"The variational Gaussian approximation revisited","volume":"21","author":"Opper","year":"2009","journal-title":"Neural Computation"},{"key":"2025120808264658600_qkaf001-B40","first-page":"4264","volume-title":"Advances in neural information processing systems","author":"Osawa","year":"2019"},{"key":"2025120808264658600_qkaf001-B41","first-page":"1363","volume-title":"Proceedings of the 29th International Conference on Machine Learning","author":"Paisley","year":"2012"},{"issue":"2","key":"2025120808264658600_qkaf001-B42","doi-asserted-by":"publisher","first-page":"69","DOI":"10.1109\/TIT.1958.1057444","article-title":"A useful theorem for nonlinear devices having Gaussian inputs","volume":"4","author":"Price","year":"1958","journal-title":"IRE Transactions on Information Theory"},{"key":"2025120808264658600_qkaf001-B43","first-page":"814","volume-title":"Proceedings of the 17th International Conference on Artificial Intelligence and Statistics","author":"Ranganath","year":"2014"},{"key":"2025120808264658600_qkaf001-B44","first-page":"324","volume-title":"Proceedings of The 33rd International Conference on Machine Learning","author":"Ranganath","year":"2016"},{"issue":"24","key":"2025120808264658600_qkaf001-B45","doi-asserted-by":"publisher","first-page":"5461","DOI":"10.1103\/PhysRevLett.81.5461","article-title":"Natural gradient descent for on-line learning","volume":"81","author":"Rattray","year":"1998","journal-title":"Physical Review Letters"},{"issue":"3","key":"2025120808264658600_qkaf001-B46","doi-asserted-by":"publisher","first-page":"400","DOI":"10.1214\/aoms\/1177729586","article-title":"A stochastic approximation method","volume":"22","author":"Robbins","year":"1951","journal-title":"The Annals of Mathematical Statistics"},{"key":"2025120808264658600_qkaf001-B47","author":"Ruiz","year":"2016"},{"issue":"4","key":"2025120808264658600_qkaf001-B48","doi-asserted-by":"publisher","first-page":"837","DOI":"10.1214\/13-BA858","article-title":"Fixed-form variational posterior approximation through stochastic linear regression","volume":"8","author":"Salimans","year":"2013","journal-title":"Bayesian Analysis"},{"key":"2025120808264658600_qkaf001-B49","author":"Salimbeni","year":"2018"},{"issue":"7","key":"2025120808264658600_qkaf001-B50","doi-asserted-by":"publisher","first-page":"1649","DOI":"10.1162\/089976601750265045","article-title":"Online model selection based on the variational Bayes","volume":"13","author":"Sato","year":"2001","journal-title":"Neural Computation"},{"issue":"4","key":"2025120808264658600_qkaf001-B51","doi-asserted-by":"publisher","first-page":"729","DOI":"10.1080\/10618600.2020.1740097","article-title":"High-dimensional copula variational approximation through transformation","volume":"29","author":"Smith","year":"2020","journal-title":"Journal of Computational and Graphical Statistics"},{"key":"2025120808264658600_qkaf001-B52","doi-asserted-by":"crossref","DOI":"10.1002\/0471722138","volume-title":"Introduction to stochastic search and optimization: Estimation, simulation and control","author":"Spall","year":"2003"},{"issue":"6","key":"2025120808264658600_qkaf001-B53","doi-asserted-by":"publisher","first-page":"1135","DOI":"10.1214\/aos\/1176345632","article-title":"Estimation of the mean of a multivariate normal distribution","volume":"9","author":"Stein","year":"1981","journal-title":"Annals of Statistics"},{"issue":"86","key":"2025120808264658600_qkaf001-B54","doi-asserted-by":"publisher","first-page":"5361","DOI":"10.21105\/joss","article-title":"normflows: A pytorch package for normalizing flows","volume":"8","author":"Stimper","year":"2023","journal-title":"Journal of Open Source Software"},{"key":"2025120808264658600_qkaf001-B55","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1155\/2014\/781670","article-title":"Impact of HbA1c measurement on hospital readmission rates: Analysis of 70,000 clinical database patient records","volume":"2014","author":"Strack","year":"2014","journal-title":"BioMed Research International"},{"issue":"1","key":"2025120808264658600_qkaf001-B56","doi-asserted-by":"publisher","first-page":"30","DOI":"10.1111\/rssb.12399","article-title":"Use of model reparametrization to improve variational Bayes","volume":"83","author":"Tan","year":"2021","journal-title":"Journal of the Royal Statistical Society: Series B, Statistical Methodology"},{"issue":"4","key":"2025120808264658600_qkaf001-B57","doi-asserted-by":"publisher","first-page":"910","DOI":"10.1080\/10618600.2020.1740714","article-title":"Bayesian variational inference for exponential random graph models","volume":"29","author":"Tan","year":"2020","journal-title":"Journal of Computational and Graphical Statistics"},{"issue":"2","key":"2025120808264658600_qkaf001-B58","doi-asserted-by":"publisher","first-page":"168","DOI":"10.1214\/13-STS418","article-title":"Variational inference for generalized linear mixed models using partially non-centered parametrizations","volume":"28","author":"Tan","year":"2013","journal-title":"Statistical Science"},{"issue":"2","key":"2025120808264658600_qkaf001-B59","doi-asserted-by":"publisher","first-page":"259","DOI":"10.1007\/s11222-017-9729-7","article-title":"Gaussian variational approximation with sparse precision matrices","volume":"28","author":"Tan","year":"2018","journal-title":"Statistics and Computing"},{"key":"2025120808264658600_qkaf001-B60","author":"Tang","year":"2019"},{"issue":"3","key":"2025120808264658600_qkaf001-B61","doi-asserted-by":"publisher","first-page":"657","DOI":"10.2307\/2532086","article-title":"Some covariance models for longitudinal count data with overdispersion","volume":"46","author":"Thall","year":"1990","journal-title":"Biometrics"},{"key":"2025120808264658600_qkaf001-B62","author":"Titsias","year":"2014"},{"issue":"6","key":"2025120808264658600_qkaf001-B63","doi-asserted-by":"publisher","first-page":"Article 71","DOI":"10.1007\/s11222-021-10047-1","article-title":"Variational Bayes on manifolds","volume":"31","author":"Tran","year":"2021","journal-title":"Statistics and Computing"},{"issue":"1","key":"2025120808264658600_qkaf001-B64","doi-asserted-by":"publisher","first-page":"97","DOI":"10.1080\/10618600.2019.1637747","article-title":"Bayesian deep net GLM and GLMM","volume":"29","author":"Tran","year":"2020","journal-title":"Journal of Computational and Graphical Statistics"},{"key":"2025120808264658600_qkaf001-B65","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1561\/2200000001","article-title":"Graphical models, exponential families, and variational inference","volume":"1","author":"Wainwright","year":"2008","journal-title":"Foundations and Trends in Machine Learning"},{"key":"2025120808264658600_qkaf001-B66","first-page":"2711","volume-title":"Proceedings of the 22nd International Conference on Artificial Intelligence and Statistics","author":"Xu","year":"2019"},{"key":"2025120808264658600_qkaf001-B67","first-page":"24365","volume-title":"Advances in neural information processing systems","author":"Yun","year":"2021"},{"key":"2025120808264658600_qkaf001-B68","first-page":"5852","volume-title":"Proceedings of the 35th International Conference on Machine Learning","author":"Zhang","year":"2018"},{"key":"2025120808264658600_qkaf001-B69","first-page":"1","article-title":"Fast expectation propagation for heteroscedastic, lasso-penalized, and quantile regression","volume":"24","author":"Zhou","year":"2023","journal-title":"Journal of Machine Learning Research"}],"container-title":["Journal of the Royal Statistical Society Series B: Statistical Methodology"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/academic.oup.com\/jrsssb\/article-pdf\/87\/4\/930\/61670827\/qkaf001.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"syndication"},{"URL":"https:\/\/academic.oup.com\/jrsssb\/article-pdf\/87\/4\/930\/61670827\/qkaf001.pdf","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,12,8]],"date-time":"2025-12-08T13:26:58Z","timestamp":1765200418000},"score":1,"resource":{"primary":{"URL":"https:\/\/academic.oup.com\/jrsssb\/article\/87\/4\/930\/7986692"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,1,29]]},"references-count":69,"journal-issue":{"issue":"4","published-online":{"date-parts":[[2025,1,29]]},"published-print":{"date-parts":[[2025,9,15]]}},"URL":"https:\/\/doi.org\/10.1093\/jrsssb\/qkaf001","relation":{},"ISSN":["1369-7412","1467-9868"],"issn-type":[{"value":"1369-7412","type":"print"},{"value":"1467-9868","type":"electronic"}],"subject":[],"published-other":{"date-parts":[[2025,9]]},"published":{"date-parts":[[2025,1,29]]}}}