{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,2,2]],"date-time":"2026-02-02T06:14:44Z","timestamp":1770012884874,"version":"3.49.0"},"reference-count":42,"publisher":"Institute of Electrical and Electronics Engineers (IEEE)","issue":"1","license":[{"start":{"date-parts":[[2022,1,1]],"date-time":"2022-01-01T00:00:00Z","timestamp":1640995200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/legalcode"},{"start":{"date-parts":[[2022,1,1]],"date-time":"2022-01-01T00:00:00Z","timestamp":1640995200000},"content-version":"am","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/legalcode"}],"funder":[{"name":"JSPS KAKENHI","award":["JP18H03248"],"award-info":[{"award-number":["JP18H03248"]}]},{"name":"JSPS KAKENHI","award":["JP19J13477"],"award-info":[{"award-number":["JP19J13477"]}]},{"name":"JST CREST","award":["JPMJCR19F5"],"award-info":[{"award-number":["JPMJCR19F5"]}]},{"name":"Joint Usage\/Research Center for Interdisciplinary Large-scale Information Infrastructures","award":["jh180012-NAHI"],"award-info":[{"award-number":["jh180012-NAHI"]}]}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["IEEE Trans. Pattern Anal. Mach. Intell."],"published-print":{"date-parts":[[2022,1,1]]},"DOI":"10.1109\/tpami.2020.3004354","type":"journal-article","created":{"date-parts":[[2020,6,23]],"date-time":"2020-06-23T20:37:52Z","timestamp":1592944672000},"page":"404-415","source":"Crossref","is-referenced-by-count":26,"title":["Scalable and Practical Natural Gradient for Large-Scale Deep Learning"],"prefix":"10.1109","volume":"44","author":[{"ORCID":"https:\/\/orcid.org\/0000-0001-6390-9797","authenticated-orcid":false,"given":"Kazuki","family":"Osawa","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-8108-2324","authenticated-orcid":false,"given":"Yohei","family":"Tsuji","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-8763-2075","authenticated-orcid":false,"given":"Yuichiro","family":"Ueno","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-3140-0854","authenticated-orcid":false,"given":"Akira","family":"Naruse","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-4748-5792","authenticated-orcid":false,"given":"Chuan-Sheng","family":"Foo","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-7573-7873","authenticated-orcid":false,"given":"Rio","family":"Yokota","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"263","reference":[{"key":"ref1","article-title":"On large-batch training for deep learning: Generalization gap and sharp minima","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Keskar"},{"key":"ref2","first-page":"2160","article-title":"Norm matters: Efficient and accurate normalization schemes in deep networks","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Hoffer"},{"key":"ref3","article-title":"Accurate, large minibatch SGD: Training ImageNet in 1 hour","author":"Goyal","year":"2017"},{"key":"ref4","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2009.5206848"},{"key":"ref5","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2015.123"},{"issue":"112","key":"ref6","first-page":"1","article-title":"Measuring the effects of data parallelism on neural network training","volume":"20","author":"Shallue","year":"2019","journal-title":"J. Mach. Learn. Res."},{"key":"ref7","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2016.308"},{"key":"ref8","first-page":"26","article-title":"Lecture 6.5\u2014RmsProp: Divide the gradient by a running average of its recent magnitude","volume":"4","author":"Tieleman","year":"2012","journal-title":"COURSERA: Neural Netw. Mach. Learn."},{"key":"ref9","doi-asserted-by":"publisher","DOI":"10.1007\/s13398-014-0173-7.2"},{"key":"ref10","article-title":"Extremely large minibatch SGD: Training ResNet-50 on ImageNet in 15 minutes","author":"Akiba","year":"2017"},{"key":"ref11","article-title":"Large batch training of convolutional networks","author":"You","year":"2017"},{"key":"ref12","article-title":"Highly scalable deep learning training system with mixed-precision: Training ImageNet in four minutes","author":"Jia","year":"2018"},{"key":"ref13","article-title":"Massively distributed SGD: ImageNet\/ResNet-50 training in a flash","author":"Mikami","year":"2018"},{"key":"ref14","article-title":"Yet another accelerated SGD: ResNet-50 training on ImageNet in 74.7 seconds","author":"Yamazaki","year":"2019"},{"key":"ref15","article-title":"Don\u2019t use large mini-batches, Use Local SGD","author":"Lin"},{"key":"ref16","article-title":"Image classification at supercomputer scale","author":"Ying","year":"2018"},{"key":"ref17","doi-asserted-by":"publisher","DOI":"10.1162\/089976698300017746"},{"key":"ref18","article-title":"Mixup: Beyond empirical risk minimization","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Zhang"},{"key":"ref19","first-page":"2408","article-title":"Optimizing neural networks with Kronecker-factored approximate curvature","volume-title":"Proc. 32nd Int. Conf. Int. Conf. Mach. Learn.","author":"Martens"},{"key":"ref20","first-page":"735","article-title":"Deep learning via Hessian-free optimization","volume-title":"Proc. 27th Int. Conf. Mach. Learn.","author":"Martens"},{"key":"ref21","first-page":"573","article-title":"A Kronecker-factored approximate fisher matrix for convolution layers","volume-title":"Proc. 33rd Int. Conf. Int. Conf. Mach. Learn.","author":"Grosse"},{"key":"ref22","article-title":"Distributed second-order optimization using Kronecker-factored approximations","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Ba"},{"key":"ref23","article-title":"Kronecker-factored curvature approximations for recurrent neural networks","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Martens"},{"key":"ref24","first-page":"5852","article-title":"Noisy natural gradient as variational inference","volume-title":"Proc. 35th Int. Conf. Mach. Learn.","author":"Zhang"},{"key":"ref25","first-page":"5279","article-title":"Scalable trust-region method for deep reinforcement learning using Kronecker-factored approximation","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Wu"},{"key":"ref26","first-page":"8196","article-title":"Which algorithmic choices matter at which batch sizes? Insights from a noisy quadratic model","author":"Zhang"},{"key":"ref27","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2019.01264"},{"key":"ref28","doi-asserted-by":"publisher","DOI":"10.1145\/3339186.3339202"},{"key":"ref29","first-page":"4287","article-title":"Practical deep learning with Bayesian principles","author":"Osawa"},{"key":"ref30","first-page":"557","article-title":"Practical Gauss-Newton optimisation for deep learning","volume-title":"Proc. 34th Int. Conf. Mach. Learn.","author":"Botev"},{"key":"ref31","first-page":"265","article-title":"TensorFlow: A system for large-scale machine learning","volume-title":"Proc. 12th USENIX Symp. Operating Syst. Des. Implementation","author":"Abadi"},{"key":"ref32","article-title":"New insights and perspectives on the natural gradient method","author":"Martens","year":"2014"},{"key":"ref33","doi-asserted-by":"publisher","DOI":"10.1145\/3292500.3330756"},{"key":"ref34","doi-asserted-by":"publisher","DOI":"10.5555\/3454287.3455008"},{"key":"ref35","article-title":"Information matrices and generalization","author":"Thomas","year":"2019"},{"key":"ref36","first-page":"4156","article-title":"Limitations of the empirical fisher approximation","author":"Kunstner"},{"key":"ref37","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2016.90"},{"key":"ref38","first-page":"694","article-title":"Fisher information and natural gradient learning in random deep networks","volume-title":"Proc. 22nd Int. Conf. Artif. Intell. Statist.","author":"Amari"},{"key":"ref39","doi-asserted-by":"publisher","DOI":"10.1109\/CCGRID.2019.00057"},{"key":"ref40","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v34i07.7000"},{"key":"ref41","article-title":"Three mechanisms of weight decay regularization","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Zhang"},{"key":"ref42","article-title":"L2 regularization versus batch and weight normalization","author":"van Laarhoven","year":"2017"}],"container-title":["IEEE Transactions on Pattern Analysis and Machine Intelligence"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx7\/34\/9639876\/09123671.pdf?arnumber=9123671","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,1,9]],"date-time":"2024-01-09T22:36:25Z","timestamp":1704839785000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/9123671\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2022,1,1]]},"references-count":42,"journal-issue":{"issue":"1"},"URL":"https:\/\/doi.org\/10.1109\/tpami.2020.3004354","relation":{},"ISSN":["0162-8828","2160-9292","1939-3539"],"issn-type":[{"value":"0162-8828","type":"print"},{"value":"2160-9292","type":"electronic"},{"value":"1939-3539","type":"electronic"}],"subject":[],"published":{"date-parts":[[2022,1,1]]}}}