{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,23]],"date-time":"2026-07-23T15:37:41Z","timestamp":1784821061946,"version":"3.55.0"},"reference-count":64,"publisher":"Institute of Electrical and Electronics Engineers (IEEE)","issue":"9","license":[{"start":{"date-parts":[[2024,9,1]],"date-time":"2024-09-01T00:00:00Z","timestamp":1725148800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/ieeexplore.ieee.org\/Xplorehelp\/downloads\/license-information\/IEEE.html"},{"start":{"date-parts":[[2024,9,1]],"date-time":"2024-09-01T00:00:00Z","timestamp":1725148800000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2024,9,1]],"date-time":"2024-09-01T00:00:00Z","timestamp":1725148800000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"}],"funder":[{"DOI":"10.13039\/501100001459","name":"Ministry of Education - Singapore","doi-asserted-by":"publisher","id":[{"id":"10.13039\/501100001459","id-type":"DOI","asserted-by":"publisher"}]},{"name":"NSF China","award":["62276004"],"award-info":[{"award-number":["62276004"]}]},{"name":"PCL, China","award":["PCL2021A12"],"award-info":[{"award-number":["PCL2021A12"]}]}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["IEEE Trans. Pattern Anal. Mach. Intell."],"published-print":{"date-parts":[[2024,9]]},"DOI":"10.1109\/tpami.2024.3382294","type":"journal-article","created":{"date-parts":[[2024,3,27]],"date-time":"2024-03-27T19:00:30Z","timestamp":1711566030000},"page":"6486-6493","source":"Crossref","is-referenced-by-count":270,"title":["Towards Understanding Convergence and Generalization of AdamW"],"prefix":"10.1109","volume":"46","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-3400-8943","authenticated-orcid":false,"given":"Pan","family":"Zhou","sequence":"first","affiliation":[{"name":"School of Computing and Information Systems, Singapore Management University, Singapore"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-8756-5981","authenticated-orcid":false,"given":"Xingyu","family":"Xie","sequence":"additional","affiliation":[{"name":"National Key Lab of General AI, School of Intelligence Science and Technology, Peking University, Beijing, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-1493-7569","authenticated-orcid":false,"given":"Zhouchen","family":"Lin","sequence":"additional","affiliation":[{"name":"National Key Lab of General AI, School of Intelligence Science and Technology, Peking University, Beijing, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-8906-3777","authenticated-orcid":false,"given":"Shuicheng","family":"Yan","sequence":"additional","affiliation":[{"name":"Skywork AI, Mingyang International Center, Beijing, China"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"263","reference":[{"key":"ref1","article-title":"Adam: A method for stochastic optimization","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Kingma"},{"key":"ref2","doi-asserted-by":"publisher","DOI":"10.1214\/aoms\/1177729586"},{"key":"ref3","doi-asserted-by":"publisher","DOI":"10.48550\/arXiv.2010.11929"},{"key":"ref4","article-title":"Mugs: A multi-granular self-supervised learning framework","author":"Zhou","year":"2022"},{"key":"ref5","doi-asserted-by":"publisher","DOI":"10.1109\/TASLP.2014.2339736"},{"key":"ref6","doi-asserted-by":"publisher","DOI":"10.1017\/CBO9780511801181"},{"key":"ref7","article-title":"Improving generalization performance by switching from Adam to SGD","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Keskar"},{"key":"ref8","article-title":"Adaptive gradient methods with dynamic bound of learning rate","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Luo"},{"key":"ref9","first-page":"21285","article-title":"Towards theoretically understanding why SGD generalizes better than Adam in deep learning","volume-title":"Proc. Conf. Neural Inf. Process. Syst.","author":"Zhou"},{"key":"ref10","article-title":"Win: Weight-decay-integrated Nesterov acceleration for adaptive gradient algorithms","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Zhou"},{"key":"ref11","article-title":"Decoupled weight decay regularization","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Loshchilov"},{"key":"ref12","first-page":"10347","article-title":"Training data-efficient image transformers & distillation through attention","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Touvron"},{"key":"ref13","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2016.90"},{"key":"ref14","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52688.2022.01167"},{"key":"ref15","doi-asserted-by":"publisher","DOI":"10.1007\/s10107-022-01822-7"},{"key":"ref16","article-title":"On the convergence of adaptive gradient methods for nonconvex optimization","volume-title":"Proc. Workshop Optim. Mach. Learn.","author":"Zhou"},{"key":"ref17","first-page":"3267","article-title":"Closing the generalization gap of adaptive gradient methods in training deep neural networks","volume-title":"Proc. Int. Joint Conf. Artif. Intell.","author":"Chen"},{"key":"ref18","article-title":"A novel convergence analysis for algorithms of the Adam family","volume-title":"Proc. Workshop Optim. Mach. Learn.","author":"Guo"},{"key":"ref19","article-title":"On the convergence of Adam and beyond","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Reddi"},{"key":"ref20","article-title":"On the convergence of a class of Adam-type algorithms for non-convex optimization","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Chen"},{"key":"ref21","first-page":"18795","article-title":"Adabelief optimizer: Adapting stepsizes by the belief in observed gradients","volume-title":"Proc. Conf. Neural Inf. Process. Syst.","author":"Zhuang"},{"key":"ref22","first-page":"354","article-title":"A variational analysis of stochastic gradient algorithms","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Mandt"},{"key":"ref23","first-page":"7654","article-title":"The anisotropic noise in stochastic gradient descent: Its behavior of escaping from sharp minima and regularization effects","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Zhu"},{"key":"ref24","article-title":"Three factors influencing minima in SGD","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Jastrzkebski"},{"key":"ref25","article-title":"L2 regularization versus batch and weight normalization","year":"2017"},{"key":"ref26","article-title":"Three mechanisms of weight decay regularization","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Zhang"},{"key":"ref27","first-page":"2164","article-title":"Norm matters: Efficient and accurate normalization schemes in deep networks","volume-title":"Proc. Conf. Neural Inf. Process. Syst.","author":"Hoffer"},{"key":"ref28","doi-asserted-by":"publisher","DOI":"10.5555\/3045118.3045167"},{"key":"ref29","article-title":"FixNorm: Dissecting weight decay for training deep neural networks","author":"Zhou","year":"2021"},{"key":"ref30","doi-asserted-by":"publisher","DOI":"10.1023\/A:1007618624809"},{"key":"ref31","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV48922.2021.00951"},{"key":"ref32","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-58539-6_42"},{"key":"ref33","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v35i8.16837"},{"key":"ref34","first-page":"2121","article-title":"Adaptive subgradient methods for online learning and stochastic optimization","volume-title":"Proc. J. Mach. Learn. Res.","volume":"12","author":"Duchi"},{"key":"ref35","first-page":"459","article-title":"Faster first-order methods for stochastic non-convex optimization on Riemannian manifolds","volume-title":"Proc. Int. Conf. Artif. Intell. Statist.","author":"Zhou"},{"key":"ref36","doi-asserted-by":"publisher","DOI":"10.1109\/TPAMI.2021.3087328"},{"key":"ref37","article-title":"Identity matters in deep learning","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Moritz Hardt"},{"key":"ref38","first-page":"1216","article-title":"Diverse neural network learns true target functions","volume-title":"Proc. Int. Conf. Artif. Intell. Statist.","author":"Xie"},{"key":"ref39","first-page":"745","article-title":"Stability and generalization of learning algorithms that converge to global optima","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Charles"},{"key":"ref40","article-title":"Towards understanding why lookahead generalizes better than SGD and beyond","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Zhou"},{"key":"ref41","doi-asserted-by":"publisher","DOI":"10.4135\/9781412983907.n1717"},{"key":"ref42","doi-asserted-by":"publisher","DOI":"10.1109\/tpami.2024.3423382"},{"key":"ref43","first-page":"382","article-title":"Algorithmic regularization in learning deep homogeneous models: Layers are automatically balanced","volume-title":"Proc. Conf. Neural Inf. Process. Syst.","author":"Du"},{"key":"ref44","article-title":"Large batch optimization for deep learning: Training BERT in 76 minutes","volume-title":"Proc. Int. Conf. Learn. Representations","author":"You"},{"key":"ref45","doi-asserted-by":"publisher","DOI":"10.1109\/TPAMI.2016.2544314"},{"key":"ref46","first-page":"11448","article-title":"Positive-negative momentum: Manipulating stochastic gradient noise to improve generalization","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Xie"},{"issue":"134","key":"ref47","first-page":"1","article-title":"Stochastic gradient descent as approximate Bayesian inference","volume":"18","author":"Stephan","year":"2017","journal-title":"J. Mach. Learn. Res."},{"key":"ref48","article-title":"A Bayesian perspective on generalization and stochastic gradient descent","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Smith"},{"key":"ref49","first-page":"2232","article-title":"An investigation into neural net optimization via Hessian eigenvalue density","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Ghorbani"},{"key":"ref50","article-title":"Very deep convolutional networks for large-scale image recognition","author":"Simonyan","year":"2014"},{"key":"ref51","first-page":"5956","article-title":"Escaping saddle points with adaptive gradient methods","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Staib"},{"key":"ref52","doi-asserted-by":"publisher","DOI":"10.4324\/9781410605337-29"},{"key":"ref53","doi-asserted-by":"publisher","DOI":"10.1007\/0-387-34239-7"},{"key":"ref54","first-page":"1225","article-title":"Train faster, generalize better: Stability of stochastic gradient descent","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Hardt"},{"key":"ref55","article-title":"Empirical risk landscape analysis for understanding deep neural networks","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Zhou"},{"key":"ref56","first-page":"5960","article-title":"Understanding generalization and optimization performance of deep CNNs","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Zhou"},{"key":"ref57","first-page":"34689","article-title":"Understanding the generalization benefit of normalization layers: Sharpness reduction","volume-title":"Proc. Conf. Neural Inf. Process. Syst.","author":"Lyu"},{"key":"ref58","first-page":"7697","article-title":"On the SDEs and scaling rules for adaptive gradient algorithms","volume-title":"Proc. Conf. Neural Inf. Process. Syst.","author":"Malladi"},{"key":"ref59","article-title":"Eigenvalues of the Hessian in deep learning: Singularity and beyond","author":"Sagun","year":"2016"},{"key":"ref60","article-title":"Empirical analysis of the Hessian of over-parametrized neural networks","author":"Sagun","year":"2017"},{"key":"ref61","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v35i11.17142"},{"key":"ref62","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2015.123"},{"key":"ref63","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2009.5206848"},{"key":"ref64","doi-asserted-by":"publisher","DOI":"10.1109\/BigData50022.2020.9378171"}],"container-title":["IEEE Transactions on Pattern Analysis and Machine Intelligence"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx7\/34\/10627928\/10480574.pdf?arnumber=10480574","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,8,7]],"date-time":"2024-08-07T17:43:47Z","timestamp":1723052627000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/10480574\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,9]]},"references-count":64,"journal-issue":{"issue":"9"},"URL":"https:\/\/doi.org\/10.1109\/tpami.2024.3382294","relation":{},"ISSN":["0162-8828","2160-9292","1939-3539"],"issn-type":[{"value":"0162-8828","type":"print"},{"value":"2160-9292","type":"electronic"},{"value":"1939-3539","type":"electronic"}],"subject":[],"published":{"date-parts":[[2024,9]]}}}