{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,6,16]],"date-time":"2026-06-16T20:00:33Z","timestamp":1781640033053,"version":"3.54.5"},"reference-count":43,"publisher":"Institute of Electrical and Electronics Engineers (IEEE)","issue":"3","license":[{"start":{"date-parts":[[2026,5,1]],"date-time":"2026-05-01T00:00:00Z","timestamp":1777593600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/ieeexplore.ieee.org\/Xplorehelp\/downloads\/license-information\/IEEE.html"},{"start":{"date-parts":[[2026,5,1]],"date-time":"2026-05-01T00:00:00Z","timestamp":1777593600000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2026,5,1]],"date-time":"2026-05-01T00:00:00Z","timestamp":1777593600000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"}],"funder":[{"DOI":"10.13039\/501100001711","name":"Schweizerischer Nationalfonds zur F\u00f6rderung der Wissenschaftlichen Forschung","doi-asserted-by":"publisher","award":["200021_205011"],"award-info":[{"award-number":["200021_205011"]}],"id":[{"id":"10.13039\/501100001711","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000183","name":"Army Research Office","doi-asserted-by":"publisher","award":["W911NF-24-1-0048"],"award-info":[{"award-number":["W911NF-24-1-0048"]}],"id":[{"id":"10.13039\/100000183","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["IEEE Signal Process. Mag."],"published-print":{"date-parts":[[2026,5]]},"DOI":"10.1109\/msp.2026.3667310","type":"journal-article","created":{"date-parts":[[2026,6,12]],"date-time":"2026-06-12T19:44:54Z","timestamp":1781293494000},"page":"21-36","source":"Crossref","is-referenced-by-count":0,"title":["Training Neural Networks at Any Scale: An exposition"],"prefix":"10.1109","volume":"43","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-0278-6268","authenticated-orcid":false,"given":"Thomas","family":"Pethick","sequence":"first","affiliation":[{"name":"Laboratory for Information and Inference Systems, École Polytechnique Fédérale de Lausanne, Lausanne, Switzerland"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-2931-4420","authenticated-orcid":false,"given":"Kimon","family":"Antonakopoulos","sequence":"additional","affiliation":[{"name":"Laboratory for Information and Inference Systems, École Polytechnique Fédérale de Lausanne, Lausanne, Switzerland"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Antonio","family":"Silveti-Falls","sequence":"additional","affiliation":[{"name":"Department of Mathematics at CentraleSupélec, Université Paris-Saclay, Inria, Gif-sur-Yvette, France"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Leena Chennuru","family":"Vankadara","sequence":"additional","affiliation":[{"name":"Gatsby Computational Neuroscience Unit, University College London, London, U.K."}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-5004-201X","authenticated-orcid":false,"given":"Volkan","family":"Cevher","sequence":"additional","affiliation":[{"name":"Laboratory for Information and Inference Systems (LIONS), École Polytechnique Fédérale de Lausanne (EPFL), Lausanne, Switzerland"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"263","reference":[{"key":"ref1","first-page":"49,069","article-title":"Training deep learning models with norm-constrained LMOs","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Pethick","year":"2025"},{"key":"ref2","article-title":"Muon: An optimizer for hidden layers in neural networks","volume-title":"Keller Jordan blog","author":"Jordan","year":"2024"},{"key":"ref3","first-page":"3920","article-title":"Modular duality in deep learning","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Bernstein","year":"2025"},{"key":"ref4","first-page":"8580","article-title":"Neural tangent kernel: Convergence and generalization in neural networks","volume-title":"Proc. 32nd Int. Conf. Neural Inf. Process. Syst. (NeurIPS)","author":"Jacot","year":"2018"},{"key":"ref5","first-page":"2937","article-title":"On lazy training in differentiable programming","volume-title":"Proc. 33rd Int. Conf. Neural Inf. Process. Syst. (NeurIPS)","author":"Chizat","year":"2019"},{"key":"ref6","first-page":"11,727","article-title":"Tensor programs IV: Feature learning in infinite-width neural networks","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Yang","year":"2021"},{"key":"ref7","first-page":"1842","article-title":"Shampoo: Preconditioned stochastic tensor optimization","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Gupta","year":"2018"},{"key":"ref8","first-page":"111","article-title":"Stochastic spectral descent for restricted Boltzmann machines","volume-title":"Proc. 18th Int. Conf. Artif. Intell. Statist.","author":"Carlson","year":"2015"},{"issue":"1","key":"ref9","first-page":"4232","article-title":"Stochastic conditional gradient methods: From convex minimization to submodular maximization","volume":"21","author":"Mokhtari","year":"2020","journal-title":"J. Mach. Learn. Res."},{"key":"ref10","volume-title":"Introduction to Optimization","author":"Polyak","year":"1987"},{"key":"ref11","first-page":"2121","article-title":"Adaptive subgradient methods for online learning and stochastic optimization","volume":"12","author":"Duchi","year":"2011","journal-title":"J. Mach. Learn. Res."},{"key":"ref12","first-page":"244","article-title":"Adaptive bound optimization for online convex optimization","volume-title":"Proc. Conf. Learn. Theory","author":"McMahan","year":"2010"},{"key":"ref13","volume-title":"Neural networks for machine learning lecture 6a overview of mini-batch gradient descent","author":"Hinton","year":"2012"},{"key":"ref14","first-page":"1","article-title":"Adam: A method for stochastic optimization","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Kingma","year":"2014"},{"key":"ref15","first-page":"1","article-title":"Decoupled weight decay regularization","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Loshchilov","year":"2019"},{"key":"ref16","doi-asserted-by":"publisher","DOI":"10.1109\/jstsp.2015.2505684"},{"key":"ref17","first-page":"560","article-title":"signSGD: Compressed optimisation for non-convex problems","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Bernstein","year":"2018"},{"key":"ref18","doi-asserted-by":"publisher","DOI":"10.1137\/100802001"},{"key":"ref19","first-page":"2971","article-title":"Preconditioned spectral descent for deep learning","volume-title":"Proc. 28th Int. Conf. Neural Inf. Process. Syst.","author":"Carlson","year":"2015"},{"key":"ref20","doi-asserted-by":"publisher","DOI":"10.1002\/nav.3800030109"},{"key":"ref21","doi-asserted-by":"publisher","DOI":"10.1145\/1824777.1824783"},{"key":"ref22","first-page":"427","article-title":"Revisiting Frank-Wolfe: Projection-free sparse convex optimization","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Jaggi","year":"2013"},{"key":"ref23","first-page":"1","article-title":"Lion secretly solves a constrained optimization: As Lyapunov predicts","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Chen","year":"2024"},{"key":"ref24","first-page":"1594","article-title":"Beyond convexity: Stochastic quasi-convex optimization","volume-title":"Proc. 29th Int. Conf. Neural Inf. Process. Syst. (NeurIPS)","volume":"1","author":"Hazan","year":"2015"},{"key":"ref25","first-page":"2260","article-title":"Momentum improves normalized SGD","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Cutkosky","year":"2020"},{"key":"ref26","article-title":"PolarGrad: A class of matrix-gradient optimizers from a unifying preconditioning perspective","author":"Lau","year":"2025"},{"key":"ref27","article-title":"Muon is scalable for LLM training","author":"Liu","year":"2025"},{"key":"ref28","first-page":"17,084","article-title":"Tensor programs V: Tuning large neural networks via zero-shot hyperparameter transfer","volume-title":"Proc. 35th Int. Conf. Neural Inf. Process. Syst. (NeurIPS)","author":"Yang","year":"2021"},{"key":"ref29","first-page":"7282","article-title":"Conditional gradient methods via stochastic path-integrated differential estimator","volume-title":"Proc. 36th Int. Conf. Mach. Learn.","author":"Yurtsever","year":"2019"},{"key":"ref30","doi-asserted-by":"publisher","DOI":"10.1137\/24m1717762"},{"key":"ref31","doi-asserted-by":"publisher","DOI":"10.52202\/079017-0320"},{"key":"ref32","article-title":"The polar express: Optimal matrix sign methods and their application to the muon algorithm","author":"Amsel","year":"2025"},{"key":"ref33","article-title":"Dion: Distributed orthonormalized updates","author":"Ahn","year":"2025"},{"key":"ref34","first-page":"86,145","article-title":"On feature learning in structured state space models","volume-title":"Proc. 38th Int. Conf. Neural Inf. Process. Syst. (NeurIPS)","author":"Vankadara","year":"2024"},{"key":"ref35","first-page":"30,016","article-title":"Training compute-optimal large language models","volume-title":"Proc. 36th Int. Conf. Neural Inf. Process. Syst. (NeurIPS)","author":"Hoffmann","year":"2022"},{"key":"ref36","first-page":"1","article-title":"Emergent abilities of large language models","volume-title":"Proc. Trans. Mach. Learn. Res.","author":"Wei","year":"2022"},{"key":"ref37","first-page":"1","article-title":"Mamba: Linear-time sequence modeling with selective state spaces","volume-title":"Proc. 1st Conf. Lang. Model.","author":"Gu","year":"2024"},{"key":"ref38","article-title":"Dont be lazy: CompleteP enables compute-efficient deep transformers","author":"Dey","year":"2025"},{"key":"ref39","first-page":"73,501","article-title":"Scalable optimization in the modular norm","volume-title":"Proc. 38th Int. Conf. Neural Inf. Process. Syst. (NeurIPS)","author":"Large","year":"2024"},{"key":"ref40","article-title":"A spectral condition for feature learning","author":"Yang","year":"2023"},{"key":"ref41","first-page":"38,888","article-title":"P2: Effective sharpness aware minimization requires layerwise perturbation scaling","volume-title":"Proc. 38th Int. Conf. Neural Inf. Process. Syst. (NeurIPS)","author":"Haas","year":"2024"},{"key":"ref42","doi-asserted-by":"publisher","DOI":"10.1109\/iccv.2015.123"},{"key":"ref43","article-title":"The duality structure gradient descent algorithm: Analysis and applications to neural networks","author":"Flynn","year":"2017"}],"container-title":["IEEE Signal Processing Magazine"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx8\/79\/11561017\/11561049.pdf?arnumber=11561049","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,6,16]],"date-time":"2026-06-16T19:45:59Z","timestamp":1781639159000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/11561049\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026,5]]},"references-count":43,"journal-issue":{"issue":"3"},"URL":"https:\/\/doi.org\/10.1109\/msp.2026.3667310","relation":{},"ISSN":["1053-5888","1558-0792"],"issn-type":[{"value":"1053-5888","type":"print"},{"value":"1558-0792","type":"electronic"}],"subject":[],"published":{"date-parts":[[2026,5]]}}}