{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,5,4]],"date-time":"2026-05-04T10:22:50Z","timestamp":1777890170642,"version":"3.51.4"},"reference-count":65,"publisher":"IEEE","license":[{"start":{"date-parts":[[2025,10,19]],"date-time":"2025-10-19T00:00:00Z","timestamp":1760832000000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2025,10,19]],"date-time":"2025-10-19T00:00:00Z","timestamp":1760832000000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"}],"funder":[{"DOI":"10.13039\/501100001459","name":"Ministry of Education, Singapore","doi-asserted-by":"publisher","award":["T2EP20224-0048"],"award-info":[{"award-number":["T2EP20224-0048"]}],"id":[{"id":"10.13039\/501100001459","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2025,10,19]]},"DOI":"10.1109\/iccv51701.2025.02101","type":"proceedings-article","created":{"date-parts":[[2026,4,29]],"date-time":"2026-04-29T19:45:49Z","timestamp":1777491949000},"page":"22633-22643","source":"Crossref","is-referenced-by-count":0,"title":["Memory-Efficient 4-bit Preconditioned Stochastic Optimization"],"prefix":"10.1109","author":[{"given":"Jingyang","family":"Li","sequence":"first","affiliation":[{"name":"National University of Singapore"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Kuangyu","family":"Ding","sequence":"additional","affiliation":[{"name":"National University of Singapore"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Kim-Chuan","family":"Toh","sequence":"additional","affiliation":[{"name":"National University of Singapore"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Pan","family":"Zhou","sequence":"additional","affiliation":[{"name":"Singapore Management University"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"263","reference":[{"key":"ref1","article-title":"Disentangling adaptive gradient methods from learning rates","author":"Agarwal","year":"2020","journal-title":"arXiv preprint"},{"key":"ref2","article-title":"Qsgd: Communication-efficient sgd via gradient quantization and encoding","volume":"30","author":"Alistarh","year":"2017","journal-title":"Advances in neural information processing systems"},{"key":"ref3","article-title":"Scalable second order optimization for deep learning","volume":"1","author":"Anil","year":"2020","journal-title":"arXiv preprint"},{"key":"ref4","doi-asserted-by":"publisher","DOI":"10.1137\/S0363012904439301"},{"key":"ref5","doi-asserted-by":"publisher","DOI":"10.1007\/s10107-020-01501-5"},{"key":"ref6","doi-asserted-by":"publisher","DOI":"10.1007\/978-981-99-8277-6"},{"key":"ref7","article-title":"Language models are few-shot learners","author":"Brown","year":"2020","journal-title":"arXiv preprint"},{"key":"ref8","doi-asserted-by":"publisher","DOI":"10.1007\/s10107-019-01431-x"},{"key":"ref9","doi-asserted-by":"publisher","DOI":"10.1137\/1.9781611971309"},{"key":"ref10","article-title":"Autoaugment: Learning augmentation policies from data","author":"Cubuk","year":"2018","journal-title":"arXiv preprint"},{"key":"ref11","doi-asserted-by":"publisher","DOI":"10.1109\/CVPRW50498.2020.00359"},{"key":"ref12","doi-asserted-by":"publisher","DOI":"10.1137\/19M1298147"},{"key":"ref13","doi-asserted-by":"publisher","DOI":"10.1007\/s10208-018-09409-5"},{"key":"ref14","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2009.5206848"},{"key":"ref15","article-title":"8-bit optimizers via block-wise quantization","volume":"2","author":"Dettmers","year":"2021","journal-title":"arXiv preprint"},{"key":"ref16","doi-asserted-by":"publisher","DOI":"10.48550\/arXiv.2010.11929"},{"issue":"7","key":"ref17","article-title":"Adaptive subgradient methods for online learning and stochastic optimization","volume":"12","author":"Duchi","year":"2011","journal-title":"Journal of Machine Learning Research"},{"key":"ref18","doi-asserted-by":"publisher","DOI":"10.1137\/17M1135086"},{"key":"ref19","doi-asserted-by":"publisher","DOI":"10.1201\/9781003162810-13"},{"key":"ref20","first-page":"2386","article-title":"Practical quasi-newton methods for training deep neural networks","volume":"33","author":"Goldfarb","year":"2020","journal-title":"Advances in Neural Information Processing Systems"},{"key":"ref21","doi-asserted-by":"publisher","DOI":"10.1137\/050643374"},{"key":"ref22","first-page":"1842","article-title":"Shampoo: Preconditioned stochastic tensor optimization","volume-title":"In International Conference on Machine Learning","volume":"1","author":"Gupta"},{"key":"ref23","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2016.90"},{"key":"ref24","article-title":"Training compute-optimal large language models","author":"Hoffmann","year":"2022","journal-title":"arXiv preprint"},{"key":"ref25","article-title":"Scaling laws for neural language models","author":"Kaplan","year":"2020","journal-title":"arXiv preprint"},{"key":"ref26","article-title":"Adam: A method for stochastic optimization","author":"Kingma","year":"2014","journal-title":"arXiv preprint"},{"key":"ref27","author":"Krizhevsky","year":"2009","journal-title":"Learning multiple layers of features from tiny images"},{"key":"ref28","doi-asserted-by":"publisher","DOI":"10.1145\/3065386"},{"issue":"7","key":"ref29","first-page":"3","article-title":"Tiny imagenet visual recognition challenge","volume":"7","author":"Le","year":"2015","journal-title":"CS 231N"},{"key":"ref30","doi-asserted-by":"publisher","DOI":"10.1093\/bioinformatics\/btz682"},{"key":"ref31","article-title":"Vision transformer for small-size datasets","volume":"6","author":"Hoon Lee","year":"2021","journal-title":"arXiv preprint"},{"key":"ref32","first-page":"2","article-title":"Memory efficient optimizers with 4-bit states","volume":"36","author":"Li","year":"2024","journal-title":"Advances in Neural Information Processing Systems"},{"key":"ref33","article-title":"Relora: High-rank training through low-rank updates","volume":"6","author":"Lialin","year":"2023","journal-title":"arXiv preprint"},{"key":"ref34","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV48922.2021.00986"},{"key":"ref35","article-title":"Decoupled weight decay regularization","volume-title":"In International Conference on Learning Representations","author":"Loshchilov"},{"key":"ref36","first-page":"2408","article-title":"Optimizing neural networks with kronecker-factored approximate curvature","volume-title":"In International Conference on Machine Learning","volume":"1","author":"Martens"},{"key":"ref37","article-title":"A new perspective on shampoo\u2019s preconditioner","author":"Morwani","year":"2024","journal-title":"arXiv preprint"},{"key":"ref38","first-page":"8748","article-title":"Learning transferable visual models from natural language supervision","volume-title":"International conference on machine learning","author":"Radford"},{"issue":"140","key":"ref39","first-page":"1","article-title":"Exploring the limits of transfer learning with a unified text-to-text transformer","volume":"21","author":"Raffel","year":"2020","journal-title":"Journal of machine learning research"},{"key":"ref40","first-page":"4384","article-title":"Ef21: A new, simpler, theoretically better, and practically faster error feedback","volume":"34","author":"Richt\u00e1rik","year":"2021","journal-title":"Advances in Neural Information Processing Systems"},{"key":"ref41","doi-asserted-by":"publisher","DOI":"10.1214\/aoms\/1177729586"},{"key":"ref42","doi-asserted-by":"publisher","DOI":"10.21437\/interspeech.2014-274"},{"key":"ref43","article-title":"A distributed data-parallel pytorch implementation of the distributed shampoo optimizer for training neural networks at-scale","volume":"1","author":"Michael Shi","year":"2023","journal-title":"arXiv preprint"},{"key":"ref44","article-title":"Very deep convolutional networks for large-scale image recognition","author":"Simonyan","year":"2014","journal-title":"arXiv preprint"},{"key":"ref45","first-page":"1139","article-title":"On the importance of initialization and momentum in deep learning","volume-title":"In International Conference on Machine Learning","volume":"1","author":"Sutskever"},{"key":"ref46","first-page":"10118","article-title":"1-bit adam: Communication efficient largescale training with adam\u2019s convergence speed","volume-title":"International Conference on Machine Learning","author":"Tang"},{"issue":"2","key":"ref47","first-page":"26","article-title":"Lecture 6.5rmsprop: Divide the gradient by a running average of its recent magnitude","volume":"4","author":"Tieleman","year":"2012","journal-title":"COURSERA: Neural networks for machine learning"},{"key":"ref48","article-title":"Llama 2: Open foundation and fine-tuned chat models","volume":"1","author":"Touvron","year":"2023","journal-title":"arXiv preprint"},{"key":"ref49","article-title":"Powersgd: Practical low-rank gradient compression for distributed optimization","volume":"32","author":"Vogels","year":"2019","journal-title":"Advances in Neural Information Processing Systems"},{"key":"ref50","article-title":"Soap: Improving and stabilizing shampoo using adam","author":"Vyas","year":"2024","journal-title":"arXiv preprint"},{"key":"ref51","article-title":"4-bit shampoo for memory-efficient network training","volume":"2","author":"Wang","year":"2024","journal-title":"arXiv preprint"},{"key":"ref52","doi-asserted-by":"publisher","DOI":"10.5555\/3294771.3294915"},{"key":"ref53","article-title":"Resnet strikes back: An improved training procedure in timm","author":"Wightman","year":"2021","journal-title":"arXiv preprint"},{"key":"ref54","article-title":"Bloomberggpt: A large language model for finance","author":"Wu","year":"2023","journal-title":"arXiv preprint"},{"issue":"48","key":"ref55","first-page":"1","article-title":"Adam-family methods for nonsmooth optimization with convergence guarantees","volume":"25","author":"Xiao","year":"2024","journal-title":"Journal of Machine Learning Research"},{"key":"ref56","doi-asserted-by":"publisher","DOI":"10.1103\/PhysRevLett.120.145301"},{"key":"ref57","article-title":"Loco: Low-bit communication adaptor for large-scale model training","volume":"2","author":"Xie","year":"2024","journal-title":"arXiv preprint"},{"key":"ref58","doi-asserted-by":"publisher","DOI":"10.1109\/TPAMI.2024.3423382"},{"key":"ref59","doi-asserted-by":"publisher","DOI":"10.52202\/068431-1970"},{"key":"ref60","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52729.2023.00760"},{"key":"ref61","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2019.00612"},{"key":"ref62","doi-asserted-by":"publisher","DOI":"10.1007\/978-1-4899-7687-1_79"},{"key":"ref63","first-page":"15383","article-title":"Why are adaptive methods good for attention models?","volume":"33","author":"Zhang","year":"2020","journal-title":"Advances in Neural Information Processing Systems"},{"key":"ref64","article-title":"Galore: Memoryefficient 11 m training by gradient low-rank projection","volume":"6","author":"Zhao","year":"2024","journal-title":"arXiv preprint"},{"key":"ref65","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v34i07.7000"}],"event":{"name":"2025 IEEE\/CVF International Conference on Computer Vision (ICCV)","location":"Honolulu, HI, USA","start":{"date-parts":[[2025,10,19]]},"end":{"date-parts":[[2025,10,25]]}},"container-title":["2025 IEEE\/CVF International Conference on Computer Vision (ICCV)"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx8\/11443115\/11443287\/11443419.pdf?arnumber=11443419","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,5,1]],"date-time":"2026-05-01T05:28:30Z","timestamp":1777613310000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/11443419\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,10,19]]},"references-count":65,"URL":"https:\/\/doi.org\/10.1109\/iccv51701.2025.02101","relation":{},"subject":[],"published":{"date-parts":[[2025,10,19]]}}}