{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,11,27]],"date-time":"2025-11-27T20:58:39Z","timestamp":1764277119036,"version":"3.28.0"},"reference-count":37,"publisher":"IEEE","license":[{"start":{"date-parts":[[2022,5,23]],"date-time":"2022-05-23T00:00:00Z","timestamp":1653264000000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2022,5,23]],"date-time":"2022-05-23T00:00:00Z","timestamp":1653264000000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2022,5,23]]},"DOI":"10.1109\/icassp43922.2022.9746839","type":"proceedings-article","created":{"date-parts":[[2022,4,27]],"date-time":"2022-04-27T19:50:34Z","timestamp":1651089034000},"page":"3958-3962","source":"Crossref","is-referenced-by-count":16,"title":["Demon: Improved Neural Network Training With Momentum Decay"],"prefix":"10.1109","author":[{"given":"John","family":"Chen","sequence":"first","affiliation":[{"name":"Rice University,Houston,Texas"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Cameron","family":"Wolfe","sequence":"additional","affiliation":[{"name":"Rice University,Houston,Texas"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Zhao","family":"Li","sequence":"additional","affiliation":[{"name":"UTHealth,Houston,Texas"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Anastasios","family":"Kyrillidis","sequence":"additional","affiliation":[{"name":"Rice University,Houston,Texas"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"263","reference":[{"article-title":"Distilbert, a distilled version of bert: smaller, faster, cheaper and lighter","year":"2020","author":"sanh","key":"ref33"},{"article-title":"Huggingface’s transformers: State-of-the-art natural language processing","year":"2020","author":"wolf","key":"ref32"},{"key":"ref31","doi-asserted-by":"publisher","DOI":"10.1137\/15M1009597"},{"article-title":"Aggregated momentum: Stability through passive damping","year":"2018","author":"lucas","key":"ref30"},{"key":"ref37","article-title":"GLUE: A multi-task bench-mark and analysis platform for natural language understanding","author":"wang","year":"2019","journal-title":"Proceedings of ICLR"},{"article-title":"Generative modeling by estimating gradients of the data distribution","year":"2019","author":"song","key":"ref36"},{"article-title":"Auto-encoding variational bayes","year":"2015","author":"kingma","key":"ref35"},{"article-title":"Very deep convolutional networks for large-scale image recognition","year":"2014","author":"simonyan","key":"ref34"},{"key":"ref10","doi-asserted-by":"crossref","DOI":"10.5244\/C.30.87","article-title":"Wide residual networks","author":"zagoruyko","year":"2016"},{"key":"ref11","first-page":"4700","article-title":"Densely connected convolutional networks","author":"huang","year":"2017","journal-title":"Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition"},{"article-title":"Mobilenets: Efficient convolutional neural networks for mobile vision applications","year":"2017","author":"howard","key":"ref12"},{"article-title":"Bert: Pre-training of deep bidirectional transformers for language understanding","year":"2019","author":"devlin","key":"ref13"},{"article-title":"Revisiting distributed synchronous SGD","year":"2016","author":"chen","key":"ref14"},{"article-title":"Automatic differentiation in pytorch","year":"2017","author":"paszke","key":"ref15"},{"key":"ref16","doi-asserted-by":"publisher","DOI":"10.1109\/ALLERTON.2016.7852343"},{"article-title":"Yellowfin and the art of momentum tuning","year":"2017","author":"zhang","key":"ref17"},{"article-title":"Conditional generative adversarial nets","year":"2014","author":"mirza","key":"ref18"},{"article-title":"Unsupervised representation learning with deep convolutional generative adversarial networks","year":"2015","author":"radford","key":"ref19"},{"article-title":"On the convergence of Adam and beyond","year":"2019","author":"reddi","key":"ref28"},{"article-title":"Adadelta: an adaptive learning rate method","year":"2012","author":"zeiler","key":"ref4"},{"key":"ref3","first-page":"2121","article-title":"Adaptive subgradient methods for online learning and stochastic optimization","volume":"12","author":"duchi","year":"2011","journal-title":"Journal of Machine Learning Research"},{"key":"ref27","first-page":"265","article-title":"Tensorflow: A system for large-scale machine learning","author":"abadi","year":"2016","journal-title":"12th USENIX Symposium on OSDI"},{"article-title":"Adam: A method for stochastic optimization","year":"2014","author":"kingma","key":"ref6"},{"article-title":"Fixing weight decay regularization in adam","year":"2017","author":"loshchilov","key":"ref29"},{"key":"ref5","first-page":"8","article-title":"Neural networks for machine learning lecture 6a overview of mini-batch gradient descent","volume":"14","author":"hinton","year":"2012"},{"key":"ref8","first-page":"1097","article-title":"Imagenet classification with deep convolutional neural networks","author":"krizhevsky","year":"2012","journal-title":"Adv in NeurIPS"},{"article-title":"Quasi-hyperbolic momentum and Adam for deep learning","year":"2018","author":"ma","key":"ref7"},{"article-title":"The cost of training nlp models: A concise overview","year":"2020","author":"sharir","key":"ref2"},{"key":"ref9","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2016.90"},{"article-title":"Language models are few-shot learners","year":"2020","author":"brown","key":"ref1"},{"article-title":"Global convergence of the heavyball method for convex optimization","year":"2014","author":"ghadimi","key":"ref20"},{"article-title":"Squeeze-and-excitation networks","year":"2017","author":"hu","key":"ref22"},{"article-title":"A sufficient condition for convergences of adam and rmsprop","year":"2018","author":"zou","key":"ref21"},{"key":"ref24","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2017.683"},{"key":"ref23","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2017.106"},{"article-title":"A disciplined approach to neural network hyper-parameters: Part 1 – learning rate, batch size, momentum, and weight decay","year":"2018","author":"smith","key":"ref26"},{"article-title":"Sgdr: Stochastic gradient descent with warm restarts","year":"2017","author":"loshchilov","key":"ref25"}],"event":{"name":"ICASSP 2022 - 2022 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)","start":{"date-parts":[[2022,5,23]]},"location":"Singapore, Singapore","end":{"date-parts":[[2022,5,27]]}},"container-title":["ICASSP 2022 - 2022 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx7\/9745891\/9746004\/09746839.pdf?arnumber=9746839","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2022,8,15]],"date-time":"2022-08-15T20:05:33Z","timestamp":1660593933000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/9746839\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2022,5,23]]},"references-count":37,"URL":"https:\/\/doi.org\/10.1109\/icassp43922.2022.9746839","relation":{},"subject":[],"published":{"date-parts":[[2022,5,23]]}}}