{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,6,19]],"date-time":"2025-06-19T05:00:53Z","timestamp":1750309253183,"version":"3.41.0"},"publisher-location":"New York, NY, USA","reference-count":48,"publisher":"ACM","license":[{"start":{"date-parts":[[2024,7,14]],"date-time":"2024-07-14T00:00:00Z","timestamp":1720915200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2024,7,14]]},"DOI":"10.1145\/3638529.3654187","type":"proceedings-article","created":{"date-parts":[[2024,7,8]],"date-time":"2024-07-08T16:33:04Z","timestamp":1720456384000},"page":"1100-1109","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":0,"title":["Neural Optimizer Equation, Decay Function, and Learning Rate Schedule Joint Evolution"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0009-0008-4874-2149","authenticated-orcid":false,"given":"Brandon","family":"Morgan","sequence":"first","affiliation":[{"name":"University of Oklahoma, Norman, United States of America"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-5393-1480","authenticated-orcid":false,"given":"Dean","family":"Hougen","sequence":"additional","affiliation":[{"name":"University of Oklahoma, Norman, United States of America"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2024,7,14]]},"reference":[{"key":"e_1_3_2_2_1_1","unstructured":"Mart\u00edn Abadi Ashish Agarwal Paul Barham Eugene Brevdo Zhifeng Chen Craig Citro Greg S. Corrado Andy Davis Jeffrey Dean Matthieu Devin Sanjay Ghemawat Ian Goodfellow Andrew Harp Geoffrey Irving Michael Isard Yangqing Jia Rafal Jozefowicz Lukasz Kaiser Manjunath Kudlur Josh Levenberg Dandelion Man\u00e9 Rajat Monga Sherry Moore Derek Murray Chris Olah Mike Schuster Jonathon Shlens Benoit Steiner Ilya Sutskever Kunal Talwar Paul Tucker Vincent Vanhoucke Vijay Vasudevan Fernanda Vi\u00e9gas Oriol Vinyals Pete Warden Martin Wattenberg Martin Wicke Yuan Yu and Xiaoqiang Zheng. 2015. TensorFlow: Large-Scale Machine Learning on Heterogeneous Systems. https:\/\/www.tensorflow.org\/ Software available from tensorflow.org."},{"key":"e_1_3_2_2_2_1","volume-title":"International Conference on Machine Learning. PMLR, 459--468","author":"Bello Irwan","year":"2017","unstructured":"Irwan Bello, Barret Zoph, Vijay Vasudevan, and Quoc V Le. 2017. Neural optimizer search with reinforcement learning. In International Conference on Machine Learning. PMLR, 459--468."},{"key":"e_1_3_2_2_3_1","doi-asserted-by":"publisher","DOI":"10.1145\/3377930.3389841"},{"volume-title":"Evolving Adaptive Neural Network Optimizers for Image Classification","author":"Carvalho Pedro","key":"e_1_3_2_2_4_1","unstructured":"Pedro Carvalho, Nuno Louren\u00e7o, and Penousal Machado. 2022. Evolving Adaptive Neural Network Optimizers for Image Classification. In Genetic Programming, Eric Medvet, Gisele Pappa, and Bing Xue (Eds.). Springer International Publishing, Cham, 3--18."},{"key":"e_1_3_2_2_5_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICASSP43922.2022.9746839"},{"key":"e_1_3_2_2_6_1","volume-title":"Evolved Optimizer for Vision. In First Conference on Automated Machine Learning (Late-Breaking Workshop).","author":"Chen Xiangning","year":"2022","unstructured":"Xiangning Chen, Chen Liang, Da Huang, Esteban Real, Yao Liu, Kaiyuan Wang, Cho-Jui Hsieh, Yifeng Lu, and Quoc V Le. 2022. Evolved Optimizer for Vision. In First Conference on Automated Machine Learning (Late-Breaking Workshop)."},{"key":"e_1_3_2_2_7_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPRW50498.2020.00359"},{"key":"e_1_3_2_2_8_1","volume-title":"ICLR Workshops","author":"Dozat T.","year":"2016","unstructured":"T. Dozat. 2016. Incorporating Nesterov Momentum into Adam. ICLR Workshops (2016)."},{"key":"e_1_3_2_2_9_1","volume-title":"Adaptive Subgradient Methods for Online Learning and Stochastic Optimization. Journal of Machine Learning Research","author":"Duchi John","year":"2011","unstructured":"John Duchi, Elad Hazan, and Yoram Singer. 2011. Adaptive Subgradient Methods for Online Learning and Stochastic Optimization. Journal of Machine Learning Research (2011)."},{"key":"e_1_3_2_2_10_1","doi-asserted-by":"publisher","DOI":"10.5555\/3322706.3361996"},{"key":"e_1_3_2_2_11_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2004.383"},{"key":"e_1_3_2_2_12_1","volume-title":"Cohen","author":"Ginsburg Boris","year":"2019","unstructured":"Boris Ginsburg, Patrice Castonguay, Oleksii Hrinchuk, Oleksii Kuchaiev, Vitaly Lavrukhin, Ryan Leary, Jason Li, Huyen Nguyen, and Jonathan M. Cohen. 2019. Stochastic Gradient Methods with Layer-wise Adaptive Moments for Training of Deep Networks. CoRR abs\/1905.11286 (2019). arXiv:1905.11286 http:\/\/arxiv.org\/abs\/1905.11286"},{"volume-title":"Deep Learning","author":"Goodfellow Ian","key":"e_1_3_2_2_13_1","unstructured":"Ian Goodfellow, Yoshua Bengio, and Aaron Courville. 2016. Deep Learning. MIT Press."},{"key":"e_1_3_2_2_14_1","volume-title":"Caiming Xiong, and Richard Socher.","author":"Gotmare Akhilesh","year":"2018","unstructured":"Akhilesh Gotmare, Nitish Shirish Keskar, Caiming Xiong, and Richard Socher. 2018. A closer look at deep learning heuristics: Learning rate restarts, warmup and distillation. arXiv preprint arXiv:1810.13243 (2018)."},{"key":"e_1_3_2_2_15_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.knosys.2020.106622"},{"key":"e_1_3_2_2_16_1","volume-title":"Proceedings of the 32nd International Conference on Machine Learning (Proceedings of Machine Learning Research","volume":"456","author":"Ioffe Sergey","year":"2015","unstructured":"Sergey Ioffe and Christian Szegedy. 2015. Batch Normalization: Accelerating Deep Network Training by Reducing Internal Covariate Shift. In Proceedings of the 32nd International Conference on Machine Learning (Proceedings of Machine Learning Research, Vol. 37), Francis Bach and David Blei (Eds.). PMLR, Lille, France, 448--456."},{"key":"e_1_3_2_2_17_1","doi-asserted-by":"publisher","unstructured":"Prateek Jain Sham M. Kakade Rahul Kidambi Praneeth Netrapalli and Aaron Sidford. 2017. Accelerating Stochastic Gradient Descent For Least Squares Regression. 10.48550\/ARXIV.1704.08227","DOI":"10.48550\/ARXIV.1704.08227"},{"key":"e_1_3_2_2_18_1","volume-title":"Kingma and Jimmy Ba","author":"Diederik","year":"2017","unstructured":"Diederik P. Kingma and Jimmy Ba. 2017. Adam: A Method for Stochastic Optimization. ICLR (2017)."},{"key":"e_1_3_2_2_19_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICCVW.2013.77"},{"key":"e_1_3_2_2_21_1","volume-title":"Yang","author":"Le Ya","year":"2015","unstructured":"Ya Le and Xuan S. Yang. 2015. Tiny ImageNet Visual Recognition Challenge."},{"key":"e_1_3_2_2_22_1","first-page":"13539","article-title":"Evolving normalization-activation layers","volume":"33","author":"Liu Hanxiao","year":"2020","unstructured":"Hanxiao Liu, Andy Brock, Karen Simonyan, and Quoc Le. 2020. Evolving normalization-activation layers. Advances in Neural Information Processing Systems 33 (2020), 13539--13550.","journal-title":"Advances in Neural Information Processing Systems"},{"key":"e_1_3_2_2_23_1","volume-title":"Loss function discovery for object detection via convergence-simulation driven search. arXiv preprint arXiv:2102.04700","author":"Liu Peidong","year":"2021","unstructured":"Peidong Liu, Gengwei Zhang, Bochao Wang, Hang Xu, Xiaodan Liang, Yong Jiang, and Zhenguo Li. 2021. Loss function discovery for object detection via convergence-simulation driven search. arXiv preprint arXiv:2102.04700 (2021)."},{"key":"e_1_3_2_2_24_1","volume-title":"Decoupled Weight Decay Regularization. ICLR","author":"Loshchilov Ilya","year":"2019","unstructured":"Ilya Loshchilov and Frank Hutter. 2019. Decoupled Weight Decay Regularization. ICLR (2019)."},{"key":"e_1_3_2_2_25_1","volume-title":"Grosse","author":"Lucas James","year":"2018","unstructured":"James Lucas, Richard S. Zemel, and Roger B. Grosse. 2018. Aggregated Momentum: Stability Through Passive Damping. CoRR abs\/1804.00325 (2018). arXiv:1804.00325 http:\/\/arxiv.org\/abs\/1804.00325"},{"key":"e_1_3_2_2_26_1","volume-title":"Quasi-hyperbolic momentum and Adam for deep learning. CoRR abs\/1810.06801","author":"Ma Jerry","year":"2018","unstructured":"Jerry Ma and Denis Yarats. 2018. Quasi-hyperbolic momentum and Adam for deep learning. CoRR abs\/1810.06801 (2018). arXiv:1810.06801 http:\/\/arxiv.org\/abs\/1810.06801"},{"key":"e_1_3_2_2_27_1","doi-asserted-by":"publisher","DOI":"10.5555\/972470.972475"},{"key":"e_1_3_2_2_28_1","volume-title":"Nitish Shirish Keskar, and Richard Socher","author":"Merity Stephen","year":"2017","unstructured":"Stephen Merity, Nitish Shirish Keskar, and Richard Socher. 2017. Regularizing and optimizing LSTM language models. arXiv preprint arXiv:1708.02182 (2017)."},{"key":"e_1_3_2_2_29_1","volume-title":"Neural Loss Function Evolution for Large-Scale Image Classifier Convolutional Neural Networks. arXiv preprint arXiv","author":"Morgan Brandon","year":"2024","unstructured":"Brandon Morgan and Dean Hougen. 2024. Neural Loss Function Evolution for Large-Scale Image Classifier Convolutional Neural Networks. arXiv preprint arXiv (2024)."},{"key":"e_1_3_2_2_30_1","unstructured":"YU. E. Nesterov. 1983. A Method of Solving A Convex Programming Problem with Convergence Rate O(1\/k2). (1983)."},{"volume-title":"Proceedings of the Indian Conference on Computer Vision, Graphics and Image Processing.","author":"Nilsback M-E.","key":"e_1_3_2_2_31_1","unstructured":"M-E. Nilsback and A. Zisserman. 2008. Automated Flower Classification over a Large Number of Classes. In Proceedings of the Indian Conference on Computer Vision, Graphics and Image Processing."},{"volume-title":"PyTorch: An Imperative Style","author":"Paszke Adam","key":"e_1_3_2_2_32_1","unstructured":"Adam Paszke, Sam Gross, Francisco Massa, Adam Lerer, James Bradbury, Gregory Chanan, Trevor Killeen, Zeming Lin, Natalia Gimelshein, Luca Antiga, Alban Desmaison, Andreas Kopf, Edward Yang, Zachary DeVito, Martin Raison, Alykhan Tejani, Sasank Chilamkurthy, Benoit Steiner, Lu Fang, Junjie Bai, and Soumith Chintala. 2019. PyTorch: An Imperative Style, High-Performance Deep Learning Library. In Advances in Neural Information Processing Systems 32. Curran Associates, Inc., 8024--8035."},{"key":"e_1_3_2_2_33_1","volume-title":"On the Convergence Proof of AMSGrad and a New Version. CoRR abs\/1904.03590","author":"Phuong Tran Thi","year":"2019","unstructured":"Tran Thi Phuong and Le Trieu Phong. 2019. On the Convergence Proof of AMSGrad and a New Version. CoRR abs\/1904.03590 (2019). arXiv:1904.03590 http:\/\/arxiv.org\/abs\/1904.03590"},{"key":"e_1_3_2_2_34_1","volume-title":"Searching for activation functions. arXiv preprint arXiv:1710.05941","author":"Ramachandran Prajit","year":"2017","unstructured":"Prajit Ramachandran, Barret Zoph, and Quoc V Le. 2017. Searching for activation functions. arXiv preprint arXiv:1710.05941 (2017)."},{"key":"e_1_3_2_2_35_1","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v33i01.33014780"},{"key":"e_1_3_2_2_36_1","volume-title":"Adafactor: Adaptive Learning Rates with Sublinear Memory Cost. CoRR abs\/1804.04235","author":"Shazeer Noam","year":"2018","unstructured":"Noam Shazeer and Mitchell Stern. 2018. Adafactor: Adaptive Learning Rates with Sublinear Memory Cost. CoRR abs\/1804.04235 (2018). arXiv:1804.04235 http:\/\/arxiv.org\/abs\/1804.04235"},{"volume-title":"Cyclical learning rates for training neural networks. In 2017 IEEE winter conference on applications of computer vision (WACV)","author":"Smith Leslie N","key":"e_1_3_2_2_37_1","unstructured":"Leslie N Smith. 2017. Cyclical learning rates for training neural networks. In 2017 IEEE winter conference on applications of computer vision (WACV). IEEE, 464--472."},{"key":"e_1_3_2_2_38_1","volume-title":"Super-convergence: Very fast training of neural networks using large learning rates. In Artificial intelligence and machine learning for multi-domain operations applications","author":"Smith Leslie N","year":"2019","unstructured":"Leslie N Smith and Nicholay Topin. 2019. Super-convergence: Very fast training of neural networks using large learning rates. In Artificial intelligence and machine learning for multi-domain operations applications, Vol. 11006. SPIE, 369--386."},{"key":"e_1_3_2_2_39_1","volume-title":"International Conference on Machine Learning. PMLR, 10096--10106","author":"Tan Mingxing","year":"2021","unstructured":"Mingxing Tan and Quoc Le. 2021. Efficientnetv2: Smaller models and faster training. In International Conference on Machine Learning. PMLR, 10096--10106."},{"key":"e_1_3_2_2_40_1","volume-title":"Calibrating the Learning Rate for Adaptive Gradient Methods to Improve Generalization Performance. CoRR abs\/1908.00700","author":"Tong Qianqian","year":"2019","unstructured":"Qianqian Tong, Guannan Liang, and Jinbo Bi. 2019. Calibrating the Learning Rate for Adaptive Gradient Methods to Improve Generalization Performance. CoRR abs\/1908.00700 (2019). arXiv:1908.00700 http:\/\/arxiv.org\/abs\/1908.00700"},{"key":"e_1_3_2_2_41_1","volume-title":"Learning an adaptive learning rate schedule. arXiv preprint arXiv:1909.09712","author":"Xu Zhen","year":"2019","unstructured":"Zhen Xu, Andrew M Dai, Jonas Kemp, and Luke Metz. 2019. Learning an adaptive learning rate schedule. arXiv preprint arXiv:1909.09712 (2019)."},{"key":"e_1_3_2_2_42_1","volume-title":"Garnett (Eds.)","volume":"31","author":"Zaheer Manzil","year":"2018","unstructured":"Manzil Zaheer, Sashank Reddi, Devendra Sachan, Satyen Kale, and Sanjiv Kumar. 2018. Adaptive Methods for Nonconvex Optimization. In Advances in Neural Information Processing Systems, S. Bengio, H. Wallach, H. Larochelle, K. Grauman, N. Cesa-Bianchi, and R. Garnett (Eds.), Vol. 31. Curran Associates, Inc. https:\/\/proceedings.neurips.cc\/paper\/2018\/file\/90365351ccc7437a1309dc64e4db32a3-Paper.pdf"},{"key":"e_1_3_2_2_43_1","volume-title":"ADADELTA: An Adaptive Learning Rate Method. CoRR abs\/1212.5701","author":"Zeiler Matthew D.","year":"2012","unstructured":"Matthew D. Zeiler. 2012. ADADELTA: An Adaptive Learning Rate Method. CoRR abs\/1212.5701 (2012). arXiv:1212.5701 http:\/\/arxiv.org\/abs\/1212.5701"},{"key":"e_1_3_2_2_44_1","volume-title":"Why gradient clipping accelerates training: A theoretical justification for adaptivity. arXiv preprint arXiv:1905.11881","author":"Zhang Jingzhao","year":"2019","unstructured":"Jingzhao Zhang, Tianxing He, Suvrit Sra, and Ali Jadbabaie. 2019. Why gradient clipping accelerates training: A theoretical justification for adaptivity. arXiv preprint arXiv:1905.11881 (2019)."},{"key":"e_1_3_2_2_45_1","doi-asserted-by":"publisher","unstructured":"Jian Zhang and Ioannis Mitliagkas. 2017. YellowFin and the Art of Momentum Tuning. 10.48550\/ARXIV.1706.03471","DOI":"10.48550\/ARXIV.1706.03471"},{"key":"e_1_3_2_2_46_1","doi-asserted-by":"publisher","unstructured":"Shen-Yi Zhao Yin-Peng Xie and Wu-Jun Li. 2020. Stochastic Normalized Gradient Descent with Momentum for Large Batch Training. 10.48550\/ARXIV.2007.13985","DOI":"10.48550\/ARXIV.2007.13985"},{"key":"e_1_3_2_2_47_1","volume-title":"Duncan","author":"Zhuang Juntang","year":"2020","unstructured":"Juntang Zhuang, Tommy Tang, Yifan Ding, Sekhar Tatikonda, Nicha C. Dvornek, Xenophon Papademetris, and James S. Duncan. 2020. AdaBelief Optimizer: Adapting Stepsizes by the Belief in Observed Gradients. CoRR abs\/2010.07468 (2020). arXiv:2010.07468 https:\/\/arxiv.org\/abs\/2010.07468"},{"key":"e_1_3_2_2_48_1","volume-title":"Neural architecture search with reinforcement learning. arXiv preprint arXiv:1611.01578","author":"Zoph Barret","year":"2016","unstructured":"Barret Zoph and Quoc V Le. 2016. Neural architecture search with reinforcement learning. arXiv preprint arXiv:1611.01578 (2016)."},{"volume-title":"Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition (CVPR).","author":"Zoph Barret","key":"e_1_3_2_2_49_1","unstructured":"Barret Zoph, Vijay Vasudevan, Jonathon Shlens, and Quoc V. Le. 2018. Learning Transferable Architectures for Scalable Image Recognition. In Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition (CVPR)."}],"event":{"name":"GECCO '24: Genetic and Evolutionary Computation Conference","sponsor":["SIGEVO ACM Special Interest Group on Genetic and Evolutionary Computation"],"location":"Melbourne VIC Australia","acronym":"GECCO '24"},"container-title":["Proceedings of the Genetic and Evolutionary Computation Conference"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3638529.3654187","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3638529.3654187","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,18]],"date-time":"2025-06-18T23:56:54Z","timestamp":1750291014000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3638529.3654187"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,7,14]]},"references-count":48,"alternative-id":["10.1145\/3638529.3654187","10.1145\/3638529"],"URL":"https:\/\/doi.org\/10.1145\/3638529.3654187","relation":{},"subject":[],"published":{"date-parts":[[2024,7,14]]},"assertion":[{"value":"2024-07-14","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}