{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,2,21]],"date-time":"2025-02-21T07:43:51Z","timestamp":1740123831426,"version":"3.37.3"},"reference-count":48,"publisher":"Springer Science and Business Media LLC","issue":"6","license":[{"start":{"date-parts":[[2023,8,8]],"date-time":"2023-08-08T00:00:00Z","timestamp":1691452800000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2023,8,8]],"date-time":"2023-08-08T00:00:00Z","timestamp":1691452800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Ann Math Artif Intell"],"published-print":{"date-parts":[[2024,12]]},"DOI":"10.1007\/s10472-023-09887-6","type":"journal-article","created":{"date-parts":[[2023,8,8]],"date-time":"2023-08-08T03:25:57Z","timestamp":1691465157000},"page":"1679-1698","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":2,"title":["To raise or not to raise: the autonomous learning rate question"],"prefix":"10.1007","volume":"92","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-2869-7159","authenticated-orcid":false,"given":"Xiaomeng","family":"Dong","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Tao","family":"Tan","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Michael","family":"Potter","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Yun-Chan","family":"Tsai","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Gaurav","family":"Kumar","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"V.\u00a0Ratna","family":"Saripalli","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Theodore","family":"Trafalis","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2023,8,8]]},"reference":[{"issue":"140","key":"9887_CR1","first-page":"1","volume":"21","author":"C Raffel","year":"2020","unstructured":"Raffel, C., Shazeer, N., Roberts, A., Lee, K., Narang, S., Matena, M., Zhou, Y., Li, W., Liu, P.J.: Exploring the limits of transfer learning with a unified text-to-text transformer. J. Mach. Learn. Res. 21(140), 1\u201367 (2020)","journal-title":"J. Mach. Learn. Res."},{"key":"9887_CR2","doi-asserted-by":"publisher","unstructured":"Wang, C., Bochkovskiy, A., Liao, H.M.: Scaled-yolov4: Scaling cross stage partial network. In: IEEE Conference on Computer Vision and Pattern Recognition, CVPR 2021, virtual, June 19-25, 2021, pp. 13029\u201313038. Computer Vision Foundation \/ IEEE (2021). https:\/\/doi.org\/10.1109\/CVPR46437.2021.01283","DOI":"10.1109\/CVPR46437.2021.01283"},{"key":"9887_CR3","doi-asserted-by":"crossref","unstructured":"Wang, C.Y., Mark\u00a0Liao, H.Y., Wu, Y.H., Chen, P.Y., Hsieh, J.W., Yeh, I.H.: Cspnet: A new backbone that can enhance learning capability of cnn. In: Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition Workshops, pp. 390\u2013391. (2020)","DOI":"10.1109\/CVPRW50498.2020.00203"},{"key":"9887_CR4","unstructured":"Liu, Y., Ott, M., Goyal, N., Du, J., Joshi, M., Chen, D., Levy, O., Lewis, M., Zettlemoyer, L., Stoyanov. V.: RoBERTa: A robustly optimized bert pretraining approach. (2020). https:\/\/openreview.net\/forum?id=SyxS0T4tvS"},{"key":"9887_CR5","unstructured":"Coleman, C.A., Narayanan, D., Kang, D., Zhao, T., Zhang, J., Nardi, L., Bailis, P., Olukotun, K., Re, C., Zaharia, M.: Dawnbench: An end-to-end deep learning benchmark and competition. NIPS ML Systems Workshop (2017)"},{"key":"9887_CR6","unstructured":"Loshchilov, I., Hutter, F.:SGDR: stochastic gradient descent with warm restarts. In: 5th International Conference on Learning Representations, ICLR 2017, Toulon, France, April 24-26, 2017, Conference Track Proceedings. OpenReview.net, Toulon (2017). https:\/\/openreview.net\/forum?id=Skq89Scxx"},{"key":"9887_CR7","unstructured":"Goyal, P., Doll\u00e1r, P., Girshick, R.B., Noordhuis, P., Wesolowski, L., Kyrola, A., Tulloch, A., Jia, Y., He, K.: Accurate, large minibatch SGD: training imagenet in 1 hour. (2017). CoRR abs\/1706.02677"},{"key":"9887_CR8","unstructured":"Smith, L.N., Topin, N.: Super-convergence: Very fast training of residual networks using large learning rates. (2017). CoRR abs\/1708.07120"},{"key":"9887_CR9","unstructured":"Vaswani, S., Mishkin, A., Laradji, I., Schmidt, M., Gidel, G., Lacoste-Julien, S.: Painless stochastic gradient: Interpolation, line-search, and convergence rates. In: Advances in Neural Information Processing Systems, pp. 3727\u20133740. (2019)"},{"issue":"1","key":"9887_CR10","first-page":"4262","volume":"18","author":"M Mahsereci","year":"2017","unstructured":"Mahsereci, M., Hennig, P.: Probabilistic line searches for stochastic optimization. J. Mach. Learn. Res. 18(1), 4262\u20134320 (2017)","journal-title":"J. Mach. Learn. Res."},{"key":"9887_CR11","unstructured":"Zoph, B., Le, Q.V.: Neural architecture search with reinforcement learning. In: 5th International Conference on Learning Representations, ICLR 2017, Toulon, France, April 24-26, 2017, Conference Track Proceedings. OpenReview.net, Toulon (2017). https:\/\/openreview.net\/forum?id=r1Ue8Hcxg"},{"key":"9887_CR12","unstructured":"Pham, H., Guan, M.Y., Zoph, B., Le, Q.V., Dean, J.: Efficient neural architecture search via parameter sharing. In: Dy J.G., Krause A. (eds.) Proceedings of the 35th International Conference on Machine Learning, ICML 2018, Stockholmsm\u00e4ssan, Stockholm, Sweden, July 10-15, 2018, Proceedings of Machine Learning Research, vol. 80, pp. 4092\u20134101. PMLR, Stockholm (2018). http:\/\/proceedings.mlr.press\/v80\/pham18a.html"},{"key":"9887_CR13","unstructured":"Liu, H., Simonyan, K., Yang, Y.: DARTS: differentiable architecture search. In: 7th International Conference on Learning Representations, ICLR 2019, New Orleans, LA, USA, May 6-9, 2019. OpenReview.net, New Orleans (2019). https:\/\/openreview.net\/forum?id=S1eYHoC5FX"},{"key":"9887_CR14","unstructured":"Bello, I., Zoph, B., Vasudevan, V., Le, Q.V.: Neural optimizer search with reinforcement learning. In: Precup, D., Teh, Y.W. (eds.) Proceedings of the 34th International Conference on Machine Learning, ICML 2017, Sydney, NSW, Australia, 6-11 August 2017, Proceedings of Machine Learning Research, vol. 70, pp. 459\u2013468. PMLR, Sydney (2017). http:\/\/proceedings.mlr.press\/v70\/bello17a.html"},{"key":"9887_CR15","unstructured":"Xu, Z., Dai, A.M., Kemp, J., Metz, L.: Learning an adaptive learning rate schedule. (2019). CoRR abs\/1909.09712"},{"key":"9887_CR16","unstructured":"Shu, J., Zhu, Y., Zhao, Q., Meng, D., Xu, Z.: Meta-lr-schedule-net: Learned lr schedules that scale and generalize. (2020)"},{"key":"9887_CR17","doi-asserted-by":"crossref","unstructured":"Daniel, C., Taylor, J., Nowozin, S.: Learning step size controllers for robust neural network training. In Proceedings of the AAAI Conference on Artificial Intelligence, vol. 30. (2016)","DOI":"10.1609\/aaai.v30i1.10187"},{"key":"9887_CR18","unstructured":"Wu, Y., Ren, M., Liao, R., Grosse, R.B.: Understanding short-horizon bias in stochastic meta-optimization. In: 6th International Conference on Learning Representations, ICLR 2018, Vancouver, BC, Canada, April 30 - May 3, 2018, Conference Track Proceedings. OpenReview.net, Vancouver (2018). https:\/\/openreview.net\/forum?id=H1MczcgR-"},{"key":"9887_CR19","unstructured":"Duchi, J., Hazan, E., Singer, Y.: Adaptive subgradient methods for online learning and stochastic optimization. J. Mach. Learn. Res. 12(61), 2121\u20132159 (2011). http:\/\/jmlr.org\/papers\/v12\/duchi11a.html"},{"key":"9887_CR20","unstructured":"Kingma, D.P., Ba, J.: In: Bengio Y., LeCun Y. (eds.) 3rd International Conference on Learning Representations, ICLR 2015, San Diego, CA, USA, May 7-9, 2015, Conference Track Proceedings. Adam: A method for stochastic optimization. (2015). http:\/\/arxiv.org\/abs\/1412.6980"},{"key":"9887_CR21","unstructured":"Tieleman, T., Hinton, G.: Lecture 6.5\u2014RmsProp: Divide the gradient by a running average of its recent magnitude. COURSERA: Neural Networks for Machine Learning (2012)"},{"key":"9887_CR22","unstructured":"Netzer, Y., Wang, T., Coates, A., Bissacco, A., Wu, B., Ng, A.Y.: Reading digits in natural images with unsupervised feature learning. NIPS Deep Learning and Unsupervised Feature Learning Workshop (2011)"},{"key":"9887_CR23","unstructured":"Simonyan, K., Zisserman, A.: Very deep convolutional networks for large-scale image recognition. In: Bengio Y., LeCun Y. (eds.) 3rd International Conference on Learning Representations, ICLR 2015, San Diego, CA, USA, May 7-9, 2015, Conference Track Proceedings. (2015). http:\/\/arxiv.org\/abs\/1409.1556"},{"key":"9887_CR24","unstructured":"Ioffe, S., Szegedy, C.: In: Bach F.R., Blei D.M. (eds.) Proceedings of the 32nd International Conference on Machine Learning, ICML 2015, Lille, France, 6-11 July 2015, JMLR Workshop and Conference Proceedings, vol. 37, pp. 448\u2013456. Batch normalization: Accelerating deep network training by reducing internal covariate shift. JMLR.org, Lille (2015). http:\/\/proceedings.mlr.press\/v37\/ioffe15.html"},{"key":"9887_CR25","unstructured":"Verma, G., Swami, A.: Error correcting output codes improve probability estimation and adversarial robustness of deep neural networks. In: Wallach H.M., Larochelle H., Beygelzimer A., d\u2019Alch\u00e9-Buc F., Fox E.B., Garnett R. (eds.) Advances in Neural Information Processing Systems 32: Annual Conference on Neural Information Processing Systems 2019, NeurIPS 2019, 8-14 December 2019, Vancouver, BC, Canada, pp. 8643\u20138653. (2019). https:\/\/papers.nips.cc\/paper\/2019\/hash\/cd61a580392a70389e27b0bc2b439f49-Abstract.html"},{"key":"9887_CR26","unstructured":"Goodfellow, I.J., Shlens, J., Szegedy, C.: Explaining and harnessing adversarial examples. In: Bengio Y., LeCun Y. (eds.) 3rd International Conference on Learning Representations, ICLR 2015, San Diego, CA, USA, May 7-9, 2015, Conference Track Proceedings. (2015). http:\/\/arxiv.org\/abs\/1412.6572"},{"key":"9887_CR27","doi-asserted-by":"crossref","unstructured":"Bossard, L., Guillaumin, M., Van\u00a0Gool, L.: Food-101 \u2013 mining discriminative components with random forests. In: European Conference on Computer Vision. (2014)","DOI":"10.1007\/978-3-319-10599-4_29"},{"key":"9887_CR28","doi-asserted-by":"publisher","unstructured":"Huang, G., Liu, Z., van\u00a0der\u00a0Maaten, L., Weinberger, K.Q.: Densely connected convolutional networks. In: 2017 IEEE Conference on Computer Vision and Pattern Recognition, CVPR 2017, Honolulu, HI, USA, July 21-26, 2017, pp. 2261\u20132269. IEEE Computer Society, Honolulu (2017). https:\/\/doi.org\/10.1109\/CVPR.2017.243","DOI":"10.1109\/CVPR.2017.243"},{"key":"9887_CR29","doi-asserted-by":"publisher","unstructured":"Szegedy, C., Vanhoucke, V., Ioffe, S., Shlens, J., Wojna, Z.: Rethinking the inception architecture for computer vision. In: 2016 IEEE Conference on Computer Vision and Pattern Recognition, CVPR 2016, Las Vegas, NV, USA, June 27-30, 2016 IEEE Computer Society, Las Vegas (2016), pp. 2818\u20132826. https:\/\/doi.org\/10.1109\/CVPR.2016.308","DOI":"10.1109\/CVPR.2016.308"},{"key":"9887_CR30","doi-asserted-by":"publisher","unstructured":"Kendall, A., Gal, Y., Cipolla, R.: Multi-task learning using uncertainty to weigh losses for scene geometry and semantics. In: 2018 IEEE Conference on Computer Vision and Pattern Recognition, CVPR 2018, Salt Lake City, UT, USA, June 18-22, 2018, pp. 7482\u20137491. IEEE Computer Society, Salt Lake City (2018). https:\/\/doi.org\/10.1109\/CVPR.2018.00781","DOI":"10.1109\/CVPR.2018.00781"},{"key":"9887_CR31","unstructured":"Welinder, P., Branson, S., Mita, T., Wah, C., Schroff, F., Belongie, S., Perona, P.: Caltech-UCSD Birds 200. Technical Report CNS-TR-2010-001, California Institute of Technology (2010)"},{"key":"9887_CR32","doi-asserted-by":"publisher","unstructured":"K. He, X. Zhang, S. Ren, J. Sun, Deep residual learning for image recognition. In: 2016 IEEE Conference on Computer Vision and Pattern Recognition, CVPR 2016, Las Vegas, NV, USA, June 27-30, 2016, pp. 770\u2013778. IEEE Computer Society, Las Vegas (2016) https:\/\/doi.org\/10.1109\/CVPR.2016.90","DOI":"10.1109\/CVPR.2016.90"},{"key":"9887_CR33","doi-asserted-by":"publisher","unstructured":"Ronneberger, O., Fischer, P., Brox, T.: U-net: Convolutional networks for biomedical image segmentation. In: Navab N., Hornegger J., Wells\u00a0III W.M., Frangi A.F. (eds.) Medical Image Computing and Computer-Assisted Intervention - MICCAI 2015 - 18th International Conference Munich, Germany, October 5 - 9, 2015, Proceedings, Part III, Lecture Notes in Computer Science, vol. 9351, pp. 234\u2013241. Springer, Munich (2015). https:\/\/doi.org\/10.1007\/978-3-319-24574-4_28","DOI":"10.1007\/978-3-319-24574-4_28"},{"key":"9887_CR34","unstructured":"Maas, A.L., Daly, R.E., Pham, P.T., Huang, D., Ng, A.Y., Potts, C.: Learning word vectors for sentiment analysis. In: Proceedings of the 49th Annual Meeting of the Association for Computational Linguistics: Human Language Technologies, pp. 142\u2013150. Association for Computational Linguistics, Portland (2011). http:\/\/www.aclweb.org\/anthology\/P11-1015"},{"key":"9887_CR35","doi-asserted-by":"crossref","unstructured":"Liu, J., Cyphers, S., Pasupat, P., McGraw, I., Glass, J.R.: A conversational movie search system based on conditional random fields. In: INTERSPEECH 2012, 13th Annual Conference of the International Speech Communication Association, Portland, Oregon, USA, September 9-13, 2012, pp. 2454\u20132457. ISCA, Portland (2012). http:\/\/www.isca-speech.org\/archive\/interspeech_2012\/i12_2454.html","DOI":"10.21437\/Interspeech.2012-563"},{"key":"9887_CR36","doi-asserted-by":"publisher","unstructured":"Devlin, J., Chang, M., Lee, K., Toutanova, K.: BERT: pre-training of deep bidirectional transformers for language understanding. In: Burstein J., Doran C., Solorio T. (eds.) Proceedings of the 2019 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, NAACL-HLT 2019, Minneapolis, MN, USA, June 2-7, 2019, Volume 1 (Long and Short Papers), pp. 4171\u20134186. Association for Computational Linguistics, Minneapolis (2019). https:\/\/doi.org\/10.18653\/v1\/n19-1423","DOI":"10.18653\/v1\/n19-1423"},{"issue":"6266","key":"9887_CR37","doi-asserted-by":"publisher","first-page":"1332","DOI":"10.1126\/science.aab3050","volume":"350","author":"BM Lake","year":"2015","unstructured":"Lake, B.M., Salakhutdinov, R., Tenenbaum, J.B.: Human-level concept learning through probabilistic program induction. Science 350(6266), 1332\u20131338 (2015). https:\/\/doi.org\/10.1126\/science.aab3050","journal-title":"Science"},{"key":"9887_CR38","unstructured":"Gregory, K., Zemel, R., Salakhutdinov, R.: Siamese neural networks for one-shot image recognition. ICML Deep Learning Workshop (2015)"},{"key":"9887_CR39","unstructured":"Karpathy, A.: The unreasonable effectiveness of recurrent neural networks. http:\/\/karpathy.github.io\/2015\/05\/21\/rnn-effectiveness\/. Accessed 4 Nov 2020"},{"key":"9887_CR40","doi-asserted-by":"publisher","unstructured":"Cho, K., van\u00a0Merrienboer, B., Bahdanau, D., Bengio, Y.: In: Proceedings of SSST@EMNLP 2014, Eighth Workshop on Syntax, Semantics and Structure in Statistical Translation, Doha, Qatar, 25 October 2014, ed. by D. Wu, M. Carpuat, X. Carreras, E.M. Vecchi, On the properties of neural machine translation: Encoder-decoder approaches (Association for Computational Linguistics, Doha, Qatar, 2014), pp. 103\u2013111. https:\/\/doi.org\/10.3115\/v1\/W14-4012","DOI":"10.3115\/v1\/W14-4012"},{"issue":"6","key":"9887_CR41","first-page":"475","volume":"4","author":"S Jaeger","year":"2014","unstructured":"Jaeger, S., Candemir, S., Antani, S., Wang, Y.X., Lu, P.X., Thoma, G.: Two public chest X-ray datasets for computer-aided screening of pulmonary diseases. Quant. Imaging Med. Surg. 4(6), 475\u2013477 (2014)","journal-title":"Quant. Imaging Med. Surg."},{"key":"9887_CR42","doi-asserted-by":"publisher","first-page":"1735","DOI":"10.1162\/neco.1997.9.8.1735","volume":"9","author":"S Hochreiter","year":"1997","unstructured":"Hochreiter, S., Schmidhuber, J.: Long short-term memory. Neural Comput. 9, 1735\u20131780 (1997). https:\/\/doi.org\/10.1162\/neco.1997.9.8.1735","journal-title":"Neural Comput."},{"key":"9887_CR43","unstructured":"Kumar, A., Liang, P., Ma, T.: Verified uncertainty calibration. In: Wallach H.M., Larochelle H., Beygelzimer A., d\u2019Alch\u00e9-Buc F., Fox E.B., Garnett R. (eds.) Advances in Neural Information Processing Systems 32: Annual Conference on Neural Information Processing Systems 2019, NeurIPS 2019, December 8-14, 2019, Vancouver, BC, Canada, pp. 3787\u20133798. (2019). https:\/\/proceedings.neurips.cc\/paper\/2019\/hash\/f8c0c968632845cd133308b1a494967f-Abstract.html"},{"issue":"1","key":"9887_CR44","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1186\/s12864-019-6413-7","volume":"21","author":"D Chicco","year":"2020","unstructured":"Chicco, D., Jurman, G.: The advantages of the matthews correlation coefficient (mcc) over f1 score and accuracy in binary classification evaluation. BMC Genomics 21(1), 1\u201313 (2020)","journal-title":"BMC Genomics"},{"key":"9887_CR45","unstructured":"Zagoruyko, S., Komodakis, N.: Wide residual networks. In: Wilson R.C., Hancock E.R., Smith W.A.P. (eds.) Proceedings of the British Machine Vision Conference 2016, BMVC 2016, York, UK, September 19-22, 2016. BMVA Press, York (2016). http:\/\/www.bmva.org\/bmvc\/2016\/papers\/paper087\/index.html"},{"key":"9887_CR46","unstructured":"Page, D.C.: Cifar10-fast dawn benchmark implementation. https:\/\/github.com\/davidcpage\/cifar10-fast. Accessed 04 Nov 2020"},{"key":"9887_CR47","doi-asserted-by":"publisher","unstructured":"Lin, T., Goyal, P., Girshick, R.B., He, K., Doll\u00e1r, P.: Focal loss for dense object detection. In: IEEE International Conference on Computer Vision, ICCV 2017, Venice, Italy, October 22-29, 2017, pp. 2999\u20133007. IEEE Computer Society, Venice (2017). https:\/\/doi.org\/10.1109\/ICCV.2017.324","DOI":"10.1109\/ICCV.2017.324"},{"key":"9887_CR48","unstructured":"Marcus, M.P.: Treebank-3 ldc99t42. web download. https:\/\/catalog.ldc.upenn.edu\/LDC99T42. Accessed 11 Nov 2020"}],"container-title":["Annals of Mathematics and Artificial Intelligence"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10472-023-09887-6.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s10472-023-09887-6\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10472-023-09887-6.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,12,23]],"date-time":"2024-12-23T02:03:27Z","timestamp":1734919407000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s10472-023-09887-6"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023,8,8]]},"references-count":48,"journal-issue":{"issue":"6","published-print":{"date-parts":[[2024,12]]}},"alternative-id":["9887"],"URL":"https:\/\/doi.org\/10.1007\/s10472-023-09887-6","relation":{},"ISSN":["1012-2443","1573-7470"],"issn-type":[{"type":"print","value":"1012-2443"},{"type":"electronic","value":"1573-7470"}],"subject":[],"published":{"date-parts":[[2023,8,8]]},"assertion":[{"value":"9 July 2023","order":1,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"8 August 2023","order":2,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Declarations"}},{"value":"The authors declare that they have no conflict of interest.","order":2,"name":"Ethics","group":{"name":"EthicsHeading","label":"Conflicts of interest"}}]}}