{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2024,10,29]],"date-time":"2024-10-29T15:09:23Z","timestamp":1730214563831,"version":"3.28.0"},"reference-count":44,"publisher":"IEEE","license":[{"start":{"date-parts":[[2021,11,1]],"date-time":"2021-11-01T00:00:00Z","timestamp":1635724800000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2021,11,1]],"date-time":"2021-11-01T00:00:00Z","timestamp":1635724800000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2021,11]]},"DOI":"10.1109\/dicta52665.2021.9647143","type":"proceedings-article","created":{"date-parts":[[2021,12,23]],"date-time":"2021-12-23T23:04:18Z","timestamp":1640300658000},"page":"1-10","source":"Crossref","is-referenced-by-count":1,"title":["A Chaos Theory Approach to Understand Neural Network Optimization"],"prefix":"10.1109","author":[{"given":"Michele","family":"Sasdelli","sequence":"first","affiliation":[{"name":"Australian Institute for Machine Learning, School of Computer Science, The University of Adelaide"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Thalaiyasingam","family":"Ajanthan","sequence":"additional","affiliation":[{"name":"Australian National University & Amazon Australia, Work done prior to joining Amazon"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Tat-Jun","family":"Chin","sequence":"additional","affiliation":[{"name":"Australian Institute for Machine Learning, School of Computer Science, The University of Adelaide"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Gustavo","family":"Carneiro","sequence":"additional","affiliation":[{"name":"Australian Institute for Machine Learning, School of Computer Science, The University of Adelaide"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"263","reference":[{"key":"ref39","first-page":"604","article-title":"Fast large-scale optimization by unifying stochastic gradient and quasi-newton methods","author":"sohl-dickstein","year":"0","journal-title":"International Conference on Machine Learning"},{"key":"ref38","article-title":"Don't decay the learning rate, increase the batch size","author":"smith","year":"2017","journal-title":"ArXiv Preprint"},{"key":"ref33","article-title":"Adaptive learning rates and parallelization for stochastic, sparse, non-smooth gradients","author":"schaul","year":"2013","journal-title":"ArXiv Preprint"},{"key":"ref32","first-page":"2483","article-title":"How does batch normalization help optimization?","author":"santurkar","year":"2018","journal-title":"Advances in neural information processing systems"},{"key":"ref31","article-title":"An overview of gradient descent optimization algorithms","author":"ruder","year":"2016","journal-title":"ArXiv Preprint"},{"key":"ref30","doi-asserted-by":"publisher","DOI":"10.1214\/aoms\/1177729586"},{"key":"ref37","doi-asserted-by":"publisher","DOI":"10.1117\/12.2520589"},{"key":"ref36","article-title":"Very deep convolutional networks for large-scale image recognition","author":"simonyan","year":"2014","journal-title":"ArXiv Preprint"},{"key":"ref35","article-title":"Deep information propagation","author":"schoenholz","year":"2016","journal-title":"ArXiv Preprint"},{"key":"ref34","first-page":"343","article-title":"No more pesky learning rates","author":"schaul","year":"0","journal-title":"International Conference on Machine Learning"},{"key":"ref10","article-title":"Adaptive subgradient methods for online learning and stochastic optimization","volume":"12","author":"duchi","year":"2011","journal-title":"Journal of Machine Learning Research"},{"key":"ref40","volume":"69","author":"sprott","year":"2003","journal-title":"Chaos and Time-Series Analysis"},{"key":"ref11","first-page":"1225","article-title":"Train faster, generalize better: Stability of stochastic gradient descent","author":"hardt","year":"0","journal-title":"International Conference on Machine Learning"},{"key":"ref12","first-page":"1143","article-title":"Control batch size and learning rate to generalize well: Theoretical and empirical evidence","author":"he","year":"2019","journal-title":"Advances in neural information processing systems"},{"key":"ref13","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2016.90"},{"key":"ref14","article-title":"Three factors influencing minima in sgd","author":"jastrz?bski","year":"2017","journal-title":"ArXiv Preprint"},{"key":"ref15","article-title":"On large-batch training for deep learning: Generalization gap and sharp minima","author":"keskar","year":"2016","journal-title":"ArXiv Preprint"},{"key":"ref16","article-title":"Adam: A method for stochastic optimization","author":"kingma","year":"2014","journal-title":"ArXiv Preprint"},{"key":"ref17","article-title":"An alternative view: When does sgd escape local minima?","author":"kleinberg","year":"2018","journal-title":"ArXiv Preprint"},{"key":"ref18","first-page":"1097","article-title":"Imagenet classification with deep convolutional neural networks","author":"krizhevsky","year":"2012","journal-title":"Advances in neural information processing systems"},{"key":"ref19","first-page":"396","article-title":"Handwritten digit recognition with a back-propagation network","author":"yann","year":"1990","journal-title":"Advances in neural information processing systems"},{"journal-title":"Numerical Optimization","year":"2006","author":"nocedal","key":"ref28"},{"key":"ref4","doi-asserted-by":"publisher","DOI":"10.1007\/BF02128236"},{"key":"ref27","first-page":"2408","article-title":"Optimizing neural networks with kronecker-factored approximate curvature","author":"martens","year":"0","journal-title":"International Conference on Machine Learning"},{"key":"ref3","article-title":"Stability of stochastic gradient descent on nonsmooth convex losses","author":"bassily","year":"2020","journal-title":"ArXiv Preprint"},{"key":"ref6","first-page":"1737","article-title":"Sgd-qn: Careful quasi-newton stochastic gradient descent","volume":"10","author":"bordes","year":"2009","journal-title":"Journal of Machine Learning Research"},{"key":"ref29","doi-asserted-by":"publisher","DOI":"10.1063\/1.5028373"},{"key":"ref5","article-title":"Quasi-newton methods for deep learning: Forget the past, just sample","author":"berahas","year":"2019","journal-title":"ArXiv Preprint"},{"key":"ref8","doi-asserted-by":"publisher","DOI":"10.1088\/1751-8113\/46\/25\/254019"},{"key":"ref7","doi-asserted-by":"publisher","DOI":"10.1137\/16M1080173"},{"key":"ref2","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-83334-2_13"},{"key":"ref9","doi-asserted-by":"publisher","DOI":"10.1109\/ITA.2018.8503224"},{"key":"ref1","article-title":"Stochastic modified equations for the asynchronous stochastic gradient descent","author":"an","year":"2018","journal-title":"Information and Inference A Journal of the IMA"},{"key":"ref20","doi-asserted-by":"publisher","DOI":"10.1109\/5.726791"},{"key":"ref22","article-title":"Understanding the effects of data parallelism and sparsity on neural network training","author":"lee","year":"2020","journal-title":"CoRR"},{"key":"ref21","first-page":"156","article-title":"Automatic learning rate maximization by on-line estimation of the hessian's eigenvectors","author":"yann","year":"1993","journal-title":"Advances in neural information processing systems"},{"key":"ref42","first-page":"1139","article-title":"On the importance of initialization and momentum in deep learning","author":"sutskever","year":"0","journal-title":"International Conference on Machine Learning"},{"key":"ref24","article-title":"Deep learning theory review: An optimal control and dynamical systems perspective","author":"liu","year":"2019","journal-title":"ArXiv Preprint"},{"key":"ref41","doi-asserted-by":"publisher","DOI":"10.29007\/21r5"},{"key":"ref23","first-page":"2101","article-title":"Stochastic modified equations and adaptive stochastic gradient algorithms","author":"li","year":"0","journal-title":"International Conference on Machine Learning"},{"key":"ref44","article-title":"Why gradient clipping accelerates training: A theoretical justification for adaptivity","author":"zhang","year":"2019","journal-title":"ArXiv Preprint"},{"key":"ref26","first-page":"735","article-title":"Deep learning via hessian-free optimization","volume":"27","author":"martens","year":"2010","journal-title":"ICML"},{"key":"ref43","first-page":"4148","article-title":"The marginal value of adaptive gradient methods in machine learning","author":"wilson","year":"2017","journal-title":"Advances in neural information processing systems"},{"key":"ref25","first-page":"3325","article-title":"The power of interpolation: Understanding the effectiveness of sgd in modern over-parametrized learning","author":"ma","year":"0","journal-title":"International Conference on Machine Learning"}],"event":{"name":"2021 Digital Image Computing: Techniques and Applications (DICTA)","start":{"date-parts":[[2021,11,29]]},"location":"Gold Coast, Australia","end":{"date-parts":[[2021,12,1]]}},"container-title":["2021 Digital Image Computing: Techniques and Applications (DICTA)"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx7\/9647036\/9647042\/09647143.pdf?arnumber=9647143","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2022,8,2]],"date-time":"2022-08-02T23:29:21Z","timestamp":1659482961000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/9647143\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2021,11]]},"references-count":44,"URL":"https:\/\/doi.org\/10.1109\/dicta52665.2021.9647143","relation":{},"subject":[],"published":{"date-parts":[[2021,11]]}}}