{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,8,1]],"date-time":"2026-08-01T05:58:12Z","timestamp":1785563892853,"version":"3.56.0"},"reference-count":43,"publisher":"IEEE","license":[{"start":{"date-parts":[[2026,5,1]],"date-time":"2026-05-01T00:00:00Z","timestamp":1777593600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/ieeexplore.ieee.org\/Xplorehelp\/downloads\/license-information\/IEEE.html"},{"start":{"date-parts":[[2026,5,1]],"date-time":"2026-05-01T00:00:00Z","timestamp":1777593600000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2026,5,1]],"date-time":"2026-05-01T00:00:00Z","timestamp":1777593600000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2026,5]]},"DOI":"10.1109\/ccgrid68966.2026.00032","type":"proceedings-article","created":{"date-parts":[[2026,7,31]],"date-time":"2026-07-31T18:11:58Z","timestamp":1785521518000},"page":"231-242","source":"Crossref","is-referenced-by-count":0,"title":["Tula: Optimizing Time, Cost, and Generalization in Distributed Large-Batch Training"],"prefix":"10.1109","author":[{"given":"Sahil","family":"Tyagi","sequence":"first","affiliation":[{"name":"Oak Ridge National Laboratory","place":["Oak Ridge, Tennessee, USA"]}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Feiyi","family":"Wang","sequence":"additional","affiliation":[{"name":"Oak Ridge National Laboratory","place":["Oak Ridge, Tennessee, USA"]}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"263","reference":[{"key":"ref1","article-title":"Scaling Laws for Neural Language Models","volume":"abs\/2001.08361","author":"Jared","year":"2020","journal-title":"ArXiv"},{"key":"ref2","doi-asserted-by":"publisher","DOI":"10.1109\/BigData59044.2023.10386724"},{"key":"ref3","doi-asserted-by":"publisher","DOI":"10.1109\/HORA49412.2020.9152915"},{"key":"ref4","article-title":"Pollux: Co-adaptive Cluster Scheduling for Goodput-Optimized Deep Learning","author":"Aurick","year":"2021","journal-title":"2021 USENIX Operating Systems Design and Principles (OSDI)"},{"key":"ref5","article-title":"An Empirical Model of Large-Batch Training","volume":"abs\/1812.06162","author":"Mc Candlish","year":"2018","journal-title":"OpenAI Dota Team"},{"key":"ref6","article-title":"KungFu: Making Training in Distributed Machine Learning Adaptive","volume-title":"USENIX Symposium on Operating Systems Design and Implementation","author":"Luo","year":"2020"},{"key":"ref7","article-title":"Accurate, Large Minibatch SGD: Training ImageNet in 1 Hour","volume":"abs\/1706.02677","author":"Priya","year":"2017","journal-title":"ArXiv"},{"key":"ref8","article-title":"On Large-Batch Training for Deep Learning: Generalization Gap and Sharp Minima","volume-title":"2017 International Conference on Learning Representations (ICLR)","author":"Keskar","year":"2017"},{"key":"ref9","article-title":"Large Batch Training of Convolutional Networks","author":"Yang","year":"2017","journal-title":"arXiv: Computer Vision and Pattern Recognition"},{"key":"ref10","doi-asserted-by":"publisher","DOI":"10.1109\/ACSOS49614.2020.00041"},{"key":"ref11","doi-asserted-by":"publisher","DOI":"10.1145\/3405671.3405810"},{"key":"ref12","first-page":"770","article-title":"Deep Residual Learning for Image Recognition","volume-title":"2016 IEEE Conference on Computer Vision and Pattern Recognition (CVPR)","author":"Kaiming","year":"2015"},{"key":"ref13","article-title":"Very Deep Convolutional Networks for Large-Scale Image Recognition","volume":"abs\/1409.1556","author":"Karen","year":"2014","journal-title":"CoRR"},{"key":"ref14","first-page":"84","article-title":"ImageNet classification with deep convolutional neural networks","volume":"60","author":"Alex","year":"2012","journal-title":"Communications of the ACM"},{"key":"ref15","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2019.00140"},{"key":"ref16","doi-asserted-by":"publisher","DOI":"10.1109\/CCGrid57682.2023.00045"},{"key":"ref17","article-title":"Communication-Efficient Distributed Deep Learning: A Comprehensive Survey","volume":"abs\/2003.06307","author":"Zhenheng","year":"2020","journal-title":"ArXiv"},{"key":"ref18","article-title":"Train longer, generalize better: closing the generalization gap in large batch training of neural networks","author":"Elad","year":"2017","journal-title":"Neural Information Processing Systems (NeurIPS)"},{"key":"ref19","article-title":"On the Computational Inefficiency of Large Batch Sizes for Stochastic Gradient Descent","volume":"abs\/1811.12941","author":"Noah","year":"2018","journal-title":"ArXiv"},{"key":"ref20","article-title":"Hessian-based Analysis of Large Batch Training and Robustness to Adversaries","author":"Zhewei","year":"2018","journal-title":"w2018 Neural Information Processing Systems (NeurIPS)"},{"key":"ref21","article-title":"The Early Phase of Neural Network Training","volume-title":"2020 International Conference on Learning Representations (ICLR)","author":"Jonathan","year":"2020"},{"key":"ref22","article-title":"Critical Learning Periods in Deep Neural Networks","volume":"abs\/1711.08856","author":"Alessandro","year":"2017","journal-title":"ArXiv"},{"key":"ref23","doi-asserted-by":"publisher","DOI":"10.1109\/CLUSTER52292.2023.00008"},{"key":"ref24","article-title":"On the Relation Between the Sharpest Directions of DNN Loss and the SGD Step Length","author":"Stanislaw","year":"2018","journal-title":"arXiv: Machine Learning"},{"key":"ref25","article-title":"Adaptive Gradient Communication via Critical Learning Regime Identification","volume-title":"Conference on Machine Learning and Systems","author":"Saurabh","year":"2021"},{"key":"ref26","doi-asserted-by":"publisher","DOI":"10.1145\/3731599.3767397"},{"key":"ref27","doi-asserted-by":"publisher","DOI":"10.1109\/TPDS.2025.3553066"},{"key":"ref28","first-page":"1","article-title":"ZeRO: Memory optimizations Toward Training Trillion Parameter Models","volume-title":"SC20: International Conference for High Performance Computing, Networking, Storage and Analysis","author":"Samyam","year":"2019"},{"key":"ref29","article-title":"An Empirical Study of Stochastic Gradient Descent with Structured Covariance Noise","volume-title":"2020 International Conference on Artificial Intelligence and Statistics (AISTATS)","author":"Yeming","year":"2020"},{"key":"ref30","article-title":"Don\u2019t Decay the Learning Rate, Increase the Batch Size","volume-title":"International Conference on Learning Representations (ICLR)","author":"Samuel L.","year":"2017"},{"key":"ref31","first-page":"1","article-title":"Large-Batch Training for LSTM and Beyond","volume-title":"SC19: International Conference for High Performance Computing, Networking, Storage and Analysis (SC)","author":"Yang","year":"2019"},{"key":"ref32","doi-asserted-by":"publisher","DOI":"10.1109\/ICDCSW.2011.20"},{"key":"ref33","article-title":"Don\u2019t Use Large MiniBatches, Use Local SGD","volume-title":"2018 International Conference on Learning Representations (ICLR)","author":"Tao","year":"2018"},{"key":"ref34","first-page":"1","article-title":"Scalable Deep Learning on Distributed Infrastructures","volume":"53","author":"Ruben","year":"2019","journal-title":"ACM Computing Surveys (CSUR)"},{"key":"ref35","article-title":"AdaScale SGD: A User-Friendly Algorithm for Distributed Training","volume-title":"2020 International Conference on Machine Learning (ICML)","author":"Tyler B.","year":"2020"},{"key":"ref36","article-title":"A Bayesian Perspective on Generalization and Stochastic Gradient Descent","volume-title":"2017 International Conference on Learning Representations (ICLR)","author":"Smith","year":"2017"},{"key":"ref37","article-title":"Extrapolation for Large-batch Training in Deep Learning","volume-title":"2020 International Conference on Machine Learning (ICML)","author":"Tao","year":"2020"},{"key":"ref38","article-title":"Large Batch Optimization for Deep Learning: Training BERT in 76 minutes","volume-title":"2020 International Conference on Learning Representations (ICLR)","author":"Yang"},{"key":"ref39","doi-asserted-by":"publisher","DOI":"10.1109\/CLOUD60044.2023.00045"},{"key":"ref40","first-page":"7330","article-title":"Learned Gradient Compression for Distributed Deep Learning","volume":"33","author":"Lusine","year":"2021","journal-title":"IEEE Transactions on Neural Networks and Learning Systems"},{"key":"ref41","article-title":"DataRater: Meta-Learned Dataset Curation","volume":"abs\/2505.17895","author":"Dan Andrei","year":"2025","journal-title":"ArXiv"},{"key":"ref42","article-title":"Compressing Gradient Optimizers via Count-Sketches","volume-title":"2019 International Conference on Machine Learning (ICML)","author":"Spring","year":"2019"},{"key":"ref43","doi-asserted-by":"publisher","DOI":"10.1109\/BigData55660.2022.10020597"}],"event":{"name":"2026 IEEE 26th International Symposium on Cluster, Cloud and Internet Computing (CCGrid)","location":"Sydney, Australia","start":{"date-parts":[[2026,5,18]]},"end":{"date-parts":[[2026,5,21]]}},"container-title":["2026 IEEE 26th International Symposium on Cluster, Cloud and Internet Computing (CCGrid)"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx8\/11618979\/11618983\/11619205.pdf?arnumber=11619205","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,8,1]],"date-time":"2026-08-01T05:11:10Z","timestamp":1785561070000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/11619205\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026,5]]},"references-count":43,"URL":"https:\/\/doi.org\/10.1109\/ccgrid68966.2026.00032","relation":{},"subject":[],"published":{"date-parts":[[2026,5]]}}}