{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,6,18]],"date-time":"2025-06-18T04:16:38Z","timestamp":1750220198625,"version":"3.41.0"},"publisher-location":"New York, NY, USA","reference-count":36,"publisher":"ACM","license":[{"start":{"date-parts":[[2022,8,29]],"date-time":"2022-08-29T00:00:00Z","timestamp":1661731200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2022,8,29]]},"DOI":"10.1145\/3545008.3545010","type":"proceedings-article","created":{"date-parts":[[2023,1,15]],"date-time":"2023-01-15T01:04:08Z","timestamp":1673744648000},"page":"1-10","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":1,"title":["SMEGA2: Distributed Asynchronous Deep Neural Network Training With a Single Momentum Buffer"],"prefix":"10.1145","author":[{"given":"Refael","family":"Cohen","sequence":"first","affiliation":[{"name":"Department of Computer Science, Technion - Israel Institute of Technology, Israel"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Ido","family":"Hakimi","sequence":"additional","affiliation":[{"name":"Department of Computer Science, Technion - Israel Institute of Technology, Israel"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Assaf","family":"Schuster","sequence":"additional","affiliation":[{"name":"Department of Computer Science, Technion - Israel Institute of Technology, Israel"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2023,1,13]]},"reference":[{"key":"e_1_3_2_1_1_1","unstructured":"Mart\u00edn Abadi Ashish Agarwal Paul Barham Eugene Brevdo Zhifeng Chen Craig Citro Greg\u00a0S. Corrado Andy Davis Jeffrey Dean Matthieu Devin Sanjay Ghemawat Ian Goodfellow Andrew Harp Geoffrey Irving Michael Isard Yangqing Jia Rafal Jozefowicz Lukasz Kaiser Manjunath Kudlur Josh Levenberg Dandelion Man\u00e9 Rajat Monga Sherry Moore Derek Murray Chris Olah Mike Schuster Jonathon Shlens Benoit Steiner Ilya Sutskever Kunal Talwar Paul Tucker Vincent Vanhoucke Vijay Vasudevan Fernanda Vi\u00e9gas Oriol Vinyals Pete Warden Martin Wattenberg Martin Wicke Yuan Yu and Xiaoqiang Zheng. 2015. TensorFlow: Large-scale Machine Learning on Heterogeneous Systems. https:\/\/www.tensorflow.org\/ Software available from tensorflow.org."},{"key":"e_1_3_2_1_2_1","unstructured":"Takuya Akiba Shuji Suzuki and Keisuke Fukuda. 2017. Extremely Large Minibatch Sgd: Training Resnet-50 on Imagenet in 15 Minutes. (2017)."},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1109\/HCW.2000.843743"},{"volume-title":"Memory-efficient Adaptive Optimization","author":"Anil Rohan","key":"e_1_3_2_1_4_1","unstructured":"Rohan Anil, Vineet Gupta, Tomer Koren, and Yoram Singer. 2019. Memory-efficient Adaptive Optimization. Curran Associates Inc."},{"key":"e_1_3_2_1_5_1","volume-title":"International Conference on Machine Learning. PMLR, 436\u2013445","author":"Aviv Rotem\u00a0Zamir","year":"2021","unstructured":"Rotem\u00a0Zamir Aviv, Ido Hakimi, Assaf Schuster, and Kfir\u00a0Yehuda Levy. 2021. Asynchronous distributed learning: Adapting to gradient delays without prior knowledge. In International Conference on Machine Learning. PMLR, 436\u2013445."},{"key":"e_1_3_2_1_6_1","volume-title":"Gap-aware Mitigation of Gradient Staleness. In International Conference on Learning Representations. https:\/\/openreview.net\/forum?id=B1lLw6EYwB","author":"Barkai Saar","year":"2020","unstructured":"Saar Barkai, Ido Hakimi, and Assaf Schuster. 2020. Gap-aware Mitigation of Gradient Staleness. In International Conference on Learning Representations. https:\/\/openreview.net\/forum?id=B1lLw6EYwB"},{"key":"e_1_3_2_1_7_1","unstructured":"Chi-Chung Chen Chia-Lin Yang and Hsiang-Yun Cheng. 2018. Efficient and Robust Parallel Dnn Training through Model Parallelism on Multi-gpu Platform. (2018)."},{"key":"e_1_3_2_1_8_1","volume-title":"Revisiting Distributed Synchronous Sgd. In International Conference on Learning Representations Workshop Track. arxiv:1604","author":"Chen Jianmin","year":"2016","unstructured":"Jianmin Chen, Rajat Monga, Samy Bengio, and Rafal Jozefowicz. 2016. Revisiting Distributed Synchronous Sgd. In International Conference on Learning Representations Workshop Track. arxiv:1604.00981"},{"key":"e_1_3_2_1_9_1","volume-title":"Mxnet: A Flexible and Efficient Machine Learning Library for Heterogeneous Distributed Systems.","author":"Chen Tianqi","year":"2015","unstructured":"Tianqi Chen, Mu Li, Yutian Li, Min Lin, Naiyan Wang, Minjie Wang, Tianjun Xiao, Bing Xu, Chiyuan Zhang, and Zheng Zhang. 2015. Mxnet: A Flexible and Efficient Machine Learning Library for Heterogeneous Distributed Systems. (2015)."},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.1145\/2901318.2901323"},{"key":"e_1_3_2_1_11_1","unstructured":"Jeffrey Dean Greg\u00a0S. Corrado Rajat Monga Kai Chen Matthieu Devin Quoc\u00a0V. Le Mark\u00a0Z. Mao Marc\u00a0Aurelio Ranzato Andrew Senior Paul Tucker Ke Yang and Andrew\u00a0Y. Ng. 2012. Large scale distributed deep networks. In Advances in Neural Information Processing Systems Vol.\u00a02. 1223\u20131231."},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2009.5206848"},{"key":"e_1_3_2_1_13_1","volume-title":"2021 USENIX Annual Technical Conference (USENIX ATC 21)","author":"Eliad Saar","year":"2021","unstructured":"Saar Eliad, Ido Hakimi, Alon De\u00a0Jagger, Mark Silberstein, and Assaf Schuster. 2021. Fine-tuning giant neural networks on commodity hardware with automatic pipeline model parallelism. In 2021 USENIX Annual Technical Conference (USENIX ATC 21). 381\u2013396."},{"key":"e_1_3_2_1_14_1","volume-title":"International Conference on Learning Representations. https:\/\/openreview.net\/forum?id=Bkeb7lHtvH","author":"Giladi Niv","year":"2020","unstructured":"Niv Giladi, Mor\u00a0Shpigel Nacson, Elad Hoffer, and Daniel Soudry. 2020. At Stability\u2019s Edge: How to Adjust Hyperparameters to Preserve Minima Selection in Asynchronous Training of Neural Networks?. In International Conference on Learning Representations. https:\/\/openreview.net\/forum?id=Bkeb7lHtvH"},{"key":"e_1_3_2_1_15_1","volume-title":"Large Minibatch SGD: Training Imagenet in 1 Hour. abs\/1706.02677","author":"Goyal Priya","year":"2017","unstructured":"Priya Goyal, Piotr Doll\u00e1r, Ross\u00a0B. Girshick, Pieter Noordhuis, Lukasz Wesolowski, Aapo Kyrola, Andrew Tulloch, Yangqing Jia, and Kaiming He. 2017. Accurate, Large Minibatch SGD: Training Imagenet in 1 Hour. abs\/1706.02677 (2017). arxiv:1706.02677"},{"key":"e_1_3_2_1_16_1","volume-title":"LAGA: Lagged AllReduce with Gradient Accumulation for Minimal Idle Time. In 2021 IEEE International Conference on Data Mining (ICDM). IEEE, 171\u2013180","author":"Hakimi Ido","year":"2021","unstructured":"Ido Hakimi, Rotem\u00a0Zamir Aviv, Kfir\u00a0Y Levy, and Assaf Schuster. 2021. LAGA: Lagged AllReduce with Gradient Accumulation for Minimal Idle Time. In 2021 IEEE International Conference on Data Mining (ICDM). IEEE, 171\u2013180."},{"key":"e_1_3_2_1_17_1","unstructured":"Ido Hakimi Saar Barkai Moshe Gabel and Assaf Schuster. 2020. Taming Momentum in a Distributed Asynchronous Environment. arXiv:1907.11612\u00a0[cs.LG]"},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2016.90"},{"key":"e_1_3_2_1_19_1","unstructured":"Atli Kosson Vitaliy Chiley Abhinav Venigalla Joel Hestness and Urs K\u00f6ster. 2020. Pipelined Backpropagation at Scale: Training Large Models without Batches. (2020)."},{"key":"e_1_3_2_1_20_1","unstructured":"Alex Krizhevsky Geoffrey Hinton 2009. Learning Multiple Layers of Features from Tiny Images. (2009)."},{"key":"e_1_3_2_1_21_1","volume-title":"Object Detection, and Visual Relationship Detection at Scale.","author":"Kuznetsova Alina","year":"2018","unstructured":"Alina Kuznetsova, Hassan Rom, Neil Alldrin, Jasper Uijlings, Ivan Krasin, Jordi Pont-Tuset, Shahab Kamali, Stefan Popov, Matteo Malloci, Tom Duerig, 2018. The Open Images Dataset V4: Unified Image Classification, Object Detection, and Visual Relationship Detection at Scale. (2018)."},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.14778\/3415478.3415530"},{"key":"e_1_3_2_1_23_1","volume-title":"Microsoft Coco: Common Objects in Context. In Computer Vision \u2013 ECCV 2014 (Cham), David Fleet, Tomas Pajdla","author":"Lin Tsung-Yi","year":"2014","unstructured":"Tsung-Yi Lin, Michael Maire, Serge Belongie, James Hays, Pietro Perona, Deva Ramanan, Piotr Doll\u00e1r, and C.\u00a0Lawrence Zitnick. 2014. Microsoft Coco: Common Objects in Context. In Computer Vision \u2013 ECCV 2014 (Cham), David Fleet, Tomas Pajdla, Bernt Schiele, and Tinne Tuytelaars (Eds.). Springer International Publishing, 740\u2013755."},{"volume-title":"Advances in Neural Information Processing Systems 31, S.\u00a0Bengio, H.\u00a0Wallach, H.\u00a0Larochelle, K.\u00a0Grauman, N.\u00a0Cesa-Bianchi, and R.\u00a0Garnett (Eds.). Curran Associates","author":"Liu Tianyi","key":"e_1_3_2_1_24_1","unstructured":"Tianyi Liu, Shiyang Li, Jianping Shi, Enlu Zhou, and Tuo Zhao. 2018. Towards Understanding Acceleration Tradeoff between Momentum and Asynchrony in Nonconvex Stochastic Optimization. In Advances in Neural Information Processing Systems 31, S.\u00a0Bengio, H.\u00a0Wallach, H.\u00a0Larochelle, K.\u00a0Grauman, N.\u00a0Cesa-Bianchi, and R.\u00a0Garnett (Eds.). Curran Associates, Inc., 3682\u20133692. http:\/\/papers.nips.cc\/paper\/7626-towards-understanding-acceleration-tradeoff-between-momentum-and-asynchrony-in-nonconvex-stochastic-optimization.pdf"},{"volume-title":"2016 54thAnnual Allerton Conference on Communication, Control, and Computing (Allerton). 997\u20131004","author":"Mitliagkas I.","key":"e_1_3_2_1_25_1","unstructured":"I. Mitliagkas, C. Zhang, S. Hadjis, and C. R\u00e9. 2016. Asynchrony Begets Momentum, with an Application to Deep Learning. In 2016 54thAnnual Allerton Conference on Communication, Control, and Computing (Allerton). 997\u20131004."},{"key":"e_1_3_2_1_26_1","volume-title":"A Method for Solving the Convex Programming Problem with Convergence Rate O(1\/k2). 269","author":"Nesterov E.","year":"1983","unstructured":"Y.\u00a0E. Nesterov. 1983. A Method for Solving the Convex Programming Problem with Convergence Rate O(1\/k2). 269 (1983), 543\u2013547. https:\/\/ci.nii.ac.jp\/naid\/10029946121\/en\/"},{"key":"e_1_3_2_1_27_1","volume-title":"Pytorch: An Imperative Style","author":"Paszke Adam","year":"2019","unstructured":"Adam Paszke, Sam Gross, Francisco Massa, Adam Lerer, James Bradbury, Gregory Chanan, Trevor Killeen, Zeming Lin, Natalia Gimelshein, Luca Antiga, Alban Desmaison, Andreas Kopf, Edward Yang, Zachary DeVito, Martin Raison, Alykhan Tejani, Sasank Chilamkurthy, Benoit Steiner, Lu Fang, Junjie Bai, and Soumith Chintala. 2019. Pytorch: An Imperative Style, High-performance Deep Learning Library. In Advances in Neural Information Processing Systems 32, H.\u00a0Wallach, H.\u00a0Larochelle, A.\u00a0Beygelzimer, F.\u00a0d'Alch\u00e9-Buc, E.\u00a0Fox, and R.\u00a0Garnett (Eds.). Curran Associates, Inc., 8024\u20138035. http:\/\/papers.neurips.cc\/paper\/9015-pytorch-an-imperative-style-high-performance-deep-learning-library.pdf"},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"publisher","unstructured":"B.\u00a0T. Polyak. 1964-01. Some Methods of Speeding up the Convergence of Iteration Methods. 4 5 (1964-01) 1\u201317. https:\/\/doi.org\/10.1016\/0041-5553(64)90137-5","DOI":"10.1016\/0041-5553(64)90137-5"},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.1145\/2939672.2945397"},{"key":"e_1_3_2_1_30_1","volume-title":"3rdInternational Conference on Learning Representations, ICLR","author":"Simonyan Karen","year":"2015","unstructured":"Karen Simonyan and Andrew Zisserman. 2015. Very Deep Convolutional Networks for Large-scale Image Recognition. In 3rdInternational Conference on Learning Representations, ICLR 2015, San Diego, CA, USA, May 7-9, 2015, Conference Track Proceedings, Yoshua Bengio and Yann LeCun (Eds.). arxiv:1409.1556"},{"key":"e_1_3_2_1_31_1","volume-title":"Proceedings of the 36th International Conference on Machine Learning(Proceedings of Machine Learning Research, Vol.\u00a097)","author":"Spring Ryan","year":"2019","unstructured":"Ryan Spring, Anastasios Kyrillidis, Vijai Mohan, and Anshumali Shrivastava. 2019. Compressing Gradient Optimizers Via Count-sketches. In Proceedings of the 36th International Conference on Machine Learning(Proceedings of Machine Learning Research, Vol.\u00a097), Kamalika Chaudhuri and Ruslan Salakhutdinov (Eds.). PMLR, 5946\u20135955. https:\/\/proceedings.mlr.press\/v97\/spring19a.html"},{"key":"e_1_3_2_1_32_1","volume-title":"Adaptive Braking for Mitigating Gradient Delay. abs\/2007.01397","author":"Venigalla Abhinav","year":"2020","unstructured":"Abhinav Venigalla, Atli Kosson, Vitaliy Chiley, and Urs K\u00f6ster. 2020. Adaptive Braking for Mitigating Gradient Delay. abs\/2007.01397 (2020). arxiv:2007.01397"},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"publisher","DOI":"10.1145\/3225058.3225069"},{"key":"e_1_3_2_1_34_1","unstructured":"Sixin Zhang Anna Choromanska and Yann Lecun. 2015-12. Deep Learning with Elastic Averaging Sgd. In Advances in Neural Information Processing Systems Vol.\u00a02015-Janua. Neural information processing systems foundation 685\u2013693. arxiv:1412.6651"},{"key":"e_1_3_2_1_35_1","doi-asserted-by":"publisher","DOI":"10.5555\/3060832.3060950"},{"key":"e_1_3_2_1_36_1","volume-title":"Proceedings of the 34th International Conference on Machine Learning(Proceedings of Machine Learning Research, Vol.\u00a070)","author":"Zheng Shuxin","year":"2017","unstructured":"Shuxin Zheng, Qi Meng, Taifeng Wang, Wei Chen, Nenghai Yu, Zhi-Ming Ma, and Tie-Yan Liu. 2017. Asynchronous Stochastic Gradient Descent with Delay Compensation. In Proceedings of the 34th International Conference on Machine Learning(Proceedings of Machine Learning Research, Vol.\u00a070), Doina Precupand Yee\u00a0Whye Teh (Eds.). PMLR, 4120\u20134129. https:\/\/proceedings.mlr.press\/v70\/zheng17b.html"}],"event":{"name":"ICPP '22: 51st International Conference on Parallel Processing","acronym":"ICPP '22","location":"Bordeaux France"},"container-title":["Proceedings of the 51st International Conference on Parallel Processing"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3545008.3545010","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3545008.3545010","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T19:02:43Z","timestamp":1750186963000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3545008.3545010"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2022,8,29]]},"references-count":36,"alternative-id":["10.1145\/3545008.3545010","10.1145\/3545008"],"URL":"https:\/\/doi.org\/10.1145\/3545008.3545010","relation":{},"subject":[],"published":{"date-parts":[[2022,8,29]]},"assertion":[{"value":"2023-01-13","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}