{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,9,13]],"date-time":"2026-09-13T12:54:37Z","timestamp":1789304077669,"version":"build-2803163510"},"reference-count":48,"publisher":"Springer Science and Business Media LLC","issue":"6","license":[{"start":{"date-parts":[[2023,5,22]],"date-time":"2023-05-22T00:00:00Z","timestamp":1684713600000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2023,5,22]],"date-time":"2023-05-22T00:00:00Z","timestamp":1684713600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Sci. China Inf. Sci."],"published-print":{"date-parts":[[2023,6]]},"DOI":"10.1007\/s11432-021-3532-8","type":"journal-article","created":{"date-parts":[[2023,5,27]],"date-time":"2023-05-27T09:02:32Z","timestamp":1685178152000},"update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":6,"title":["Accelerate distributed deep learning with cluster-aware sketch quantization"],"prefix":"10.1007","volume":"66","author":[{"given":"Keshi","family":"Ge","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Yiming","family":"Zhang","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Yongquan","family":"Fu","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Zhiquan","family":"Lai","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Xiaoge","family":"Deng","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Dongsheng","family":"Li","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2023,5,22]]},"reference":[{"key":"3532_CR1","doi-asserted-by":"publisher","first-page":"400","DOI":"10.1214\/aoms\/1177729586","volume":"22","author":"H Robbins","year":"1951","unstructured":"Robbins H, Monro S. A stochastic approximation method. Ann Math Statist, 1951, 22: 400\u2013407","journal-title":"Ann Math Statist"},{"key":"3532_CR2","first-page":"2121","volume":"12","author":"J Duchi","year":"2011","unstructured":"Duchi J, Hazan E, Singer Y. Adaptive subgradient methods for online learning and stochastic optimization. J Mach Learning Res, 2011, 12: 2121\u20132159","journal-title":"J Mach Learning Res"},{"key":"3532_CR3","unstructured":"Sutskever I, Martens J, Dahl G, et al. On the importance of initialization and momentum in deep learning. In: Proceedings of International Conference on Machine Learning, 2013. 1139\u20131147"},{"key":"3532_CR4","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1145\/3320060","volume":"52","author":"T Ben-Nun","year":"2020","unstructured":"Ben-Nun T, Hoefler T. Demystifying parallel and distributed deep learning. ACM Comput Surv, 2020, 52: 1\u201343","journal-title":"ACM Comput Surv"},{"key":"3532_CR5","unstructured":"Dean J, Corrado G, Monga R, et al. Large scale distributed deep networks. In: Proceedings of Conference and Workshop on Neural Information Processing Systems, 2012. 1223\u20131231"},{"key":"3532_CR6","doi-asserted-by":"crossref","unstructured":"He K, Zhang X, Ren S, et al. Deep residual learning for image recognition. In: Proceedings of IEEE Conference on Computer Vision and Pattern Recognition (CVPR), 2016. 770\u2013778","DOI":"10.1109\/CVPR.2016.90"},{"key":"3532_CR7","doi-asserted-by":"crossref","unstructured":"Mahajan D, Girshick R, Ramanathan V, et al. Exploring the limits of weakly supervised pretraining. In: Proceedings of European Conference on Computer Vision, 2018. 181\u2013196","DOI":"10.1007\/978-3-030-01216-8_12"},{"key":"3532_CR8","unstructured":"Devlin J, Chang M W, Lee K, et al. BERT: pre-training of deep bidirectional transformers for language understanding. 2018. ArXiv:1810.04805"},{"key":"3532_CR9","unstructured":"Tan M, Le Q. EfficientNet: rethinking model scaling for convolutional neural networks. In: Proceedings of International Conference on Machine Learning, 2019. 6105\u20136114"},{"key":"3532_CR10","doi-asserted-by":"crossref","unstructured":"Carion N, Massa F, Synnaeve G, et al. End-to-end object detection with transformers. In: Proceedings of European Conference on Computer Vision, 2020. 213\u2013229","DOI":"10.1007\/978-3-030-58452-8_13"},{"key":"3532_CR11","doi-asserted-by":"crossref","unstructured":"Xie Q, Luong M T, Hovy E, et al. Self-training with noisy student improves ImageNet classification. In: Proceedings of IEEE\/CVF Conference on Computer Vision and Pattern Recognition (CVPR), 2020. 10687\u201310698","DOI":"10.1109\/CVPR42600.2020.01070"},{"key":"3532_CR12","unstructured":"Alistarh D, Grubic D, Li J, et al. QSGD: communication-efficient SGD via gradient quantization and encoding. In: Proceedings of Conference and Workshop on Neural Information Processing Systems, 2017. 1709\u20131720"},{"key":"3532_CR13","unstructured":"Wen W, Xu C, Yan F, et al. TernGrad: ternary gradients to reduce communication in distributed deep learning. In: Proceedings of Conference and Workshop on Neural Information Processing Systems, 2017. 1509\u20131519"},{"key":"3532_CR14","first-page":"3174","volume":"33","author":"F Faghri","year":"2020","unstructured":"Faghri F, Tabrizian I, Markov I, et al. Adaptive gradient quantization for data-parallel SGD. In: Proceedings of the 34th International Conference on Neural Information Processing Systems, 2020. 33: 3174\u20133185","journal-title":"Proceedings of the 34th International Conference on Neural Information Processing Systems"},{"key":"3532_CR15","unstructured":"Johnson R, Zhang T. Accelerating stochastic gradient descent using predictive variance reduction. In: Proceedings of Conference and Workshop on Neural Information Processing Systems, 2013. 315\u2013323"},{"key":"3532_CR16","unstructured":"Wang H, Sievert S, Liu S, et al. ATOMO: communication-efficient learning via atomic sparsification. In: Proceedings of Conference and Workshop on Neural Information Processing Systems, 2018. 9850\u20139861"},{"key":"3532_CR17","doi-asserted-by":"crossref","unstructured":"Li M, Andersen D G, Park J W, et al. Scaling distributed machine learning with the parameter server. In: Proceedings of the 11th USENIX Symposium on Operating Systems Design and Implementation, 2014. 583\u2013598","DOI":"10.1145\/2640087.2644155"},{"key":"3532_CR18","doi-asserted-by":"publisher","first-page":"58","DOI":"10.1016\/j.jalgor.2003.12.001","volume":"55","author":"G Cormode","year":"2005","unstructured":"Cormode G, Muthukrishnan S. An improved data stream summary: the count-min sketch and its applications. J Algorithms, 2005, 55: 58\u201375","journal-title":"J Algorithms"},{"key":"3532_CR19","doi-asserted-by":"publisher","first-page":"97","DOI":"10.1145\/1562764.1562789","volume":"52","author":"G Cormode","year":"2009","unstructured":"Cormode G, Hadjieleftheriou M. Finding the frequent items in streams of data. Commun ACM, 2009, 52: 97\u2013105","journal-title":"Commun ACM"},{"key":"3532_CR20","doi-asserted-by":"crossref","unstructured":"Fu Y, Li D, Shen S, et al. Clustering-preserving network flow sketching. In: Proceedings of IEEE INFOCOM 2020, 2020. 1309\u20131318","DOI":"10.1109\/INFOCOM41043.2020.9155388"},{"key":"3532_CR21","unstructured":"Karimireddy S P, Rebjock Q, Stich S, et al. Error feedback fixes SIGNSGD and other gradient compression schemes. In: Proceedings of International Conference on Machine Learning, 2019. 3252\u20133261"},{"key":"3532_CR22","doi-asserted-by":"crossref","unstructured":"Xu H, Ho C Y, Abdelmoniem A M, et al. GRACE: a compressed communication framework for distributed machine learning. In: Proceedings of 2021 IEEE 41st International Conference on Distributed Computing Systems, 2021. 561\u2013572","DOI":"10.1109\/ICDCS51616.2021.00060"},{"key":"3532_CR23","doi-asserted-by":"crossref","unstructured":"You Y, Bulu\u00e7 A, Demmel J. Scaling deep learning on GPU and knights landing clusters. In: Proceedings of the International Conference for High Performance Computing, Networking, Storage and Analysis, New York, 2017","DOI":"10.1145\/3126908.3126912"},{"key":"3532_CR24","unstructured":"Abadi M, Barham P, Chen J, et al. TensorFlow: a system for large-scale machine learning. In: Proceedings of the 12th USENIX Symposium on Operating Systems Design and Implementation, 2016. 265\u2013283"},{"key":"3532_CR25","unstructured":"Zhang H, Zheng Z, Xu S, et al. Poseidon: an efficient communication architecture for distributed deep learning on GPU clusters. In: Proceedings of 2017 USENIX Annual Technical Conference (USENIX ATC 17), 2017. 181\u2013193"},{"key":"3532_CR26","unstructured":"Li Y, Yu M, Li S, et al. Pipe-SGD: a decentralized pipelined SGD framework for distributed deep net training. In: Proceedings of the 32nd International Conference on Neural Information Processing Systems, 2018. 8056\u20138067"},{"key":"3532_CR27","unstructured":"Huang Y, Cheng Y, Bapna A, et al. GPipe: efficient training of giant neural networks using pipeline parallelism. 2018. ArXiv:1811.06965"},{"key":"3532_CR28","doi-asserted-by":"crossref","unstructured":"Shi S, Chu X, Li B. MG-WFBP: efficient data communication for distributed synchronous SGD algorithms. In: Proceedings of IEEE INFOCOM 2019-IEEE Conference on Computer Communications, 2019. 172\u2013180","DOI":"10.1109\/INFOCOM.2019.8737367"},{"key":"3532_CR29","doi-asserted-by":"crossref","unstructured":"Narayanan D, Harlap A, Phanishayee A, et al. PipeDream: generalized pipeline parallelism for DNN training. In: Proceedings of the 27th ACM Symposium on Operating Systems Principles, 2019. 1\u201315","DOI":"10.1145\/3341301.3359646"},{"key":"3532_CR30","doi-asserted-by":"crossref","unstructured":"Rasley J, Rajbhandari S, Ruwase O, et al. DeepSpeed: system optimizations enable training deep learning models with over 100 billion parameters. In: Proceedings of the 26th ACM SIGKDD International Conference on Knowledge Discovery & Data Mining, 2020. 3505\u20133506","DOI":"10.1145\/3394486.3406703"},{"key":"3532_CR31","doi-asserted-by":"crossref","unstructured":"Fan S, Rong Y, Meng C, et al. DAPPLE: a pipelined data parallel approach for training large models. In: Proceedings of the 26th ACM SIGPLAN Symposium on Principles and Practice of Parallel Programming, 2021. 431\u2013445","DOI":"10.1145\/3437801.3441593"},{"key":"3532_CR32","unstructured":"Bernstein J, Wang Y X, Azizzadenesheli K, et al. SIGNSGD: compressed optimisation for non-convex problems. In: Proceedings of International Conference on Machine Learning, 2018. 560\u2013569"},{"key":"3532_CR33","unstructured":"Stich S U, Cordonnier J B, Jaggi M. Sparsified SGD with memory. In: Proceedings of Conference and Workshop on Neural Information Processing Systems, 2018. 4447\u20134458"},{"key":"3532_CR34","unstructured":"Lin Y, Han S, Mao H, et al. Deep gradient compression: reducing the communication bandwidth for distributed training. In: Proceedings of International Conference on Learning Representations, 2018"},{"key":"3532_CR35","unstructured":"Wangni J, Wang J, Liu J, et al. Gradient sparsification for communication-efficient distributed optimization. In: Proceedings of Conference and Workshop on Neural Information Processing Systems, 2018. 1299\u20131309"},{"key":"3532_CR36","doi-asserted-by":"crossref","unstructured":"Renggli C, Ashkboos S, Aghagolzadeh M, et al. SparCML: high-performance sparse communication for machine learning. In: Proceedings of International Conference for High Performance Computing, Networking, Storage and Analysis, 2019. 1\u201315","DOI":"10.1145\/3295500.3356222"},{"key":"3532_CR37","doi-asserted-by":"crossref","unstructured":"Shi S, Zhao K, Wang Q, et al. A convergence analysis of distributed SGD with communication-efficient gradient sparsification. In: Proceedings of International Joint Conference on Artificial Intelligence, 2019. 3411\u20133417","DOI":"10.24963\/ijcai.2019\/473"},{"key":"3532_CR38","doi-asserted-by":"crossref","unstructured":"Chen C Y, Ni J, Lu S, et al. ScaleCom: scalable sparsified gradient compression for communication-efficient distributed training. In: Proceedings of Conference and Workshop on Neural Information Processing Systems, 2020","DOI":"10.1109\/SPAWC48557.2020.9153887"},{"key":"3532_CR39","unstructured":"Yu M, Lin Z, Narra K, et al. GradiVeQ: vector quantization for bandwidth-efficient gradient aggregation in distributed CNN training. In: Proceedings of Conference and Workshop on Neural Information Processing Systems, 2018. 5123\u20135133"},{"key":"3532_CR40","unstructured":"Vogels T, Karimireddy S P, Jaggi M. PowerSGD: practical low-rank gradient compression for distributed optimization. In: Proceedings of Conference and Workshop on Neural Information Processing Systems, 2019. 14259\u201314268"},{"key":"3532_CR41","unstructured":"Zhang H, Li J, Kara K, et al. ZipML: training linear models with end-to-end low precision, and a little bit of deep learning. In: Proceedings of International Conference on Machine Learning, 2017. 4035\u20134043"},{"key":"3532_CR42","unstructured":"Ramezani-Kebrya A, Faghri F, Roy D M. NUQSGD: improved communication efficiency for data-parallel SGD via nonuniform quantization. 2019. ArXiv:1908.06077"},{"key":"3532_CR43","unstructured":"Horvath S, Ho C Y, Horvath L, et al. Natural compression for distributed deep learning. 2019. ArXiv:1905.10988"},{"key":"3532_CR44","unstructured":"Fu F, Hu Y, He Y, et al. Don\u2019t waste your bits! Squeeze activations and gradients for deep neural networks via tinyscript. In: Proceedings of International Conference on Machine Learning, 2020. 3304\u20133314"},{"key":"3532_CR45","doi-asserted-by":"publisher","first-page":"2572","DOI":"10.1109\/JSAC.2021.3087262","volume":"39","author":"L Cui","year":"2021","unstructured":"Cui L, Su X, Zhou Y, et al. Slashing communication traffic in federated learning by transmitting clustered model updates. IEEE J Sel Areas Commun, 2021, 39: 2572\u20132589","journal-title":"IEEE J Sel Areas Commun"},{"key":"3532_CR46","unstructured":"Ivkin N, Rothchild D, Ullah E, et al. Communication-efficient distributed SGD with sketching. In: Proceedings of Conference and Workshop on Neural Information Processing Systems, 2019. 13144\u201313154"},{"key":"3532_CR47","unstructured":"Rothchild D, Panda A, Ullah E, et al. FetchSGD: communication-efficient federated learning with sketching. In: Proceedings of International Conference on Machine Learning, 2020. 8253\u20138265"},{"key":"3532_CR48","doi-asserted-by":"publisher","first-page":"945","DOI":"10.1007\/s00778-019-00596-3","volume":"29","author":"J Jiang","year":"2020","unstructured":"Jiang J, Fu F, Yang T, et al. SKCompress: compressing sparse and nonuniform gradient in distributed machine learning. VLDB J, 2020, 29: 945\u2013972","journal-title":"VLDB J"}],"container-title":["Science China Information Sciences"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s11432-021-3532-8.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s11432-021-3532-8\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s11432-021-3532-8.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,7,19]],"date-time":"2024-07-19T20:26:24Z","timestamp":1721420784000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s11432-021-3532-8"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023,5,22]]},"references-count":48,"journal-issue":{"issue":"6","published-print":{"date-parts":[[2023,6]]}},"alternative-id":["3532"],"URL":"https:\/\/doi.org\/10.1007\/s11432-021-3532-8","relation":{},"ISSN":["1674-733X","1869-1919"],"issn-type":[{"value":"1674-733X","type":"print"},{"value":"1869-1919","type":"electronic"}],"subject":[],"published":{"date-parts":[[2023,5,22]]},"assertion":[{"value":"5 September 2021","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"9 December 2021","order":2,"name":"revised","label":"Revised","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"10 May 2022","order":3,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"22 May 2023","order":4,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}}],"article-number":"162102"}}