{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,8,7]],"date-time":"2026-08-07T02:21:41Z","timestamp":1786069301041,"version":"3.56.0"},"reference-count":126,"publisher":"Institute of Electrical and Electronics Engineers (IEEE)","issue":"12","license":[{"start":{"date-parts":[[2023,12,1]],"date-time":"2023-12-01T00:00:00Z","timestamp":1701388800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/ieeexplore.ieee.org\/Xplorehelp\/downloads\/license-information\/IEEE.html"},{"start":{"date-parts":[[2023,12,1]],"date-time":"2023-12-01T00:00:00Z","timestamp":1701388800000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2023,12,1]],"date-time":"2023-12-01T00:00:00Z","timestamp":1701388800000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"}],"funder":[{"name":"National Key Research and Development Program of China","award":["2022YFB4501702"],"award-info":[{"award-number":["2022YFB4501702"]}]},{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["62302512"],"award-info":[{"award-number":["62302512"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]},{"name":"Excellent Youth Foundation of Hunan Province","award":["2021JJ10050"],"award-info":[{"award-number":["2021JJ10050"]}]}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["IEEE Trans. Parallel Distrib. Syst."],"published-print":{"date-parts":[[2023,12]]},"DOI":"10.1109\/tpds.2023.3323282","type":"journal-article","created":{"date-parts":[[2023,10,10]],"date-time":"2023-10-10T19:34:31Z","timestamp":1696966471000},"page":"3294-3308","source":"Crossref","is-referenced-by-count":13,"title":["Communication Optimization Algorithms for Distributed Deep Learning Systems: A Survey"],"prefix":"10.1109","volume":"34","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-2661-0889","authenticated-orcid":false,"given":"Enda","family":"Yu","sequence":"first","affiliation":[{"name":"College of Computer, National University of Defense Technology, Changsha, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-6243-8479","authenticated-orcid":false,"given":"Dezun","family":"Dong","sequence":"additional","affiliation":[{"name":"College of Computer, National University of Defense Technology, Changsha, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-6125-3330","authenticated-orcid":false,"given":"Xiangke","family":"Liao","sequence":"additional","affiliation":[{"name":"College of Computer, National University of Defense Technology, Changsha, China"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"263","reference":[{"key":"ref57","doi-asserted-by":"publisher","DOI":"10.1145\/3126908.3126912"},{"key":"ref56","first-page":"1","article-title":"Deep learning with elastic averaging SGD","author":"zhang","year":"2015","journal-title":"Proc Int Conf Neural Inf Process"},{"key":"ref59","doi-asserted-by":"publisher","DOI":"10.21437\/Interspeech.2014-274"},{"key":"ref58","doi-asserted-by":"publisher","DOI":"10.1145\/3417607"},{"key":"ref53","doi-asserted-by":"publisher","DOI":"10.1109\/ICDM.2016.0028"},{"key":"ref52","doi-asserted-by":"publisher","DOI":"10.1109\/ICDCS.2018.00020"},{"key":"ref55","first-page":"4120","article-title":"Asynchronous stochastic gradient descent with delay compensation","author":"zheng","year":"2017","journal-title":"Proc Int Conf Mach Learn"},{"key":"ref54","doi-asserted-by":"publisher","DOI":"10.1145\/3332466.3374528"},{"key":"ref51","first-page":"1","article-title":"More effective distributed ML via a stale synchronous parallel parameter server","author":"ho","year":"2013","journal-title":"Proc Int Conf Neural Inf Process"},{"key":"ref50","first-page":"1","article-title":"Solving the straggler problem with bounded staleness","author":"cipar","year":"2013","journal-title":"Proc 14th USENIX Conf Hot Topics Operating Syst"},{"key":"ref46","doi-asserted-by":"publisher","DOI":"10.1109\/TPDS.2021.3062721"},{"key":"ref45","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v33i01.33015289"},{"key":"ref48","first-page":"1","article-title":"Large scale distributed deep networks","author":"dean","year":"2012","journal-title":"Proc Int Conf Neural Inf Process"},{"key":"ref47","first-page":"1","article-title":"Hogwild!: A lock-free approach to parallelizing stochastic gradient descent","author":"recht","year":"2011","journal-title":"Proc Int Conf Neural Inf Process"},{"key":"ref42","doi-asserted-by":"publisher","DOI":"10.1109\/TPDS.2021.3052862"},{"key":"ref41","article-title":"Horovod: Fast and easy distributed deep learning in tensorflow","author":"sergeev","year":"2018"},{"key":"ref44","first-page":"1467","article-title":"Layer-wise adaptive gradient sparsification for distributed deep learning with convergence guarantees","author":"shi","year":"2020","journal-title":"Proc 26th Eur Conf Artif Intell"},{"key":"ref43","doi-asserted-by":"publisher","DOI":"10.1109\/INFOCOM42981.2021.9488803"},{"key":"ref49","first-page":"571","article-title":"Project adam: Building an efficient and scalable deep learning training system","author":"chilimbi","year":"2014","journal-title":"Proc 11th USENIX Conf Operating Syst Des Implementation"},{"key":"ref8","doi-asserted-by":"publisher","DOI":"10.1109\/MM.2008.31"},{"key":"ref7","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2009.5206848"},{"key":"ref9","doi-asserted-by":"publisher","DOI":"10.1145\/3079856.3080246"},{"key":"ref4","first-page":"1","article-title":"Very deep convolutional networks for large-scale image recognition","author":"simonyan","year":"2015","journal-title":"Proc Int Conf Learn Representations"},{"key":"ref3","first-page":"1106","article-title":"Imagenet classification with deep convolutional neural networks","author":"krizhevsky","year":"2012","journal-title":"Proc Int Conf Neural Inf Process"},{"key":"ref6","first-page":"740","article-title":"Microsoft COCO: Common objects in context","author":"lin","year":"2014","journal-title":"Proc Eur Conf Comput Vis"},{"key":"ref5","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2016.90"},{"key":"ref100","doi-asserted-by":"publisher","DOI":"10.1109\/ALLERTON.2018.8635933"},{"key":"ref101","first-page":"2545","article-title":"Trading redundancy for communication: Speeding up distributed SGD for non-convex optimization","author":"haddadpour","year":"2019","journal-title":"Proc Int Conf Mach Learn"},{"key":"ref40","doi-asserted-by":"publisher","DOI":"10.1145\/3472456.3472467"},{"key":"ref35","first-page":"418","article-title":"TicTac: Accelerating distributed deep learning with communication scheduling","volume":"1","author":"hashemi","year":"2019","journal-title":"Proc Mach Learn Syst"},{"key":"ref34","first-page":"1","article-title":"Optimizing network performance in distributed machine learning","author":"mai","year":"2015","journal-title":"Proc 7th USENIX Workshop Hot Topics Cloud Comput"},{"key":"ref37","doi-asserted-by":"publisher","DOI":"10.1145\/3341301.3359642"},{"key":"ref36","first-page":"132","article-title":"Priority-based parameter propagation for distributed DNN training","volume":"1","author":"jayarajan","year":"2019","journal-title":"Proc Mach Learn Syst"},{"key":"ref31","doi-asserted-by":"publisher","DOI":"10.1145\/3341301.3359646"},{"key":"ref30","article-title":"MXNet: A flexible and efficient machine learning library for heterogeneous distributed systems","author":"chen","year":"2015"},{"key":"ref33","doi-asserted-by":"publisher","DOI":"10.1145\/3458817.3476145"},{"key":"ref32","doi-asserted-by":"publisher","DOI":"10.1109\/SC41405.2020.00049"},{"key":"ref39","doi-asserted-by":"publisher","DOI":"10.1109\/TSC.2020.2985684"},{"key":"ref38","doi-asserted-by":"publisher","DOI":"10.1109\/INFOCOM41043.2020.9155282"},{"key":"ref24","doi-asserted-by":"publisher","DOI":"10.1145\/3394486.3406703"},{"key":"ref23","doi-asserted-by":"publisher","DOI":"10.1109\/IPDPS49936.2021.00071"},{"key":"ref26","doi-asserted-by":"publisher","DOI":"10.1109\/TNET.2021.3117042"},{"key":"ref25","article-title":"Megatron-LM: Training multi-billion parameter language models using model parallelism","author":"shoeybi","year":"2019"},{"key":"ref20","first-page":"1","article-title":"GPipe: Efficient training of giant neural networks using pipeline parallelism","author":"huang","year":"2019","journal-title":"Proc Int Conf Neural Inf Process"},{"key":"ref22","first-page":"1","article-title":"Beyond data and model parallelism for deep neural networks","volume":"1","author":"jia","year":"2019","journal-title":"Proc Mach Learn Syst"},{"key":"ref21","doi-asserted-by":"publisher","DOI":"10.1145\/3437801.3441593"},{"key":"ref28","first-page":"265","article-title":"TensorFlow: A system for large-scale machine learning","author":"abadi","year":"2016","journal-title":"Proc 12th USENIX Conf Operating Syst Des Implementation"},{"key":"ref27","article-title":"Bringing HPC techniques to deep learning","author":"gibiansky","year":"2017","journal-title":"Baidu Res"},{"key":"ref29","first-page":"1","article-title":"PyTorch: An imperative style, high-performance deep learning library","author":"paszke","year":"2019","journal-title":"Proc Int Conf Neural Inf Process"},{"key":"ref13","doi-asserted-by":"publisher","DOI":"10.1109\/INFOCOM41043.2020.9155269"},{"key":"ref12","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2016.284"},{"key":"ref15","doi-asserted-by":"publisher","DOI":"10.1145\/3363554"},{"key":"ref14","doi-asserted-by":"publisher","DOI":"10.1145\/3320060"},{"key":"ref97","first-page":"1","article-title":"Lighter-communication distributed machine learning via sufficient factor broadcasting","author":"xie","year":"2016","journal-title":"Proc Conf Uncertainty of Artificial Intelligence"},{"key":"ref126","first-page":"785","article-title":"Scaling distributed machine learning with in-network aggregation","author":"sapio","year":"2021","journal-title":"Proc 18th USENIX Symp Netw Syst Des Implementation"},{"key":"ref96","first-page":"1","article-title":"Communication complexity of distributed convex learning and optimization","author":"arjevani","year":"2015","journal-title":"Proc Int Conf Neural Inf Process"},{"key":"ref11","article-title":"PaLM: Scaling language modeling with pathways","author":"chowdhery","year":"2022"},{"key":"ref99","article-title":"Revisiting distributed synchronous SGD","author":"chen","year":"2016"},{"key":"ref124","doi-asserted-by":"publisher","DOI":"10.1145\/3502181.3531473"},{"key":"ref10","doi-asserted-by":"publisher","DOI":"10.1007\/s11023-020-09548-1"},{"key":"ref98","first-page":"344","article-title":"Stochastic gradient push for distributed deep learning","author":"assran","year":"2019","journal-title":"Proc Int Conf Mach Learn"},{"key":"ref125","doi-asserted-by":"publisher","DOI":"10.1109\/ICDE48307.2020.00124"},{"key":"ref17","first-page":"1","article-title":"Research progress on network performance optimization of distributed machine learning system","volume":"45","author":"wang","year":"2022","journal-title":"Proc 7th USENIX Conf Hot Topics Cloud Comput"},{"key":"ref16","doi-asserted-by":"publisher","DOI":"10.1016\/j.jpdc.2020.11.005"},{"key":"ref19","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-7908-2604-3_16"},{"key":"ref18","article-title":"Communication-efficient distributed deep learning: A comprehensive survey","author":"tang","year":"2020"},{"key":"ref93","article-title":"Yet another accelerated SGD: Resnet-50 training on imagenet in 74.7 seconds","author":"yamazaki","year":"2019"},{"key":"ref92","article-title":"Imagenet\/resnet-50 training in 224 seconds","author":"mikami","year":"2018"},{"key":"ref95","first-page":"456","article-title":"Distributed training strategies for the structured perceptron","author":"mcdonald","year":"2010","journal-title":"Proc Conf North Amer Chapter Assoc Comput Linguistics"},{"key":"ref94","first-page":"2595","article-title":"Parallelized stochastic gradient descent","author":"zinkevich","year":"2010","journal-title":"Proc Int Conf Neural Inf Process"},{"key":"ref91","first-page":"4171","article-title":"BERT: Pre-training of deep bidirectional transformers for language understanding","author":"devlin","year":"2019","journal-title":"Proc NAACL-HLT"},{"key":"ref90","doi-asserted-by":"publisher","DOI":"10.1109\/TPDS.2019.2913833"},{"key":"ref89","first-page":"1","article-title":"Large batch optimization for deep learning: Training bert in 76 minutes","author":"you","year":"2019","journal-title":"Proc Int Conf Learn Representations"},{"key":"ref86","doi-asserted-by":"publisher","DOI":"10.1145\/3369583.3392681"},{"key":"ref85","doi-asserted-by":"publisher","DOI":"10.1145\/3295500.3356222"},{"key":"ref88","doi-asserted-by":"publisher","DOI":"10.1145\/3295500.3356137"},{"key":"ref87","article-title":"Accurate, large minibatch SGD: Training imageNet in 1 hour","author":"goyal","year":"2017"},{"key":"ref82","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v32i1.11728"},{"key":"ref81","first-page":"1","article-title":"The convergence of sparsified gradient methods","author":"alistarh","year":"2018","journal-title":"Proc Int Conf Neural Inf Process"},{"key":"ref84","doi-asserted-by":"publisher","DOI":"10.1109\/IPCCC47392.2019.8958738"},{"key":"ref83","first-page":"297","article-title":"An efficient statistical-based gradient compression technique for distributed training systems","volume":"3","author":"abdelmoniem","year":"2021","journal-title":"Proc Mach Learn Syst"},{"key":"ref80","doi-asserted-by":"publisher","DOI":"10.1109\/TNNLS.2021.3084806"},{"key":"ref79","first-page":"1","article-title":"Gradient sparsification for communication-efficient distributed optimization","author":"wangni","year":"2018","journal-title":"Proc Int Conf Neural Inf Process"},{"key":"ref108","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v33i01.33015693"},{"key":"ref78","doi-asserted-by":"publisher","DOI":"10.1145\/3503221.3508399"},{"key":"ref109","doi-asserted-by":"publisher","DOI":"10.24963\/ijcai.2018\/447"},{"key":"ref106","first-page":"1","article-title":"Local SGD converges fast and communicates little","author":"stich","year":"2019","journal-title":"Proc Int Conf Learn Representations"},{"key":"ref107","first-page":"1","article-title":"Don't use large mini-batches, use local SGD","author":"lin","year":"2019","journal-title":"Proc Int Conf Learn Representations"},{"key":"ref75","first-page":"1","article-title":"A linear speedup analysis of distributed deep learning with sparse and quantized communication","author":"jiang","year":"2018","journal-title":"Proc Int Conf Neural Inf Process"},{"key":"ref104","article-title":"Extremely large minibatch SGD: Training resnet-50 on imagenet in 15 minutes","author":"akiba","year":"2017"},{"key":"ref74","first-page":"1","article-title":"Deep gradient compression: Reducing the communication bandwidth for distributed training","author":"lin","year":"2018","journal-title":"Proc Int Conf Learn Representations"},{"key":"ref105","article-title":"Image classification at supercomputer scale","author":"ying","year":"2018"},{"key":"ref77","doi-asserted-by":"publisher","DOI":"10.1109\/ICDCS.2019.00220"},{"key":"ref102","article-title":"Powerai DDL","author":"cho","year":"2017"},{"key":"ref76","doi-asserted-by":"publisher","DOI":"10.1016\/j.jpdc.2019.05.016"},{"key":"ref103","article-title":"Don't decay the learning rate, increase the batch size","author":"smith","year":"2017"},{"key":"ref2","doi-asserted-by":"publisher","DOI":"10.1145\/2959100.2959190"},{"key":"ref1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPRW56347.2022.00309"},{"key":"ref71","first-page":"145","article-title":"Synchronous multi-GPU deep learning with low-precision communication: An experimental study","author":"grubic","year":"2018","journal-title":"Proc Intl Conf Extending Database Technology"},{"key":"ref111","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v35i11.17153"},{"key":"ref70","doi-asserted-by":"publisher","DOI":"10.1016\/j.sysarc.2023.102927"},{"key":"ref112","first-page":"9709","article-title":"Cooperative SGD: A unified framework for the design and analysis of local-update SGD algorithms","volume":"22","author":"wang","year":"2021","journal-title":"J Mach Learn Res"},{"key":"ref73","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D17-1045"},{"key":"ref72","doi-asserted-by":"publisher","DOI":"10.21437\/Interspeech.2015-354"},{"key":"ref110","first-page":"1","article-title":"Local SGD with periodic averaging: Tighter analysis and adaptive synchronization","author":"haddadpour","year":"2019","journal-title":"Proc Int Conf Neural Inf Process"},{"key":"ref68","first-page":"1","article-title":"ATOMO: Communication-efficient learning via atomic sparsification","author":"wang","year":"2018","journal-title":"Proc Int Conf Neural Inf Process"},{"key":"ref119","doi-asserted-by":"publisher","DOI":"10.1109\/ICASSP40776.2020.9053834"},{"key":"ref67","first-page":"1","article-title":"GradiVeQ: Vector quantization for bandwidth-efficient gradient aggregation in distributed CNN training","author":"yu","year":"2018","journal-title":"Proc Int Conf Neural Inf Process"},{"key":"ref117","doi-asserted-by":"publisher","DOI":"10.1016\/j.jpdc.2022.05.014"},{"key":"ref69","first-page":"1","article-title":"PowerSGD: Practical low-rank gradient compression for distributed optimization","author":"vogels","year":"2019","journal-title":"Proc Int Conf Neural Inf Process"},{"key":"ref118","article-title":"DaSGD: Squeezing SGD parallelization performance in distributed training using delayed averaging","author":"zhou","year":"2020"},{"key":"ref64","first-page":"560","article-title":"signSGD: Compressed optimisation for non-convex problems","author":"bernstein","year":"2018","journal-title":"Proc Int Conf Mach Learn"},{"key":"ref115","doi-asserted-by":"publisher","DOI":"10.24963\/ijcai.2019\/637"},{"key":"ref63","first-page":"4035","article-title":"ZipML: Training linear models with end-to-end low precision, and a little bit of deep learning","author":"zhang","year":"2017","journal-title":"Proc Int Conf Mach Learn"},{"key":"ref116","doi-asserted-by":"publisher","DOI":"10.1145\/3472456.3472508"},{"key":"ref66","first-page":"53","article-title":"3LC: Lightweight and effective traffic compression for distributed machine learning","volume":"1","author":"lim","year":"2019","journal-title":"Proc Mach Learn Syst"},{"key":"ref113","first-page":"1","article-title":"Communication-efficient SGD: From local SGD to one-shot averaging","author":"spiridonoff","year":"2021","journal-title":"Proc Int Conf Neural Inf Process"},{"key":"ref65","first-page":"3252","article-title":"Error feedback fixes signSGD and other gradient compression schemes","author":"karimireddy","year":"2019","journal-title":"Proc Int Conf Mach Learn"},{"key":"ref114","first-page":"5332","article-title":"Federated accelerated stochastic gradient descent","author":"yuan","year":"2020","journal-title":"Proc Int Conf Neural Inf Process"},{"key":"ref60","first-page":"1","article-title":"QSGD: Communication-efficient SGD via gradient quantization and encoding","author":"alistarh","year":"2017","journal-title":"Proc Int Conf Neural Inf Process"},{"key":"ref122","first-page":"1","article-title":"Communication-efficient distributed learning via lazily aggregated quantized gradients","author":"sun","year":"2019","journal-title":"Proc Int Conf Neural Inf Process"},{"key":"ref123","article-title":"C-Coll: Introducing error-bounded lossy compression into MPI collectives","author":"huang","year":"2023"},{"key":"ref62","first-page":"5325","article-title":"Error compensated quantized SGD and its applications to large-scale distributed optimization","author":"wu","year":"2018","journal-title":"Proc Int Conf Mach Learn"},{"key":"ref120","doi-asserted-by":"publisher","DOI":"10.1109\/IJCNN.2019.8852172"},{"key":"ref61","first-page":"1509","article-title":"TernGrad: Ternary gradients to reduce communication in distributed deep learning","author":"wen","year":"2017","journal-title":"Proc Int Conf Neural Inf Process"},{"key":"ref121","first-page":"2021","article-title":"FedPAQ: A communication-efficient federated learning method with periodic averaging and quantization","author":"reisizadeh","year":"2020","journal-title":"Proc Int Conf Artif Intell Statist"}],"container-title":["IEEE Transactions on Parallel and Distributed Systems"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx7\/71\/10272650\/10275049.pdf?arnumber=10275049","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2023,12,11]],"date-time":"2023-12-11T20:52:05Z","timestamp":1702327925000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/10275049\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023,12]]},"references-count":126,"journal-issue":{"issue":"12"},"URL":"https:\/\/doi.org\/10.1109\/tpds.2023.3323282","relation":{},"ISSN":["1045-9219","1558-2183","2161-9883"],"issn-type":[{"value":"1045-9219","type":"print"},{"value":"1558-2183","type":"electronic"},{"value":"2161-9883","type":"electronic"}],"subject":[],"published":{"date-parts":[[2023,12]]}}}