{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,3,22]],"date-time":"2026-03-22T06:39:53Z","timestamp":1774161593333,"version":"3.50.1"},"reference-count":58,"publisher":"Institute of Electrical and Electronics Engineers (IEEE)","issue":"8","funder":[{"DOI":"10.13039\/501100012166","name":"National Key Research and Development Program of China","doi-asserted-by":"publisher","award":["2016YFB0800402"],"award-info":[{"award-number":["2016YFB0800402"]}],"id":[{"id":"10.13039\/501100012166","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100003012","name":"Funding from Hong Kong RGC Research Impact Fund","doi-asserted-by":"publisher","award":["R5060-19"],"award-info":[{"award-number":["R5060-19"]}],"id":[{"id":"10.13039\/100003012","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100003012","name":"Funding from Hong Kong RGC Research Impact Fund","doi-asserted-by":"publisher","award":["R5034-18"],"award-info":[{"award-number":["R5034-18"]}],"id":[{"id":"10.13039\/100003012","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100002920","name":"General Research Fund","doi-asserted-by":"publisher","award":["152221\/19E"],"award-info":[{"award-number":["152221\/19E"]}],"id":[{"id":"10.13039\/501100002920","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100002920","name":"General Research Fund","doi-asserted-by":"publisher","award":["15220320\/20E"],"award-info":[{"award-number":["15220320\/20E"]}],"id":[{"id":"10.13039\/501100002920","id-type":"DOI","asserted-by":"publisher"}]},{"name":"Collaborative Research Fund","award":["C5026-18G"],"award-info":[{"award-number":["C5026-18G"]}]},{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["61872310"],"award-info":[{"award-number":["61872310"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["U1836204"],"award-info":[{"award-number":["U1836204"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["U1936108"],"award-info":[{"award-number":["U1936108"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100010877","name":"Shenzhen Science and Technology Innovation Commission","doi-asserted-by":"publisher","award":["R2020A045"],"award-info":[{"award-number":["R2020A045"]}],"id":[{"id":"10.13039\/501100010877","id-type":"DOI","asserted-by":"publisher"}]},{"name":"Shenzhen Basic Research Funding Scheme","award":["JCYJ20170818103849343"],"award-info":[{"award-number":["JCYJ20170818103849343"]}]},{"DOI":"10.13039\/501100002858","name":"China Postdoctoral Science Foundation","doi-asserted-by":"publisher","award":["2019M661709"],"award-info":[{"award-number":["2019M661709"]}],"id":[{"id":"10.13039\/501100002858","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["IEEE J. Select. Areas Commun."],"published-print":{"date-parts":[[2021,8]]},"DOI":"10.1109\/jsac.2021.3087272","type":"journal-article","created":{"date-parts":[[2021,6,8]],"date-time":"2021-06-08T11:20:48Z","timestamp":1623151248000},"page":"2541-2557","source":"Crossref","is-referenced-by-count":21,"title":["LOSP: Overlap Synchronization Parallel With Local Compensation for Fast Distributed Training"],"prefix":"10.1109","volume":"39","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-7591-5315","authenticated-orcid":false,"given":"Haozhao","family":"Wang","sequence":"first","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0001-7538-1985","authenticated-orcid":false,"given":"Zhihao","family":"Qu","sequence":"additional","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0001-9831-2202","authenticated-orcid":false,"given":"Song","family":"Guo","sequence":"additional","affiliation":[]},{"given":"Ningqi","family":"Wang","sequence":"additional","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0002-7791-5511","authenticated-orcid":false,"given":"Ruixuan","family":"Li","sequence":"additional","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0003-0488-511X","authenticated-orcid":false,"given":"Weihua","family":"Zhuang","sequence":"additional","affiliation":[]}],"member":"263","reference":[{"key":"ref39","doi-asserted-by":"crossref","first-page":"2184","DOI":"10.1109\/JSAC.2019.2933969","article-title":"Machine learning in the air","volume":"37","author":"g\u00fcnd\u00fcz","year":"2019","journal-title":"IEEE J Sel Areas Commun"},{"key":"ref38","doi-asserted-by":"publisher","DOI":"10.1109\/TC.2019.2931716"},{"key":"ref33","first-page":"8386","article-title":"Robust and communication-efficient collaborative learning","author":"reisizadeh","year":"2019","journal-title":"Proc Adv Neural Inf Process Syst Annu Conf Neural Inf Process Syst (NeurIPS)"},{"key":"ref32","doi-asserted-by":"publisher","DOI":"10.1145\/3274808.3274828"},{"key":"ref31","doi-asserted-by":"publisher","DOI":"10.1109\/ICDCS.2019.00150"},{"key":"ref30","first-page":"1998","article-title":"Gradient diversity: A key ingredient for scalable distributed learning","author":"yin","year":"2018","journal-title":"Proc Int Conf Artif Intell Statist (AISTATS)"},{"key":"ref37","doi-asserted-by":"publisher","DOI":"10.1109\/MICRO.2018.00023"},{"key":"ref36","first-page":"2165","article-title":"Order optimal one-shot distributed learning","author":"sharif-nassab","year":"2019","journal-title":"Proc Adv Neural Inf Process Syst Annu Conf Neural Inf Process Syst (NeurIPS)"},{"key":"ref35","doi-asserted-by":"publisher","DOI":"10.1109\/TC.2017.2780237"},{"key":"ref34","doi-asserted-by":"publisher","DOI":"10.1109\/INFOCOM.2019.8737587"},{"key":"ref28","doi-asserted-by":"publisher","DOI":"10.1145\/3225058.3225069"},{"key":"ref27","article-title":"Scaling SGD batch size to 32k for imagenet training","volume":"abs 1708 3888","author":"you","year":"2017","journal-title":"CoRR"},{"key":"ref29","article-title":"Extremely large minibatch SGD: Training resnet-50 on imagenet in 15 minutes","volume":"abs 1711 4325","author":"akiba","year":"2017","journal-title":"CoRR"},{"key":"ref2","doi-asserted-by":"publisher","DOI":"10.1109\/ICASSP.2013.6638947"},{"key":"ref1","doi-asserted-by":"publisher","DOI":"10.1145\/3337821.3337828"},{"key":"ref20","doi-asserted-by":"publisher","DOI":"10.1109\/TNET.2018.2814779"},{"key":"ref22","article-title":"Local SGD converges fast and communicates little","author":"stich","year":"2019","journal-title":"Proc 7th Int Conf Learn Represent (ICLR)"},{"key":"ref21","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v33i01.33015693"},{"key":"ref24","doi-asserted-by":"publisher","DOI":"10.1109\/JSAC.2019.2904348"},{"key":"ref23","article-title":"Cooperative SGD: A unified framework for the design and analysis of communication-efficient SGD algorithms","author":"wang","year":"2019","journal-title":"Proc Workshop Int Conf Mach Learn (ICML Workshop)"},{"key":"ref26","doi-asserted-by":"publisher","DOI":"10.1109\/JSAC.2019.2934002"},{"key":"ref25","article-title":"Adaptive communication strategies to achieve the best error-runtime trade-off in local-update SGD","author":"wang","year":"2019","journal-title":"Proc Mach Learn Syst (MLSys)"},{"key":"ref50","year":"2020","journal-title":"Source Code for MnistCNN"},{"key":"ref51","article-title":"Learning multiple layers of features from tiny images","author":"krizhevsky","year":"2009"},{"key":"ref58","author":"wang","year":"2020","journal-title":"Source Code for LOSP"},{"key":"ref57","year":"2021","journal-title":"Aliyun"},{"key":"ref56","article-title":"Tying word vectors and word classifiers: A loss framework for language modeling","author":"inan","year":"2017","journal-title":"Proc 5th Int Conf Learn Represent (ICLR)"},{"key":"ref55","doi-asserted-by":"publisher","DOI":"10.21236\/ADA273556"},{"key":"ref54","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2017.243"},{"key":"ref53","first-page":"1106","article-title":"ImageNet classification with deep convolutional neural networks","author":"krizhevsky","year":"2012","journal-title":"Proc Annu Conf Neural Inf Process Syst (NeurIPS)"},{"key":"ref52","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2016.90"},{"key":"ref10","first-page":"1232","article-title":"Large scale distributed deep networks","author":"dean","year":"2012","journal-title":"Proc Annu Conf Neural Inf Process Systems (NeurIPS)"},{"key":"ref11","article-title":"Solving the straggler problem with bounded staleness","author":"cipar","year":"2013","journal-title":"Proceedings of Workshop on Hot Topics in Operating Systems (HotOS)"},{"key":"ref40","first-page":"785","article-title":"Scaling distributed machine learning with in-network aggregation","author":"sapio","year":"2021","journal-title":"Proc 18th USENIX Symp Netw Syst Design Implement (NSDI)"},{"key":"ref12","first-page":"1223","article-title":"More effective distributed ML via a stale synchronous parallel parameter server","author":"ho","year":"2013","journal-title":"Proc Annu Conf Neural Inf Process Syst (NeurIPS)"},{"key":"ref13","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v33i01.33015289"},{"key":"ref14","first-page":"4120","article-title":"Asynchronous stochastic gradient descent with delay compensation","author":"zheng","year":"2017","journal-title":"Proc 34th Int Conf Mach Learn (ICML)"},{"key":"ref15","doi-asserted-by":"publisher","DOI":"10.1109\/TPDS.2017.2752706"},{"key":"ref16","doi-asserted-by":"publisher","DOI":"10.1109\/TC.2017.2777863"},{"key":"ref17","doi-asserted-by":"publisher","DOI":"10.1109\/TC.2018.2876312"},{"key":"ref18","doi-asserted-by":"publisher","DOI":"10.1109\/TPDS.2018.2866582"},{"key":"ref19","first-page":"5330","article-title":"Can decentralized algorithms outperform centralized algorithms? A case study for decentralized parallel stochastic gradient descent","author":"lian","year":"2017","journal-title":"Proc Adv Neural Inf Process Syst (NeurIPS)"},{"key":"ref4","doi-asserted-by":"publisher","DOI":"10.1016\/j.patrec.2018.02.010"},{"key":"ref3","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2014.223"},{"key":"ref6","first-page":"19","article-title":"Communication efficient distributed machine learning with the parameter server","volume":"27","author":"li","year":"2014","journal-title":"Proc Adv Neural Inf Process Syst"},{"key":"ref5","first-page":"583","article-title":"Scaling distributed machine learning with the parameter server","author":"li","year":"2014","journal-title":"Proc of USENIX Symp on Operating Systems Design and Implementation (OSDI)"},{"key":"ref8","first-page":"181","article-title":"Poseidon: An efficient communication architecture for distributed deep learning on GPU clusters","author":"zhang","year":"2017","journal-title":"Proc USENIX Annu Tech Conf (ATC)"},{"key":"ref7","doi-asserted-by":"publisher","DOI":"10.1145\/3064176.3064182"},{"key":"ref49","author":"lecun","year":"1998","journal-title":"The MNIST Database of Handwritten Digits"},{"key":"ref9","article-title":"Revisiting distributed synchronous SGD","volume":"abs 1604 981","author":"chen","year":"2016","journal-title":"CoRR"},{"key":"ref46","first-page":"2737","article-title":"Asynchronous parallel stochastic gradient for nonconvex optimization","author":"lian","year":"2015","journal-title":"Proc Annu Conf Neural Inf Process Syst (NeurIPS)"},{"key":"ref45","doi-asserted-by":"publisher","DOI":"10.1137\/16M1080173"},{"key":"ref48","doi-asserted-by":"publisher","DOI":"10.24963\/ijcai.2019\/637"},{"key":"ref47","doi-asserted-by":"publisher","DOI":"10.1145\/3035918.3035933"},{"key":"ref42","doi-asserted-by":"publisher","DOI":"10.1109\/ICASSP40776.2020.9053834"},{"key":"ref41","first-page":"344","article-title":"Stochastic gradient push for distributed deep learning","author":"assran","year":"2019","journal-title":"Proc 36th Int Conf Mach Learn (ICML)"},{"key":"ref44","first-page":"957","article-title":"AdaDelay: Delay Adaptive distributed stochastic optimization","author":"sra","year":"2016","journal-title":"Proc 19th Int Conf Artif Intell Statist (AISTATS)"},{"key":"ref43","doi-asserted-by":"publisher","DOI":"10.24963\/ijcai.2018\/447"}],"container-title":["IEEE Journal on Selected Areas in Communications"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx7\/49\/9486979\/09448017.pdf?arnumber=9448017","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2022,1,27]],"date-time":"2022-01-27T11:47:15Z","timestamp":1643284035000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/9448017\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2021,8]]},"references-count":58,"journal-issue":{"issue":"8"},"URL":"https:\/\/doi.org\/10.1109\/jsac.2021.3087272","relation":{},"ISSN":["0733-8716","1558-0008"],"issn-type":[{"value":"0733-8716","type":"print"},{"value":"1558-0008","type":"electronic"}],"subject":[],"published":{"date-parts":[[2021,8]]}}}