{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,1,18]],"date-time":"2026-01-18T07:10:28Z","timestamp":1768720228517,"version":"3.49.0"},"reference-count":44,"publisher":"Institute of Electrical and Electronics Engineers (IEEE)","issue":"7","license":[{"start":{"date-parts":[[2021,7,1]],"date-time":"2021-07-01T00:00:00Z","timestamp":1625097600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/ieeexplore.ieee.org\/Xplorehelp\/downloads\/license-information\/IEEE.html"},{"start":{"date-parts":[[2021,7,1]],"date-time":"2021-07-01T00:00:00Z","timestamp":1625097600000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2021,7,1]],"date-time":"2021-07-01T00:00:00Z","timestamp":1625097600000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"}],"funder":[{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["61972376"],"award-info":[{"award-number":["61972376"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["61502450"],"award-info":[{"award-number":["61502450"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["61432018"],"award-info":[{"award-number":["61432018"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["61521092"],"award-info":[{"award-number":["61521092"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]},{"name":"National Key Research and Development Program of China","award":["2016YFB0200800"],"award-info":[{"award-number":["2016YFB0200800"]}]},{"name":"National Key Research and Development Program of China","award":["2016YFB0200803"],"award-info":[{"award-number":["2016YFB0200803"]}]},{"name":"National Key Research and Development Program of China","award":["2017YFB0202302"],"award-info":[{"award-number":["2017YFB0202302"]}]},{"name":"National Key Research and Development Program of China","award":["2017YFB0202105"],"award-info":[{"award-number":["2017YFB0202105"]}]},{"name":"State Key Laboratory of Computer Architecture Foundation","award":["CARCH3504"],"award-info":[{"award-number":["CARCH3504"]}]},{"DOI":"10.13039\/501100004826","name":"Natural Science Foundation of Beijing Municipality","doi-asserted-by":"publisher","award":["L182053"],"award-info":[{"award-number":["L182053"]}],"id":[{"id":"10.13039\/501100004826","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["IEEE Trans. Parallel Distrib. Syst."],"published-print":{"date-parts":[[2021,7,1]]},"DOI":"10.1109\/tpds.2020.3048836","type":"journal-article","created":{"date-parts":[[2021,1,7]],"date-time":"2021-01-07T06:41:30Z","timestamp":1610001690000},"page":"1702-1712","source":"Crossref","is-referenced-by-count":21,"title":["Why Dataset Properties Bound the Scalability of Parallel Machine Learning Training Algorithms"],"prefix":"10.1109","volume":"32","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-9217-6980","authenticated-orcid":false,"given":"Daning","family":"Cheng","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-0022-7865","authenticated-orcid":false,"given":"Shigang","family":"Li","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Hanping","family":"Zhang","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Fen","family":"Xia","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Yunquan","family":"Zhang","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"263","reference":[{"key":"ref39","doi-asserted-by":"publisher","DOI":"10.1080\/10556788.2016.1278445"},{"key":"ref38","first-page":"629","article-title":"Trading computation for communication: Distributed stochastic dual coordinate ascent","author":"yang","year":"2013","journal-title":"Proc 26th Int Conf Neural Inf Process Syst"},{"key":"ref33","doi-asserted-by":"publisher","DOI":"10.1137\/130943170"},{"key":"ref32","first-page":"2595","article-title":"Parallelized stochastic gradient descent","author":"zinkevich","year":"2010","journal-title":"Proc Conf Neural Inf Process Syst"},{"key":"ref31","first-page":"693","article-title":"Hogwild!: A lock-free approach to parallelizing stochastic gradient descent","author":"feng","year":"2011","journal-title":"Proc Advances Neural Inf Process Syst"},{"key":"ref30","doi-asserted-by":"publisher","DOI":"10.1145\/3332466.3374528"},{"key":"ref37","first-page":"1973","article-title":"Adding vs. averaging in distributed primal-dual optimization","author":"ma","year":"2015"},{"key":"ref36","first-page":"3068","article-title":"Communication-efficient distributed dual coordinate ascent","author":"jaggi","year":"2014","journal-title":"Proc Advances Neural Inf Process Syst"},{"key":"ref35","article-title":"Stochastic dual coordinate ascent methods for regularized loss minimization","volume":"14","author":"shalevshwartz","year":"2013","journal-title":"J Mach Learn Res"},{"key":"ref34","first-page":"-64i","article-title":"Accelerated proximal stochastic dual coordinate ascent for regularized loss minimization","author":"shalevshwartz","year":"2014","journal-title":"Proc Int Conf Int Conf Machi Learn"},{"key":"ref10","article-title":"cuDNN: Efficient primitives for deep learning","author":"chetlur","year":"2014"},{"key":"ref40","first-page":"2832","article-title":"Estimation, optimization, and parallelism when data is sparse","author":"duchi","year":"2013","journal-title":"Proc Advances Neural Inf Process Syst"},{"key":"ref11","doi-asserted-by":"publisher","DOI":"10.1109\/SC.2018.00054"},{"key":"ref12","article-title":"Scaling SGD batch size to 32K for imagenet training","author":"you","year":"2017","journal-title":"arXiv 1708 03888"},{"key":"ref13","doi-asserted-by":"publisher","DOI":"10.1145\/3295500.3356207"},{"key":"ref14","doi-asserted-by":"publisher","DOI":"10.1145\/3295500.3356137"},{"key":"ref15","first-page":"693","article-title":"HOGWILD!: A lock-free approach to parallelizing stochastic gradient descent","author":"niu","year":"2011","journal-title":"Proc Advances Neural Inf Process Syst"},{"key":"ref16","first-page":"7652","article-title":"Communication compression for decentralized training","author":"tang","year":"2018"},{"key":"ref17","first-page":"1","article-title":"A general distributed dual coordinate optimization framework for regularized loss minimization","volume":"18","author":"zheng","year":"2017","journal-title":"J Mach Learn Res"},{"key":"ref18","doi-asserted-by":"publisher","DOI":"10.1145\/3022670.2976746"},{"key":"ref19","article-title":"MXNet: A flexible and efficient machine learning library for heterogeneous distributed systems","author":"chen","year":"2015","journal-title":"arXiv 1512 01274"},{"key":"ref28","doi-asserted-by":"publisher","DOI":"10.1145\/3332466.3374528"},{"key":"ref4","first-page":"257","article-title":"Adaptive subgradient methods for online learning and stochastic optimization","volume":"12","author":"duchi","year":"2010","journal-title":"J Mach Learn Res"},{"key":"ref27","first-page":"165","article-title":"Optimal distributed online prediction using mini-batches","volume":"13","author":"dekel","year":"2012","journal-title":"J Mach Learn Res"},{"key":"ref3","doi-asserted-by":"publisher","DOI":"10.1016\/j.neulet.2012.05.025"},{"key":"ref6","first-page":"12 350","article-title":"Theoretical limits of pipeline parallel optimization and application to distributed deep learning","author":"colin igor and","year":"2019","journal-title":"Proc Advances Neural Inf Process Syst"},{"key":"ref29","doi-asserted-by":"publisher","DOI":"10.1016\/j.jpdc.2020.06.011"},{"key":"ref5","first-page":"550","article-title":"Distributed dual averaging in networks","author":"duchi","year":"2010","journal-title":"Proc Conf Neural Inf Process Syst"},{"key":"ref8","article-title":"Large scale distributed neural network training through online distillation","author":"anil","year":"2018"},{"key":"ref7","first-page":"2331","article-title":"Slow learners are fast","author":"langford","year":"2009","journal-title":"Proc Int Conf Neural Inf Process"},{"key":"ref2","article-title":"Optimal mini-batch size selection for fast gradient descent","author":"perrone","year":"2019"},{"key":"ref9","doi-asserted-by":"publisher","DOI":"10.1145\/3225058.3225069"},{"key":"ref1","article-title":"Accurate, large minibatch SGD: Training imageNet in 1 hour","author":"goyal","year":"2017"},{"key":"ref20","article-title":"Reducing bert pre-training time from 3 days to 76 minutes","author":"you","year":"2019"},{"key":"ref22","doi-asserted-by":"publisher","DOI":"10.1137\/070704277"},{"key":"ref21","first-page":"72","article-title":"On the stochastic quasi-gradient method and stochastic quasi-feyer sequences","volume":"2","author":"ermoliev","year":"1969"},{"key":"ref42","article-title":"SGD with shuffling: Optimal rates without component convexity and large epoch requirements","year":"2020"},{"key":"ref24","first-page":"265","article-title":"Tensorflow: A system for large-scale machine learning","year":"0"},{"key":"ref41","first-page":"3065","article-title":"Minimax bounds on stochastic batched convex optimization","author":"duchi","year":"2018","journal-title":"Proc Conf Learn Theory"},{"key":"ref23","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-7908-2604-3_16"},{"key":"ref44","author":"cam","year":"2012","journal-title":"Asymptotics in Statistics Some Basic Concepts"},{"key":"ref26","author":"bertsekas","year":"2003","journal-title":"Parallel and Distributed Computation Numerical Methods"},{"key":"ref43","first-page":"1","article-title":"Improved asynchronous parallel optimization analysis for stochastic incremental methods","volume":"19","author":"leblond","year":"2018","journal-title":"J Mach Learn Res"},{"key":"ref25","doi-asserted-by":"publisher","DOI":"10.1145\/2647868.2654889"}],"container-title":["IEEE Transactions on Parallel and Distributed Systems"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx7\/71\/9357351\/09316159.pdf?arnumber=9316159","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2022,5,10]],"date-time":"2022-05-10T14:50:30Z","timestamp":1652194230000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/9316159\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2021,7,1]]},"references-count":44,"journal-issue":{"issue":"7"},"URL":"https:\/\/doi.org\/10.1109\/tpds.2020.3048836","relation":{},"ISSN":["1045-9219","1558-2183","2161-9883"],"issn-type":[{"value":"1045-9219","type":"print"},{"value":"1558-2183","type":"electronic"},{"value":"2161-9883","type":"electronic"}],"subject":[],"published":{"date-parts":[[2021,7,1]]}}}