{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,14]],"date-time":"2026-07-14T16:27:39Z","timestamp":1784046459895,"version":"3.55.0"},"reference-count":51,"publisher":"Institute of Electrical and Electronics Engineers (IEEE)","issue":"12","license":[{"start":{"date-parts":[[2020,12,1]],"date-time":"2020-12-01T00:00:00Z","timestamp":1606780800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/ieeexplore.ieee.org\/Xplorehelp\/downloads\/license-information\/IEEE.html"},{"start":{"date-parts":[[2020,12,1]],"date-time":"2020-12-01T00:00:00Z","timestamp":1606780800000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2020,12,1]],"date-time":"2020-12-01T00:00:00Z","timestamp":1606780800000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["IEEE Trans. Parallel Distrib. Syst."],"published-print":{"date-parts":[[2020,12,1]]},"DOI":"10.1109\/tpds.2020.3003307","type":"journal-article","created":{"date-parts":[[2020,6,18]],"date-time":"2020-06-18T20:46:32Z","timestamp":1592513192000},"page":"2802-2818","source":"Crossref","is-referenced-by-count":96,"title":["Distributed Training of Deep Learning Models: A Taxonomic Perspective"],"prefix":"10.1109","volume":"31","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-1776-8000","authenticated-orcid":false,"given":"Matthias","family":"Langer","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-0302-5775","authenticated-orcid":false,"given":"Zhen","family":"He","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-2657-4849","authenticated-orcid":false,"given":"Wenny","family":"Rahayu","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-5999-1521","authenticated-orcid":false,"given":"Yanbo","family":"Xue","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"263","reference":[{"key":"ref39","first-page":"997","article-title":"Asynchrony begets momentum, with an application to deep learning","author":"mitliagkas","year":"2017","journal-title":"Proc 54th Allerton Conf Commun Control Comput"},{"key":"ref38","first-page":"79","article-title":"High-performance distributed ML at scale through parameter server consistency models","author":"dai","year":"2015","journal-title":"Proc 29th Conf Artif Intell"},{"key":"ref33","article-title":"Adam: A method for stochastic optimization","author":"kingma","year":"2015","journal-title":"Proc 3rd Int Conf Learn Representations"},{"key":"ref32","first-page":"1139","article-title":"On the importance of initialization and momentum in deep learning","volume":"28","author":"sutskever","year":"2013","journal-title":"Proc 30th Int Conf Mach Learn"},{"key":"ref31","article-title":"On large-batch training for deep learning: Generalization gap and sharp minima","author":"keskar","year":"2017","journal-title":"Proc Intl Conf on Learning Representations"},{"key":"ref30","first-page":"685","article-title":"Deep learning with elastic averaging SGD","volume":"28","author":"zhang","year":"2015","journal-title":"Proc Int Conf Neural Inf Process"},{"key":"ref37","first-page":"873","article-title":"Distributed delayed stochastic optimization","volume":"24","author":"agarwal","year":"2011","journal-title":"Proc Int Conf Neural Inf Process"},{"key":"ref36","first-page":"3368","article-title":"Gradient coding: Avoiding stragglers in distributed learning","volume":"70","author":"tandon","year":"2017","journal-title":"Proc 34th Int Conf Mach Learn"},{"key":"ref35","first-page":"1223","article-title":"More effective distributed ML via a stale synchronous parallel parameter server","volume":"26","author":"ho","year":"2013","journal-title":"Proc Int Conf Neural Inf Process"},{"key":"ref34","article-title":"Revisiting distributed synchronous SGD","author":"chen","year":"2017","journal-title":"Proc Intl Conf on Learning Representations"},{"key":"ref28","article-title":"Qualitatively characterizing neural network optimization problems","author":"goodfellow","year":"2015","journal-title":"Proc 3rd Int Conf Learn Representations"},{"key":"ref27","doi-asserted-by":"publisher","DOI":"10.1145\/3357223.3362707"},{"key":"ref29","doi-asserted-by":"publisher","DOI":"10.1109\/CDC.2015.7402404"},{"key":"ref2","doi-asserted-by":"publisher","DOI":"10.1016\/j.neunet.2014.09.003"},{"key":"ref1","doi-asserted-by":"crossref","first-page":"436","DOI":"10.1038\/nature14539","article-title":"Deep learning","volume":"521","author":"lecun","year":"2015","journal-title":"Nature"},{"key":"ref20","first-page":"103","article-title":"GPipe: Efficient training of giant neural networks using pipeline parallelism","author":"huang","year":"2019","journal-title":"Advances in Neural IInformation Processing Systems"},{"key":"ref22","first-page":"448","article-title":"Batch normalization: Accelerating deep network training by reducing internal covariate shift","volume":"37","author":"ioffe","year":"2015","journal-title":"Proc 32nd Int Conf Mach Learn"},{"key":"ref21","first-page":"1223","article-title":"Large scale distributed deep networks","author":"dean","year":"2012","journal-title":"Proc Int Conf Neural Inf Process"},{"key":"ref24","article-title":"CaffeOnSpark open sourced for distributed deep learning on big data clusters","author":"feng","year":"2016"},{"key":"ref23","first-page":"571","article-title":"Project adam: Building an efficient and scalable deep learning training system","author":"chilimbi","year":"2014","journal-title":"Proc 11th USENIX Symp Operating Syst Des Implementation"},{"key":"ref26","first-page":"5330","article-title":"Can decentralized algorithms outperform centralized algorithms? A case study for decentralized parallel stochastic gradient descent","volume":"30","author":"lian","year":"2017","journal-title":"Proc Int Conf Neural Inf Process"},{"key":"ref25","article-title":"Distributed stochastic optimization for deep learning","author":"zhang","year":"2016"},{"key":"ref50","doi-asserted-by":"publisher","DOI":"10.1109\/IPDPS.2019.00018"},{"key":"ref51","article-title":"AIBench: An industry standard internet service AI benchmark suite","author":"gao","year":"2019","journal-title":"CoRR Arxiv"},{"key":"ref10","article-title":"SparkNet: Training deep networks in spark","author":"moritz","year":"2016","journal-title":"Proc Intl Conf on Learning Representations"},{"key":"ref11","doi-asserted-by":"publisher","DOI":"10.1109\/TPDS.2018.2833074"},{"key":"ref40","first-page":"4120","article-title":"Asynchronous stochastic gradient descent with delay compensation","author":"zheng","year":"2017","journal-title":"Proc 34th Int Conf Mach Learn"},{"key":"ref12","doi-asserted-by":"publisher","DOI":"10.1109\/ICCCN.2017.8038464"},{"key":"ref13","doi-asserted-by":"publisher","DOI":"10.1145\/3320060"},{"key":"ref14","first-page":"19","article-title":"Communication efficient distributed machine learning with the parameter server","volume":"27","author":"li","year":"2014","journal-title":"Proc Int Conf Neural Inf Process"},{"key":"ref15","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2016.284"},{"key":"ref16","doi-asserted-by":"publisher","DOI":"10.1109\/TBDATA.2015.2472014"},{"key":"ref17","first-page":"265","article-title":"TensorFlow: A system for large-scale machine learning","author":"abadi","year":"2016","journal-title":"Proc 12th USENIX Symp Operating Syst Des Implementation"},{"key":"ref18","doi-asserted-by":"publisher","DOI":"10.1109\/ICASSP.2013.6639343"},{"key":"ref19","article-title":"MXNet: A flexible and efficient machine learning library for heterogeneous distributed systems","author":"chen","year":"2015","journal-title":"Proc 29th Conf Neural Inf Process Syst"},{"key":"ref4","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2016.90"},{"key":"ref3","first-page":"1097","article-title":"ImageNet classification with deep convolutional neural networks","volume":"25","author":"krizhevsky","year":"2012","journal-title":"Proc Int Conf Neural Inf Process"},{"key":"ref6","article-title":"Outrageously large neural networks: The sparsely-gated mixture-of-experts layer","author":"shazeer","year":"2017","journal-title":"Proc Intl Conf on Learning Representations"},{"key":"ref5","doi-asserted-by":"publisher","DOI":"10.1109\/CCBD.2016.029"},{"key":"ref8","article-title":"Distributed deep learning in bandwidth-constrained environments","author":"langer","year":"2018"},{"key":"ref7","article-title":"Very deep convolutional networks for large-scale image recoginition","author":"simonyan","year":"2015","journal-title":"Proc 3rd Int Conf Learn Representations"},{"key":"ref49","first-page":"336","article-title":"MLPerf training benchmark","author":"mattson","year":"2020","journal-title":"Proc Conf Syst Mach Learning"},{"key":"ref9","first-page":"1337","article-title":"Deep learning with COTS HPC systems","author":"coates","year":"2013","journal-title":"Proc 30th Int Conf Mach Learn"},{"key":"ref46","doi-asserted-by":"publisher","DOI":"10.1109\/TIT.2006.874516"},{"key":"ref45","article-title":"Gossip training for deep learning","author":"blot","year":"2016","journal-title":"CoRR Arxiv"},{"key":"ref48","article-title":"DAWNBench: An end-to-end deep learning benchmark and competition","author":"coleman","year":"2017","journal-title":"Proc Conf Neural Inf Process Syst Mach Learn Syst Workshop"},{"key":"ref47","doi-asserted-by":"publisher","DOI":"10.1007\/978-1-4842-2766-4_12"},{"key":"ref42","article-title":"DeepSpark: Spark-based deep learning supporting asynchronous updates and caffe compatibility","author":"kim","year":"2016","journal-title":"CoRR Arxiv"},{"key":"ref41","first-page":"693","article-title":"HOGWILD!: A lock-free approach to parallelizing stochastic gradient descent","volume":"24","author":"niu","year":"2011","journal-title":"Proc Int Conf Neural Inf Process"},{"key":"ref44","doi-asserted-by":"publisher","DOI":"10.1177\/1094342005051521"},{"key":"ref43","first-page":"2737","article-title":"Asynchronous parallel stochastic gradient for nonconvex optimization","volume":"28","author":"lian","year":"2015","journal-title":"Proc Int Conf Neural Inf Process"}],"container-title":["IEEE Transactions on Parallel and Distributed Systems"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx7\/71\/9126280\/09120226.pdf?arnumber=9120226","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2022,4,27]],"date-time":"2022-04-27T14:40:33Z","timestamp":1651070433000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/9120226\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2020,12,1]]},"references-count":51,"journal-issue":{"issue":"12"},"URL":"https:\/\/doi.org\/10.1109\/tpds.2020.3003307","relation":{},"ISSN":["1045-9219","1558-2183","2161-9883"],"issn-type":[{"value":"1045-9219","type":"print"},{"value":"1558-2183","type":"electronic"},{"value":"2161-9883","type":"electronic"}],"subject":[],"published":{"date-parts":[[2020,12,1]]}}}