{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,20]],"date-time":"2026-07-20T15:52:11Z","timestamp":1784562731870,"version":"3.55.0"},"reference-count":61,"publisher":"Institute of Electrical and Electronics Engineers (IEEE)","issue":"1","funder":[{"name":"GRF","award":["14208318"],"award-info":[{"award-number":["14208318"]}]}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["IEEE Trans. Parallel Distrib. Syst."],"published-print":{"date-parts":[[2022,1,1]]},"DOI":"10.1109\/tpds.2021.3064966","type":"journal-article","created":{"date-parts":[[2021,3,9]],"date-time":"2021-03-09T20:58:43Z","timestamp":1615323523000},"page":"144-158","source":"Crossref","is-referenced-by-count":39,"title":["Elastic Deep Learning in Multi-Tenant GPU Clusters"],"prefix":"10.1109","volume":"33","author":[{"given":"Yidi","family":"Wu","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Kaihao","family":"Ma","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-2122-915X","authenticated-orcid":false,"given":"Xiao","family":"Yan","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Zhi","family":"Liu","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Zhenkun","family":"Cai","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Yuzhen","family":"Huang","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"James","family":"Cheng","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Han","family":"Yuan","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Fan","family":"Yu","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"263","reference":[{"key":"ref39","first-page":"1","article-title":"Curiously fast convergence of some stochastic gradient descent algorithms","author":"bottou","year":"2009","journal-title":"Proc Symp Learn Data Sci"},{"key":"ref38","first-page":"2624","article-title":"Random shuffling beats SGD after finite epochs","author":"haochen","year":"2019","journal-title":"Proc 36th Int Conf Mach Learn"},{"key":"ref33","doi-asserted-by":"publisher","DOI":"10.1109\/5.726791"},{"key":"ref32","article-title":"Scaling SGD batch size to 32k for ImageNet training","author":"you","year":"2017","journal-title":"CoRR"},{"key":"ref31","article-title":"Accurate, large minibatch SGD: training ImageNet in 1 hour","author":"goyal","year":"2017","journal-title":"CoRR"},{"key":"ref30","author":"asia","year":"0","journal-title":"Project philly traces"},{"key":"ref37","first-page":"130","article-title":"DGCL: An efficient communication library for distributed GNN training","author":"cai","year":"2021","journal-title":"Proc 14th EuroSys Conf"},{"key":"ref36","first-page":"359","article-title":"Seastar: Vertex-centric programming for graph neural networks","author":"wu","year":"2021","journal-title":"Proc 14th EuroSys Conf"},{"key":"ref35","first-page":"1731","article-title":"Train longer, generalize better: Closing the generalization gap in large batch training of neural networks","author":"hoffer","year":"2017","journal-title":"Proc Int Conf Neural Inf Process"},{"key":"ref34","article-title":"On large-batch training for deep learning: Generalization gap and sharp minima","author":"keskar","year":"2017","journal-title":"Proc 5th Int Conf Learn Representations"},{"key":"ref60","doi-asserted-by":"publisher","DOI":"10.1145\/3064176.3064181"},{"key":"ref61","doi-asserted-by":"publisher","DOI":"10.1145\/2987550.2987576"},{"key":"ref28","article-title":"Priority-based parameter propagation for distributed DNN training","author":"jayarajan","year":"2019","journal-title":"CoRR"},{"key":"ref27","doi-asserted-by":"publisher","DOI":"10.1145\/3341301.3359642"},{"key":"ref29","article-title":"Horovod: Fast and easy distributed deep learning in tensorflow","author":"sergeev","year":"2018","journal-title":"CoRR"},{"key":"ref2","article-title":"Mesos: A platform for fine-grained resource sharing in the data center","author":"hindman","year":"2011","journal-title":"Proc 8th USENIX Symp Netw Syst Des Implementation"},{"key":"ref1","doi-asserted-by":"publisher","DOI":"10.1145\/2523616.2523633"},{"key":"ref20","doi-asserted-by":"publisher","DOI":"10.1145\/3302424.3303957"},{"key":"ref22","article-title":"Nvidia nccl","year":"0"},{"key":"ref21","doi-asserted-by":"publisher","DOI":"10.1016\/j.jpdc.2008.09.002"},{"key":"ref24","first-page":"583","article-title":"Scaling distributed machine learning with the parameter server","author":"li","year":"2014","journal-title":"Proc 11th USENIX Symp Operating Syst Des Implementation"},{"key":"ref23","doi-asserted-by":"publisher","DOI":"10.1145\/3187009.3177734"},{"key":"ref26","doi-asserted-by":"publisher","DOI":"10.1109\/TBDATA.2015.2472014"},{"key":"ref25","first-page":"181","article-title":"Poseidon: An efficient communication architecture for distributed deep learning on GPU clusters","author":"zhang","year":"2017","journal-title":"Proc USENIX Annu Tech Conf"},{"key":"ref50","first-page":"601","article-title":"Latent Dirichlet allocation","author":"blei","year":"2001","journal-title":"Proc Int Conf Neural Inf Process"},{"key":"ref51","article-title":"Paddlepaddle","year":"2019"},{"key":"ref59","doi-asserted-by":"publisher","DOI":"10.1145\/2901318.2901319"},{"key":"ref58","first-page":"1","article-title":"Tributary: Spot-dancing for elastic services with latency slos","author":"harlap","year":"2018","journal-title":"Proc USENIX Annu Tech Conf"},{"key":"ref57","doi-asserted-by":"publisher","DOI":"10.1145\/3064176.3064182"},{"key":"ref56","article-title":"Google preemptible VM instances","author":"cloud","year":"2019"},{"key":"ref55","article-title":"Amazon EC2 spot instance","year":"2019"},{"key":"ref54","first-page":"400","article-title":"Resource elasticity in distributed deep learning","author":"or","year":"2020","journal-title":"Proc 3rd Conf Mach Learn Syst"},{"key":"ref53","article-title":"DL2: A deep learning-driven scheduler for deep learning clusters","author":"peng","year":"2019","journal-title":"CoRR"},{"key":"ref52","article-title":"Elasticdl","author":"financial","year":"2019"},{"key":"ref10","doi-asserted-by":"publisher","DOI":"10.1145\/3267809.3267818"},{"key":"ref11","first-page":"631","article-title":"Litz: Elastic framework for high-performance distributed machine learning","author":"qiao","year":"2018","journal-title":"Proc USENIX Annu Tech Conf"},{"key":"ref40","article-title":"ZooKeeper: Wait-free coordination for internet-scale systems","author":"hunt","year":"2010","journal-title":"Proc USENIX Annu Tech Conf"},{"key":"ref12","doi-asserted-by":"publisher","DOI":"10.1109\/ICDCS.2019.00203"},{"key":"ref13","first-page":"265","article-title":"TensorFlow: A system for large-scale machine learning","author":"abadi","year":"2016","journal-title":"Proc 12th USENIX Symp Operating Syst Des Implementation"},{"key":"ref14","article-title":"Automatic differentiation in pytorch","author":"paszke","year":"2017"},{"key":"ref15","article-title":"Mindspore","year":"2021"},{"key":"ref16","article-title":"MXNet: A flexible and efficient machine learning library for heterogeneous distributed systems","author":"chen","year":"2015","journal-title":"CoRR"},{"key":"ref17","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2016.90"},{"key":"ref18","article-title":"Very deep convolutional networks for large-scale image recognition","author":"simonyan","year":"2015","journal-title":"Proc 3rd Int Conf Learn Representations"},{"key":"ref19","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2016.308"},{"key":"ref4","first-page":"595","article-title":"Gandiva: Introspective cluster scheduling for deep learning","author":"xiao","year":"2018","journal-title":"Proc 13th USENIX Symp Operating Syst Des Implementation"},{"key":"ref3","doi-asserted-by":"publisher","DOI":"10.1145\/3190508.3190517"},{"key":"ref6","article-title":"Themis: Fair and efficient GPU cluster scheduling for machine learning workloads","author":"mahajan","year":"2019","journal-title":"CoRR"},{"key":"ref5","first-page":"485","article-title":"Tiresias: A GPU cluster manager for distributed deep learning","author":"gu","year":"2019","journal-title":"Proc 16th USENIX Symp Netw Syst Des Implementation"},{"key":"ref8","first-page":"947","article-title":"Analysis of large-scale multi-tenant GPU clusters for DNN training workloads","author":"jeon","year":"2019","journal-title":"Proc USENIX Annu Tech Conf"},{"key":"ref7","doi-asserted-by":"publisher","DOI":"10.1145\/3127479.3127490"},{"key":"ref49","doi-asserted-by":"publisher","DOI":"10.1145\/2741948.2741964"},{"key":"ref9","doi-asserted-by":"publisher","DOI":"10.1145\/3302424.3303964"},{"key":"ref46","doi-asserted-by":"publisher","DOI":"10.1145\/2901318.2901323"},{"key":"ref45","first-page":"5998","article-title":"Attention is all you need","author":"vaswani","year":"2017","journal-title":"Proc Int Conf Neural Inf Process"},{"key":"ref48","doi-asserted-by":"publisher","DOI":"10.1145\/2465351.2465386"},{"key":"ref47","article-title":"TF CNN benchmark","year":"2019"},{"key":"ref42","doi-asserted-by":"publisher","DOI":"10.1109\/MSST.2010.5496972"},{"key":"ref41","article-title":"etcd","year":"2019"},{"key":"ref44","doi-asserted-by":"publisher","DOI":"10.1145\/1024908.1024910"},{"key":"ref43","doi-asserted-by":"publisher","DOI":"10.1145\/2517349.2522738"}],"container-title":["IEEE Transactions on Parallel and Distributed Systems"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx7\/71\/9465725\/09373916.pdf?arnumber=9373916","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2022,1,27]],"date-time":"2022-01-27T21:23:05Z","timestamp":1643318585000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/9373916\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2022,1,1]]},"references-count":61,"journal-issue":{"issue":"1"},"URL":"https:\/\/doi.org\/10.1109\/tpds.2021.3064966","relation":{},"ISSN":["1045-9219","1558-2183","2161-9883"],"issn-type":[{"value":"1045-9219","type":"print"},{"value":"1558-2183","type":"electronic"},{"value":"2161-9883","type":"electronic"}],"subject":[],"published":{"date-parts":[[2022,1,1]]}}}