{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,12]],"date-time":"2026-07-12T00:39:56Z","timestamp":1783816796555,"version":"3.55.0"},"reference-count":48,"publisher":"Institute of Electrical and Electronics Engineers (IEEE)","issue":"8","license":[{"start":{"date-parts":[[2022,8,1]],"date-time":"2022-08-01T00:00:00Z","timestamp":1659312000000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/ieeexplore.ieee.org\/Xplorehelp\/downloads\/license-information\/IEEE.html"},{"start":{"date-parts":[[2022,8,1]],"date-time":"2022-08-01T00:00:00Z","timestamp":1659312000000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2022,8,1]],"date-time":"2022-08-01T00:00:00Z","timestamp":1659312000000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["IEEE Trans. Parallel Distrib. Syst."],"published-print":{"date-parts":[[2022,8,1]]},"DOI":"10.1109\/tpds.2021.3132413","type":"journal-article","created":{"date-parts":[[2021,12,6]],"date-time":"2021-12-06T20:46:19Z","timestamp":1638823579000},"page":"1967-1981","source":"Crossref","is-referenced-by-count":40,"title":["TensorOpt: Exploring the Tradeoffs in Distributed DNN Training With Auto-Parallelism"],"prefix":"10.1109","volume":"33","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-0199-4866","authenticated-orcid":false,"given":"Zhenkun","family":"Cai","sequence":"first","affiliation":[{"name":"Department of Computer Science and Engineering, The Chinese University of Hong Kong, Hong Kong"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-2122-915X","authenticated-orcid":false,"given":"Xiao","family":"Yan","sequence":"additional","affiliation":[{"name":"Department of Computer Science and Engineering, Southern University of Science and Technology, Shenzhen, Guangdong, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Kaihao","family":"Ma","sequence":"additional","affiliation":[{"name":"Department of Computer Science and Engineering, The Chinese University of Hong Kong, Hong Kong"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Yidi","family":"Wu","sequence":"additional","affiliation":[{"name":"Department of Computer Science and Engineering, The Chinese University of Hong Kong, Hong Kong"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Yuzhen","family":"Huang","sequence":"additional","affiliation":[{"name":"Department of Computer Science and Engineering, The Chinese University of Hong Kong, Hong Kong"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"James","family":"Cheng","sequence":"additional","affiliation":[{"name":"Department of Computer Science and Engineering, The Chinese University of Hong Kong, Hong Kong"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Teng","family":"Su","sequence":"additional","affiliation":[{"name":"Huawei Technologies Co. Ltd., Shenzhen, Guangdong, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Fan","family":"Yu","sequence":"additional","affiliation":[{"name":"Huawei Technologies Co. Ltd., Shenzhen, Guangdong, China"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"263","reference":[{"key":"ref39","doi-asserted-by":"publisher","DOI":"10.1145\/3373376.3378505"},{"key":"ref38","first-page":"4125","article-title":"Memory-efficient backpropagation through time","author":"gruslys","year":"2016","journal-title":"Proc Adv Neural Inf Process Syst"},{"key":"ref33","first-page":"2430","article-title":"Device placement optimization with reinforcement learning","author":"mirhoseini","year":"2017","journal-title":"Proc 34th Int Conf Mach Learn"},{"key":"ref32","article-title":"Mxnet: A flexible and efficient machine learning library for heterogeneous distributed systems","author":"chen","year":"0"},{"key":"ref31","article-title":"Automatic differentiation in pytorch","author":"paszke","year":"2017"},{"key":"ref30","first-page":"1097","article-title":"Imagenet classification with deep convolutional neural networks","author":"krizhevsky","year":"2012","journal-title":"Proc Adv Neural Inf Process Syst"},{"key":"ref37","article-title":"Gshard: Scaling giant models with conditional computation and automatic sharding","author":"lepikhin","year":"2020"},{"key":"ref36","article-title":"Megatron-lm: Training multi-billion parameter language models using model parallelism","author":"shoeybi","year":"2019"},{"key":"ref35","article-title":"One weird trick for parallelizing convolutional neural networks","author":"krizhevsky","year":"0"},{"key":"ref34","first-page":"1223","article-title":"More effective distributed ml via a stale synchronous parallel parameter server","author":"ho","year":"2013","journal-title":"Proc Adv Neural Inf Process Syst"},{"key":"ref10","first-page":"265","article-title":"Tensorflow: A system for large-scale machine learning","author":"abadi","year":"2016","journal-title":"Proc 12th USENIX Conf Oper Syst Des Implementation"},{"key":"ref40","article-title":"Checkmate: Breaking the memory wall with optimal tensor rematerialization","author":"jain","year":"2020","journal-title":"Proc MLSys"},{"key":"ref11","first-page":"1866","article-title":"Splitnet: Learning to semantically split deep networks for parameter reduction and model parallelization","author":"kim","year":"2017","journal-title":"Proc Int Conf Mach Learn"},{"key":"ref12","first-page":"2279","article-title":"Exploring hidden dimensions in parallelizing convolutional neural networks","author":"jia","year":"2018","journal-title":"Proc Int Conf Mach Learn"},{"key":"ref13","article-title":"Beyond data and model parallelism for deep neural networks","author":"jia","year":"2019","journal-title":"Proc Mach Learn Syst"},{"key":"ref14","doi-asserted-by":"publisher","DOI":"10.1145\/3302424.3303953"},{"key":"ref15","doi-asserted-by":"crossref","DOI":"10.21437\/Interspeech.2014-80","article-title":"Long short-term memory recurrent neural network architectures for large scale acoustic modeling","author":"sak","year":"2014"},{"key":"ref16","doi-asserted-by":"publisher","DOI":"10.1145\/3341301.3359646"},{"key":"ref17","first-page":"10 414","article-title":"Mesh-tensorflow: Deep learning for supercomputers","author":"shazeer","year":"2018","journal-title":"Proc Adv Neural Inf Process Syst"},{"key":"ref18","article-title":"Bert: Pre-training of deep bidirectional transformers for language understanding","author":"devlin","year":"2018"},{"key":"ref19","doi-asserted-by":"publisher","DOI":"10.1145\/3368089.3417050"},{"key":"ref28","doi-asserted-by":"publisher","DOI":"10.1109\/MICRO.2016.7783721"},{"key":"ref4","article-title":"Very deep convolutional networks for large-scale image recognition","author":"simonyan","year":"2014"},{"key":"ref27","article-title":"Horovod: Fast and easy distributed deep learning in TensorFlow","author":"sergeev","year":"2018"},{"key":"ref3","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v33i01.33014780"},{"key":"ref6","article-title":"Google's neural machine translation system: Bridging the gap between human and machine translation","author":"wu","year":"2016"},{"key":"ref29","article-title":"Training deep nets with sublinear memory cost","author":"chen","year":"2016"},{"key":"ref5","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2016.308"},{"key":"ref8","first-page":"103","article-title":"GPipe: Efficient training of giant neural networks using pipeline parallelism","author":"huang","year":"2019","journal-title":"Proc Adv Neural Inf Process Syst"},{"key":"ref7","article-title":"Neural machine translation by jointly learning to align and translate","author":"bahdanau","year":"2014"},{"key":"ref2","first-page":"5998","article-title":"Attention is all you need","author":"vaswani","year":"2017","journal-title":"Proc Adv Neural Inf Process Syst"},{"key":"ref9","first-page":"1223","article-title":"Large scale distributed deep networks","author":"dean","year":"2012","journal-title":"Proc Adv Neural Inf Process Syst"},{"key":"ref1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2016.90"},{"key":"ref46","first-page":"578","article-title":"TVM: An automated end-to-end optimizing compiler for deep learning","author":"chen","year":"2018","journal-title":"Proc Symp Oper Syst Des Implementation"},{"key":"ref20","doi-asserted-by":"publisher","DOI":"10.1145\/3447786.3456233"},{"key":"ref45","article-title":"Tensorflow xla","year":"2020"},{"key":"ref48","article-title":"An analysis of deep neural network models for practical applications","author":"canziani","year":"2016"},{"key":"ref22","doi-asserted-by":"crossref","first-page":"117","DOI":"10.1016\/j.jpdc.2008.09.002","article-title":"Bandwidth optimal all-reduce algorithms for clusters of workstations","volume":"69","author":"patarasuk","year":"2009","journal-title":"J Parallel Distrib Comput"},{"key":"ref47","article-title":"Tpu","year":"2020"},{"key":"ref21","article-title":"Nvidia collective communications library (NCCL)","year":"2020"},{"key":"ref42","first-page":"387","article-title":"Flashneuron: Ssd-enabled large-batch training of very deep neural networks","author":"bae","year":"0","journal-title":"Proc 19th USENIX Conf File Storage Technol"},{"key":"ref24","article-title":"Nvlink and nvswitch","year":"2020"},{"key":"ref41","first-page":"1161","article-title":"A graph theoretic framework of recomputation algorithms for memory-efficient backpropagation","author":"kusumoto","year":"2019","journal-title":"Proc Adv Neural Inf Process Syst Annu Conf Neural Inf Process Syst"},{"key":"ref23","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-540-30218-6_19"},{"key":"ref44","article-title":"Mindspore","year":"2020"},{"key":"ref26","doi-asserted-by":"publisher","DOI":"10.5244\/C.30.87"},{"key":"ref43","doi-asserted-by":"publisher","DOI":"10.1145\/2939672.2945397"},{"key":"ref25","article-title":"Infiniband","year":"2020","journal-title":"Wikipedia"}],"container-title":["IEEE Transactions on Parallel and Distributed Systems"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx7\/71\/9642402\/09635821.pdf?arnumber=9635821","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2022,5,11]],"date-time":"2022-05-11T20:10:59Z","timestamp":1652299859000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/9635821\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2022,8,1]]},"references-count":48,"journal-issue":{"issue":"8"},"URL":"https:\/\/doi.org\/10.1109\/tpds.2021.3132413","relation":{},"ISSN":["1045-9219","1558-2183","2161-9883"],"issn-type":[{"value":"1045-9219","type":"print"},{"value":"1558-2183","type":"electronic"},{"value":"2161-9883","type":"electronic"}],"subject":[],"published":{"date-parts":[[2022,8,1]]}}}