{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,6,10]],"date-time":"2026-06-10T16:43:54Z","timestamp":1781109834917,"version":"3.54.1"},"reference-count":37,"publisher":"Institute of Electrical and Electronics Engineers (IEEE)","issue":"2","license":[{"start":{"date-parts":[[2025,2,1]],"date-time":"2025-02-01T00:00:00Z","timestamp":1738368000000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/ieeexplore.ieee.org\/Xplorehelp\/downloads\/license-information\/IEEE.html"},{"start":{"date-parts":[[2025,2,1]],"date-time":"2025-02-01T00:00:00Z","timestamp":1738368000000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2025,2,1]],"date-time":"2025-02-01T00:00:00Z","timestamp":1738368000000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"}],"funder":[{"name":"unrestricted grant\/gift from Meta, Inc"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["IEEE Trans. Parallel Distrib. Syst."],"published-print":{"date-parts":[[2025,2]]},"DOI":"10.1109\/tpds.2024.3507814","type":"journal-article","created":{"date-parts":[[2024,11,28]],"date-time":"2024-11-28T19:00:39Z","timestamp":1732820439000},"page":"226-238","source":"Crossref","is-referenced-by-count":4,"title":["Towards Universal Performance Modeling for Machine Learning Training on Multi-GPU Platforms"],"prefix":"10.1109","volume":"36","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-5992-3913","authenticated-orcid":false,"given":"Zhongyi","family":"Lin","sequence":"first","affiliation":[{"name":"Advanced Micro Devices, Inc, San Jose, CA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Ning","family":"Sun","sequence":"additional","affiliation":[{"name":"Meta, Inc, Menlo Park, CA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0005-7653-744X","authenticated-orcid":false,"given":"Pallab","family":"Bhattacharya","sequence":"additional","affiliation":[{"name":"NVIDIA, Santa Clara, CA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Xizhou","family":"Feng","sequence":"additional","affiliation":[{"name":"Meta, Inc, Menlo Park, CA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Louis","family":"Feng","sequence":"additional","affiliation":[{"name":"Meta, Inc, Menlo Park, CA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-6582-8237","authenticated-orcid":false,"given":"John D.","family":"Owens","sequence":"additional","affiliation":[{"name":"Department of Electrical and Computer Engineering, University of California, Davis, CA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"263","reference":[{"key":"ref1","article-title":"Deep learning recommendation model for personalization and recommendation systems","author":"Naumov","year":"2019"},{"key":"ref2","doi-asserted-by":"publisher","DOI":"10.1145\/3470496.3533727"},{"key":"ref3","first-page":"1877","article-title":"Language models are few-shot learners","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","author":"Brown"},{"key":"ref4","article-title":"Using deepspeed and megatron to train megatron-turing NLG 530B, a large-scale generative language model","author":"Smith","year":"2022"},{"key":"ref5","article-title":"PaLM: Scaling language modeling with pathways","author":"Chowdhery","year":"2022"},{"key":"ref6","article-title":"OPT: Open pre-trained transformer language models","author":"Zhang","year":"2022"},{"key":"ref7","article-title":"LLaMA: Open and efficient foundation language models","author":"Touvron","year":"2023"},{"key":"ref8","doi-asserted-by":"publisher","DOI":"10.1109\/BigData.2018.8622396"},{"key":"ref9","doi-asserted-by":"publisher","DOI":"10.1109\/ACCESS.2019.2916550"},{"key":"ref10","doi-asserted-by":"publisher","DOI":"10.1109\/ICDCS47774.2020.00097"},{"key":"ref11","doi-asserted-by":"publisher","DOI":"10.1109\/ICS51289.2020.00039"},{"key":"ref12","first-page":"503","article-title":"Habitat: A runtime-based computational performance predictor for deep neural network training","volume-title":"Proc. USENIX Annu. Tech. Conf.","author":"Yu"},{"key":"ref13","doi-asserted-by":"publisher","DOI":"10.1109\/ICCVW54120.2021.00112"},{"key":"ref14","doi-asserted-by":"publisher","DOI":"10.1109\/HiPC56025.2022.00019"},{"key":"ref15","doi-asserted-by":"publisher","DOI":"10.48550\/arXiv.1810.04805"},{"key":"ref16","article-title":"Language models are unsupervised multitask learners","volume-title":"OpenAI Blog","author":"Radford","year":"2019"},{"key":"ref17","first-page":"5753","article-title":"XLNet: Generalized autoregressive pretraining for language understanding","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","author":"Yang"},{"key":"ref18","doi-asserted-by":"publisher","DOI":"10.1145\/2783258.2783270"},{"key":"ref19","doi-asserted-by":"publisher","DOI":"10.1109\/BigData.2016.7840590"},{"key":"ref20","article-title":"Paleo: A performance model for deep neural networks","volume-title":"Proc. 5th Int. Conf. Learn. Representations","author":"Qi"},{"key":"ref21","doi-asserted-by":"publisher","DOI":"10.1109\/IISWC47752.2019.9042047"},{"key":"ref22","article-title":"Chakra: Advancing performance benchmarking and co-design using standardized execution traces","author":"Sridharan","year":"2023"},{"key":"ref23","doi-asserted-by":"publisher","DOI":"10.1109\/CloudIntelligence52565.2021.00012"},{"key":"ref24","article-title":"Gshard: Scaling giant models with conditional computation and automatic sharding","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Lepikhin"},{"key":"ref25","doi-asserted-by":"publisher","DOI":"10.1109\/ISPASS51385.2021.00033"},{"key":"ref26","doi-asserted-by":"publisher","DOI":"10.1145\/3503222.3507777"},{"key":"ref27","doi-asserted-by":"publisher","DOI":"10.1145\/3534678.3539034"},{"key":"ref28","first-page":"15190","article-title":"Dreamshard: Generalizable embedding table placement for recommender systems","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","author":"Zha"},{"key":"ref29","article-title":"DLRM open-source datasets","year":"2021"},{"key":"ref31","volume-title":"Introduction to Parallel Computing","author":"Grama","year":"2003"},{"key":"ref32","doi-asserted-by":"publisher","DOI":"10.1109\/TPDS.2019.2928289"},{"key":"ref33","doi-asserted-by":"publisher","DOI":"10.1145\/3079856.3080216"},{"key":"ref34","article-title":"PARAM benchmark","year":"2020"},{"key":"ref35","article-title":"FBGEMM: Enabling high-performance low-precision deep learning inference","author":"Khudia","year":"2021"},{"key":"ref36","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.emnlp-demos.6"},{"key":"ref37","article-title":"Comparison between DataParallel and DistributedDataParallel","year":"2023"},{"key":"ref38","doi-asserted-by":"publisher","DOI":"10.14778\/3415478.3415530"}],"container-title":["IEEE Transactions on Parallel and Distributed Systems"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx8\/71\/10795769\/10770554.pdf?arnumber=10770554","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,12,28]],"date-time":"2024-12-28T06:20:06Z","timestamp":1735366806000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/10770554\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,2]]},"references-count":37,"journal-issue":{"issue":"2"},"URL":"https:\/\/doi.org\/10.1109\/tpds.2024.3507814","relation":{},"ISSN":["1045-9219","1558-2183","2161-9883"],"issn-type":[{"value":"1045-9219","type":"print"},{"value":"1558-2183","type":"electronic"},{"value":"2161-9883","type":"electronic"}],"subject":[],"published":{"date-parts":[[2025,2]]}}}