{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,3,24]],"date-time":"2026-03-24T00:13:46Z","timestamp":1774311226940,"version":"3.50.1"},"reference-count":67,"publisher":"Institute of Electrical and Electronics Engineers (IEEE)","issue":"5","license":[{"start":{"date-parts":[[2026,5,1]],"date-time":"2026-05-01T00:00:00Z","timestamp":1777593600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/ieeexplore.ieee.org\/Xplorehelp\/downloads\/license-information\/IEEE.html"},{"start":{"date-parts":[[2026,5,1]],"date-time":"2026-05-01T00:00:00Z","timestamp":1777593600000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2026,5,1]],"date-time":"2026-05-01T00:00:00Z","timestamp":1777593600000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"}],"funder":[{"name":"National Key R&D Program of China","award":["2022YFB4501400"],"award-info":[{"award-number":["2022YFB4501400"]}]},{"DOI":"10.13039\/501100012659","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["62421002"],"award-info":[{"award-number":["62421002"]}],"id":[{"id":"10.13039\/501100012659","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100012659","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["62025208"],"award-info":[{"award-number":["62025208"]}],"id":[{"id":"10.13039\/501100012659","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100012659","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["62302512"],"award-info":[{"award-number":["62302512"]}],"id":[{"id":"10.13039\/501100012659","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["IEEE Trans. Parallel Distrib. Syst."],"published-print":{"date-parts":[[2026,5]]},"DOI":"10.1109\/tpds.2026.3668887","type":"journal-article","created":{"date-parts":[[2026,3,2]],"date-time":"2026-03-02T20:57:22Z","timestamp":1772485042000},"page":"1121-1135","source":"Crossref","is-referenced-by-count":0,"title":["Pro-Prophet: A Systematic Load Balancing Method for Efficient Parallel Training of Large-Scale MoE Models"],"prefix":"10.1109","volume":"37","author":[{"ORCID":"https:\/\/orcid.org\/0009-0008-9377-1678","authenticated-orcid":false,"given":"Wei","family":"Wang","sequence":"first","affiliation":[{"name":"National Key Laboratory of Parallel and Distributed Computing, College of Computer Science and Technology, National University of Defense Technology, Changsha, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-3458-4732","authenticated-orcid":false,"given":"Zhiquan","family":"Lai","sequence":"additional","affiliation":[{"name":"National Key Laboratory of Parallel and Distributed Computing, College of Computer Science and Technology, National University of Defense Technology, Changsha, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-9743-2034","authenticated-orcid":false,"given":"Dongsheng","family":"Li","sequence":"additional","affiliation":[{"name":"National Key Laboratory of Parallel and Distributed Computing, College of Computer Science and Technology, National University of Defense Technology, Changsha, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-7419-1511","authenticated-orcid":false,"given":"Shengwei","family":"Li","sequence":"additional","affiliation":[{"name":"National Key Laboratory of Parallel and Distributed Computing, College of Computer Science and Technology, National University of Defense Technology, Changsha, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-0354-9467","authenticated-orcid":false,"given":"Weijie","family":"Liu","sequence":"additional","affiliation":[{"name":"National Key Laboratory of Parallel and Distributed Computing, College of Computer Science and Technology, National University of Defense Technology, Changsha, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-0669-6892","authenticated-orcid":false,"given":"Keshi","family":"Ge","sequence":"additional","affiliation":[{"name":"National Key Laboratory of Parallel and Distributed Computing, College of Computer Science and Technology, National University of Defense Technology, Changsha, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Ao","family":"Shen","sequence":"additional","affiliation":[{"name":"National Key Laboratory of Parallel and Distributed Computing, College of Computer Science and Technology, National University of Defense Technology, Changsha, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-3587-0917","authenticated-orcid":false,"given":"Huayou","family":"Su","sequence":"additional","affiliation":[{"name":"National Key Laboratory of Parallel and Distributed Computing, College of Computer Science and Technology, National University of Defense Technology, Changsha, China"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"263","reference":[{"key":"ref1","article-title":"GShard: Scaling giant models with conditional computation and automatic sharding","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Lepikhin","year":"2021"},{"key":"ref2","first-page":"5547","article-title":"GLaM: Efficient scaling of language models with mixture-of-experts","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Du","year":"2022"},{"issue":"1","key":"ref3","first-page":"5232","article-title":"Switch transformers: Scaling to trillion parameter models with simple and efficient sparsity","volume":"23","author":"Fedus","year":"2022","journal-title":"J. Mach. Learn. Res."},{"key":"ref4","first-page":"18332","article-title":"DeepSpeed-MoE: Advancing mixture-of-experts inference and training to power next-generation AI scale","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Rajbhandari","year":"2022"},{"key":"ref5","first-page":"269","article-title":"Tutel: Adaptive mixture-of-experts at scale","volume-title":"Proc. Mach. Learn. Syst.","volume":"5","author":"Hwang","year":"2023"},{"key":"ref6","article-title":"Outrageously large neural networks: The sparsely-gated mixture-of-experts layer","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Shazeer","year":"2017"},{"key":"ref7","doi-asserted-by":"publisher","DOI":"10.1109\/TKDE.2025.3554028"},{"key":"ref8","doi-asserted-by":"publisher","DOI":"10.1145\/3503221.3508418"},{"key":"ref9","doi-asserted-by":"publisher","DOI":"10.1145\/3588964"},{"key":"ref10","doi-asserted-by":"publisher","DOI":"10.1109\/SC41405.2020.00024"},{"key":"ref11","article-title":"Scaling laws for neural language models","author":"Kaplan","year":"2020"},{"key":"ref12","doi-asserted-by":"publisher","DOI":"10.24818\/ida-ql\/2019.5"},{"issue":"1","key":"ref13","first-page":"5485","article-title":"Exploring the limits of transfer learning with a unified text-to-text transformer","volume":"21","author":"Raffel","year":"2020","journal-title":"J. Mach. Learn. Res."},{"key":"ref14","first-page":"5753","article-title":"XLNet: Generalized autoregressive pretraining for language understanding","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","author":"Yang","year":"2019"},{"key":"ref15","article-title":"RoBERTa: A robustly optimized BERT pretraining approach","author":"Liu","year":"2019"},{"issue":"8","key":"ref16","article-title":"Language models are unsupervised multitask learners","volume":"1","author":"Radford","year":"2019","journal-title":"OpenAI Blog"},{"key":"ref17","first-page":"1877","article-title":"Language models are few-shot learners","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","author":"Brown","year":"2020"},{"key":"ref18","article-title":"Using DeepSpeed and megatron to train megatron-turing NLG 530B, a large-scale generative language model","author":"Smith","year":"2022"},{"key":"ref19","article-title":"Mixture of a million experts","author":"He","year":"2024"},{"key":"ref20","article-title":"Taming sparsely activated transformer with stochastic experts","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Zuo","year":"2022"},{"key":"ref21","first-page":"33270","article-title":"Scaling laws for fine-grained mixture of experts","volume-title":"Proc. 41st Int. Conf. Mach. Learn.","author":"Ludziejewski","year":"2024"},{"key":"ref22","article-title":"Sparse upcycling: Training mixture-of-experts from dense checkpoints","volume-title":"Proc. Eleventh Int. Conf. Learn. Representations","author":"Komatsuzaki","year":"2023"},{"key":"ref23","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v36i8.20858"},{"key":"ref24","article-title":"One student knows all experts know: From sparse to dense","author":"Xue","year":"2022"},{"key":"ref25","doi-asserted-by":"publisher","DOI":"10.1109\/ipdpsw55747.2022.00171"},{"key":"ref26","article-title":"DeepSeek-V2: A strong, economical, and efficient mixture-of-experts language model","author":"Liu","year":"2024"},{"key":"ref27","article-title":"The llama 4 herd: The beginning of a new era of natively multimodal AI innovation","year":"2025"},{"key":"ref28","article-title":"GPT-4 technical report","author":"Achiam","year":"2023"},{"key":"ref29","article-title":"DeepSeek-v3 technical report","author":"Liu","year":"2024"},{"key":"ref30","article-title":"Qwen3 technical report","author":"Yang","year":"2025"},{"key":"ref31","article-title":"Announcing the availability of Phi-3.5-MoE in azure AI studio and GitHub","year":"2024"},{"key":"ref32","article-title":"OLMoE: Open mixture-of-experts language models","author":"Muennighoff","year":"2024"},{"key":"ref33","doi-asserted-by":"publisher","DOI":"10.1109\/TAC.2007.898077"},{"key":"ref34","doi-asserted-by":"publisher","DOI":"10.1109\/ICPP.2008.83"},{"key":"ref35","doi-asserted-by":"publisher","DOI":"10.1007\/3-540-45706-2_112"},{"key":"ref36","article-title":"Megatron-LM: Training multi-billion parameter language models using model parallelism","author":"Shoeybi","year":"2019"},{"key":"ref37","article-title":"HetuMoE: An efficient trillion-scale mixture-of-expert distributed training system","author":"Nie","year":"2022"},{"key":"ref38","article-title":"FastMoE: A fast mixture-of-expert training system","author":"He","year":"2021"},{"key":"ref39","doi-asserted-by":"publisher","DOI":"10.1109\/TSC.2024.3399654"},{"key":"ref40","article-title":"OpenWebText corpus","author":"Gokaslan","year":"2019"},{"key":"ref41","first-page":"945","article-title":"Accelerating distributed MoE training and inference with Lina","volume-title":"Proc. USENIX Annu. Tech. Conf.","author":"Li","year":"2023"},{"key":"ref42","doi-asserted-by":"publisher","DOI":"10.1109\/CLUSTER52292.2023.00015"},{"key":"ref43","doi-asserted-by":"publisher","DOI":"10.1145\/3603269.3604869"},{"key":"ref44","doi-asserted-by":"publisher","DOI":"10.52202\/068431-0480"},{"key":"ref45","doi-asserted-by":"publisher","DOI":"10.1109\/TPDS.2023.3247001"},{"key":"ref46","doi-asserted-by":"publisher","DOI":"10.1145\/3472456.3472497"},{"key":"ref47","doi-asserted-by":"publisher","DOI":"10.1145\/3605573.3605613"},{"key":"ref48","first-page":"24829","article-title":"Piper: Multidimensional planner for DNN parallelization","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","author":"Tarnawski","year":"2021"},{"key":"ref49","doi-asserted-by":"publisher","DOI":"10.1360\/SSI-2023-0051"},{"key":"ref50","doi-asserted-by":"publisher","DOI":"10.14778\/3611540.3611569"},{"key":"ref51","doi-asserted-by":"publisher","DOI":"10.14778\/3561261.3561265"},{"key":"ref52","article-title":"Maximizing parallelism in distributed training for huge neural networks","author":"Bian","year":"2021"},{"key":"ref53","doi-asserted-by":"publisher","DOI":"10.1109\/TC.2024.3517748"},{"key":"ref54","doi-asserted-by":"publisher","DOI":"10.1109\/CLUSTER51413.2022.00043"},{"key":"ref55","doi-asserted-by":"publisher","DOI":"10.1145\/3722113"},{"key":"ref56","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2023.acl-long.134"},{"key":"ref57","first-page":"341","article-title":"Reducing activation recomputation in large transformer models","volume-title":"Proc. Mach. Learn. Syst.","volume":"5","author":"Korthikanti","year":"2023"},{"key":"ref58","doi-asserted-by":"publisher","DOI":"10.1145\/3662158.3662806"},{"key":"ref59","article-title":"Ring attention with blockwise transformers for near-infinite context","volume-title":"Proc. Twelfth Int. Conf. Learn. Representations","author":"Liu","year":"2023"},{"key":"ref60","doi-asserted-by":"publisher","DOI":"10.1145\/3503221.3508417"},{"key":"ref61","doi-asserted-by":"publisher","DOI":"10.1109\/INFOCOM52122.2024.10621327"},{"key":"ref62","doi-asserted-by":"publisher","DOI":"10.1145\/3577193.3593704"},{"key":"ref63","doi-asserted-by":"publisher","DOI":"10.1109\/INFOCOM.2019.8737367"},{"key":"ref64","first-page":"4150","article-title":"PipeTransformer: Automated elastic pipelining for distributed training of transformers","volume-title":"Proc. 38th Int. Conf. on Mach. Learn.","volume":"139","author":"He","year":"2021"},{"key":"ref65","doi-asserted-by":"publisher","DOI":"10.1109\/TPDS.2024.3406420"},{"key":"ref66","doi-asserted-by":"publisher","DOI":"10.1145\/3627703.3650083"},{"key":"ref67","doi-asserted-by":"publisher","DOI":"10.1109\/INFOCOM53939.2023.10228874"}],"container-title":["IEEE Transactions on Parallel and Distributed Systems"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx8\/71\/11434582\/11417698.pdf?arnumber=11417698","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,3,23]],"date-time":"2026-03-23T23:23:13Z","timestamp":1774308193000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/11417698\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026,5]]},"references-count":67,"journal-issue":{"issue":"5"},"URL":"https:\/\/doi.org\/10.1109\/tpds.2026.3668887","relation":{},"ISSN":["1045-9219","1558-2183","2161-9883"],"issn-type":[{"value":"1045-9219","type":"print"},{"value":"1558-2183","type":"electronic"},{"value":"2161-9883","type":"electronic"}],"subject":[],"published":{"date-parts":[[2026,5]]}}}