{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,7]],"date-time":"2026-04-07T21:00:19Z","timestamp":1775595619421,"version":"3.50.1"},"reference-count":58,"publisher":"Institute of Electrical and Electronics Engineers (IEEE)","issue":"5","license":[{"start":{"date-parts":[[2026,5,1]],"date-time":"2026-05-01T00:00:00Z","timestamp":1777593600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/ieeexplore.ieee.org\/Xplorehelp\/downloads\/license-information\/IEEE.html"},{"start":{"date-parts":[[2026,5,1]],"date-time":"2026-05-01T00:00:00Z","timestamp":1777593600000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2026,5,1]],"date-time":"2026-05-01T00:00:00Z","timestamp":1777593600000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"}],"funder":[{"DOI":"10.13039\/501100021171","name":"Basic and Applied Basic Research Foundation of Guangdong Province","doi-asserted-by":"publisher","award":["2024B1515020032"],"award-info":[{"award-number":["2024B1515020032"]}],"id":[{"id":"10.13039\/501100021171","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100021171","name":"Basic and Applied Basic Research Foundation of Guangdong Province","doi-asserted-by":"publisher","award":["2024A1515010253"],"award-info":[{"award-number":["2024A1515010253"]}],"id":[{"id":"10.13039\/501100021171","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["62276279"],"award-info":[{"award-number":["62276279"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["62306344"],"award-info":[{"award-number":["62306344"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["62372483"],"award-info":[{"award-number":["62372483"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["62276280"],"award-info":[{"award-number":["62276280"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["U22B2060"],"award-info":[{"award-number":["U22B2060"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]},{"name":"Research Foundation of Science and in part by Technology Plan Project of Guangzhou City","award":["2023B01J0001"],"award-info":[{"award-number":["2023B01J0001"]}]},{"name":"Research Foundation of Science and in part by Technology Plan Project of Guangzhou City","award":["2024B01W0004"],"award-info":[{"award-number":["2024B01W0004"]}]}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["IEEE Trans. Pattern Anal. Mach. Intell."],"published-print":{"date-parts":[[2026,5]]},"DOI":"10.1109\/tpami.2026.3651754","type":"journal-article","created":{"date-parts":[[2026,1,12]],"date-time":"2026-01-12T22:00:30Z","timestamp":1768255230000},"page":"5505-5519","source":"Crossref","is-referenced-by-count":0,"title":["Accelerated Optimization of Large Mixture-of-Experts Models by Density-Aware Multi-Stage Learning"],"prefix":"10.1109","volume":"48","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-1340-3995","authenticated-orcid":false,"given":"Jianxing","family":"Yu","sequence":"first","affiliation":[{"name":"School of Artificial Intelligence, Sun Yat-Sen University, Zhuhai, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-5619-5252","authenticated-orcid":false,"given":"Haowei","family":"Jiang","sequence":"additional","affiliation":[{"name":"School of Artificial Intelligence, Sun Yat-Sen University, Zhuhai, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-8263-9032","authenticated-orcid":false,"given":"Huaijie","family":"Zhu","sequence":"additional","affiliation":[{"name":"School of Artificial Intelligence, Sun Yat-Sen University, Zhuhai, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-8739-2216","authenticated-orcid":false,"given":"Wenqing","family":"Chen","sequence":"additional","affiliation":[{"name":"School of Software Engineering, Sun Yat-sen University, Zhuhai, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-1610-9599","authenticated-orcid":false,"given":"Yanghui","family":"Rao","sequence":"additional","affiliation":[{"name":"School of Computer Science and Engineering, Sun Yat-sen University, Guangzhou, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-5903-2504","authenticated-orcid":false,"given":"Qinliang","family":"Su","sequence":"additional","affiliation":[{"name":"School of Computer Science and Engineering, Sun Yat-sen University, Guangzhou, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-1214-5384","authenticated-orcid":false,"given":"Jian","family":"Yin","sequence":"additional","affiliation":[{"name":"School of Artificial Intelligence, Sun Yat-Sen University, Zhuhai, China"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"263","reference":[{"key":"ref1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-10599-4_29"},{"key":"ref2","first-page":"1","article-title":"Sparse MoE as the new dropout: Scaling dense and self-slimmable transformers","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Chen","year":"2023"},{"key":"ref3","first-page":"1","article-title":"Net2Net: Accelerating learning via knowledge transfer","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Chen","year":"2016"},{"key":"ref4","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2022.acl-long.581"},{"key":"ref5","doi-asserted-by":"publisher","DOI":"10.1109\/TC.2019.2914438"},{"key":"ref6","first-page":"1","article-title":"Universal transformers","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Dehghani","year":"2019"},{"key":"ref7","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2009.5206848"},{"key":"ref8","doi-asserted-by":"publisher","DOI":"10.48550\/arXiv.1810.04805"},{"key":"ref9","first-page":"2616","article-title":"Towards adaptive residual network training: A neural-ODE perspective","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Dong","year":"2020"},{"key":"ref10","first-page":"1","article-title":"An image is worth 16 \u00d7 16 words: Transformers for image recognition at scale","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Dosovitskiy","year":"2021"},{"key":"ref11","first-page":"8469","article-title":"PaLM-E: An embodied multimodal language model","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Driess","year":"2023"},{"key":"ref12","first-page":"5547","article-title":"GLaM: Efficient scaling of language models with mixture-of-experts","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Du","year":"2022"},{"key":"ref13","first-page":"2943","article-title":"Rigging the lottery: Making all tickets winners","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Evci","year":"2020"},{"key":"ref14","first-page":"1","article-title":"Switch transformers: Scaling to trillion parameter models with simple and efficient sparsity","volume":"23","author":"Fedus","year":"2022","journal-title":"J. Mach. Learn. Res."},{"key":"ref15","doi-asserted-by":"publisher","DOI":"10.1016\/j.asoc.2023.110788"},{"key":"ref16","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2021.emnlp-main.446"},{"key":"ref17","first-page":"2337","article-title":"Efficient training of BERT by progressively stacking","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Gong","year":"2019"},{"key":"ref18","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2021.naacl-main.406"},{"key":"ref19","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2023.findings-acl.580"},{"key":"ref20","doi-asserted-by":"publisher","DOI":"10.1016\/j.neunet.2021.08.032"},{"key":"ref21","first-page":"5562","article-title":"The Lipschitz constant of self-attention","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Kim","year":"2021"},{"key":"ref22","doi-asserted-by":"publisher","DOI":"10.1109\/ICCVW.2013.77"},{"key":"ref23","first-page":"1","article-title":"Learning multiple layers of features from tiny images","volume-title":"","author":"Krizhevsky","year":"2009"},{"key":"ref24","first-page":"1","article-title":"ALBERT: A lite BERT for self-supervised learning of language representations","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Lan","year":"2020"},{"key":"ref25","first-page":"1","article-title":"GShard: Scaling giant models with conditional computation and automatic sharding","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Lepikhin","year":"2021"},{"key":"ref26","first-page":"6265","article-title":"BASE layers: Simplifying training of large, sparse models","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Lewis","year":"2021"},{"key":"ref27","doi-asserted-by":"publisher","DOI":"10.1525\/9780520940420-020"},{"key":"ref28","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52688.2022.01216"},{"key":"ref29","doi-asserted-by":"publisher","DOI":"10.1109\/IJCNN60899.2024.10651105"},{"key":"ref30","doi-asserted-by":"publisher","DOI":"10.1016\/j.engappai.2023.106300"},{"key":"ref31","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV48922.2021.00986"},{"key":"ref32","first-page":"1","article-title":"Decoupled weight decay regularization","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Loshchilov","year":"2019"},{"key":"ref33","first-page":"1","article-title":"Mixed precision training","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Micikevicius","year":"2018"},{"key":"ref34","doi-asserted-by":"publisher","DOI":"10.1109\/ICVGIP.2008.47"},{"key":"ref35","first-page":"1","article-title":"Language models are unsupervised multitask learners","volume":"1","author":"Radford","year":"2019","journal-title":"Blog OpenAI"},{"key":"ref36","first-page":"14011","article-title":"DeepSpeed-MoE: Advancing Mixture-of-Experts inference and training to power next-generation AI scale","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Rajbhandari","year":"2022"},{"key":"ref37","first-page":"8583","article-title":"Scaling vision with sparse mixture of experts","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","author":"Riquelme","year":"2021"},{"key":"ref38","first-page":"1","article-title":"Outrageously large neural networks: The sparsely-gated mixture-of-experts layer","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Shazeer","year":"2017"},{"key":"ref39","first-page":"19893","article-title":"Staged training for transformer language models","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Shen","year":"2022"},{"key":"ref40","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2023.findings-emnlp.758"},{"key":"ref41","first-page":"10347","article-title":"Training data-efficient image transformers & distillation through attention","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Touvron","year":"2021"},{"key":"ref42","doi-asserted-by":"publisher","DOI":"10.48550\/ARXIV.1706.03762"},{"key":"ref43","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/w18-5446"},{"key":"ref44","first-page":"1","article-title":"Learning to grow pretrained models for efficient transformer training","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Wang","year":"2023"},{"key":"ref45","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2023.acl-long.114"},{"key":"ref46","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52729.2023.01385"},{"key":"ref47","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2017.369"},{"key":"ref48","first-page":"564","article-title":"Network morphism","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Wei","year":"2016"},{"key":"ref49","doi-asserted-by":"publisher","DOI":"10.1145\/3394486.3403126"},{"key":"ref50","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2024.acl-long.352"},{"key":"ref51","first-page":"22373","article-title":"Firefly neural architecture descent: A general approach for growing neural networks","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","author":"Wu","year":"2020"},{"key":"ref52","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v36i8.20858"},{"key":"ref53","doi-asserted-by":"publisher","DOI":"10.21437\/Interspeech.2021-478"},{"key":"ref54","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52733.2024.02191"},{"key":"ref55","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52733.2024.00550"},{"key":"ref56","doi-asserted-by":"publisher","DOI":"10.1109\/IJCNN60899.2024.10650737"},{"key":"ref57","first-page":"14011","article-title":"Accelerating training of transformer-based language models with progressive layer dropping","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","author":"Zhang","year":"2020"},{"key":"ref58","doi-asserted-by":"publisher","DOI":"10.52202\/068431-0515"}],"container-title":["IEEE Transactions on Pattern Analysis and Machine Intelligence"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx8\/34\/11474534\/11342294.pdf?arnumber=11342294","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,4,7]],"date-time":"2026-04-07T19:59:32Z","timestamp":1775591972000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/11342294\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026,5]]},"references-count":58,"journal-issue":{"issue":"5"},"URL":"https:\/\/doi.org\/10.1109\/tpami.2026.3651754","relation":{},"ISSN":["0162-8828","2160-9292","1939-3539"],"issn-type":[{"value":"0162-8828","type":"print"},{"value":"2160-9292","type":"electronic"},{"value":"1939-3539","type":"electronic"}],"subject":[],"published":{"date-parts":[[2026,5]]}}}