{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,7]],"date-time":"2026-04-07T16:15:45Z","timestamp":1775578545385,"version":"3.50.1"},"reference-count":76,"publisher":"Institute of Electrical and Electronics Engineers (IEEE)","issue":"12","license":[{"start":{"date-parts":[[2025,12,1]],"date-time":"2025-12-01T00:00:00Z","timestamp":1764547200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/ieeexplore.ieee.org\/Xplorehelp\/downloads\/license-information\/IEEE.html"},{"start":{"date-parts":[[2025,12,1]],"date-time":"2025-12-01T00:00:00Z","timestamp":1764547200000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2025,12,1]],"date-time":"2025-12-01T00:00:00Z","timestamp":1764547200000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"}],"funder":[{"DOI":"10.13039\/501100012166","name":"National Key Research and Development Program of China","doi-asserted-by":"publisher","award":["2022ZD0160101"],"award-info":[{"award-number":["2022ZD0160101"]}],"id":[{"id":"10.13039\/501100012166","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100007219","name":"Natural Science Foundation of Shanghai Municipality","doi-asserted-by":"publisher","award":["23ZR1402900"],"award-info":[{"award-number":["23ZR1402900"]}],"id":[{"id":"10.13039\/100007219","id-type":"DOI","asserted-by":"publisher"}]},{"name":"Shanghai Science and Technology Commission Explorer Program Project","award":["24TS1401300"],"award-info":[{"award-number":["24TS1401300"]}]},{"DOI":"10.13039\/501100010256","name":"Shanghai Municipal Science and Technology Major Project","doi-asserted-by":"publisher","award":["2021SHZDZX0103"],"award-info":[{"award-number":["2021SHZDZX0103"]}],"id":[{"id":"10.13039\/501100010256","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["IEEE Trans. Circuits Syst. Video Technol."],"published-print":{"date-parts":[[2025,12]]},"DOI":"10.1109\/tcsvt.2025.3586550","type":"journal-article","created":{"date-parts":[[2025,7,7]],"date-time":"2025-07-07T13:53:44Z","timestamp":1751896424000},"page":"12329-12340","source":"Crossref","is-referenced-by-count":1,"title":["Sparse-to-Dense Training: A Novel Training Scheme to Enhance Vision Transformers"],"prefix":"10.1109","volume":"35","author":[{"ORCID":"https:\/\/orcid.org\/0009-0009-7190-4983","authenticated-orcid":false,"given":"Yongqi","family":"Huang","sequence":"first","affiliation":[{"name":"College of Future Information Technology, Fudan University, Shanghai, China"}]},{"given":"Peng","family":"Ye","sequence":"additional","affiliation":[{"name":"College of Future Information Technology, Fudan University, Shanghai, China"}]},{"ORCID":"https:\/\/orcid.org\/0009-0003-7405-2022","authenticated-orcid":false,"given":"Chongjun","family":"Tu","sequence":"additional","affiliation":[{"name":"College of Future Information Technology, Fudan University, Shanghai, China"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-0779-9818","authenticated-orcid":false,"given":"Tao","family":"Chen","sequence":"additional","affiliation":[{"name":"College of Future Information Technology, Fudan University, Shanghai, China"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-2772-9320","authenticated-orcid":false,"given":"Tong","family":"He","sequence":"additional","affiliation":[{"name":"Shanghai Artificial Intelligence Laboratory, Shanghai, China"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-9163-2761","authenticated-orcid":false,"given":"Wanli","family":"Ouyang","sequence":"additional","affiliation":[{"name":"Shanghai Artificial Intelligence Laboratory, Shanghai, China"}]}],"member":"263","reference":[{"key":"ref1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR46437.2021.01352"},{"key":"ref2","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2019.00200"},{"key":"ref3","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR46437.2021.01074"},{"key":"ref4","article-title":"RepMLP: Re-parameterizing convolutions into fully-connected layers for image recognition","author":"Ding","year":"2021","journal-title":"arXiv:2105.01883"},{"key":"ref5","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2016.90"},{"key":"ref6","article-title":"An image is worth 16\u00d716 words: Transformers for image recognition at scale","author":"Dosovitskiy","year":"2020","journal-title":"arXiv:2010.11929"},{"key":"ref7","article-title":"GShard: Scaling giant models with conditional computation and automatic sharding","author":"Lepikhin","year":"2020","journal-title":"arXiv:2006.16668"},{"key":"ref8","article-title":"Outrageously large neural networks: The sparsely-gated mixture-of-experts layer","author":"Shazeer","year":"2017","journal-title":"arXiv:1701.06538"},{"issue":"1","key":"ref9","first-page":"5232","article-title":"Switch transformers: Scaling to trillion parameter models with simple and efficient sparsity","volume":"23","author":"Fedus","year":"2021","journal-title":"J. Mach. Learn. Res."},{"key":"ref10","first-page":"8583","article-title":"Scaling vision with sparse mixture of experts","volume-title":"Proc. NIPS","volume":"34","author":"Riquelme"},{"key":"ref11","first-page":"269","article-title":"Tutel: Adaptive mixture-of-experts at scale","volume-title":"Proc. Mach. Learn. Syst.","volume":"5","author":"Hwang"},{"key":"ref12","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52688.2022.01170"},{"key":"ref13","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2017.97"},{"key":"ref14","article-title":"Scale out for large minibatch SGD: Residual network training on ImageNet-1K with improved accuracy and reduced time to train","author":"Codreanu","year":"2017","journal-title":"arXiv:1711.04291"},{"key":"ref15","doi-asserted-by":"publisher","DOI":"10.1007\/s11263-015-0816-y"},{"key":"ref16","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2016.170"},{"key":"ref17","first-page":"6265","article-title":"BASE layers: Simplifying training of large, sparse models","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Lewis"},{"key":"ref18","first-page":"7103","article-title":"Mixture-of-Experts with expert choice routing","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","volume":"35","author":"Zhou"},{"key":"ref19","first-page":"17555","article-title":"Hash layers for large sparse models","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","author":"Roller"},{"key":"ref20","article-title":"Taming sparsely activated transformer with stochastic experts","author":"Zuo","year":"2021","journal-title":"arXiv:2110.04260"},{"key":"ref21","doi-asserted-by":"publisher","DOI":"10.1145\/3503221.3508418"},{"key":"ref22","first-page":"18332","article-title":"DeepSpeed-MoE: Advancing mixture-of-experts inference and training to power next-generation AI scale","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Rajbhandari"},{"key":"ref23","doi-asserted-by":"publisher","DOI":"10.1145\/3588964"},{"key":"ref24","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2024.acl-long.70"},{"key":"ref25","article-title":"Multimodal contrastive learning with LIMoE: The language-image mixture of experts","author":"Mustafa","year":"2022","journal-title":"arXiv:2206.02770"},{"key":"ref26","article-title":"MoE-LLaVA: Mixture of experts for large vision-language models","author":"Lin","year":"2024","journal-title":"arXiv:2401.15947"},{"key":"ref27","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v36i8.20858"},{"key":"ref28","article-title":"Sparse upcycling: Training mixture-of-experts from dense checkpoints","author":"Komatsuzaki","year":"2022","journal-title":"arXiv:2212.05055"},{"key":"ref29","article-title":"One Student knows all experts know: From sparse to dense","author":"Xue","year":"2022","journal-title":"arXiv:2201.10890"},{"key":"ref30","doi-asserted-by":"publisher","DOI":"10.1109\/TCSVT.2024.3411804"},{"key":"ref31","first-page":"1298","article-title":"ExpandNets: Linear over-parameterization to train compact convolutional networks","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","volume":"33","author":"Guo"},{"key":"ref32","article-title":"RepNAS: Searching for efficient re-parameterizing blocks","author":"Zhang","year":"2021","journal-title":"arXiv:2109.03508"},{"key":"ref33","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV48922.2021.00447"},{"key":"ref34","first-page":"1195","article-title":"Mean teachers are better role models: Weight-averaged consistency targets improve semi-supervised deep learning results","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","volume":"30","author":"Tarvainen"},{"key":"ref35","doi-asserted-by":"publisher","DOI":"10.1109\/TCSVT.2024.3405069"},{"key":"ref36","doi-asserted-by":"publisher","DOI":"10.1109\/TCSVT.2024.3402533"},{"key":"ref37","doi-asserted-by":"publisher","DOI":"10.1109\/TCSVT.2023.3243205"},{"key":"ref38","doi-asserted-by":"publisher","DOI":"10.1109\/TCSVT.2023.3301854"},{"key":"ref39","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52688.2022.00706"},{"key":"ref40","article-title":"Averaging weights leads to wider optima and better generalization","author":"Izmailov","year":"2018","journal-title":"arXiv:1803.05407"},{"key":"ref41","first-page":"23965","article-title":"Model soups: Averaging weights of multiple fine-tuned models improves accuracy without increasing inference time","volume-title":"Proc. 39th Int. Conf. Mach. Learn.","author":"Wortsman"},{"key":"ref42","first-page":"28656","article-title":"Model ratatouille: Recycling diverse models for out-of-distribution generalization","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Ram\u00e9"},{"key":"ref43","article-title":"Learning multiple layers of features from tiny images","author":"Krizhevsky","year":"2009"},{"key":"ref44","article-title":"Combining weakly and webly supervised learning for classifying food images","author":"Kaur","year":"2017","journal-title":"arXiv:1712.08730"},{"issue":"7","key":"ref45","first-page":"3","article-title":"Tiny imagenet visual recognition challenge","volume":"7","author":"Le","year":"2015","journal-title":"CS 231N"},{"key":"ref46","article-title":"Are we done with ImageNet?","author":"Beyer","year":"2020","journal-title":"arXiv:2006.07159"},{"key":"ref47","first-page":"5389","article-title":"Do ImageNet classifiers generalize to ImageNet","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Recht"},{"key":"ref48","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR46437.2021.01501"},{"key":"ref49","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV48922.2021.00823"},{"key":"ref50","article-title":"Benchmarking neural network robustness to common corruptions and perturbations","author":"Hendrycks","year":"2019","journal-title":"arXiv:1903.12261"},{"key":"ref51","article-title":"Vision transformer for small-size datasets","author":"Lee","year":"2021","journal-title":"arXiv:2112.13492"},{"key":"ref52","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV48922.2021.01172"},{"key":"ref53","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV48922.2021.00060"},{"key":"ref54","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV48922.2021.00986"},{"key":"ref55","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-19803-8_5"},{"key":"ref56","article-title":"Vision transformer adapter for dense predictions","author":"Chen","year":"2022","journal-title":"arXiv:2205.08534"},{"key":"ref57","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-10602-1_48"},{"key":"ref58","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2017.322"},{"key":"ref59","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2017.544"},{"key":"ref60","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-01228-1_26"},{"key":"ref61","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2015.7298801"},{"key":"ref62","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52688.2022.01871"},{"key":"ref63","doi-asserted-by":"publisher","DOI":"10.1109\/3DV62453.2024.00113"},{"key":"ref64","article-title":"Mistral 7B","author":"Jiang","year":"2023","journal-title":"arXiv:2310.06825"},{"key":"ref65","article-title":"Stable code technical report","volume-title":"arXiv:2404.01226","author":"Pinnaparaju","year":"2024"},{"key":"ref66","article-title":"Code llama: Open foundation models for code","author":"Rozi\u00e8re","year":"2023","journal-title":"arXiv:2308.12950"},{"key":"ref67","article-title":"DeepSeek-coder: When the large language model meets programming\u2014The rise of code intelligence","author":"Guo","year":"2024","journal-title":"arXiv:2401.14196"},{"key":"ref68","article-title":"BERT: Pre-training of deep bidirectional transformers for language understanding","author":"Devlin","year":"2018","journal-title":"arXiv:1810.04805"},{"key":"ref69","doi-asserted-by":"publisher","DOI":"10.48550\/ARXIV.1907.11692"},{"key":"ref70","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/W18-5446"},{"key":"ref71","article-title":"WizardCoder: Empowering code large language models with evol-instruct","author":"Luo","year":"2023","journal-title":"arXiv:2306.08568"},{"key":"ref72","article-title":"Evaluating large language models trained on code","author":"Chen","year":"2021","journal-title":"arXiv:2107.03374"},{"key":"ref73","article-title":"Is your code generated by ChatGPT really correct? Rigorous evaluation of large language models for code generation","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","volume":"36","author":"Liu"},{"key":"ref74","article-title":"Program synthesis with large language models","author":"Austin","year":"2021","journal-title":"arXiv:2108.07732"},{"key":"ref75","first-page":"18319","article-title":"DS-1000: A natural and reliable benchmark for data science code generation","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Lai"},{"key":"ref76","article-title":"Magicoder: Empowering code generation with OSS-instruct","author":"Wei","year":"2023","journal-title":"arXiv:2312.02120"}],"container-title":["IEEE Transactions on Circuits and Systems for Video Technology"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx8\/76\/11278843\/11072250.pdf?arnumber=11072250","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,1,5]],"date-time":"2026-01-05T18:41:08Z","timestamp":1767638468000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/11072250\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,12]]},"references-count":76,"journal-issue":{"issue":"12"},"URL":"https:\/\/doi.org\/10.1109\/tcsvt.2025.3586550","relation":{},"ISSN":["1051-8215","1558-2205"],"issn-type":[{"value":"1051-8215","type":"print"},{"value":"1558-2205","type":"electronic"}],"subject":[],"published":{"date-parts":[[2025,12]]}}}