{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,8,5]],"date-time":"2026-08-05T17:58:59Z","timestamp":1785952739696,"version":"3.56.0"},"publisher-location":"New York, NY, USA","reference-count":53,"publisher":"ACM","license":[{"start":{"date-parts":[[2025,11,15]],"date-time":"2025-11-15T00:00:00Z","timestamp":1763164800000},"content-version":"vor","delay-in-days":0,"URL":"http:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"DOI":"10.13039\/100000015","name":"DOE U.S. Department of Energy","doi-asserted-by":"publisher","award":["DEAC02\u201306CH11357, 0F--60169"],"award-info":[{"award-number":["DEAC02\u201306CH11357, 0F--60169"]}],"id":[{"id":"10.13039\/100000015","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000001","name":"NSF (National Science Foundation)","doi-asserted-by":"publisher","award":["2411386, 2411387, 2106635"],"award-info":[{"award-number":["2411386, 2411387, 2106635"]}],"id":[{"id":"10.13039\/100000001","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2025,11,16]]},"DOI":"10.1145\/3712285.3759864","type":"proceedings-article","created":{"date-parts":[[2025,11,12]],"date-time":"2025-11-12T16:04:47Z","timestamp":1762963487000},"page":"1381-1394","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":1,"title":["MLP-Offload: Multi-Level, Multi-Path Offloading for LLM Pre-training to Break the GPU Memory Wall"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-8200-0148","authenticated-orcid":false,"given":"Avinash Kumar","family":"Maurya","sequence":"first","affiliation":[{"name":"Argonne National Laboratory, Lemont, Illinois, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-5034-2880","authenticated-orcid":false,"given":"M. Mustafa","family":"Rafique","sequence":"additional","affiliation":[{"name":"Rochester Institute of Technology, Rochester, New York, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-7890-3934","authenticated-orcid":false,"given":"Franck","family":"Cappello","sequence":"additional","affiliation":[{"name":"Argonne National Laboratory, Lemont, Illinois, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-0661-7509","authenticated-orcid":false,"given":"Bogdan","family":"Nicolae","sequence":"additional","affiliation":[{"name":"Argonne National Laboratory, Lemont, Illinois, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2025,11,15]]},"reference":[{"key":"e_1_3_3_3_2_2","unstructured":"Amazon Web Services. 2025. Amazon FSx for Lustre Pricing. https:\/\/aws.amazon.com\/fsx\/lustre\/pricing\/. Accessed: 2025-02-05."},{"key":"e_1_3_3_3_3_2","unstructured":"Argonne Joint Laboratory for System Evaluation (JLSE). 2023. NVIDIA H100. https:\/\/www.jlse.anl.gov\/nvidia-h100. Accessed: 2025-02-04."},{"key":"e_1_3_3_3_4_2","unstructured":"Argonne Leadership Computing Facility. 2025. Polaris. https:\/\/www.alcf.anl.gov\/polaris."},{"key":"e_1_3_3_3_5_2","unstructured":"Tom\u00a0B. Brown Benjamin Mann Nick Ryder Melanie Subbiah Jared Kaplan et\u00a0al. 2020. Language models are few-shot learners. Advances in neural information processing systems 33 (2020) 1877\u20131901."},{"key":"e_1_3_3_3_6_2","doi-asserted-by":"crossref","unstructured":"Zhenkun Cai Xiao Yan Kaihao Ma Yidi Wu Yuzhen Huang James Cheng Teng Su and Fan Yu. 2021. Tensoropt: Exploring the tradeoffs in distributed dnn training with auto-parallelism. IEEE Transactions on Parallel and Distributed Systems 33 8 (2021) 1967\u20131981.","DOI":"10.1109\/TPDS.2021.3132413"},{"key":"e_1_3_3_3_7_2","doi-asserted-by":"publisher","DOI":"10.1109\/ICPADS63350.2024.00097"},{"key":"e_1_3_3_3_8_2","unstructured":"DeepSpeed Team. n. d.. Memory Requirements- DeepSpeed Documentation. https:\/\/deepspeed.readthedocs.io\/en\/latest\/memory.html. Accessed: 2025-04-09."},{"key":"e_1_3_3_3_9_2","doi-asserted-by":"publisher","DOI":"10.1145\/3534056.3534945"},{"key":"e_1_3_3_3_10_2","unstructured":"Jiangfei Duan Shuo Zhang Zerui Wang Lijuan Jiang Wenwen Qu Qinghao Hu Guoteng Wang Qizhen Weng Hang Yan Xingcheng Zhang Xipeng Qiu Dahua Lin Yonggang Wen Xin Jin Tianwei Zhang and Peng Sun. 2024. Efficient Training of Large Language Models on Distributed Infrastructures: A Survey. arxiv:https:\/\/arXiv.org\/abs\/2407.20018\u00a0[cs.DC] https:\/\/arxiv.org\/abs\/2407.20018"},{"key":"e_1_3_3_3_11_2","doi-asserted-by":"crossref","unstructured":"Amir Gholami Zhewei Yao Sehoon Kim Coleman Hooper Michael\u00a0W Mahoney and Kurt Keutzer. 2024. AI and memory wall. Micro 44 3 (2024) 33\u201339.","DOI":"10.1109\/MM.2024.3373763"},{"key":"e_1_3_3_3_12_2","doi-asserted-by":"crossref","unstructured":"Mikaila\u00a0J. Gossman Bogdan Nicolae and Jon\u00a0C. Calhoun. 2024. Scalable I\/O aggregation for asynchronous multi-level checkpointing. Future Generation Computer Systems 160 (2024) 420\u2013432.","DOI":"10.1016\/j.future.2024.06.003"},{"key":"e_1_3_3_3_13_2","unstructured":"Matt Hamblen. 2023. Update: ChatGPT runs 10K Nvidia training GPUs with potential for thousands more. https:\/\/www.fierceelectronics.com\/sensors\/chatgpt-runs-10k-nvidia-training-gpus-potential-thousands-more."},{"key":"e_1_3_3_3_14_2","volume-title":"GPipe: efficient training of giant neural networks using pipeline parallelism","author":"Huang Yanping","year":"2019","unstructured":"Yanping Huang, Youlong Cheng, Ankur Bapna, Orhan Firat, Mia\u00a0Xu Chen, Dehao Chen, HyoukJoong Lee, Jiquan Ngiam, Quoc\u00a0V. Le, Yonghui Wu, and Zhifeng Chen. 2019. GPipe: efficient training of giant neural networks using pipeline parallelism. Curran Associates Inc., Red Hook, NY, USA."},{"key":"e_1_3_3_3_15_2","doi-asserted-by":"publisher","DOI":"10.1109\/HPCA57654.2024.00034"},{"key":"e_1_3_3_3_16_2","unstructured":"Xuanlin Jiang Yang Zhou Shiyi Cao Ion Stoica and Minlan Yu. 2024. NEO: Saving GPU Memory Crisis with CPU Offloading for Online LLM Inference. arxiv:https:\/\/arXiv.org\/abs\/2411.01142\u00a0[cs.DC] https:\/\/arxiv.org\/abs\/2411.01142"},{"key":"e_1_3_3_3_17_2","doi-asserted-by":"crossref","unstructured":"Yao Jiang Xinyu Yan Ge-Peng Ji Keren Fu Meijun Sun Huan Xiong Deng-Ping Fan and Fahad\u00a0Shahbaz Khan. 2024. Effectiveness assessment of recent large vision-language models. Visual Intelligence 2 1 (2024) 17.","DOI":"10.1007\/s44267-024-00050-1"},{"key":"e_1_3_3_3_18_2","unstructured":"Diederik\u00a0P Kingma and Jimmy Ba. 2014. Adam: A method for stochastic optimization. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/1412.6980 (2014)."},{"key":"e_1_3_3_3_19_2","unstructured":"Joonhyung Lee Jeongin Bae Byeongwook Kim Se\u00a0Jung Kwon and Dongsoo Lee. 2024. To FP8 and Back Again: Quantifying the Effects of Reducing Precision on LLM Training Stability. arxiv:https:\/\/arXiv.org\/abs\/2405.18710\u00a0[cs.LG] https:\/\/arxiv.org\/abs\/2405.18710"},{"key":"e_1_3_3_3_20_2","doi-asserted-by":"publisher","DOI":"10.1145\/3605573.3605613"},{"key":"e_1_3_3_3_21_2","unstructured":"Xiang Li Yiqun Yao Xin Jiang Xuezhi Fang Chao Wang Xinzhang Liu Zihan Wang Yu Zhao Xin Wang Yuyao Huang Shuangyong Song Yongxiang Li Zheng Zhang Bo Zhao Aixin Sun Yequan Wang Zhongjiang He Zhongyuan Wang Xuelong Li and Tiejun Huang. 2024. Tele-FLM Technical Report. arxiv:https:\/\/arXiv.org\/abs\/2404.16645"},{"key":"e_1_3_3_3_22_2","unstructured":"Changyue Liao Mo Sun Zihan Yang Kaiqi Chen Binhang Yuan Fei Wu and Zeke Wang. 2024. Adding NVMe SSDs to Enable and Accelerate 100B Model Fine-tuning on a Single GPU. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2403.06504 (2024)."},{"key":"e_1_3_3_3_23_2","unstructured":"Changyue Liao Mo Sun Zihan Yang Jun Xie Kaiqi Chen Binhang Yuan Fei Wu and Zeke Wang. 2024. LoHan: Low-Cost High-Performance Framework to Fine-Tune 100B Model on a Consumer GPU. arxiv:https:\/\/arXiv.org\/abs\/2403.06504\u00a0[cs.DC] https:\/\/arxiv.org\/abs\/2403.06504"},{"key":"e_1_3_3_3_24_2","unstructured":"Aixin Liu Bei Feng Bing Xue Bingxuan Wang Bochao Wu Chengda Lu Chenggang Zhao Chengqi Deng Chenyu Zhang Chong Ruan et\u00a0al. 2024. Deepseek-v3 technical report. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2412.19437 (2024)."},{"key":"e_1_3_3_3_25_2","doi-asserted-by":"publisher","DOI":"10.1145\/3625549.3658685"},{"key":"e_1_3_3_3_26_2","doi-asserted-by":"publisher","DOI":"10.1145\/3659995.3660038"},{"key":"e_1_3_3_3_27_2","doi-asserted-by":"publisher","DOI":"10.1145\/3652892.3700781"},{"key":"e_1_3_3_3_28_2","unstructured":"Paulius Micikevicius Sharan Narang Jonah Alben Gregory Diamos Erich Elsen David Garcia Boris Ginsburg Michael Houston Oleksii Kuchaiev Ganesh Venkatesh and Hao Wu. 2018. Mixed Precision Training. arxiv:https:\/\/arXiv.org\/abs\/1710.03740\u00a0[cs.AI] https:\/\/arxiv.org\/abs\/1710.03740"},{"key":"e_1_3_3_3_29_2","unstructured":"Jack\u00a0W. Rae Sebastian Borgeaud Trevor Cai Katie Millican et\u00a0al. 2022. Scaling Language Models: Methods Analysis & Insights from Training Gopher. arxiv:https:\/\/arXiv.org\/abs\/2112.11446\u00a0[cs.CL] https:\/\/arxiv.org\/abs\/2112.11446"},{"key":"e_1_3_3_3_30_2","doi-asserted-by":"publisher","DOI":"10.1109\/SC41405.2020.00024"},{"key":"e_1_3_3_3_31_2","doi-asserted-by":"publisher","DOI":"10.1145\/3458817.3476205"},{"key":"e_1_3_3_3_32_2","doi-asserted-by":"publisher","DOI":"10.1145\/3394486.3406703"},{"key":"e_1_3_3_3_33_2","first-page":"551","volume-title":"2021 USENIX Annual Technical Conference (USENIX ATC 21)","author":"Ren Jie","year":"2021","unstructured":"Jie Ren, Samyam Rajbhandari, Reza\u00a0Yazdani Aminabadi, Olatunji Ruwase, Shuangyan Yang, Minjia Zhang, Dong Li, and Yuxiong He. 2021. { Zero-offload} : Democratizing { billion-scale} model training. In 2021 USENIX Annual Technical Conference (USENIX ATC 21). 551\u2013564."},{"key":"e_1_3_3_3_34_2","unstructured":"Mohammad Shoeybi Mostofa Patwary Raul Puri Patrick LeGresley Jared Casper and Bryan Catanzaro. 2019. Megatron-LM: Training multi-billion parameter language models using model parallelism. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/1909.08053 (2019)."},{"key":"e_1_3_3_3_35_2","unstructured":"Shuaiwen\u00a0Leon Song Bonnie Kruft Minjia Zhang Conglong Li Shiyang Chen et\u00a0al. 2023. DeepSpeed4Science Initiative: Enabling Large-Scale Scientific Discovery through Sophisticated AI System Technologies. arXiv:https:\/\/arXiv.org\/abs\/2310.04610\u00a0[cs]"},{"key":"e_1_3_3_3_36_2","unstructured":"Ross Taylor Marcin Kardas Guillem Cucurull Thomas Scialom Anthony Hartshorn Elvis Saravia Andrew Poulton Viktor Kerkez and Robert Stojnic. 2022. Galactica: A Large Language Model for Science. arxiv:https:\/\/arXiv.org\/abs\/2211.09085\u00a0[cs.CL] https:\/\/arxiv.org\/abs\/2211.09085"},{"key":"e_1_3_3_3_37_2","unstructured":"DeepSpeed Team. 2022. GitHub Issue: Delayed parameter update in ZeRO-Offload. https:\/\/github.com\/deepspeedai\/DeepSpeed\/discussions\/2461."},{"key":"e_1_3_3_3_38_2","unstructured":"HPCAI Tech. 2025. TensorNVMe: A Python library for transferring PyTorch tensors between CPU and NVMe. https:\/\/github.com\/hpcaitech\/TensorNVMe. Accessed: 2025-02-06."},{"key":"e_1_3_3_3_39_2","unstructured":"Alan Thompson. [n. d.]. Inside Large Language Models. https:\/\/lifearchitect.ai\/models. Accessed: 2025-04-12."},{"key":"e_1_3_3_3_40_2","unstructured":"Hugo Touvron Louis Martin Kevin Stone Peter Albert Amjad Almahairi et\u00a0al. 2023. Llama 2: Open Foundation and Fine-Tuned Chat Models. arXiv:https:\/\/arXiv.org\/abs\/2307.09288\u00a0[cs]"},{"key":"e_1_3_3_3_41_2","unstructured":"Hugo Touvron Louis Martin Kevin Stone Peter Albert Amjad Almahairi et\u00a0al. 2023. Llama 2: Open Foundation and Fine-Tuned Chat Models. arXiv:https:\/\/arXiv.org\/abs\/2307.09288"},{"key":"e_1_3_3_3_42_2","unstructured":"Jonathan Vanian and Kif Leswing. [n. d.]. ChatGPT and generative AI are booming but the costs can be extraordinary. https:\/\/www.cnbc.com\/2023\/03\/13\/chatgpt-and-generative-ai-are-booming-but-at-a-very-expensive-price.html. Accessed: 2023-03-15."},{"key":"e_1_3_3_3_43_2","unstructured":"VAST Data. 2023. The VAST Data Platform Whitepaper. https:\/\/www.vastdata.com\/whitepaper. Accessed: 2025-04-12."},{"key":"e_1_3_3_3_44_2","unstructured":"Guanhua Wang Masahiro Tanaka Xiaoxia Wu Lok\u00a0Chand Koppaka Samyam Rajbhandari Olatunji Ruwase and Yuxiong He. 2023. DeepSpeed ZeRO-Offload++: 6x Higher Training Throughput via Collaborative CPU\/GPU Twin-Flow."},{"key":"e_1_3_3_3_45_2","doi-asserted-by":"publisher","DOI":"10.1145\/3600006.3613145"},{"key":"e_1_3_3_3_46_2","unstructured":"BigScience Workshop. 2023. BLOOM: A 176B-Parameter Open-Access Multilingual Language Model. arXiv:https:\/\/arXiv.org\/abs\/2211.05100\u00a0[cs]"},{"key":"e_1_3_3_3_47_2","doi-asserted-by":"crossref","unstructured":"Gang Xian Wenxiang Yang Yusong Tan Jinghua Feng Yuqi Li Jian Zhang and Jie Yu. 2024. Mobilizing underutilized storage nodes via job path: A job-aware file striping approach. Parallel Comput. 121 (2024) 103095.","DOI":"10.1016\/j.parco.2024.103095"},{"key":"e_1_3_3_3_48_2","unstructured":"Aohan Zeng Xiao Liu Zhengxiao Du Zihan Wang Hanyu Lai Ming Ding Zhuoyi Yang Yifan Xu Wendi Zheng Xiao Xia et\u00a0al. 2022. Glm-130b: An open bilingual pre-trained model. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2210.02414 (2022)."},{"key":"e_1_3_3_3_49_2","unstructured":"Aohan Zeng Xiao Liu Zhengxiao Du Zihan Wang Hanyu Lai Ming Ding Zhuoyi Yang Yifan Xu Wendi Zheng Xiao Xia Weng\u00a0Lam Tam Zixuan Ma Yufei Xue Jidong Zhai Wenguang Chen Peng Zhang Yuxiao Dong and Jie Tang. 2023. GLM-130B: An Open Bilingual Pre-trained Model. arxiv:https:\/\/arXiv.org\/abs\/2210.02414\u00a0[cs.CL] https:\/\/arxiv.org\/abs\/2210.02414"},{"key":"e_1_3_3_3_50_2","unstructured":"Susan Zhang Stephen Roller Naman Goyal Mikel Artetxe Moya Chen Shuohui Chen Christopher Dewan Mona Diab Xian Li et\u00a0al. 2022. Opt: Open pre-trained transformer language models. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2205.01068 (2022)."},{"key":"e_1_3_3_3_51_2","doi-asserted-by":"crossref","unstructured":"Shuai Zhang Lina Yao Aixin Sun and Yi Tay. 2019. Deep learning based recommender system: A survey and new perspectives. ACM computing surveys (CSUR) (2019).","DOI":"10.1145\/3285029"},{"key":"e_1_3_3_3_52_2","unstructured":"Wayne\u00a0Xin Zhao Kun Zhou Junyi Li Tianyi Tang Xiaolei Wang Yupeng Hou Yingqian Min Beichen Zhang Junjie Zhang Zican Dong et\u00a0al. 2023. A survey of large language models. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2303.18223 (2023)."},{"key":"e_1_3_3_3_53_2","doi-asserted-by":"crossref","unstructured":"Xia Zhao Limin Wang Yufei Zhang Xuming Han Muhammet Deveci and Milan Parmar. 2024. A review of convolutional neural networks in computer vision. Artificial Intelligence Review 57 4 (2024) 99.","DOI":"10.1007\/s10462-024-10721-6"},{"key":"e_1_3_3_3_54_2","doi-asserted-by":"crossref","unstructured":"Yanli Zhao Andrew Gu Rohan Varma Liang Luo Chien-Chin Huang Min Xu Less Wright Hamid Shojanazeri Myle Ott Sam Shleifer Alban Desmaison Can Balioglu Pritam Damania Bernard Nguyen Geeta Chauhan Yuchen Hao Ajit Mathews and Shen Li. 2023. PyTorch FSDP: Experiences on Scaling Fully Sharded Data Parallel. Proc. VLDB Endow. 16 12 (Aug. 2023) 3848\u20133860.","DOI":"10.14778\/3611540.3611569"}],"event":{"name":"SC '25: The International Conference for High Performance Computing, Networking, Storage and Analysis","location":"St. Louis MO USA","acronym":"SC '25","sponsor":["SIGHPC ACM Special Interest Group on High Performance Computing, Special Interest Group on High Performance Computing"]},"container-title":["Proceedings of the International Conference for High Performance Computing, Networking, Storage and Analysis"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/abs\/10.1145\/3712285.3759864","content-type":"text\/html","content-version":"vor","intended-application":"syndication"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3712285.3759864","content-type":"application\/pdf","content-version":"vor","intended-application":"syndication"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3712285.3759864","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,3,11]],"date-time":"2026-03-11T18:52:10Z","timestamp":1773255130000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3712285.3759864"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,11,15]]},"references-count":53,"alternative-id":["10.1145\/3712285.3759864","10.1145\/3712285"],"URL":"https:\/\/doi.org\/10.1145\/3712285.3759864","relation":{},"subject":[],"published":{"date-parts":[[2025,11,15]]},"assertion":[{"value":"2025-11-15","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}