{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,6,27]],"date-time":"2026-06-27T19:45:23Z","timestamp":1782589523405,"version":"3.54.5"},"publisher-location":"New York, NY, USA","reference-count":51,"publisher":"ACM","content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2025,9,8]]},"DOI":"10.1145\/3754598.3754606","type":"proceedings-article","created":{"date-parts":[[2025,12,20]],"date-time":"2025-12-20T08:34:32Z","timestamp":1766219672000},"page":"678-688","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":1,"title":["Amber: Towards Fast and Space-Efficient Incremental Checkpointing in Large Language Model Training"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0009-0005-1710-5014","authenticated-orcid":false,"given":"Zhiqiang","family":"Wang","sequence":"first","affiliation":[{"name":"University of Science and Technology of China, Hefei, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-3965-2597","authenticated-orcid":false,"given":"Wenzhe","family":"Zhu","sequence":"additional","affiliation":[{"name":"University of Science and Technology of China, Hefei, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0009-6075-1221","authenticated-orcid":false,"given":"Zaigui","family":"Zhang","sequence":"additional","affiliation":[{"name":"Jinan Inspur Data Technology Co., Ltd., Jinan, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0007-5793-1497","authenticated-orcid":false,"given":"Chaomei","family":"Yan","sequence":"additional","affiliation":[{"name":"University of Science and Technology of China, Hefei, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0007-6842-5899","authenticated-orcid":false,"given":"Fan","family":"Guo","sequence":"additional","affiliation":[{"name":"University of Science and Technology of China, Hefei, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-3743-8511","authenticated-orcid":false,"given":"Yongkun","family":"Li","sequence":"additional","affiliation":[{"name":"University of Science and Technology of China, Hefei, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-9586-0561","authenticated-orcid":false,"given":"Yinlong","family":"Xu","sequence":"additional","affiliation":[{"name":"Anhui Provincial Key Laboratory of High Performance Computing, USTC, Hefei, China and University of Science and Technology of China, Hefei, China"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2025,12,20]]},"reference":[{"key":"e_1_3_3_1_2_2","unstructured":"Josh Achiam Steven Adler Sandhini Agarwal Lama Ahmad Ilge Akkaya Florencia\u00a0Leoni Aleman Diogo Almeida Janko Altenschmidt Sam Altman Shyamal Anadkat et\u00a0al. 2023. Gpt-4 technical report. arXiv:https:\/\/arXiv.org\/abs\/2303.08774 (2023)."},{"key":"e_1_3_3_1_3_2","doi-asserted-by":"publisher","DOI":"10.1145\/3698038.3698553"},{"key":"e_1_3_3_1_4_2","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D17-1045"},{"key":"e_1_3_3_1_5_2","doi-asserted-by":"publisher","DOI":"10.1145\/3492321.3519584"},{"key":"e_1_3_3_1_6_2","unstructured":"Jinze Bai Shuai Bai Yunfei Chu Zeyu Cui Kai Dang Xiaodong Deng Yang Fan Wenbin Ge Yu Han Fei Huang et\u00a0al. 2023. Qwen technical report. arXiv:https:\/\/arXiv.org\/abs\/2309.16609 (2023)."},{"key":"e_1_3_3_1_7_2","volume-title":"Proc. of ICML","author":"Biderman Stella","year":"2023","unstructured":"Stella Biderman, Hailey Schoelkopf, Quentin\u00a0Gregory Anthony, Herbie Bradley, Kyle O\u2019Brien, Eric Hallahan, Mohammad\u00a0Aflah Khan, Shivanshu Purohit, USVSN\u00a0Sai Prashanth, Edward Raff, et\u00a0al. 2023. Pythia: A suite for analyzing large language models across training and scaling. In Proc. of ICML."},{"key":"e_1_3_3_1_8_2","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-7908-2604-3_16"},{"key":"e_1_3_3_1_9_2","unstructured":"Tom Brown Benjamin Mann Nick Ryder Melanie Subbiah Jared\u00a0D Kaplan Prafulla Dhariwal Arvind Neelakantan Pranav Shyam Girish Sastry Amanda Askell et\u00a0al. 2020. Language models are few-shot learners. Proc. of NeurIPS (2020)."},{"key":"e_1_3_3_1_10_2","doi-asserted-by":"publisher","DOI":"10.1109\/ICCD58817.2023.00031"},{"key":"e_1_3_3_1_11_2","volume-title":"Proc. of ICML","author":"Chen Yu","year":"2020","unstructured":"Yu Chen and Zhenming Liu. 2020. On Efficient Constructions of Checkpoints. In Proc. of ICML."},{"key":"e_1_3_3_1_12_2","unstructured":"Aakanksha Chowdhery Sharan Narang Jacob Devlin Maarten Bosma Gaurav Mishra Adam Roberts Paul Barham Hyung\u00a0Won Chung Charles Sutton Sebastian Gehrmann et\u00a0al. 2023. Palm: Scaling language modeling with pathways. Journal of Machine Learning Research 24 240 (2023) 1\u2013113."},{"key":"e_1_3_3_1_13_2","unstructured":"Contributors Opencompass. [n. d.]. Opencompass: A universal evaluation platform for foundation models. https:\/\/github.com\/open-compass\/opencompass."},{"key":"e_1_3_3_1_14_2","doi-asserted-by":"crossref","unstructured":"Leonardo Dagum and Ramesh Menon. 1998. OpenMP: an industry standard API for shared-memory programming. IEEE computational science and engineering 5 1 (1998) 46\u201355.","DOI":"10.1109\/99.660313"},{"key":"e_1_3_3_1_15_2","unstructured":"Abhimanyu Dubey Abhinav Jauhri Abhinav Pandey Abhishek Kadian Ahmad Al-Dahle Aiesha Letman Akhil Mathur Alan Schelten Amy Yang Angela Fan et\u00a0al. 2024. The llama 3 herd of models. arXiv:https:\/\/arXiv.org\/abs\/2407.21783 (2024)."},{"key":"e_1_3_3_1_16_2","volume-title":"Proc. of USENIX NSDI","author":"Eisenman Assaf","year":"2022","unstructured":"Assaf Eisenman, Kiran\u00a0Kumar Matam, Steven Ingram, Dheevatsa Mudigere, Raghuraman Krishnamoorthi, Krishnakumar Nair, Misha Smelyanskiy, and Murali Annavaram. 2022. Check-N-Run: A checkpointing system for training deep learning recommendation models. In Proc. of USENIX NSDI."},{"key":"e_1_3_3_1_17_2","unstructured":"William Fedus Barret Zoph and Noam Shazeer. 2022. Switch transformers: Scaling to trillion parameter models with simple and efficient sparsity. Journal of Machine Learning Research 23 120 (2022) 1\u201339."},{"key":"e_1_3_3_1_18_2","doi-asserted-by":"publisher","DOI":"10.1145\/3452296.3472904"},{"key":"e_1_3_3_1_19_2","doi-asserted-by":"publisher","DOI":"10.1145\/3694715.3695960"},{"key":"e_1_3_3_1_20_2","unstructured":"Daya Guo Dejian Yang Haowei Zhang Junxiao Song Ruoyu Zhang Runxin Xu Qihao Zhu Shirong Ma Peiyi Wang Xiao Bi et\u00a0al. 2025. Deepseek-r1: Incentivizing reasoning capability in llms via reinforcement learning. arXiv:https:\/\/arXiv.org\/abs\/2501.12948 (2025)."},{"key":"e_1_3_3_1_21_2","doi-asserted-by":"publisher","DOI":"10.1145\/3627703.3650085"},{"key":"e_1_3_3_1_22_2","unstructured":"Tao He Xue Li Zhibin Wang Kun Qian Jingbo Xu Wenyuan Yu and Jingren Zhou. 2023. Unicron: Economizing self-healing llm training at scale. arXiv:https:\/\/arXiv.org\/abs\/2401.00134 (2023)."},{"key":"e_1_3_3_1_23_2","doi-asserted-by":"publisher","DOI":"10.1145\/3404397.3404408"},{"key":"e_1_3_3_1_24_2","doi-asserted-by":"publisher","DOI":"10.1145\/3600006.3613152"},{"key":"e_1_3_3_1_25_2","volume-title":"Proc. of USENIX NSDI","author":"Jiang Ziheng","year":"2024","unstructured":"Ziheng Jiang, Haibin Lin, Yinmin Zhong, Qi Huang, Yangrui Chen, Zhi Zhang, Yanghua Peng, Xiang Li, Cong Xie, Shibiao Nong, et\u00a0al. 2024. MegaScale: Scaling large language model training to more than 10,000 GPUs. In Proc. of USENIX NSDI."},{"key":"e_1_3_3_1_26_2","doi-asserted-by":"crossref","unstructured":"Haoyu Jin Donglei Wu Shuyu Zhang Xiangyu Zou Sian Jin Dingwen Tao Qing Liao and Wen Xia. 2023. Design of a quantization-based dnn delta compression framework for model snapshots and federated learning. IEEE Trans. on Parallel and Distributed Systems 34 3 (2023) 923\u2013937.","DOI":"10.1109\/TPDS.2022.3230840"},{"key":"e_1_3_3_1_27_2","unstructured":"Diederik\u00a0P Kingma. 2014. Adam: A method for stochastic optimization. arXiv:https:\/\/arXiv.org\/abs\/1412.6980 (2014)."},{"key":"e_1_3_3_1_28_2","doi-asserted-by":"publisher","DOI":"10.1109\/MM.2023.3237548"},{"key":"e_1_3_3_1_29_2","volume-title":"Proc. of ICML","author":"Li Wenshuo","year":"2024","unstructured":"Wenshuo Li, Xinghao Chen, Han Shu, Yehui Tang, and Yunhe Wang. 2024. ExCP: Extreme LLM Checkpoint Compression via Weight-Momentum Joint Shrinking. In Proc. of ICML."},{"key":"e_1_3_3_1_30_2","volume-title":"Proc. of ICLR","author":"Lin Yujun","year":"2018","unstructured":"Yujun Lin, Song Han, Huizi Mao, Yu Wang, and Bill Dally. 2018. Deep Gradient Compression: Reducing the Communication Bandwidth for Distributed Training. In Proc. of ICLR."},{"key":"e_1_3_3_1_31_2","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV48922.2021.00986"},{"key":"e_1_3_3_1_32_2","volume-title":"Proc. of MLSys","author":"Maeng Kiwan","year":"2021","unstructured":"Kiwan Maeng, Shivam Bharuka, Isabel Gao, Mark Jeffrey, Vikram Saraph, Bor-Yiing Su, Caroline Trippel, Jiyan Yang, Mike Rabbat, Brandon Lucia, et\u00a0al. 2021. Understanding and improving failure tolerant training for deep learning recommendation with partial recovery. In Proc. of MLSys."},{"key":"e_1_3_3_1_33_2","doi-asserted-by":"publisher","DOI":"10.1145\/3625549.3658685"},{"key":"e_1_3_3_1_34_2","unstructured":"Microsoft Azure. [n. d.]. Boost Checkpoint Speed and Reduce Cost with Nebula. https:\/\/learn.microsoft.com\/en-us\/azure\/machine-learning\/reference-checkpoint-performance-for-large-models?view=azureml-api-2&tabs=PYTORCH."},{"key":"e_1_3_3_1_35_2","volume-title":"Proc. of USENIX FAST","author":"Mohan Jayashree","year":"2021","unstructured":"Jayashree Mohan, Amar Phanishayee, and Vijay Chidambaram. 2021. CheckFreq: Frequent, Fine-Grained DNN Checkpointing. In Proc. of USENIX FAST."},{"key":"e_1_3_3_1_36_2","doi-asserted-by":"publisher","DOI":"10.1145\/3458817.3476209"},{"key":"e_1_3_3_1_37_2","doi-asserted-by":"publisher","DOI":"10.1109\/CCGrid49817.2020.00-76"},{"key":"e_1_3_3_1_38_2","volume-title":"Proc. of NeurIPS","author":"Paszke Adam","year":"2019","unstructured":"Adam Paszke, Sam Gross, Francisco Massa, Adam Lerer, James Bradbury, Gregory Chanan, Trevor Killeen, Zeming Lin, Natalia Gimelshein, Luca Antiga, et\u00a0al. 2019. Pytorch: An imperative style, high-performance deep learning library. In Proc. of NeurIPS."},{"key":"e_1_3_3_1_39_2","unstructured":"Alec Radford Jeffrey Wu Rewon Child David Luan Dario Amodei Ilya Sutskever et\u00a0al. 2019. Language models are unsupervised multitask learners. OpenAI blog 1 8 (2019) 9."},{"key":"e_1_3_3_1_40_2","unstructured":"Colin Raffel Noam Shazeer Adam Roberts Katherine Lee Sharan Narang Michael Matena Yanqi Zhou Wei Li and Peter\u00a0J Liu. 2020. Exploring the limits of transfer learning with a unified text-to-text transformer. Journal of machine learning research 21 140 (2020) 1\u201367."},{"key":"e_1_3_3_1_41_2","doi-asserted-by":"publisher","DOI":"10.1145\/3394486.3406703"},{"key":"e_1_3_3_1_42_2","volume-title":"Proc. of USENIX NSDI","author":"Thorpe John","year":"2023","unstructured":"John Thorpe, Pengzhan Zhao, Jonathan Eyolfson, Yifan Qiao, Zhihao Jia, Minjia Zhang, Ravi Netravali, and Guoqing\u00a0Harry Xu. 2023. Bamboo: Making preemptible instances resilient for affordable training of large DNN. In Proc. of USENIX NSDI."},{"key":"e_1_3_3_1_43_2","unstructured":"A Vaswani. 2017. Attention is all you need. Proc. of NeurIPS (2017)."},{"key":"e_1_3_3_1_44_2","unstructured":"Borui Wan Mingji Han Yiyao Sheng Yanghua Peng Haibin Lin Mofan Zhang Zhichao Lai Menghan Yu Junda Zhang Zuquan Song et\u00a0al. 2024. ByteCheckpoint: A Unified Checkpointing System for Large Foundation Model Development. arXiv:https:\/\/arXiv.org\/abs\/2407.20143 (2024)."},{"key":"e_1_3_3_1_45_2","unstructured":"Guanhua Wang Olatunji Ruwase Bing Xie and Yuxiong He. 2024. FastPersist: Accelerating Model Checkpointing in Deep Learning. arXiv:https:\/\/arXiv.org\/abs\/2406.13768 (2024)."},{"key":"e_1_3_3_1_46_2","doi-asserted-by":"publisher","DOI":"10.1145\/3600006.3613145"},{"key":"e_1_3_3_1_47_2","volume-title":"Proc. of USENIX ATC","author":"Wang Zheng","year":"2024","unstructured":"Zheng Wang, Yuke Wang, Boyuan Feng, Guyue Huang, Dheevatsa Mudigere, Bharath Muthiah, Ang Li, and Yufei Ding. 2024. OPER:Optimality-Guided Embedding Table Parallelization for Large-scale Recommendation Model. In Proc. of USENIX ATC."},{"key":"e_1_3_3_1_48_2","unstructured":"BigScience Workshop Teven\u00a0Le Scao Angela Fan Christopher Akiki Ellie Pavlick Suzana Ili\u0107 Daniel Hesslow Roman Castagn\u00e9 Alexandra\u00a0Sasha Luccioni Fran\u00e7ois Yvon et\u00a0al. 2022. Bloom: A 176b-parameter open-access multilingual language model. arXiv:https:\/\/arXiv.org\/abs\/2211.05100 (2022)."},{"key":"e_1_3_3_1_49_2","unstructured":"Baodong Wu Lei Xia Qingping Li Kangyu Li Xu Chen Yongqiang Guo Tieyao Xiang Yuheng Chen and Shigang Li. 2023. TRANSOM: An efficient fault-tolerant system for training LLMs. arXiv:https:\/\/arXiv.org\/abs\/2310.10046 (2023)."},{"key":"e_1_3_3_1_50_2","doi-asserted-by":"publisher","DOI":"10.1109\/ISCAS58744.2024.10557880"},{"key":"e_1_3_3_1_51_2","unstructured":"Susan Zhang Stephen Roller Naman Goyal Mikel Artetxe Moya Chen Shuohui Chen Christopher Dewan Mona Diab Xian Li Xi\u00a0Victoria Lin et\u00a0al. 2022. Opt: Open pre-trained transformer language models. arXiv:https:\/\/arXiv.org\/abs\/2205.01068 (2022)."},{"key":"e_1_3_3_1_52_2","doi-asserted-by":"publisher","DOI":"10.1145\/3572848.3577510"}],"event":{"name":"ICPP '25: 54th International Conference on Parallel Processing","location":"San Diego CA USA","acronym":"ICPP '25"},"container-title":["Proceedings of the 54th International Conference on Parallel Processing"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3754598.3754606","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,12,20]],"date-time":"2025-12-20T08:36:36Z","timestamp":1766219796000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3754598.3754606"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,9,8]]},"references-count":51,"alternative-id":["10.1145\/3754598.3754606","10.1145\/3754598"],"URL":"https:\/\/doi.org\/10.1145\/3754598.3754606","relation":{},"subject":[],"published":{"date-parts":[[2025,9,8]]},"assertion":[{"value":"2025-12-20","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}