{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,7]],"date-time":"2026-04-07T21:47:40Z","timestamp":1775598460948,"version":"3.50.1"},"reference-count":71,"publisher":"IEEE","license":[{"start":{"date-parts":[[2025,6,22]],"date-time":"2025-06-22T00:00:00Z","timestamp":1750550400000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2025,6,22]],"date-time":"2025-06-22T00:00:00Z","timestamp":1750550400000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"}],"funder":[{"DOI":"10.13039\/100016684","name":"Center for Cognitive Computing Systems Research","doi-asserted-by":"publisher","id":[{"id":"10.13039\/100016684","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100006785","name":"Google","doi-asserted-by":"publisher","id":[{"id":"10.13039\/100006785","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100017223","name":"National Energy Research Scientific Computing Center","doi-asserted-by":"publisher","id":[{"id":"10.13039\/100017223","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2025,6,22]]},"DOI":"10.1109\/dac63849.2025.11132754","type":"proceedings-article","created":{"date-parts":[[2025,9,15]],"date-time":"2025-09-15T17:35:41Z","timestamp":1757957741000},"page":"1-7","source":"Crossref","is-referenced-by-count":2,"title":["SSDTrain: An Activation Offloading Framework to SSDs for Faster Large Language Model Training"],"prefix":"10.1109","author":[{"given":"Kun","family":"Wu","sequence":"first","affiliation":[{"name":"NVIDIA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Jeongmin Brian","family":"Park","sequence":"additional","affiliation":[{"name":"NVIDIA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Xiaofan","family":"Zhang","sequence":"additional","affiliation":[{"name":"Stanford University"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Mert","family":"Hidayeto\u011flu","sequence":"additional","affiliation":[{"name":"University of California,Irvine"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Vikram Sharma","family":"Mailthody","sequence":"additional","affiliation":[{"name":"NVIDIA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Sitao","family":"Huang","sequence":"additional","affiliation":[{"name":"University of Illinois Urbana-Champaign"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Steve","family":"Lumetta","sequence":"additional","affiliation":[{"name":"Google"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Wen-Mei","family":"Hwu","sequence":"additional","affiliation":[{"name":"NVIDIA"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"263","reference":[{"key":"ref1","article-title":"ChatGPT","year":"2022"},{"key":"ref2","volume-title":"Bing Chat - Microsoft Edge","year":"2023"},{"key":"ref3","article-title":"Midjourney","year":"2022"},{"key":"ref4","article-title":"LangChain","year":"2022"},{"key":"ref5","article-title":"Emergent Abilities of Large Language Models","author":"Wei","year":"2022"},{"key":"ref6","article-title":"The cost of training AI could soon become too much to bear","author":"Meyer","year":"2024","journal-title":"Yahoo Finance."},{"key":"ref7","article-title":"Winner-Take-All Column Row Sampling for Memory Efficient Adaptation of Language Model","author":"Liu","year":"2023"},{"key":"ref8","article-title":"Reducing Activation Recomputation in Large Transformer Models","author":"Korthikanti","year":"2022"},{"key":"ref9","article-title":"MegaScale: Scaling Large Language Model Training to More Than 10,000 GPUs","author":"Jiang","year":"2024"},{"key":"ref10","article-title":"Megatron-LM: Training Multi-Billion Parameter Language Models Using Model Parallelism","author":"Shoeybi","year":"2020"},{"key":"ref11","article-title":"BLOOM: A 176B-Parameter Open-Access Multilingual Language Model","author":"Scao","year":"2023"},{"key":"ref12","article-title":"Dissecting Batching Effects in GPT Inference","author":"Chen","year":"2023"},{"key":"ref13","doi-asserted-by":"publisher","DOI":"10.1145\/3673038.3673136"},{"key":"ref14","doi-asserted-by":"publisher","DOI":"10.1109\/SC41404.2022.00051"},{"key":"ref15","article-title":"Scaling Laws for Neural Language Models","author":"Kaplan","year":"2020"},{"key":"ref16","article-title":"An Empirical Model of Large-Batch Training","author":"McCandlish","year":"2018"},{"key":"ref17","article-title":"Training Deep Nets with Sublinear Memory Cost","author":"Chen","year":"2016"},{"key":"ref18","year":"2023","journal-title":"Announcing Epoch AI\u2019s Updated Parameter, Compute and Data Trends Database"},{"key":"ref19","article-title":"ND A100 v4-series - Azure Virtual Machines","year":"2024"},{"key":"ref20","article-title":"GPU machine types Compute Engine Documentation","year":"2017"},{"key":"ref21","article-title":"Delta Project Profile","year":"2022"},{"key":"ref22","article-title":"Fiddler: CPU-GPU Orchestration for Fast Inference of Mixture-of-Experts Models","author":"Kamahori","year":"2024"},{"key":"ref23","article-title":"ZeRO-Offload: Democratizing Billion-Scale Model Training","author":"Ren","year":"2021","journal-title":"USENIX ATC"},{"key":"ref24","article-title":"PowerInfer: Fast Large Language Model Serving with a Consumer-grade GPU","author":"Song","year":"2023"},{"key":"ref25","article-title":"FlashNeuron: SSD-Enabled Large-Batch Training of Very Deep Neural Networks","author":"Bae","year":"2021","journal-title":"FAST"},{"key":"ref26","article-title":"About google cloud hyperdisk - compute engine documentation","year":"2022"},{"key":"ref27","doi-asserted-by":"publisher","DOI":"10.1002\/cpe.8143"},{"key":"ref28","article-title":"Megatron-DeepSpeed: Ongoing research training transformer language models at scale, including: BERT & GPT-2","year":"2019"},{"key":"ref29","article-title":"Language Models are Unsupervised Multitask Learners","author":"Radford","year":"2019"},{"key":"ref30","doi-asserted-by":"publisher","DOI":"10.48550\/ARXIV.1706.03762"},{"key":"ref31","article-title":"BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding","author":"Devlin","year":"2019"},{"key":"ref32","article-title":"Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer","author":"Raffel","year":"2023"},{"key":"ref33","article-title":"GSPMD: General and Scalable Parallelization for ML Computation Graphs","author":"Xu","year":"2021"},{"key":"ref34","doi-asserted-by":"publisher","DOI":"10.1109\/SC41405.2020.00024"},{"key":"ref35","article-title":"Training Compute-Optimal Large Language Models","author":"Hoffmann","year":"2022"},{"key":"ref36","article-title":"Language models are few-shot learners","author":"Brown","year":"2020"},{"key":"ref37","article-title":"Ultra-Low Latency with Samsung Z-NAND SSD","year":"2017"},{"key":"ref38","article-title":"JESD218B: Solid-State Drive (SSD) Requirements and Endurance Test Method"},{"key":"ref39","article-title":"What do I need to know about SSD endurance and overprovisioning?","year":"2023"},{"key":"ref40","article-title":"QNAP NAS Solution: QTS SSD Extra OverProvisioning","year":"2018"},{"key":"ref41","year":"2024","journal-title":"Why SMART\u2019s OverProvisioning?"},{"key":"ref42","article-title":"Solidigm\u2122 SSD Endurance Estimator","year":"2022"},{"key":"ref43","article-title":"Over-Provisioning NAND-Based Intel\u00ae SSDs for Better Endurance","year":"2018"},{"key":"ref44","article-title":"Over-Provisioning Benefits for Samsung Data Center SSDs","year":"2019"},{"key":"ref45","article-title":"Operational Characteristics of SSDs in Enterprise Storage Systems: A Large-Scale Field Study","author":"Maneas","year":"2022","journal-title":"FAST"},{"key":"ref46","article-title":"FlexGen: High-Throughput Generative Inference of Large Language Models with a Single GPU","author":"Sheng","year":"2023"},{"key":"ref47","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2024.acl-long.678"},{"key":"ref48","doi-asserted-by":"publisher","DOI":"10.1145\/3458817.3476205"},{"key":"ref49","doi-asserted-by":"publisher","DOI":"10.1109\/NAS55553.2022.9925516"},{"key":"ref50","article-title":"ProTrain: Efficient LLM Training via MemoryAware Techniques","author":"Yang","year":"2024"},{"key":"ref51","doi-asserted-by":"publisher","DOI":"10.1109\/SC41404.2022.00076"},{"key":"ref52","article-title":"Rapidsai\/kvikio: KvikIO - High Performance File IO","year":"2022"},{"key":"ref53","article-title":"Monkey patch","year":"2006"},{"key":"ref54","article-title":"LLM-Analysis: Latency and Memory Analysis of Transformer Models for Training and Inference","author":"Li","year":"2023"},{"key":"ref55","doi-asserted-by":"publisher","DOI":"10.1109\/iccd.2012.6378623"},{"key":"ref56","doi-asserted-by":"publisher","DOI":"10.1109\/date.2012.6176524"},{"key":"ref57","article-title":"Optimizing NAND Flash-Based SSDs via Retention Relaxation","author":"Liu","year":"2012","journal-title":"FAST. USENIX Association"},{"key":"ref58","first-page":"371","article-title":"Behemoth: A Flash-centric Training Accelerator for Extreme-scale dnns","author":"Kim","year":"2021"},{"key":"ref59","article-title":"A monolingual approach to contextualized word embeddings for mid-resource languages","author":"Su\u2019arez","year":"2020","journal-title":"ACL"},{"key":"ref60","first-page":"9","article-title":"Asynchronous pipelines for processing huge corpora on medium to low resource infrastructures","author":"Su\u2019arez","year":"2019","journal-title":"ser. CMLC-7. Mannheim: LeibnizInstitut f\u2019ur Deutsche Sprache"},{"key":"ref61","article-title":"Llama 2: Open Foundation and Fine-Tuned Chat Models","author":"Touvron","year":"2023"},{"key":"ref62","article-title":"FlashAttention-2: Faster Attention with Better Parallelism and Work Partitioning","author":"Dao","year":"2023"},{"key":"ref63","article-title":"FlashAttention: Fast and Memory-Efficient Exact Attention with IO-Awareness","author":"Dao","year":"2022"},{"key":"ref64","article-title":"Scaling large language model training with Pax on GPUs","author":"Nitin","year":"2023"},{"key":"ref65","article-title":"NVIDIA A100 40GB Ampere PCIe GPU","year":"2021"},{"key":"ref66","article-title":"Intel optane p.5800x 1.6TB","year":"2021"},{"key":"ref67","doi-asserted-by":"publisher","DOI":"10.1145\/3600006.3613165"},{"key":"ref68","doi-asserted-by":"publisher","DOI":"10.1145\/3373376.3378505"},{"key":"ref69","doi-asserted-by":"publisher","DOI":"10.1145\/3200691.3178491"},{"key":"ref70","doi-asserted-by":"publisher","DOI":"10.1109\/MICRO.2016.7783721"},{"key":"ref71","doi-asserted-by":"publisher","DOI":"10.1145\/3373376.3378530"}],"event":{"name":"2025 62nd ACM\/IEEE Design Automation Conference (DAC)","location":"San Francisco, CA, USA","start":{"date-parts":[[2025,6,22]]},"end":{"date-parts":[[2025,6,25]]}},"container-title":["2025 62nd ACM\/IEEE Design Automation Conference (DAC)"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx8\/11132383\/11132091\/11132754.pdf?arnumber=11132754","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,12,2]],"date-time":"2025-12-02T18:44:25Z","timestamp":1764701065000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/11132754\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,6,22]]},"references-count":71,"URL":"https:\/\/doi.org\/10.1109\/dac63849.2025.11132754","relation":{},"subject":[],"published":{"date-parts":[[2025,6,22]]}}}