{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,10]],"date-time":"2026-10-10T23:49:22Z","timestamp":1791676162951,"version":"4.3.4"},"publisher-location":"New York, NY, USA","reference-count":92,"publisher":"ACM","license":[{"start":{"date-parts":[[2024,11,4]],"date-time":"2024-11-04T00:00:00Z","timestamp":1730678400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"DOI":"10.13039\/100000001","name":"NSF (National Science Foundation)","doi-asserted-by":"publisher","award":["CNS-2104243"],"award-info":[{"award-number":["CNS-2104243"]}],"id":[{"id":"10.13039\/100000001","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000001","name":"NSF (National Science Foundation)","doi-asserted-by":"publisher","award":["CNS-2106184"],"award-info":[{"award-number":["CNS-2106184"]}],"id":[{"id":"10.13039\/100000001","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000001","name":"NSF (National Science Foundation)","doi-asserted-by":"publisher","award":["CCF-2327011"],"award-info":[{"award-number":["CCF-2327011"]}],"id":[{"id":"10.13039\/100000001","id-type":"DOI","asserted-by":"publisher"}]},{"name":"NWO VICI","award":["639.023.812"],"award-info":[{"award-number":["639.023.812"]}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2024,11,4]]},"DOI":"10.1145\/3694715.3695970","type":"proceedings-article","created":{"date-parts":[[2024,11,15]],"date-time":"2024-11-15T14:28:18Z","timestamp":1731680898000},"page":"144-159","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":25,"title":["Reducing Energy Bloat in Large Model Training"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0001-5924-3427","authenticated-orcid":false,"given":"Jae-Won","family":"Chung","sequence":"first","affiliation":[{"name":"University of Michigan, Ann Arbor, MI, United States"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0009-8292-7232","authenticated-orcid":false,"given":"Yile","family":"Gu","sequence":"additional","affiliation":[{"name":"University of Washington, Seattle, WA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0007-5206-2333","authenticated-orcid":false,"given":"Insu","family":"Jang","sequence":"additional","affiliation":[{"name":"University of Michigan, Ann Arbor, MI, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0009-0645-834X","authenticated-orcid":false,"given":"Luoxi","family":"Meng","sequence":"additional","affiliation":[{"name":"University of California, San Diego, San Diego, CA, United States"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-6290-0894","authenticated-orcid":false,"given":"Nikhil","family":"Bansal","sequence":"additional","affiliation":[{"name":"University of Michigan, Ann Arbor, MI, United States"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-0884-6740","authenticated-orcid":false,"given":"Mosharaf","family":"Chowdhury","sequence":"additional","affiliation":[{"name":"University of Michigan, Ann Arbor, MI, United States"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2024,11,15]]},"reference":[{"key":"e_1_3_2_2_1_1","unstructured":"DeepSpeed. https:\/\/github.com\/microsoft\/DeepSpeed."},{"key":"e_1_3_2_2_2_1","unstructured":"How much electricity does an American home use? https:\/\/www.eia.gov\/tools\/faqs\/faq.php?id=97&t=3."},{"key":"e_1_3_2_2_3_1","unstructured":"Megatron-LM. https:\/\/github.com\/NVIDIA\/Megatron-LM."},{"key":"e_1_3_2_2_4_1","unstructured":"NVIDIA Management Library (NVML). https:\/\/developer.nvidia.com\/nvidia-management-library-nvml."},{"key":"e_1_3_2_2_5_1","volume-title":"The falcon series of open language models. arXiv preprint arXiv:2311.16867","author":"Almazrouei Ebtesam","year":"2023","unstructured":"Ebtesam Almazrouei, Hamza Alobeidli, Abdulaziz Alshamsi, Alessandro Cappelli, Ruxandra Cojocaru, M\u00e9rouane Debbah, \u00c9tienne Goffinet, Daniel Hesslow, Julien Launay, Quentin Malartic, et al. The falcon series of open language models. arXiv preprint arXiv:2311.16867, 2023."},{"key":"e_1_3_2_2_6_1","first-page":"9","author":"Andonian Alex","year":"2023","unstructured":"Alex Andonian, Quentin Anthony, Stella Biderman, Sid Black, Preetham Gali, Leo Gao, Eric Hallahan, Josh Levy-Kramer, Connor Leahy, Lucas Nestler, Kip Parker, Michael Pieler, Jason Phang, Shivanshu Purohit, Hailey Schoelkopf, Dashiell Stander, Tri Songz, Curt Tigges, Benjamin Th\u00e9rien, Phil Wang, and Samuel Weinbach. GPT-NeoX: Large Scale Autoregressive Language Modeling in PyTorch, 9 2023.","journal-title":"Large Scale Autoregressive Language Modeling in PyTorch"},{"key":"e_1_3_2_2_7_1","doi-asserted-by":"publisher","DOI":"10.1145\/3297858.3304049"},{"key":"e_1_3_2_2_8_1","volume-title":"NeurIPS","author":"Brown Tom","year":"2020","unstructured":"Tom Brown, Benjamin Mann, Nick Ryder, Melanie Subbiah, Jared D Kaplan, Prafulla Dhariwal, Arvind Neelakantan, Pranav Shyam, Girish Sastry, Amanda Askell, Sandhini Agarwal, Ariel Herbert-Voss, Gretchen Krueger, Tom Henighan, Rewon Child, Aditya Ramesh, Daniel Ziegler, Jeffrey Wu, Clemens Winter, Chris Hesse, Mark Chen, Eric Sigler, Mateusz Litwin, Scott Gray, Benjamin Chess, Jack Clark, Christopher Berner, Sam McCandlish, Alec Radford, Ilya Sutskever, and Dario Amodei. Language models are few-shot learners. In NeurIPS, 2020."},{"key":"e_1_3_2_2_9_1","doi-asserted-by":"crossref","unstructured":"Trevor Cai. Predictable scaling and infrastructure (HotChips 2024 keynote talk). OpenAI.","DOI":"10.1109\/HCS61935.2024.10664929"},{"key":"e_1_3_2_2_10_1","doi-asserted-by":"publisher","DOI":"10.1007\/s11227-016-1653-7"},{"key":"e_1_3_2_2_11_1","volume-title":"https:\/\/www.cbre.com\/insights\/reports\/global-data-center-trends-2023","author":"CBRE.","year":"2023","unstructured":"CBRE. Global data center trends 2023. https:\/\/www.cbre.com\/insights\/reports\/global-data-center-trends-2023, 2023."},{"key":"e_1_3_2_2_12_1","volume-title":"https:\/\/www.cbre.com\/insights\/reports\/global-data-center-trends-2024","author":"CBRE.","year":"2024","unstructured":"CBRE. Global data center trends 2024. https:\/\/www.cbre.com\/insights\/reports\/global-data-center-trends-2024, 2024."},{"key":"e_1_3_2_2_13_1","volume-title":"Training deep nets with sublinear memory cost","author":"Chen Tianqi","year":"2016","unstructured":"Tianqi Chen, Bing Xu, Chiyuan Zhang, and Carlos Guestrin. Training deep nets with sublinear memory cost. 2016."},{"key":"e_1_3_2_2_14_1","doi-asserted-by":"publisher","DOI":"10.1109\/JSSC.2016.2616357"},{"key":"e_1_3_2_2_15_1","volume-title":"ATC","author":"Choi Sangjin","year":"2023","unstructured":"Sangjin Choi, Inhoe Koo, Jeongseob Ahn, Myeongjae Jeon, and Youngjin Kwon. EnvPipe: Performance-preserving DNN training framework for saving energy. In ATC, 2023."},{"key":"e_1_3_2_2_16_1","volume-title":"IGSC","author":"Chow Marcus","year":"2024","unstructured":"Marcus Chow and Daniel Wong. CoFRIS: Coordinated frequency and resource scaling for GPU inference servers. In IGSC, 2024."},{"key":"e_1_3_2_2_17_1","volume-title":"Energy-Efficient Computing for Science Workshop","author":"Chung Jae-Won","year":"2024","unstructured":"Jae-Won Chung, Nishil Talati, and Mosharaf Chowdhury. Toward cross-layer energy optimizations in AI systems. In Energy-Efficient Computing for Science Workshop, 2024."},{"key":"e_1_3_2_2_18_1","unstructured":"ML COMMONS. MLPerf training v3.1 benchmark results. https:\/\/github.com\/mlcommons\/training_results_v3.1."},{"key":"e_1_3_2_2_19_1","volume-title":"Proceedings of the 2019 Conference of the North American Chapter of the Association for Computational Linguistics (NAACL)","author":"Devlin Jacob","year":"2019","unstructured":"Jacob Devlin, Ming-Wei Chang, Kenton Lee, and Kristina Toutanova. BERT: Pre-training of deep bidirectional transformers for language understanding. In Proceedings of the 2019 Conference of the North American Chapter of the Association for Computational Linguistics (NAACL), 2019."},{"key":"e_1_3_2_2_20_1","doi-asserted-by":"publisher","DOI":"10.1145\/3531146.3533234"},{"key":"e_1_3_2_2_21_1","doi-asserted-by":"publisher","DOI":"10.1145\/321694.321699"},{"key":"e_1_3_2_2_22_1","volume-title":"Extensions of maximum flow. https:\/\/courses.engr.illinois.edu\/cs498dl1\/sp2015\/notes\/25-maxflowext.pdf. [Online","author":"Erickson Jeff","year":"2023","unstructured":"Jeff Erickson. Extensions of maximum flow. https:\/\/courses.engr.illinois.edu\/cs498dl1\/sp2015\/notes\/25-maxflowext.pdf. [Online; accessed 05-April-2023]."},{"key":"e_1_3_2_2_23_1","volume-title":"ACM PPoPP","author":"Fan Shiqing","year":"2021","unstructured":"Shiqing Fan, Yi Rong, Chen Meng, Zongyan Cao, Siyu Wang, Zhen Zheng, Chuan Wu, Guoping Long, Jun Yang, Lixue Xia, Lansong Diao, Xiaoyong Liu, and Wei Lin. DAPPLE: A pipelined data parallel approach for training large models. In ACM PPoPP, 2021."},{"key":"e_1_3_2_2_24_1","doi-asserted-by":"publisher","DOI":"10.1515\/9781400875184"},{"key":"e_1_3_2_2_25_1","volume-title":"SOSP","author":"Gandhi Swapnil","year":"2024","unstructured":"Swapnil Gandhi, Mark Zhao, Athinagoras Skiadopoulos, and Christos Kozyrakis. ReCycle: Pipeline adaptation for the resilient distributed training of large DNNs. In SOSP, 2024."},{"key":"e_1_3_2_2_26_1","volume-title":"On the computational ineficiency of large batch sizes for stochastic gradient descent. arXiv preprint arXiv:1811.12941","author":"Golmant Noah","year":"2018","unstructured":"Noah Golmant, Nikita Vemuri, Zhewei Yao, Vladimir Feinberg, Amir Gholami, Kai Rothauge, Michael W Mahoney, and Joseph Gonzalez. On the computational ineficiency of large batch sizes for stochastic gradient descent. arXiv preprint arXiv:1811.12941, 2018."},{"key":"e_1_3_2_2_27_1","volume-title":"OSDI","author":"Gujarati Arpan","year":"2020","unstructured":"Arpan Gujarati, Reza Karimi, Safya Alzayat, Wei Hao, Antoine Kaufmann, Ymir Vigfusson, and Jonathan Mace. Serving DNNs like clockwork: Performance predictability from the bottom up. In OSDI, 2020."},{"key":"e_1_3_2_2_28_1","doi-asserted-by":"publisher","DOI":"10.1145\/3126908.3126937"},{"key":"e_1_3_2_2_29_1","unstructured":"James Hamilton. Constraint-driven innovation (CIDR 2024 keynote talk). https:\/\/mvdirona.com\/jrh\/talksandpapers\/JamesHamiltonCIDR2024.pdf."},{"key":"e_1_3_2_2_30_1","volume-title":"ICML","author":"He Chaoyang","year":"2021","unstructured":"Chaoyang He, Shen Li, Mahdi Soltanolkotabi, and Salman Avestimehr. PipeTransformer: Automated elastic pipelining for distributed training of large-scale models. In ICML, 2021."},{"key":"e_1_3_2_2_31_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.cie.2016.02.018"},{"key":"e_1_3_2_2_32_1","unstructured":"Jordan Hoffmann Sebastian Borgeaud Arthur Mensch Elena Buchatskaya Trevor Cai Eliza Rutherford Diego de Las Casas Lisa Anne Hendricks Johannes Welbl Aidan Clark Thomas Hennigan Eric Noland Katherine Millican George van den Driessche Bogdan Damoc Aurelia Guy Simon Osindero Kar\u00e9n Simonyan Erich Elsen Oriol Vinyals Jack Rae and Laurent Sifre. An empirical analysis of compute-optimal large language model training. In NeurIPS 2022."},{"key":"e_1_3_2_2_33_1","volume-title":"OSDI","author":"Hu Qinghao","year":"2023","unstructured":"Qinghao Hu, Zhisheng Ye, Meng Zhang, Qiaoling Chen, Peng Sun, Yonggang Wen, and Tianwei Zhang. Hydro: Surrogate-Based hyperparameter tuning service in datacenters. In OSDI, 2023."},{"key":"e_1_3_2_2_34_1","volume-title":"NeurIPS","author":"Huang Yanping","year":"2019","unstructured":"Yanping Huang, Youlong Cheng, Ankur Bapna, Orhan Firat, Mia Xu Chen, Dehao Chen, HyoukJoong Lee, Jiquan Ngiam, Quoc V. Le, Yonghui Wu, and Zhifeng Chen. GPipe: Efficient training of giant neural networks using pipeline parallelism. In NeurIPS, 2019."},{"key":"e_1_3_2_2_35_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.future.2015.01.005"},{"key":"e_1_3_2_2_36_1","doi-asserted-by":"publisher","DOI":"10.1109\/CCGrid49817.2020.00-35"},{"key":"e_1_3_2_2_37_1","doi-asserted-by":"publisher","DOI":"10.1145\/3600006.3613152"},{"key":"e_1_3_2_2_38_1","volume-title":"ATC","author":"Jeon Myeongjae","year":"2019","unstructured":"Myeongjae Jeon, Shivaram Venkataraman, Amar Phanishayee, Junjie Qian, Wencong Xiao, and Fan Yang. Analysis of Large-Scale Multi-Tenant GPU clusters for DNN training workloads. In ATC, 2019."},{"key":"e_1_3_2_2_39_1","volume-title":"NSDI","author":"Jiang Ziheng","year":"2024","unstructured":"Ziheng Jiang, Haibin Lin, Yinmin Zhong, Qi Huang, Yangrui Chen, Zhi Zhang, Yanghua Peng, Xiang Li, Cong Xie, Shibiao Nong, Yulu Jia, Sun He, Hongmin Chen, Zhihao Bai, Qi Hou, Shipeng Yan, Ding Zhou, Yiyao Sheng, Zhuo Jiang, Haohan Xu, Haoran Wei, Zhang Zhang, Pengfei Nie, Leqi Zou, Sida Zhao, Liang Xiang, Zherui Liu, Zhe Li, Xiaoying Jia, Jianxi Ye, Xin Jin, and Xin Liu. MegaScale: Scaling large language model training to more than 10,000 GPUs. In NSDI, 2024."},{"key":"e_1_3_2_2_40_1","volume-title":"Scaling laws for neural language models. arXiv preprint arXiv:2001.08361","author":"Kaplan Jared","year":"2020","unstructured":"Jared Kaplan, Sam McCandlish, Tom Henighan, Tom B. Brown, Benjamin Chess, Rewon Child, Scott Gray, Alec Radford, Jeffrey Wu, and Dario Amodei. Scaling laws for neural language models. arXiv preprint arXiv:2001.08361, 2020."},{"key":"e_1_3_2_2_41_1","volume-title":"ATC","author":"Keahey Kate","year":"2020","unstructured":"Kate Keahey, Jason Anderson, Zhuo Zhen, Pierre Riteau, Paul Ruth, Dan Stanzione, Mert Cevik, Jacob Colleran, Haryadi S Gunawi, Cody Hammock, et al. Lessons learned from the chameleon testbed. In ATC, 2020."},{"key":"e_1_3_2_2_42_1","volume-title":"ICLR","author":"Keskar Nitish Shirish","year":"2017","unstructured":"Nitish Shirish Keskar, Dheevatsa Mudigere, Jorge Nocedal, Mikhail Smelyanskiy, and Ping Tak Peter Tang. On large-batch training for deep learning: Generalization gap and sharp minima. In ICLR, 2017."},{"key":"e_1_3_2_2_43_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.future.2023.03.041"},{"key":"e_1_3_2_2_44_1","volume-title":"Quantifying the carbon emissions of machine learning. arXiv preprint arXiv:1910.09700","author":"Lacoste Alexandre","year":"2019","unstructured":"Alexandre Lacoste, Alexandra Luccioni, Victor Schmidt, and Thomas Dandres. Quantifying the carbon emissions of machine learning. arXiv preprint arXiv:1910.09700, 2019."},{"key":"e_1_3_2_2_45_1","doi-asserted-by":"publisher","DOI":"10.1109\/TPDS.2023.3247001"},{"key":"e_1_3_2_2_46_1","doi-asserted-by":"publisher","DOI":"10.1007\/s11227-021-03740-5"},{"key":"e_1_3_2_2_47_1","volume-title":"OSDI","author":"Li Shaohong","year":"2020","unstructured":"Shaohong Li, Xi Wang, Xiao Zhang, Vasileios Kontorinis, Sreekumar Kodakara, David Lo, and Parthasarathy Ranganathan. Thunderbolt: Throughput-Optimized, Quality-of-Service-Aware power capping at scale. In OSDI, 2020."},{"key":"e_1_3_2_2_48_1","volume-title":"OSDI","author":"Lin Zhiqi","year":"2024","unstructured":"Zhiqi Lin, Youshan Miao, Quanlu Zhang, Fan Yang, Yi Zhu, Cheng Li, Saeed Maleki, Xu Cao, Ning Shang, Yilei Yang, Weijiang Xu, Mao Yang, Lintao Zhang, and Lidong Zhou. nnScaler: Constraint-Guided parallelization plan generation for deep learning training. In OSDI, 2024."},{"key":"e_1_3_2_2_49_1","volume-title":"Estimating the carbon footprint of BLOOM, a 176b parameter language model","author":"Luccioni Alexandra Sasha","year":"2022","unstructured":"Alexandra Sasha Luccioni, Sylvain Viguier, and Anne-Laure Ligozat. Estimating the carbon footprint of BLOOM, a 176b parameter language model. 2022."},{"key":"e_1_3_2_2_50_1","unstructured":"Tobias Mann. NVIDIA turns up the AI heat with 1 200w blackwell gpus. https:\/\/www.theregister.com\/2024\/03\/18\/nvidia_turns_up_the_ai."},{"key":"e_1_3_2_2_51_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICAC.2017.43"},{"key":"e_1_3_2_2_52_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.joule.2024.07.018"},{"key":"e_1_3_2_2_53_1","volume-title":"Investing in the rising data center economy. https:\/\/www.mckinsey.com\/industries\/technology-media-and-telecommunications\/our-insights\/investing-in-the-rising-data-center-economy#\/","author":"Company McKinsey","year":"2023","unstructured":"McKinsey & Company. Investing in the rising data center economy. https:\/\/www.mckinsey.com\/industries\/technology-media-and-telecommunications\/our-insights\/investing-in-the-rising-data-center-economy#\/, 2023."},{"key":"e_1_3_2_2_54_1","volume-title":"Analyzing and mitigating data stalls in dnn training. 14(5):771--784, jan","author":"Mohan Jayashree","year":"2021","unstructured":"Jayashree Mohan, Amar Phanishayee, Ashish Raniwala, and Vijay Chidambaram. Analyzing and mitigating data stalls in dnn training. 14(5):771--784, jan 2021."},{"key":"e_1_3_2_2_55_1","doi-asserted-by":"publisher","DOI":"10.1145\/3341301.3359646"},{"key":"e_1_3_2_2_56_1","volume-title":"ICML","author":"Narayanan Deepak","year":"2021","unstructured":"Deepak Narayanan, Amar Phanishayee, Kaiyu Shi, Xie Chen, and Matei Zaharia. Memory-efficient pipeline-parallel DNN training. In ICML, 2021."},{"key":"e_1_3_2_2_57_1","doi-asserted-by":"publisher","DOI":"10.1145\/3458817.3476209"},{"key":"e_1_3_2_2_58_1","unstructured":"NVIDIA. Nvidia H100 tensor core GPU architecture overview. https:\/\/resources.nvidia.com\/en-us-tensor-core\/gtc22-whitepaper-hopper."},{"key":"e_1_3_2_2_59_1","volume-title":"MLSys","author":"Osawa Kazuki","year":"2023","unstructured":"Kazuki Osawa, Shigang Li, and Torsten Hoefler. PipeFisher: Efficient training of large language models using pipelining and fisher information matrices. In MLSys, 2023."},{"key":"e_1_3_2_2_60_1","volume-title":"NeurIPS","author":"Paszke Adam","year":"2019","unstructured":"Adam Paszke, Sam Gross, Francisco Massa, Adam Lerer, James Bradbury, Gregory Chanan, Trevor Killeen, Zeming Lin, Natalia Gimelshein, Luca Antiga, et al. Pytorch: An imperative style, high-performance deep learning library. NeurIPS, 2019."},{"key":"e_1_3_2_2_61_1","doi-asserted-by":"publisher","DOI":"10.1145\/3620666.3651329"},{"key":"e_1_3_2_2_62_1","doi-asserted-by":"publisher","DOI":"10.1109\/LCA.2023.3278652"},{"key":"e_1_3_2_2_63_1","volume-title":"Carbon emissions and large neural network training. arXiv preprint arXiv:2104.10350","author":"Patterson David","year":"2021","unstructured":"David Patterson, Joseph Gonzalez, Quoc Le, Chen Liang, Lluis-Miquel Munguia, Daniel Rothchild, David So, Maud Texier, and Jeff Dean. Carbon emissions and large neural network training. arXiv preprint arXiv:2104.10350, 2021."},{"key":"e_1_3_2_2_64_1","doi-asserted-by":"publisher","DOI":"10.1287\/mnsc.24.4.393"},{"key":"e_1_3_2_2_65_1","doi-asserted-by":"publisher","DOI":"10.1145\/3617232.3624853"},{"issue":"140","key":"e_1_3_2_2_66_1","first-page":"1","article-title":"Exploring the limits of transfer learning with a unified text-to-text transformer","volume":"21","author":"Raffel Colin","year":"2020","unstructured":"Colin Raffel, Noam Shazeer, Adam Roberts, Katherine Lee, Sharan Narang, Michael Matena, Yanqi Zhou, Wei Li, and Peter J. Liu. Exploring the limits of transfer learning with a unified text-to-text transformer. Journal of Machine Learning Research, 21(140):1--67, 2020.","journal-title":"Journal of Machine Learning Research"},{"key":"e_1_3_2_2_67_1","doi-asserted-by":"publisher","DOI":"10.1145\/3373376.3378533"},{"key":"e_1_3_2_2_68_1","doi-asserted-by":"publisher","DOI":"10.1145\/3381831"},{"key":"e_1_3_2_2_69_1","volume-title":"Megatron-LM: Training multibillion parameter language models using model parallelism. arXiv preprint arXiv:1909.08053","author":"Shoeybi Mohammad","year":"2019","unstructured":"Mohammad Shoeybi, Mostofa Patwary, Raul Puri, Patrick LeGresley, Jared Casper, and Bryan Catanzaro. Megatron-LM: Training multibillion parameter language models using model parallelism. arXiv preprint arXiv:1909.08053, 2019."},{"key":"e_1_3_2_2_70_1","doi-asserted-by":"publisher","DOI":"10.5555\/2804694.2804701"},{"key":"e_1_3_2_2_72_1","unstructured":"Shaden Smith Mostofa Patwary Brandon Norick Patrick LeGresley Samyam Rajbhandari Jared Casper Zhun Liu Shrimai Prabhumoye George Zerveas Vijay Korthikanti Elton Zhang Rewon Child Reza Yazdani Aminabadi Julie Bernauer Xia Song Mohammad Shoeybi Yuxiong He Michael Houston Saurabh Tiwary and Bryan Catanzaro. Using deepspeed and megatron to train Megatron-Turing NLG 530b a Large-Scale generative language model. 2022."},{"key":"e_1_3_2_2_73_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/P19-1355"},{"key":"e_1_3_2_2_74_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-32512-0_26"},{"key":"e_1_3_2_2_75_1","doi-asserted-by":"publisher","DOI":"10.1109\/JPROC.2017.2761740"},{"key":"e_1_3_2_2_76_1","volume-title":"NSDI","author":"Thorpe John","year":"2023","unstructured":"John Thorpe, Pengzhan Zhao, Jonathan Eyolfson, Yifan Qiao, Zhihao Jia, Minjia Zhang, Ravi Netravali, and Guoqing Harry Xu. Bamboo: Making preemptible instances resilient for affordable training of large DNNs. In NSDI, 2023."},{"key":"e_1_3_2_2_77_1","volume-title":"NeurIPS","author":"Vaswani Ashish","year":"2017","unstructured":"Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, Llion Jones, Aidan N. Gomez, \u0141ukasz Kaiser, and Illia Polosukhin. Attention is all you need. In NeurIPS, 2017."},{"key":"e_1_3_2_2_78_1","article-title":"Dynamic GPU energy optimization for machine learning training workloads","author":"Wang Farui","year":"2021","unstructured":"Farui Wang, Weizhe Zhang, Shichao Lai, Meng Hao, and Zheng Wang. Dynamic GPU energy optimization for machine learning training workloads. IEEE Transactions on Parallel and Distributed Systems, 2021.","journal-title":"IEEE Transactions on Parallel and Distributed Systems"},{"key":"e_1_3_2_2_79_1","volume-title":"NSDI","author":"Weng Qizhen","year":"2022","unstructured":"Qizhen Weng, Wencong Xiao, Yinghao Yu, Wei Wang, Cheng Wang, Jian He, Yong Li, Liping Zhang, Wei Lin, and Yu Ding. MLaaS in the wild: Workload analysis and scheduling in large-scale heterogeneous GPU clusters. In NSDI, 2022."},{"key":"e_1_3_2_2_80_1","doi-asserted-by":"publisher","DOI":"10.1109\/IGCC.2011.6008564"},{"key":"e_1_3_2_2_81_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.emnlp-demos.6"},{"key":"e_1_3_2_2_82_1","volume-title":"Workshop. BLOOM: A 176b-parameter open-access multilingual language model.","year":"2023","unstructured":"BigScience Workshop. BLOOM: A 176b-parameter open-access multilingual language model. 2023."},{"key":"e_1_3_2_2_83_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICDE60146.2024.00394"},{"key":"e_1_3_2_2_84_1","doi-asserted-by":"publisher","DOI":"10.1109\/JSSC.2023.3316648"},{"key":"e_1_3_2_2_85_1","volume-title":"Chasing low-carbon electricity for practical and sustainable dnn training","author":"Yang Zhenning","year":"2023","unstructured":"Zhenning Yang, Luoxi Meng, Jae-Won Chung, and Mosharaf Chowdhury. Chasing low-carbon electricity for practical and sustainable dnn training. 2023."},{"key":"e_1_3_2_2_86_1","volume-title":"USENIX NSDI","author":"You Jie","year":"2023","unstructured":"Jie You, Jae-Won Chung, and Mosharaf Chowdhury. Zeus: Understanding and optimizing GPU energy consumption of DNN training. In USENIX NSDI, 2023."},{"key":"e_1_3_2_2_87_1","doi-asserted-by":"publisher","DOI":"10.5244\/C.30.87"},{"key":"e_1_3_2_2_88_1","doi-asserted-by":"publisher","DOI":"10.1145\/3542929.3563487"},{"key":"e_1_3_2_2_89_1","volume-title":"ISCA","author":"Zhao Mark","year":"2022","unstructured":"Mark Zhao, Niket Agarwal, Aarti Basant, Bu\u011fra Gedik, Satadru Pan, Mustafa Ozdal, Rakesh Komuravelli, Jerry Pan, Tianshu Bao, Haowei Lu, Sundaram Narayanan, Jack Langman, Kevin Wilfong, Harsha Rastogi, Carole-Jean Wu, Christos Kozyrakis, and Parik Pol. Understanding data storage and ingestion for large-scale deep recommendation model training: Industrial product. In ISCA, 2022."},{"key":"e_1_3_2_2_90_1","volume-title":"USENIX OSDI","author":"Zheng Lianmin","year":"2022","unstructured":"Lianmin Zheng, Zhuohan Li, Hao Zhang, Yonghao Zhuang, Zhifeng Chen, Yanping Huang, Yida Wang, Yuanzhong Xu, Danyang Zhuo, Eric P. Xing, Joseph E. Gonzalez, and Ion Stoica. Alpa: Automating inter- and Intra-Operator parallelism for distributed deep learning. In USENIX OSDI, 2022."},{"key":"e_1_3_2_2_91_1","doi-asserted-by":"publisher","DOI":"10.1109\/CCGrid49817.2020.00-37"},{"key":"e_1_3_2_2_92_1","volume-title":"NSDI","author":"Zu Yazhou","year":"2024","unstructured":"Yazhou Zu, Alireza Ghaffarkhah, Hoang-Vu Dang, Brian Towles, Steven Hand, Safeen Huda, Adekunle Bello, Alexander Kolbasov, Arash Rezaei, Dayou Du, Steve Lacy, Hang Wang, Aaron Wisner, Chris Lewis, and Henri Bahini. Resiliency at scale: Managing Google's TPUv4 machine learning supercomputer. In NSDI, 2024."},{"key":"e_1_3_2_2_93_1","volume-title":"DGX-A100 face to face DGX-2 - performance, power and thermal behavior evaluation. Energies, 14(2)","author":"\u0160pe\u0165ko Matej","year":"2021","unstructured":"Matej \u0160pe\u0165ko, Ond\u0159ej Vysock\u00fd, Branislav Jans\u00edk, and Lubom\u00edr \u0158\u00edha. DGX-A100 face to face DGX-2 - performance, power and thermal behavior evaluation. Energies, 14(2), 2021."}],"event":{"name":"SOSP '24: ACM SIGOPS 30th Symposium on Operating Systems Principles","location":"Austin TX USA","acronym":"SOSP '24","sponsor":["SIGOPS ACM Special Interest Group on Operating Systems","USENIX"]},"container-title":["Proceedings of the ACM SIGOPS 30th Symposium on Operating Systems Principles"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3694715.3695970","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3694715.3695970","content-type":"application\/pdf","content-version":"vor","intended-application":"syndication"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3694715.3695970","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,18]],"date-time":"2025-06-18T20:05:48Z","timestamp":1750277148000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3694715.3695970"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,11,4]]},"references-count":92,"alternative-id":["10.1145\/3694715.3695970","10.1145\/3694715"],"URL":"https:\/\/doi.org\/10.1145\/3694715.3695970","relation":{},"subject":[],"published":{"date-parts":[[2024,11,4]]},"assertion":[{"value":"2024-11-15","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}