{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,9]],"date-time":"2026-07-09T16:08:26Z","timestamp":1783613306374,"version":"3.55.0"},"publisher-location":"New York, NY, USA","reference-count":72,"publisher":"ACM","funder":[{"name":"IBM IIDAI","award":["NA"],"award-info":[{"award-number":["NA"]}]},{"name":"NRF Korea","award":["RS-2024-00405857"],"award-info":[{"award-number":["RS-2024-00405857"]}]},{"name":"MSIT IITP","award":["RS-2024-00456287"],"award-info":[{"award-number":["RS-2024-00456287"]}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2025,10,18]]},"DOI":"10.1145\/3725843.3756079","type":"proceedings-article","created":{"date-parts":[[2025,10,17]],"date-time":"2025-10-17T17:19:56Z","timestamp":1760721596000},"page":"609-625","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":2,"title":["NetZIP: Algorithm\/Hardware Co-design of In-network Lossless Compression for Distributed Large Model Training"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0009-0009-2314-2734","authenticated-orcid":false,"given":"Jinghan","family":"Huang","sequence":"first","affiliation":[{"name":"University of Illinois Urbana-Champaign, Urbana, Illinois, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-1157-0127","authenticated-orcid":false,"given":"Hyungyo","family":"Kim","sequence":"additional","affiliation":[{"name":"University of Illinois Urbana-Champaign, Urbana, Illinois, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0000-1431-0067","authenticated-orcid":false,"given":"Nachuan","family":"Wang","sequence":"additional","affiliation":[{"name":"University of Illinois Urbana-Champaign, Urbana, Illinois, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0006-0023-743X","authenticated-orcid":false,"given":"Jaeyoung","family":"Kang","sequence":"additional","affiliation":[{"name":"University of Illinois Urbana-Champaign, Urbana, Illinois, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0001-5373-5743","authenticated-orcid":false,"given":"Hrishi","family":"Shah","sequence":"additional","affiliation":[{"name":"University of Illinois Urbana-Champaign, Urbana, Illinois, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-2335-4798","authenticated-orcid":false,"given":"Eun Kyung","family":"Lee","sequence":"additional","affiliation":[{"name":"IBM Research, Yorktown Heights, New York, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-8165-166X","authenticated-orcid":false,"given":"Minjia","family":"Zhang","sequence":"additional","affiliation":[{"name":"University of Illinois Urbana-Champaign, Urbana, Illinois, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0005-0472-107X","authenticated-orcid":false,"given":"Fan","family":"Lai","sequence":"additional","affiliation":[{"name":"University of Illinois Urbana-Champaign, Urbana, Illinois, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-0442-5634","authenticated-orcid":false,"given":"Nam Sung","family":"Kim","sequence":"additional","affiliation":[{"name":"University of Illinois Urbana-Champaign, Urbana, Illinois, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2025,10,17]]},"reference":[{"key":"e_1_3_3_2_2_2","unstructured":"Mart\u00edn Abadi Ashish Agarwal Paul Barham Eugene Brevdo Zhifeng Chen Craig Citro Greg\u00a0S. Corrado Andy Davis Jeffrey Dean Matthieu Devin Sanjay Ghemawat Ian Goodfellow Andrew Harp Geoffrey Irving Michael Isard Yangqing Jia Rafal Jozefowicz Lukasz Kaiser Manjunath Kudlur Josh Levenberg Dan Mane Rajat Monga Sherry Moore Derek Murray Chris Olah Mike Schuster Jonathon Shlens Benoit Steiner Ilya Sutskever Kunal Talwar Paul Tucker Vincent Vanhoucke Vijay Vasudevan Fernanda Viegas Oriol Vinyals Pete Warden Martin Wattenberg Martin Wicke Yuan Yu and Xiaoqiang Zheng. 2016. TensorFlow: Large-Scale Machine Learning on Heterogeneous Distributed Systems. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/1603.04467 (2016)."},{"key":"e_1_3_3_2_3_2","unstructured":"Dan Alistarh Demjan Grubic Jerry Li Ryota Tomioka and Milan Vojnovic. 2017. QSGD: Communication-Efficient SGD via Gradient Quantization and Encoding. Advances in neural information processing systems 30 (2017)."},{"key":"e_1_3_3_2_4_2","unstructured":"Amazon. accessed in 2025. Amazon EC2 Instance Types. https:\/\/docs.aws.amazon.com\/ec2\/latest\/instancetypes\/ec2-types.pdf."},{"key":"e_1_3_3_2_5_2","unstructured":"AMD. accessed in 2025. Alveo U280 Data Center Accelerator Card. https:\/\/www.xilinx.com\/publications\/product-briefs\/alveo-u280-product-brief.pdf."},{"key":"e_1_3_3_2_6_2","unstructured":"AMD. accessed in 2025. Vitis Data Compression Library. https:\/\/github.com\/Xilinx\/Vitis_Libraries\/tree\/master\/data_compression."},{"key":"e_1_3_3_2_7_2","unstructured":"Anton Shilov. accessed in 2025. GlobalFoundries Details 7 nm Plans: Three Generations 700 mm2 HVM in 2018. https:\/\/www.anandtech.com\/show\/11558\/globalfoundries-details-7-nm-plans-three-generations-700-mm-hvm-in-2018."},{"key":"e_1_3_3_2_8_2","unstructured":"BigScience Workshop. accessed in 2025. bloom. https:\/\/huggingface.co\/bigscience\/bloom."},{"key":"e_1_3_3_2_9_2","unstructured":"CloudLab. accessed in 2025. CloudLab Portal Hardware. https:\/\/www.cloudlab.us\/portal-hardware.php."},{"key":"e_1_3_3_2_10_2","first-page":"1","volume-title":"USENIX annual technical conference (USENIX ATC)","author":"Duplyakin Dmitry","year":"2019","unstructured":"Dmitry Duplyakin, Robert Ricci, Aleksander Maricq, Gary Wong, Jonathon Duerig, Eric Eide, Leigh Stoller, Mike Hibler, David Johnson, Kirk Webb, Aditya Akella, Kuangching Wang, Glenn Ricart, Larry Landweber, Chip Elliott, Michael Zink, Emmanuel Cecchet, Snigdhaswin Kar, and Prabodh Mishra. 2019. The Design and Operation of CloudLab. In USENIX annual technical conference (USENIX ATC). 1\u201314."},{"key":"e_1_3_3_2_11_2","unstructured":"Abhinav Dutta Sanjeev Krishnan Nipun Kwatra and Ramachandran Ramjee. 2024. Accuracy is Not All You Need. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2407.09141 (2024)."},{"key":"e_1_3_3_2_12_2","unstructured":"Facebook. accessed in 2025. zstd. https:\/\/github.com\/facebook\/zstd."},{"key":"e_1_3_3_2_13_2","unstructured":"Jared Fernandez Luca Wehrstedt Leonid Shamis Mostafa Elhoushi Kalyan Saladi Yonatan Bisk Emma Strubell and Jacob Kahn. 2024. Hardware Scaling Trends and Diminishing Returns in Large-Scale Distributed Training. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2411.13055 (2024)."},{"key":"e_1_3_3_2_14_2","first-page":"57","volume-title":"Proceedings of the ACM SIGCOMM Conference (SIGCOMM)","author":"Gangidi Adithya","year":"2024","unstructured":"Adithya Gangidi, Rui Miao, Shengbao Zheng, Sai\u00a0Jayesh Bondu, Guilherme Goes, Hany Morsy, Rohit Puri, Mohammad Riftadi, Ashmitha\u00a0Jeevaraj Shetty, Jingyi Yang, Shuqiang Zhang, Mikel\u00a0Jimenez Fernandez, Shashidhar Gandham, and Hongyi Zeng. 2024. RDMA over Ethernet for Distributed AI Training at Meta Scale. In Proceedings of the ACM SIGCOMM Conference (SIGCOMM). 57\u201370."},{"key":"e_1_3_3_2_15_2","unstructured":"Google. accessed in 2025. snappy. https:\/\/github.com\/google\/snappy."},{"key":"e_1_3_3_2_16_2","unstructured":"Aaron Grattafiori Abhimanyu Dubey Abhinav Jauhri Abhinav Pandey Abhishek Kadian Ahmad Al-Dahle Aiesha Letman Akhil Mathur Alan Schelten Alex Vaughan Amy Yang Angela Fan Anirudh Goyal Anthony Hartshorn Aobo Yang Archi Mitra Archie Sravankumar Artem Korenev Arthur Hinsvark Arun Rao Aston Zhang Aurelien Rodriguez Austen Gregerson Ava Spataru Baptiste Roziere Bethany Biron Binh Tang Bobbie Chern Charlotte Caucheteux Chaya Nayak Chloe Bi Chris Marra Chris McConnell Christian Keller Christophe Touret Chunyang Wu Corinne Wong Cristian\u00a0Canton Ferrer Cyrus Nikolaidis Damien Allonsius Daniel Song Danielle Pintz Danny Livshits Danny Wyatt David Esiobu Dhruv Choudhary Dhruv Mahajan Diego Garcia-Olano Diego Perino Dieuwke Hupkes Egor Lakomkin Ehab AlBadawy Elina Lobanova Emily Dinan Eric\u00a0Michael Smith Filip Radenovic Francisco Guzm\u00e1n Frank Zhang Gabriel Synnaeve Gabrielle Lee Georgia\u00a0Lewis Anderson Govind Thattai Graeme Nail Gregoire Mialon Guan Pang Guillem Cucurell Hailey Nguyen Hannah Korevaar Hu Xu Hugo Touvron Iliyan Zarov Imanol\u00a0Arrieta Ibarra Isabel Kloumann Ishan Misra Ivan Evtimov Jack Zhang Jade Copet Jaewon Lee Jan Geffert Jana Vranes Jason Park Jay Mahadeokar Jeet Shah Jelmer van\u00a0der Linde Jennifer Billock Jenny Hong Jenya Lee Jeremy Fu Jianfeng Chi Jianyu Huang Jiawen Liu Jie Wang Jiecao Yu Joanna Bitton Joe Spisak Jongsoo Park Joseph Rocca Joshua Johnstun Joshua Saxe Junteng Jia Kalyan\u00a0Vasuden Alwala Karthik Prasad Kartikeya Upasani Kate Plawiak Ke Li Kenneth Heafield Kevin Stone Khalid El-Arini Krithika Iyer Kshitiz Malik Kuenley Chiu Kunal Bhalla Kushal Lakhotia Lauren Rantala-Yeary Laurens van\u00a0der Maaten Lawrence Chen Liang Tan Liz Jenkins Louis Martin Lovish Madaan Lubo Malo Lukas Blecher Lukas Landzaat Luke de Oliveira Madeline Muzzi Mahesh Pasupuleti Mannat Singh Manohar Paluri Marcin Kardas Maria Tsimpoukelli Mathew Oldham Mathieu Rita Maya Pavlova Melanie Kambadur Mike Lewis Min Si Mitesh\u00a0Kumar Singh Mona Hassan Naman Goyal Narjes Torabi Nikolay Bashlykov Nikolay Bogoychev Niladri Chatterji Ning Zhang Olivier Duchenne Onur \u00c7elebi Patrick Alrassy Pengchuan Zhang Pengwei Li Petar Vasic Peter Weng Prajjwal Bhargava Pratik Dubal Praveen Krishnan Punit\u00a0Singh Koura Puxin Xu Qing He Qingxiao Dong Ragavan Srinivasan Raj Ganapathy Ramon Calderer Ricardo\u00a0Silveira Cabral Robert Stojnic Roberta Raileanu Rohan Maheswari Rohit Girdhar Rohit Patel Romain Sauvestre Ronnie Polidoro Roshan Sumbaly Ross Taylor Ruan Silva Rui Hou Rui Wang Saghar Hosseini Sahana Chennabasappa Sanjay Singh Sean Bell Seohyun\u00a0Sonia Kim Sergey Edunov Shaoliang Nie Sharan Narang Sharath Raparthy Sheng Shen Shengye Wan Shruti Bhosale Shun Zhang Simon Vandenhende Soumya Batra Spencer Whitman Sten Sootla Stephane Collot Suchin Gururangan Sydney Borodinsky Tamar Herman Tara Fowler Tarek Sheasha Thomas Georgiou Thomas Scialom Tobias Speckbacher Todor Mihaylov Tong Xiao Ujjwal Karn Vedanuj Goswami Vibhor Gupta Vignesh Ramanathan Viktor Kerkez Vincent Gonguet Virginie Do Vish Vogeti V\u00edtor Albiero Vladan Petrovic Weiwei Chu Wenhan Xiong Wenyin Fu Whitney Meers Xavier Martinet Xiaodong Wang Xiaofang Wang Xiaoqing\u00a0Ellen Tan Xide Xia Xinfeng Xie Xuchao Jia Xuewei Wang Yaelle Goldschlag Yashesh Gaur Yasmine Babaei Yi Wen Yiwen Song Yuchen Zhang Yue Li Yuning Mao Zacharie\u00a0Delpierre Coudert Zheng Yan Zhengxing Chen Zoe Papakipos Aaditya Singh Aayushi Srivastava Abha Jain Adam Kelsey Adam Shajnfeld Adithya Gangidi Adolfo Victoria Ahuva Goldstand Ajay Menon Ajay Sharma Alex Boesenberg Alexei Baevski Allie Feinstein Amanda Kallet Amit Sangani Amos Teo Anam Yunus Andrei Lupu Andres Alvarado Andrew Caples Andrew Gu Andrew Ho Andrew Poulton Andrew Ryan Ankit Ramchandani Annie Dong Annie Franco Anuj Goyal Aparajita Saraf Arkabandhu Chowdhury Ashley Gabriel Ashwin Bharambe Assaf Eisenman Azadeh Yazdan Beau James Ben Maurer Benjamin Leonhardi Bernie Huang Beth Loyd Beto\u00a0De Paola Bhargavi Paranjape Bing Liu Bo Wu Boyu Ni Braden Hancock Bram Wasti Brandon Spence Brani Stojkovic Brian Gamido Britt Montalvo Carl Parker Carly Burton Catalina Mejia Ce Liu Changhan Wang Changkyu Kim Chao Zhou Chester Hu Ching-Hsiang Chu Chris Cai Chris Tindal Christoph Feichtenhofer Cynthia Gao Damon Civin Dana Beaty Daniel Kreymer Daniel Li David Adkins David Xu Davide Testuggine Delia David Devi Parikh Diana Liskovich Didem Foss Dingkang Wang Duc Le Dustin Holland Edward Dowling Eissa Jamil Elaine Montgomery Eleonora Presani Emily Hahn Emily Wood Eric-Tuan Le Erik Brinkman Esteban Arcaute Evan Dunbar Evan Smothers Fei Sun Felix Kreuk Feng Tian Filippos Kokkinos Firat Ozgenel Francesco Caggioni Frank Kanayet Frank Seide Gabriela\u00a0Medina Florez Gabriella Schwarz Gada Badeer Georgia Swee Gil Halpern Grant Herman Grigory Sizov Guangyi Zhang Guna Lakshminarayanan Hakan Inan Hamid Shojanazeri Han Zou Hannah Wang Hanwen Zha Haroun Habeeb Harrison Rudolph Helen Suk Henry Aspegren Hunter Goldman Hongyuan Zhan Ibrahim Damlaj Igor Molybog Igor Tufanov Ilias Leontiadis Irina-Elena Veliche Itai Gat Jake Weissman James Geboski James Kohli Janice Lam Japhet Asher Jean-Baptiste Gaya Jeff Marcus Jeff Tang Jennifer Chan Jenny Zhen Jeremy Reizenstein Jeremy Teboul Jessica Zhong Jian Jin Jingyi Yang Joe Cummings Jon Carvill Jon Shepard Jonathan McPhie Jonathan Torres Josh Ginsburg Junjie Wang Kai Wu Kam\u00a0Hou U Karan Saxena Kartikay Khandelwal Katayoun Zand Kathy Matosich Kaushik Veeraraghavan Kelly Michelena Keqian Li Kiran Jagadeesh Kun Huang Kunal Chawla Kyle Huang Lailin Chen Lakshya Garg Lavender A Leandro Silva Lee Bell Lei Zhang Liangpeng Guo Licheng Yu Liron Moshkovich Luca Wehrstedt Madian Khabsa Manav Avalani Manish Bhatt Martynas Mankus Matan Hasson Matthew Lennie Matthias Reso Maxim Groshev Maxim Naumov Maya Lathi Meghan Keneally Miao Liu Michael\u00a0L. Seltzer Michal Valko Michelle Restrepo Mihir Patel Mik Vyatskov Mikayel Samvelyan Mike Clark Mike Macey Mike Wang Miquel\u00a0Jubert Hermoso Mo Metanat Mohammad Rastegari Munish Bansal Nandhini Santhanam Natascha Parks Natasha White Navyata Bawa Nayan Singhal Nick Egebo Nicolas Usunier Nikhil Mehta Nikolay\u00a0Pavlovich Laptev Ning Dong Norman Cheng Oleg Chernoguz Olivia Hart Omkar Salpekar Ozlem Kalinli Parkin Kent Parth Parekh Paul Saab Pavan Balaji Pedro Rittner Philip Bontrager Pierre Roux Piotr Dollar Polina Zvyagina Prashant Ratanchandani Pritish Yuvraj Qian Liang Rachad Alao Rachel Rodriguez Rafi Ayub Raghotham Murthy Raghu Nayani Rahul Mitra Rangaprabhu Parthasarathy Raymond Li Rebekkah Hogan Robin Battey Rocky Wang Russ Howes Ruty Rinott Sachin Mehta Sachin Siby Sai\u00a0Jayesh Bondu Samyak Datta Sara Chugh Sara Hunt Sargun Dhillon Sasha Sidorov Satadru Pan Saurabh Mahajan Saurabh Verma Seiji Yamamoto Sharadh Ramaswamy Shaun Lindsay Shaun Lindsay Sheng Feng Shenghao Lin Shengxin\u00a0Cindy Zha Shishir Patil Shiva Shankar Shuqiang Zhang Shuqiang Zhang Sinong Wang Sneha Agarwal Soji Sajuyigbe Soumith Chintala Stephanie Max Stephen Chen Steve Kehoe Steve Satterfield Sudarshan Govindaprasad Sumit Gupta Summer Deng Sungmin Cho Sunny Virk Suraj Subramanian Sy Choudhury Sydney Goldman Tal Remez Tamar Glaser Tamara Best Thilo Koehler Thomas Robinson Tianhe Li Tianjun Zhang Tim Matthews Timothy Chou Tzook Shaked Varun Vontimitta Victoria Ajayi Victoria Montanez Vijai Mohan Vinay\u00a0Satish Kumar Vishal Mangla Vlad Ionescu Vlad Poenaru Vlad\u00a0Tiberiu Mihailescu Vladimir Ivanov Wei Li Wenchen Wang Wenwen Jiang Wes Bouaziz Will Constable Xiaocheng Tang Xiaojian Wu Xiaolan Wang Xilun Wu Xinbo Gao Yaniv Kleinman Yanjun Chen Ye Hu Ye Jia Ye Qi Yenda Li Yilin Zhang Ying Zhang Yossi Adi Youngjin Nam Yu Wang Yu Zhao Yuchen Hao Yundi Qian Yunlu Li Yuzi He Zach Rait Zachary DeVito Zef Rosnbrick Zhaoduo Wen Zhenyu Yang Zhiwei Zhao and Zhiyu Ma. 2024. The Llama 3 Herd of Models. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2407.21783 (2024)."},{"key":"e_1_3_3_2_17_2","unstructured":"Yutong He Pengrui Li Yipeng Hu Chuyan Chen and Kun Yuan. 2024. Subspace Optimization for Large Language Models with Convergence Guarantees. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2410.11289 (2024)."},{"key":"e_1_3_3_2_18_2","doi-asserted-by":"publisher","DOI":"10.1109\/ISCA59077.2024.00051"},{"key":"e_1_3_3_2_19_2","doi-asserted-by":"crossref","unstructured":"David\u00a0A Huffman. 1952. A Method for the Construction of Minimum-Redundancy Codes. Proceedings of the IRE 40 9 (1952) 1098\u20131101.","DOI":"10.1109\/JRPROC.1952.273898"},{"key":"e_1_3_3_2_20_2","unstructured":"Hugging Face. accessed in 2025. Hugging Face. https:\/\/huggingface.co."},{"key":"e_1_3_3_2_21_2","unstructured":"Ajay Jaiswal Zhe Gan Xianzhi Du Bowen Zhang Zhangyang Wang and Yinfei Yang. 2023. Compressing LLMs: The Truth is Rarely Pure and Never Simple. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2310.01382 (2023)."},{"key":"e_1_3_3_2_22_2","unstructured":"Justin Pietsch. accessed in 2025. A summary of High Speed Ethernet ASICs. https:\/\/elegantnetwork.github.io\/posts\/A-Summary-of-Network-ASICs\/."},{"key":"e_1_3_3_2_23_2","doi-asserted-by":"crossref","unstructured":"Prachi Kashikar Olivier Sentieys and Sharad Sinha. 2023. Lossless Neural Network Model Compression Through Exponent Sharing. IEEE Transactions on Very Large Scale Integration Systems (VLSI) (2023).","DOI":"10.1109\/TVLSI.2023.3307607"},{"key":"e_1_3_3_2_24_2","doi-asserted-by":"crossref","unstructured":"Yousun Ko Alex Chadwick Daniel Bates and Robert Mullins. 2021. Lane Compression: A Lightweight Lossless Compression Method for Machine Learning on Embedded Systems. ACM Transactions on Embedded Computing Systems (TECS) 20 2 (2021) 1\u201326.","DOI":"10.1145\/3431815"},{"key":"e_1_3_3_2_25_2","doi-asserted-by":"publisher","DOI":"10.1007\/978-1-4419-0739-4"},{"key":"e_1_3_3_2_26_2","first-page":"769","volume-title":"USENIX Symposium on Networked Systems Design and Implementation (NSDI)","author":"Lai Fan","year":"2023","unstructured":"Fan Lai, Yinwei Dai, Harsha\u00a0V. Madhyastha, and Mosharaf Chowdhury. 2023. ModelKeeper: Accelerating DNN Training via Automated Training Warmup. In USENIX Symposium on Networked Systems Design and Implementation (NSDI). 769\u2013785."},{"key":"e_1_3_3_2_27_2","first-page":"273","volume-title":"USENIX Symposium on Networked Systems Design and Implementation (NSDI)","author":"Lai Fan","year":"2020","unstructured":"Fan Lai, Jie You, Xiangfeng Zhu, Harsha\u00a0V. Madhyastha, and Mosharaf Chowdhury. 2020. Sol: Fast Distributed Computation Over Slow Networks. In USENIX Symposium on Networked Systems Design and Implementation (NSDI). 273\u2013288."},{"key":"e_1_3_3_2_28_2","doi-asserted-by":"publisher","DOI":"10.1145\/3663408.3663409"},{"key":"e_1_3_3_2_29_2","doi-asserted-by":"publisher","DOI":"10.1109\/MICRO.2018.00023"},{"key":"e_1_3_3_2_30_2","unstructured":"Xiangru Lian Yijun Huang Yuncheng Li and Ji Liu. 2015. Asynchronous Parallel Stochastic Gradient for Nonconvex Optimization. Advances in Neural Information Processing Systems 28 (2015) 2737\u20132745."},{"key":"e_1_3_3_2_31_2","unstructured":"Xiangru Lian Ce Zhang Huan Zhang Cho-Jui Hsieh Wei Zhang and Ji Liu. 2017. Can Decentralized Algorithms Outperform Centralized Algorithms? A Case Study for Decentralized Parallel Stochastic Gradient Descent. Advances in neural information processing systems 30 (2017)."},{"key":"e_1_3_3_2_32_2","unstructured":"Tao Lin Sebastian\u00a0U Stich Kumar\u00a0Kshitij Patel and Martin Jaggi. 2018. Don\u2019t Use Large Mini-Batches Use Local SGD. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/1808.07217 (2018)."},{"key":"e_1_3_3_2_33_2","first-page":"7111","volume-title":"International Conference on Machine Learning","author":"Lu Yucheng","year":"2021","unstructured":"Yucheng Lu and Christopher De\u00a0Sa. 2021. Optimal Complexity in Decentralized Training. In International Conference on Machine Learning. PMLR, 7111\u20137123."},{"key":"e_1_3_3_2_34_2","unstructured":"lz4. accessed in 2025. lz4. https:\/\/github.com\/lz4\/lz4."},{"key":"e_1_3_3_2_35_2","unstructured":"madler. accessed in 2025. zlib. https:\/\/github.com\/madler\/zlib."},{"key":"e_1_3_3_2_36_2","first-page":"809","volume-title":"USENIX Symposium on Networked Systems Design and Implementation (NSDI)","author":"Mahajan Kshiteej","year":"2023","unstructured":"Kshiteej Mahajan, Ching-Hsiang Chu, Srinivas Sridharan, and Aditya Akella. 2023. Better Together: Jointly Optimizing ML Collective Scheduling and Execution Planning using SYNDICATE. In USENIX Symposium on Networked Systems Design and Implementation (NSDI). 809\u2013824."},{"key":"e_1_3_3_2_37_2","unstructured":"Meta Llama. accessed in 2025. Llama-3.1-405B. https:\/\/huggingface.co\/meta-llama\/Llama-3.1-405B."},{"key":"e_1_3_3_2_38_2","unstructured":"Shervin Minaee Tomas Mikolov Narjes Nikzad Meysam Chenaghlu Richard Socher Xavier Amatriain and Jianfeng Gao. 2024. Large Language Models: A Survey. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2402.06196 (2024)."},{"key":"e_1_3_3_2_39_2","unstructured":"Mistral AI. accessed in 2025. Mixtral-8x7B. https:\/\/huggingface.co\/mistralai."},{"key":"e_1_3_3_2_40_2","unstructured":"Jayashree Mohan Amar Phanishayee Ashish Raniwala and Vijay Chidambaram. 2020. Analyzing and Mitigating Data Stalls in DNN Training. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2007.06775 (2020)."},{"key":"e_1_3_3_2_41_2","unstructured":"Feng Niu Benjamin Recht Christopher R\u00e9 and Stephen\u00a0J Wright. 2011. Hogwild!: A Lock-Free Approach to Parallelizing Stochastic Gradient Descent. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/1106.5730 (2011)."},{"key":"e_1_3_3_2_42_2","unstructured":"NVIDIA. accessed in 2025. DOCA SDK Document. https:\/\/docs.nvidia.com\/doca\/sdk\/index.html."},{"key":"e_1_3_3_2_43_2","unstructured":"NVIDIA. accessed in 2025. NVIDIA Collective Communications Library (NCCL). https:\/\/developer.nvidia.com\/nccl."},{"key":"e_1_3_3_2_44_2","unstructured":"NVIDIA. accessed in 2025. NVIDIA ConnectX-7 Adapter Cards User Manual. https:\/\/docs.nvidia.com\/networking\/display\/connectx7vpi\/specifications."},{"key":"e_1_3_3_2_45_2","unstructured":"NVIDIA. accessed in 2025. NVIDIA DGX H100. https:\/\/openzeka.com\/en\/wp-content\/uploads\/2022\/04\/ai-for-enterprise-dgx-h100-datasheet-nvidia-a4-2146027-r3-web.pdf."},{"key":"e_1_3_3_2_46_2","unstructured":"NVIDIA. accessed in 2025. NVIDIA nvCOMP. https:\/\/developer.nvidia.com\/nvcomp."},{"key":"e_1_3_3_2_47_2","unstructured":"NVIDIA. accessed in 2025. NVIDIA Spectrum-X Network Platform Architecture. https:\/\/resources.nvidia.com\/en-us-accelerated-networking-resource-library\/nvidia-spectrum-x."},{"key":"e_1_3_3_2_48_2","unstructured":"NVIDIA. accessed in 2025. NVLink and NVLink Switch. https:\/\/www.nvidia.com\/en-us\/data-center\/nvlink\/."},{"key":"e_1_3_3_2_49_2","unstructured":"PyTorch. accessed in 2025. PyTorch. https:\/\/pytorch.org."},{"key":"e_1_3_3_2_50_2","doi-asserted-by":"publisher","DOI":"10.1145\/3651890.3672265"},{"key":"e_1_3_3_2_51_2","unstructured":"Alec Radford Karthik Narasimhan Tim Salimans and Ilya Sutskever. 2018. Improving Language Understanding by Generative Pre-Training. (2018)."},{"key":"e_1_3_3_2_52_2","first-page":"1403","volume-title":"USENIX Symposium on Networked Systems Design and Implementation (NSDI)","author":"Rajasekaran Sudarsanan","year":"2024","unstructured":"Sudarsanan Rajasekaran, Manya Ghobadi, and Aditya Akella. 2024. CASSINI: Network-Aware Job Scheduling in Machine Learning Clusters. In USENIX Symposium on Networked Systems Design and Implementation (NSDI). 1403\u20131420."},{"key":"e_1_3_3_2_53_2","doi-asserted-by":"publisher","DOI":"10.1145\/3394486.3406703"},{"key":"e_1_3_3_2_54_2","doi-asserted-by":"publisher","DOI":"10.1134\/S1064562423701314"},{"key":"e_1_3_3_2_55_2","unstructured":"Mohammad Shoeybi Mostofa Patwary Raul Puri Patrick LeGresley Jared Casper and Bryan Catanzaro. 2019. Megatron-LM: Training Multi-Billion Parameter Language Models Using Model Parallelism. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/1909.08053 (2019)."},{"key":"e_1_3_3_2_56_2","doi-asserted-by":"publisher","DOI":"10.1145\/3575693.3575712"},{"key":"e_1_3_3_2_57_2","unstructured":"Sebastian\u00a0U Stich. 2018. Local SGD Converges Fast and Communicates Little. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/1805.09767 (2018)."},{"key":"e_1_3_3_2_58_2","unstructured":"Hanlin Tang Shaoduo Gan Ce Zhang Tong Zhang and Ji Liu. 2018. Communication Compression for Decentralized Training. Advances in Neural Information Processing Systems 31 (2018)."},{"key":"e_1_3_3_2_59_2","unstructured":"Rohan Taori Ishaan Gulrajani Tianyi Zhang Yann Dubois Xuechen Li Carlos Guestrin Percy Liang and Tatsunori\u00a0B Hashimoto. 2023. Alpaca: A Strong Replicable Instruction-Following Model. Stanford Center for Research on Foundation Models. https:\/\/crfm. stanford. edu\/2023\/03\/13\/alpaca. html 3 6 (2023) 7."},{"key":"e_1_3_3_2_60_2","first-page":"563","volume-title":"USENIX Annual Technical Conference (USENIX ATC)","author":"Um Taegeon","year":"2024","unstructured":"Taegeon Um, Byungsoo Oh, Minyoung Kang, Woo-Yeon Lee, Goeun Kim, Dongseob Kim, Youngtaek Kim, Mohd Muzzammil, and Myeongjae Jeon. 2024. Metis: Fast Automatic Distributed Training on Heterogeneous GPUs. In USENIX Annual Technical Conference (USENIX ATC). 563\u2013578."},{"key":"e_1_3_3_2_61_2","first-page":"431","volume-title":"Proceedings of the International Conference on Computational Linguistics: Industry Track (COLING)","author":"Wang Chengyu","year":"2025","unstructured":"Chengyu Wang, Yuanhao Yue, Jun Huang, and Peng Wang. 2025. Building a Family of Data Augmentation Models for Low-cost LLM Fine-tuning on the Cloud. In Proceedings of the International Conference on Computational Linguistics: Industry Track (COLING). 431\u2013444."},{"key":"e_1_3_3_2_62_2","unstructured":"Guanhua Wang Chengming Zhang Zheyu Shen Ang Li and Olatunji Ruwase. 2024. Domino: Eliminating Communication in LLM Training via Generic Tensor Slicing and Overlapping. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2409.15241 (2024)."},{"key":"e_1_3_3_2_63_2","unstructured":"Hongyi Wang Scott Sievert Zachary Charles Shengchao Liu Stephen Wright and Dimitris Papailiopoulos. 2018. Atomo: Communication-efficient learning via atomic sparsification. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/1806.04090 (2018)."},{"key":"e_1_3_3_2_64_2","volume-title":"USENIX Symposium on Networked Systems Design and Implementation (NSDI)","author":"Wang Xizheng","year":"2025","unstructured":"Xizheng Wang, Qingxu Li, Yichi Xu, Gang Lu, Dan Li, Li Chen, Heyang Zhou, Linkang Zheng, Sen Zhang, Yikai Zhu, Yang Liu, Pengcheng Zhang, Kun Qian, Kunling He, Jiaqi Gao, Ennan Zhai, Dennis Cai, and Binzhang Fu. 2025. SimAI: Unifying Architecture Design and Performance Tunning for Large-Scale Large Language Model Training with Scalability and Precision. In USENIX Symposium on Networked Systems Design and Implementation (NSDI)."},{"key":"e_1_3_3_2_65_2","doi-asserted-by":"publisher","DOI":"10.1145\/3552326.3587451"},{"key":"e_1_3_3_2_66_2","doi-asserted-by":"crossref","unstructured":"Zeqin Wang Ming Wen Yuedong Xu Yipeng Zhou Jessie\u00a0Hui Wang and Liang Zhang. 2023. Communication compression techniques in distributed deep learning: A survey. Journal of Systems Architecture 142 (2023) 102927.","DOI":"10.1016\/j.sysarc.2023.102927"},{"key":"e_1_3_3_2_67_2","unstructured":"Jianqiao Wangni Jialei Wang Ji Liu and Tong Zhang. 2017. Gradient Sparsification for Communication-Efficient Distributed Optimization. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/1710.09854 (2017)."},{"key":"e_1_3_3_2_68_2","unstructured":"Wei Wen Cong Xu Feng Yan Chunpeng Wu Yandan Wang Yiran Chen and Hai Li. 2017. TernGrad: Ternary Gradients to Reduce Communication in Distributed Deep Learning. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/1705.07878 (2017)."},{"key":"e_1_3_3_2_69_2","unstructured":"Wikipedia. accessed in 2025. LZ77 and LZ78. https:\/\/en.wikipedia.org\/wiki\/LZ77_and_LZ78."},{"key":"e_1_3_3_2_70_2","unstructured":"BigScience Workshop : Teven\u00a0Le Scao Angela Fan Christopher Akiki Ellie Pavlick Suzana Ili\u0107 Daniel Hesslow Roman Castagn\u00e9 Alexandra\u00a0Sasha Luccioni Fran\u00e7ois Yvon Matthias Gall\u00e9 Jonathan Tow Alexander\u00a0M. Rush Stella Biderman Albert Webson Pawan\u00a0Sasanka Ammanamanchi Thomas Wang Beno\u00eet Sagot Niklas Muennighoff Albert\u00a0Villanova del Moral Olatunji Ruwase Rachel Bawden Stas Bekman Angelina McMillan-Major Iz Beltagy Huu Nguyen Lucile Saulnier Samson Tan Pedro\u00a0Ortiz Suarez Victor Sanh Hugo Lauren\u00e7on Yacine Jernite Julien Launay Margaret Mitchell Colin Raffel Aaron Gokaslan Adi Simhi Aitor Soroa Alham\u00a0Fikri Aji Amit Alfassy Anna Rogers Ariel\u00a0Kreisberg Nitzav Canwen Xu Chenghao Mou Chris Emezue Christopher Klamm Colin Leong Daniel van Strien David\u00a0Ifeoluwa Adelani Dragomir Radev Eduardo\u00a0Gonz\u00e1lez Ponferrada Efrat Levkovizh Ethan Kim Eyal\u00a0Bar Natan Francesco\u00a0De Toni G\u00e9rard Dupont Germ\u00e1n Kruszewski Giada Pistilli Hady Elsahar Hamza Benyamina Hieu Tran Ian Yu Idris Abdulmumin Isaac Johnson Itziar Gonzalez-Dios Javier de\u00a0la Rosa Jenny Chim Jesse Dodge Jian Zhu Jonathan Chang J\u00f6rg Frohberg Joseph Tobing Joydeep Bhattacharjee Khalid Almubarak Kimbo Chen Kyle Lo Leandro\u00a0Von Werra Leon Weber Long Phan Loubna\u00a0Ben allal Ludovic Tanguy Manan Dey Manuel\u00a0Romero Mu\u00f1oz Maraim Masoud Mar\u00eda Grandury Mario \u0160a\u0161ko Max Huang Maximin Coavoux Mayank Singh Mike Tian-Jian Jiang Minh\u00a0Chien Vu Mohammad\u00a0A. Jauhar Mustafa Ghaleb Nishant Subramani Nora Kassner Nurulaqilla Khamis Olivier Nguyen Omar Espejel Ona de Gibert Paulo Villegas Peter Henderson Pierre Colombo Priscilla Amuok Quentin Lhoest Rheza Harliman Rishi Bommasani Roberto\u00a0Luis L\u00f3pez Rui Ribeiro Salomey Osei Sampo Pyysalo Sebastian Nagel Shamik Bose Shamsuddeen\u00a0Hassan Muhammad Shanya Sharma Shayne Longpre Somaieh Nikpoor Stanislav Silberberg Suhas Pai Sydney Zink Tiago\u00a0Timponi Torrent Timo Schick Tristan Thrush Valentin Danchev Vassilina Nikoulina Veronika Laippala Violette Lepercq Vrinda Prabhu Zaid Alyafeai Zeerak Talat Arun Raja Benjamin Heinzerling Chenglei Si Davut\u00a0Emre Ta\u015far Elizabeth Salesky Sabrina\u00a0J. Mielke Wilson\u00a0Y. Lee Abheesht Sharma Andrea Santilli Antoine Chaffin Arnaud Stiegler Debajyoti Datta Eliza Szczechla Gunjan Chhablani Han Wang Harshit Pandey Hendrik Strobelt Jason\u00a0Alan Fries Jos Rozen Leo Gao Lintang Sutawika M\u00a0Saiful Bari Maged\u00a0S. Al-shaibani Matteo Manica Nihal Nayak Ryan Teehan Samuel Albanie Sheng Shen Srulik Ben-David Stephen\u00a0H. Bach Taewoon Kim Tali Bers Thibault Fevry Trishala Neeraj Urmish Thakker Vikas Raunak Xiangru Tang Zheng-Xin Yong Zhiqing Sun Shaked Brody Yallow Uri Hadar Tojarieh Adam Roberts Hyung\u00a0Won Chung Jaesung Tae Jason Phang Ofir Press Conglong Li Deepak Narayanan Hatim Bourfoune Jared Casper Jeff Rasley Max Ryabinin Mayank Mishra Minjia Zhang Mohammad Shoeybi Myriam Peyrounette Nicolas Patry Nouamane Tazi Omar Sanseviero Patrick von Platen Pierre Cornette Pierre\u00a0Fran\u00e7ois Lavall\u00e9e R\u00e9mi Lacroix Samyam Rajbhandari Sanchit Gandhi Shaden Smith St\u00e9phane Requena Suraj Patil Tim Dettmers Ahmed Baruwa Amanpreet Singh Anastasia Cheveleva Anne-Laure Ligozat Arjun Subramonian Aur\u00e9lie N\u00e9v\u00e9ol Charles Lovering Dan Garrette Deepak Tunuguntla Ehud Reiter Ekaterina Taktasheva Ekaterina Voloshina Eli Bogdanov Genta\u00a0Indra Winata Hailey Schoelkopf Jan-Christoph Kalo Jekaterina Novikova Jessica\u00a0Zosa Forde Jordan Clive Jungo Kasai Ken Kawamura Liam Hazan Marine Carpuat Miruna Clinciu Najoung Kim Newton Cheng Oleg Serikov Omer Antverg Oskar van\u00a0der Wal Rui Zhang Ruochen Zhang Sebastian Gehrmann Shachar Mirkin Shani Pais Tatiana Shavrina Thomas Scialom Tian Yun Tomasz Limisiewicz Verena Rieser Vitaly Protasov Vladislav Mikhailov Yada Pruksachatkun Yonatan Belinkov Zachary Bamberger Zden\u011bk Kasner Alice Rueda Amanda Pestana Amir Feizpour Ammar Khan Amy Faranak Ana Santos Anthony Hevia Antigona Unldreaj Arash Aghagol Arezoo Abdollahi Aycha Tammour Azadeh HajiHosseini Bahareh Behroozi Benjamin Ajibade Bharat Saxena Carlos\u00a0Mu\u00f1oz Ferrandis Daniel McDuff Danish Contractor David Lansky Davis David Douwe Kiela Duong\u00a0A. Nguyen Edward Tan Emi Baylor Ezinwanne Ozoani Fatima Mirza Frankline Ononiwu Habib Rezanejad Hessie Jones Indrani Bhattacharya Irene Solaiman Irina Sedenko Isar Nejadgholi Jesse Passmore Josh Seltzer Julio\u00a0Bonis Sanz Livia Dutra Mairon Samagaio Maraim Elbadri Margot Mieskes Marissa Gerchick Martha Akinlolu Michael McKenna Mike Qiu Muhammed Ghauri Mykola Burynok Nafis Abrar Nazneen Rajani Nour Elkott Nour Fahmy Olanrewaju Samuel Ran An Rasmus Kromann Ryan Hao Samira Alizadeh Sarmad Shubber Silas Wang Sourav Roy Sylvain Viguier Thanh Le Tobi Oyebade Trieu Le Yoyo Yang Zach Nguyen Abhinav\u00a0Ramesh Kashyap Alfredo Palasciano Alison Callahan Anima Shukla Antonio Miranda-Escalada Ayush Singh Benjamin Beilharz Bo Wang Caio Brito Chenxi Zhou Chirag Jain Chuxin Xu Cl\u00e9mentine Fourrier Daniel\u00a0Le\u00f3n Peri\u00f1\u00e1n Daniel Molano Dian Yu Enrique Manjavacas Fabio Barth Florian Fuhrimann Gabriel Altay Giyaseddin Bayrak Gully Burns Helena\u00a0U. Vrabec Imane Bello Ishani Dash Jihyun Kang John Giorgi Jonas Golde Jose\u00a0David Posada Karthik\u00a0Rangasai Sivaraman Lokesh Bulchandani Lu Liu Luisa Shinzato Madeleine\u00a0Hahn de Bykhovetz Maiko Takeuchi Marc P\u00e0mies Maria\u00a0A Castillo Marianna Nezhurina Mario S\u00e4nger Matthias Samwald Michael Cullan Michael Weinberg Michiel\u00a0De Wolf Mina Mihaljcic Minna Liu Moritz Freidank Myungsun Kang Natasha Seelam Nathan Dahlberg Nicholas\u00a0Michio Broad Nikolaus Muellner Pascale Fung Patrick Haller Ramya Chandrasekhar Renata Eisenberg Robert Martin Rodrigo Canalli Rosaline Su Ruisi Su Samuel Cahyawijaya Samuele Garda Shlok\u00a0S Deshmukh Shubhanshu Mishra Sid Kiblawi Simon Ott Sinee Sang-aroonsiri Srishti Kumar Stefan Schweter Sushil Bharati Tanmay Laud Th\u00e9o Gigant Tomoya Kainuma Wojciech Kusa Yanis Labrak Yash\u00a0Shailesh Bajaj Yash Venkatraman Yifan Xu Yingxin Xu Yu Xu Zhe Tan Zhongli Xie Zifan Ye Mathilde Bras Younes Belkada and Thomas Wolf. 2023. BLOOM: A 176B-Parameter Open-Access Multilingual Language Model. arxiv:https:\/\/arXiv.org\/abs\/2211.05100\u00a0[cs.CL] https:\/\/arxiv.org\/abs\/2211.05100"},{"key":"e_1_3_3_2_71_2","first-page":"10495","volume-title":"International Conference on Machine Learning (ICML)","author":"Xie Cong","year":"2020","unstructured":"Cong Xie, Sanmi Koyejo, and Indranil Gupta. 2020. Zeno++: Robust Fully Asynchronous SGD. In International Conference on Machine Learning (ICML). 10495\u201310503."},{"key":"e_1_3_3_2_72_2","unstructured":"Zhichao Xu Ashim Gupta Tao Li Oliver Bentham and Vivek Srikumar. 2024. Beyond Perplexity: Multi-dimensional Safety Evaluation of LLM Compression. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2407.04965 (2024)."},{"key":"e_1_3_3_2_73_2","first-page":"559","volume-title":"USENIX Symposium on Operating Systems Design and Implementation (OSDI)","author":"Zheng Lianmin","year":"2022","unstructured":"Lianmin Zheng, Zhuohan Li, Hao Zhang, Yonghao Zhuang, Zhifeng Chen, Yanping Huang, Yida Wang, Yuanzhong Xu, Danyang Zhuo, Eric\u00a0P. Xing, Joseph\u00a0E. Gonzalez, and Ion Stoica. 2022. Alpa: Automating Inter- and Intra-Operator Parallelism for Distributed Deep Learning. In USENIX Symposium on Operating Systems Design and Implementation (OSDI). 559\u2013578."}],"event":{"name":"MICRO 2025: 58th IEEE\/ACM International Symposium on Microarchitecture","location":"Seoul Korea","acronym":"MICRO 2025","sponsor":["SIGMICRO ACM Special Interest Group on Microarchitectural Research and Processing"]},"container-title":["Proceedings of the 58th IEEE\/ACM International Symposium on Microarchitecture"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3725843.3756079","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,1,26]],"date-time":"2026-01-26T21:41:47Z","timestamp":1769463707000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3725843.3756079"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,10,17]]},"references-count":72,"alternative-id":["10.1145\/3725843.3756079","10.1145\/3725843"],"URL":"https:\/\/doi.org\/10.1145\/3725843.3756079","relation":{},"subject":[],"published":{"date-parts":[[2025,10,17]]},"assertion":[{"value":"2025-10-17","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}