{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,2]],"date-time":"2026-07-02T13:45:02Z","timestamp":1782999902252,"version":"3.54.5"},"publisher-location":"New York, NY, USA","reference-count":62,"publisher":"ACM","license":[{"start":{"date-parts":[[2026,7,5]],"date-time":"2026-07-05T00:00:00Z","timestamp":1783209600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/legalcode"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2026,7,6]]},"DOI":"10.1145\/3797905.3807839","type":"proceedings-article","created":{"date-parts":[[2026,7,2]],"date-time":"2026-07-02T11:50:37Z","timestamp":1782993037000},"page":"314-327","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":0,"title":["Closing the Efficiency Gap: AI Datacenter Co-design Roadmap for Scalable Training of LLMs"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-2861-8393","authenticated-orcid":false,"given":"Jesmin Jahan","family":"Tithi","sequence":"first","affiliation":[{"name":"Intel, Santa Clara, California, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0003-3718-5272","authenticated-orcid":false,"given":"Hanjiang","family":"Wu","sequence":"additional","affiliation":[{"name":"Georgia Institute of Technology, Atlanta, Georgia, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0000-8406-0817","authenticated-orcid":false,"given":"Joongun","family":"Park","sequence":"additional","affiliation":[{"name":"Georgia Institute of Technology, Atlanta, Georgia, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0000-4000-3132","authenticated-orcid":false,"given":"Avishaii","family":"Abuhatzera","sequence":"additional","affiliation":[{"name":"Ex-Intel, Haifa, Israel"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-4977-7107","authenticated-orcid":false,"given":"Fabrizio","family":"Petrini","sequence":"additional","affiliation":[{"name":"Intel, Santa Clara, California, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-5738-6942","authenticated-orcid":false,"given":"Tushar","family":"Krishna","sequence":"additional","affiliation":[{"name":"Georgia Institute of Technology, Atlanta, Georgia, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2026,7,5]]},"reference":[{"key":"e_1_3_3_2_2_2","unstructured":"AMD. 2024. AMD Instinct MI300X Platform Data Sheet. https:\/\/www.amd.com\/content\/dam\/amd\/en\/documents\/instinct-tech-docs\/data-sheets\/amd-instinct-mi300x-platform-data-sheet.pdf Accessed: 2025-02-25."},{"key":"e_1_3_3_2_3_2","unstructured":"astra sim. 2025. symbolic_tensor_graph. GitHub repository. https:\/\/github.com\/astra-sim\/symbolic_tensor_graph"},{"key":"e_1_3_3_2_4_2","unstructured":"Weilin Cai Juyong Jiang Fan Wang Jing Tang Sunghun Kim and Jiayi Huang. 2024. A survey on mixture of experts. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2407.06204 (2024)."},{"key":"e_1_3_3_2_5_2","unstructured":"Dr. Yu-Han Chang. 2024. Co-Packaged Optics (CPO) 2025-2035: Technologies Market and Forecasts. https:\/\/www.idtechex.com\/en\/research-report\/co-packaged-optics\/1019 Comprehensive analysis of CPO technology packaging trends and market forecasts."},{"key":"e_1_3_3_2_6_2","unstructured":"Dr. Yu-Han Chang. 2024. Co-Packaged Optics (CPO): Evaluating Different Packaging Technologies. IDTechEx Research Article (2024). https:\/\/www.idtechex.com\/en\/research-article\/co-packaged-optics-cpo-evaluating-different-packaging-technologies\/31608 In-depth insights into CPO packaging approaches and technology benchmarks."},{"key":"e_1_3_3_2_7_2","unstructured":"Dr. Yu-Han Chang. 2024. Co-Packaged Optics (CPO) Market Outlook & Packaging Technology Trends. IDTechEx Research Article (2024). https:\/\/www.idtechex.com\/en\/research-article\/co-packaged-optics-cpo-market-outlook-and-packaging-technology-trends\/31440 Market trajectory and packaging technology trends for CPO."},{"key":"e_1_3_3_2_8_2","unstructured":"Li-Wen Chang Wenlei Bao Qi Hou Chengquan Jiang Ningxin Zheng Yinmin Zhong Xuanrun Zhang Zuquan Song Chengji Yao Ziheng Jiang Haibin Lin Xin Jin and Xin Liu. 2024. FLUX: Fast Software-based Communication Overlap On GPUs Through Kernel Fusion. arxiv:https:\/\/arXiv.org\/abs\/2406.06858\u00a0[cs.LG] https:\/\/arxiv.org\/abs\/2406.06858"},{"key":"e_1_3_3_2_9_2","unstructured":"Fabio Checconi Jesmin\u00a0Jahan Tithi and Fabrizio Petrini. 2022. Ridgeline: A 2D Roofline Model for Distributed Systems. arxiv:https:\/\/arXiv.org\/abs\/2209.01368\u00a0[cs.DC] https:\/\/arxiv.org\/abs\/2209.01368"},{"key":"e_1_3_3_2_10_2","unstructured":"Cisco. 2024. Cisco Massively Scalable Data Center Network Fabric Design and Operation White Paper. https:\/\/www.cisco.com\/c\/en\/us\/products\/collateral\/switches\/nexus-9000-series-switches\/white-paper-c11-743245.html Accessed: 2025-02-14."},{"key":"e_1_3_3_2_11_2","unstructured":"John D\u2019Ambrosia. 2024. Ethernet\u2019s Next Bar is Now \u2013 800 Gb\/s! IEEE Standards Association: Beyond Standards Blog. https:\/\/standards.ieee.org\/beyond-standards\/ethernets-next-bar\/ Accessed: 2026-02-02."},{"key":"e_1_3_3_2_12_2","unstructured":"DeepSeek-AI : Xiao Bi Deli Chen Guanting Chen Shanhuang Chen Damai Dai Chengqi Deng Honghui Ding Kai Dong Qiushi Du Zhe Fu Huazuo Gao Kaige Gao Wenjun Gao Ruiqi Ge Kang Guan Daya Guo Jianzhong Guo Guangbo Hao Zhewen Hao Ying He Wenjie Hu Panpan Huang Erhang Li Guowei Li Jiashi Li Yao Li Y.\u00a0K. Li Wenfeng Liang Fangyun Lin A.\u00a0X. Liu Bo Liu Wen Liu Xiaodong Liu Xin Liu Yiyuan Liu Haoyu Lu Shanghao Lu Fuli Luo Shirong Ma Xiaotao Nie Tian Pei Yishi Piao Junjie Qiu Hui Qu Tongzheng Ren Zehui Ren Chong Ruan Zhangli Sha Zhihong Shao Junxiao Song Xuecheng Su Jingxiang Sun Yaofeng Sun Minghui Tang Bingxuan Wang Peiyi Wang Shiyu Wang Yaohui Wang Yongji Wang Tong Wu Y. Wu Xin Xie Zhenda Xie Ziwei Xie Yiliang Xiong Hanwei Xu R.\u00a0X. Xu Yanhong Xu Dejian Yang Yuxiang You Shuiping Yu Xingkai Yu B. Zhang Haowei Zhang Lecong Zhang Liyue Zhang Mingchuan Zhang Minghua Zhang Wentao Zhang Yichao Zhang Chenggang Zhao Yao Zhao Shangyan Zhou Shunfeng Zhou Qihao Zhu and Yuheng Zou. 2024. deepseekai2024deepseekllmscalingopensource. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2401.02954 (2024)."},{"key":"e_1_3_3_2_13_2","doi-asserted-by":"crossref","unstructured":"Jens Domke Satoshi Matsuoka Ivan\u00a0R Ivanov Yuki Tsushima Tomoya Yuki Akihiro Nomura Shinichi Miura Nie McDonald Dennis\u00a0L Floyd and Nicolas Dube. 2019. HyperX topology: First at-scale implementation and comparison to the fat-tree. (2019) 1\u201323.","DOI":"10.1145\/3295500.3356140"},{"key":"e_1_3_3_2_14_2","series-title":"Proceedings of Machine Learning Research","first-page":"5547","volume-title":"Proceedings of the 39th International Conference on Machine Learning (ICML)","volume":"162","author":"Du Nan","year":"2022","unstructured":"Nan Du, Yanping Huang, Andrew\u00a0M. Dai, Simon Qi, Zhifeng Chen, Yuanzhong Xu, Jiaming Song, Hanxiao Liu, Zhengdong Lu, Denny Zhou, Quoc\u00a0V. Le, and Yonghui Wu. 2022. GLaM: Efficient Scaling with Mixture-of-Experts. In Proceedings of the 39th International Conference on Machine Learning (ICML)(Proceedings of Machine Learning Research, Vol.\u00a0162). PMLR, 5547\u20135569. https:\/\/proceedings.mlr.press\/v162\/du22c.html"},{"key":"e_1_3_3_2_15_2","doi-asserted-by":"crossref","unstructured":"Jiangfei Duan Xiuhong Li Ping Xu Xingcheng Zhang Shengen Yan Yun Liang and Dahua Lin. 2024. Proteus: Simulating the performance of distributed DNN training. IEEE Transactions on Parallel and Distributed Systems (2024).","DOI":"10.1109\/TPDS.2024.3443255"},{"key":"e_1_3_3_2_16_2","unstructured":"William Fedus Barret Zoph and Noam Shazeer. 2022. Switch Transformers: Scaling to Trillion Parameter Models with Simple and Efficient Sparsity. Journal of Machine Learning Research (JMLR) 23 120 (2022) 1\u201339. https:\/\/jmlr.org\/papers\/v23\/21-0998.html"},{"key":"e_1_3_3_2_17_2","doi-asserted-by":"crossref","unstructured":"Luciano Floridi and Massimo Chiriatti. 2020. GPT-3: Its nature scope limits and consequences. Minds and Machines 30 (2020) 681\u2013694.","DOI":"10.1007\/s11023-020-09548-1"},{"key":"e_1_3_3_2_18_2","unstructured":"Seokjin Go Joongun Park Spandan More Hanjiang Wu Irene Wang Aaron Jezghani Tushar Krishna and Divya Mahajan. 2025. Characterizing the Efficiency of Distributed Training: A Power Performance and Thermal Perspective. arxiv:https:\/\/arXiv.org\/abs\/2509.10371\u00a0[cs.DC] https:\/\/arxiv.org\/abs\/2509.10371"},{"key":"e_1_3_3_2_19_2","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-50743-5_3"},{"key":"e_1_3_3_2_20_2","unstructured":"Aaron Grattafiori Abhimanyu Dubey Abhinav Jauhri Abhinav Pandey Abhishek Kadian Ahmad Al-Dahle Aiesha Letman Akhil Mathur Alan Schelten Alex Vaughan Amy Yang Angela Fan Anirudh Goyal Anthony Hartshorn Aobo Yang Archi Mitra Archie Sravankumar Artem Korenev Arthur Hinsvark Arun Rao Aston Zhang Aurelien Rodriguez Austen Gregerson Ava Spataru Baptiste Roziere Bethany Biron Binh Tang Bobbie Chern Charlotte Caucheteux Chaya Nayak Chloe Bi Chris Marra Chris McConnell Christian Keller Christophe Touret Chunyang Wu Corinne Wong Cristian\u00a0Canton Ferrer Cyrus Nikolaidis Damien Allonsius Daniel Song Danielle Pintz Danny Livshits Danny Wyatt David Esiobu Dhruv Choudhary Dhruv Mahajan Diego Garcia-Olano Diego Perino Dieuwke Hupkes Egor Lakomkin Ehab AlBadawy Elina Lobanova Emily Dinan Eric\u00a0Michael Smith Filip Radenovic Francisco Guzm\u00e1n Frank Zhang Gabriel Synnaeve Gabrielle Lee Georgia\u00a0Lewis Anderson Govind Thattai Graeme Nail Gregoire Mialon Guan Pang Guillem Cucurell Hailey Nguyen Hannah Korevaar Hu Xu Hugo Touvron Iliyan Zarov Imanol\u00a0Arrieta Ibarra Isabel Kloumann Ishan Misra Ivan Evtimov Jack Zhang Jade Copet Jaewon Lee Jan Geffert Jana Vranes Jason Park Jay Mahadeokar Jeet Shah Jelmer van\u00a0der Linde Jennifer Billock Jenny Hong Jenya Lee Jeremy Fu Jianfeng Chi Jianyu Huang Jiawen Liu Jie Wang Jiecao Yu Joanna Bitton Joe Spisak Jongsoo Park Joseph Rocca Joshua Johnstun Joshua Saxe Junteng Jia Kalyan\u00a0Vasuden Alwala Karthik Prasad Kartikeya Upasani Kate Plawiak Ke Li Kenneth Heafield Kevin Stone Khalid El-Arini Krithika Iyer Kshitiz Malik Kuenley Chiu Kunal Bhalla Kushal Lakhotia Lauren Rantala-Yeary Laurens van\u00a0der Maaten Lawrence Chen Liang Tan Liz Jenkins Louis Martin Lovish Madaan Lubo Malo Lukas Blecher Lukas Landzaat Luke de Oliveira Madeline Muzzi Mahesh Pasupuleti Mannat Singh Manohar Paluri Marcin Kardas Maria Tsimpoukelli Mathew Oldham Mathieu Rita Maya Pavlova Melanie Kambadur Mike Lewis Min Si Mitesh\u00a0Kumar Singh Mona Hassan Naman Goyal Narjes Torabi Nikolay Bashlykov Nikolay Bogoychev Niladri Chatterji Ning Zhang Olivier Duchenne Onur \u00c7elebi Patrick Alrassy Pengchuan Zhang Pengwei Li Petar Vasic Peter Weng Prajjwal Bhargava Pratik Dubal Praveen Krishnan Punit\u00a0Singh Koura Puxin Xu Qing He Qingxiao Dong Ragavan Srinivasan Raj Ganapathy Ramon Calderer Ricardo\u00a0Silveira Cabral Robert Stojnic Roberta Raileanu Rohan Maheswari Rohit Girdhar Rohit Patel Romain Sauvestre Ronnie Polidoro Roshan Sumbaly Ross Taylor Ruan Silva Rui Hou Rui Wang Saghar Hosseini Sahana Chennabasappa Sanjay Singh Sean Bell Seohyun\u00a0Sonia Kim Sergey Edunov Shaoliang Nie Sharan Narang Sharath Raparthy Sheng Shen Shengye Wan Shruti Bhosale Shun Zhang Simon Vandenhende Soumya Batra Spencer Whitman Sten Sootla Stephane Collot Suchin Gururangan Sydney Borodinsky Tamar Herman Tara Fowler Tarek Sheasha Thomas Georgiou Thomas Scialom Tobias Speckbacher Todor Mihaylov Tong Xiao Ujjwal Karn Vedanuj Goswami Vibhor Gupta Vignesh Ramanathan Viktor Kerkez Vincent Gonguet Virginie Do Vish Vogeti V\u00edtor Albiero Vladan Petrovic Weiwei Chu Wenhan Xiong Wenyin Fu Whitney Meers Xavier Martinet Xiaodong Wang Xiaofang Wang Xiaoqing\u00a0Ellen Tan Xide Xia Xinfeng Xie Xuchao Jia Xuewei Wang Yaelle Goldschlag Yashesh Gaur Yasmine Babaei Yi Wen Yiwen Song Yuchen Zhang Yue Li Yuning Mao Zacharie\u00a0Delpierre Coudert Zheng Yan Zhengxing Chen Zoe Papakipos Aaditya Singh Aayushi Srivastava Abha Jain Adam Kelsey Adam Shajnfeld Adithya Gangidi Adolfo Victoria Ahuva Goldstand Ajay Menon Ajay Sharma Alex Boesenberg Alexei Baevski Allie Feinstein Amanda Kallet Amit Sangani Amos Teo Anam Yunus Andrei Lupu Andres Alvarado Andrew Caples Andrew Gu Andrew Ho Andrew Poulton Andrew Ryan Ankit Ramchandani Annie Dong Annie Franco Anuj Goyal Aparajita Saraf Arkabandhu Chowdhury Ashley Gabriel Ashwin Bharambe Assaf Eisenman Azadeh Yazdan Beau James Ben Maurer Benjamin Leonhardi Bernie Huang Beth Loyd Beto\u00a0De Paola Bhargavi Paranjape Bing Liu Bo Wu Boyu Ni Braden Hancock Bram Wasti Brandon Spence Brani Stojkovic Brian Gamido Britt Montalvo Carl Parker Carly Burton Catalina Mejia Ce Liu Changhan Wang Changkyu Kim Chao Zhou Chester Hu Ching-Hsiang Chu Chris Cai Chris Tindal Christoph Feichtenhofer Cynthia Gao Damon Civin Dana Beaty Daniel Kreymer Daniel Li David Adkins David Xu Davide Testuggine Delia David Devi Parikh Diana Liskovich Didem Foss Dingkang Wang Duc Le Dustin Holland Edward Dowling Eissa Jamil Elaine Montgomery Eleonora Presani Emily Hahn Emily Wood Eric-Tuan Le Erik Brinkman Esteban Arcaute Evan Dunbar Evan Smothers Fei Sun Felix Kreuk Feng Tian Filippos Kokkinos Firat Ozgenel Francesco Caggioni Frank Kanayet Frank Seide Gabriela\u00a0Medina Florez Gabriella Schwarz Gada Badeer Georgia Swee Gil Halpern Grant Herman Grigory Sizov Guangyi Zhang Guna Lakshminarayanan Hakan Inan Hamid Shojanazeri Han Zou Hannah Wang Hanwen Zha Haroun Habeeb Harrison Rudolph Helen Suk Henry Aspegren Hunter Goldman Hongyuan Zhan Ibrahim Damlaj Igor Molybog Igor Tufanov Ilias Leontiadis Irina-Elena Veliche Itai Gat Jake Weissman James Geboski James Kohli Janice Lam Japhet Asher Jean-Baptiste Gaya Jeff Marcus Jeff Tang Jennifer Chan Jenny Zhen Jeremy Reizenstein Jeremy Teboul Jessica Zhong Jian Jin Jingyi Yang Joe Cummings Jon Carvill Jon Shepard Jonathan McPhie Jonathan Torres Josh Ginsburg Junjie Wang Kai Wu Kam\u00a0Hou U Karan Saxena Kartikay Khandelwal Katayoun Zand Kathy Matosich Kaushik Veeraraghavan Kelly Michelena Keqian Li Kiran Jagadeesh Kun Huang Kunal Chawla Kyle Huang Lailin Chen Lakshya Garg Lavender A Leandro Silva Lee Bell Lei Zhang Liangpeng Guo Licheng Yu Liron Moshkovich Luca Wehrstedt Madian Khabsa Manav Avalani Manish Bhatt Martynas Mankus Matan Hasson Matthew Lennie Matthias Reso Maxim Groshev Maxim Naumov Maya Lathi Meghan Keneally Miao Liu Michael\u00a0L. Seltzer Michal Valko Michelle Restrepo Mihir Patel Mik Vyatskov Mikayel Samvelyan Mike Clark Mike Macey Mike Wang Miquel\u00a0Jubert Hermoso Mo Metanat Mohammad Rastegari Munish Bansal Nandhini Santhanam Natascha Parks Natasha White Navyata Bawa Nayan Singhal Nick Egebo Nicolas Usunier Nikhil Mehta Nikolay\u00a0Pavlovich Laptev Ning Dong Norman Cheng Oleg Chernoguz Olivia Hart Omkar Salpekar Ozlem Kalinli Parkin Kent Parth Parekh Paul Saab Pavan Balaji Pedro Rittner Philip Bontrager Pierre Roux Piotr Dollar Polina Zvyagina Prashant Ratanchandani Pritish Yuvraj Qian Liang Rachad Alao Rachel Rodriguez Rafi Ayub Raghotham Murthy Raghu Nayani Rahul Mitra Rangaprabhu Parthasarathy Raymond Li Rebekkah Hogan Robin Battey Rocky Wang Russ Howes Ruty Rinott Sachin Mehta Sachin Siby Sai\u00a0Jayesh Bondu Samyak Datta Sara Chugh Sara Hunt Sargun Dhillon Sasha Sidorov Satadru Pan Saurabh Mahajan Saurabh Verma Seiji Yamamoto Sharadh Ramaswamy Shaun Lindsay Shaun Lindsay Sheng Feng Shenghao Lin Shengxin\u00a0Cindy Zha Shishir Patil Shiva Shankar Shuqiang Zhang Shuqiang Zhang Sinong Wang Sneha Agarwal Soji Sajuyigbe Soumith Chintala Stephanie Max Stephen Chen Steve Kehoe Steve Satterfield Sudarshan Govindaprasad Sumit Gupta Summer Deng Sungmin Cho Sunny Virk Suraj Subramanian Sy Choudhury Sydney Goldman Tal Remez Tamar Glaser Tamara Best Thilo Koehler Thomas Robinson Tianhe Li Tianjun Zhang Tim Matthews Timothy Chou Tzook Shaked Varun Vontimitta Victoria Ajayi Victoria Montanez Vijai Mohan Vinay\u00a0Satish Kumar Vishal Mangla Vlad Ionescu Vlad Poenaru Vlad\u00a0Tiberiu Mihailescu Vladimir Ivanov Wei Li Wenchen Wang Wenwen Jiang Wes Bouaziz Will Constable Xiaocheng Tang Xiaojian Wu Xiaolan Wang Xilun Wu Xinbo Gao Yaniv Kleinman Yanjun Chen Ye Hu Ye Jia Ye Qi Yenda Li Yilin Zhang Ying Zhang Yossi Adi Youngjin Nam Yu Wang Yu Zhao Yuchen Hao Yundi Qian Yunlu Li Yuzi He Zach Rait Zachary DeVito Zef Rosnbrick Zhaoduo Wen Zhenyu Yang Zhiwei Zhao and Zhiyu Ma. 2024. The Llama 3 Herd of Models. arxiv:https:\/\/arXiv.org\/abs\/2407.21783\u00a0[cs.AI] https:\/\/arxiv.org\/abs\/2407.21783"},{"key":"e_1_3_3_2_21_2","unstructured":"Dallin Grimm. 2024. UALink Consortium Poised to Compete with Nvidia\u2019s NVLink. Tom\u2019s Hardware (2024). https:\/\/www.tomshardware.com\/tech-industry\/ualink-consortium-officially-incorporates-nvlink-competitor-headed-by-amd-and-intel-opens-doors-to-contributor-members Accessed: 2025-02-27."},{"key":"e_1_3_3_2_22_2","doi-asserted-by":"publisher","DOI":"10.1109\/ISCA59077.2024.00064"},{"key":"e_1_3_3_2_23_2","volume-title":"Intel(R) Tofino(TM) Native Architecture \u2013 Public Version","year":"2020","unstructured":"Intel. 2020. Intel(R) Tofino(TM) Native Architecture \u2013 Public Version. Technical Report. Intel Corporation. https:\/\/raw.githubusercontent.com\/barefootnetworks\/Open-Tofino\/master\/PUBLIC_Tofino-Native-Arch.pdf"},{"key":"e_1_3_3_2_24_2","doi-asserted-by":"publisher","DOI":"10.1145\/3581784.3607102"},{"key":"e_1_3_3_2_25_2","unstructured":"Albert\u00a0Q. Jiang Alexandre Sablayrolles Antoine Roux Arthur Mensch Blanche Savary Chris Bamford Devendra\u00a0Singh Chaplot Diego de\u00a0las Casas Emma\u00a0Bou Hanna Florian Bressand Gianna Lengyel Guillaume Bour Guillaume Lample L\u00e9lio\u00a0Renard Lavaud Lucile Saulnier Marie-Anne Lachaux Pierre Stock Sandeep Subramanian Sophia Yang Szymon Antoniak Teven\u00a0Le Scao Th\u00e9ophile Gervet Thibaut Lavril Thomas Wang Timoth\u00e9e Lacroix and William\u00a0El Sayed. 2024. Mixtral of Experts. arxiv:https:\/\/arXiv.org\/abs\/2401.04088\u00a0[cs.LG] https:\/\/arxiv.org\/abs\/2401.04088"},{"key":"e_1_3_3_2_26_2","doi-asserted-by":"publisher","DOI":"10.1109\/HCS55958.2022.9895631"},{"key":"e_1_3_3_2_27_2","unstructured":"Vijay Korthikanti Jared Casper Sangkug Lym Lawrence McAfee Michael Andersch Mohammad Shoeybi and Bryan Catanzaro. 2022. Reducing Activation Recomputation in Large Transformer Models. arxiv:https:\/\/arXiv.org\/abs\/2205.05198\u00a0[cs.LG] https:\/\/arxiv.org\/abs\/2205.05198"},{"key":"e_1_3_3_2_28_2","unstructured":"Jakub Krajewski Jan Ludziejewski Kamil Adamczewski Maciej Pi\u00f3ro Micha\u0142 Krutul Szymon Antoniak Kamil Ciebiera Krystian Kr\u00f3l Tomasz Odrzyg\u00f3\u017ad\u017a Piotr Sankowski Marek Cygan and Sebastian Jaszczur. 2024. Scaling Laws for Fine-Grained Mixture of Experts. arxiv:https:\/\/arXiv.org\/abs\/2402.07871\u00a0[cs.LG] https:\/\/arxiv.org\/abs\/2402.07871"},{"key":"e_1_3_3_2_29_2","doi-asserted-by":"publisher","DOI":"10.1145\/3558481.3591073"},{"key":"e_1_3_3_2_30_2","doi-asserted-by":"crossref","unstructured":"Kartik Lakhotia Laura Monroe Kelly Isham Maciej Besta Nils Blach Torsten Hoefler and Fabrizio Petrini. 2024. PolarStar: Expanding the Horizon of Diameter-3 Networks. (2024) 345\u2013357.","DOI":"10.1145\/3626183.3659975"},{"key":"e_1_3_3_2_31_2","unstructured":"Nathan Lambert. 2024. OLMoE and the hidden simplicity in training better foundation models. Blog post on Interconnects.ai. https:\/\/www.interconnects.ai\/p\/olmoe-and-building-better-llms"},{"key":"e_1_3_3_2_32_2","unstructured":"Dmitry Lepikhin HyoukJoong Lee Yuanzhong Xu Dehao Chen Orhan Firat Yanping Huang Maxim Krikun Noam Shazeer and Zhifeng Chen. 2020. GShard: Scaling Giant Models with Conditional Computation and Automatic Sharding. arxiv:https:\/\/arXiv.org\/abs\/2006.16668\u00a0[cs.CL] https:\/\/arxiv.org\/abs\/2006.16668"},{"key":"e_1_3_3_2_33_2","first-page":"945","volume-title":"2023 USENIX Annual Technical Conference (USENIX ATC 23)","author":"Li Jiamin","year":"2023","unstructured":"Jiamin Li, Yimin Jiang, Yibo Zhu, Cong Wang, and Hong Xu. 2023. Accelerating distributed { MoE} training and inference with lina. In 2023 USENIX Annual Technical Conference (USENIX ATC 23). 945\u2013959."},{"key":"e_1_3_3_2_34_2","unstructured":"Lightmatter. 2025. Lightmatter Announces Passage L200 the Fastest Co-Packaged Optics for AI. https:\/\/lightmatter.co\/press-release\/lightmatter-announces-passage-l200-the-fastest-co-packaged-optics-for-ai\/. Accessed: 2025-04-02."},{"key":"e_1_3_3_2_35_2","unstructured":"Austin Lyons. 2025. DeepSeek Mixture of Experts & V2. Blog post on Chipstrat. https:\/\/www.chipstrat.com\/p\/deepseek-moe-and-v2 Paid article."},{"key":"e_1_3_3_2_36_2","unstructured":"Mistral AI. 2025. Models Overview. https:\/\/docs.mistral.ai\/getting-started\/models\/models_overview\/ Accessed: 2025-04-14."},{"key":"e_1_3_3_2_37_2","unstructured":"MOE. 2023. GPT-4 Architecture and Infrastructure. https:\/\/152334h.github.io\/blog\/knowing-enough-about-moe\/ Accessed: 2025-02-14."},{"key":"e_1_3_3_2_38_2","doi-asserted-by":"publisher","DOI":"10.1109\/ISPASS57527.2023.00037"},{"key":"e_1_3_3_2_39_2","unstructured":"Samuel\u00a0K. Moore. 2025. A Crucial Optical Technology Has Finally Arrived. IEEE Spectrum 62 3 (March 2025). https:\/\/spectrum.ieee.org\/co-packaged-optics Accessed: 2026-02-02."},{"key":"e_1_3_3_2_40_2","unstructured":"NVIDIA. 2024. NeMo: Neural Modules Toolkit. https:\/\/github.com\/NVIDIA\/NeMo. Accessed: 2024-12-19."},{"key":"e_1_3_3_2_41_2","unstructured":"NVIDIA. 2025. NeMo: Neural Modules Framework. https:\/\/github.com\/NVIDIA\/NeMo. Accessed: 2025-02-27."},{"key":"e_1_3_3_2_42_2","volume-title":"Point-to-point communication \u2014 NCCL (All-to-All section)","year":"2025","unstructured":"NVIDIA. 2025. Point-to-point communication \u2014 NCCL (All-to-All section). NVIDIA Corporation. https:\/\/docs.nvidia.com\/deeplearning\/nccl\/user-guide\/docs\/usage\/p2p.html#all-to-all Accessed: 2025-10-06."},{"key":"e_1_3_3_2_43_2","volume-title":"NVIDIA SHARP Collective Library (revision 3.0.0 ed.)","author":"Corporation NVIDIA","year":"2023","unstructured":"NVIDIA Corporation. 2023. NVIDIA SHARP Collective Library (revision 3.0.0 ed.). NVIDIA. https:\/\/docs.nvidia.com\/networking\/display\/sharpv300 Accessed: 2026-02-02."},{"key":"e_1_3_3_2_44_2","unstructured":"OpenAI Josh Achiam Steven Adler Sandhini Agarwal Lama Ahmad Ilge Akkaya Florencia\u00a0Leoni Aleman Diogo Almeida Janko Altenschmidt Sam Altman Shyamal Anadkat Red Avila Igor Babuschkin Suchir Balaji Valerie Balcom Paul Baltescu Haiming Bao Mohammad Bavarian Jeff Belgum Irwan Bello Jake Berdine Gabriel Bernadett-Shapiro Christopher Berner Lenny Bogdonoff Oleg Boiko Madelaine Boyd Anna-Luisa Brakman Greg Brockman Tim Brooks Miles Brundage Kevin Button Trevor Cai Rosie Campbell Andrew Cann Brittany Carey Chelsea Carlson Rory Carmichael Brooke Chan Che Chang Fotis Chantzis Derek Chen Sully Chen Ruby Chen Jason Chen Mark Chen Ben Chess Chester Cho Casey Chu Hyung\u00a0Won Chung Dave Cummings Jeremiah Currier Yunxing Dai Cory Decareaux Thomas Degry Noah Deutsch Damien Deville Arka Dhar David Dohan Steve Dowling Sheila Dunning Adrien Ecoffet Atty Eleti Tyna Eloundou David Farhi Liam Fedus Niko Felix Sim\u00f3n\u00a0Posada Fishman Juston Forte Isabella Fulford Leo Gao Elie Georges Christian Gibson Vik Goel Tarun Gogineni Gabriel Goh Rapha Gontijo-Lopes Jonathan Gordon Morgan Grafstein Scott Gray Ryan Greene Joshua Gross Shixiang\u00a0Shane Gu Yufei Guo Chris Hallacy Jesse Han Jeff Harris Yuchen He Mike Heaton Johannes Heidecke Chris Hesse Alan Hickey Wade Hickey Peter Hoeschele Brandon Houghton Kenny Hsu Shengli Hu Xin Hu Joost Huizinga Shantanu Jain Shawn Jain Joanne Jang Angela Jiang Roger Jiang Haozhun Jin Denny Jin Shino Jomoto Billie Jonn Heewoo Jun Tomer Kaftan \u0141ukasz Kaiser Ali Kamali Ingmar Kanitscheider Nitish\u00a0Shirish Keskar Tabarak Khan Logan Kilpatrick Jong\u00a0Wook Kim Christina Kim Yongjik Kim Jan\u00a0Hendrik Kirchner Jamie Kiros Matt Knight Daniel Kokotajlo \u0141ukasz Kondraciuk Andrew Kondrich Aris Konstantinidis Kyle Kosic Gretchen Krueger Vishal Kuo Michael Lampe Ikai Lan Teddy Lee Jan Leike Jade Leung Daniel Levy Chak\u00a0Ming Li Rachel Lim Molly Lin Stephanie Lin Mateusz Litwin Theresa Lopez Ryan Lowe Patricia Lue Anna Makanju Kim Malfacini Sam Manning Todor Markov Yaniv Markovski Bianca Martin Katie Mayer Andrew Mayne Bob McGrew Scott\u00a0Mayer McKinney Christine McLeavey Paul McMillan Jake McNeil David Medina Aalok Mehta Jacob Menick Luke Metz Andrey Mishchenko Pamela Mishkin Vinnie Monaco Evan Morikawa Daniel Mossing Tong Mu Mira Murati Oleg Murk David M\u00e9ly Ashvin Nair Reiichiro Nakano Rajeev Nayak Arvind Neelakantan Richard Ngo Hyeonwoo Noh Long Ouyang Cullen O\u2019Keefe Jakub Pachocki Alex Paino Joe Palermo Ashley Pantuliano Giambattista Parascandolo Joel Parish Emy Parparita Alex Passos Mikhail Pavlov Andrew Peng Adam Perelman Filipe de Avila Belbute\u00a0Peres Michael Petrov Henrique\u00a0Ponde de Oliveira\u00a0Pinto Michael Pokorny Michelle Pokrass Vitchyr\u00a0H. Pong Tolly Powell Alethea Power Boris Power Elizabeth Proehl Raul Puri Alec Radford Jack Rae Aditya Ramesh Cameron Raymond Francis Real Kendra Rimbach Carl Ross Bob Rotsted Henri Roussez Nick Ryder Mario Saltarelli Ted Sanders Shibani Santurkar Girish Sastry Heather Schmidt David Schnurr John Schulman Daniel Selsam Kyla Sheppard Toki Sherbakov Jessica Shieh Sarah Shoker Pranav Shyam Szymon Sidor Eric Sigler Maddie Simens Jordan Sitkin Katarina Slama Ian Sohl Benjamin Sokolowsky Yang Song Natalie Staudacher Felipe\u00a0Petroski Such Natalie Summers Ilya Sutskever Jie Tang Nikolas Tezak Madeleine\u00a0B. Thompson Phil Tillet Amin Tootoonchian Elizabeth Tseng Preston Tuggle Nick Turley Jerry Tworek Juan Felipe\u00a0Cer\u00f3n Uribe Andrea Vallone Arun Vijayvergiya Chelsea Voss Carroll Wainwright Justin\u00a0Jay Wang Alvin Wang Ben Wang Jonathan Ward Jason Wei CJ Weinmann Akila Welihinda Peter Welinder Jiayi Weng Lilian Weng Matt Wiethoff Dave Willner Clemens Winter Samuel Wolrich Hannah Wong Lauren Workman Sherwin Wu Jeff Wu Michael Wu Kai Xiao Tao Xu Sarah Yoo Kevin Yu Qiming Yuan Wojciech Zaremba Rowan Zellers Chong Zhang Marvin Zhang Shengjia Zhao Tianhao Zheng Juntang Zhuang William Zhuk and Barret Zoph. 2024. GPT-4 Technical Report. arxiv:https:\/\/arXiv.org\/abs\/2303.08774\u00a0[cs.CL] https:\/\/arxiv.org\/abs\/2303.08774"},{"key":"e_1_3_3_2_45_2","unstructured":"deepspeedai\u00a0(GitHub organization). 2025. Megatron-DeepSpeed: Ongoing research training transformer language models at scale. GitHub repository. Retrieved 2025-09-10 from https:\/\/github.com\/deepspeedai\/Megatron-DeepSpeed Fork of NVIDIA\u2019s Megatron-LM supporting DeepSpeed features such as MoE 3D parallelism curriculum learning etc.."},{"key":"e_1_3_3_2_46_2","unstructured":"Dylan Patel Daniel Nishball Myron Xie Ivan Chiam Clara Ee and Patrick Zhou. 2026. Co-Packaged Optics (CPO) Book \u2013 Scaling with Light for the Next Wave of Interconnect. SemiAnalysis. https:\/\/newsletter.semianalysis.com\/p\/co-packaged-optics-cpo-book-scaling Accessed: 2026-01-16."},{"key":"e_1_3_3_2_47_2","unstructured":"Dylan Patel Myron Xie Daniel Nishball Ivan Chiam Patrick Zhou Doug O\u2019Laughlin and Wega Chu. 2025. NVIDIA GTC 2025 \u2013 Built For Reasoning Vera Rubin Kyber CPO Dynamo Inference Jensen Math Feynman. SemiAnalysis (19 March 2025). https:\/\/newsletter.semianalysis.com\/p\/nvidia-gtc-2025-built-for-reasoning-vera-rubin-kyber-cpo-dynamo-inference-jensen-math-feynman"},{"key":"e_1_3_3_2_48_2","unstructured":"Kishore Punniyamurthy Khaled Hamidouche and Bradford\u00a0M. Beckmann. 2024. Optimizing Distributed ML Communication with Fused Computation-Collective Operations. arxiv:https:\/\/arXiv.org\/abs\/2305.06942\u00a0[cs.DC] https:\/\/arxiv.org\/abs\/2305.06942"},{"key":"e_1_3_3_2_49_2","first-page":"18332","volume-title":"International conference on machine learning","author":"Rajbhandari Samyam","year":"2022","unstructured":"Samyam Rajbhandari, Conglong Li, Zhewei Yao, Minjia Zhang, Reza\u00a0Yazdani Aminabadi, Ammar\u00a0Ahmad Awan, Jeff Rasley, and Yuxiong He. 2022. Deepspeed-moe: Advancing mixture-of-experts inference and training to power next-generation ai scale. In International conference on machine learning. PMLR, 18332\u201318346."},{"key":"e_1_3_3_2_50_2","unstructured":"Samyam Rajbhandari Jeff Rasley Olatunji Ruwase and Yuxiong He. 2020. ZeRO: Memory Optimizations Toward Training Trillion Parameter Models. arxiv:https:\/\/arXiv.org\/abs\/1910.02054\u00a0[cs.LG] https:\/\/arxiv.org\/abs\/1910.02054"},{"key":"e_1_3_3_2_51_2","unstructured":"Jie Ren Samyam Rajbhandari Reza\u00a0Yazdani Aminabadi Olatunji Ruwase Shuangyan Yang Minjia Zhang Dong Li and Yuxiong He. 2021. ZeRO-Offload: Democratizing Billion-Scale Model Training. arxiv:https:\/\/arXiv.org\/abs\/2101.06840\u00a0[cs.DC] https:\/\/arxiv.org\/abs\/2101.06840"},{"key":"e_1_3_3_2_52_2","unstructured":"SemiAnalysis. 2023. GPT-4 Architecture and Infrastructure. https:\/\/semianalysis.com\/2023\/07\/10\/gpt-4-architecture-infrastructure\/ Accessed: 2025-02-14."},{"key":"e_1_3_3_2_53_2","unstructured":"Nicole Sheynin. 2024. 9 Data Center Trends and Outlook for 2025. https:\/\/www.alpha-sense.com\/blog\/trends\/data-center-trends\/ Accessed: 2025-02-14."},{"key":"e_1_3_3_2_54_2","unstructured":"Mohammad Shoeybi Mostofa Patwary Raul Puri Patrick LeGresley Jared Casper and Bryan Catanzaro. 2019. Megatron-lm: Training multi-billion parameter language models using model parallelism. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/1909.08053 (2019)."},{"key":"e_1_3_3_2_55_2","volume-title":"Proceedings of the Ninth Annual Conference on Machine Learning and Systems (MLSys 2026), Industry Track","author":"Sridharan Srinivas","year":"2026","unstructured":"Srinivas Sridharan, Andy Balogh, Bradford\u00a0M. Beckmann, Brian Coutinho, Louis Feng, Sheng Fu, Sanshan Gao, Mehryar Garakani, Taekyung Heo, David Kanter, Josh Ladd, Ziwei Li, Winston Liu, Changhai Man, Dan Mihailescu, Spandan More, Joongun Park, Ashwin Ramachandran, Vinay Ramakrishnaiah, Saeed Rashidi, Vijay\u00a0Janapa Reddi, Puneet Sharma, Phio Tian, William Won, Hanjiang Wu, Huan Xu, Jinsun Yoo, and Tushar Krishna. 2026. MLCommons Chakra: Advancing Performance Benchmarking and Co-design using Standardized Execution Traces. In Proceedings of the Ninth Annual Conference on Machine Learning and Systems (MLSys 2026), Industry Track. Bellevue, WA, USA."},{"key":"e_1_3_3_2_56_2","unstructured":"Srinivas Sridharan Taekyung Heo Louis Feng Zhaodong Wang Matt Bergeron Wenyin Fu Shengbao Zheng Brian Coutinho Saeed Rashidi Changhai Man and Tushar Krishna. 2023. Chakra: Advancing Performance Benchmarking and Co-design using Standardized Execution Traces. arxiv:https:\/\/arXiv.org\/abs\/2305.14516\u00a0[cs.LG] https:\/\/arxiv.org\/abs\/2305.14516"},{"key":"e_1_3_3_2_57_2","unstructured":"Cerebras Systems. 2025. DARPA Selects Cerebras to Deliver Next Generation Real-Time Compute Platform for Advanced Military and Commercial Applications. https:\/\/www.businesswire.com\/news\/home\/20250401398990\/en\/DARPA-Selects-Cerebras-to-Deliver-Next-Generation-Real-Time-Compute-Platform-for-Advanced-Military-and-Commercial-Applications. Accessed: 2025-04-03."},{"key":"e_1_3_3_2_58_2","unstructured":"Nouamane Tazi Ferdinand Mom Haojun Zhao Phuc Nguyen Mohamed Mekkouri Leandro Werra and Thomas Wolf. 2025. The Ultra-Scale Playbook: Training LLMs on GPU Clusters. Hugging Face. https:\/\/huggingface.co\/spaces\/nanotron\/ultrascale-playbook Accessed: 2026-02-02."},{"key":"e_1_3_3_2_59_2","unstructured":"Ultra Ethernet Consortium. 2025. Ultra Ethernet Consortium. https:\/\/ultraethernet.org\/. Accessed: 2025-02-27."},{"key":"e_1_3_3_2_60_2","unstructured":"James Wang. 2024. Cerebras CS-3: The World\u2019s Fastest and Most Scalable AI Accelerator. Cerebras Systems Blog. https:\/\/www.cerebras.ai\/blog\/cerebras-cs3 Accessed: 2026-01-21."},{"key":"e_1_3_3_2_61_2","doi-asserted-by":"publisher","DOI":"10.1109\/ISPASS57527.2023.00035"},{"key":"e_1_3_3_2_62_2","doi-asserted-by":"publisher","unstructured":"Yawei Yin Roberto Proietti Xiaohui Ye Christopher\u00a0J. Nitta Venkatesh Akella and S.\u00a0J.\u00a0Ben Yoo. 2013. LIONS: An AWGR-Based Low-Latency Optical Switch for High-Performance Computing and Data Centers. IEEE Journal of Selected Topics in Quantum Electronics 19 2 (2013) 3600409\u20133600409. 10.1109\/JSTQE.2012.2209174","DOI":"10.1109\/JSTQE.2012.2209174"},{"key":"e_1_3_3_2_63_2","unstructured":"Yanli Zhao Andrew Gu Rohan Varma Liang Luo Chien-Chin Huang Min Xu Less Wright Hamid Shojanazeri Myle Ott Sam Shleifer Alban Desmaison Can Balioglu Pritam Damania Bernard Nguyen Geeta Chauhan Yuchen Hao Ajit Mathews and Shen Li. 2023. PyTorch FSDP: Experiences on Scaling Fully Sharded Data Parallel. arxiv:https:\/\/arXiv.org\/abs\/2304.11277\u00a0[cs.DC] https:\/\/arxiv.org\/abs\/2304.11277"}],"event":{"name":"ICS '26: 2026 International Conference on Supercomputing","location":"Belfast United Kingdom","acronym":"ICS '26","sponsor":["SIGHPC ACM Special Interest Group on High Performance Computing, Special Interest Group on High Performance Computing","SIGARCH ACM Special Interest Group on Computer Architecture"]},"container-title":["Proceedings of the 40th ACM International Conference on Supercomputing"],"original-title":[],"deposited":{"date-parts":[[2026,7,2]],"date-time":"2026-07-02T13:09:16Z","timestamp":1782997756000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3797905.3807839"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026,7,5]]},"references-count":62,"alternative-id":["10.1145\/3797905.3807839","10.1145\/3797905"],"URL":"https:\/\/doi.org\/10.1145\/3797905.3807839","relation":{},"subject":[],"published":{"date-parts":[[2026,7,5]]},"assertion":[{"value":"2026-07-05","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}