{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,27]],"date-time":"2026-10-27T19:21:41Z","timestamp":1793128901169,"version":"4.5.0"},"publisher-location":"New York, NY, USA","reference-count":35,"publisher":"ACM","license":[{"start":{"date-parts":[[2024,6,17]],"date-time":"2024-06-17T00:00:00Z","timestamp":1718582400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2024,6,17]]},"DOI":"10.1145\/3662158.3662806","type":"proceedings-article","created":{"date-parts":[[2024,6,5]],"date-time":"2024-06-05T10:38:06Z","timestamp":1717583886000},"page":"121-130","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":32,"title":["System Optimizations for Enabling Training of Extreme Long Sequence Transformer Models"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-3425-5602","authenticated-orcid":false,"given":"Sam Ade","family":"Jacobs","sequence":"first","affiliation":[{"name":"Microsoft Inc, Redmond, United States of America"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0001-3972-8133","authenticated-orcid":false,"given":"Masahiro","family":"Tanaka","sequence":"additional","affiliation":[{"name":"Microsoft Inc, Redmond, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-3008-9133","authenticated-orcid":false,"given":"Chengming","family":"Zhang","sequence":"additional","affiliation":[{"name":"Microsoft Inc, Redmond, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-8165-166X","authenticated-orcid":false,"given":"Minjia","family":"Zhang","sequence":"additional","affiliation":[{"name":"Microsoft Inc, Redmond, United States of America"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-7949-6453","authenticated-orcid":false,"given":"Reza Yazdani","family":"Aminadabi","sequence":"additional","affiliation":[{"name":"Microsoft Inc, Redmond, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-8402-1436","authenticated-orcid":false,"given":"Shuaiwen Leon","family":"Song","sequence":"additional","affiliation":[{"name":"Microsoft Inc, Redmond, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-0386-8759","authenticated-orcid":false,"given":"Samyam","family":"Rajbhandari","sequence":"additional","affiliation":[{"name":"Microsoft Inc, Redmond, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-0478-8854","authenticated-orcid":false,"given":"Yuxiong","family":"He","sequence":"additional","affiliation":[{"name":"Microsoft Inc, Redmond, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2024,6,17]]},"reference":[{"key":"e_1_3_2_1_1_1","volume-title":"Longformer: The Long-Document Transformer. arXiv:2004.05150 [cs.CL]","author":"Beltagy Iz","year":"2020","unstructured":"Iz Beltagy, Matthew E. Peters, and Arman Cohan. 2020. Longformer: The Long-Document Transformer. arXiv:2004.05150 [cs.CL]"},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.1145\/3320060"},{"key":"e_1_3_2_1_3_1","volume-title":"Generating Long Sequences with Sparse Transformers. CoRR abs\/1904.10509","author":"Child Rewon","year":"2019","unstructured":"Rewon Child, Scott Gray, Alec Radford, and Ilya Sutskever. 2019. Generating Long Sequences with Sparse Transformers. CoRR abs\/1904.10509 (2019)."},{"key":"e_1_3_2_1_4_1","volume-title":"Rethinking Attention with Performers. CoRR abs\/2009.14794","author":"Choromanski Krzysztof","year":"2020","unstructured":"Krzysztof Choromanski, Valerii Likhosherstov, David Dohan, Xingyou Song, Andreea Gane, Tam\u00e1s Sarl\u00f3s, Peter Hawkins, Jared Davis, Afroz Mohiuddin, Lukasz Kaiser, David Belanger, Lucy J. Colwell, and Adrian Weller. 2020. Rethinking Attention with Performers. CoRR abs\/2009.14794 (2020). arXiv:2009.14794 https:\/\/arxiv.org\/abs\/2009.14794"},{"key":"e_1_3_2_1_5_1","unstructured":"Tri Dao. 2023. FlashAttention-2: Faster Attention with Better Parallelism and Work Partitioning. arXiv:2307.08691 [cs.LG]"},{"key":"e_1_3_2_1_6_1","unstructured":"Tri Dao Daniel Y. Fu Stefano Ermon Atri Rudra and Christopher R\u00e9. 2022. FlashAttention: Fast and Memory-Efficient Exact Attention with IO-Awareness. arXiv:2205.14135 [cs.LG]"},{"key":"e_1_3_2_1_7_1","unstructured":"Jeffrey Dean Greg Corrado Rajat Monga Kai Chen Matthieu Devin Mark Mao Marc'aurelio Ranzato Andrew Senior Paul Tucker Ke Yang et al. 2012. Large scale distributed deep networks. Advances in neural information processing systems 25 (2012)."},{"key":"e_1_3_2_1_8_1","volume-title":"9th International Conference on Learning Representations, ICLR 2021","author":"Dosovitskiy Alexey","year":"2021","unstructured":"Alexey Dosovitskiy, Lucas Beyer, Alexander Kolesnikov, Dirk Weissenborn, Xiaohua Zhai, Thomas Unterthiner, Mostafa Dehghani, Matthias Minderer, Georg Heigold, Sylvain Gelly, Jakob Uszkoreit, and Neil Houlsby. 2021. An Image is Worth 16\u00d716 Words: Transformers for Image Recognition at Scale. In 9th International Conference on Learning Representations, ICLR 2021, Virtual Event, Austria, May 3-7, 2021. OpenReview.net."},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1109\/JBHI.2021.3062322"},{"key":"e_1_3_2_1_10_1","volume-title":"GPipe: Efficient Training of Giant Neural Networks using Pipeline Parallelism. ArXiv abs\/1811.06965","author":"Huang Yanping","year":"2018","unstructured":"Yanping Huang, Yonglong Cheng, Dehao Chen, HyoukJoong Lee, Jiquan Ngiam, Quoc V. Le, and Zhifeng Chen. 2018. GPipe: Efficient Training of Giant Neural Networks using Pipeline Parallelism. ArXiv abs\/1811.06965 (2018)."},{"key":"e_1_3_2_1_11_1","volume-title":"On large-batch training for deep learning: Generalization gap and sharp minima. arXiv preprint arXiv:1609.04836","author":"Keskar Nitish Shirish","year":"2016","unstructured":"Nitish Shirish Keskar, Dheevatsa Mudigere, Jorge Nocedal, Mikhail Smelyanskiy, and Ping Tak Peter Tang. 2016. On large-batch training for deep learning: Generalization gap and sharp minima. arXiv preprint arXiv:1609.04836 (2016)."},{"key":"e_1_3_2_1_12_1","unstructured":"Vijay Korthikanti Jared Casper Sangkug Lym Lawrence McAfee Michael Andersch Mohammad Shoeybi and Bryan Catanzaro. 2022. Reducing Activation Recomputation in Large Transformer Models. arXiv:2205.05198 [cs.LG]"},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"crossref","unstructured":"Wojciech Kry\u015bci\u0144ski Nazneen Rajani Divyansh Agarwal Caiming Xiong and Dragomir Radev. 2022. BookSum: A Collection of Datasets for Long-form Narrative Summarization. arXiv:2105.08209 [cs.CL]","DOI":"10.18653\/v1\/2022.findings-emnlp.488"},{"key":"e_1_3_2_1_14_1","volume-title":"Sequence Parallelism: Long Sequence Training from System Perspective. arXiv:2105.13120 [cs.LG]","author":"Li Shenggui","year":"2022","unstructured":"Shenggui Li, Fuzhao Xue, Chaitanya Baranwal, Yongbin Li, and Yang You. 2022. Sequence Parallelism: Long Sequence Training from System Perspective. arXiv:2105.13120 [cs.LG]"},{"key":"e_1_3_2_1_15_1","volume-title":"Clinical-Longformer and Clinical-BigBird: Transformers for long clinical sequences. CoRR abs\/2201.11838","author":"Li Yikuan","year":"2022","unstructured":"Yikuan Li, Ramsey M. Wehbe, Faraz S. Ahmad, Hanyin Wang, and Yuan Luo. 2022. Clinical-Longformer and Clinical-BigBird: Transformers for long clinical sequences. CoRR abs\/2201.11838 (2022). arXiv:2201.11838 https:\/\/arxiv.org\/abs\/2201.11838"},{"key":"e_1_3_2_1_16_1","unstructured":"MosaicML. 2023. Introducing MPT-7B: A New Standard for Open-Source Commercially Usable LLMs. https:\/\/https:\/\/www.mosaicml.com\/blog\/mpt-7b."},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.1145\/3341301.3359646"},{"key":"e_1_3_2_1_18_1","volume-title":"International Conference on Machine Learning. PMLR, 7937--7947","author":"Narayanan Deepak","year":"2021","unstructured":"Deepak Narayanan, Amar Phanishayee, Kaiyu Shi, Xie Chen, and Matei Zaharia. 2021. Memory-efficient pipeline-parallel dnn training. In International Conference on Machine Learning. PMLR, 7937--7947."},{"key":"e_1_3_2_1_19_1","unstructured":"Tung Nguyen Johannes Brandstetter Ashish Kapoor Jayesh K. Gupta and Aditya Grover. 2023. ClimaX: A foundation model for weather and climate. arXiv:2301.10343 [cs.LG]"},{"key":"e_1_3_2_1_21_1","unstructured":"Bowen Peng Jeffrey Quesnelle Honglu Fan and Enrico Shippole. 2023. YaRN: Efficient Context Window Extension of Large Language Models. arXiv:2309.00071 [cs.CL]"},{"key":"e_1_3_2_1_22_1","unstructured":"PyTorch. 2023. Lightning AI. https:\/\/lightning.ai\/"},{"key":"e_1_3_2_1_23_1","volume-title":"Rabe and Charles Staats","author":"Markus","year":"2021","unstructured":"Markus N. Rabe and Charles Staats. 2021. Self-attention Does Not Need O(n2) Memory. CoRR abs\/2112.05682 (2021). arXiv:2112.05682 https:\/\/arxiv.org\/abs\/2112.05682"},{"key":"e_1_3_2_1_24_1","unstructured":"Alec Radford Jeff Wu Rewon Child David Luan Dario Amodei and Ilya Sutskever. 2019. Language Models are Unsupervised Multitask Learners. (2019)."},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.1109\/SC41405.2020.00024"},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.1145\/3458817.3476205"},{"key":"e_1_3_2_1_27_1","unstructured":"Mohammad Shoeybi Mostofa Patwary Raul Puri Patrick LeGresley Jared Casper and Bryan Catanzaro. 2019. Megatron-LM: Training Multi-Billion Parameter Language Models Using Model Parallelism. arXiv:1909.08053 [cs.CL]"},{"key":"e_1_3_2_1_28_1","unstructured":"Shaden Smith Mostofa Patwary Brandon Norick Patrick LeGresley Samyam Rajbhandari Jared Casper Zhun Liu Shrimai Prabhumoye George Zerveas Vijay Korthikanti et al. 2022. Using DeepSpeed and Megatron to Train Megatron-Turing NLG 530B A Large-Scale Generative Language Model. arXiv preprint arXiv:2201.11990 (2022)."},{"key":"e_1_3_2_1_29_1","unstructured":"Yi Tay Mostafa Dehghani Samira Abnar Yikang Shen Dara Bahri Philip Pham Jinfeng Rao Liu Yang Sebastian Ruder and Donald Metzler. 2020. Long Range Arena: A Benchmark for Efficient Transformers. arXiv:2011.04006 [cs.LG]"},{"key":"e_1_3_2_1_30_1","unstructured":"DeepSpeed Team and Rangan Majumder. 2020. DeepSpeed: Extreme-scale model training for everyone. https:\/\/www.microsoft.com\/en-us\/research\/blog\/deepspeed-extreme-scale-model-training-for-everyone\/."},{"key":"e_1_3_2_1_31_1","unstructured":"Hugo Touvron Louis Martin Kevin Stone Peter Albert Amjad Almahairi Yasmine Babaei Nikolay Bashlykov Soumya Batra Prajjwal Bhargava Shruti Bhosale Dan Bikel Lukas Blecher Cristian Canton Ferrer Moya Chen Guillem Cucurull David Esiobu Jude Fernandes Jeremy Fu Wenyin Fu Brian Fuller Cynthia Gao Vedanuj Goswami Naman Goyal Anthony Hartshorn Saghar Hosseini Rui Hou Hakan Inan Marcin Kardas Viktor Kerkez Madian Khabsa Isabel Kloumann Artem Korenev Punit Singh Koura Marie-Anne Lachaux Thibaut Lavril Jenya Lee Diana Liskovich Yinghai Lu Yuning Mao Xavier Martinet Todor Mihaylov Pushkar Mishra Igor Molybog Yixin Nie Andrew Poulton Jeremy Reizenstein Rashi Rungta Kalyan Saladi Alan Schelten Ruan Silva Eric Michael Smith Ranjan Subramanian Xiaoqing Ellen Tan Binh Tang Ross Taylor Adina Williams Jian Xiang Kuan Puxin Xu Zheng Yan Iliyan Zarov Yuchen Zhang Angela Fan Melanie Kambadur Sharan Narang Aurelien Rodriguez Robert Stojnic Sergey Edunov and Thomas Scialom. 2023. Llama 2: Open Foundation and Fine-Tuned Chat Models. arXiv:2307.09288 [cs.CL]"},{"key":"e_1_3_2_1_32_1","unstructured":"Ashish Vaswani Noam Shazeer Niki Parmar Jakob Uszkoreit Llion Jones Aidan N Gomez \u0141ukasz Kaiser and Illia Polosukhin. 2017. Attention is all you need. In Advances in neural information processing systems. 5998--6008."},{"key":"e_1_3_2_1_33_1","volume-title":"Connor Holmes, Samyam Rajbhandari, Olatunji Ruwase, Feng Yan, Lei Yang, and Yuxiong He.","author":"Wang Guanhua","year":"2023","unstructured":"Guanhua Wang, Heyang Qin, Sam Ade Jacobs, Connor Holmes, Samyam Rajbhandari, Olatunji Ruwase, Feng Yan, Lei Yang, and Yuxiong He. 2023. ZeRO++: Extremely Efficient Collective Communication for Giant Model Training. arXiv:2306.10209 [cs.DC]"},{"key":"e_1_3_2_1_34_1","volume-title":"Barlas Oguz, Madian Khabsa, Han Fang, Yashar Mehdad, Sharan Narang, Kshitiz Malik, Angela Fan, Shruti Bhosale, Sergey Edunov, Mike Lewis, Sinong Wang, and Hao Ma.","author":"Xiong Wenhan","year":"2023","unstructured":"Wenhan Xiong, Jingyu Liu, Igor Molybog, Hejia Zhang, Prajjwal Bhargava, Rui Hou, Louis Martin, Rashi Rungta, Karthik Abinav Sankararaman, Barlas Oguz, Madian Khabsa, Han Fang, Yashar Mehdad, Sharan Narang, Kshitiz Malik, Angela Fan, Shruti Bhosale, Sergey Edunov, Mike Lewis, Sinong Wang, and Hao Ma. 2023. Effective Long-Context Scaling of Foundation Models. arXiv:2309.16039 [cs.CL]"},{"key":"e_1_3_2_1_35_1","volume-title":"Big Bird: Transformers for Longer Sequences. arXiv:2007.14062 [cs.LG]","author":"Zaheer Manzil","year":"2021","unstructured":"Manzil Zaheer, Guru Guruganesh, Avinava Dubey, Joshua Ainslie, Chris Alberti, Santiago Ontanon, Philip Pham, Anirudh Ravula, Qifan Wang, Li Yang, and Amr Ahmed. 2021. Big Bird: Transformers for Longer Sequences. arXiv:2007.14062 [cs.LG]"},{"key":"e_1_3_2_1_36_1","volume-title":"Austin Clyde, Bharat Kale, Danilo Perez-Rivera, Heng Ma, et al.","author":"Zvyagin Maxim","year":"2022","unstructured":"Maxim Zvyagin, Alexander Brace, Kyle Hippe, Yuntian Deng, Bin Zhang, Cindy Orozco Bohorquez, Austin Clyde, Bharat Kale, Danilo Perez-Rivera, Heng Ma, et al. 2022. GenSLMs: Genome-scale language models reveal SARS-CoV-2 evolutionary dynamics. bioRxiv (2022), 2022--10."}],"event":{"name":"PODC '24: 43rd ACM Symposium on Principles of Distributed Computing","location":"Nantes France","acronym":"PODC '24","sponsor":["SIGACT ACM Special Interest Group on Algorithms and Computation Theory","SIGOPS ACM Special Interest Group on Operating Systems"]},"container-title":["Proceedings of the 43rd ACM Symposium on Principles of Distributed Computing"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3662158.3662806","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"}],"deposited":{"date-parts":[[2025,6,18]],"date-time":"2025-06-18T19:43:41Z","timestamp":1750275821000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3662158.3662806"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,6,17]]},"references-count":35,"alternative-id":["10.1145\/3662158.3662806","10.1145\/3662158"],"URL":"https:\/\/doi.org\/10.1145\/3662158.3662806","relation":{},"subject":[],"published":{"date-parts":[[2024,6,17]]},"assertion":[{"value":"2024-06-17","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}