{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,23]],"date-time":"2026-04-23T07:55:26Z","timestamp":1776930926455,"version":"3.51.2"},"publisher-location":"New York, NY, USA","reference-count":42,"publisher":"ACM","license":[{"start":{"date-parts":[[2025,11,15]],"date-time":"2025-11-15T00:00:00Z","timestamp":1763164800000},"content-version":"vor","delay-in-days":0,"URL":"http:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"DOI":"10.13039\/100000015","name":"DOE U.S. Department of Energy","doi-asserted-by":"publisher","award":["DE-AC05-00OR22725"],"award-info":[{"award-number":["DE-AC05-00OR22725"]}],"id":[{"id":"10.13039\/100000015","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2025,11,16]]},"DOI":"10.1145\/3712285.3759859","type":"proceedings-article","created":{"date-parts":[[2025,11,12]],"date-time":"2025-11-12T16:04:47Z","timestamp":1762963487000},"page":"1395-1408","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":0,"title":["RingX: Scalable Parallel Attention for Long-Context Learning on HPC"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-3843-5520","authenticated-orcid":false,"given":"Junqi","family":"Yin","sequence":"first","affiliation":[{"name":"Oak Ridge National Laboratory (ORNL), Oak Ridge, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-9751-1370","authenticated-orcid":false,"given":"Mijanur","family":"Palash","sequence":"additional","affiliation":[{"name":"Oak Ridge National Laboratory (ORNL), Oak Ridge, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-5289-7460","authenticated-orcid":false,"given":"Mallikarjun","family":"Shankar","sequence":"additional","affiliation":[{"name":"Oak Ridge National Laboratory (ORNL), Oak Ridge, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-0099-1559","authenticated-orcid":false,"given":"Feiyi","family":"Wang","sequence":"additional","affiliation":[{"name":"Oak Ridge National Laboratory (ORNL), Oak Ridge, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2025,11,15]]},"reference":[{"key":"e_1_3_3_3_2_2","unstructured":"Together AI. 2024. Long-Data-Collections. https:\/\/huggingface.co\/datasets\/togethercomputer\/Long-Data-Collections\/."},{"key":"e_1_3_3_3_3_2","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2023.emnlp-main.298"},{"key":"e_1_3_3_3_4_2","unstructured":"Hangbo Bao Li Dong and Furu Wei. 2021. BEiT: BERT Pre-Training of Image Transformers. arxiv:https:\/\/arXiv.org\/abs\/2106.08254\u00a0[cs.CV]"},{"key":"e_1_3_3_3_5_2","unstructured":"Cristian Bodnar Wessel\u00a0P. Bruinsma Ana Lucic Megan Stanley Anna Vaughan Johannes Brandstetter Patrick Garvan Maik Riechert Jonathan\u00a0A. Weyn Haiyu Dong Jayesh\u00a0K. Gupta Kit Thambiratnam Alexander\u00a0T. Archibald Chun-Chieh Wu Elizabeth Heider Max Welling Richard\u00a0E. Turner and Paris Perdikaris. 2024. A Foundation Model for the Earth System. arxiv:https:\/\/arXiv.org\/abs\/2405.13063\u00a0[physics.ao-ph] https:\/\/arxiv.org\/abs\/2405.13063"},{"key":"e_1_3_3_3_6_2","unstructured":"William Brandon Aniruddha Nrusimha Kevin Qian Zachary Ankner Tian Jin Zhiye Song and Jonathan Ragan-Kelley. 2023. Striped Attention: Faster Ring Attention for Causal Transformers. arxiv:https:\/\/arXiv.org\/abs\/2311.09431\u00a0[cs.LG] https:\/\/arxiv.org\/abs\/2311.09431"},{"key":"e_1_3_3_3_7_2","first-page":"1877","volume-title":"Advances in Neural Information Processing Systems","author":"Brown Tom","year":"2020","unstructured":"Tom Brown, Benjamin Mann, Nick Ryder, Melanie Subbiah, Jared\u00a0D Kaplan, Prafulla Dhariwal, Arvind Neelakantan, Pranav Shyam, Girish Sastry, Amanda Askell, Sandhini Agarwal, Ariel Herbert-Voss, Gretchen Krueger, Tom Henighan, Rewon Child, Aditya Ramesh, Daniel Ziegler, Jeffrey Wu, Clemens Winter, Chris Hesse, Mark Chen, Eric Sigler, Mateusz Litwin, Scott Gray, Benjamin Chess, Jack Clark, Christopher Berner, Sam McCandlish, Alec Radford, Ilya Sutskever, and Dario Amodei. 2020. Language Models are Few-Shot Learners. In Advances in Neural Information Processing Systems , H.\u00a0Larochelle, M.\u00a0Ranzato, R.\u00a0Hadsell, M.F. Balcan, and H.\u00a0Lin (Eds.), Vol.\u00a033. Curran Associates, Inc., virtual, 1877\u20131901. https:\/\/proceedings.neurips.cc\/paper\/2020\/file\/1457c0d6bfcb4967418bfb8ac142f64a-Paper.pdf"},{"key":"e_1_3_3_3_8_2","unstructured":"Jean-Baptiste Cordonnier Andreas Loukas and Martin Jaggi. 2021. Multi-Head Attention: Collaborate Instead of Concatenate. arxiv:https:\/\/arXiv.org\/abs\/2006.16362\u00a0[cs.LG] https:\/\/arxiv.org\/abs\/2006.16362"},{"key":"e_1_3_3_3_9_2","doi-asserted-by":"crossref","unstructured":"Tri Dao Daniel\u00a0Y. Fu Stefano Ermon Atri Rudra and Christopher R\u00e9. 2022. FlashAttention: Fast and Memory-Efficient Exact Attention with IO-Awareness. arxiv:https:\/\/arXiv.org\/abs\/2205.14135\u00a0[cs.LG]","DOI":"10.52202\/068431-1189"},{"key":"e_1_3_3_3_10_2","unstructured":"DeepSeek-AI. 2024. DeepSeek-V2: A Strong Economical and Efficient Mixture-of-Experts Language Model. arxiv:https:\/\/arXiv.org\/abs\/2405.04434\u00a0[cs.CL] https:\/\/arxiv.org\/abs\/2405.04434"},{"key":"e_1_3_3_3_11_2","unstructured":"DeepSeek-AI. 2025. DeepSeek-R1: Incentivizing Reasoning Capability in LLMs via Reinforcement Learning. arxiv:https:\/\/arXiv.org\/abs\/2501.12948\u00a0[cs.CL] https:\/\/arxiv.org\/abs\/2501.12948"},{"key":"e_1_3_3_3_12_2","unstructured":"DeepSeek-AI Aixin Liu Bei Feng Bing Xue and et al. 2025. DeepSeek-V3 Technical Report. arxiv:https:\/\/arXiv.org\/abs\/2412.19437\u00a0[cs.CL] https:\/\/arxiv.org\/abs\/2412.19437"},{"key":"e_1_3_3_3_13_2","unstructured":"Mostafa Dehghani Josip Djolonga Basil Mustafa and et al. 2023. Scaling Vision Transformers to 22 Billion Parameters. arxiv:https:\/\/arXiv.org\/abs\/2302.05442\u00a0[cs.CV] https:\/\/arxiv.org\/abs\/2302.05442"},{"key":"e_1_3_3_3_14_2","volume-title":"International Conference on Learning Representations","author":"Dosovitskiy Alexey","year":"2021","unstructured":"Alexey Dosovitskiy, Lucas Beyer, Alexander Kolesnikov, Dirk Weissenborn, Xiaohua Zhai, Thomas Unterthiner, Mostafa Dehghani, Matthias Minderer, Georg Heigold, Sylvain Gelly, Jakob Uszkoreit, and Neil Houlsby. 2021. An Image is Worth 16x16 Words: Transformers for Image Recognition at Scale. In International Conference on Learning Representations. https:\/\/openreview.net\/forum?id=YicbFdNTTy"},{"key":"e_1_3_3_3_15_2","unstructured":"Jiarui Fang and Shangchun Zhao. 2024. USP: A Unified Sequence Parallelism Approach for Long Context Generative AI. arxiv:https:\/\/arXiv.org\/abs\/2405.07719\u00a0[cs.LG] https:\/\/arxiv.org\/abs\/2405.07719"},{"key":"e_1_3_3_3_16_2","unstructured":"Google. 2025. Gemini 2.5 Pro. https:\/\/blog.google\/technology\/google-deepmind\/gemini-model-thinking-updates-march-2025\/."},{"key":"e_1_3_3_3_17_2","unstructured":"Gradient. 2024. Llama3 with 1M context Length. https:\/\/www.gradient.ai\/. Accessed: 2025-03-26."},{"key":"e_1_3_3_3_18_2","unstructured":"Aaron Grattafiori Abhimanyu Dubey Abhinav Jauhri and et al. 2024. The Llama 3 Herd of Models. arxiv:https:\/\/arXiv.org\/abs\/2407.21783\u00a0[cs.AI] https:\/\/arxiv.org\/abs\/2407.21783"},{"key":"e_1_3_3_3_19_2","unstructured":"Diandian Gu Peng Sun Qinghao Hu Ting Huang Xun Chen Yingtong Xiong Guoteng Wang Qiaoling Chen Shangchun Zhao Jiarui Fang Yonggang Wen Tianwei Zhang Xin Jin and Xuanzhe Liu. 2024. LoongTrain: Efficient Training of Long-Sequence LLMs with Head-Context Parallelism. arxiv:https:\/\/arXiv.org\/abs\/2406.18485\u00a0[cs.DC] https:\/\/arxiv.org\/abs\/2406.18485"},{"key":"e_1_3_3_3_20_2","doi-asserted-by":"publisher","unstructured":"Hans Hersbach Bill Bell Paul Berrisford and et al. 2020. The ERA5 global reanalysis. Quarterly Journal of the Royal Meteorological Society 146 730 (2020) 1999\u20132049. 10.1002\/qj.3803","DOI":"10.1002\/qj.3803"},{"key":"e_1_3_3_3_21_2","doi-asserted-by":"publisher","DOI":"10.1145\/3662158.3662806"},{"key":"e_1_3_3_3_22_2","volume-title":"First Conference on Language Modeling","author":"Li Dacheng","year":"2024","unstructured":"Dacheng Li, Rulin Shao, Anze Xie, Eric\u00a0P. Xing, Xuezhe Ma, Ion Stoica, Joseph\u00a0E. Gonzalez, and Hao Zhang. 2024. DISTFLASHATTN: Distributed Memory-efficient Attention for Long-context LLMs Training. In First Conference on Language Modeling. https:\/\/openreview.net\/forum?id=pUEDkZyPDl"},{"key":"e_1_3_3_3_23_2","doi-asserted-by":"crossref","unstructured":"Y. Li E. Perlman M. Wan Y. Yang C. Meneveau R. Burns S. Chen A. Szalay and G. Eyink. 2008. A public turbulence database cluster and applications to study Lagrangian evolution of velocity increments in turbulence. Journal of Turbulence 9 31 (2008). Article 31.","DOI":"10.1080\/14685240802376389"},{"key":"e_1_3_3_3_24_2","volume-title":"The Twelfth International Conference on Learning Representations","author":"Liu Hao","year":"2024","unstructured":"Hao Liu, Matei Zaharia, and Pieter Abbeel. 2024. RingAttention with Blockwise Transformers for Near-Infinite Context. In The Twelfth International Conference on Learning Representations. https:\/\/openreview.net\/forum?id=WsRHpHH4s0"},{"key":"e_1_3_3_3_25_2","doi-asserted-by":"publisher","DOI":"10.1145\/3458817.3476209"},{"key":"e_1_3_3_3_26_2","unstructured":"NERSC. 2023. Numeric Weather prediction with ViT. https:\/\/github.com\/NERSC\/sc23-dl-tutorial. Accessed: 2025-3-26."},{"key":"e_1_3_3_3_27_2","unstructured":"Tung Nguyen Johannes Brandstetter Ashish Kapoor Jayesh\u00a0K. Gupta and Aditya Grover. 2023. ClimaX: A foundation model for weather and climate. arxiv:https:\/\/arXiv.org\/abs\/2301.10343\u00a0[cs.LG] https:\/\/arxiv.org\/abs\/2301.10343"},{"key":"e_1_3_3_3_28_2","unstructured":"OpenAI. 2023. ChatGPT O1 Reasoning Model. https:\/\/openai.com\/. Accessed: 2025-03-26."},{"key":"e_1_3_3_3_29_2","unstructured":"OpenAI. 2023. GPT-4 Technical Report. arxiv:https:\/\/arXiv.org\/abs\/2303.08774\u00a0[cs.CL]"},{"key":"e_1_3_3_3_30_2","unstructured":"OpenAI. 2024. Learning to reason with LLMs. https:\/\/openai.com\/index\/learning-to-reason-with-llms\/."},{"key":"e_1_3_3_3_31_2","unstructured":"OpenAI. 2025. ChatGPT (Mar 23 2025 Version). Large language model. Retrieved from https:\/\/chat.openai.com\/."},{"key":"e_1_3_3_3_32_2","unstructured":"OpenAI. 2025. GPT-4.5 (Mar 23 2025 Version). Large language model. Retrieved from https:\/\/chat.openai.com\/."},{"key":"e_1_3_3_3_33_2","doi-asserted-by":"publisher","DOI":"10.1145\/1362622.1362654"},{"key":"e_1_3_3_3_34_2","unstructured":"Alec Radford Karthik Narasimhan Tim Salimans Ilya Sutskever et\u00a0al. 2018. Improving language understanding by generative pre-training. (2018)."},{"key":"e_1_3_3_3_35_2","unstructured":"Alexander Sergeev and Mike\u00a0Del Balso. 2018. Horovod: fast and easy distributed deep learning in TensorFlow. arxiv:https:\/\/arXiv.org\/abs\/1802.05799\u00a0[cs.LG] https:\/\/arxiv.org\/abs\/1802.05799"},{"key":"e_1_3_3_3_36_2","doi-asserted-by":"publisher","DOI":"10.5555\/1964238.1964240"},{"key":"e_1_3_3_3_37_2","unstructured":"Vasudev Shyam Jonathan Pilault Emily Shepperd Quentin Anthony and Beren Millidge. 2025. Tree Attention: Topology-aware Decoding for Long-Context Attention on GPU clusters. arxiv:https:\/\/arXiv.org\/abs\/2408.04093\u00a0[cs.LG] https:\/\/arxiv.org\/abs\/2408.04093"},{"key":"e_1_3_3_3_38_2","doi-asserted-by":"publisher","unstructured":"Rajeev Thakur Rolf Rabenseifner and William Gropp. 2005. Optimization of Collective Communication Operations in MPICH. The International Journal of High Performance Computing Applications 19 1 (2005) 49\u201366. 10.1177\/1094342005051521 arXiv:https:\/\/doi.org\/10.1177\/1094342005051521","DOI":"10.1177\/1094342005051521"},{"key":"e_1_3_3_3_39_2","volume-title":"International Conference on Machine Learning","author":"Touvron Hugo","year":"2021","unstructured":"Hugo Touvron, Matthieu Cord, Matthieu Douze, Francisco Massa, Alexandre Sablayrolles, and Herv\u00e9 J\u00e9gou. 2021. Training data-efficient image transformers & distillation through attention. In International Conference on Machine Learning."},{"key":"e_1_3_3_3_40_2","unstructured":"Ashish Vaswani Noam Shazeer Niki Parmar Jakob Uszkoreit Llion Jones Aidan\u00a0N Gomez \u0141ukasz Kaiser and Illia Polosukhin. 2017. Attention is all you need. Advances in neural information processing systems 30 (2017)."},{"key":"e_1_3_3_3_41_2","doi-asserted-by":"publisher","DOI":"10.1109\/SC41406.2024.00007"},{"key":"e_1_3_3_3_42_2","doi-asserted-by":"publisher","DOI":"10.1145\/3581784.3613215"},{"key":"e_1_3_3_3_43_2","doi-asserted-by":"publisher","DOI":"10.1109\/SCW63240.2024.00009"}],"event":{"name":"SC '25: The International Conference for High Performance Computing, Networking, Storage and Analysis","location":"St. Louis MO USA","acronym":"SC '25","sponsor":["SIGHPC ACM Special Interest Group on High Performance Computing, Special Interest Group on High Performance Computing"]},"container-title":["Proceedings of the International Conference for High Performance Computing, Networking, Storage and Analysis"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/abs\/10.1145\/3712285.3759859","content-type":"text\/html","content-version":"vor","intended-application":"syndication"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3712285.3759859","content-type":"application\/pdf","content-version":"vor","intended-application":"syndication"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3712285.3759859","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,3,11]],"date-time":"2026-03-11T18:30:11Z","timestamp":1773253811000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3712285.3759859"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,11,15]]},"references-count":42,"alternative-id":["10.1145\/3712285.3759859","10.1145\/3712285"],"URL":"https:\/\/doi.org\/10.1145\/3712285.3759859","relation":{},"subject":[],"published":{"date-parts":[[2025,11,15]]},"assertion":[{"value":"2025-11-15","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}