{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,23]],"date-time":"2026-04-23T07:55:26Z","timestamp":1776930926301,"version":"3.51.2"},"publisher-location":"New York, NY, USA","reference-count":53,"publisher":"ACM","content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2025,11,16]]},"DOI":"10.1145\/3712285.3759894","type":"proceedings-article","created":{"date-parts":[[2025,11,12]],"date-time":"2025-11-12T16:04:47Z","timestamp":1762963487000},"page":"1769-1784","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":0,"title":["UltraAttn: Efficiently Parallelizing Attention through Hierarchical Context-Tiling"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0009-0001-4585-8305","authenticated-orcid":false,"given":"Haoyu","family":"Yang","sequence":"first","affiliation":[{"name":"Tsinghua University, Beijing, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-7828-9030","authenticated-orcid":false,"given":"Zan","family":"Zong","sequence":"additional","affiliation":[{"name":"Tsinghua University, Beijing, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-2358-3395","authenticated-orcid":false,"given":"Yuyang","family":"Jin","sequence":"additional","affiliation":[{"name":"Tsinghua University, Beijing, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0009-0006-7633-9010","authenticated-orcid":false,"given":"Kinman","family":"Lei","sequence":"additional","affiliation":[{"name":"Tsinghua University, Beijing, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-8578-5158","authenticated-orcid":false,"given":"Jiaao","family":"He","sequence":"additional","affiliation":[{"name":"Tsinghua University, Beijing, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0009-0000-2995-9213","authenticated-orcid":false,"given":"Qigang","family":"Yang","sequence":"additional","affiliation":[{"name":"Tsinghua University, Beijing, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-7656-6428","authenticated-orcid":false,"given":"Jidong","family":"Zhai","sequence":"additional","affiliation":[{"name":"Tsinghua University, Beijing, China and Qinghai University, Xining, China"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2025,11,15]]},"reference":[{"key":"e_1_3_3_3_2_2","unstructured":"Shantanu Acharya Fei Jia and Boris Ginsburg. 2024. Star attention: Efficient llm inference over long sequences. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2411.17116 (2024)."},{"key":"e_1_3_3_3_3_2","doi-asserted-by":"crossref","unstructured":"Joshua Ainslie James Lee-Thorp Michiel de Jong Yury Zemlyanskiy Federico Lebr\u00f3n and Sumit Sanghai. 2023. Gqa: Training generalized multi-query transformer models from multi-head checkpoints. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2305.13245 (2023).","DOI":"10.18653\/v1\/2023.emnlp-main.298"},{"key":"e_1_3_3_3_4_2","doi-asserted-by":"crossref","unstructured":"Joshua Ainslie Santiago Ontanon Chris Alberti Vaclav Cvicek Zachary Fisher Philip Pham Anirudh Ravula Sumit Sanghai Qifan Wang and Li Yang. 2020. ETC: Encoding long and structured inputs in transformers. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2004.08483 (2020).","DOI":"10.18653\/v1\/2020.emnlp-main.19"},{"key":"e_1_3_3_3_5_2","doi-asserted-by":"publisher","DOI":"10.1109\/SC.2012.71"},{"key":"e_1_3_3_3_6_2","unstructured":"Iz Beltagy Matthew\u00a0E Peters and Arman Cohan. 2020. Longformer: The long-document transformer. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2004.05150 (2020)."},{"key":"e_1_3_3_3_7_2","unstructured":"William Brandon Aniruddha Nrusimha Kevin Qian Zachary Ankner Tian Jin Zhiye Song and Jonathan Ragan-Kelley. 2023. Striped attention: Faster ring attention for causal transformers. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2311.09431 (2023)."},{"key":"e_1_3_3_3_8_2","unstructured":"Rewon Child Scott Gray Alec Radford and Ilya Sutskever. 1904. Generating long sequences with sparse transformers. arXiv 2019. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/1904.10509 (1904)."},{"key":"e_1_3_3_3_9_2","unstructured":"Rewon Child Scott Gray Alec Radford and Ilya Sutskever. 2019. Generating long sequences with sparse transformers. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/1904.10509 (2019)."},{"key":"e_1_3_3_3_10_2","unstructured":"Claude. 2024. Claude 3. https:\/\/claude.ai. Accessed 08\/2024."},{"key":"e_1_3_3_3_11_2","unstructured":"Tri Dao. 2023. Flashattention-2: Faster attention with better parallelism and work partitioning. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2307.08691 (2023)."},{"key":"e_1_3_3_3_12_2","doi-asserted-by":"crossref","unstructured":"Tri Dao Dan Fu Stefano Ermon Atri Rudra and Christopher R\u00e9. 2022. Flashattention: Fast and memory-efficient exact attention with io-awareness. Advances in Neural Information Processing Systems 35 (2022) 16344\u201316359.","DOI":"10.52202\/068431-1189"},{"key":"e_1_3_3_3_13_2","unstructured":"Jiarui Fang and Shangchun Zhao. 2024. A Unified Sequence Parallelism Approach for Long Context Generative AI. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2405.07719 (2024)."},{"key":"e_1_3_3_3_14_2","unstructured":"Google. 2024. Yi LLM. https:\/\/blog.google\/technology\/ai\/google-gemini-next-generation-model-february-2024\/. Accessed 08\/2024."},{"key":"e_1_3_3_3_15_2","unstructured":"LLC Gurobi\u00a0Optimization. 2025. Gurobi Optimizer. https:\/\/www.gurobi.com. Version 12.0."},{"key":"e_1_3_3_3_16_2","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52729.2023.00599"},{"key":"e_1_3_3_3_17_2","unstructured":"Jonathan Ho Nal Kalchbrenner Dirk Weissenborn and Tim Salimans. 2019. Axial attention in multidimensional transformers. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/1912.12180 (2019)."},{"key":"e_1_3_3_3_18_2","unstructured":"Yanping Huang Youlong Cheng Ankur Bapna Orhan Firat Dehao Chen Mia Chen HyoukJoong Lee Jiquan Ngiam Quoc\u00a0V Le Yonghui Wu et\u00a0al. 2019. Gpipe: Efficient training of giant neural networks using pipeline parallelism. Advances in neural information processing systems 32 (2019)."},{"key":"e_1_3_3_3_19_2","unstructured":"Sam\u00a0Ade Jacobs Masahiro Tanaka Chengming Zhang Minjia Zhang Shuaiwen\u00a0Leon Song Samyam Rajbhandari and Yuxiong He. 2023. DeepSpeed Ulysses: System Optimizations for Enabling Training of Extreme Long Sequence Transformer Models.(2023). arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2309.14509 (2023)."},{"key":"e_1_3_3_3_20_2","volume-title":"Proceedings of the Second Conference on Machine Learning and Systems, SysML 2019, Stanford, CA, USA, March 31 - April 2, 2019","author":"Jia Zhihao","year":"2019","unstructured":"Zhihao Jia, Matei Zaharia, and Alex Aiken. 2019. Beyond Data and Model Parallelism for Deep Neural Networks. In Proceedings of the Second Conference on Machine Learning and Systems, SysML 2019, Stanford, CA, USA, March 31 - April 2, 2019, Ameet Talwalkar, Virginia Smith, and Matei Zaharia (Eds.). mlsys.org. https:\/\/proceedings.mlsys.org\/paper_files\/paper\/2019\/hash\/b422680f3db0986ddd7f8f126baaf0fa-Abstract.html"},{"key":"e_1_3_3_3_21_2","unstructured":"Nikita Kitaev \u0141ukasz Kaiser and Anselm Levskaya. 2020. Reformer: The efficient transformer. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2001.04451 (2020)."},{"key":"e_1_3_3_3_22_2","unstructured":"Dacheng Li Rulin Shao Anze Xie Eric\u00a0P Xing Joseph\u00a0E Gonzalez Ion Stoica Xuezhe Ma and Hao Zhang. 2023. Lightseq: Sequence level parallelism for distributed training of long context transformers. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2310.03294 (2023)."},{"key":"e_1_3_3_3_23_2","unstructured":"Shenggui Li Fuzhao Xue Chaitanya Baranwal Yongbin Li and Yang You. 2021. Sequence parallelism: Long sequence training from system perspective. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2105.13120 (2021)."},{"key":"e_1_3_3_3_24_2","doi-asserted-by":"publisher","DOI":"10.1109\/ICME52920.2022.9859720"},{"key":"e_1_3_3_3_25_2","unstructured":"lingyiwanwu. 2024. Yi LLM. https:\/\/www.lingyiwanwu.com\/. Accessed 08\/2024."},{"key":"e_1_3_3_3_26_2","unstructured":"Aixin Liu Bei Feng Bing Xue Bingxuan Wang Bochao Wu Chengda Lu Chenggang Zhao Chengqi Deng Chenyu Zhang Chong Ruan et\u00a0al. 2024. DeepSeek-V3 Technical Report. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2412.19437 (2024)."},{"key":"e_1_3_3_3_27_2","unstructured":"Hao Liu Wilson Yan Matei Zaharia and Pieter Abbeel. 2024. World model on million-length video and language with ringattention. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2402.08268 (2024)."},{"key":"e_1_3_3_3_28_2","unstructured":"Hao Liu Matei Zaharia and Pieter Abbeel. 2023. Ring attention with blockwise transformers for near-infinite context. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2310.01889 (2023)."},{"key":"e_1_3_3_3_29_2","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV48922.2021.00986"},{"key":"e_1_3_3_3_30_2","unstructured":"META. 2024. llama2. https:\/\/llama.meta.com\/llama2\/. Accessed 08\/2024."},{"key":"e_1_3_3_3_31_2","unstructured":"Maxim Milakov and Natalia Gimelshein. 2018. Online normalizer calculation for softmax. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/1805.02867 (2018)."},{"key":"e_1_3_3_3_32_2","first-page":"561","volume-title":"13th USENIX symposium on operating systems design and implementation (OSDI 18)","author":"Moritz Philipp","year":"2018","unstructured":"Philipp Moritz, Robert Nishihara, Stephanie Wang, Alexey Tumanov, Richard Liaw, Eric Liang, Melih Elibol, Zongheng Yang, William Paul, Michael\u00a0I Jordan, et\u00a0al. 2018. Ray: A distributed framework for emerging { AI} applications. In 13th USENIX symposium on operating systems design and implementation (OSDI 18). 561\u2013577."},{"key":"e_1_3_3_3_33_2","unstructured":"Tsendsuren Munkhdalai Manaal Faruqui and Siddharth Gopal. 2024. Leave no context behind: Efficient infinite context transformers with infini-attention. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2404.07143 101 (2024)."},{"key":"e_1_3_3_3_34_2","doi-asserted-by":"publisher","DOI":"10.1145\/3458817.3476209"},{"key":"e_1_3_3_3_35_2","unstructured":"NVIDIA. 2019. CudaGraph. https:\/\/developer.nvidia.com\/blog\/cuda-graphs\/. Accessed 04\/2025."},{"key":"e_1_3_3_3_36_2","unstructured":"Pytorch. 2025. FlexAttention. https:\/\/pytorch.org\/blog\/flexattention\/. Accessed 01\/2025."},{"key":"e_1_3_3_3_37_2","doi-asserted-by":"publisher","DOI":"10.1109\/SC41405.2020.00024"},{"key":"e_1_3_3_3_38_2","unstructured":"Liliang Ren Yang Liu Yadong Lu Yelong Shen Chen Liang and Weizhu Chen. 2024. Samba: Simple hybrid state space models for efficient unlimited context language modeling. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2406.07522 (2024)."},{"key":"e_1_3_3_3_39_2","doi-asserted-by":"crossref","unstructured":"Aurko Roy Mohammad Saffar Ashish Vaswani and David Grangier. 2021. Efficient content-based sparse attention with routing transformers. Transactions of the Association for Computational Linguistics 9 (2021) 53\u201368.","DOI":"10.1162\/tacl_a_00353"},{"key":"e_1_3_3_3_40_2","doi-asserted-by":"crossref","unstructured":"Jay Shah Ganesh Bikshandi Ying Zhang Vijay Thakkar Pradeep Ramani and Tri Dao. 2024. Flashattention-3: Fast and accurate attention with asynchrony and low-precision. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2407.08608 (2024).","DOI":"10.52202\/079017-2193"},{"key":"e_1_3_3_3_41_2","unstructured":"Mohammad Shoeybi Mostofa Patwary Raul Puri Patrick LeGresley Jared Casper and Bryan Catanzaro. 2019. Megatron-lm: Training multi-billion parameter language models using model parallelism. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/1909.08053 (2019)."},{"key":"e_1_3_3_3_42_2","doi-asserted-by":"publisher","DOI":"10.1145\/3620666.3651359"},{"key":"e_1_3_3_3_43_2","first-page":"267","volume-title":"16th USENIX Symposium on Operating Systems Design and Implementation (OSDI 22)","author":"Unger Colin","year":"2022","unstructured":"Colin Unger, Zhihao Jia, Wei Wu, Sina Lin, Mandeep Baines, Carlos Efrain\u00a0Quintero Narvaez, Vinay Ramakrishnaiah, Nirmal Prajapati, Pat McCormick, Jamaludin Mohd-Yusof, et\u00a0al. 2022. Unity: Accelerating { DNN} training through joint optimization of algebraic transformations and parallelization. In 16th USENIX Symposium on Operating Systems Design and Implementation (OSDI 22). 267\u2013284."},{"key":"e_1_3_3_3_44_2","doi-asserted-by":"crossref","unstructured":"Ethan Waisberg Joshua Ong Mouayad Masalkhi Sharif\u00a0Amit Kamran Nasif Zaman Prithul Sarker Andrew\u00a0G Lee and Alireza Tavakkoli. 2023. GPT-4: a new era of artificial intelligence in medicine. Irish Journal of Medical Science (1971-) 192 6 (2023) 3197\u20133200.","DOI":"10.1007\/s11845-023-03377-8"},{"key":"e_1_3_3_3_45_2","doi-asserted-by":"publisher","DOI":"10.1109\/BigData47090.2019.9006180"},{"key":"e_1_3_3_3_46_2","doi-asserted-by":"publisher","DOI":"10.1145\/3302424.3303953"},{"key":"e_1_3_3_3_47_2","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52729.2023.02172"},{"key":"e_1_3_3_3_48_2","unstructured":"Guangxuan Xiao Yuandong Tian Beidi Chen Song Han and Mike Lewis. 2023. Efficient streaming language models with attention sinks. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2309.17453 (2023)."},{"key":"e_1_3_3_3_49_2","unstructured":"Wilson Yan Yunzhi Zhang Pieter Abbeel and Aravind Srinivas. 2021. Videogpt: Video generation using vq-vae and transformers. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2104.10157 (2021)."},{"key":"e_1_3_3_3_50_2","unstructured":"Jingyi Yang Aya Ibrahim Xinfeng Xie Bangsheng Tang Grigory Sizov Jongsoo Park Jianyu Huang et\u00a0al. 2024. Context Parallelism for Scalable Million-Token Inference. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2411.01783 (2024)."},{"key":"e_1_3_3_3_51_2","unstructured":"Zihao Ye Lequn Chen Ruihang Lai Wuwei Lin Yineng Zhang Stephanie Wang Tianqi Chen Baris Kasikci Vinod Grover Arvind Krishnamurthy et\u00a0al. 2025. FlashInfer: Efficient and Customizable Attention Engine for LLM Inference Serving. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2501.01005 (2025)."},{"key":"e_1_3_3_3_52_2","unstructured":"Manzil Zaheer Guru Guruganesh Kumar\u00a0Avinava Dubey Joshua Ainslie Chris Alberti Santiago Ontanon Philip Pham Anirudh Ravula Qifan Wang Li Yang et\u00a0al. 2020. Big bird: Transformers for longer sequences. Advances in neural information processing systems 33 (2020) 17283\u201317297."},{"key":"e_1_3_3_3_53_2","unstructured":"Yanli Zhao Andrew Gu Rohan Varma Liang Luo Chien-Chin Huang Min Xu Less Wright Hamid Shojanazeri Myle Ott Sam Shleifer et\u00a0al. 2023. Pytorch fsdp: experiences on scaling fully sharded data parallel. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2304.11277 (2023)."},{"key":"e_1_3_3_3_54_2","first-page":"559","volume-title":"16th USENIX Symposium on Operating Systems Design and Implementation (OSDI 22)","author":"Zheng Lianmin","year":"2022","unstructured":"Lianmin Zheng, Zhuohan Li, Hao Zhang, Yonghao Zhuang, Zhifeng Chen, Yanping Huang, Yida Wang, Yuanzhong Xu, Danyang Zhuo, Eric\u00a0P Xing, et\u00a0al. 2022. Alpa: Automating inter-and { Intra-Operator} parallelism for distributed deep learning. In 16th USENIX Symposium on Operating Systems Design and Implementation (OSDI 22). 559\u2013578."}],"event":{"name":"SC '25: The International Conference for High Performance Computing, Networking, Storage and Analysis","location":"St. Louis MO USA","acronym":"SC '25","sponsor":["SIGHPC ACM Special Interest Group on High Performance Computing, Special Interest Group on High Performance Computing"]},"container-title":["Proceedings of the International Conference for High Performance Computing, Networking, Storage and Analysis"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3712285.3759894","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,3,11]],"date-time":"2026-03-11T18:45:21Z","timestamp":1773254721000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3712285.3759894"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,11,15]]},"references-count":53,"alternative-id":["10.1145\/3712285.3759894","10.1145\/3712285"],"URL":"https:\/\/doi.org\/10.1145\/3712285.3759894","relation":{},"subject":[],"published":{"date-parts":[[2025,11,15]]},"assertion":[{"value":"2025-11-15","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}