{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,2]],"date-time":"2026-07-02T13:44:40Z","timestamp":1782999880404,"version":"3.54.5"},"publisher-location":"New York, NY, USA","reference-count":47,"publisher":"ACM","license":[{"start":{"date-parts":[[2026,7,5]],"date-time":"2026-07-05T00:00:00Z","timestamp":1783209600000},"content-version":"vor","delay-in-days":0,"URL":"http:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"DOI":"10.13039\/100000001","name":"NSF (National Science Foundation)","doi-asserted-by":"publisher","award":["2514351"],"award-info":[{"award-number":["2514351"]}],"id":[{"id":"10.13039\/100000001","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000001","name":"NSF (National Science Foundation)","doi-asserted-by":"publisher","award":["2505118"],"award-info":[{"award-number":["2505118"]}],"id":[{"id":"10.13039\/100000001","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000015","name":"DOE U.S. Department of Energy","doi-asserted-by":"publisher","award":["DE-AC05-00OR22725"],"award-info":[{"award-number":["DE-AC05-00OR22725"]}],"id":[{"id":"10.13039\/100000015","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2026,7,6]]},"DOI":"10.1145\/3797905.3800525","type":"proceedings-article","created":{"date-parts":[[2026,7,2]],"date-time":"2026-07-02T11:50:37Z","timestamp":1782993037000},"page":"17-29","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":0,"title":["FLYING SERVING: On-the-Fly Parallelism Switching for Large Language Model Serving"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0009-0003-6998-6090","authenticated-orcid":false,"given":"Shouwei","family":"Gao","sequence":"first","affiliation":[{"name":"Oregon State University, Corvallis, Oregon, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-3843-5520","authenticated-orcid":false,"given":"Junqi","family":"Yin","sequence":"additional","affiliation":[{"name":"Oak Ridge National Laboratory, Oak Ridge, TN, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-0099-1559","authenticated-orcid":false,"given":"Feiyi","family":"Wang","sequence":"additional","affiliation":[{"name":"Oak Ridge National Laboratory, Oak Ridge, TN, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0003-7076-3230","authenticated-orcid":false,"given":"Wenqian","family":"Dong","sequence":"additional","affiliation":[{"name":"Oregon State University, Corvallis, OR, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2026,7,5]]},"reference":[{"key":"e_1_3_3_2_2_2","series-title":"(OSDI\u201924)","volume-title":"Proceedings of the 18th USENIX Conference on Operating Systems Design and Implementation","author":"Agrawal Amey","year":"2024","unstructured":"Amey Agrawal, Nitin Kedia, Ashish Panwar, Jayashree Mohan, Nipun Kwatra, Bhargav\u00a0S. Gulavani, Alexey Tumanov, and Ramachandran Ramjee. 2024. Taming throughput-latency tradeoff in LLM inference with sarathi-serve. In Proceedings of the 18th USENIX Conference on Operating Systems Design and Implementation (Santa Clara, CA, USA) (OSDI\u201924). USENIX Association, USA, Article 7, 18\u00a0pages."},{"key":"e_1_3_3_2_3_2","unstructured":"Amey Agrawal Ashish Panwar Jayashree Mohan Nipun Kwatra Bhargav\u00a0S Gulavani and Ramachandran Ramjee. 2023. Sarathi: Efficient llm inference by piggybacking decodes with chunked prefills. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2308.16369 (2023)."},{"key":"e_1_3_3_2_4_2","doi-asserted-by":"crossref","unstructured":"Joshua Ainslie James Lee-Thorp Michiel De\u00a0Jong Yury Zemlyanskiy Federico Lebr\u00f3n and Sumit Sanghai. 2023. Gqa: Training generalized multi-query transformer models from multi-head checkpoints. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2305.13245 (2023).","DOI":"10.18653\/v1\/2023.emnlp-main.298"},{"key":"e_1_3_3_2_5_2","first-page":"1877","volume-title":"Advances in Neural Information Processing Systems","author":"Brown Tom","year":"2020","unstructured":"Tom Brown, Benjamin Mann, Nick Ryder, Melanie Subbiah, Jared\u00a0D Kaplan, Prafulla Dhariwal, Arvind Neelakantan, Pranav Shyam, Girish Sastry, Amanda Askell, et\u00a0al. 2020. Language models are few-shot learners. In Advances in Neural Information Processing Systems , Vol.\u00a033. 1877\u20131901."},{"key":"e_1_3_3_2_6_2","unstructured":"Haoyu Chen Xue Li Kun Qian and Xin Wang. 2025. Gyges: Dynamic Cross-Instance Parallelism Transformation for Efficient LLM Inference. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2509.19729 (2025)."},{"key":"e_1_3_3_2_7_2","unstructured":"Mark Chen Jerry Tworek Heewoo Jun Qiming Yuan Henrique\u00a0Ponde de Oliveira\u00a0Pinto Jared Kaplan et\u00a0al. 2021. Evaluating Large Language Models Trained on Code. arxiv:https:\/\/arXiv.org\/abs\/2107.03374\u00a0[cs.LG] https:\/\/arxiv.org\/abs\/2107.03374"},{"key":"e_1_3_3_2_8_2","unstructured":"DeepSeek-AI Aixin Liu Bei Feng Bing Xue Bingxuan Wang Bochao Wu Chengda Lu Chenggang Zhao and et al.2025. DeepSeek-V3 Technical Report. arxiv:https:\/\/arXiv.org\/abs\/2412.19437\u00a0[cs.CL] https:\/\/arxiv.org\/abs\/2412.19437"},{"key":"e_1_3_3_2_9_2","unstructured":"Amr Elmeleegy Lequn Chen and Kevin Hu. 2024. Spotlight: Perplexity AI Serves 400 Million Search Queries a Month Using NVIDIA Inference Stack."},{"key":"e_1_3_3_2_10_2","doi-asserted-by":"crossref","unstructured":"Mohammed\u00a0Daffalla Elradi. 2025. Prometheus & Grafana: A Metrics-focused Monitoring Stack. Journal of Computer Allied Intelligence (JCAI ISSN: 2584-2676) 3 3 (2025) 28\u201339.","DOI":"10.69996\/jcai.2025015"},{"key":"e_1_3_3_2_11_2","unstructured":"Aaron Grattafiori Abhimanyu Dubey et\u00a0al. 2024. The Llama 3 Herd of Models. arxiv:https:\/\/arXiv.org\/abs\/2407.21783\u00a0[cs.AI] https:\/\/arxiv.org\/abs\/2407.21783"},{"key":"e_1_3_3_2_12_2","doi-asserted-by":"publisher","DOI":"10.1145\/3779212.3790219"},{"key":"e_1_3_3_2_13_2","unstructured":"Krystal Hu et\u00a0al. 2023. ChatGPT sets record for fastest-growing user base-analyst note. Reuters 12 (2023) 2023."},{"key":"e_1_3_3_2_14_2","volume-title":"GPipe: efficient training of giant neural networks using pipeline parallelism","author":"Huang Yanping","year":"2019","unstructured":"Yanping Huang, Youlong Cheng, Ankur Bapna, Orhan Firat, Mia\u00a0Xu Chen, Dehao Chen, HyoukJoong Lee, Jiquan Ngiam, Quoc\u00a0V. Le, Yonghui Wu, and Zhifeng Chen. 2019. GPipe: efficient training of giant neural networks using pipeline parallelism. Curran Associates Inc., Red Hook, NY, USA."},{"key":"e_1_3_3_2_15_2","volume-title":"ACM Sigmetrics 2026","author":"Jain Kunal","year":"2026","unstructured":"Kunal Jain, A. Parayil, Ankur Mallick, Rujia Wang, Renee St.\u00a0Amant, Chetan Bansal, Victor Ruehle, Saravan Rajmohan, Shashwat Jaiswal, Yogesh Simmhan, Anoop Kulkarni, and Steve Kofsky. 2026. Serving Models, Fast and Slow:Optimizing Heterogeneous LLM Inferencing Workloads at Scale. In ACM Sigmetrics 2026. https:\/\/www.microsoft.com\/en-us\/research\/publication\/serving-models-fast-and-slowoptimizing-heterogeneous-llm-inferencing-workloads-at-scale\/"},{"key":"e_1_3_3_2_16_2","doi-asserted-by":"publisher","DOI":"10.1145\/3600006.3613165"},{"key":"e_1_3_3_2_17_2","unstructured":"Ruiqi Lai Hongrui Liu Chengzhi Lu Zonghao Liu Siyu Cao Siyang Shao Yixin Zhang Luo Mai and Dmitrii Ustiugov. 2025. TokenScale: Timely and Accurate Autoscaling for Disaggregated LLM Serving with Token Velocity. arxiv:https:\/\/arXiv.org\/abs\/2512.03416\u00a0[cs.DC] https:\/\/arxiv.org\/abs\/2512.03416"},{"key":"e_1_3_3_2_18_2","doi-asserted-by":"crossref","unstructured":"Yujia Li David Choi Junyoung Chung Nate Kushman Julian Schrittwieser R\u00e9mi Leblond Tom Eccles James Keeling Felix Gimeno Agustin Dal\u00a0Lago et\u00a0al. 2022. Competition-level code generation with alphacode. Science 378 6624 (2022) 1092\u20131097.","DOI":"10.1126\/science.abq1158"},{"key":"e_1_3_3_2_19_2","volume-title":"17th USENIX Symposium on Operating Systems Design and Implementation (OSDI 23)","author":"Li Zhuohan","year":"2023","unstructured":"Zhuohan Li et\u00a0al. 2023. AlpaServe: Statistical Multiplexing with Model Parallelism for Deep Learning Serving. In 17th USENIX Symposium on Operating Systems Design and Implementation (OSDI 23)."},{"key":"e_1_3_3_2_20_2","unstructured":"Zikun Li Zhuofu Chen Remi Delacourt Gabriele Oliaro Zeyu Wang Qinghan Chen Shuhuai Lin April Yang Zhihao Zhang Zhuoming Chen Sean Lai Xinhao Cheng Xupeng Miao and Zhihao Jia. 2025. AdaServe: Accelerating Multi-SLO LLM Serving with SLO-Customized Speculative Decoding. arxiv:https:\/\/arXiv.org\/abs\/2501.12162\u00a0[cs.CL] https:\/\/arxiv.org\/abs\/2501.12162"},{"key":"e_1_3_3_2_21_2","unstructured":"Aixin Liu et\u00a0al. 2025. HAP: Hybrid Adaptive Parallelism for Efficient Mixture-of-Experts Inference. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2508.19373 (2025)."},{"key":"e_1_3_3_2_22_2","unstructured":"Qunyou Liu Darong Huang Marina Zapater and David Atienza. 2025. GreenLLM: SLO-Aware Dynamic Frequency Scaling for Energy-Efficient LLM Serving. arxiv:https:\/\/arXiv.org\/abs\/2508.16449\u00a0[cs.PF] https:\/\/arxiv.org\/abs\/2508.16449"},{"key":"e_1_3_3_2_23_2","unstructured":"MLCommons. 2025. MLPerf Inference Benchmarks Documentation. https:\/\/docs.mlcommons.org\/inference\/."},{"key":"e_1_3_3_2_24_2","unstructured":"MLCommons. 2025. MLPerf Inference v5.0 Advances Language Model Capabilities for GenAI. https:\/\/mlcommons.org\/2025\/04\/llm-inference-v5\/."},{"key":"e_1_3_3_2_25_2","unstructured":"Zizhao Mo Jianxiong Liao Huanle Xu Zhi Zhou and Chengzhong Xu. 2025. Hetis: Serving LLMs in Heterogeneous GPU Clusters with Fine-grained and Dynamic Parallelism. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2509.08309 (2025)."},{"key":"e_1_3_3_2_26_2","unstructured":"Vinh Nguyen Wenwen Gao Emily Apsey Ganesh Kudleppanavar Neelay Shah and Elias Bermudez. 2025. LLM Inference Benchmarking: Fundamental Concepts. https:\/\/developer.nvidia.com\/blog\/llm-benchmarking-fundamental-concepts\/."},{"key":"e_1_3_3_2_27_2","unstructured":"Vinh Nguyen Wenwen Gao Emily Apsey Ganesh Kudleppanavar Neelay Shah and Elias Bermudez. 2025. LLM Inference Benchmarking: Fundamental Concepts. https:\/\/developer.nvidia.com\/blog\/llm-benchmarking-fundamental-concepts\/."},{"key":"e_1_3_3_2_28_2","unstructured":"NVIDIA Corporation. 2017. NCCL: NVIDIA Collective Communications Library. https:\/\/github.com\/NVIDIA\/nccl."},{"key":"e_1_3_3_2_29_2","unstructured":"Oak Ridge Leadership Computing Facility (OLCF). [n. d.]. Defiant Quick-Start Guide. https:\/\/docs.olcf.ornl.gov\/ace_testbed\/defiant_quick_start_guide.html."},{"key":"e_1_3_3_2_30_2","unstructured":"OpenAI : Sandhini Agarwal et\u00a0al. 2025. gpt-oss-120b & gpt-oss-20b Model Card. arxiv:https:\/\/arXiv.org\/abs\/2508.10925\u00a0[cs.CL] https:\/\/arxiv.org\/abs\/2508.10925"},{"key":"e_1_3_3_2_31_2","unstructured":"Shantanu\u00a0Deepak Patankar and Hugo Affaticati. 2024. Optimizing Language Model Inference on Azure. https:\/\/techcommunity.microsoft.com\/blog\/azurehighperformancecomputingblog\/optimizing-language-model-inference-on-azure\/4248271."},{"key":"e_1_3_3_2_32_2","first-page":"606","volume-title":"Proceedings of Machine Learning and Systems","volume":"5","author":"Pope Reiner","year":"2023","unstructured":"Reiner Pope, Sholto Douglas, Aakanksha Chowdhery, Jacob Devlin, James Bradbury, Jonathan Heek, Kefan Xiao, Shivani Agrawal, and Jeff Dean. 2023. Efficiently Scaling Transformer Inference. In Proceedings of Machine Learning and Systems , D.\u00a0Song, M.\u00a0Carbin, and T.\u00a0Chen (Eds.), Vol.\u00a05. Curan, 606\u2013624. https:\/\/proceedings.mlsys.org\/paper_files\/paper\/2023\/file\/c4be71ab8d24cdfb45e3d06dbfca2780-Paper-mlsys2023.pdf"},{"key":"e_1_3_3_2_33_2","unstructured":"PyTorch Contributors. 2017. Gloo: Collective Communications Library. https:\/\/github.com\/pytorch\/gloo."},{"key":"e_1_3_3_2_34_2","unstructured":"Samyam Rajbhandari Mert Hidayetoglu Aurick Qiao Ye Wang Juncheng Yang Jeff Rasley Michael Wyatt and Yuxiong He. 2025. Arctic Inference with Shift Parallelism: Fast and Efficient Open Source Inference System for Enterprise AI. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2507.11830 (2025)."},{"key":"e_1_3_3_2_35_2","doi-asserted-by":"publisher","DOI":"10.1109\/ISCA45697.2020.00045"},{"key":"e_1_3_3_2_36_2","unstructured":"Mohammad Shoeybi Mostofa Patwary Raul Puri Patrick LeGresley Jared Casper and Bryan Catanzaro. 2020. Megatron-LM: Training Multi-Billion Parameter Language Models Using Model Parallelism. arxiv:https:\/\/arXiv.org\/abs\/1909.08053\u00a0[cs.CL] https:\/\/arxiv.org\/abs\/1909.08053"},{"key":"e_1_3_3_2_37_2","doi-asserted-by":"publisher","DOI":"10.1109\/HPCA61900.2025.00102"},{"key":"e_1_3_3_2_38_2","unstructured":"Qidong Su Wei Zhao Xin Li Muralidhar Andoorveedu Chenhao Jiang Zhanda Zhu Kevin Song Christina Giannoula and Gennady Pekhimenko. 2025. Seesaw: High-throughput LLM Inference via Model Re-sharding. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2503.06433 (2025)."},{"key":"e_1_3_3_2_39_2","unstructured":"Gemini Team Rohan Anil Sebastian Borgeaud Jean-Baptiste Alayrac Jiahui Yu Radu Soricut and et al.2025. Gemini: A Family of Highly Capable Multimodal Models. arxiv:https:\/\/arXiv.org\/abs\/2312.11805\u00a0[cs.CL] https:\/\/arxiv.org\/abs\/2312.11805"},{"key":"e_1_3_3_2_40_2","unstructured":"Kimi Team Yifan Bai Yiping Bao Guanduo Chen Jiahao Chen Ningxin Chen Ruijue Chen Yanru Chen Yuankun Chen Yutian Chen et\u00a0al. 2025. Kimi k2: Open agentic intelligence. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2507.20534 (2025)."},{"key":"e_1_3_3_2_41_2","unstructured":"The Prometheus Authors. 2025. Prometheus - Monitoring system & time series database. https:\/\/prometheus.io\/. https:\/\/prometheus.io\/"},{"key":"e_1_3_3_2_42_2","unstructured":"Hugo Touvron Thibaut Lavril Gautier Izacard Xavier Martinet Marie-Anne Lachaux Timoth\u00e9e Lacroix Baptiste Rozi\u00e8re Naman Goyal Eric Hambro Faisal Azhar et\u00a0al. 2023. Llama: Open and efficient foundation language models. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2302.13971 (2023)."},{"key":"e_1_3_3_2_43_2","doi-asserted-by":"publisher","DOI":"10.1145\/3694715.3695975"},{"key":"e_1_3_3_2_44_2","unstructured":"Xingyao Wang Yangyi Chen Lifan Yuan Yizhe Zhang Yunzhu Li Hao Peng and Heng Ji. 2024. Executable Code Actions Elicit Better LLM Agents. arxiv:https:\/\/arXiv.org\/abs\/2402.01030\u00a0[cs.CL]"},{"key":"e_1_3_3_2_45_2","doi-asserted-by":"publisher","DOI":"10.1145\/3711896.3737413"},{"key":"e_1_3_3_2_46_2","unstructured":"Bingyang Wu et\u00a0al. 2023. Fast Distributed Inference Serving for Large Language Models. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2305.05920 (2023)."},{"key":"e_1_3_3_2_47_2","doi-asserted-by":"publisher","DOI":"10.1145\/3694715.3695948"},{"key":"e_1_3_3_2_48_2","unstructured":"Chejian Xu Wei Ping Peng Xu Zihan Liu Boxin Wang Mohammad Shoeybi Bo Li and Bryan Catanzaro. 2025. From 128K to 4M: Efficient Training of Ultra-Long Context Large Language Models. arxiv:https:\/\/arXiv.org\/abs\/2504.06214\u00a0[cs.CL] https:\/\/arxiv.org\/abs\/2504.06214"}],"event":{"name":"ICS '26: 2026 International Conference on Supercomputing","location":"Belfast United Kingdom","acronym":"ICS '26","sponsor":["SIGHPC ACM Special Interest Group on High Performance Computing, Special Interest Group on High Performance Computing","SIGARCH ACM Special Interest Group on Computer Architecture"]},"container-title":["Proceedings of the 40th ACM International Conference on Supercomputing"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/abs\/10.1145\/3797905.3800525","content-type":"text\/html","content-version":"vor","intended-application":"syndication"}],"deposited":{"date-parts":[[2026,7,2]],"date-time":"2026-07-02T13:03:45Z","timestamp":1782997425000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3797905.3800525"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026,7,5]]},"references-count":47,"alternative-id":["10.1145\/3797905.3800525","10.1145\/3797905"],"URL":"https:\/\/doi.org\/10.1145\/3797905.3800525","relation":{},"subject":[],"published":{"date-parts":[[2026,7,5]]},"assertion":[{"value":"2026-07-05","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}