{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,3,15]],"date-time":"2026-03-15T15:30:23Z","timestamp":1773588623759,"version":"3.50.1"},"publisher-location":"New York, NY, USA","reference-count":55,"publisher":"ACM","funder":[{"name":"National Key Research and Development Program of China","award":["2024YFB4505703"],"award-info":[{"award-number":["2024YFB4505703"]}]},{"name":"National Natural Science Foundation of China","award":["62232011"],"award-info":[{"award-number":["62232011"]}]},{"name":"Natural Science Foundation of Shanghai Municipality","award":["24ZR1430500"],"award-info":[{"award-number":["24ZR1430500"]}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2026,3,22]]},"DOI":"10.1145\/3779212.3790236","type":"proceedings-article","created":{"date-parts":[[2026,3,10]],"date-time":"2026-03-10T13:55:26Z","timestamp":1773150926000},"page":"2030-2047","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":0,"title":["Towards High-Goodput LLM Serving with Prefill-decode Multiplexing"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0009-0007-7719-1377","authenticated-orcid":false,"given":"Yukang","family":"Chen","sequence":"first","affiliation":[{"name":"Shanghai Jiao Tong University, Shanghai, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-6646-5260","authenticated-orcid":false,"given":"Weihao","family":"Cui","sequence":"additional","affiliation":[{"name":"Shanghai Jiao Tong University, Shanghai, China and National University of Singapore, Singapore, Singapore"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-1561-5329","authenticated-orcid":false,"given":"Han","family":"Zhao","sequence":"additional","affiliation":[{"name":"Shanghai Jiao Tong University, Shanghai, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0009-0000-4411-9773","authenticated-orcid":false,"given":"Ziyi","family":"Xu","sequence":"additional","affiliation":[{"name":"Shanghai Jiao Tong University, Shanghai, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0009-0004-8742-606X","authenticated-orcid":false,"given":"Xiaoze","family":"Fan","sequence":"additional","affiliation":[{"name":"Shanghai Jiao Tong University, Shanghai, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-2807-9780","authenticated-orcid":false,"given":"Xusheng","family":"Chen","sequence":"additional","affiliation":[{"name":"Researcher, Shanghai, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-3652-5437","authenticated-orcid":false,"given":"Yangjie","family":"Zhou","sequence":"additional","affiliation":[{"name":"National University of Singapore, Singapore, Singapore"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-4060-9438","authenticated-orcid":false,"given":"Shixuan","family":"Sun","sequence":"additional","affiliation":[{"name":"Shanghai Jiao Tong University, Shanghai, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-8618-4581","authenticated-orcid":false,"given":"Bingsheng","family":"He","sequence":"additional","affiliation":[{"name":"National University of Singapore, Singapore, Singapore"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-5832-0347","authenticated-orcid":false,"given":"Quan","family":"Chen","sequence":"additional","affiliation":[{"name":"Shanghai Jiao Tong University, Shanghai, China"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2026,3,22]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"publisher","DOI":"10.5555\/3691938.3691945"},{"key":"e_1_3_2_1_2_1","volume-title":"SARATHI: Efficient LLM, Inference by Piggybacking Decodes with Chunked Prefills. arXiv:2308.16369 (Aug.","author":"Agrawal Amey","year":"2023","unstructured":"Amey Agrawal, Ashish Panwar, Jayashree Mohan, Nipun Kwatra, Bhargav S. Gulavani, and Ramachandran Ramjee. 2023. SARATHI: Efficient LLM, Inference by Piggybacking Decodes with Chunked Prefills. arXiv:2308.16369 (Aug. 2023). arXiv:2308.16369 [cs]"},{"key":"e_1_3_2_1_3_1","unstructured":"Michael Andersch Greg Palmer Ronny Krashinsky Nick Stam Vishal Mehta Gonzalo Brito and Sridhar Ramaswamy. 2025. NVIDIA Hopper Architecture In-Depth - Thread block clusters. https:\/\/developer.nvidia.com\/blog\/nvidia-hopper-architecture-in-depth\/##thread_block_clusters. Accessed 2026-01-10."},{"key":"e_1_3_2_1_4_1","unstructured":"anon8231489123. 2023. ShareGPT Vicuna Unfiltered \u2013 Cleaned Split (v3). https:\/\/huggingface.co\/datasets\/anon8231489123\/ShareGPT_Vicuna_unfiltered\/resolve\/main\/ShareGPT_V3_unfiltered_cleaned_split.json. Accessed: 2025-04-16."},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","unstructured":"Jinze Bai Shuai Bai Yunfei Chu Zeyu Cui Kai Dang Xiaodong Deng Yang Fan Wenbin Ge Yu Han Fei Huang Binyuan Hui Luo Ji Mei Li Junyang Lin Runji Lin and et al. 2023. Qwen Technical Report. arXiv:2309.16609 (Sept. 2023). arXiv:2309.16609 [cs] doi:10.48550\/arXiv.2309.16609","DOI":"10.48550\/arXiv.2309.16609"},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"crossref","unstructured":"Yushi Bai Shangqing Tu Jiajie Zhang Hao Peng Xiaozhi Wang Xin Lv Shulin Cao Jiazheng Xu Lei Hou Yuxiao Dong et al. 2024. LongBench v2: Towards deeper understanding and reasoning on realistic long-context multitasks. arXiv preprint arXiv:2412.15204 (2024).","DOI":"10.18653\/v1\/2025.acl-long.183"},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"publisher","DOI":"10.1109\/RTAS58335.2023.00012"},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.1145\/2872362.2872368"},{"key":"e_1_3_2_1_9_1","first-page":"199","volume-title":"2022 USENIX Annual Technical Conference (USENIX ATC 22)","author":"Choi Seungbeom","year":"2022","unstructured":"Seungbeom Choi, Sunho Lee, Yeonjae Kim, Jongse Park, Youngjin Kwon, and Jaehyuk Huh. 2022. Serving heterogeneous machine learning models on servers with sharing. In 2022 USENIX Annual Technical Conference (USENIX ATC 22). 199-216."},{"key":"e_1_3_2_1_10_1","unstructured":"NVIDIA Corporation. 2025a. CUDA Driver API: Green Contexts. https:\/\/docs.nvidia.com\/cuda\/cuda-driver-api\/group__CUDA__GREEN__CONTEXTS.html. Accessed: 2025-03-29."},{"key":"e_1_3_2_1_11_1","unstructured":"NVIDIA Corporation. 2025b. CUDA Runtime API: Stream Management. https:\/\/docs.nvidia.com\/cuda\/cuda-runtime-api\/group__CUDART__STREAM.html. Accessed: 2025-03-30."},{"key":"e_1_3_2_1_12_1","unstructured":"NVIDIA Corporation. 2025c. Multi-Instance GPU (MIG). https:\/\/www.nvidia.com\/en-sg\/technologies\/multi-instance-gpu\/. Accessed: 2025-03-30."},{"key":"e_1_3_2_1_13_1","unstructured":"NVIDIA Corporation. 2025d. Multi-Process Service. Version 570."},{"key":"e_1_3_2_1_14_1","unstructured":"NVIDIA Corporation. 2025 e. NVIDIA Dynamo: A Datacenter Scale Distributed Inference Serving Framework. https:\/\/github.com\/ai-dynamo\/dynamo. Accessed: 2025-04-07."},{"key":"e_1_3_2_1_15_1","unstructured":"Abhimanyu Dubey Abhinav Jauhri Abhinav Pandey Abhishek Kadian Ahmad Al-Dahle Aiesha Letman Akhil Mathur Alan Schelten Amy Yang Angela Fan Anirudh Goyal Anthony Hartshorn Aobo Yang Archi Mitra Archie Sravankumar and et al. 2024. The Llama 3 Herd of Models. arXiv:2407.21783 (Aug. 2024). arXiv:2407.21783"},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.1145\/3695053.3730999"},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","unstructured":"Etash Guha Ryan Marten Sedrick Keh Negin Raoof Georgios Smyrnis Hritik Bansal Marianna Nezhurina Jean Mercat Trung Vu Zayne Sprague Ashima Suvarna Benjamin Feuer Liangyu Chen Zaid Khan Eric Frankel Sachin Grover Caroline Choi Niklas Muennighoff Shiye Su Wanjia Zhao John Yang Shreyas Pimpalgaonkar Kartik Sharma Charlie Cheng-Jie Ji Yichuan Deng Sarah Pratt Vivek Ramanujan Jon Saad-Falcon Jeffrey Li Achal Dave Alon Albalak Kushal Arora Blake Wulfe Chinmay Hegde Greg Durrett Sewoong Oh Mohit Bansal Saadia Gabriel Aditya Grover Kai-Wei Chang Vaishaal Shankar Aaron Gokaslan Mike A. Merrill Tatsunori Hashimoto Yejin Choi Jenia Jitsev Reinhard Heckel Maheswaran Sathiamoorthy Alexandros G. Dimakis and Ludwig Schmidt. 2025. OpenThoughts: data recipes for reasoning models. doi:10.48550\/arXiv.2506.04178 arXiv:2506.04178 [cs].","DOI":"10.48550\/arXiv.2506.04178"},{"key":"e_1_3_2_1_18_1","unstructured":"Ke Hong Lufang Chen Zhong Wang Xiuhong Li Qiuli Mao Jianping Ma Chao Xiong Guanyu Wu Buhe Han Guohao Dai Yun Liang and Yu Wang. 2025. semi-PD: Towards Efficient LLM Serving via Phase-Wise Disaggregated Computation and Unified Storage. arXiv:2504.19867 [cs.CL] https:\/\/arxiv.org\/abs\/2504.19867"},{"key":"e_1_3_2_1_19_1","volume-title":"Cursor: The AI Code Editor","author":"Anysphere Inc.","year":"2025","unstructured":"Anysphere Inc. 2025. Cursor: The AI Code Editor. https:\/\/www.cursor.com\/. Accessed: 2025-04-05."},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.1109\/HPCA.2018.00034"},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.1145\/3676641.3715996"},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10586-021-03429-7"},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.1145\/3600006.3613165"},{"key":"e_1_3_2_1_24_1","unstructured":"Patrick Lewis Ethan Perez Aleksandra Piktus Fabio Petroni Vladimir Karpukhin Naman Goyal Heinrich K\u00fcttler Mike Lewis Wen-tau Yih Tim Rockt\u00e4schel et al. 2020. Retrieval-augmented generation for knowledge-intensive nlp tasks. Advances in neural information processing systems Vol. 33 (2020) 9459-9474."},{"key":"e_1_3_2_1_25_1","unstructured":"Jiaqi Li Mengmeng Wang Zilong Zheng and Muhan Zhang. 2024. LooGLE: Can Long-Context Language Models Understand Long Contexts? arXiv:2311.04939 [cs.CL] https:\/\/arxiv.org\/abs\/2311.04939"},{"key":"e_1_3_2_1_26_1","first-page":"929","volume-title":"Applications with Semantic Variable. In 18th USENIX Symposium, on Operating Systems Design, and Implementation (OSDI, 24)","author":"Lin Chaofan","year":"2024","unstructured":"Chaofan Lin, Zhenhua Han, Chengruidong Zhang, Yuqing Yang, Fan Yang, Chen Chen, and Lili Qiu. 2024. Parrot: Efficient Serving of LLM-based, Applications with Semantic Variable. In 18th USENIX Symposium, on Operating Systems Design, and Implementation (OSDI, 24). 929-945."},{"key":"e_1_3_2_1_27_1","doi-asserted-by":"publisher","DOI":"10.1109\/CloudCom.2019.00025"},{"key":"e_1_3_2_1_28_1","unstructured":"Zejia Lin Hongxin Xu Guanyi Chen Zhiguang Chen Yutong Lu and Xianwei Zhang. 2025. Boosting LLM Serving through Spatial-Temporal GPU Resource Sharing. arXiv:2504.19516 [cs.DC] https:\/\/arxiv.org\/abs\/2504.19516"},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.1109\/DAC63849.2025.11132617"},{"key":"e_1_3_2_1_30_1","unstructured":"OpenAI. 2025. ChatGPT. https:\/\/chatgpt.com\/. Accessed: 2025-04-05."},{"key":"e_1_3_2_1_31_1","first-page":"8026","volume-title":"Proceedings of the 33rd International Conference, on Neural Information Processing Systems. Curran Associates Inc.","author":"Paszke Adam","unstructured":"Adam Paszke, Sam Gross, Francisco Massa, Adam Lerer, James Bradbury, Gregory Chanan, Trevor Killeen, Zeming Lin, Natalia Gimelshein, Luca Antiga, Alban Desmaison, Andreas K\u00f6pf, Edward Yang, Zach DeVito, Martin Raison, and et al., 2019. PyTorch: An Imperative Style, High-Performance Deep Learning Library. In Proceedings of the 33rd International Conference, on Neural Information Processing Systems. Curran Associates Inc., Red Hook, NY, USA, 8026-8037."},{"key":"e_1_3_2_1_32_1","doi-asserted-by":"publisher","DOI":"10.1109\/ISCA59077.2024.00019"},{"key":"e_1_3_2_1_33_1","volume-title":"Fan Yang, and Mao Yang.","author":"Qi Zhenting","year":"2024","unstructured":"Zhenting Qi, Mingyuan Ma, Jiahang Xu, Li Lyna Zhang, Fan Yang, and Mao Yang. 2024. Mutual Reasoning Makes Smaller LLMs, Stronger Problem-Solvers. arXiv:2408.06195 (Aug. 2024). arXiv:2408.06195"},{"key":"e_1_3_2_1_34_1","first-page":"155","volume-title":"Chatbot. In 23rd USENIX Conference, on File, and Storage Technologies (FAST, 25)","author":"Qin Ruoyu","year":"2025","unstructured":"Ruoyu Qin, Zheming Li, Weiran He, Jialei Cui, Feng Ren, Mingxing Zhang, Yongwei Wu, Weimin Zheng, and Xinran Xu. 2025. Mooncake: Trading More Storage for Less Computation -- a KVCache-centric, Architecture for Serving LLM, Chatbot. In 23rd USENIX Conference, on File, and Storage Technologies (FAST, 25). 155-170."},{"key":"e_1_3_2_1_35_1","doi-asserted-by":"publisher","DOI":"10.1109\/HPCA61900.2025.00102"},{"key":"e_1_3_2_1_36_1","doi-asserted-by":"publisher","DOI":"10.1145\/3627703.3629578"},{"key":"e_1_3_2_1_37_1","doi-asserted-by":"publisher","DOI":"10.48550\/arXiv.2302.13971"},{"key":"e_1_3_2_1_38_1","unstructured":"Hugo Touvron Louis Martin Kevin Stone Peter Albert Amjad Almahairi Yasmine Babaei Nikolay Bashlykov Soumya Batra Prajjwal Bhargava Shruti Bhosale Dan Bikel Lukas Blecher Cristian Canton Ferrer Moya Chen Guillem Cucurull David Esiobu Jude Fernandes Jeremy Fu Wenyin Fu Brian Fuller Cynthia Gao Vedanuj Goswami Naman Goyal Anthony Hartshorn Saghar Hosseini Rui Hou Hakan Inan Marcin Kardas Viktor Kerkez Madian Khabsa Isabel Kloumann Artem Korenev Punit Singh Koura Marie-Anne Lachaux Thibaut Lavril Jenya Lee Diana Liskovich Yinghai Lu Yuning Mao Xavier Martinet Todor Mihaylov Pushkar Mishra Igor Molybog Yixin Nie Andrew Poulton Jeremy Reizenstein Rashi Rungta Kalyan Saladi Alan Schelten Ruan Silva Eric Michael Smith Ranjan Subramanian Xiaoqing Ellen Tan Binh Tang Ross Taylor Adina Williams Jian Xiang Kuan Puxin Xu Zheng Yan Iliyan Zarov Yuchen Zhang Angela Fan Melanie Kambadur Sharan Narang Aurelien Rodriguez Robert Stojnic Sergey Edunov and Thomas Scialom. 2023b. Llama 2: Open Foundation and Fine-Tuned Chat Models. arXiv:2307.09288 [cs.CL] https:\/\/arxiv.org\/abs\/2307.09288"},{"key":"e_1_3_2_1_39_1","doi-asserted-by":"publisher","DOI":"10.48550\/arXiv.1706.03762"},{"key":"e_1_3_2_1_40_1","unstructured":"vLLM Team. [n.d.]. CUDA Graphs \u2014 vLLM Design Documentation. https:\/\/docs.vllm.ai\/en\/stable\/design\/cuda_graphs\/. Accessed: 2026-01-09."},{"key":"e_1_3_2_1_41_1","unstructured":"Jiahao Wang Jinbo Han Xingda Wei Sijie Shen Dingyan Zhang Chenguang Fang Rong Chen Wenyuan Yu and Haibo Chen. 2025a. KVCache cache in the wild: characterizing and optimizing KVCache cache at a large cloud provider. 465-482. https:\/\/www.usenix.org\/conference\/atc25\/presentation\/wang-jiahao"},{"key":"e_1_3_2_1_42_1","unstructured":"Zhibin Wang Shipeng Li Yuhang Zhou Xue Li Zhonghui Zhang Nguyen Cam-Tu Rong Gu Chen Tian Guihai Chen and Sheng Zhong. 2025b. Revisiting Service Level Objectives and System Level Metrics in Large Language Model Serving. arXiv:2410.14257 [cs.LG] https:\/\/arxiv.org\/abs\/2410.14257"},{"key":"e_1_3_2_1_43_1","volume-title":"Quoc V Le, and Denny Zhou.","author":"Wei Jason","year":"2022","unstructured":"Jason Wei, Xuezhi Wang, Dale Schuurmans, Maarten Bosma, Brian Ichter, Fei Xia, Ed H Chi, Quoc V Le, and Denny Zhou. 2022. Chain-of-Thought Prompting Elicits Reasoning in Large Language Models. Advances in neural information processing systems, Vol. 35 (2022), 24824-24837."},{"key":"e_1_3_2_1_44_1","doi-asserted-by":"publisher","DOI":"10.1145\/3694715.3695948"},{"key":"e_1_3_2_1_45_1","doi-asserted-by":"publisher","DOI":"10.1145\/3689031.3696098"},{"key":"e_1_3_2_1_46_1","volume-title":"Inference Serving. In Eighth Conference, on Machine Learning, and Systems.","author":"Ye Zihao","year":"2025","unstructured":"Zihao Ye, Lequn Chen, Ruihang Lai, Wuwei Lin, Yineng Zhang, Stephanie Wang, Tianqi Chen, Baris Kasikci, Vinod Grover, Arvind Krishnamurthy, and Luis Ceze. 2025. FlashInfer: Efficient and Customizable Attention Engine for LLM, Inference Serving. In Eighth Conference, on Machine Learning, and Systems."},{"key":"e_1_3_2_1_47_1","first-page":"521","volume-title":"Orca: A Distributed Serving System for Transformer-Based Generative Models. In 16th USENIX Symposium, on Operating Systems Design, and Implementation (OSDI, 22)","author":"Yu Gyeong-In","year":"2022","unstructured":"Gyeong-In Yu, Joo Seong Jeong, Geon-Woo Kim, Soojeong Kim, and Byung-Gon Chun. 2022. Orca: A Distributed Serving System for Transformer-Based Generative Models. In 16th USENIX Symposium, on Operating Systems Design, and Implementation (OSDI, 22). 521-538."},{"key":"e_1_3_2_1_48_1","doi-asserted-by":"publisher","DOI":"10.1145\/3689031.3696070"},{"key":"e_1_3_2_1_49_1","doi-asserted-by":"publisher","DOI":"10.1145\/3330345.3330351"},{"key":"e_1_3_2_1_50_1","first-page":"1371","volume-title":"HSM: A Hybrid Slowdown Model for Multitasking GPUs. In Proceedings of the Twenty-Fifth International Conference on Architectural Support for Programming Languages and Operating Systems (ASPLOS","author":"Zhao Xia","year":"2022","unstructured":"Xia Zhao, Magnus Jahre, and Lieven Eeckhout. [n.d.]. HSM: A Hybrid Slowdown Model for Multitasking GPUs. In Proceedings of the Twenty-Fifth International Conference on Architectural Support for Programming Languages and Operating Systems (ASPLOS 2022). 1371-1385."},{"key":"e_1_3_2_1_51_1","first-page":"559","volume-title":"Alpa: Automating Inter- and Intra-Operator Parallelism for Distributed Deep Learning. In 16th USENIX Symposium, on Operating Systems Design, and Implementation (OSDI, 22)","author":"Zheng Lianmin","year":"2022","unstructured":"Lianmin Zheng, Zhuohan Li, Hao Zhang, Yonghao Zhuang, Zhifeng Chen, Yanping Huang, Yida Wang, Yuanzhong Xu, Danyang Zhuo, Eric P. Xing, Joseph E. Gonzalez, and Ion Stoica. 2022. Alpa: Automating Inter- and Intra-Operator Parallelism for Distributed Deep Learning. In 16th USENIX Symposium, on Operating Systems Design, and Implementation (OSDI, 22). 559-578."},{"key":"e_1_3_2_1_52_1","volume-title":"Proceedings of the 38th International Conference on Neural Information Processing Systems","author":"Zheng Lianmin","year":"2024","unstructured":"Lianmin Zheng, Liangsheng Yin, Zhiqiang Xie, Chuyue Sun, Jeff Huang, Cody Hao Yu, Shiyi Cao, Christos Kozyrakis, Ion Stoica, Joseph E. Gonzalez, Clark Barrett, and Ying Sheng. 2024. SGLang: efficient execution of structured language model programs. In Proceedings of the 38th International Conference on Neural Information Processing Systems (Vancouver, BC, Canada) (NIPS '24). Curran Associates Inc., Red Hook, NY, USA, Article 2000, 27 pages."},{"key":"e_1_3_2_1_53_1","first-page":"193","volume-title":"DistServe: Disaggregating Prefill and Decoding for Goodput-Optimized Large Language Model Serving. In 18th USENIX Symposium, on Operating Systems Design, and Implementation (OSDI, 24)","author":"Zhong Yinmin","year":"2024","unstructured":"Yinmin Zhong, Shengyu Liu, Junda Chen, Jianbo Hu, Yibo Zhu, Xuanzhe Liu, Xin Jin, and Hao Zhang. 2024. DistServe: Disaggregating Prefill and Decoding for Goodput-Optimized Large Language Model Serving. In 18th USENIX Symposium, on Operating Systems Design, and Implementation (OSDI, 24). 193-210."},{"key":"e_1_3_2_1_54_1","first-page":"749","article-title":"NanoFlow","author":"Zhu Kan","year":"2025","unstructured":"Kan Zhu, Yufei Gao, Yilong Zhao, Liangyu Zhao, Gefei Zuo, Yile Gu, Dedong Xie, Zihao Ye, Keisuke Kamahori, Chien-Yu Lin, Ziren Wang, Stephanie Wang, Arvind Krishnamurthy, and Baris Kasikci. 2025. NanoFlow: Towards Optimal Large Language Model Serving Throughput. 749-765. https:\/\/www.usenix.org\/conference\/osdi25\/presentation\/zhu-kan","journal-title":"Towards Optimal Large Language Model Serving Throughput."},{"key":"e_1_3_2_1_55_1","volume-title":"Hany Hassan, Ruofei Zhang, Tuo Zhao, and Jianfeng Gao.","author":"Zuo Simiao","year":"2022","unstructured":"Simiao Zuo, Xiaodong Liu, Jian Jiao, Young Jin Kim, Hany Hassan, Ruofei Zhang, Tuo Zhao, and Jianfeng Gao. 2022. Taming Sparsely Activated Transformer with Stochastic Experts. arXiv:2110.04260 (Feb. 2022). arXiv:2110.04"}],"event":{"name":"ASPLOS '26: 31st ACM International Conference on Architectural Support for Programming Languages and Operating Systems","location":"Pittsburgh PA USA","sponsor":["SIGOPS ACM Special Interest Group on Operating Systems","SIGPLAN ACM Special Interest Group on Programming Languages","SIGARCH ACM Special Interest Group on Computer Architecture","SIGBED ACM Special Interest Group on Embedded Systems"]},"container-title":["Proceedings of the 31st ACM International Conference on Architectural Support for Programming Languages and Operating Systems, Volume 2"],"original-title":[],"deposited":{"date-parts":[[2026,3,15]],"date-time":"2026-03-15T13:58:31Z","timestamp":1773583111000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3779212.3790236"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026,3,22]]},"references-count":55,"alternative-id":["10.1145\/3779212.3790236","10.1145\/3779212"],"URL":"https:\/\/doi.org\/10.1145\/3779212.3790236","relation":{},"subject":[],"published":{"date-parts":[[2026,3,22]]},"assertion":[{"value":"2026-03-22","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}