{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,9,16]],"date-time":"2026-09-16T19:01:59Z","timestamp":1789585319332,"version":"build-2803163510"},"publisher-location":"New York, NY, USA","reference-count":41,"publisher":"ACM","license":[{"start":{"date-parts":[[2026,9,27]],"date-time":"2026-09-27T00:00:00Z","timestamp":1790467200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/legalcode"}],"funder":[{"DOI":"10.13039\/100000001","name":"National Science Foundation","doi-asserted-by":"publisher","award":["NSF-2421782"],"award-info":[{"award-number":["NSF-2421782"]}],"id":[{"id":"10.13039\/100000001","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000001","name":"National Science Foundation","doi-asserted-by":"publisher","award":["NSF-2350425"],"award-info":[{"award-number":["NSF-2350425"]}],"id":[{"id":"10.13039\/100000001","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000001","name":"National Science Foundation","doi-asserted-by":"publisher","award":["NSF-2319988"],"award-info":[{"award-number":["NSF-2319988"]}],"id":[{"id":"10.13039\/100000001","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000001","name":"National Science Foundation","doi-asserted-by":"publisher","award":["NSF-2206522"],"award-info":[{"award-number":["NSF-2206522"]}],"id":[{"id":"10.13039\/100000001","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100006112","name":"Microsoft Research","doi-asserted-by":"publisher","award":["Microsoft Research Faculty Fellowship 8300751"],"award-info":[{"award-number":["Microsoft Research Faculty Fellowship 8300751"]}],"id":[{"id":"10.13039\/100006112","id-type":"DOI","asserted-by":"publisher"}]},{"name":"Amazon research","award":["Amazon research award"],"award-info":[{"award-number":["Amazon research award"]}]},{"DOI":"10.13039\/100008536","name":"Amazon Web Services","doi-asserted-by":"publisher","award":["AWS Cloud Credit for Research"],"award-info":[{"award-number":["AWS Cloud Credit for Research"]}],"id":[{"id":"10.13039\/100008536","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2026,9,28]]},"DOI":"10.1145\/3832810.3832922","type":"proceedings-article","created":{"date-parts":[[2026,9,13]],"date-time":"2026-09-13T00:19:06Z","timestamp":1789258746000},"page":"208-217","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":0,"title":["Heterogeneous SLO Guaranteed Multi-Resource-Aware Batching in LLM Serving"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-7548-6223","authenticated-orcid":false,"given":"Haiying","family":"Shen","sequence":"first","affiliation":[{"name":"University of Virginia, Charlottesville, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-7677-3358","authenticated-orcid":false,"given":"Tanmoy","family":"Sen","sequence":"additional","affiliation":[{"name":"University of Virginia, Charlottesville, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-0478-8854","authenticated-orcid":false,"given":"Yuxiong","family":"He","sequence":"additional","affiliation":[{"name":"Snowflake, Bellevue, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2026,9,27]]},"reference":[{"key":"e_1_3_3_1_2_2","unstructured":"[n.d.]. OpenAI API Documentation. https:\/\/help.openai.com\/en\/articles\/ 4936856-what-are-tokens-and-how-to-count-them."},{"key":"e_1_3_3_1_3_2","unstructured":"2023. [FEATURE] Implement Dynamic SplitFuse #1562. https:\/\/github.com\/vllm-project\/vllm\/issues\/1562."},{"key":"e_1_3_3_1_4_2","volume-title":"Proc. of MLSys","author":"Agrawal Amey","year":"2024","unstructured":"Amey Agrawal, Nitin Kedia, Jayashree Mohan, Ashish Panwar, Nipun Kwatra, Bhargav Gulavani, Ramachandran Ramjee, and Alexey Tumanov. 2024. VIDUR: A Large-Scale Simulation Framework for LLM Inference. In Proc. of MLSys."},{"key":"e_1_3_3_1_5_2","first-page":"117","volume-title":"18th USENIX Symposium on Operating Systems Design and Implementation (OSDI 24)","author":"Agrawal Amey","year":"2024","unstructured":"Amey Agrawal, Nitin Kedia, Ashish Panwar, Jayashree Mohan, Nipun Kwatra, Bhargav Gulavani, Alexey Tumanov, and Ramachandran Ramjee. 2024. Taming Throughput-Latency Tradeoff in LLM Inference with Sarathi-Serve. In 18th USENIX Symposium on Operating Systems Design and Implementation (OSDI 24). USENIX Association, Santa Clara, CA, 117\u2013134."},{"key":"e_1_3_3_1_6_2","doi-asserted-by":"crossref","unstructured":"Marc Brysbaert. 2019. How many words do we read per minute? A review and meta-analysis of reading rate. Journal of Memory and Language 109 (08 2019).","DOI":"10.1016\/j.jml.2019.104047"},{"key":"e_1_3_3_1_7_2","unstructured":"Shweta Jain and Saurabh Jain. 2016. Analysis of Multi Level Feedback Queue Scheduling Using Markov Chain Model with Data Model Approach. International Journal of Advanced Networking and Applications 7 (06 2016) 2915\u20132924."},{"key":"e_1_3_3_1_8_2","doi-asserted-by":"publisher","DOI":"10.52202\/079017-1663"},{"key":"e_1_3_3_1_9_2","doi-asserted-by":"publisher","DOI":"10.52202\/075280-0791"},{"key":"e_1_3_3_1_10_2","unstructured":"Jongwook Choi. [n.d.]. gpustat. https:\/\/github.com\/wookayin\/gpustat. Online; accessed 26 August 2023."},{"key":"e_1_3_3_1_11_2","unstructured":"Sosuke Kobayashi. 2018. Homemade BookCorpus. https:\/\/github.com\/soskek\/bookcorpus."},{"key":"e_1_3_3_1_12_2","doi-asserted-by":"publisher","DOI":"10.1145\/3600006.3613165"},{"key":"e_1_3_3_1_13_2","volume-title":"Proc. of OSDI","author":"Lee Wonbeom","year":"2024","unstructured":"Wonbeom Lee, Jungi Lee, Junghwan Seo, and Jaewoong Sim. 2024. InfiniGen: Efficient Generative Inference of Large Language Models with Dynamic KV Cache Management. In Proc. of OSDI."},{"key":"e_1_3_3_1_14_2","volume-title":"USENIX Symposium on Operating Systems Design and Implementation","author":"Li Zhuohan","year":"2023","unstructured":"Zhuohan Li, Lianmin Zheng, Yinmin Zhong, Vincent Liu, Ying Sheng, Xin Jin, Yanping Huang, Z. Chen, Hao Zhang, Joseph\u00a0E. Gonzalez, and Ion Stoica. 2023. AlpaServe: Statistical Multiplexing with Model Parallelism for Deep Learning Serving. In USENIX Symposium on Operating Systems Design and Implementation."},{"key":"e_1_3_3_1_15_2","unstructured":"Lianmin Zheng Liangsheng Yin Zhiqiang Xie Jeff Huang Chuyue Sun Cody Hao Yu Shiyi Cao Christos Kozyrakis Ion Stoica Joseph E. Gonzalez Clark Barrett Ying Sheng. [n.d.]. Fast and Expressive LLM Inference with RadixAttention and SGLang. https:\/\/lmsys.org\/blog\/2024-01-17-sglang\/."},{"key":"e_1_3_3_1_16_2","first-page":"929","volume-title":"18th USENIX Symposium on Operating Systems Design and Implementation (OSDI 24)","author":"Lin Chaofan","year":"2024","unstructured":"Chaofan Lin et\u00a0al. 2024. Parrot: Efficient Serving of LLM-based Applications with Semantic Variable. In 18th USENIX Symposium on Operating Systems Design and Implementation (OSDI 24). USENIX Association, Santa Clara, CA, 929\u2013945."},{"key":"e_1_3_3_1_17_2","doi-asserted-by":"publisher","DOI":"10.52202\/075280-2279"},{"key":"e_1_3_3_1_18_2","volume-title":"Proc. of ICML","author":"Liu Zichang","year":"2023","unstructured":"Zichang Liu, Jue Wang, Tri Dao, Tianyi Zhou, Binhang Yuan, Zhao Song, Anshumali Shrivastava, Ce Zhang, Yuandong Tian, Christopher Re, and Beidi Chen. 2023. Deja vu: Contextual sparsity for efficient llms at inference time. In Proc. of ICML."},{"key":"e_1_3_3_1_19_2","unstructured":"MicroSoft. 2023. DeepSpeed-FastGen: High-throughput Text Generation for LLMs via MII and DeepSpeed-Inference. https:\/\/github.com\/microsoft\/DeepSpeed\/tree\/master\/blogs\/deepspeed-fastgen."},{"key":"e_1_3_3_1_20_2","doi-asserted-by":"publisher","DOI":"10.1145\/3620665.3640383"},{"key":"e_1_3_3_1_21_2","doi-asserted-by":"crossref","unstructured":"Pratyush Patel Esha Choukse Chaojie Zhang \u00cd\u00f1igo Goiri Aashaka Shah Saeed Maleki and Ricardo Bianchini. 2024. Splitwise: Efficient generative LLM inference using phase splitting. Proc. of ISCA (2024).","DOI":"10.1109\/ISCA59077.2024.00019"},{"key":"e_1_3_3_1_22_2","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2021.eacl-main.24"},{"key":"e_1_3_3_1_23_2","volume-title":"Proc. of the 21st ACM European Conference on Computer Systems (EuroSys)","author":"A.\u00a0Goel D.\u00a0Z. Tootaghaj K. Diab H. Soni K. K. Ramakrishnan P.\u00a0Sharma S.\u00a0S.\u00a0Shubha,","year":"2026","unstructured":"D.\u00a0Z. Tootaghaj K. Diab H. Soni K. K. Ramakrishnan P.\u00a0Sharma S.\u00a0S.\u00a0Shubha, A.\u00a0Goel and H. Shen. 2026. AdaGen: Workload-Adaptive Cluster Scheduler for Latency-Optimal LLM Inference Serving. In Proc. of the 21st ACM European Conference on Computer Systems (EuroSys)."},{"key":"e_1_3_3_1_24_2","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/P17-1099"},{"key":"e_1_3_3_1_25_2","unstructured":"ShareGPT. 2023. sharegpt-english. https:\/\/huggingface.co\/datasets\/theblackcat102\/sharegpt-english."},{"key":"e_1_3_3_1_26_2","first-page":"965","volume-title":"18th USENIX Symposium on Operating Systems Design and Implementation (OSDI 24)","author":"Sheng Ying","year":"2024","unstructured":"Ying Sheng, Shiyi Cao, Dacheng Li, Banghua Zhu, Zhuohan Li, Danyang Zhuo, Joseph\u00a0E Gonzalez, and Ion Stoica. 2024. Fairness in serving large language models. In 18th USENIX Symposium on Operating Systems Design and Implementation (OSDI 24). 965\u2013988."},{"key":"e_1_3_3_1_27_2","first-page":"31094","volume-title":"International Conference on Machine Learning","author":"Sheng Ying","year":"2023","unstructured":"Ying Sheng, Lianmin Zheng, Binhang Yuan, Zhuohan Li, Max Ryabinin, Beidi Chen, Percy Liang, Christopher R\u00e9, Ion Stoica, and Ce Zhang. 2023. Flexgen: High-throughput generative inference of large language models with a single GPU. In International Conference on Machine Learning. PMLR, 31094\u201331116."},{"key":"e_1_3_3_1_28_2","doi-asserted-by":"publisher","DOI":"10.1145\/3603269.3604830"},{"key":"e_1_3_3_1_29_2","first-page":"173","volume-title":"18th USENIX Symposium on Operating Systems Design and Implementation (OSDI 24)","author":"Sun Biao","year":"2024","unstructured":"Biao Sun, Ziming Huang, Hanyu Zhao, Wencong Xiao, Xinyi Zhang, Yong Li, and Wei Lin. 2024. Llumnix: Dynamic Scheduling for Large Language Model Serving. In 18th USENIX Symposium on Operating Systems Design and Implementation (OSDI 24). USENIX Association, Santa Clara, CA, 173\u2013191."},{"key":"e_1_3_3_1_30_2","unstructured":"Rohan Taori Ishaan Gulrajani Tianyi Zhang Yann Dubois Xuechen Li Carlos Guestrin Percy Liang and Tatsunori\u00a0B. Hashimoto. 2023. Stanford Alpaca: An Instruction-following LLaMA model. https:\/\/github.com\/tatsu-lab\/stanford_alpaca."},{"key":"e_1_3_3_1_31_2","doi-asserted-by":"publisher","DOI":"10.1145\/3466752.3480129"},{"key":"e_1_3_3_1_32_2","doi-asserted-by":"publisher","DOI":"10.1145\/3694715.3695948"},{"key":"e_1_3_3_1_33_2","unstructured":"Bingyang Wu Yinmin Zhong Zili Zhang Gang Huang Xuanzhe Liu and Xin Jin. 2023. Fast Distributed Inference Serving for Large Language Models. ArXiv abs\/2305.05920 (2023). https:\/\/api.semanticscholar.org\/CorpusID:258588170"},{"key":"e_1_3_3_1_34_2","volume-title":"16th USENIX Symposium on Operating Systems Design and Implementation (OSDI 22)","author":"Yu Gyeong-In","year":"2022","unstructured":"Gyeong-In Yu, Joo\u00a0Seong Jeong, Geon-Woo Kim, Soojeong Kim, and Byung-Gon Chun. 2022. Orca: A Distributed Serving System for Transformer-Based Generative Models. In 16th USENIX Symposium on Operating Systems Design and Implementation (OSDI 22)."},{"key":"e_1_3_3_1_35_2","doi-asserted-by":"publisher","DOI":"10.1145\/3731569.3764823"},{"key":"e_1_3_3_1_36_2","unstructured":"Susan Zhang Stephen Roller Naman Goyal Mikel Artetxe Moya Chen Shuohui Chen Christopher Dewan Mona\u00a0T. Diab Xian Li Xi\u00a0Victoria Lin Todor Mihaylov Myle Ott Sam Shleifer Kurt Shuster Daniel Simig Punit\u00a0Singh Koura Anjali Sridhar Tianlu Wang and Luke Zettlemoyer. 2022. OPT: Open Pre-trained Transformer Language Models. ArXiv abs\/2205.01068 (2022). https:\/\/api.semanticscholar.org\/CorpusID:248496292"},{"key":"e_1_3_3_1_37_2","doi-asserted-by":"publisher","DOI":"10.1145\/3731569.3764810"},{"key":"e_1_3_3_1_38_2","doi-asserted-by":"crossref","unstructured":"Youpeng Zhao Di Wu and Jun Wang. 2024. ALISA: Accelerating Large Language Model Inference via Sparsity-Aware KV Caching. 2024 ACM\/IEEE 51st Annual International Symposium on Computer Architecture (ISCA) (2024) 1005\u20131017. https:\/\/api.semanticscholar.org\/CorpusID:268691374","DOI":"10.1109\/ISCA59077.2024.00077"},{"key":"e_1_3_3_1_39_2","doi-asserted-by":"publisher","DOI":"10.1145\/3600006.3613139"},{"key":"e_1_3_3_1_40_2","volume-title":"16th USENIX Symposium on Operating Systems Design and Implementation (OSDI 22)","author":"Zheng Ningxin","year":"2022","unstructured":"Ningxin Zheng, Bin Lin, Quanlu Zhang, Lingxiao Ma, Yuqing Yang, Fan Yang, Yang Wang, Mao Yang, and Lidong Zhou. 2022. SparTA: Deep-Learning Model Sparsity via Tensor-with-Sparsity-Attribute. In 16th USENIX Symposium on Operating Systems Design and Implementation (OSDI 22)."},{"key":"e_1_3_3_1_41_2","doi-asserted-by":"publisher","DOI":"10.52202\/075280-2859"},{"key":"e_1_3_3_1_42_2","volume-title":"Proc. of OSDI","author":"Zhong Yinmin","year":"2024","unstructured":"Yinmin Zhong, Shengyu Liu, Junda Chen, Jianbo Hu, Yibo Zhu, Xuanzhe Liu, Xin Jin, and Hao Zhang. 2024. DistServe: Disaggregating Prefill and Decoding for Goodput-optimized Large Language Model Serving. In Proc. of OSDI."}],"event":{"name":"ICPP '26: Proceedings of the 55th International Conference on Parallel Processing","location":"Singapore , Singapore","acronym":"ICPP '26"},"container-title":["Proceedings of the 55th International Conference on Parallel Processing"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/abs\/10.1145\/3832810.3832922","content-type":"text\/html","content-version":"vor","intended-application":"syndication"}],"deposited":{"date-parts":[[2026,9,16]],"date-time":"2026-09-16T18:02:33Z","timestamp":1789581753000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3832810.3832922"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026,9,27]]},"references-count":41,"alternative-id":["10.1145\/3832810.3832922","10.1145\/3832810"],"URL":"https:\/\/doi.org\/10.1145\/3832810.3832922","relation":{},"subject":[],"published":{"date-parts":[[2026,9,27]]},"assertion":[{"value":"2026-09-27","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}