{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,6,5]],"date-time":"2026-06-05T13:06:56Z","timestamp":1780664816747,"version":"3.54.1"},"publisher-location":"New York, NY, USA","reference-count":39,"publisher":"ACM","license":[{"start":{"date-parts":[[2026,4,26]],"date-time":"2026-04-26T00:00:00Z","timestamp":1777161600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by-nc-nd\/4.0\/legalcode"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2026,4,27]]},"DOI":"10.1145\/3767295.3769345","type":"proceedings-article","created":{"date-parts":[[2026,4,24]],"date-time":"2026-04-24T20:20:04Z","timestamp":1777062004000},"page":"1111-1127","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":0,"title":["AdaGen: Workload-Adaptive Cluster Scheduler for Latency-Optimal LLM Inference Serving"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0009-0002-9284-507X","authenticated-orcid":false,"given":"Sudipta Saha","family":"Shubha","sequence":"first","affiliation":[{"name":"University of Virginia, Charlottesville, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-2343-670X","authenticated-orcid":false,"given":"Ayush","family":"Goel","sequence":"additional","affiliation":[{"name":"HPE Labs, Milpitas, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-3664-4484","authenticated-orcid":false,"given":"Diman Zad","family":"Tootaghaj","sequence":"additional","affiliation":[{"name":"HPE Labs, Milpitas, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0009-1986-5104","authenticated-orcid":false,"given":"Khaled","family":"Diab","sequence":"additional","affiliation":[{"name":"HPE Labs, Burnaby, Canada"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-6812-8751","authenticated-orcid":false,"given":"Hardik","family":"Soni","sequence":"additional","affiliation":[{"name":"HPE Labs, Heidelberg, Germany"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-1849-5155","authenticated-orcid":false,"given":"K. K.","family":"Ramakrishnan","sequence":"additional","affiliation":[{"name":"University of California, Riverside, Riverside, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-4594-8164","authenticated-orcid":false,"given":"Puneet","family":"Sharma","sequence":"additional","affiliation":[{"name":"HPE Labs, Milpitas, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-7548-6223","authenticated-orcid":false,"given":"Haiying","family":"Shen","sequence":"additional","affiliation":[{"name":"University of Virginia, Charlottesville, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2026,4,26]]},"reference":[{"key":"e_1_3_2_1_1_1","volume-title":"Proc. of OSDI","author":"Agrawal Amey","year":"2024","unstructured":"Amey Agrawal, Nitin Kedia, Ashish Panwar, Jayashree Mohan, Nipun Kwatra, Bhargav Gulavani, Alexey Tumanov, and Ramachandran Ramjee. Taming {Throughput-Latency} tradeoff in {LLM} inference with {Sarathi-Serve}. In Proc. of OSDI, 2024."},{"key":"e_1_3_2_1_2_1","volume-title":"Proc. of OSDI","author":"Zhong Yinmin","year":"2024","unstructured":"Yinmin Zhong, Shengyu Liu, Junda Chen, Jianbo Hu, Yibo Zhu, Xuanzhe Liu, Xin Jin, and Hao Zhang. {DistServe}: Disaggregating prefill and decoding for goodput-optimized large language model serving. In Proc. of OSDI, 2024."},{"key":"e_1_3_2_1_3_1","volume-title":"Proc. of OSDI","author":"Sun Biao","year":"2024","unstructured":"Biao Sun, Ziming Huang, Hanyu Zhao, Wencong Xiao, Xinyi Zhang, Yong Li, and Wei Lin. Llumnix: Dynamic scheduling for large language model serving. In Proc. of OSDI, 2024."},{"key":"e_1_3_2_1_4_1","volume-title":"Deepspeed mii. https:\/\/github.com\/deepspeedai\/DeepSpeed-MII","author":"Microsoft DeepSpeed","year":"2025","unstructured":"DeepSpeed Microsoft. Deepspeed mii. https:\/\/github.com\/deepspeedai\/DeepSpeed-MII, 2025."},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1145\/3721146.3721947"},{"key":"e_1_3_2_1_6_1","volume-title":"Symphony: Improving memory management for llm inference workloads. arXiv preprint arXiv:2412.16434","author":"Agarwal Saurabh","year":"2024","unstructured":"Saurabh Agarwal, Anyong Mao, Aditya Akella, and Shivaram Venkataraman. Symphony: Improving memory management for llm inference workloads. arXiv preprint arXiv:2412.16434, 2024."},{"key":"e_1_3_2_1_7_1","volume-title":"Mell: Memory-efficient large language model serving via multi-gpu kv cache management. arXiv preprint arXiv:2501.06709","author":"Qianli Liu","year":"2025","unstructured":"Liu Qianli, Hong Zicong, Chen Fahao, Li Peng, and Guo Song. Mell: Memory-efficient large language model serving via multi-gpu kv cache management. arXiv preprint arXiv:2501.06709, 2025."},{"key":"e_1_3_2_1_8_1","unstructured":"Connor Holmes Masahiro Tanaka Michael Wyatt Ammar Ahmad Awan Jeff Rasley Samyam Rajbhandari Reza Yazdani Aminabadi Heyang Qin Arash Bakhtiari Lev Kurilenko et al. Deepspeed-fastgen: High-throughput text generation for llms via mii and deepspeed-inference. arXiv preprint arXiv:2401.08671 2024."},{"key":"e_1_3_2_1_9_1","volume-title":"Sarathi: Efficient llm inference by piggybacking decodes with chunked prefills. arXiv preprint arXiv:2308.16369","author":"Agrawal Amey","year":"2023","unstructured":"Amey Agrawal, Ashish Panwar, Jayashree Mohan, Nipun Kwatra, Bhargav S Gulavani, and Ramachandran Ramjee. Sarathi: Efficient llm inference by piggybacking decodes with chunked prefills. arXiv preprint arXiv:2308.16369, 2023."},{"key":"e_1_3_2_1_10_1","volume-title":"Fast distributed inference serving for large language models. arXiv preprint arXiv:2305.05920","author":"Wu Bingyang","year":"2023","unstructured":"Bingyang Wu, Yinmin Zhong, Zili Zhang, Gang Huang, Xuanzhe Liu, and Xin Jin. Fast distributed inference serving for large language models. arXiv preprint arXiv:2305.05920, 2023."},{"key":"e_1_3_2_1_11_1","volume-title":"Tempo: Application-aware llm serving with mixed slo requirements. arXiv preprint arXiv:2504.20068","author":"Zhang Wei","year":"2025","unstructured":"Wei Zhang, Zhiyu Wu, Yi Mu, Banruo Liu, Myungjin Lee, and Fan Lai. Tempo: Application-aware llm serving with mixed slo requirements. arXiv preprint arXiv:2504.20068, 2025."},{"key":"e_1_3_2_1_12_1","volume-title":"Chetan Bansal, Victor R\u00fchle, Anoop Kulkarni, et al. Serving models, fast and slow: optimizing heterogeneous llm inferencing workloads at scale. arXiv preprint arXiv:2502.14617","author":"Jaiswal Shashwat","year":"2025","unstructured":"Shashwat Jaiswal, Kunal Jain, Yogesh Simmhan, Anjaly Parayil, Ankur Mallick, Rujia Wang, Renee St Amant, Chetan Bansal, Victor R\u00fchle, Anoop Kulkarni, et al. Serving models, fast and slow: optimizing heterogeneous llm inferencing workloads at scale. arXiv preprint arXiv:2502.14617, 2025."},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1109\/ISCA59077.2024.00019"},{"key":"e_1_3_2_1_14_1","volume-title":"Inference without interference: Disaggregate llm inference for mixed downstream workloads. arXiv preprint arXiv:2401.11181","author":"Hu Cunchen","year":"2024","unstructured":"Cunchen Hu, Heyang Huang, Liangliang Xu, Xusheng Chen, Jiang Xu, Shuang Chen, Hao Feng, Chenxi Wang, Sa Wang, Yungang Bao, et al. Inference without interference: Disaggregate llm inference for mixed downstream workloads. arXiv preprint arXiv:2401.11181, 2024."},{"key":"e_1_3_2_1_15_1","volume-title":"Kv-cache streaming for fast, fault-tolerant generative llm serving. arXiv preprint arXiv:2403.01876","author":"Strati Foteini","year":"2024","unstructured":"Foteini Strati, Sara Mcallister, Amar Phanishayee, Jakub Tarnawski, and Ana Klimovic. D\\'ej\\avu: Kv-cache streaming for fast, fault-tolerant generative llm serving. arXiv preprint arXiv:2403.01876, 2024."},{"key":"e_1_3_2_1_16_1","volume-title":"Nvidia dynamo. https:\/\/developer.nvidia.com\/dynamo","year":"2025","unstructured":"Nvidia. Nvidia dynamo. https:\/\/developer.nvidia.com\/dynamo, 2025."},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.1145\/3600006.3613165"},{"key":"e_1_3_2_1_18_1","volume-title":"Amelie Chi Zhou, and Xiaowen Chu. Towards efficient and reliable llm serving: A real-world workload study. arXiv e-prints","author":"Wang Yuxin","year":"2024","unstructured":"Yuxin Wang, Yuhan Chen, Zeyu Li, Zhenheng Tang, Rui Guo, Xin Wang, Qiang Wang, Amelie Chi Zhou, and Xiaowen Chu. Towards efficient and reliable llm serving: A real-world workload study. arXiv e-prints, pages arXiv-2401, 2024."},{"key":"e_1_3_2_1_19_1","volume-title":"Openchat: Advancing open-source language models with mixed-quality data. arXiv preprint arXiv:2309.11235","author":"Wang Guan","year":"2023","unstructured":"Guan Wang, Sijie Cheng, Xianyuan Zhan, Xiangang Li, Sen Song, and Yang Liu. Openchat: Advancing open-source language models with mixed-quality data. arXiv preprint arXiv:2309.11235, 2023."},{"key":"e_1_3_2_1_20_1","first-page":"93","volume-title":"2024 USENIX Annual Technical Conference (USENIX ATC 24)","author":"Qiu Haoran","year":"2024","unstructured":"Haoran Qiu, Weichao Mao, Archit Patke, Shengkun Cui, Saurabh Jha, Chen Wang, Hubertus Franke, Zbigniew Kalbarczyk, Tamer Ba\u015far, and Ravishankar K Iyer. Power-aware deep learning model serving with {\u03bc-Serve}. In 2024 USENIX Annual Technical Conference (USENIX ATC 24), pages 75\u201393, 2024."},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.1145\/2619239.2626334"},{"key":"e_1_3_2_1_22_1","volume-title":"Proc. of OSDI","author":"Shubha Sudipta Saha","year":"2024","unstructured":"Sudipta Saha Shubha, Haiying Shen, and Anand Iyer. {USHER}: Holistic interference avoidance for resource optimized {ML} inference. In Proc. of OSDI, 2024."},{"key":"e_1_3_2_1_23_1","volume-title":"Hans Peter Kriegel, and Xiaowei Xu. Dbscan revisited, revisited: why and how you should (still) use dbscan. ACM Transactions on Database Systems (TODS), 42(3)","author":"Schubert Erich","year":"2017","unstructured":"Erich Schubert, J\u00f6rg Sander, Martin Ester, Hans Peter Kriegel, and Xiaowei Xu. Dbscan revisited, revisited: why and how you should (still) use dbscan. ACM Transactions on Database Systems (TODS), 42(3), 2017."},{"key":"e_1_3_2_1_24_1","volume-title":"Proc. of OSDI","author":"Moritz Philipp","year":"2018","unstructured":"Philipp Moritz, Robert Nishihara, Stephanie Wang, Alexey Tumanov, Richard Liaw, Eric Liang, Melih Elibol, Zongheng Yang, William Paul, Michael I Jordan, et al. Ray: A distributed framework for emerging {AI} applications. In Proc. of OSDI, 2018."},{"key":"e_1_3_2_1_25_1","volume-title":"Flashattention: Fast and memory-efficient exact attention with ioawareness. Advances in neural information processing systems, 35:16344\u201316359","author":"Dao Tri","year":"2022","unstructured":"Tri Dao, Dan Fu, Stefano Ermon, Atri Rudra, and Christopher R\u00e9. Flashattention: Fast and memory-efficient exact attention with ioawareness. Advances in neural information processing systems, 35:16344\u201316359, 2022."},{"key":"e_1_3_2_1_26_1","volume-title":"Nvidia nccl. https:\/\/developer.nvidia.com\/nccl","year":"2025","unstructured":"Nvidia. Nvidia nccl. https:\/\/developer.nvidia.com\/nccl, 2025."},{"key":"e_1_3_2_1_27_1","volume-title":"Proc. of NSDI","author":"Crankshaw Daniel","year":"2017","unstructured":"Daniel Crankshaw, Xin Wang, Guilio Zhou, Michael J Franklin, Joseph E Gonzalez, and Ion Stoica. Clipper: A {Low-Latency} online prediction serving system. In Proc. of NSDI, 2017."},{"key":"e_1_3_2_1_28_1","volume-title":"Proc. of SOSP","author":"Shen Haichen","year":"2019","unstructured":"Haichen Shen, Lequn Chen, Yuchen Jin, Liangyu Zhao, Bingyu Kong, Matthai Philipose, Arvind Krishnamurthy, and Ravi Sundaram. Nexus: A gpu cluster engine for accelerating dnn-based video analysis. In Proc. of SOSP, 2019."},{"key":"e_1_3_2_1_29_1","volume-title":"Nvidia triton. https:\/\/github.com\/triton-inference-server\/server","year":"2025","unstructured":"Nvidia. Nvidia triton. https:\/\/github.com\/triton-inference-server\/server, 2025."},{"key":"e_1_3_2_1_30_1","volume-title":"Proc. of OSDI","author":"Gujarati Arpan","year":"2020","unstructured":"Arpan Gujarati, Reza Karimi, Safya Alzayat, Wei Hao, Antoine Kaufmann, Ymir Vigfusson, and Jonathan Mace. Serving { DNNs} like clockwork: Performance predictability from the bottom up. In Proc. of OSDI, 2020."},{"key":"e_1_3_2_1_31_1","volume-title":"Proc. of NSDI","author":"Zhang Hong","year":"2023","unstructured":"Hong Zhang, Yupeng Tang, Anurag Khandelwal, and Ion Stoica. {SHEPHERD}: Serving {DNNs} in the wild. In Proc. of NSDI, 2023."},{"key":"e_1_3_2_1_32_1","volume-title":"Proc. of OSDI","author":"Li Zhuohan","year":"2023","unstructured":"Zhuohan Li, Lianmin Zheng, Yinmin Zhong, Vincent Liu, Ying Sheng, Xin Jin, Yanping Huang, Zhifeng Chen, Hao Zhang, Joseph E Gonzalez, et al. {AlpaServe}: Statistical multiplexing with model parallelism for deep learning serving. In Proc. of OSDI, 2023."},{"key":"e_1_3_2_1_33_1","volume-title":"Proc. of ATC","author":"Romero Francisco","year":"2021","unstructured":"Francisco Romero, Qian Li, Neeraja J Yadwadkar, and Christos Kozyrakis. {INFaaS}: Automated model-less inference serving. In Proc. of ATC, 2021."},{"key":"e_1_3_2_1_34_1","volume-title":"Proc. of OSDI","author":"Lin Chaofan","year":"2024","unstructured":"Chaofan Lin, Zhenhua Han, Chengruidong Zhang, Yuqing Yang, Fan Yang, Chen Chen, and Lili Qiu. Parrot: Efficient serving of {LLM-based} applications with semantic variable. In Proc. of OSDI, 2024."},{"key":"e_1_3_2_1_35_1","volume-title":"Nvidia fastertransformer. https:\/\/github.com\/NVIDIA\/Faster Transformer","year":"2023","unstructured":"Nvidia. Nvidia fastertransformer. https:\/\/github.com\/NVIDIA\/Faster Transformer, 2023."},{"key":"e_1_3_2_1_36_1","volume-title":"Proc. of SIGPLAN","author":"Fang Jiarui","year":"2021","unstructured":"Jiarui Fang, Yang Yu, Chengduo Zhao, and Jie Zhou. Turbotransformers: an efficient gpu serving system for transformer models. In Proc. of SIGPLAN, 2021."},{"key":"e_1_3_2_1_37_1","volume-title":"Proc. of OSDI","author":"Yu Gyeong-In","year":"2022","unstructured":"Gyeong-In Yu, Joo Seong Jeong, Geon-Woo Kim, Soojeong Kim, and Byung-Gon Chun. Orca: A distributed serving system for {Transformer-Based} generative models. In Proc. of OSDI, 2022."},{"key":"e_1_3_2_1_38_1","volume-title":"Jared Kaplan, Harri Edwards, Yuri Burda, Nicholas Joseph, Greg Brockman, et al. Evaluating large language models trained on code. arXiv preprint arXiv:2107.03374","author":"Chen Mark","year":"2021","unstructured":"Mark Chen, Jerry Tworek, Heewoo Jun, Qiming Yuan, Henrique Ponde De Oliveira Pinto, Jared Kaplan, Harri Edwards, Yuri Burda, Nicholas Joseph, Greg Brockman, et al. Evaluating large language models trained on code. arXiv preprint arXiv:2107.03374, 2021."},{"key":"e_1_3_2_1_39_1","volume-title":"et al. Long-bench: A bilingual, multitask benchmark for long context understanding. arXiv preprint arXiv:2308.14508","author":"Bai Yushi","year":"2023","unstructured":"Yushi Bai, Xin Lv, Jiajie Zhang, Hongchang Lyu, Jiankai Tang, Zhidian Huang, Zhengxiao Du, Xiao Liu, Aohan Zeng, Lei Hou, et al. Long-bench: A bilingual, multitask benchmark for long context understanding. arXiv preprint arXiv:2308.14508, 2023."}],"event":{"name":"EUROSYS '26: 21st European Conference on Computer Systems","location":"McEwan Hall\/The University of Edinburgh Edinburgh Scotland UK","acronym":"EUROSYS '26","sponsor":["SIGOPS ACM Special Interest Group on Operating Systems"]},"container-title":["Proceedings of the 21st European Conference on Computer Systems"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3767295.3769345","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,6,5]],"date-time":"2026-06-05T12:22:12Z","timestamp":1780662132000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3767295.3769345"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026,4,26]]},"references-count":39,"alternative-id":["10.1145\/3767295.3769345","10.1145\/3767295"],"URL":"https:\/\/doi.org\/10.1145\/3767295.3769345","relation":{},"subject":[],"published":{"date-parts":[[2026,4,26]]},"assertion":[{"value":"2026-04-26","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}