{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,29]],"date-time":"2026-10-29T08:41:37Z","timestamp":1793263297789,"version":"4.5.0"},"publisher-location":"New York, NY, USA","reference-count":86,"publisher":"ACM","license":[{"start":{"date-parts":[[2025,3,30]],"date-time":"2025-03-30T00:00:00Z","timestamp":1743292800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"funder":[{"name":"SRC and DARPA","award":["ACE, one of the seven centers in JUMP 2.0"],"award-info":[{"award-number":["ACE, one of the seven centers in JUMP 2.0"]}]},{"DOI":"10.13039\/501100006374","name":"NSF (National Science Foundation)","doi-asserted-by":"publisher","award":["CNS 1956007, CCF 2107470, CCF 2316233"],"award-info":[{"award-number":["CNS 1956007, CCF 2107470, CCF 2316233"]}],"id":[{"id":"10.13039\/501100006374","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2025,3,30]]},"DOI":"10.1145\/3676641.3716025","type":"proceedings-article","created":{"date-parts":[[2025,3,27]],"date-time":"2025-03-27T16:47:32Z","timestamp":1743094052000},"page":"1266-1281","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":41,"title":["TAPAS: Thermal- and Power-Aware Scheduling for LLM Inference in Cloud Platforms"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0009-0007-4914-336X","authenticated-orcid":false,"given":"Jovan","family":"Stojkovic","sequence":"first","affiliation":[{"name":"University of Illinois at Urbana-Champaign, Champaign, IL, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0002-8334-1291","authenticated-orcid":false,"given":"Chaojie","family":"Zhang","sequence":"additional","affiliation":[{"name":"Microsoft Azure Research, Redmond, WA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-2591-4012","authenticated-orcid":false,"given":"\u00cd\u00f1igo","family":"Goiri","sequence":"additional","affiliation":[{"name":"Microsoft Azure Research, Redmond, WA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-0371-5522","authenticated-orcid":false,"given":"Esha","family":"Choukse","sequence":"additional","affiliation":[{"name":"Microsoft Azure Research, Redmond, WA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-8071-1130","authenticated-orcid":false,"given":"Haoran","family":"Qiu","sequence":"additional","affiliation":[{"name":"Microsoft Azure Research, Redmond, WA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-9662-2661","authenticated-orcid":false,"given":"Rodrigo","family":"Fonseca","sequence":"additional","affiliation":[{"name":"Microsoft Azure Research, Redmond, WA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-2595-5228","authenticated-orcid":false,"given":"Josep","family":"Torrellas","sequence":"additional","affiliation":[{"name":"University of Illinois at Urbana-Champaign, Champaign, IL, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-5971-5084","authenticated-orcid":false,"given":"Ricardo","family":"Bianchini","sequence":"additional","affiliation":[{"name":"Microsoft Azure, Redmond, WA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2025,3,30]]},"reference":[{"key":"e_1_3_2_1_1_1","volume-title":"Ramachandran Ramjee. Taming Throughput-Latency Tradeoff in LLM Inference with Sarathi-Serve. In 18th USENIX Symposium on Operating Systems Design and Implementation (OSDI)","author":"Agrawal Amey","year":"2024","unstructured":"Amey Agrawal, Nitin Kedia, Ashish Panwar, Jayashree Mohan, Nipun Kwatra, Bhargav Gulavani, Alexey Tumanov, and Ramachandran Ramjee. Taming Throughput-Latency Tradeoff in LLM Inference with Sarathi-Serve. In 18th USENIX Symposium on Operating Systems Design and Implementation (OSDI), 2024."},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.7873\/DATE2014.189"},{"key":"e_1_3_2_1_3_1","volume-title":"Gemini API Developer Docs. https:\/\/ai.google.dev\/geminiapi\/docs","author":"Google","year":"2024","unstructured":"Google AI. Gemini API Developer Docs. https:\/\/ai.google.dev\/geminiapi\/docs, 2024."},{"key":"e_1_3_2_1_4_1","volume-title":"Mohammad Rastegari, and Mehrdad Farajtabar. LLM in a flash: Efficient Large Language Model Inference with Limited Memory. arXiv preprint arXiv:2312.11514","author":"Alizadeh Keivan","year":"2024","unstructured":"Keivan Alizadeh, Iman Mirzadeh, Dmitry Belenko, Karen Khatamifard, Minsik Cho, Carlo C Del Mundo, Mohammad Rastegari, and Mehrdad Farajtabar. LLM in a flash: Efficient Large Language Model Inference with Limited Memory. arXiv preprint arXiv:2312.11514, 2024."},{"key":"e_1_3_2_1_5_1","volume-title":"Azure Machine Learning - ML as a Service. https:\/\/azure. microsoft.com\/en-us\/products\/machine-learning\/","year":"2024","unstructured":"Azure. Azure Machine Learning - ML as a Service. https:\/\/azure. microsoft.com\/en-us\/products\/machine-learning\/, 2024."},{"key":"e_1_3_2_1_6_1","volume-title":"ND' sub-family GPU accelerated virtual machine size series. https:\/\/learn.microsoft.com\/en-us\/azure\/virtual-machines\/sizes\/gpu-accelerated\/nd-family","author":"Azure Microsoft","year":"2024","unstructured":"Microsoft Azure. 'ND' sub-family GPU accelerated virtual machine size series. https:\/\/learn.microsoft.com\/en-us\/azure\/virtual-machines\/sizes\/gpu-accelerated\/nd-family, 2024."},{"key":"e_1_3_2_1_7_1","volume-title":"November","author":"Badri Hicham","year":"2023","unstructured":"Hicham Badri and Appu Shaji. Half-quadratic quantization of large machine learning models, November 2023."},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-01741-4"},{"key":"e_1_3_2_1_9_1","volume-title":"Using Large Language Models in Learning and Teaching. https:\/\/biomedicalsciences.unimelb.edu.au\/study\/dlh\/assets\/documents\/large-language-models-in-education\/llms-in-education","author":"Bowne Jairus","year":"2024","unstructured":"Jairus Bowne. Using Large Language Models in Learning and Teaching. https:\/\/biomedicalsciences.unimelb.edu.au\/study\/dlh\/assets\/documents\/large-language-models-in-education\/llms-in-education, 2024."},{"key":"e_1_3_2_1_10_1","volume-title":"Atif Manzoor, Syed Asad Hussain, and Jongwon Kim. Thermal-aware scheduling in green data centers. ACM Computing Surveys, 47(3), feb","author":"Chaudhry Muhammad Tayyab","year":"2015","unstructured":"Muhammad Tayyab Chaudhry, Teck Chaw Ling, Atif Manzoor, Syed Asad Hussain, and Jongwon Kim. Thermal-aware scheduling in green data centers. ACM Computing Surveys, 47(3), feb 2015."},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.future.2023.03.049"},{"key":"e_1_3_2_1_12_1","volume-title":"Andrew Warfield. Live Migration of Virtual Machines. In Proceedings of the Symposium on Networked Systems Design and Implementation (NSDI)","author":"Clark Christopher","year":"2005","unstructured":"Christopher Clark, Keir Fraser, Steven Hand, Jacob Gorm Hansen, Erik Jul, Christian Limpach, Ian Pratt, and Andrew Warfield. Live Migration of Virtual Machines. In Proceedings of the Symposium on Networked Systems Design and Implementation (NSDI), 2005."},{"key":"e_1_3_2_1_13_1","volume-title":"GPU machine types. https:\/\/cloud.google.com\/compute\/docs\/gpus\/","author":"Cloud Google","year":"2024","unstructured":"Google Cloud. GPU machine types. https:\/\/cloud.google.com\/compute\/docs\/gpus\/, 2024."},{"key":"e_1_3_2_1_14_1","volume-title":"FlashAttention: Fast and Memory-Efficient Exact Attention with IO-Awareness. arXiv preprint arXiv:2205.14135","author":"Dao Tri","year":"2022","unstructured":"Tri Dao, Daniel Y. Fu, Stefano Ermon, Atri Rudra, and Christopher R\u00e9. FlashAttention: Fast and Memory-Efficient Exact Attention with IO-Awareness. arXiv preprint arXiv:2205.14135, 2022."},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.applthermaleng.2016.10.093"},{"key":"e_1_3_2_1_16_1","volume-title":"https:\/\/www.energy.gov\/energysaver\/evaporative-coolers","author":"Coolers Evaporative","year":"2024","unstructured":"Energy.gov. Evaporative Coolers. https:\/\/www.energy.gov\/energysaver\/evaporative-coolers, 2024."},{"key":"e_1_3_2_1_17_1","volume-title":"Pengfei Zuo. Cost-Efficient Large Language Model Serving for Multi-turn Conversations with CachedAttention. In USENIX Annual Technical Conference (USENIX ATC)","author":"Gao Bin","year":"2024","unstructured":"Bin Gao, Zhuomin He, Puru Sharma, Qingxuan Kang, Djordje Jevdjic, Junbo Deng, Xingkun Yang, Zhou Yu, and Pengfei Zuo. Cost-Efficient Large Language Model Serving for Multi-turn Conversations with CachedAttention. In USENIX Annual Technical Conference (USENIX ATC), 2024."},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1145\/3673038.3673144"},{"key":"e_1_3_2_1_19_1","volume-title":"The world's most widely adopted AI developer tool. https: \/\/github.com\/features\/copilot","year":"2024","unstructured":"GitHub. The world's most widely adopted AI developer tool. https: \/\/github.com\/features\/copilot, 2024."},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.1145\/2694344.2694378"},{"key":"e_1_3_2_1_21_1","volume-title":"Proceedings of the 14th USENIX Symposium on Operating Systems Design and Implementation (OSDI)","author":"Hadary Ori","year":"2020","unstructured":"Ori Hadary, Luke Marshall, Ishai Menache, Abhisek Pan, Esaias E Greeff, David Dion, Star Dorminey, Shailesh Joshi, Yang Chen, Mark Russinovich, and Thomas Moscibroda. Protean: VM Allocation Service at Scale. In Proceedings of the 14th USENIX Symposium on Operating Systems Design and Implementation (OSDI), 2020."},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.1145\/3173162.3173190"},{"key":"e_1_3_2_1_23_1","volume-title":"Our commitment to climate-conscious data center cooling. https:\/\/blog.google\/outreach-initiatives\/sustainability\/ourcommitment-to-climate-conscious-data-center-cooling\/","author":"H\u00f6lzle Urs","year":"2022","unstructured":"Urs H\u00f6lzle. Our commitment to climate-conscious data center cooling. https:\/\/blog.google\/outreach-initiatives\/sustainability\/ourcommitment-to-climate-conscious-data-center-cooling\/, 2022."},{"key":"e_1_3_2_1_24_1","volume-title":"Ricardo Bianchini. Cost-Efficient Overclocking in Immersion-Cooled Datacenters. In Proceedings of the 48th Annual International Symposium on Computer Architecture (ISCA)","author":"Jalili Majid","year":"2021","unstructured":"Majid Jalili, Ioannis Manousakis, \u00cd\u00f1igo Goiri, Pulkit A. Misra, Ashish Raniwala, Husam Alissa, Bharath Ramakrishnan, Phillip Tuma, Christian Belady, Marcus Fontoura, and Ricardo Bianchini. Cost-Efficient Overclocking in Immersion-Cooled Datacenters. In Proceedings of the 48th Annual International Symposium on Computer Architecture (ISCA), 2021."},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.1145\/3307650.3322236"},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.1109\/LCA.2024.3406038"},{"key":"e_1_3_2_1_27_1","volume-title":"Ion Stoica. Efficient Memory Management for Large Language Model Serving with PagedAttention. In Proceedings of the Symposium on Operating Systems Principles (SOSP)","author":"Kwon Woosuk","year":"2023","unstructured":"Woosuk Kwon, Zhuohan Li, Siyuan Zhuang, Ying Sheng, Lianmin Zheng, Cody Hao Yu, Joseph E. Gonzalez, Hao Zhang, and Ion Stoica. Efficient Memory Management for Large Language Model Serving with PagedAttention. In Proceedings of the Symposium on Operating Systems Principles (SOSP), 2023."},{"key":"e_1_3_2_1_28_1","volume-title":"60 ChatGPT Statistics And Facts You Need to Know","author":"Lammertyn Marina","year":"2024","unstructured":"Marina Lammertyn. 60 ChatGPT Statistics And Facts You Need to Know in 2024. https:\/\/blog.invgate.com\/chatgpt-statistics, 2024."},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.1145\/3358235"},{"key":"e_1_3_2_1_30_1","volume-title":"LLM Inference Serving: Survey of Recent Advances and Opportunities. arXiv preprint arXiv:2407.12391","author":"Li Baolin","year":"2024","unstructured":"Baolin Li, Yankai Jiang, Vijay Gadepally, and Devesh Tiwari. LLM Inference Serving: Survey of Recent Advances and Opportunities. arXiv preprint arXiv:2407.12391, 2024."},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"publisher","DOI":"10.1145\/3632775.3661938"},{"key":"e_1_3_2_1_32_1","volume-title":"Proceedings of the 14th USENIX Symposium on Operating Systems Design and Implementation (OSDI)","author":"Li Shaohong","year":"2020","unstructured":"Shaohong Li, Xi Wang, Xiao Zhang, Vasileios Kontorinis, Sreekumar Kodakara, David Lo, and Parthasarathy Ranganathan. Thunderbolt: Throughput-Optimized, Quality-of-Service-Aware Power Capping at Scale. In Proceedings of the 14th USENIX Symposium on Operating Systems Design and Implementation (OSDI), 2020."},{"key":"e_1_3_2_1_33_1","volume-title":"Evaluating quantized large language models. arXiv preprint arXiv:2402.18158","author":"Li Shiyao","year":"2024","unstructured":"Shiyao Li, Xuefei Ning, Luning Wang, Tengxuan Liu, Xiangsheng Shi, Shengen Yan, Guohao Dai, Huazhong Yang, and Yu Wang. Evaluating quantized large language models. arXiv preprint arXiv:2402.18158, 2024."},{"key":"e_1_3_2_1_34_1","doi-asserted-by":"publisher","DOI":"10.1109\/HPCA.2019.00067"},{"key":"e_1_3_2_1_35_1","volume-title":"The Unseen AI Disruptions for Power Grids: LLM-Induced Transients. arXiv preprint arXiv:2409.11416","author":"Li Yuzhuo","year":"2024","unstructured":"Yuzhuo Li, Mariam Mughees, Yize Chen, and Yunwei Ryan Li. The Unseen AI Disruptions for Power Grids: LLM-Induced Transients. arXiv preprint arXiv:2409.11416, 2024."},{"key":"e_1_3_2_1_36_1","volume-title":"Ion Stoica. AlpaServe: Statistical Multiplexing with Model Parallelism for Deep Learning Serving. In Proceedings of the Symposium on Operating Systems Design and Implementation (OSDI '23)","author":"Li Zhuohan","year":"2023","unstructured":"Zhuohan Li, Lianmin Zheng, Yinmin Zhong, Vincent Liu, Ying Sheng, Xin Jin, Yanping Huang, Zhifeng Chen, Hao Zhang, Joseph E. Gonzalez, and Ion Stoica. AlpaServe: Statistical Multiplexing with Model Parallelism for Deep Learning Serving. In Proceedings of the Symposium on Operating Systems Design and Implementation (OSDI '23), 2023."},{"key":"e_1_3_2_1_37_1","volume-title":"QServe: W4A8KV4 Quantization and System Co-design for Efficient LLM Serving","author":"Lin Yujun","year":"2024","unstructured":"Yujun Lin, Haotian Tang, Shang Yang, Zhekai Zhang, Guangxuan Xiao, Chuang Gan, and Song Han. QServe: W4A8KV4 Quantization and System Co-design for Efficient LLM Serving, 2024."},{"key":"e_1_3_2_1_38_1","doi-asserted-by":"publisher","DOI":"10.1145\/2806777.2806938"},{"key":"e_1_3_2_1_39_1","doi-asserted-by":"publisher","DOI":"10.1145\/2528521.1508269"},{"key":"e_1_3_2_1_40_1","volume-title":"2024 Sustainability Report. https:\/\/sustainability.atmeta.com\/ 2024-sustainability-report\/","year":"2024","unstructured":"Meta. 2024 Sustainability Report. https:\/\/sustainability.atmeta.com\/ 2024-sustainability-report\/, 2024."},{"key":"e_1_3_2_1_41_1","volume-title":"https:\/\/huggingface.co\/meta-llama\/Meta-Llama-3-70B-Instruct","author":"B.","year":"2024","unstructured":"Meta. Llama3--70B. https:\/\/huggingface.co\/meta-llama\/Meta-Llama-3-70B-Instruct, 2024."},{"key":"e_1_3_2_1_42_1","doi-asserted-by":"publisher","DOI":"10.1145\/3620665.3640411"},{"key":"e_1_3_2_1_43_1","doi-asserted-by":"publisher","DOI":"10.1145\/3664925"},{"key":"e_1_3_2_1_44_1","volume-title":"Cooling and Airflow Optimization. https:\/\/docs.nvidia.com\/dgx-superpod\/design-guides\/dgx-superpoddata-center-design-h100\/latest\/cooling.html","author":"NVIDIA.","year":"2024","unstructured":"NVIDIA. Cooling and Airflow Optimization. https:\/\/docs.nvidia.com\/dgx-superpod\/design-guides\/dgx-superpoddata-center-design-h100\/latest\/cooling.html, 2024."},{"key":"e_1_3_2_1_45_1","volume-title":"DGX H100: AI for Enterprise. https:\/\/www.nvidia.com\/enus\/data-center\/dgx-h100\/","author":"NVIDIA.","year":"2024","unstructured":"NVIDIA. DGX H100: AI for Enterprise. https:\/\/www.nvidia.com\/enus\/data-center\/dgx-h100\/, 2024."},{"key":"e_1_3_2_1_46_1","volume-title":"Introduction to the NVIDIA DGX A100 System. https:\/\/docs.nvidia.com\/dgx\/dgxa100-user-guide\/introductionto-dgxa100.html","author":"NVIDIA.","year":"2024","unstructured":"NVIDIA. Introduction to the NVIDIA DGX A100 System. https:\/\/docs.nvidia.com\/dgx\/dgxa100-user-guide\/introductionto-dgxa100.html, 2024."},{"key":"e_1_3_2_1_47_1","volume-title":"https:\/\/nvidia.github.io\/TensorRT-LLM\/","author":"M's Documentation NVIDIA.","year":"2024","unstructured":"NVIDIA. TensorRT-LLM's Documentation. https:\/\/nvidia.github.io\/TensorRT-LLM\/, 2024."},{"key":"e_1_3_2_1_48_1","doi-asserted-by":"publisher","DOI":"10.1145\/3620666.3651329"},{"key":"e_1_3_2_1_49_1","doi-asserted-by":"publisher","DOI":"10.1109\/ISCA59077.2024.00019"},{"key":"e_1_3_2_1_50_1","volume-title":"Proceedings of the 27th ACM International Conference on Architectural Support for Programming Languages and Operating Systems (ASPLOS)","author":"Pei Qiangyu","year":"2022","unstructured":"Qiangyu Pei, Shutong Chen, Qixia Zhang, Xinhui Zhu, Fangming Liu, Ziyang Jia, Yishuo Wang, and Yongjie Yuan. CoolEdge: hotspotrelievable warm water cooling for energy-efficient edge datacenters. In Proceedings of the 27th ACM International Conference on Architectural Support for Programming Languages and Operating Systems (ASPLOS), 2022."},{"key":"e_1_3_2_1_51_1","doi-asserted-by":"publisher","DOI":"10.1145\/1736020.1736047"},{"key":"e_1_3_2_1_52_1","volume-title":"A study of generative large language model for medical research and healthcare. npj Digital Medicine","author":"Peng Cheng","year":"2023","unstructured":"Cheng Peng, Xi Yang, Aokun Chen, Kaleb Smith, Nima PourNejatian, Anthony Costa, Cheryl Martin, Mona Flores, Ying Zhang, Tanja Magoc, Gloria Lipori, Mitchell Duane, Naykky Ospina, Mustafa Ahmed, William Hogan, Elizabeth Shenkman, Yi Guo, Jiang Bian, and Yonghui Wu. A study of generative large language model for medical research and healthcare. npj Digital Medicine, 2023."},{"key":"e_1_3_2_1_53_1","volume-title":"USENIX Annual Technical Conference (USENIX ATC)","author":"Qiu Haoran","year":"2024","unstructured":"Haoran Qiu, Weichao Mao, Archit Patke, Shengkun Cui, Saurabh Jha, Chen Wang, Hubertus Franke, Zbigniew Kalbarczyk, Tamer Ba\u015far, and Ravishankar K. Iyer. Power-aware Deep Learning Model Serving with \u03bc-Serve. In USENIX Annual Technical Conference (USENIX ATC), 2024."},{"key":"e_1_3_2_1_54_1","volume-title":"Iyer. Efficient Interactive LLM Serving with Proxy Model-based Sequence Length Prediction. In The 5th International Workshop on Cloud Intelligence \/ AIOps at ASPLOS 2024","author":"Qiu Haoran","year":"2024","unstructured":"Haoran Qiu, Weichao Mao, Archit Patke, Shengkun Cui, Saurabh Jha, Chen Wang, Hubertus Franke, Zbigniew T. Kalbarczyk, Tamer Ba\u015far, and Ravishankar K. Iyer. Efficient Interactive LLM Serving with Proxy Model-based Sequence Length Prediction. In The 5th International Workshop on Cloud Intelligence \/ AIOps at ASPLOS 2024, 2024."},{"key":"e_1_3_2_1_55_1","doi-asserted-by":"publisher","DOI":"10.1145\/3373376.3378533"},{"key":"e_1_3_2_1_56_1","volume-title":"Proceedings of the High Performance Extreme Computing Conference (HPEC)","author":"Samsi Siddharth","year":"2023","unstructured":"Siddharth Samsi, Dan Zhao, Joseph McDonald, Baolin Li, Adam Michaleas, Michael Jones, William Bergeron, Jeremy Kepner, Devesh Tiwari, and Vijay Gadepally. FromWords toWatts: Benchmarking the Energy Costs of Large Language Model Inference. In Proceedings of the High Performance Extreme Computing Conference (HPEC), 2023."},{"key":"e_1_3_2_1_57_1","volume-title":"Deploy LLMs with Hugging Face Inference Endpoints. https:\/\/huggingface.co\/blog\/inference-endpoints-llm","author":"Schmid Philipp","year":"2024","unstructured":"Philipp Schmid. Deploy LLMs with Hugging Face Inference Endpoints. https:\/\/huggingface.co\/blog\/inference-endpoints-llm, 2024."},{"key":"e_1_3_2_1_58_1","volume-title":"Amazon EC2 P4 Instances. https:\/\/aws.amazon.com\/ec2\/instance-types\/p4\/","author":"Services Amazon Web","year":"2020","unstructured":"Amazon Web Services. Amazon EC2 P4 Instances. https:\/\/aws.amazon.com\/ec2\/instance-types\/p4\/, 2020."},{"key":"e_1_3_2_1_59_1","volume-title":"Amazon EC2 P5 Instances. https:\/\/aws.amazon.com\/ec2\/instance-types\/p5\/","author":"Services Amazon Web","year":"2024","unstructured":"Amazon Web Services. Amazon EC2 P5 Instances. https:\/\/aws.amazon.com\/ec2\/instance-types\/p5\/, 2024."},{"key":"e_1_3_2_1_60_1","volume-title":"Amazon SageMaker - Machine Learning Services. https:\/\/aws.amazon.com\/sagemaker\/","author":"Services Amazon Web","year":"2024","unstructured":"Amazon Web Services. Amazon SageMaker - Machine Learning Services. https:\/\/aws.amazon.com\/sagemaker\/, 2024."},{"key":"e_1_3_2_1_61_1","volume-title":"Meta Llama 3 models are now available in Amazon SageMaker JumpStart. https:\/\/aws.amazon.com\/blogs\/machinelearning\/ meta-llama-3-models-are-now-available-in-amazonsagemaker- jumpstart\/","author":"Services AmazonWeb","year":"2024","unstructured":"AmazonWeb Services. Meta Llama 3 models are now available in Amazon SageMaker JumpStart. https:\/\/aws.amazon.com\/blogs\/machinelearning\/ meta-llama-3-models-are-now-available-in-amazonsagemaker- jumpstart\/, 2024."},{"key":"e_1_3_2_1_62_1","volume-title":"International Conference on Green Computing (ICGC)","author":"Shi Bing","year":"2010","unstructured":"Bing Shi and Ankur Srivastava. Thermal and power-aware task scheduling for Hadoop based storage centric datacenters. In International Conference on Green Computing (ICGC), 2010."},{"key":"e_1_3_2_1_63_1","volume-title":"Megatron-LM: Training Multi- Billion Parameter Language Models Using Model Parallelism. arXiv preprint arXiv:1909.08053","author":"Shoeybi Mohammad","year":"2019","unstructured":"Mohammad Shoeybi, Mostofa Patwary, Raul Puri, Patrick LeGresley, Jared Casper, and Bryan Catanzaro. Megatron-LM: Training Multi- Billion Parameter Language Models Using Model Parallelism. arXiv preprint arXiv:1909.08053, 2019."},{"key":"e_1_3_2_1_64_1","doi-asserted-by":"publisher","DOI":"10.1109\/ISCA.2018.00013"},{"key":"e_1_3_2_1_65_1","doi-asserted-by":"publisher","DOI":"10.3390\/en14020376"},{"key":"e_1_3_2_1_66_1","volume-title":"Preble: Efficient Distributed Prompt Scheduling for LLM Serving. arXiv preprint arXiv:2407.00023","author":"Srivatsa Vikranth","year":"2024","unstructured":"Vikranth Srivatsa, Zijian He, Reyna Abhyankar, Dongming Li, and Yiying Zhang. Preble: Efficient Distributed Prompt Scheduling for LLM Serving. arXiv preprint arXiv:2407.00023, 2024."},{"key":"e_1_3_2_1_67_1","volume-title":"Towards Greener LLMs: Bringing Energy-Efficiency to the Forefront of LLM Inference. arXiv preprint arXiv:2403.20306","author":"Stojkovic Jovan","year":"2024","unstructured":"Jovan Stojkovic, Esha Choukse, Chaojie Zhang, \u00cd\u00f1igo Goiri, and Josep Torrellas. Towards Greener LLMs: Bringing Energy-Efficiency to the Forefront of LLM Inference. arXiv preprint arXiv:2403.20306, 2024."},{"key":"e_1_3_2_1_68_1","volume-title":"Ricardo Bianchini. SmartOClock: Workload- and Risk- Aware Overclocking in the Cloud. In Proceedings of the International Symposium on Computer Architecture (ISCA)","author":"Stojkovic Jovan","year":"2024","unstructured":"Jovan Stojkovic, Pulkit Misra, \u00cd\u00f1igo Goiri, Sam Whitlock, Esha Choukse, Mayukh Das, Chetan Bansal, Jason Lee, Zoey Sun, Haoran Qiu, Reed Zimmermann, Savyasachi Samal, Brijesh Warrier, Ashish Raniwala, and Ricardo Bianchini. SmartOClock: Workload- and Risk- Aware Overclocking in the Cloud. In Proceedings of the International Symposium on Computer Architecture (ISCA), 2024."},{"key":"e_1_3_2_1_69_1","volume-title":"Esha Choukse. DynamoLLM: Designing LLM Inference Clusters for Performance and Energy Efficiency. In Proceedings of the IEEE International Symposium on High Performance Computer Architecture (HPCA)","author":"Stojkovic Jovan","year":"2025","unstructured":"Jovan Stojkovic, Chaojie Zhang, \u00cd\u00f1igo Goiri, Josep Torrellas, and Esha Choukse. DynamoLLM: Designing LLM Inference Clusters for Performance and Energy Efficiency. In Proceedings of the IEEE International Symposium on High Performance Computer Architecture (HPCA), 2025."},{"key":"e_1_3_2_1_70_1","unstructured":"Christopher Tearpak. LLM RAG: Deploy LLM Inference Endpoints & Optimize Output with RAG. https:\/\/techcommunity.microsoft.com\/t5\/startups-at-microsoft\/llm-rag-deploy-llm-inference-endpointsamp-optimize-output-with\/ba-p\/4222636 2024."},{"key":"e_1_3_2_1_71_1","volume-title":"https:\/\/ huggingface.co\/tiiuae\/falcon-180B","author":"Technology Innovation Institute (TII). Falcon-180B.","year":"2024","unstructured":"Technology Innovation Institute (TII). Falcon-180B. https:\/\/ huggingface.co\/tiiuae\/falcon-180B, 2024."},{"key":"e_1_3_2_1_72_1","volume-title":"Llama 2: Open foundation and finetuned chat models. arXiv preprint arXiv:2307.09288","author":"Touvron Hugo","year":"2023","unstructured":"Hugo Touvron, Louis Martin, Kevin Stone, Peter Albert, Amjad Almahairi, Yasmine Babaei, Nikolay Bashlykov, Soumya Batra, Prajjwal Bhargava, Shruti Bhosale, et al. Llama 2: Open foundation and finetuned chat models. arXiv preprint arXiv:2307.09288, 2023."},{"key":"e_1_3_2_1_73_1","doi-asserted-by":"publisher","DOI":"10.1145\/1809049.1809076"},{"key":"e_1_3_2_1_74_1","volume-title":"Tom W Keller. SHIP: Scalable Hierarchical Power Control for Large-Scale Data Centers. In Proceedings of the 18th International Conference on Parallel Architectures and Compilation Techniques (PACT)","author":"Wang Xiaorui","year":"2009","unstructured":"Xiaorui Wang, Ming Chen, Charles Lefurgy, and Tom W Keller. SHIP: Scalable Hierarchical Power Control for Large-Scale Data Centers. In Proceedings of the 18th International Conference on Parallel Architectures and Compilation Techniques (PACT), 2009."},{"key":"e_1_3_2_1_75_1","first-page":"945","volume-title":"Proceedings of the 19th USENIX Symposium on Networked Systems Design and Implementation (NSDI 2022","author":"Weng Qizhen","year":"2022","unstructured":"Qizhen Weng, Wencong Xiao, Yinghao Yu, Wei Wang, Cheng Wang, Jian He, Yong Li, Liping Zhang, Wei Lin, and Yu Ding. MLaaS in the wild: Workload analysis and scheduling in large-scale heterogeneous GPU clusters. In Proceedings of the 19th USENIX Symposium on Networked Systems Design and Implementation (NSDI 2022), pages 945--960, 2022."},{"key":"e_1_3_2_1_76_1","doi-asserted-by":"publisher","DOI":"10.1145\/3632775.3662830"},{"key":"e_1_3_2_1_77_1","volume-title":"LoongServe: Efficiently Serving Long-context Large Language Models with Elastic Sequence Parallelism. arXiv preprint arXiv:2404.09526","author":"Wu Bingyang","year":"2024","unstructured":"Bingyang Wu, Shengyu Liu, Yinmin Zhong, Peng Sun, Xuanzhe Liu, and Xin Jin. LoongServe: Efficiently Serving Long-context Large Language Models with Elastic Sequence Parallelism. arXiv preprint arXiv:2404.09526, 2024."},{"key":"e_1_3_2_1_78_1","volume-title":"Fast distributed inference serving for large language models. arXiv preprint arXiv:2305.05920","author":"Wu Bingyang","year":"2023","unstructured":"Bingyang Wu, Yinmin Zhong, Zili Zhang, Gang Huang, Xuanzhe Liu, and Xin Jin. Fast distributed inference serving for large language models. arXiv preprint arXiv:2305.05920, 2023."},{"key":"e_1_3_2_1_79_1","doi-asserted-by":"publisher","DOI":"10.1109\/ISCA.2016.48"},{"key":"e_1_3_2_1_80_1","doi-asserted-by":"publisher","DOI":"10.1145\/3638757"},{"key":"e_1_3_2_1_81_1","volume-title":"Byung-Gon Chun. Orca: A Distributed Serving System for Transformer-Based Generative Models. In Proceedings of the 16th USENIX Symposium on Operating Systems Design and Implementation (OSDI)","author":"Yu Gyeong-In","year":"2022","unstructured":"Gyeong-In Yu, Joo Seong Jeong, Geon-Woo Kim, Soojeong Kim, and Byung-Gon Chun. Orca: A Distributed Serving System for Transformer-Based Generative Models. In Proceedings of the 16th USENIX Symposium on Operating Systems Design and Implementation (OSDI), 2022."},{"key":"e_1_3_2_1_82_1","volume-title":"Stateful Large Language Model Serving with Pensieve. arXiv preprint arXiv:2312.05516","author":"Yu Lingfan","year":"2023","unstructured":"Lingfan Yu and Jinyang Li. Stateful Large Language Model Serving with Pensieve. arXiv preprint arXiv:2312.05516, 2023."},{"key":"e_1_3_2_1_83_1","volume-title":"Ricardo Bianchini. Flex: High-Availability Datacenters with Zero Reserved Power. In Proceedings of the 48th Annual International Symposium on Computer Architecture (ISCA)","author":"Zhang Chaojie","year":"2021","unstructured":"Chaojie Zhang, Alok Gautam Kumbhare, Ioannis Manousakis, Deli Zhang, Pulkit A. Misra, RodAssis, KyleWoolcock, Nithish Mahalingam, Brijesh Warrier, David Gauthier, Lalu Kunnath, Steve Solomon, Osvaldo Morales, Marcus Fontoura, and Ricardo Bianchini. Flex: High-Availability Datacenters with Zero Reserved Power. In Proceedings of the 48th Annual International Symposium on Computer Architecture (ISCA), 2021."},{"key":"e_1_3_2_1_84_1","doi-asserted-by":"publisher","DOI":"10.1109\/ISCA59077.2024.00077"},{"key":"e_1_3_2_1_85_1","volume-title":"Hao Zhang. DistServe: Disaggregating Prefill and Decoding for Goodput-optimized Large Language Model Serving. In 18th USENIX Symposium on Operating Systems Design and Implementation (OSDI)","author":"Zhong Yinmin","year":"2024","unstructured":"Yinmin Zhong, Shengyu Liu, Junda Chen, Jianbo Hu, Yibo Zhu, Xuanzhe Liu, Xin Jin, and Hao Zhang. DistServe: Disaggregating Prefill and Decoding for Goodput-optimized Large Language Model Serving. In 18th USENIX Symposium on Operating Systems Design and Implementation (OSDI), 2024."},{"key":"e_1_3_2_1_86_1","volume-title":"Guangyu Sun. PetS: A Unified Framework for Parameter-Efficient Transformers Serving. In Proceedings of the USENIX Annual Technical Conference (USENIX ATC)","author":"Zhou Zhe","year":"2022","unstructured":"Zhe Zhou, Xuechao Wei, Jiejing Zhang, and Guangyu Sun. PetS: A Unified Framework for Parameter-Efficient Transformers Serving. In Proceedings of the USENIX Annual Technical Conference (USENIX ATC), 2022."}],"event":{"name":"ASPLOS '25: 30th ACM International Conference on Architectural Support for Programming Languages and Operating Systems","location":"Rotterdam Netherlands","acronym":"ASPLOS '25","sponsor":["SIGPLAN ACM Special Interest Group on Programming Languages","SIGOPS ACM Special Interest Group on Operating Systems","SIGARCH ACM Special Interest Group on Computer Architecture"]},"container-title":["Proceedings of the 30th ACM International Conference on Architectural Support for Programming Languages and Operating Systems, Volume 2"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3676641.3716025","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3676641.3716025","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,8,21]],"date-time":"2025-08-21T11:12:27Z","timestamp":1755774747000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3676641.3716025"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,3,30]]},"references-count":86,"alternative-id":["10.1145\/3676641.3716025","10.1145\/3676641"],"URL":"https:\/\/doi.org\/10.1145\/3676641.3716025","relation":{},"subject":[],"published":{"date-parts":[[2025,3,30]]},"assertion":[{"value":"2025-03-30","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}