{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,5,4]],"date-time":"2026-05-04T09:37:33Z","timestamp":1777887453954,"version":"3.51.4"},"reference-count":22,"publisher":"IEEE","license":[{"start":{"date-parts":[[2025,10,21]],"date-time":"2025-10-21T00:00:00Z","timestamp":1761004800000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2025,10,21]],"date-time":"2025-10-21T00:00:00Z","timestamp":1761004800000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"}],"funder":[{"DOI":"10.13039\/501100001321","name":"National Research Foundation","doi-asserted-by":"publisher","id":[{"id":"10.13039\/501100001321","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2025,10,21]]},"DOI":"10.1109\/mascots67699.2025.11283352","type":"proceedings-article","created":{"date-parts":[[2026,2,16]],"date-time":"2026-02-16T21:02:16Z","timestamp":1771275736000},"page":"1-8","source":"Crossref","is-referenced-by-count":0,"title":["FlexLLM: Flexible and Cost-Efficient LLM Serving with Heterogeneous GPUs"],"prefix":"10.1109","author":[{"given":"Kihyun","family":"Kim","sequence":"first","affiliation":[{"name":"Sogang University,Seoul,Republic of Korea"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Jinwoo","family":"Kim","sequence":"additional","affiliation":[{"name":"Sogang University,Seoul,Republic of Korea"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"James J.","family":"Kim","sequence":"additional","affiliation":[{"name":"Soteria Inc."}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Dong","family":"Li","sequence":"additional","affiliation":[{"name":"University of California,Merced,CA,USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Youngjae","family":"Kim","sequence":"additional","affiliation":[{"name":"Sogang University,Seoul,Republic of Korea"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"263","reference":[{"key":"ref1","article-title":"A survey of large language models","author":"Zhao","year":"2023","journal-title":"arXiv preprint arXiv:2303.18223"},{"key":"ref2","doi-asserted-by":"publisher","DOI":"10.48550\/arxiv.1811.06965"},{"key":"ref3","article-title":"Efficient and economic large language model inference with attention offloading","author":"Chen","year":"2024","journal-title":"arXiv preprint arXiv:2405.01814"},{"key":"ref4","doi-asserted-by":"publisher","DOI":"10.14778\/3415478.3415530"},{"key":"ref5","doi-asserted-by":"publisher","DOI":"10.51593\/20190014"},{"key":"ref6","doi-asserted-by":"publisher","DOI":"10.1145\/3672198.3673797"},{"key":"ref7","article-title":"Openai api pricing","year":"2025","journal-title":"OpenAI"},{"key":"ref8","article-title":"Message batches api: Run jobs up to 50% cheaper","year":"2024","journal-title":"Anthropic"},{"key":"ref9","doi-asserted-by":"publisher","DOI":"10.48550\/ARXIV.1706.03762"},{"key":"ref10","first-page":"117","article-title":"Taming throughput-latency tradeoff in 11 m inference with sarathi-serve","volume-title":"Proc. of the 18th USENIX Symposium on Operating Systems Design and Implementation","author":"Agrawal"},{"key":"ref11","article-title":"NQKV: A KV cache quantization scheme based on normal distribution characteristics","author":"Cai","year":"2025","journal-title":"arXiv preprint arXiv:2505.16210"},{"key":"ref12","first-page":"606","article-title":"Efficiently scaling transformer inference","volume-title":"Proc. of the 5th Conference on Machine Learning and Systems (MLSys)","author":"Pope"},{"key":"ref13","article-title":"Megatron-LM: Training multi-billion parameter language models using model parallelism","author":"Shoeybi","year":"2019","journal-title":"arXiv preprint arXiv:1909.08053"},{"key":"ref14","article-title":"Deepspeed pipeline parallelism tutorial","author":"Team","year":"2025"},{"key":"ref15","article-title":"Demystifying cost-efficiency in llm serving over heterogeneous gpus","author":"Jiang","year":"2025","journal-title":"arXiv preprint arXiv:2502.00722"},{"key":"ref16","doi-asserted-by":"publisher","DOI":"10.1145\/3669940.3707215"},{"key":"ref17","article-title":"Hexgen: Generative inference of large language model over heterogeneous environment","author":"Jiang","year":"2023","journal-title":"arXiv preprint arXiv:2311.11514"},{"key":"ref18","article-title":"Serving models, fast and slow: Optimizing heterogeneous 11 m inferencing workloads at scale","author":"Jaiswal","year":"2025","journal-title":"arXiv preprint arXiv:2502.14617"},{"key":"ref19","doi-asserted-by":"publisher","DOI":"10.1109\/hipcw63042.2024.00088"},{"key":"ref20","article-title":"Predicting llm inference latency: A roofline-driven ml method","volume-title":"Proc. of the Annual Conference on Neural Information Processing Systems (NeurIPS)","author":"Imai"},{"key":"ref21","first-page":"3109431116","article-title":"Flexgen: High-throughput generative inference of large language models with a single gpu","volume-title":"Proc. of the International Conference on Machine Learning (ICML)","author":"Sheng"},{"key":"ref22","doi-asserted-by":"publisher","DOI":"10.1177\/1529100615623267"}],"event":{"name":"2025 33rd International Symposium on Modeling, Analysis and Simulation of Computer and Telecommunication Systems (MASCOTS)","location":"Paris, France","start":{"date-parts":[[2025,10,21]]},"end":{"date-parts":[[2025,10,23]]}},"container-title":["2025 33rd International Symposium on Modeling, Analysis and Simulation of Computer and Telecommunication Systems (MASCOTS)"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx8\/11283137\/11283151\/11283352.pdf?arnumber=11283352","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,5,1]],"date-time":"2026-05-01T04:13:10Z","timestamp":1777608790000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/11283352\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,10,21]]},"references-count":22,"URL":"https:\/\/doi.org\/10.1109\/mascots67699.2025.11283352","relation":{},"subject":[],"published":{"date-parts":[[2025,10,21]]}}}