{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,23]],"date-time":"2026-07-23T14:10:09Z","timestamp":1784815809598,"version":"3.55.0"},"reference-count":62,"publisher":"IEEE","license":[{"start":{"date-parts":[[2024,9,23]],"date-time":"2024-09-23T00:00:00Z","timestamp":1727049600000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2024,9,23]],"date-time":"2024-09-23T00:00:00Z","timestamp":1727049600000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2024,9,23]]},"DOI":"10.1109\/hpec62836.2024.10938426","type":"proceedings-article","created":{"date-parts":[[2025,4,3]],"date-time":"2025-04-03T19:07:19Z","timestamp":1743707239000},"page":"1-8","source":"Crossref","is-referenced-by-count":32,"title":["LLM Inference Serving: Survey of Recent Advances and Opportunities"],"prefix":"10.1109","author":[{"given":"Baolin","family":"Li","sequence":"first","affiliation":[{"name":"Northeastern University"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Yankai","family":"Jiang","sequence":"additional","affiliation":[{"name":"Northeastern University"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Vijay","family":"Gadepally","sequence":"additional","affiliation":[{"name":"MIT Lincoln Laboratory"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Devesh","family":"Tiwari","sequence":"additional","affiliation":[{"name":"Northeastern University"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"263","reference":[{"key":"ref1","article-title":"Medusa: Simple llm inference acceleration framework with multiple decoding heads","author":"Cai","year":"2024","journal-title":"arXiv preprint"},{"key":"ref2","article-title":"Break the sequential de-pendency of llm inference using lookahead decoding","author":"Fu","year":"2024","journal-title":"arXiv preprint"},{"key":"ref3","first-page":"114","article-title":"Keyformer: Kv cache reduction through key tokens selection for efficient generative inference","volume-title":"Proceedings of Machine Learning and Systems","volume":"6","author":"Adnan","year":"2024"},{"key":"ref4","article-title":"Towards efficient generative large language model serving: A survey from algorithms to systems","author":"Miao","year":"2023","journal-title":"arXiv preprint"},{"key":"ref5","article-title":"Llm inference unveiled: Survey and roofline model insights","author":"Yuan","year":"2024","journal-title":"arXiv preprint"},{"key":"ref6","article-title":"A survey on efficient inference for large language models","author":"Zhou","year":"2024","journal-title":"arXiv preprint"},{"key":"ref7","doi-asserted-by":"publisher","DOI":"10.48550\/ARXIV.1706.03762"},{"key":"ref8","author":"Radford","year":"2018","journal-title":"Improving language understanding by generative pretraining"},{"key":"ref9","first-page":"606","article-title":"Efficiently scaling transformer inference","volume-title":"Proceedings of Machine Learning and Systems","volume":"5","author":"Pope","year":"2023"},{"key":"ref10","doi-asserted-by":"crossref","first-page":"611","DOI":"10.1145\/3600006.3613165","article-title":"Efficient memory management for large language model serving with pagedattention","volume-title":"Proceedings of the 29th Symposium on Operating Systems Principles","author":"Kwon","year":"2023"},{"key":"ref11","volume-title":"Text generation inference: A rust, python and grpc server for text generation inference"},{"key":"ref12","volume-title":"Tensorrt-llm: A tensorrt toolbox for optimized large language model inference"},{"key":"ref13","article-title":"vattention: Dynamic memory management for serving llms without pagedattention","author":"Prabhu","year":"2024","journal-title":"arXiv preprint"},{"key":"ref14","first-page":"325","article-title":"Prompt cache: Modular attention reuse for low-latency inference","volume-title":"Proceedings of Machine Learning and Systems","volume":"6","author":"Gim","year":"2024"},{"key":"ref15","first-page":"111","article-title":"Cost-Efficient large language model serving for multi-turn conversations with CachedAttention","volume-title":"2024 USENIX Annual Technical Conference (USENIX ATC 24)","author":"Gao","year":"2024"},{"key":"ref16","article-title":"Ring attention with blockwise transformers for near-infinite context","author":"Liu","year":"2023","journal-title":"arXiv preprint"},{"key":"ref17","article-title":"Infinite-llm: Efficient llm service for long context with distattention and distributed kvcache","author":"Lin","year":"2024","journal-title":"arXiv preprint"},{"key":"ref18","article-title":"Memserve: Context caching for disaggregated llm serving with elastic memory pool","author":"Hu","year":"2024","journal-title":"arXiv preprint"},{"key":"ref19","article-title":"Infinigen: Efficient generative inference of large language models with dynamic kv cache management","author":"Lee","year":"2024","journal-title":"arXiv preprint"},{"key":"ref20","article-title":"Loongserve: Efficiently serving long-context large language models with elastic sequence parallelism","author":"Wu","year":"2024","journal-title":"arXiv preprint"},{"key":"ref21","first-page":"31094","article-title":"Flexgen: High-throughput generative inference of large language models with a single gpu","volume-title":"International Conference on Machine Learning","author":"Sheng","year":"2023"},{"key":"ref22","article-title":"Kivi: A tuning-free asymmetric 2bit quantization for kv cache","author":"Liu","year":"2024","journal-title":"arXiv preprint"},{"key":"ref23","article-title":"Gear: An efficient kv cache compression recipefor near-lossless generative inference of llm","author":"Kang","year":"2024","journal-title":"arXiv preprint"},{"key":"ref24","article-title":"Minicache: Kv cache compression in depth dimension for large language models","author":"Liu","year":"2024","journal-title":"arXiv preprint"},{"key":"ref25","article-title":"Response length perception and sequence scheduling: An llm-empowered llm inference pipeline","volume":"36","author":"Zheng","year":"2024","journal-title":"Advances in Neural Information Processing Systems"},{"key":"ref26","first-page":"18015","article-title":"S3: Increasing gpu utilization during generative inference for higher throughput","volume":"36","author":"Jin","year":"2023","journal-title":"Advances in Neural Information Processing Systems"},{"key":"ref27","first-page":"521","article-title":"Orca: A distributed serving system for {Transformer-Based} generative models","volume-title":"16th USENIX Symposium on Operating Systems Design and Implementation (OSDI 22)","author":"Yu","year":"2022"},{"key":"ref28","article-title":"Deepspeed-fastgen: High-throughput text generation for llms via mii and deepspeed-inference","author":"Holmes","year":"2024","journal-title":"arXiv preprint"},{"key":"ref29","article-title":"Taming throughput-latency tradeoff in llm inference with sarathi-serve","author":"Agrawal","year":"2024","journal-title":"arXiv preprint"},{"key":"ref30","article-title":"Inference without interference: Disaggre-gate llm inference for mixed downstream workloads","author":"Hu","year":"2024","journal-title":"arXiv preprint"},{"key":"ref31","article-title":"Splitwise: Efficient generative llm inference using phase splitting","author":"Patel","year":"2023","journal-title":"arXiv preprint"},{"key":"ref32","article-title":"Distserve: Disaggregating prefill and decoding for goodput-optimized large language model serving","author":"Zhong","year":"2024","journal-title":"arXiv preprint"},{"key":"ref33","first-page":"162","article-title":"Hetegen: Efficient heterogeneous parallel inference for large language models on resource-constrained devices","volume-title":"Proceedings of Machine Learning and Systems","volume":"6","author":"Xuanlei","year":"2024"},{"key":"ref34","first-page":"369","article-title":"Exegpt: Constraint-aware resource scheduling for llm inference","volume-title":"Proceedings of the 29th ACM International Conference on Architectural Support for Programming Languages and Operating Systems","volume":"2","author":"Oh","year":"2024"},{"key":"ref35","article-title":"Helix: Distributed serving of large language models via max-flow on heterogeneous gpus","author":"Mei","year":"2024","journal-title":"arXiv preprint"},{"key":"ref36","first-page":"1112","article-title":"Spotserve: Serving generative large language models on preemptible instances","volume-title":"Proceedings of the 29th ACM International Conference on Architectural Support for Programming Languages and Operating Systems","volume":"2","author":"Miao","year":"2024"},{"key":"ref37","article-title":"Serverlessllm: Locality-enhanced serverless inference for large language models","author":"Fu","year":"2024","journal-title":"arXiv preprint"},{"key":"ref38","article-title":"M\\\u2019 elange: Cost efficient large language model serving by exploiting gpu heterogeneity","author":"Griggs","year":"2024","journal-title":"arXiv preprint"},{"key":"ref39","article-title":"Llumnix: Dynamic scheduling for large language model serving","author":"Sun","year":"2024","journal-title":"arXiv preprint"},{"key":"ref40","first-page":"207","article-title":"Characterizing power management opportunities for llms in the cloud","volume-title":"Proceedings of the 29th ACM International Conference on Architectural Support for Programming Languages and Operating Systems","volume":"3","author":"Patel","year":"2024"},{"key":"ref41","article-title":"Perllm: Personalized inference scheduling with edge-cloud collaboration for diverse llm services","author":"Yang","year":"2024","journal-title":"arXiv preprint"},{"key":"ref42","article-title":"Flexllm: A system for co-serving large language model inference and parameter-efficient finetuning","author":"Miao","year":"2024","journal-title":"arXiv preprint"},{"key":"ref43","article-title":"Andes: Defining and enhancing quality-of-experience in llm-based text streaming services","author":"Liu","year":"2024","journal-title":"arXiv preprint"},{"key":"ref44","first-page":"9459","article-title":"Retrieval-augmented generation for knowledge-intensive nlp tasks","volume":"33","author":"Lewis","year":"2020","journal-title":"Advances in Neural Information Processing Systems"},{"key":"ref45","article-title":"Retrieval augmentation reduces hallucination in conversation","author":"Shuster","year":"2021","journal-title":"arXiv preprint"},{"key":"ref46","article-title":"Accelerating inference of retrieval-augmented generation via sparse context selection","author":"Zhu","year":"2024","journal-title":"arXiv preprint"},{"key":"ref47","article-title":"Ragcache: Efficient knowledge caching for retrieval-augmented generation","author":"Jin","year":"2024","journal-title":"arXiv preprint"},{"key":"ref48","article-title":"Cacheblend: Fast large language model serving with cached knowledge fusion","author":"Yao","year":"2024","journal-title":"arXiv preprint"},{"key":"ref49","first-page":"945","article-title":"Accelerating distributed {MoE} training and inference with lina","volume-title":"2023 USENIX Annual Technical Conference (USENIX ATC 23)","author":"Li","year":"2023"},{"key":"ref50","doi-asserted-by":"publisher","DOI":"10.1109\/IPDPS57955.2024.00086"},{"key":"ref51","first-page":"224","article-title":"Sida: Sparsity-inspired data-aware serving for efficient and scalable large mixture-of-experts models","volume-title":"Proceedings of Machine Learning and Systems","volume":"6","author":"Du","year":"2024"},{"key":"ref52","article-title":"Moe-infinity: Activation-aware expert offloading for efficient moe serving","author":"Xue","year":"2024","journal-title":"arXiv preprint"},{"key":"ref53","article-title":"Fiddler: Cpu-gpu orchestration for fast inference of mixture-of-experts models","author":"Kamahori","year":"2024","journal-title":"arXiv preprint"},{"key":"ref54","article-title":"Towards moe deployment: Mitigating inefficiencies in mixture-of-expert (moe) inference","author":"Huang","year":"2023","journal-title":"arXiv preprint"},{"key":"ref55","article-title":"Fairness in serving large language models","author":"Sheng","year":"2023","journal-title":"arXiv preprint"},{"key":"ref56","article-title":"Toward sustainable genai using generation directives for carbon-friendly large language model inference","author":"Li","year":"2024","journal-title":"arXiv preprint"},{"key":"ref57","first-page":"148","article-title":"Flashdecoding++: Faster large language model inference with asynchronization, flat gemm optimization, and heuristics","volume-title":"Proceedings of Machine Learning and Systems","volume":"6","author":"Hong","year":"2024"},{"key":"ref58","article-title":"Parrot: Efficient serving of llm-based applications with semantic variable","author":"Lin","year":"2024","journal-title":"arXiv preprint"},{"key":"ref59","article-title":"Flashattention-3: Fast and accurate attention with asynchrony and low-precision","author":"Shah","year":"2024","journal-title":"arXiv preprint"},{"key":"ref60","article-title":"Frugalgpt: How to use large language models while reducing cost and improving performance","author":"Chen","year":"2023","journal-title":"arXiv preprint"},{"key":"ref61","doi-asserted-by":"publisher","DOI":"10.1145\/3620666.3651335"},{"key":"ref62","article-title":"Routellm: Learning to route llms with preference data","author":"Ong","year":"2024","journal-title":"arXiv preprint"}],"event":{"name":"2024 IEEE High Performance Extreme Computing Conference (HPEC)","location":"Wakefield, MA, USA","start":{"date-parts":[[2024,9,23]]},"end":{"date-parts":[[2024,9,27]]}},"container-title":["2024 IEEE High Performance Extreme Computing Conference (HPEC)"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx8\/10938401\/10938415\/10938426.pdf?arnumber=10938426","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,27]],"date-time":"2025-06-27T17:41:43Z","timestamp":1751046103000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/10938426\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,9,23]]},"references-count":62,"URL":"https:\/\/doi.org\/10.1109\/hpec62836.2024.10938426","relation":{},"subject":[],"published":{"date-parts":[[2024,9,23]]}}}