{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,8,23]],"date-time":"2025-08-23T00:05:01Z","timestamp":1755907501515,"version":"3.44.0"},"publisher-location":"Cham","reference-count":41,"publisher":"Springer Nature Switzerland","isbn-type":[{"value":"9783031998560","type":"print"},{"value":"9783031998577","type":"electronic"}],"license":[{"start":{"date-parts":[[2025,8,23]],"date-time":"2025-08-23T00:00:00Z","timestamp":1755907200000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2025,8,23]],"date-time":"2025-08-23T00:00:00Z","timestamp":1755907200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2026]]},"DOI":"10.1007\/978-3-031-99857-7_24","type":"book-chapter","created":{"date-parts":[[2025,8,22]],"date-time":"2025-08-22T05:16:51Z","timestamp":1755839811000},"page":"335-348","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":0,"title":["EFIM: Efficient Serving of\u00a0LLMs for\u00a0Infilling Tasks with\u00a0Improved KV Cache Reuse"],"prefix":"10.1007","author":[{"ORCID":"https:\/\/orcid.org\/0009-0005-2979-4486","authenticated-orcid":false,"given":"Tianyu","family":"Guo","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Hande","family":"Dong","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Yichong","family":"Leng","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Feng","family":"Liu","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Cheater","family":"Lin","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-2166-977X","authenticated-orcid":false,"given":"Nong","family":"Xiao","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-3507-4299","authenticated-orcid":false,"given":"Xianwei","family":"Zhang","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2025,8,23]]},"reference":[{"key":"24_CR1","doi-asserted-by":"publisher","unstructured":"Aminabadi, R.Y., Rajbhandari, S., Awan, A.A., et\u00a0al.: Deepspeed-inference: enabling efficient inference of transformer models at unprecedented scale. In: SC (2022). https:\/\/doi.org\/10.1109\/SC41404.2022.00051","DOI":"10.1109\/SC41404.2022.00051"},{"key":"24_CR2","doi-asserted-by":"publisher","unstructured":"Anil, R., Borgeaud, S., Wu, Y., et\u00a0al.: Gemini: a family of highly capable multimodal models. arXiv (2023). https:\/\/doi.org\/10.48550\/ARXIV.2312.11805","DOI":"10.48550\/ARXIV.2312.11805"},{"key":"24_CR3","doi-asserted-by":"publisher","unstructured":"Bavarian, M., Jun, H., Tezak, N., et\u00a0al.: Efficient training of language models to fill in the middle. arXiv (2022). https:\/\/doi.org\/10.48550\/ARXIV.2207.14255","DOI":"10.48550\/ARXIV.2207.14255"},{"key":"24_CR4","unstructured":"Brown, T.B., Mann, B., Ryder, N., et\u00a0al.: Language models are few-shot learners. In: NeurIPS (2020)"},{"key":"24_CR5","unstructured":"OpenAI canvas. https:\/\/openai.com\/index\/introducing-canvas\/"},{"key":"24_CR6","unstructured":"Chen, M., Tworek, J., Jun, H., et\u00a0al.: Evaluating large language models trained on code. arXiv (2021)"},{"key":"24_CR7","unstructured":"Chowdhery, A., Narang, S., Devlin, J., et\u00a0al.: Palm: scaling language modeling with pathways. J. Mach. Learn. Res. (2023)"},{"key":"24_CR8","unstructured":"Amazon CodeWhisper. https:\/\/docs.aws.amazon.com\/codewhisperer\/"},{"key":"24_CR9","unstructured":"GitHub Copilot. https:\/\/github.com\/features\/copilot"},{"key":"24_CR10","unstructured":"Dao, T.: Flashattention-2: faster attention with better parallelism and work partitioning. In: ICLR (2024)"},{"key":"24_CR11","unstructured":"Dao, T., Fu, D.Y., Ermon, S., et\u00a0al.: Flashattention: fast and memory-efficient exact attention with io-awareness. In: NeurIPS (2022)"},{"key":"24_CR12","doi-asserted-by":"publisher","unstructured":"DeepSeek-AI, Liu, A., Feng, B., Xue, B., et\u00a0al.: Deepseek-v3 technical report. arXiv (2024). https:\/\/doi.org\/10.48550\/ARXIV.2412.19437","DOI":"10.48550\/ARXIV.2412.19437"},{"key":"24_CR13","unstructured":"DeepSeek V3 serving. https:\/\/zhuanlan.zhihu.com\/p\/27181462601"},{"key":"24_CR14","doi-asserted-by":"publisher","unstructured":"Devlin, J., Chang, M., Lee, K., Toutanova, K.: BERT: pre-training of deep bidirectional transformers for language understanding. In: NAACL-HLT (2019). https:\/\/doi.org\/10.18653\/V1\/N19-1423","DOI":"10.18653\/V1\/N19-1423"},{"key":"24_CR15","unstructured":"Ding, Y., Wang, Z., Ahmad, W.U., et\u00a0al.: Crosscodeeval: a diverse and multilingual benchmark for cross-file code completion. In: NeurIPS (2023)"},{"key":"24_CR16","doi-asserted-by":"publisher","unstructured":"Dubey, A., Jauhri, A., Pandey, A., et\u00a0al.: The llama 3 herd of models. arXiv (2024). https:\/\/doi.org\/10.48550\/ARXIV.2407.21783","DOI":"10.48550\/ARXIV.2407.21783"},{"key":"24_CR17","unstructured":"Fried, D., Aghajanyan, A., Lin, J., et\u00a0al.: Incoder: a generative model for code infilling and synthesis. In: ICLR (2023)"},{"key":"24_CR18","unstructured":"Gao, B., He, Z., Sharma, P., et\u00a0al.: Cost-efficient large language model serving for multi-turn conversations with cached attention. In: ATC (2024)"},{"key":"24_CR19","unstructured":"Gim, I., Chen, G., Lee, S., et\u00a0al.: Prompt cache: modular attention reuse for low-latency inference. In: MLSys (2024)"},{"key":"24_CR20","doi-asserted-by":"publisher","unstructured":"Guo, D., Zhu, Q., Yang, D., et\u00a0al.: Deepseek-coder: when the large language model meets programming - the rise of code intelligence. arXiv (2024). https:\/\/doi.org\/10.48550\/ARXIV.2401.14196","DOI":"10.48550\/ARXIV.2401.14196"},{"key":"24_CR21","doi-asserted-by":"publisher","unstructured":"Guo, T., et al.: EFIM: efficient serving of LLMs for infilling tasks with improved KV cache reuse (2025). https:\/\/doi.org\/10.5281\/zenodo.15580572","DOI":"10.5281\/zenodo.15580572"},{"key":"24_CR22","doi-asserted-by":"publisher","unstructured":"Hui, B., Yang, J., Cui, Z., et\u00a0al.: Qwen2.5-coder technical report. arXiv (2024). https:\/\/doi.org\/10.48550\/ARXIV.2409.12186","DOI":"10.48550\/ARXIV.2409.12186"},{"key":"24_CR23","doi-asserted-by":"publisher","unstructured":"Kwon, W., Li, Z., Zhuang, S., et\u00a0al.: Efficient memory management for large language model serving with pagedattention. In: SOSP (2023). https:\/\/doi.org\/10.1145\/3600006.3613165","DOI":"10.1145\/3600006.3613165"},{"key":"24_CR24","unstructured":"Lee, W., Lee, J., Seo, J., Sim, J.: Infinigen: efficient generative inference of large language models with dynamic KV cache management. In: OSDI (2024)"},{"key":"24_CR25","unstructured":"Li, R., Allal, L.B., Zi, Y., et\u00a0al.: Starcoder: may the source be with you! TMLR (2023)"},{"key":"24_CR26","unstructured":"Llama3.1 model card. https:\/\/huggingface.co\/meta-llama\/Llama-3.1-8B"},{"key":"24_CR27","doi-asserted-by":"publisher","unstructured":"Lozhkov, A., Li, R., Allal, L.B., et\u00a0al.: Starcoder 2 and the stack v2: the next generation. arXiv (2024). https:\/\/doi.org\/10.48550\/ARXIV.2402.19173","DOI":"10.48550\/ARXIV.2402.19173"},{"key":"24_CR28","doi-asserted-by":"publisher","unstructured":"Nijkamp, E., Hayashi, H., Xiong, C., et\u00a0al.: Codegen2: lessons for training llms on programming and natural languages. arXiv (2023). https:\/\/doi.org\/10.48550\/ARXIV.2305.02309","DOI":"10.48550\/ARXIV.2305.02309"},{"key":"24_CR29","doi-asserted-by":"publisher","unstructured":"Nijkamp, E., Pang, B., Hayashi, H., et\u00a0al.: Codegen: an open large language model for code with multi-turn program synthesis. In: ICLR (2023). https:\/\/doi.org\/10.48550\/ARXIV.2312.11805","DOI":"10.48550\/ARXIV.2312.11805"},{"key":"24_CR30","unstructured":"Pope, R., Douglas, S., Chowdhery, A., et\u00a0al.: Efficiently scaling transformer inference. In: MLSys (2023)"},{"key":"24_CR31","doi-asserted-by":"publisher","unstructured":"Rozi\u00e8re, B., Gehring, J., Gloeckle, F., et\u00a0al.: Code llama: open foundation models for code. arXiv (2023). https:\/\/doi.org\/10.48550\/ARXIV.2308.12950","DOI":"10.48550\/ARXIV.2308.12950"},{"key":"24_CR32","unstructured":"Starcoderdata. https:\/\/huggingface.co\/datasets\/bigcode\/starcoderdata"},{"key":"24_CR33","unstructured":"Vaswani, A., Shazeer, N., Parmar, N., et\u00a0al.: Attention is all you need. In: NeurIPS (2017)"},{"key":"24_CR34","unstructured":"Wang, X., Wang, Z., Liu, J., et\u00a0al.: MINT: evaluating llms in multi-turn interaction with tools and language feedback. In: ICLR (2024)"},{"key":"24_CR35","doi-asserted-by":"publisher","unstructured":"Wang, Y., Chen, K., Tan, H., Guo, K.: Tabi: an efficient multi-level inference system for large language models. In: EuroSys (2023). https:\/\/doi.org\/10.1145\/3552326.3587438","DOI":"10.1145\/3552326.3587438"},{"key":"24_CR36","unstructured":"Xiao, G., Tian, Y., Chen, B., et\u00a0al.: Efficient streaming language models with attention sinks. In: ICLR (2024)"},{"key":"24_CR37","doi-asserted-by":"publisher","unstructured":"Ye, L., Tao, Z., Huang, Y., Li, Y.: Chunkattention: efficient self-attention with prefix-aware KV cache and two-phase partition. In: ACL (2024). https:\/\/doi.org\/10.18653\/V1\/2024.ACL-LONG.623","DOI":"10.18653\/V1\/2024.ACL-LONG.623"},{"key":"24_CR38","unstructured":"Yu, G., Jeong, J.S., Kim, G., et\u00a0al.: Orca: a distributed serving system for transformer-based generative models. In: OSDI (2022)"},{"key":"24_CR39","doi-asserted-by":"publisher","unstructured":"Yu, L., Lin, J., Li, J.: Stateful large language model serving with pensieve. In: EuroSys (2025). https:\/\/doi.org\/10.1145\/3689031.3696086","DOI":"10.1145\/3689031.3696086"},{"key":"24_CR40","unstructured":"Zhang, Z., Sheng, Y., Zhou, T., et\u00a0al.: H2O: heavy-hitter oracle for efficient generative inference of large language models. In: NeurIPS (2023)"},{"key":"24_CR41","unstructured":"Zheng, L., Yin, L., Xie, Z., et\u00a0al.: Sglang: efficient execution of structured language model programs. In: NeurIPS (2024)"}],"container-title":["Lecture Notes in Computer Science","Euro-Par 2025: Parallel Processing"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-031-99857-7_24","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,8,22]],"date-time":"2025-08-22T05:17:07Z","timestamp":1755839827000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-3-031-99857-7_24"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,8,23]]},"ISBN":["9783031998560","9783031998577"],"references-count":41,"URL":"https:\/\/doi.org\/10.1007\/978-3-031-99857-7_24","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"value":"0302-9743","type":"print"},{"value":"1611-3349","type":"electronic"}],"subject":[],"published":{"date-parts":[[2025,8,23]]},"assertion":[{"value":"23 August 2025","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"The authors have no competing interests to declare that are relevant to the content of this article.","order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Disclosure of Interests"}},{"value":"Euro-Par","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"European Conference on Parallel Processing","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Dresden","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Germany","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2025","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"25 April 2025","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"29 April 2025","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"31","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"europar2025","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"https:\/\/2025.euro-par.org\/","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}}]}}