{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,10,1]],"date-time":"2025-10-01T16:29:58Z","timestamp":1759336198711,"version":"build-2065373602"},"publisher-location":"New York, NY, USA","reference-count":83,"publisher":"ACM","content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2025,10,13]]},"DOI":"10.1145\/3731569.3764829","type":"proceedings-article","created":{"date-parts":[[2025,10,1]],"date-time":"2025-10-01T12:43:24Z","timestamp":1759322604000},"page":"375-398","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":0,"title":["IC-Cache: Efficient Large Language Model Serving via In-context Caching"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0009-0009-6477-755X","authenticated-orcid":false,"given":"Yifan","family":"Yu","sequence":"first","affiliation":[{"name":"University of Illinois Urbana-Champaign, Champaign, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-2697-9950","authenticated-orcid":false,"given":"Yu","family":"Gan","sequence":"additional","affiliation":[{"name":"Google, Seattle, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0009-0001-4938-7176","authenticated-orcid":false,"given":"Nikhil","family":"Sarda","sequence":"additional","affiliation":[{"name":"Google, Mountain View, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-6157-1980","authenticated-orcid":false,"given":"Lillian","family":"Tsai","sequence":"additional","affiliation":[{"name":"Google, Sunnyvale, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-0467-4956","authenticated-orcid":false,"given":"Jiaming","family":"Shen","sequence":"additional","affiliation":[{"name":"Google, New York, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-2051-7616","authenticated-orcid":false,"given":"Yanqi","family":"Zhou","sequence":"additional","affiliation":[{"name":"Google, Mountain View, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-9505-9528","authenticated-orcid":false,"given":"Arvind","family":"Krishnamurthy","sequence":"additional","affiliation":[{"name":"Google, Seattle, USA"},{"name":"University of Washington, Seattle, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0009-0005-0472-107X","authenticated-orcid":false,"given":"Fan","family":"Lai","sequence":"additional","affiliation":[{"name":"University of Illinois Urbana-Champaign, Champaign, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0009-0008-7786-8541","authenticated-orcid":false,"given":"Hank","family":"Levy","sequence":"additional","affiliation":[{"name":"Google, Seattle, USA"},{"name":"University of Washington, Seattle, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-0460-9900","authenticated-orcid":false,"given":"David","family":"Culler","sequence":"additional","affiliation":[{"name":"Google, Seattle, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2025,10,12]]},"reference":[{"key":"e_1_3_2_1_1_1","unstructured":"AI Overviews and Your Website. https:\/\/developers.google.com\/search\/docs\/appearance\/ai-overviews."},{"key":"e_1_3_2_1_2_1","unstructured":"Amazon codewhisperer. https:\/\/aws.amazon.com\/codewhisperer\/."},{"key":"e_1_3_2_1_3_1","unstructured":"Anthropic claude. https:\/\/claude.ai\/."},{"key":"e_1_3_2_1_4_1","unstructured":"Character ai. https:\/\/character.ai\/."},{"key":"e_1_3_2_1_5_1","unstructured":"ChatGPT: Introducing deep research. https:\/\/openai.com\/index\/introducing-deep-research\/."},{"key":"e_1_3_2_1_6_1","unstructured":"DeepSeek Context caching. https:\/\/api-docs.deepseek.com\/guides\/kv_cache."},{"key":"e_1_3_2_1_7_1","unstructured":"DeepSeek-V3\/R1 Inference System Overview. https:\/\/github.com\/deepseek-ai\/open-infra-index\/blob\/main\/202502OpenSourceWeek\/day_6_one_more_thing_deepseekV3R1_inference_system_overview.md."},{"key":"e_1_3_2_1_8_1","unstructured":"Gemini 1.5 Flash-8B is now production ready. https:\/\/developers.googleblog.com\/en\/gemini-15-flash-8b-is-now-generally-\\available-for-use\/."},{"key":"e_1_3_2_1_9_1","unstructured":"Gemini Context caching. https:\/\/ai.google.dev\/gemini-api\/docs\/caching?lang=python."},{"key":"e_1_3_2_1_10_1","unstructured":"Github copilot. https:\/\/github.com\/features\/copilot\/."},{"key":"e_1_3_2_1_11_1","unstructured":"Helicone:Open source LLM observability platform. https:\/\/www.helicone.ai\/status\/provider\/Google."},{"key":"e_1_3_2_1_12_1","unstructured":"https:\/\/huggingface.co\/spaces\/lmarena-ai\/chatbot-arena-leaderboard. https:\/\/huggingface.co\/spaces\/lmarena-ai\/chatbot-arena-leaderboard."},{"key":"e_1_3_2_1_13_1","unstructured":"HuggingFace Serverless Inference API. https:\/\/huggingface.co\/docs\/api-inference\/index."},{"key":"e_1_3_2_1_14_1","unstructured":"LangChain: Build context-aware reasoning applications. https:\/\/github.com\/langchain-ai\/langchain."},{"key":"e_1_3_2_1_15_1","unstructured":"MS MARCO. https:\/\/microsoft.github.io\/msmarco\/."},{"key":"e_1_3_2_1_16_1","unstructured":"spaCy: Industrial-strength Natural Language Processing (NLP). https:\/\/github.com\/explosion\/spaCy."},{"key":"e_1_3_2_1_17_1","volume-title":"Building a cost-optimized chatbot with semantic caching. https:\/\/www.databricks.com\/blog\/building-cost-optimized-chatbot-semantic-caching","author":"Databricks","year":"2024","unstructured":"Databricks: Building a cost-optimized chatbot with semantic caching. https:\/\/www.databricks.com\/blog\/building-cost-optimized-chatbot-semantic-caching, 2024."},{"key":"e_1_3_2_1_18_1","volume-title":"Sarathi: Efficient llm inference by piggybacking decodes with chunked prefills. arXiv preprint arXiv:2308.16369","author":"Agrawal Amey","year":"2023","unstructured":"Amey Agrawal, Ashish Panwar, Jayashree Mohan, Nipun Kwatra, Bhargav S Gulavani, and Ramachandran Ramjee. Sarathi: Efficient llm inference by piggybacking decodes with chunked prefills. arXiv preprint arXiv:2308.16369, 2023."},{"key":"e_1_3_2_1_19_1","series-title":"Proceedings of Machine Learning Research","first-page":"39","volume-title":"Proceedings of the 25th Annual Conference on Learning Theory","author":"Agrawal Shipra","unstructured":"Shipra Agrawal and Navin Goyal. Analysis of thompson sampling for the multi-armed bandit problem. In Shie Mannor, Nathan Srebro, and Robert C. Williamson, editors, Proceedings of the 25th Annual Conference on Learning Theory, volume 23 of Proceedings of Machine Learning Research, pages 39.1\u201339.26, Edinburgh, Scotland, 25\u201327 Jun 2012. PMLR."},{"key":"e_1_3_2_1_20_1","volume-title":"Self-rag: Learning to retrieve, generate, and critique through self-reflection. arXiv preprint arXiv:2310.11511","author":"Asai Akari","year":"2023","unstructured":"Akari Asai, Zeqiu Wu, Yizhong Wang, Avirup Sil, and Hannaneh Hajishirzi. Self-rag: Learning to retrieve, generate, and critique through self-reflection. arXiv preprint arXiv:2310.11511, 2023."},{"key":"e_1_3_2_1_21_1","first-page":"218","volume-title":"Proceedings of the 3rd Workshop for Natural Language Processing Open Source Software (NLP-OSS 2023","author":"Bang Fu","year":"2023","unstructured":"Fu Bang. Gptcache: An open-source semantic cache for llm applications enabling faster answers and cost savings. In Proceedings of the 3rd Workshop for Natural Language Processing Open Source Software (NLP-OSS 2023), pages 212\u2013218, 2023."},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"crossref","unstructured":"Ond\u0159ej Bojar Rajen Chatterjee Christian Federmann Yvette Graham Barry Haddow Matthias Huck Antonio Jimeno Yepes Philipp Koehn Varvara Logacheva Christof Monz Matteo Negri Aur\u00e9lie N\u00e9v\u00e9ol Mariana Neves Martin Popel Matt Post Raphael Rubino Carolina Scarton Lucia Specia Marco Turchi Karin Verspoor and Marcos Zampieri. Findings of the 2016 conference on machine translation. In Ond\u0159ej Bojar Christian Buck Rajen Chatterjee Christian Federmann Liane Guillou Barry Haddow Matthias Huck Antonio Jimeno Yepes Aur\u00e9lie N\u00e9v\u00e9ol Mariana Neves Pavel Pecina Martin Popel Philipp Koehn Christof Monz Matteo Negri Matt Post Lucia Specia Karin Verspoor J\u00f6rg Tiedemann and Marco Turchi editors Proceedings of the First Conference on Machine Translation: Volume 2 Shared Task Papers pages 131\u2013198 Berlin Germany August 2016. Association for Computational Linguistics.","DOI":"10.18653\/v1\/W16-2301"},{"key":"e_1_3_2_1_23_1","volume-title":"Chris Leary, Dougal Maclaurin, George Necula, Adam Paszke, Jake VanderPlas, Skye Wanderman-Milne, and Qiao Zhang. JAX: composable transformations of Python+NumPy programs","author":"Bradbury James","year":"2018","unstructured":"James Bradbury, Roy Frostig, Peter Hawkins, Matthew James Johnson, Chris Leary, Dougal Maclaurin, George Necula, Adam Paszke, Jake VanderPlas, Skye Wanderman-Milne, and Qiao Zhang. JAX: composable transformations of Python+NumPy programs, 2018."},{"key":"e_1_3_2_1_24_1","volume-title":"Language models are few-shot learners. arXiv preprint arXiv:2005.14165","author":"Brown Tom B","year":"2020","unstructured":"Tom B Brown. Language models are few-shot learners. arXiv preprint arXiv:2005.14165, 2020."},{"key":"e_1_3_2_1_25_1","volume-title":"NeurIPS","author":"Chen Lingjiao","year":"2024","unstructured":"Lingjiao Chen, Jared Quincy Davis, Boris Hanin, Peter Bailis, Ion Stoica, Matei Zaharia, and James Zou. Are more llm calls all you need? towards scaling laws of compound inference systems. In NeurIPS, 2024."},{"key":"e_1_3_2_1_26_1","volume-title":"Safe rlhf: Safe reinforcement learning from human feedback. arXiv preprint arXiv:2310.12773","author":"Dai Josef","year":"2023","unstructured":"Josef Dai, Xuehai Pan, Ruiyang Sun, Jiaming Ji, Xinbo Xu, Mickel Liu, Yizhou Wang, and Yaodong Yang. Safe rlhf: Safe reinforcement learning from human feedback. arXiv preprint arXiv:2310.12773, 2023."},{"key":"e_1_3_2_1_27_1","volume-title":"NeurIPS","author":"Deudon Michel","year":"2018","unstructured":"Michel Deudon. Learning semantic similarity in a continuous space. In NeurIPS, 2018."},{"key":"e_1_3_2_1_28_1","volume-title":"Arxiv: 2301.00234","author":"Dong Qingxiu","year":"2023","unstructured":"Qingxiu Dong, Lei Li, Damai Dai, Ce Zheng, Jingyuan Ma, Rui Li, Heming Xia, Jingjing Xu, Zhiyong Wu, Tianyu Liu, Baobao Chang, Xu Sun, Lei Li, and Zhifang Sui. A survey on in-context learning. In Arxiv: 2301.00234, 2023."},{"key":"e_1_3_2_1_29_1","volume-title":"Gemini: A family of highly capable multimodal models. arXiv preprint arXiv:2312.11805","author":"Google Gemini Team","year":"2023","unstructured":"Gemini Team Google. Gemini: A family of highly capable multimodal models. arXiv preprint arXiv:2312.11805, 2023."},{"key":"e_1_3_2_1_30_1","volume-title":"The llama 3 herd of models. In arXiv: 2407.21783","author":"Grattafiori Aaron","year":"2024","unstructured":"Aaron Grattafiori, Abhimanyu Dubey, et al. The llama 3 herd of models. In arXiv: 2407.21783, 2024."},{"key":"e_1_3_2_1_31_1","unstructured":"Tom Gunter Zirui Wang Chong Wang Ruoming Pang Andy Narayanan Aonan Zhang Bowen Zhang Chen Chen Chung-Cheng Chiu David Qiu Deepak Gopinath Dian Ang Yap Dong Yin Feng Nan Floris Weers Guoli Yin Haoshuo Huang Jianyu Wang Jiarui Lu John Peebles Ke Ye Mark Lee Nan Du Qibin Chen Quentin Keunebroek Sam Wiseman Syd Evans Tao Lei Vivek Rathod Xiang Kong Xianzhi Du Yanghao Li Yongqiang Wang Yuan Gao Zaid Ahmed Zhaoyang Xu Zhiyun Lu Al Rashid Albin Madappally Jose Alec Doane Alfredo Bencomo Allison Vanderby Andrew Hansen Ankur Jain Anupama Mann Anupama Areeba Kamal Bugu Wu Carolina Brum Charlie Maalouf Chinguun Erdenebileg Chris Dulhanty Dominik Moritz Doug Kang Eduardo Jimenez Evan Ladd Fangping Shi Felix Bai Frank Chu Fred Hohman Hadas Kotek Hannah Gillis Coleman Jane Li Jeffrey Bigham Jeffery Cao Jeff Lai Jessica Cheung Jiulong Shan Joe Zhou John Li Jun Qin Karanjeet Singh Karla Vega Kelvin Zou Laura Heckman Lauren Gardiner Margit Bowler Maria Cordell Meng Cao Nicole Hay Nilesh Shahdadpuri Otto Godwin Pranay Dighe Pushyami Rachapudi Ramsey Tantawi Roman Frigg Sam Davarnia Sanskruti Shah Saptarshi Guha Sasha Sirovica Shen Ma Shuang Ma Simon Wang Sulgi Kim Suma Jayaram Vaishaal Shankar Varsha Paidi Vivek Kumar Xin Wang Xin Zheng Walker Cheng Yael Shrager Yang Ye Yasu Tanaka Yihao Guo Yunsong Meng Zhao Tang Luo Zhi Ouyang Alp Aygar Alvin Wan Andrew Walkingshaw Andy Narayanan Antonie Lin Arsalan Farooq Brent Ramerth Colorado Reed Chris Bartels Chris Chaney David Riazati Eric Liang Yang Erin Feldman Gabriel Hochstrasser Guillaume Seguin Irina Belousova Joris Pelemans Karen Yang Keivan Alizadeh Vahid Liangliang Cao Mahyar Najibi Marco Zuliani Max Horton Minsik Cho Nikhil Bhendawade Patrick Dong Piotr Maj Pulkit Agrawal Qi Shan Qichen Fu Regan Poston Sam Xu Shuangning Liu Sushma Rao Tashweena Heeramun Thomas Merth Uday Rayala Victor Cui Vivek Rangarajan Sridhar Wencong Zhang Wenqi Zhang Wentao Wu Xingyu Zhou Xinwen Liu Yang Zhao Yin Xia Zhile Ren and Zhongzheng Ren. Apple intelligence foundation language models. In Arxiv: 2407.21075 2024."},{"key":"e_1_3_2_1_32_1","volume-title":"A theory of emergent in-context learning as implicit structure induction. In arXiv: 2303.07971","author":"Hahn Michael","year":"2023","unstructured":"Michael Hahn and Navin Goyal. A theory of emergent in-context learning as implicit structure induction. In arXiv: 2303.07971, 2023."},{"key":"e_1_3_2_1_33_1","volume-title":"Inference without interference: Disaggregate llm inference for mixed downstream workloads. arXiv preprint arXiv:2401.11181","author":"Hu Cunchen","year":"2024","unstructured":"Cunchen Hu, Heyang Huang, Liangliang Xu, Xusheng Chen, Jiang Xu, Shuang Chen, Hao Feng, Chenxi Wang, Sa Wang, Yungang Bao, et al. Inference without interference: Disaggregate llm inference for mixed downstream workloads. arXiv preprint arXiv:2401.11181, 2024."},{"key":"e_1_3_2_1_34_1","volume-title":"An empirical study of llm-as-a-judge for llm evaluation: Fine-tuned judge models are task-specific classifiers. arXiv preprint arXiv:2403.02839","author":"Huang Hui","year":"2024","unstructured":"Hui Huang, Yingqi Qu, Jing Liu, Muyun Yang, and Tiejun Zhao. An empirical study of llm-as-a-judge for llm evaluation: Fine-tuned judge models are task-specific classifiers. arXiv preprint arXiv:2403.02839, 2024."},{"key":"e_1_3_2_1_35_1","volume-title":"Evaluation of best-of-n sampling strategies for language model alignment. In arXiv: 2502.12668","author":"Ichihara Yuki","year":"2025","unstructured":"Yuki Ichihara, Yuu Jinnai, Tetsuro Morimura, Kaito Ariu, Kenshi Abe, Mitsuki Sakamoto, and Eiji Uchibe. Evaluation of best-of-n sampling strategies for language model alignment. In arXiv: 2502.12668, 2025."},{"key":"e_1_3_2_1_36_1","volume-title":"Active retrieval augmented generation. arXiv preprint arXiv:2305.06983","author":"Jiang Zhengbao","year":"2023","unstructured":"Zhengbao Jiang, Frank F Xu, Luyu Gao, Zhiqing Sun, Qian Liu, Jane Dwivedi-Yu, Yiming Yang, Jamie Callan, and Graham Neubig. Active retrieval augmented generation. arXiv preprint arXiv:2305.06983, 2023."},{"key":"e_1_3_2_1_37_1","unstructured":"Ziheng Jiang Haibin Lin Yinmin Zhong Qi Huang Yangrui Chen Zhi Zhang Yanghua Peng Xiang Li Cong Xie Shibiao Nong Yulu Jia Sun He Hongmin Chen Zhihao Bai Qi Hou Shipeng Yan Ding Zhou Yiyao Sheng Zhuo Jiang Haohan Xu Haoran Wei Zhang Zhang Pengfei Nie Leqi Zou Sida Zhao Liang Xiang Zherui Liu Zhe Li Xiaoying Jia Jianxi Ye Xin Jin and Xin Liu. MegaScale: Scaling large language model training to more than 10 000 GPUs. In NSDI."},{"key":"e_1_3_2_1_38_1","article-title":"Billion-scale similarity search with GPUs","author":"Johnson Jeff","year":"2019","unstructured":"Jeff Johnson, Matthijs Douze, and Herv\u00e9 J\u00e9gou. Billion-scale similarity search with GPUs. IEEE Transactions on Big Data, 2019.","journal-title":"IEEE Transactions on Big Data"},{"key":"e_1_3_2_1_39_1","volume-title":"NeurIPS","author":"Kim Dongjin","year":"2023","unstructured":"Dongjin Kim, Woojeong Kim, and Suhyun Kim. Tanh works better with asymmetry. In NeurIPS, 2023."},{"key":"e_1_3_2_1_40_1","volume-title":"Tree of clarifications: Answering ambiguous questions with retrieval-augmented large language models. arXiv preprint arXiv:2310.14696","author":"Kim Gangwoo","year":"2023","unstructured":"Gangwoo Kim, Sungdong Kim, Byeongguk Jeon, Joonsuk Park, and Jaewoo Kang. Tree of clarifications: Answering ambiguous questions with retrieval-augmented large language models. arXiv preprint arXiv:2310.14696, 2023."},{"key":"e_1_3_2_1_41_1","volume-title":"Transactions of the Association of Computational Linguistics","author":"Kwiatkowski Tom","year":"2019","unstructured":"Tom Kwiatkowski, Jennimaria Palomaki, Olivia Redfield, Michael Collins, Ankur Parikh, Chris Alberti, Danielle Epstein, Illia Polosukhin, Matthew Kelcey, Jacob Devlin, Kenton Lee, Kristina N. Toutanova, Llion Jones, Ming-Wei Chang, Andrew Dai, Jakob Uszkoreit, Quoc Le, and Slav Petrov. Natural questions: a benchmark for question answering research. Transactions of the Association of Computational Linguistics, 2019."},{"key":"e_1_3_2_1_42_1","volume-title":"SOSP","author":"Kwon Woosuk","year":"2023","unstructured":"Woosuk Kwon, Zhuohan Li, Siyuan Zhuang, Ying Sheng, Lianmin Zheng, Cody Hao Yu, Joseph Gonzalez, Hao Zhang, and Ion Stoica. Efficient memory management for large language model serving with pagedattention. In SOSP, 2023."},{"key":"e_1_3_2_1_43_1","volume-title":"Auto-gda: Automatic domain adaptation for efficient grounding verification in retrieval augmented generation. arXiv preprint arXiv:2410.03461","author":"Leemann Tobias","year":"2024","unstructured":"Tobias Leemann, Periklis Petridis, Giuseppe Vietri, Dionysis Manousakas, Aaron Roth, and Sergul Aydore. Auto-gda: Automatic domain adaptation for efficient grounding verification in retrieval augmented generation. arXiv preprint arXiv:2410.03461, 2024."},{"key":"e_1_3_2_1_44_1","first-page":"9459","article-title":"Retrieval-augmented generation for knowledgeintensive nlp tasks","volume":"33","author":"Lewis Patrick","year":"2020","unstructured":"Patrick Lewis, Ethan Perez, Aleksandra Piktus, Fabio Petroni, Vladimir Karpukhin, Naman Goyal, Heinrich K\u00fcttler, Mike Lewis, Wen-tau Yih, Tim Rockt\u00e4schel, et al. Retrieval-augmented generation for knowledgeintensive nlp tasks. Advances in Neural Information Processing Systems, 33:9459\u20139474, 2020.","journal-title":"Advances in Neural Information Processing Systems"},{"key":"e_1_3_2_1_45_1","volume-title":"VLDB","author":"Li Haoran","year":"2014","unstructured":"Haoran Li, Li Xiong, Lifan Zhang, and Xiaoqian Jiang. Dpsynthesizer: differentially private data synthesizer for privacy preserving data sharing. VLDB, 2014."},{"key":"e_1_3_2_1_46_1","volume-title":"WWW","author":"Li Lihong","year":"2010","unstructured":"Lihong Li, Wei Chu, John Langford, and Robert E. Schapire. A contextual-bandit approach to personalized news article recommendation. In WWW, 2010."},{"key":"e_1_3_2_1_47_1","volume-title":"OSDI","author":"Li Zhuohan","year":"2023","unstructured":"Zhuohan Li, Lianmin Zheng, Yinmin Zhong, Vincent Liu, Ying Sheng, Xin Jin, Yanping Huang, Zhifeng Chen, Hao Zhang, Joseph E. Gonzalez, and Ion Stoica. AlpaServe: Statistical multiplexing with model parallelism for deep learning serving. In OSDI, 2023."},{"key":"e_1_3_2_1_48_1","volume-title":"Adaserve: Slocustomized llm serving with fine-grained speculative decoding. In arXiv: 2501.12162","author":"Li Zikun","year":"2025","unstructured":"Zikun Li, Zhuofu Chen, Remi Delacourt, Gabriele Oliaro, Zeyu Wang, Qinghan Chen, Shuhuai Lin, April Yang, Zhihao Zhang, Zhuoming Chen, Sean Lai, Xupeng Miao, and Zhihao Jia. Adaserve: Slocustomized llm serving with fine-grained speculative decoding. In arXiv: 2501.12162, 2025."},{"key":"e_1_3_2_1_49_1","volume-title":"Chanvichet Vong, and \"Teknium\". Openorca: An open dataset of gpt augmented flan reasoning traces. https:\/\/https:\/\/huggingface.co\/Open-Orca\/OpenOrca","author":"Lian Wing","year":"2023","unstructured":"Wing Lian, Bleys Goodson, Eugene Pentland, Austin Cook, Chanvichet Vong, and \"Teknium\". Openorca: An open dataset of gpt augmented flan reasoning traces. https:\/\/https:\/\/huggingface.co\/Open-Orca\/OpenOrca, 2023."},{"key":"e_1_3_2_1_50_1","volume-title":"OSDI","author":"Lin Chaofan","year":"2024","unstructured":"Chaofan Lin, Zhenhua Han, Chengruidong Zhang, Yuqing Yang, Fan Yang, Chen Chen, and Lili Qiu. Parrot: Efficient serving of llm-based applications with semantic variable. In OSDI, 2024."},{"key":"e_1_3_2_1_51_1","volume-title":"Andes: Defining and enhancing quality-of-experience in llm-based text streaming services","author":"Liu Jiachen","year":"2024","unstructured":"Jiachen Liu, Jae-Won Chung, Zhiyu Wu, Fan Lai, Myungjin Lee, and Mosharaf Chowdhury. Andes: Defining and enhancing quality-of-experience in llm-based text streaming services. 2024."},{"key":"e_1_3_2_1_52_1","volume-title":"In-context learning with retrieved demonstrations for language models: A survey. arXiv preprint arXiv:2401.11624","author":"Luo Man","year":"2024","unstructured":"Man Luo, Xin Xu, Yue Liu, Panupong Pasupat, and Mehran Kazemi. In-context learning with retrieved demonstrations for language models: A survey. arXiv preprint arXiv:2401.11624, 2024."},{"key":"e_1_3_2_1_53_1","volume-title":"Advances in Neural Information Processing Systems (NeurIPS)","author":"Meng Yu","year":"2024","unstructured":"Yu Meng, Mengzhou Xia, and Danqi Chen. Simpo: Simple preference optimization with a reference-free reward. In Advances in Neural Information Processing Systems (NeurIPS), 2024."},{"key":"e_1_3_2_1_54_1","volume-title":"MS MARCO: A human generated machine reading comprehension dataset. CoRR, abs\/1611.09268","author":"Nguyen Tri","year":"2016","unstructured":"Tri Nguyen, Mir Rosenberg, Xia Song, Jianfeng Gao, Saurabh Tiwary, Rangan Majumder, and Li Deng. MS MARCO: A human generated machine reading comprehension dataset. CoRR, abs\/1611.09268, 2016."},{"key":"e_1_3_2_1_55_1","volume-title":"Routellm: Learning to route llms with preference data. In arXiv: 2406.18665","author":"Ong Isaac","year":"2024","unstructured":"Isaac Ong, Amjad Almahairi, Vincent Wu, Wei-Lin Chiang, Tianhao Wu, Joseph E. Gonzalez, M Waleed Kadous, and Ion Stoica. Routellm: Learning to route llms with preference data. In arXiv: 2406.18665, 2024."},{"key":"e_1_3_2_1_56_1","volume-title":"Training language models to follow instructions with human feedback. Advances in neural information processing systems, 35:27730\u201327744","author":"Ouyang Long","year":"2022","unstructured":"Long Ouyang, Jeffrey Wu, Xu Jiang, Diogo Almeida, Carroll Wain-wright, Pamela Mishkin, Chong Zhang, Sandhini Agarwal, Katarina Slama, Alex Ray, et al. Training language models to follow instructions with human feedback. Advances in neural information processing systems, 35:27730\u201327744, 2022."},{"key":"e_1_3_2_1_57_1","first-page":"132","volume-title":"2024 ACM\/IEEE 51st Annual International Symposium on Computer Architecture (ISCA)","author":"Patel Pratyush","unstructured":"Pratyush Patel, Esha Choukse, Chaojie Zhang, Aashaka Shah, \u00cd\u00f1igo Goiri, Saeed Maleki, and Ricardo Bianchini. Splitwise: Efficient generative llm inference using phase splitting. In 2024 ACM\/IEEE 51st Annual International Symposium on Computer Architecture (ISCA), pages 118\u2013132. IEEE, 2024."},{"key":"e_1_3_2_1_58_1","volume-title":"Conserve: Harvesting gpus for low-latency and high-throughput large language model serving. In arXiv: 2410.01228","author":"Qiao Yifan","year":"2024","unstructured":"Yifan Qiao, Shu Anzai, Shan Yu, Haoran Ma, Yang Wang, Miryung Kim, and Harry Xu. Conserve: Harvesting gpus for low-latency and high-throughput large language model serving. In arXiv: 2410.01228, 2024."},{"key":"e_1_3_2_1_59_1","volume-title":"Modserve: Scalable and resource-efficient large multimodal model serving. In arXiv: 2502.00937","author":"Qiu Haoran","year":"2025","unstructured":"Haoran Qiu, Anish Biswas, Zihan Zhao, Jayashree Mohan, Alind Khare, Esha Choukse, Inigo Goiri, Zeyu Zhang, Haiying Shen, Chetan Bansal, Ramachandran Ramjee, and Rodrigo Fonseca. Modserve: Scalable and resource-efficient large multimodal model serving. In arXiv: 2502.00937, 2025."},{"issue":"140","key":"e_1_3_2_1_60_1","first-page":"1","article-title":"Exploring the limits of transfer learning with a unified text-to-text transformer","volume":"21","author":"Raffel Colin","year":"2020","unstructured":"Colin Raffel, Noam Shazeer, Adam Roberts, Katherine Lee, Sharan Narang, Michael Matena, Yanqi Zhou, Wei Li, and Peter J. Liu. Exploring the limits of transfer learning with a unified text-to-text transformer. Journal of Machine Learning Research, 21(140):1\u201367, 2020.","journal-title":"Journal of Machine Learning Research"},{"issue":"4","key":"e_1_3_2_1_61_1","doi-asserted-by":"crossref","first-page":"333","DOI":"10.1561\/1500000019","article-title":"The probabilistic relevance framework: Bm25 and beyond","volume":"3","author":"Robertson Stephen","year":"2009","unstructured":"Stephen Robertson, Hugo Zaragoza, et al. The probabilistic relevance framework: Bm25 and beyond. Foundations and Trends\u00ae in Information Retrieval, 3(4):333\u2013389, 2009.","journal-title":"Foundations and Trends\u00ae in Information Retrieval"},{"key":"e_1_3_2_1_62_1","volume-title":"Enhancing retrieval-augmented large language models with iterative retrieval-generation synergy. arXiv preprint arXiv:2305.15294","author":"Shao Zhihong","year":"2023","unstructured":"Zhihong Shao, Yeyun Gong, Yelong Shen, Minlie Huang, Nan Duan, and Weizhu Chen. Enhancing retrieval-augmented large language models with iterative retrieval-generation synergy. arXiv preprint arXiv:2305.15294, 2023."},{"key":"e_1_3_2_1_63_1","volume-title":"OSDI","author":"Sheng Ying","year":"2024","unstructured":"Ying Sheng, Shiyi Cao, Dacheng Li, Banghua Zhu, Zhuohan Li, Danyang Zhuo, Joseph E. Gonzalez, and Ion Stoica. Fairness in serving large language models. In OSDI, 2024."},{"issue":"1","key":"e_1_3_2_1_64_1","doi-asserted-by":"crossref","first-page":"11","DOI":"10.1108\/eb026526","article-title":"A statistical interpretation of term specificity and its application in retrieval","volume":"28","author":"Jones Karen Sparck","year":"1972","unstructured":"Karen Sparck Jones. A statistical interpretation of term specificity and its application in retrieval. Journal of documentation, 28(1):11\u201321, 1972.","journal-title":"Journal of documentation"},{"key":"e_1_3_2_1_65_1","volume-title":"Hygen: Efficient llm serving via elastic online-offline request co-location. In arXiv: 2501.14808","author":"Sun Ting","year":"2025","unstructured":"Ting Sun, Penghan Wang, and Fan Lai. Hygen: Efficient llm serving via elastic online-offline request co-location. In arXiv: 2501.14808, 2025."},{"key":"e_1_3_2_1_66_1","volume-title":"Stanford alpaca: An instruction-following llama model. https:\/\/github.com\/tatsu-lab\/stanford_alpaca","author":"Taori Rohan","year":"2023","unstructured":"Rohan Taori, Ishaan Gulrajani, Tianyi Zhang, Yann Dubois, Xuechen Li, Carlos Guestrin, Percy Liang, and Tatsunori B. Hashimoto. Stanford alpaca: An instruction-following llama model. https:\/\/github.com\/tatsu-lab\/stanford_alpaca, 2023."},{"key":"e_1_3_2_1_67_1","unstructured":"Gemma Team Morgane Riviere Shreya Pathak Pier Giuseppe Sessa Cassidy Hardin Surya Bhupatiraju L\u00e9onard Hussenot Thomas Mesnard Bobak Shahriari Alexandre Ram\u00e9 et al. Gemma 2: Improving open language models at a practical size. arXiv preprint arXiv:2408.00118 2024."},{"key":"e_1_3_2_1_68_1","volume-title":"ISCA","author":"Wang Jaylen","year":"2024","unstructured":"Jaylen Wang, Daniel S. Berger, Fiodar Kazhamiaka, Celine Irvene, Chaojie Zhang, Esha Choukse, Kali Frost, Rodrigo Fonseca, Brijesh Warrier, Chetan Bansal, Jonathan Stern, Ricardo Bianchini, and Akshitha Sriraman. Designing cloud servers for lower carbon. In ISCA, 2024."},{"key":"e_1_3_2_1_69_1","volume-title":"Emergent abilities of large language models. arXiv preprint arXiv:2206.07682","author":"Wei Jason","year":"2022","unstructured":"Jason Wei, Yi Tay, Rishi Bommasani, Colin Raffel, Barret Zoph, Sebastian Borgeaud, Dani Yogatama, Maarten Bosma, Denny Zhou, Donald Metzler, et al. Emergent abilities of large language models. arXiv preprint arXiv:2206.07682, 2022."},{"key":"e_1_3_2_1_70_1","volume-title":"Fast distributed inference serving for large language models. arXiv preprint arXiv:2305.05920","author":"Wu Bingyang","year":"2023","unstructured":"Bingyang Wu, Yinmin Zhong, Zili Zhang, Shengyu Liu, Fangyue Liu, Yuanhang Sun, Gang Huang, Xuanzhe Liu, and Xin Jin. Fast distributed inference serving for large language models. arXiv preprint arXiv:2305.05920, 2023."},{"key":"e_1_3_2_1_71_1","volume-title":"OSDI","author":"Wu Bingyang","year":"2024","unstructured":"Bingyang Wu, Ruidong Zhu, Zili Zhang, Peng Sun, Xuanzhe Liu, and Xin Jin. dLoRA: Dynamically orchestrating requests and adapters for LoRA LLM serving. In OSDI, 2024."},{"key":"e_1_3_2_1_72_1","volume-title":"ICLR","author":"Wu Shiguang","year":"2025","unstructured":"Shiguang Wu, Yaqing Wang, and Quanming Yao. Why in-context learning models are good few-shot learners? In ICLR, 2025."},{"key":"e_1_3_2_1_73_1","volume-title":"Powerinfer-2: Fast large language model inference on a smartphone. arXiv preprint arXiv:2406.06282","author":"Xue Zhenliang","year":"2024","unstructured":"Zhenliang Xue, Yixin Song, Zeyu Mi, Le Chen, Yubin Xia, and Haibo Chen. Powerinfer-2: Fast large language model inference on a smartphone. arXiv preprint arXiv:2406.06282, 2024."},{"key":"e_1_3_2_1_74_1","volume-title":"Cacheblend: Fast large language model serving with cached knowledge fusion. arXiv preprint arXiv:2405.16444","author":"Yao Jiayi","year":"2024","unstructured":"Jiayi Yao, Hanchen Li, Yuhan Liu, Siddhant Ray, Yihua Cheng, Qizheng Zhang, Kuntai Du, Shan Lu, and Junchen Jiang. Cacheblend: Fast large language model serving with cached knowledge fusion. arXiv preprint arXiv:2405.16444, 2024."},{"key":"e_1_3_2_1_75_1","volume-title":"Generating data for symbolic language with large language models. arXiv preprint arXiv:2305.13917","author":"Ye Jiacheng","year":"2023","unstructured":"Jiacheng Ye, Chengzu Li, Lingpeng Kong, and Tao Yu. Generating data for symbolic language with large language models. arXiv preprint arXiv:2305.13917, 2023."},{"key":"e_1_3_2_1_76_1","volume-title":"ICML","author":"Ye Jiacheng","year":"2023","unstructured":"Jiacheng Ye, Zhiyong Wu, Tao Yu, and Lingpeng Kong. Compositional exemplars for in-context learning. ICML, 2023."},{"key":"e_1_3_2_1_77_1","first-page":"538","volume-title":"16th USENIX Symposium on Operating Systems Design and Implementation (OSDI 22)","author":"Yu Gyeong-In","year":"2022","unstructured":"Gyeong-In Yu, Joo Seong Jeong, Geon-Woo Kim, Soojeong Kim, and Byung-Gon Chun. Orca: A distributed serving system for {Transformer-Based} generative models. In 16th USENIX Symposium on Operating Systems Design and Implementation (OSDI 22), pages 521\u2013538, 2022."},{"key":"e_1_3_2_1_78_1","volume-title":"EMNLP","author":"Zhao Qingfei","year":"2024","unstructured":"Qingfei Zhao, Ruobing Wang, Yukuo Cen, Daren Zha, Shicheng Tan, Yuxiao Dong, and Jie Tang. Longrag: A dual-perspective retrieval-augmented generation paradigm for long-context question answering. EMNLP, 2024."},{"key":"e_1_3_2_1_79_1","volume-title":"Lmsys-chat-1m: A large-scale real-world llm conversation dataset. arXiv: 2309.11998","author":"Zheng Lianmin","year":"2024","unstructured":"Lianmin Zheng, Wei-Lin Chiang, Ying Sheng, Tianle Li, Siyuan Zhuang, Zhanghao Wu, Yonghao Zhuang, Zhuohan Li, Zi Lin, Eric P. Xing, Joseph E. Gonzalez, Ion Stoica, and Hao Zhang. Lmsys-chat-1m: A large-scale real-world llm conversation dataset. arXiv: 2309.11998, 2024."},{"key":"e_1_3_2_1_80_1","volume-title":"NeurIPS","author":"Zheng Lianmin","year":"2023","unstructured":"Lianmin Zheng, Wei-Lin Chiang, Ying Sheng, Siyuan Zhuang, Zhanghao Wu, Yonghao Zhuang, Zi Lin, Zhuohan Li, Dacheng Li, Eric Xing, et al. Judging llm-as-a-judge with mt-bench and chatbot arena. NeurIPS, 2023."},{"key":"e_1_3_2_1_81_1","volume-title":"ASPLOS","author":"Zheng Lianmin","year":"2023","unstructured":"Lianmin Zheng, Liangsheng Yin, Zhiqiang Xie, Chuyue Sun, Jeff Huang, Cody Hao Yu, Shiyi Cao, Christos Kozyrakis, Ion Stoica, Joseph E. Gonzalez, Clark Barrett, and Ying Sheng. Sglang: Efficient execution of structured language model programs. In ASPLOS, 2023."},{"key":"e_1_3_2_1_82_1","volume-title":"Distserve: Disaggregating prefill and decoding for goodput-optimized large language model serving. arXiv preprint arXiv:2401.09670","author":"Zhong Yinmin","year":"2024","unstructured":"Yinmin Zhong, Shengyu Liu, Junda Chen, Jianbo Hu, Yibo Zhu, Xuanzhe Liu, Xin Jin, and Hao Zhang. Distserve: Disaggregating prefill and decoding for goodput-optimized large language model serving. arXiv preprint arXiv:2401.09670, 2024."},{"key":"e_1_3_2_1_83_1","volume-title":"Aditya Krishna Menon, Afshin Rostamizadeh, Sanjiv Kumar, Jean-Francois Kagy, and Rishabh Agarwal. Distillspec: Improving speculative decoding via knowledge distillation","author":"Zhou Yongchao","year":"2024","unstructured":"Yongchao Zhou, Kaifeng Lyu, Ankit Singh Rawat, Aditya Krishna Menon, Afshin Rostamizadeh, Sanjiv Kumar, Jean-Francois Kagy, and Rishabh Agarwal. Distillspec: Improving speculative decoding via knowledge distillation, 2024."}],"event":{"name":"SOSP '25: ACM SIGOPS 31st Symposium on Operating Systems Principles","location":"Lotte Hotel World Seoul Republic of Korea","acronym":"SOSP '25","sponsor":["SIGOPS ACM Special Interest Group on Operating Systems","USENIX"]},"container-title":["Proceedings of the ACM SIGOPS 31st Symposium on Operating Systems Principles"],"original-title":[],"deposited":{"date-parts":[[2025,10,1]],"date-time":"2025-10-01T12:55:45Z","timestamp":1759323345000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3731569.3764829"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,10,12]]},"references-count":83,"alternative-id":["10.1145\/3731569.3764829","10.1145\/3731569"],"URL":"https:\/\/doi.org\/10.1145\/3731569.3764829","relation":{},"subject":[],"published":{"date-parts":[[2025,10,12]]},"assertion":[{"value":"2025-10-12","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}