{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,27]],"date-time":"2026-10-27T18:01:27Z","timestamp":1793124087309,"version":"4.5.0"},"publisher-location":"New York, NY, USA","reference-count":85,"publisher":"ACM","funder":[{"DOI":"10.13039\/100005326","name":"Yale University","doi-asserted-by":"publisher","id":[{"id":"10.13039\/100005326","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000001","name":"National Science Foundation","doi-asserted-by":"publisher","award":["2112562"],"award-info":[{"award-number":["2112562"]}],"id":[{"id":"10.13039\/100000001","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2025,10,13]]},"DOI":"10.1145\/3731569.3764814","type":"proceedings-article","created":{"date-parts":[[2025,10,1]],"date-time":"2025-10-01T12:43:24Z","timestamp":1759322604000},"page":"415-430","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":4,"title":["Pie: A Programmable Serving System for Emerging LLM Applications"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-7581-9526","authenticated-orcid":false,"given":"In","family":"Gim","sequence":"first","affiliation":[{"name":"Yale University, New Haven, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-1868-4856","authenticated-orcid":false,"given":"Zhiyao","family":"Ma","sequence":"additional","affiliation":[{"name":"Yale University, New Haven, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-5224-8599","authenticated-orcid":false,"given":"Seung-seob","family":"Lee","sequence":"additional","affiliation":[{"name":"Yale University, New Haven, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-0840-167X","authenticated-orcid":false,"given":"Lin","family":"Zhong","sequence":"additional","affiliation":[{"name":"Yale University, New Haven, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2025,10,12]]},"reference":[{"key":"e_1_3_2_1_1_1","volume-title":"Proc. ICML.","author":"Abhyankar Reyna","year":"2024","unstructured":"Reyna Abhyankar, Zijian He, Vikranth Srivatsa, Hao Zhang, and Yiying Zhang. 2024. InferCept: Efficient Intercept Support for Augmented Large Language Model Inference. In Proc. ICML."},{"key":"e_1_3_2_1_2_1","volume-title":"Proc. USENIX OSDI.","author":"Agrawal Amey","year":"2024","unstructured":"Amey Agrawal, Nitin Kedia, Ashish Panwar, Jayashree Mohan, Nipun Kwatra, Bhargav S Gulavani, Alexey Tumanov, and Ramachandran Ramjee. 2024. Taming Throughput-Latency Tradeoff in LLM Inference with Sarathi-Serve. In Proc. USENIX OSDI."},{"key":"e_1_3_2_1_3_1","volume-title":"Proc. IEEE\/ACM SC Conf.","author":"Aminabadi Reza Yazdani","year":"2022","unstructured":"Reza Yazdani Aminabadi, Samyam Rajbhandari, Ammar Ahmad Awan, Cheng Li, Du Li, Elton Zheng, Olatunji Ruwase, Shaden Smith, Minjia Zhang, Jeff Rasley, and Yuxiong He. 2022. DeepSpeed- Inference: Enabling Efficient Inference of Transformer Models at Unprecedented Scale. In Proc. IEEE\/ACM SC Conf."},{"key":"e_1_3_2_1_4_1","unstructured":"Shuai Bai Keqin Chen Xuejing Liu Jialin Wang Wenbin Ge Sibo Song Kai Dang Peng Wang Shijie Wang Jun Tang et al. 2025. Qwen2. 5-vl technical report. arXiv preprint arXiv:2502.13923 (2025)."},{"key":"e_1_3_2_1_5_1","volume-title":"Longformer: The Long-Document Transformer. CoRR abs\/2004.05150","author":"Beltagy Iz","year":"2020","unstructured":"Iz Beltagy, Matthew E. Peters, and Arman Cohan. 2020. Longformer: The Long-Document Transformer. CoRR abs\/2004.05150 (2020)."},{"key":"e_1_3_2_1_6_1","volume-title":"Proc. ACM SOSP.","author":"Bershad Brian N","year":"1995","unstructured":"Brian N Bershad, Stefan Savage, Przemyslaw Pardyak, Emin G\u00fcn Sirer, Marc E Fiuczynski, David Becker, Craig Chambers, and Susan J Eggers. 1995. Extensibility, Safety and Performance in the SPIN Operating System. In Proc. ACM SOSP."},{"key":"e_1_3_2_1_7_1","volume-title":"Proc. AAAI.","author":"Besta Maciej","year":"2024","unstructured":"Maciej Besta, Nils Blach, Ales Kubicek, Robert Gerstenberger, Michal Podstawski, Lukas Gianinazzi, Joanna Gajda, Tomasz Lehmann, Hubert Niewiadomski, Piotr Nyczyk, and Torsten Hoefler. 2024. Graph of Thoughts: Solving Elaborate Problems with Large Language Models. In Proc. AAAI."},{"key":"e_1_3_2_1_8_1","volume-title":"Proceedings of the ACM on Programming Languages 7, PLDI (2023)","author":"Beurer-Kellner Luca","year":"2023","unstructured":"Luca Beurer-Kellner, Marc Fischer, and Martin Vechev. 2023. Prompting is programming: A query language for large language models. Proceedings of the ACM on Programming Languages 7, PLDI (2023), 1946\u20131969."},{"key":"e_1_3_2_1_9_1","volume-title":"Vechev","author":"Beurer-Kellner Luca","year":"2024","unstructured":"Luca Beurer-Kellner, Marc Fischer, and Martin T. Vechev. 2024. Guiding LLMs The Right Way: Fast, Non-Invasive Constrained Generation. In Proc. ICML."},{"key":"e_1_3_2_1_10_1","unstructured":"Bytecode Alliance. 2025. bytecodealliance\/wasmtime: A lightweight WebAssembly runtime that is fast secure and standards-compliant. https:\/\/github.com\/bytecodealliance\/wasmtime. Accessed: 2025-08-26."},{"key":"e_1_3_2_1_11_1","volume-title":"Pyramidkv: Dynamic kv cache compression based on pyramidal information funneling. arXiv preprint arXiv:2406.02069","author":"Cai Zefan","year":"2024","unstructured":"Zefan Cai, Yichi Zhang, Bofei Gao, Yuliang Liu, Yucheng Li, Tianyu Liu, Keming Lu, Wayne Xiong, Yue Dong, Junjie Hu, et al. 2024. Pyramidkv: Dynamic kv cache compression based on pyramidal information funneling. arXiv preprint arXiv:2406.02069 (2024)."},{"key":"e_1_3_2_1_12_1","volume-title":"Proc. NeurIPS.","author":"Dao Tri","year":"2022","unstructured":"Tri Dao, Daniel Y. Fu, Stefano Ermon, Atri Rudra, and Christopher R\u00e9. 2022. FlashAttention: Fast and Memory-Efficient Exact Attention with IO-Awareness. In Proc. NeurIPS."},{"key":"e_1_3_2_1_13_1","volume-title":"XGrammar: Flexible and Efficient Structured Generation Engine for Large Language Models. CoRR abs\/2411.15100","author":"Dong Yixin","year":"2024","unstructured":"Yixin Dong, Charlie F. Ruan, Yaxing Cai, Ruihang Lai, Ziyi Xu, Yilong Zhao, and Tianqi Chen. 2024. XGrammar: Flexible and Efficient Structured Generation Engine for Large Language Models. CoRR abs\/2411.15100 (2024)."},{"key":"e_1_3_2_1_14_1","volume-title":"Outlines: Structured Text Generation. https:\/\/github.com\/dottxt-ai\/outlines. Accessed: 2025-04-16.","author":"AI.","year":"2025","unstructured":"Dottxt-AI. 2025. Outlines: Structured Text Generation. https:\/\/github.com\/dottxt-ai\/outlines. Accessed: 2025-04-16."},{"key":"e_1_3_2_1_15_1","unstructured":"Abhimanyu Dubey Abhinav Jauhri Abhinav Pandey Abhishek Kadian Ahmad Al-Dahle Aiesha Letman Akhil Mathur Alan Schelten Amy Yang Angela Fan et al. 2024. The Llama 3 Herd of Models. CoRR abs\/2407.21783 (2024)."},{"key":"e_1_3_2_1_16_1","volume-title":"Proc. ACM SOSP.","author":"Engler Dawson R","year":"1995","unstructured":"Dawson R Engler, M Frans Kaashoek, and James W O'toole Jr. 1995. Exokernel: An Operating System Architecture for Application-Level Resource Management. In Proc. ACM SOSP."},{"key":"e_1_3_2_1_17_1","volume-title":"Ada-kv: Optimizing kv cache eviction by adaptive budget allocation for efficient llm inference. arXiv preprint arXiv:2407.11550","author":"Feng Yuan","year":"2024","unstructured":"Yuan Feng, Junlin Lv, Yukun Cao, Xike Xie, and S Kevin Zhou. 2024. Ada-kv: Optimizing kv cache eviction by adaptive budget allocation for efficient llm inference. arXiv preprint arXiv:2407.11550 (2024)."},{"key":"e_1_3_2_1_18_1","volume-title":"Beam Search Strategies for Neural Machine Translation. ACL","author":"Freitag Markus","year":"2017","unstructured":"Markus Freitag and Yaser Al-Onaizan. 2017. Beam Search Strategies for Neural Machine Translation. ACL (2017), 56."},{"key":"e_1_3_2_1_19_1","volume-title":"Proc. USENIX OSDI.","author":"Fu Yao","year":"2024","unstructured":"Yao Fu, Leyang Xue, Yeqi Huang, Andrei-Octavian Brabete, Dmitrii Ustiugov, Yuvraj Patel, and Luo Mai. 2024. ServerlessLLM: Low-Latency Serverless Inference for Large Language Models. In Proc. USENIX OSDI."},{"key":"e_1_3_2_1_20_1","volume-title":"Proc. EuroSys.","author":"Gao Shiwei","year":"2025","unstructured":"Shiwei Gao, Youmin Chen, and Jiwu Shu. 2025. Fast State Restoration in LLM Serving with HCache. In Proc. EuroSys."},{"key":"e_1_3_2_1_21_1","volume-title":"Proc. MLSys.","author":"Gim In","year":"2024","unstructured":"In Gim, Guojun Chen, Seung-Seob Lee, Nikhil Sarda, Anurag Khandelwal, and Lin Zhong. 2024. Prompt Cache: Modular Attention Reuse for Low-Latency Inference. In Proc. MLSys."},{"key":"e_1_3_2_1_22_1","volume-title":"Not Prompts. In Proceedings of the 2025 Workshop on Hot Topics in Operating Systems. 179\u2013186","author":"Gim In","year":"2025","unstructured":"In Gim and Lin Zhong. 2025. Serve Programs, Not Prompts. In Proceedings of the 2025 Workshop on Hot Topics in Operating Systems. 179\u2013186."},{"key":"e_1_3_2_1_23_1","volume-title":"Proc. ICML.","author":"Gloeckle Fabian","year":"2024","unstructured":"Fabian Gloeckle, Badr Youbi Idrissi, Baptiste Rozi\u00e8re, David Lopez-Paz, and Gabriel Synnaeve. 2024. Better & Faster Large Language Models via Multi-token Prediction. In Proc. ICML."},{"key":"e_1_3_2_1_24_1","volume-title":"Guidance: A guidance language for controlling large language models. https:\/\/github.com\/guidance-ai\/guidance. Accessed: 2025-04-16.","author":"AI.","year":"2025","unstructured":"Guidance-AI. 2025. Guidance: A guidance language for controlling large language models. https:\/\/github.com\/guidance-ai\/guidance. Accessed: 2025-04-16."},{"key":"e_1_3_2_1_25_1","volume-title":"Proc. ICLR.","author":"Gupta Neha","year":"2024","unstructured":"Neha Gupta, Harikrishna Narasimhan, Wittawat Jitkrittum, Ankit Singh Rawat, Aditya Krishna Menon, and Sanjiv Kumar. 2024. Language Model Cascades: Token-Level Uncertainty And Beyond. In Proc. ICLR."},{"key":"e_1_3_2_1_26_1","volume-title":"Proc. USENIX NSDI.","author":"Hindman Benjamin","year":"2011","unstructured":"Benjamin Hindman, Andy Konwinski, Matei Zaharia, Ali Ghodsi, Anthony D Joseph, Randy H Katz, Scott Shenker, and Ion Stoica. 2011. Mesos: A Platform for Fine-Grained Resource Sharing in the Data Center. In Proc. USENIX NSDI."},{"key":"e_1_3_2_1_27_1","volume-title":"Proc. ICLR.","author":"Holtzman Ari","year":"2020","unstructured":"Ari Holtzman, Jan Buys, Li Du, Maxwell Forbes, and Yejin Choi. 2020. The Curious Case of Neural Text Degeneration. In Proc. ICLR."},{"key":"e_1_3_2_1_28_1","first-page":"2","article-title":"2022. Lora: Low-rank adaptation of large language models","volume":"1","author":"Hu Edward J","year":"2022","unstructured":"Edward J Hu, Yelong Shen, Phillip Wallis, Zeyuan Allen-Zhu, Yuanzhi Li, Shean Wang, Lu Wang, Weizhu Chen, et al. 2022. Lora: Low-rank adaptation of large language models. ICLR 1, 2 (2022), 3.","journal-title":"ICLR"},{"key":"e_1_3_2_1_29_1","unstructured":"Hugging-Face. 2023. Text Generation Inference. https:\/\/github.com\/huggingface\/text-generation-inference. Large Language Model Text Generation Inference Server in Rust and Python."},{"key":"e_1_3_2_1_30_1","volume-title":"RAGO: Systematic Performance Optimization for Retrieval-Augmented Generation Serving. abs\/2503.14649","author":"Jiang Wenqi","year":"2025","unstructured":"Wenqi Jiang, Suvinay Subramanian, Cat Graves, Gustavo Alonso, Amir Yazdanbakhsh, and Vidushi Dadu. 2025. RAGO: Systematic Performance Optimization for Retrieval-Augmented Generation Serving. abs\/2503.14649 (2025)."},{"key":"e_1_3_2_1_31_1","volume-title":"David Hall, Percy Liang, Christopher Potts, and Matei Zaharia.","author":"Khattab Omar","year":"2022","unstructured":"Omar Khattab, Keshav Santhanam, Xiang Lisa Li, David Hall, Percy Liang, Christopher Potts, and Matei Zaharia. 2022. Demonstrate-Search-Predict: Composing retrieval and language models for knowledge-intensive NLP. CoRR abs\/2212.14024 (2022)."},{"key":"e_1_3_2_1_32_1","volume-title":"Proc. ICLR.","author":"Khattab Omar","year":"2024","unstructured":"Omar Khattab, Arnav Singhvi, Paridhi Maheshwari, Zhiyuan Zhang, Keshav Santhanam, Saiful Haq, Ashutosh Sharma, Thomas T Joshi, Hanna Moazam, Heather Miller, et al. 2024. DSPy: Compiling declarative language model calls into state-of-the-art pipelines. In Proc. ICLR."},{"key":"e_1_3_2_1_33_1","volume-title":"An LLM Compiler for Parallel Function Calling. abs\/2312.04511","author":"Kim Sehoon","year":"2023","unstructured":"Sehoon Kim, Suhong Moon, Ryan Tabrizi, Nicholas Lee, Michael W Mahoney, Kurt Keutzer, and Amir Gholami. 2023. An LLM Compiler for Parallel Function Calling. abs\/2312.04511 (2023)."},{"key":"e_1_3_2_1_34_1","volume-title":"Proc. ICML.","author":"Kirchenbauer John","year":"2023","unstructured":"John Kirchenbauer, Jonas Geiping, Yuxin Wen, Jonathan Katz, Ian Miers, and Tom Goldstein. 2023. A Watermark for Large Language Models. In Proc. ICML."},{"key":"e_1_3_2_1_35_1","volume-title":"Proc. MLSys.","author":"Kuchnik Michael","year":"2023","unstructured":"Michael Kuchnik, Virginia Smith, and George Amvrosiadis. 2023. Validating Large Language Models with ReLM. In Proc. MLSys."},{"key":"e_1_3_2_1_36_1","volume-title":"Certifying LLM Safety against Adversarial Prompting. abs\/2309.02705","author":"Kumar Aounon","year":"2023","unstructured":"Aounon Kumar, Chirag Agarwal, Suraj Srinivas, Soheil Feizi, and Hima Lakkaraju. 2023. Certifying LLM Safety against Adversarial Prompting. abs\/2309.02705 (2023)."},{"key":"e_1_3_2_1_37_1","unstructured":"Woosuk Kwon. 2024. [RFC] Drop beam search support. https:\/\/github.com\/vllm-project\/vllm\/issues\/6226. GitHub issue #6226 vLLM Project."},{"key":"e_1_3_2_1_38_1","volume-title":"Proc. ACM SOSP.","author":"Kwon Woosuk","year":"2023","unstructured":"Woosuk Kwon, Zhuohan Li, Siyuan Zhuang, Ying Sheng, Lianmin Zheng, Cody Hao Yu, Joseph Gonzalez, Hao Zhang, and Ion Stoica. 2023. Efficient Memory Management for Large Language Model Serving with PagedAttention. In Proc. ACM SOSP."},{"key":"e_1_3_2_1_39_1","unstructured":"LangChain-AI. 2025. LangChain: Build context-aware reasoning applications. https:\/\/github.com\/langchain-ai\/langchain. Accessed: 2025-04-16."},{"key":"e_1_3_2_1_40_1","volume-title":"Langflow: Low-code AI builder for agentic and RAG applications. https:\/\/www.langflow.org\/. Accessed: 2025-04-16.","year":"2025","unstructured":"Langflow. 2025. Langflow: Low-code AI builder for agentic and RAG applications. https:\/\/www.langflow.org\/. Accessed: 2025-04-16."},{"key":"e_1_3_2_1_41_1","volume-title":"Proc. ACL.","author":"Lee Soochan","year":"2023","unstructured":"Soochan Lee and Gunhee Kim. 2023. Recursion of Thought: A Divide-and-Conquer Approach to Multi-Context Reasoning with Language Models. In Proc. ACL."},{"key":"e_1_3_2_1_42_1","volume-title":"Proc. USENIX OSDI.","author":"Lee Wonbeom","year":"2024","unstructured":"Wonbeom Lee, Jungi Lee, Junghwan Seo, and Jaewoong Sim. 2024. InfiniGen: Efficient Generative Inference of Large Language Models with Dynamic KV Cache Management. In Proc. USENIX OSDI."},{"key":"e_1_3_2_1_43_1","volume-title":"Proc. ICML.","author":"Leviathan Yaniv","year":"2023","unstructured":"Yaniv Leviathan, Matan Kalman, and Yossi Matias. 2023. Fast Inference from Transformers via Speculative Decoding. In Proc. ICML."},{"key":"e_1_3_2_1_44_1","volume-title":"Proc. USENIX OSDI.","author":"Lin Chaofan","year":"2024","unstructured":"Chaofan Lin, Zhenhua Han, Chengruidong Zhang, Yuqing Yang, Fan Yang, Chen Chen, and Lili Qiu. 2024. Parrot: Efficient Serving of LLM-based Applications with Semantic Variable. In Proc. USENIX OSDI."},{"key":"e_1_3_2_1_45_1","volume-title":"Proc. MLSys.","author":"Lin Ji","year":"2024","unstructured":"Ji Lin, Jiaming Tang, Haotian Tang, Shang Yang, Wei-Ming Chen, WeiChen Wang, Guangxuan Xiao, Xingyu Dang, Chuang Gan, and Song Han. 2024. Awq: Activation-aware weight quantization for on-device LLM compression and acceleration. In Proc. MLSys."},{"key":"e_1_3_2_1_46_1","unstructured":"Linux Foundation. 2024. eBPF - Extended Berkeley Packet Filter. https:\/\/ebpf.io\/. Accessed: 2025-04-17."},{"key":"e_1_3_2_1_47_1","volume-title":"Proc. ACM SIGCOMM.","author":"Liu Yuhan","year":"2024","unstructured":"Yuhan Liu, Hanchen Li, Yihua Cheng, Siddhant Ray, Yuyang Huang, Qizheng Zhang, Kuntai Du, Jiayi Yao, Shan Lu, Ganesh Ananthanarayanan, Michael Maire, Henry Hoffmann, Ari Holtzman, and Junchen Jiang. 2024. CacheGen: KV Cache Compression and Streaming for Fast Large Language Model Serving. In Proc. ACM SIGCOMM."},{"key":"e_1_3_2_1_48_1","volume-title":"Autellix: An Efficient Serving Engine for LLM Agents as General Programs. abs\/2502.13965","author":"Luo Michael","year":"2025","unstructured":"Michael Luo, Xiaoxiang Shi, Colin Cai, Tianjun Zhang, Justin Wong, Yichuan Wang, Chi Wang, Yanping Huang, Zhifeng Chen, Joseph E Gonzalez, and Ion Stoica. 2025. Autellix: An Efficient Serving Engine for LLM Agents as General Programs. abs\/2502.13965 (2025)."},{"key":"e_1_3_2_1_49_1","volume-title":"OpenFlow: enabling innovation in campus networks. ACM SIGCOMM computer communication review","author":"McKeown Nick","year":"2008","unstructured":"Nick McKeown, Tom Anderson, Hari Balakrishnan, Guru Parulkar, Larry Peterson, Jennifer Rexford, Scott Shenker, and Jonathan Turner. 2008. OpenFlow: enabling innovation in campus networks. ACM SIGCOMM computer communication review (2008)."},{"key":"e_1_3_2_1_50_1","volume-title":"Proc. ACM SOSP.","author":"Morris Robert","year":"1999","unstructured":"Robert Morris, Eddie Kohler, John Jannotti, and M Frans Kaashoek. 1999. The Click modular router. In Proc. ACM SOSP."},{"key":"e_1_3_2_1_51_1","unstructured":"Michal Moskal Madan Musuvathi and Emre K\u0131c\u0131man. 2024. AI Controller Interface. https:\/\/github.com\/microsoft\/aici\/."},{"key":"e_1_3_2_1_52_1","volume-title":"Proc. ICML.","author":"Ni Ansong","year":"2023","unstructured":"Ansong Ni, Srini Iyer, Dragomir Radev, Veselin Stoyanov, Wen-Tau Yih, Sida I. Wang, and Xi Victoria Lin. 2023. LEVER: Learning to Verify Language-to-Code Generation with Execution. In Proc. ICML."},{"key":"e_1_3_2_1_53_1","volume-title":"Proc. ICLR.","author":"Ning Xuefei","year":"2024","unstructured":"Xuefei Ning, Zinan Lin, Zixuan Zhou, Zifu Wang, Huazhong Yang, and Yu Wang. 2024. Skeleton-of-Thought: Prompting LLMs for Efficient Parallel Generation. In Proc. ICLR."},{"key":"e_1_3_2_1_54_1","unstructured":"OpenAI. 2023. Parallel Function Calling. https:\/\/platform.openai.com\/docs\/guides\/function-calling Accessed: 2024-10-26."},{"key":"e_1_3_2_1_55_1","volume-title":"Grammar-Aligned Decoding. In Proc. NeurIPS.","author":"Park Kanghee","year":"2024","unstructured":"Kanghee Park, Jiayu Wang, Taylor Berg-Kirkpatrick, Nadia Polikarpova, and Loris D'Antoni. 2024. Grammar-Aligned Decoding. In Proc. NeurIPS."},{"key":"e_1_3_2_1_56_1","volume-title":"Proc. NIPS Wrkshp. Autodiff.","author":"Paszke Adam","year":"2017","unstructured":"Adam Paszke, Sam Gross, Soumith Chintala, Gregory Chanan, Edward Yang, Zachary DeVito, Zeming Lin, Alban Desmaison, Luca Antiga, and Adam Lerer. 2017. Automatic differentiation in pytorch. In Proc. NIPS Wrkshp. Autodiff."},{"key":"e_1_3_2_1_57_1","volume-title":"Gonzalez","author":"Patil Shishir G.","year":"2024","unstructured":"Shishir G. Patil, Tianjun Zhang, Xin Wang, and Joseph E. Gonzalez. 2024. Gorilla: Large Language Model Connected with Massive APIs. In Proc. NeurIPS."},{"key":"e_1_3_2_1_58_1","volume-title":"Proc. USENIX OSDI.","author":"Peter Simon","year":"2014","unstructured":"Simon Peter, Jialin Li, Irene Zhang, Dan R K Ports, Doug Woos, Arvind Krishnamurthy, Thomas E Anderson, and Timothy Roscoe. 2014. Arrakis: The Operating System is the Control Plane. In Proc. USENIX OSDI."},{"key":"e_1_3_2_1_59_1","volume-title":"Proc. ACM ASPLOS.","author":"Prabhu Ramya","year":"2025","unstructured":"Ramya Prabhu, Ajay Nayak, Jayashree Mohan, Ramachandran Ramjee, and Ashish Panwar. 2025. vAttention: Dynamic Memory Management for Serving LLMs without PagedAttention. In Proc. ACM ASPLOS."},{"key":"e_1_3_2_1_60_1","volume-title":"Proc. ACM EuroMLSys at EuroSys.","author":"Santhanam Keshav","year":"2024","unstructured":"Keshav Santhanam, Deepti Raghavan, Muhammad Shahir Rahman, Thejas Venkatesh, Neha Kunjal, Pratiksha Thaker, Philip Alexander Levis, and Matei Zaharia. 2024. ALTO: An Efficient Network Orchestrator for Compound AI Systems. In Proc. ACM EuroMLSys at EuroSys."},{"key":"e_1_3_2_1_61_1","volume-title":"Proc. ACL.","author":"Santilli Andrea","year":"2023","unstructured":"Andrea Santilli, Silvio Severino, Emilian Postolache, Valentino Maiorca, Michele Mancusi, Riccardo Marin, and Emanuele Rodol\u00e0. 2023. Accelerating Transformer Inference for Translation via Parallel Decoding. In Proc. ACL."},{"key":"e_1_3_2_1_62_1","unstructured":"Apoorv Saxena. 2023. Prompt Lookup Decoding. https:\/\/github.com\/apoorvumang\/prompt-lookup-decoding\/"},{"key":"e_1_3_2_1_63_1","volume-title":"Proc. NeurIPS.","author":"Schick Timo","year":"2023","unstructured":"Timo Schick, Jane Dwivedi-Yu, Roberto Dess\u00ec, Roberta Raileanu, Maria Lomeli, Eric Hambro, Luke Zettlemoyer, Nicola Cancedda, and Thomas Scialom. 2023. Toolformer: Language Models Can Teach Themselves to Use Tools. In Proc. NeurIPS."},{"key":"e_1_3_2_1_64_1","volume-title":"Proc. ICML.","author":"Sheng Ying","year":"2023","unstructured":"Ying Sheng, Lianmin Zheng, Binhang Yuan, Zhuohan Li, Max Ryabinin, Beidi Chen, Percy Liang, Christopher R\u00e9, Ion Stoica, and Ce Zhang. 2023. FlexGen: High-Throughput Generative Inference of Large Language Models with a Single GPU. In Proc. ICML."},{"key":"e_1_3_2_1_65_1","volume-title":"Proc. IEEE ICCV.","author":"Sur\u00eds D\u00eddac","year":"2023","unstructured":"D\u00eddac Sur\u00eds, Sachit Menon, and Carl Vondrick. 2023. ViperGPT: Visual Inference via Python Execution for Reasoning. In Proc. IEEE ICCV."},{"key":"e_1_3_2_1_66_1","volume-title":"Proc. ACM\/IEEE ICSE.","author":"Tang Ze","year":"2022","unstructured":"Ze Tang, Xiaoyu Shen, Chuanyi Li, Jidong Ge, Liguo Huang, Zhelin Zhu, and Bin Luo. 2022. AST-trans: Code summarization with efficient tree-structured attention. In Proc. ACM\/IEEE ICSE."},{"key":"e_1_3_2_1_67_1","volume-title":"Juliette Love, et al.","author":"Team Gemma","year":"2024","unstructured":"Gemma Team, Thomas Mesnard, Cassidy Hardin, Robert Dadashi, Surya Bhupatiraju, Shreya Pathak, Laurent Sifre, Morgane Rivi\u00e8re, Mihir Sanjay Kale, Juliette Love, et al. 2024. Gemma: Open models based on gemini research and technology. arXiv preprint arXiv:2403.08295 (2024)."},{"key":"e_1_3_2_1_68_1","doi-asserted-by":"crossref","unstructured":"Trieu H Trinh Yuhuai Wu Quoc V Le He He and Thang Luong. 2024. Solving olympiad geometry without human demonstrations. (2024).","DOI":"10.1038\/s41586-023-06747-5"},{"key":"e_1_3_2_1_69_1","volume-title":"BitNet: Scaling 1-bit Transformers for Large Language Models. CoRR abs\/2310.11453","author":"Wang Hongyu","year":"2023","unstructured":"Hongyu Wang, Shuming Ma, Li Dong, Shaohan Huang, Huaijie Wang, Lingxiao Ma, Fan Yang, Ruiping Wang, Yi Wu, and Furu Wei. 2023. BitNet: Scaling 1-bit Transformers for Large Language Models. CoRR abs\/2310.11453 (2023)."},{"key":"e_1_3_2_1_70_1","doi-asserted-by":"crossref","unstructured":"Lei Wang Chen Ma Xueyang Feng Zeyu Zhang Hao Yang Jingsen Zhang Zhiyuan Chen Jiakai Tang Xu Chen Yankai Lin et al. 2024. A survey on large language model based autonomous agents. Frontiers of Computer Science (2024).","DOI":"10.1007\/s11704-024-40231-1"},{"key":"e_1_3_2_1_71_1","volume-title":"Proc. ICML.","author":"Wang Xingyao","year":"2024","unstructured":"Xingyao Wang, Yangyi Chen, Lifan Yuan, Yizhe Zhang, Yunzhu Li, Hao Peng, and Heng Ji. 2024. Executable Code Actions Elicit Better LLM Agents. In Proc. ICML."},{"key":"e_1_3_2_1_72_1","unstructured":"Wasmtime Project. 2025. PoolingAllocationConfig \u2014 Wasmtime API Documentation. Wasmtime Documentation. https:\/\/docs.wasmtime.dev\/api\/wasmtime\/struct.PoolingAllocationConfig.html Accessed: 2025-08-26."},{"key":"e_1_3_2_1_73_1","volume-title":"AutoGen: Enabling Next-Gen LLM Applications via Multi-Agent Conversation Framework. abs\/2308.08155","author":"Wu Qingyun","year":"2023","unstructured":"Qingyun Wu, Gagan Bansal, Jieyu Zhang, Yiran Wu, Shaokun Zhang, Erkang Zhu, Beibin Li, Li Jiang, Xiaoyun Zhang, and Chi Wang. 2023. AutoGen: Enabling Next-Gen LLM Applications via Multi-Agent Conversation Framework. abs\/2308.08155 (2023)."},{"key":"e_1_3_2_1_74_1","volume-title":"Proc. ICLR.","author":"Xiao Guangxuan","year":"2024","unstructured":"Guangxuan Xiao, Yuandong Tian, Beidi Chen, Song Han, and Mike Lewis. 2024. Efficient Streaming Language Models with Attention Sinks. In Proc. ICLR."},{"key":"e_1_3_2_1_75_1","volume-title":"Proc. NeurIPS.","author":"Yao Shunyu","year":"2023","unstructured":"Shunyu Yao, Dian Yu, Jeffrey Zhao, Izhak Shafran, Tom Griffiths, Yuan Cao, and Karthik Narasimhan. 2023. Tree of Thoughts: Deliberate Problem Solving with Large Language Models. In Proc. NeurIPS."},{"key":"e_1_3_2_1_76_1","volume-title":"Proc. ICLR.","author":"Yao Shunyu","year":"2023","unstructured":"Shunyu Yao, Jeffrey Zhao, Dian Yu, Nan Du, Izhak Shafran, Karthik R Narasimhan, and Yuan Cao. 2023. ReAct: Synergizing Reasoning and Acting in Language Models. In Proc. ICLR."},{"key":"e_1_3_2_1_77_1","volume-title":"Proc. ACL.","author":"Ye Lu","year":"2024","unstructured":"Lu Ye, Ze Tao, Yong Huang, and Yang Li. 2024. ChunkAttention: Efficient Self-Attention with Prefix-Aware KV Cache and Two-Phase Partition. In Proc. ACL."},{"key":"e_1_3_2_1_78_1","volume-title":"FlashInfer: Efficient and Customizable Attention Engine for LLM Inference Serving. CoRR abs\/2501.01005","author":"Ye Zihao","year":"2025","unstructured":"Zihao Ye, Lequn Chen, Ruihang Lai, Wuwei Lin, Yineng Zhang, Stephanie Wang, Tianqi Chen, Baris Kasikci, Vinod Grover, Arvind Krishnamurthy, and Luis Ceze. 2025. FlashInfer: Efficient and Customizable Attention Engine for LLM Inference Serving. CoRR abs\/2501.01005 (2025)."},{"key":"e_1_3_2_1_79_1","volume-title":"Proc. USENIX OSDI.","author":"Yu Gyeong-In","year":"2022","unstructured":"Gyeong-In Yu, Joo Seong Jeong, Geon-Woo Kim, Soojeong Kim, and Byung-Gon Chun. 2022. Orca: A Distributed Serving System for Transformer-Based Generative Models. In Proc. USENIX OSDI."},{"key":"e_1_3_2_1_80_1","volume-title":"Proc. EuroSys.","author":"Yu Lingfan","year":"2025","unstructured":"Lingfan Yu, Jinkun Lin, and Jinyang Li. 2025. Stateful Large Language Model Serving with Pensieve. In Proc. EuroSys."},{"key":"e_1_3_2_1_81_1","volume-title":"Heather Miller, Chris Potts, James Zou, Michael Carbin, Jonathan Frankle, Naveen Rao, and Ali Ghodsi.","author":"Zaharia Matei","year":"2024","unstructured":"Matei Zaharia, Omar Khattab, Lingjiao Chen, Jared Quincy Davis, Heather Miller, Chris Potts, James Zou, Michael Carbin, Jonathan Frankle, Naveen Rao, and Ali Ghodsi. 2024. The Shift from Models to Compound AI Systems. https:\/\/bair.berkeley.edu\/blog\/2024\/02\/18\/compound-ai-systems\/."},{"key":"e_1_3_2_1_82_1","unstructured":"ZeroMQ Community. 2023. libzmq: ZeroMQ core engine in C++. https:\/\/github.com\/zeromq\/libzmq."},{"key":"e_1_3_2_1_83_1","volume-title":"Proc. USENIX OSDI.","author":"Zheng Lianmin","year":"2022","unstructured":"Lianmin Zheng, Zhuohan Li, Hao Zhang, Yonghao Zhuang, Zhifeng Chen, Yanping Huang, Yida Wang, Yuanzhong Xu, Danyang Zhuo, Eric P. Xing, Joseph E. Gonzalez, and Ion Stoica. 2022. Alpa: Automating Inter- and Intra-Operator Parallelism for Distributed Deep Learning. In Proc. USENIX OSDI."},{"key":"e_1_3_2_1_84_1","volume-title":"Proc. NeurIPS.","author":"Zheng Lianmin","year":"2024","unstructured":"Lianmin Zheng, Liangsheng Yin, Zhiqiang Xie, Chuyue Sun, Jeff Huang, Cody Hao Yu, Shiyi Cao, Christos Kozyrakis, Ion Stoica, Joseph E Gonzalez, Clark W Barrett, and Ying Sheng. 2024. SGLang: Efficient Execution of Structured Language Model Programs. In Proc. NeurIPS."},{"key":"e_1_3_2_1_85_1","volume-title":"Proc. ICML.","author":"Zhuge Mingchen","year":"2024","unstructured":"Mingchen Zhuge, Wenyi Wang, Louis Kirsch, Francesco Faccio, Dmitrii Khizbullin, and J\u00fcrgen Schmidhuber. 2024. Gptswarm: Language agents as optimizable graphs. In Proc. ICML."}],"event":{"name":"SOSP '25: ACM SIGOPS 31st Symposium on Operating Systems Principles","location":"Lotte Hotel World Seoul Republic of Korea","acronym":"SOSP '25","sponsor":["SIGOPS ACM Special Interest Group on Operating Systems","USENIX"]},"container-title":["Proceedings of the ACM SIGOPS 31st Symposium on Operating Systems Principles"],"original-title":[],"deposited":{"date-parts":[[2025,10,1]],"date-time":"2025-10-01T12:51:58Z","timestamp":1759323118000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3731569.3764814"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,10,12]]},"references-count":85,"alternative-id":["10.1145\/3731569.3764814","10.1145\/3731569"],"URL":"https:\/\/doi.org\/10.1145\/3731569.3764814","relation":{},"subject":[],"published":{"date-parts":[[2025,10,12]]},"assertion":[{"value":"2025-10-12","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}