{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,8,29]],"date-time":"2025-08-29T17:10:12Z","timestamp":1756487412085,"version":"3.44.0"},"publisher-location":"New York, NY, USA","reference-count":52,"publisher":"ACM","license":[{"start":{"date-parts":[[2025,5,14]],"date-time":"2025-05-14T00:00:00Z","timestamp":1747180800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"funder":[{"DOI":"10.13039\/501100006374","name":"National Science Foundation","doi-asserted-by":"publisher","award":["2112562"],"award-info":[{"award-number":["2112562"]}],"id":[{"id":"10.13039\/501100006374","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2025,5,14]]},"DOI":"10.1145\/3713082.3730398","type":"proceedings-article","created":{"date-parts":[[2025,6,6]],"date-time":"2025-06-06T09:53:51Z","timestamp":1749203631000},"page":"179-186","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":0,"title":["Serve Programs, Not Prompts"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-7581-9526","authenticated-orcid":false,"given":"In","family":"Gim","sequence":"first","affiliation":[{"name":"Yale University, New Haven, CT, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-0840-167X","authenticated-orcid":false,"given":"Lin","family":"Zhong","sequence":"additional","affiliation":[{"name":"Yale University, New Haven, CT, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2025,6,6]]},"reference":[{"key":"e_1_3_2_1_1_1","volume-title":"Proc. ICML","author":"Abhyankar R.","year":"2024","unstructured":"R. Abhyankar, Z. He, V. Srivatsa, H. Zhang, and Y. Zhang. InferCept: Efficient intercept support for augmented large language model inference. In Proc. ICML, 2024."},{"key":"e_1_3_2_1_2_1","unstructured":"Anthropic. Prompt caching with claude. https:\/\/www.anthropic.com\/news\/prompt-caching. Accessed: 2025-01-16."},{"key":"e_1_3_2_1_3_1","volume-title":"Longformer: The long-document transformer. CoRR, abs\/2004.05150","author":"Beltagy I.","year":"2020","unstructured":"I. Beltagy, M. E. Peters, and A. Cohan. Longformer: The long-document transformer. CoRR, abs\/2004.05150, 2020."},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.1145\/224056.224077"},{"key":"e_1_3_2_1_5_1","volume-title":"Graph of thoughts: Solving elaborate problems with large language models","author":"Besta M.","year":"2024","unstructured":"M. Besta, N. Blach, A. Kubicek, R. Gerstenberger, M. Podstawski, L. Gianinazzi, J. Gajda, T. Lehmann, H. Niewiadomski, P. Nyczyk, and T. Hoefler. Graph of thoughts: Solving elaborate problems with large language models. 2024."},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"publisher","DOI":"10.1145\/3591300"},{"key":"e_1_3_2_1_7_1","volume-title":"Proc. ICML","author":"Beurer-Kellner L.","year":"2024","unstructured":"L. Beurer-Kellner, M. Fischer, and M. T. Vechev. Guiding llms the right way: Fast, non-invasive constrained generation. In Proc. ICML, 2024."},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.1145\/2656877.2656890"},{"key":"e_1_3_2_1_9_1","volume-title":"Typefly: Flying drones with large language model. CoRR, abs\/2312.14950","author":"Chen G.","year":"2023","unstructured":"G. Chen, X. Yu, and L. Zhong. Typefly: Flying drones with large language model. CoRR, abs\/2312.14950, 2023."},{"key":"e_1_3_2_1_10_1","volume-title":"Proc. ICML","author":"Chen Y.","year":"2024","unstructured":"Y. Chen, X. Pan, Y. Li, B. Ding, and J. Zhou. EE-LLM: large-scale training and inference of early-exit large language models with 3d parallelism. In Proc. ICML, 2024."},{"key":"e_1_3_2_1_11_1","unstructured":"G.Cloud. Context caching overview. https:\/\/cloud.google.com\/vertex-ai\/generative-ai\/docs\/context-cache\/context-cache-overview. Accessed: 2025-01-16."},{"key":"e_1_3_2_1_12_1","volume-title":"Xgrammar: Flexible and efficient structured generation engine for large language models. CoRR, abs\/2411.15100","author":"Dong Y.","year":"2024","unstructured":"Y. Dong, C. F. Ruan, Y. Cai, R. Lai, Z. Xu, Y. Zhao, and T. Chen. Xgrammar: Flexible and efficient structured generation engine for large language models. CoRR, abs\/2411.15100, 2024."},{"key":"e_1_3_2_1_13_1","volume-title":"Outlines: Structured text generation. https:\/\/github.com\/dottxt-ai\/outlines","author":"AI.","year":"2025","unstructured":"Dottxt-AI. Outlines: Structured text generation. https:\/\/github.com\/dottxt-ai\/outlines, 2025. Accessed: 2025-04-16."},{"key":"e_1_3_2_1_14_1","volume-title":"The Llama 3 herd of models. CoRR, abs\/2407.21783","author":"Dubey A.","year":"2024","unstructured":"A. Dubey, A. Jauhri, A. Pandey, A. Kadian, A. Al-Dahle, A. Letman, A. Mathur, A. Schelten, A. Yang, A. Fan, et al. The Llama 3 herd of models. CoRR, abs\/2407.21783, 2024."},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.1145\/3689031.3696072"},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2023.emnlp-main.674"},{"key":"e_1_3_2_1_17_1","volume-title":"Proc. MLSys","author":"Gim I.","year":"2024","unstructured":"I. Gim, G. Chen, S.-S. Lee, N. Sarda, A. Khandelwal, and L. Zhong. Prompt cache: Modular attention reuse for low-latency inference. In Proc. MLSys, 2024."},{"key":"e_1_3_2_1_18_1","volume-title":"Asynchronous LLM function calling. abs\/2412.07017","author":"Gim I.","year":"2024","unstructured":"I. Gim, S.-S. Lee, and L. Zhong. Asynchronous LLM function calling. abs\/2412.07017, 2024."},{"key":"e_1_3_2_1_19_1","volume-title":"Guidance: A guidance language for controlling large language models. https:\/\/github.com\/guidance-ai\/guidance","author":"AI.","year":"2025","unstructured":"Guidance-AI. Guidance: A guidance language for controlling large language models. https:\/\/github.com\/guidance-ai\/guidance, 2025. Accessed: 2025-04-16."},{"key":"e_1_3_2_1_20_1","volume-title":"Proc. ICLR","author":"Gupta N.","year":"2024","unstructured":"N. Gupta, H. Narasimhan, W. Jitkrittum, A. S. Rawat, A. K. Menon, and S. Kumar. Language model cascades: Token-level uncertainty and beyond. In Proc. ICLR, 2024."},{"key":"e_1_3_2_1_21_1","volume-title":"Proc. ICLR","author":"Holtzman A.","year":"2020","unstructured":"A. Holtzman, J. Buys, L. Du, M. Forbes, and Y. Choi. The curious case of neural text degeneration. In Proc. ICLR, 2020."},{"volume-title":"Text generation inference. https:\/\/github.com\/huggingface\/text-generation-inference","year":"2023","key":"e_1_3_2_1_22_1","unstructured":"Hugging-Face. Text generation inference. https:\/\/github.com\/huggingface\/text-generation-inference, 2023. Large Language Model Text Generation Inference Server in Rust and Python."},{"key":"e_1_3_2_1_23_1","volume-title":"RAGO: Systematic performance optimization for retrieval-augmented generation serving. abs\/2503.14649","author":"Jiang W.","year":"2025","unstructured":"W. Jiang, S. Subramanian, C. Graves, G. Alonso, A. Yazdanbakhsh, and V. Dadu. RAGO: Systematic performance optimization for retrieval-augmented generation serving. abs\/2503.14649, 2025."},{"key":"e_1_3_2_1_24_1","volume-title":"Demonstrate-search-predict: Composing retrieval and language models for knowledge-intensive NLP. CoRR, abs\/2212.14024","author":"Khattab O.","year":"2022","unstructured":"O. Khattab, K. Santhanam, X. L. Li, D. Hall, P. Liang, C. Potts, and M. Zaharia. Demonstrate-search-predict: Composing retrieval and language models for knowledge-intensive NLP. CoRR, abs\/2212.14024, 2022."},{"key":"e_1_3_2_1_25_1","volume-title":"An LLM compiler for parallel function calling. abs\/2312.04511","author":"Kim S.","year":"2023","unstructured":"S. Kim, S. Moon, R. Tabrizi, N. Lee, M. W. Mahoney, K. Keutzer, and A. Gholami. An LLM compiler for parallel function calling. abs\/2312.04511, 2023."},{"key":"e_1_3_2_1_26_1","volume-title":"Proc. ICML","author":"Kirchenbauer J.","year":"2023","unstructured":"J. Kirchenbauer, J. Geiping, Y. Wen, J. Katz, I. Miers, and T. Goldstein. A watermark for large language models. In Proc. ICML, 2023."},{"key":"e_1_3_2_1_27_1","volume-title":"Proc. MLSys","author":"Kuchnik M.","year":"2023","unstructured":"M. Kuchnik, V. Smith, and G. Amvrosiadis. Validating large language models with ReLM. In Proc. MLSys, 2023."},{"key":"e_1_3_2_1_28_1","volume-title":"Proc. USENIX OSDI","author":"Kulkarni C.","year":"2018","unstructured":"C. Kulkarni, S. Moore, M. Naqvi, T. Zhang, R. Ricci, and R. Stutsman. Splinter:bare-metal extensions for multi-tenant low-latency storage. In Proc. USENIX OSDI, 2018."},{"key":"e_1_3_2_1_29_1","volume-title":"Certifying LLM safety against adversarial prompting. abs\/2309.02705","author":"Kumar A.","year":"2023","unstructured":"A. Kumar, C. Agarwal, S. Srinivas, S. Feizi, and H. Lakkaraju. Certifying LLM safety against adversarial prompting. abs\/2309.02705, 2023."},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"publisher","DOI":"10.1145\/3600006.3613165"},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2023.findings-acl.40"},{"key":"e_1_3_2_1_32_1","volume-title":"Proc. ICML","author":"Leviathan Y.","year":"2023","unstructured":"Y. Leviathan, M. Kalman, and Y. Matias. Fast inference from transformers via speculative decoding. In Proc. ICML, 2023."},{"key":"e_1_3_2_1_33_1","volume-title":"Proc. USENIX OSDI","author":"Lin C.","year":"2024","unstructured":"C. Lin, Z. Han, C. Zhang, Y. Yang, F. Yang, C. Chen, and L. Qiu. Parrot: Efficient serving of LLM-based applications with semantic variable. In Proc. USENIX OSDI, 2024."},{"key":"e_1_3_2_1_34_1","volume-title":"ebpf - extended berkeley packet filter. https:\/\/ebpf.io\/","author":"Foundation Linux","year":"2024","unstructured":"Linux Foundation. ebpf - extended berkeley packet filter. https:\/\/ebpf.io\/, 2024. Accessed: 2025-04-17."},{"key":"e_1_3_2_1_35_1","doi-asserted-by":"publisher","DOI":"10.1145\/3651890.3672274"},{"key":"e_1_3_2_1_36_1","unstructured":"OpenAI. Prompt caching guide. https:\/\/platform.openai.com\/docs\/guides\/prompt-caching. Accessed: 2025-01-16."},{"key":"e_1_3_2_1_37_1","volume-title":"Parallel function calling","author":"AI.","year":"2023","unstructured":"OpenAI. Parallel function calling, 2023. Accessed: 2024-10-26."},{"key":"e_1_3_2_1_38_1","volume-title":"Proc. NeurIPS","author":"Patil S. G.","year":"2024","unstructured":"S. G. Patil, T. Zhang, X. Wang, and J. E. Gonzalez. Gorilla: Large language model connected with massive apis. In Proc. NeurIPS, 2024."},{"key":"e_1_3_2_1_39_1","volume-title":"Proc. MLSys","author":"Pope R.","year":"2023","unstructured":"R. Pope, S. Douglas, A. Chowdhery, J. Devlin, J. Bradbury, J. Heek, K. Xiao, S. Agrawal, and J. Dean. Efficiently scaling transformer inference. In Proc. MLSys, 2023."},{"key":"e_1_3_2_1_40_1","volume-title":"Improving language understanding by generative pre-training","author":"Radford A.","year":"2018","unstructured":"A. Radford, K. Narasimhan, T. Salimans, I. Sutskever, et al. Improving language understanding by generative pre-training. 2018."},{"key":"e_1_3_2_1_41_1","volume-title":"Proc. ICML","author":"Sheng Y.","year":"2023","unstructured":"Y. Sheng, L. Zheng, B. Yuan, Z. Li, M. Ryabinin, B. Chen, P. Liang, C. R\u00e9, I. Stoica, and C. Zhang. Flexgen: High-throughput generative inference of large language models with a single GPU. In Proc. ICML, 2023."},{"key":"e_1_3_2_1_42_1","doi-asserted-by":"publisher","DOI":"10.1145\/3510003.3510224"},{"key":"e_1_3_2_1_43_1","doi-asserted-by":"publisher","DOI":"10.1038\/s41586-023-06747-5"},{"key":"e_1_3_2_1_44_1","volume-title":"Frontiers of Computer Science","author":"Wang L.","year":"2024","unstructured":"L. Wang, C. Ma, X. Feng, Z. Zhang, H. Yang, J. Zhang, Z. Chen, J. Tang, X. Chen, Y. Lin, et al. A survey on large language model based autonomous agents. Frontiers of Computer Science, 2024."},{"key":"e_1_3_2_1_45_1","volume-title":"Proc. ICML","author":"Wang X.","year":"2024","unstructured":"X. Wang, Y. Chen, L. Yuan, Y. Zhang, Y. Li, H. Peng, and H. Ji. Executable code actions elicit better LLM agents. In Proc. ICML, 2024."},{"key":"e_1_3_2_1_46_1","volume-title":"AutoGen: Enabling next-gen LLM applications via multi-agent conversation framework. abs\/2308.08155","author":"Wu Q.","year":"2023","unstructured":"Q. Wu, G. Bansal, J. Zhang, Y. Wu, S. Zhang, E. Zhu, B. Li, L. Jiang, X. Zhang, and C. Wang. AutoGen: Enabling next-gen LLM applications via multi-agent conversation framework. abs\/2308.08155, 2023."},{"key":"e_1_3_2_1_47_1","volume-title":"Proc. ICLR","author":"Xiao G.","year":"2024","unstructured":"G. Xiao, Y. Tian, B. Chen, S. Han, and M. Lewis. Efficient streaming language models with attention sinks. In Proc. ICLR, 2024."},{"key":"e_1_3_2_1_48_1","volume-title":"Proc. NeurIPS","author":"Yao S.","year":"2023","unstructured":"S. Yao, D. Yu, J. Zhao, I. Shafran, T. Griffiths, Y. Cao, and K. Narasimhan. Tree of thoughts: Deliberate problem solving with large language models. In Proc. NeurIPS, 2023."},{"key":"e_1_3_2_1_49_1","volume-title":"Proc. ICLR","author":"Yao S.","year":"2023","unstructured":"S. Yao, J. Zhao, D. Yu, N. Du, I. Shafran, K. R. Narasimhan, and Y. Cao. ReAct: Synergizing reasoning and acting in language models. In Proc. ICLR, 2023."},{"key":"e_1_3_2_1_50_1","volume-title":"The shift from models to compound AI systems. https:\/\/bair.berkeley.edu\/blog\/2024\/02\/18\/compound-ai-systems\/","author":"Zaharia M.","year":"2024","unstructured":"M. Zaharia, O. Khattab, L. Chen, J. Q. Davis, H. Miller, C. Potts, J. Zou, M. Carbin, J. Frankle, N. Rao, and A. Ghodsi. The shift from models to compound AI systems. https:\/\/bair.berkeley.edu\/blog\/2024\/02\/18\/compound-ai-systems\/, 2024."},{"key":"e_1_3_2_1_51_1","volume-title":"Proc. NeurIPS","author":"Zhang Z.","year":"2023","unstructured":"Z. Zhang, Y. Sheng, T. Zhou, T. Chen, L. Zheng, R. Cai, Z. Song, Y. Tian, C. R\u00e9, C. Barrett, et al. H2o: Heavy-hitter oracle for efficient generative inference of large language models. In Proc. NeurIPS, 2023."},{"key":"e_1_3_2_1_52_1","volume-title":"Proc. NeurIPS","author":"Zheng L.","year":"2024","unstructured":"L. Zheng, L. Yin, Z. Xie, C. Sun, J. Huang, C. H. Yu, S. Cao, C. Kozyrakis, I. Stoica, J. E. Gonzalez, C. W. Barrett, and Y. Sheng. SGLang: Efficient execution of structured language model programs. In Proc. NeurIPS, 2024."}],"event":{"name":"HOTOS '25: Workshop on Hot Topics in Operating Systems","sponsor":["SIGOPS ACM Special Interest Group on Operating Systems"],"location":"Banff AB Canada","acronym":"HOTOS '25"},"container-title":["Proceedings of the Workshop on Hot Topics in Operating Systems"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3713082.3730398","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3713082.3730398","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,8,29]],"date-time":"2025-08-29T16:48:45Z","timestamp":1756486125000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3713082.3730398"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,5,14]]},"references-count":52,"alternative-id":["10.1145\/3713082.3730398","10.1145\/3713082"],"URL":"https:\/\/doi.org\/10.1145\/3713082.3730398","relation":{},"subject":[],"published":{"date-parts":[[2025,5,14]]},"assertion":[{"value":"2025-06-06","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}