{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,10,11]],"date-time":"2025-10-11T02:54:27Z","timestamp":1760151267324,"version":"build-2065373602"},"reference-count":57,"publisher":"Institute of Electrical and Electronics Engineers (IEEE)","issue":"5","license":[{"start":{"date-parts":[[2025,9,1]],"date-time":"2025-09-01T00:00:00Z","timestamp":1756684800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/ieeexplore.ieee.org\/Xplorehelp\/downloads\/license-information\/IEEE.html"},{"start":{"date-parts":[[2025,9,1]],"date-time":"2025-09-01T00:00:00Z","timestamp":1756684800000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2025,9,1]],"date-time":"2025-09-01T00:00:00Z","timestamp":1756684800000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"}],"funder":[{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["92267105"],"award-info":[{"award-number":["92267105"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100021171","name":"Basic and Applied Basic Research Foundation of Guangdong Province","doi-asserted-by":"publisher","award":["2023B1515130002"],"award-info":[{"award-number":["2023B1515130002"]}],"id":[{"id":"10.13039\/501100021171","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100018601","name":"Guangdong Special Support Plan","doi-asserted-by":"publisher","award":["2021TQ06X990"],"award-info":[{"award-number":["2021TQ06X990"]}],"id":[{"id":"10.13039\/501100018601","id-type":"DOI","asserted-by":"publisher"}]},{"name":"Shenzhen Basic Research Program","award":["JCYJ20220818101610023","KJZD20230923113800001"],"award-info":[{"award-number":["JCYJ20220818101610023","KJZD20230923113800001"]}]}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["IEEE Trans. Serv. Comput."],"published-print":{"date-parts":[[2025,9]]},"DOI":"10.1109\/tsc.2025.3603576","type":"journal-article","created":{"date-parts":[[2025,8,28]],"date-time":"2025-08-28T18:09:50Z","timestamp":1756404590000},"page":"3164-3176","source":"Crossref","is-referenced-by-count":0,"title":["Serving LLM in Distributed GPU Cluster With Fine-Grain Pipeline Constraints"],"prefix":"10.1109","volume":"18","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-4809-9543","authenticated-orcid":false,"given":"Yanying","family":"Lin","sequence":"first","affiliation":[{"name":"Shenzhen Institutes of Advanced Technology, Chinese Academy of Sciences, Shenzhen, China"}]},{"ORCID":"https:\/\/orcid.org\/0009-0005-1776-4773","authenticated-orcid":false,"given":"Shijie","family":"Peng","sequence":"additional","affiliation":[{"name":"Shenzhen Institutes of Advanced Technology, Chinese Academy of Sciences, Shenzhen, China"}]},{"ORCID":"https:\/\/orcid.org\/0009-0002-4764-2601","authenticated-orcid":false,"given":"Shuaipeng","family":"Wu","sequence":"additional","affiliation":[{"name":"Shenzhen Institutes of Advanced Technology, Chinese Academy of Sciences, Shenzhen, China"}]},{"given":"Yanbo","family":"Li","sequence":"additional","affiliation":[{"name":"Shenzhen Institutes of Advanced Technology, Chinese Academy of Sciences, Shenzhen, China"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-6421-3828","authenticated-orcid":false,"given":"Chengzhi","family":"Lu","sequence":"additional","affiliation":[{"name":"Shenzhen Institutes of Advanced Technology, Chinese Academy of Sciences, Shenzhen, China"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-6133-407X","authenticated-orcid":false,"given":"Kejiang","family":"Ye","sequence":"additional","affiliation":[{"name":"Shenzhen Institutes of Advanced Technology, Chinese Academy of Sciences, Shenzhen, China"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-9480-0356","authenticated-orcid":false,"given":"Chengzhong","family":"Xu","sequence":"additional","affiliation":[{"name":"University of Macau, Macau, China"}]}],"member":"263","reference":[{"article-title":"LLaMA: Open and efficient foundation language models","year":"2023","author":"Touvron","key":"ref1"},{"article-title":"Robust speech recognition via large-scale weak supervision","volume-title":"Proc. 40th Int. Conf. Mach. Learn.","author":"Radford","key":"ref2"},{"article-title":"PaLM 2 technical report","year":"2023","author":"Anil","key":"ref3"},{"article-title":"Llama 2: Open foundation and fine-tuned chat models","year":"2023","author":"Touvron","key":"ref4"},{"article-title":"Mixture-of-experts meets instruction tuning: A winning combination for large language models","year":"2023","author":"Shen","key":"ref5"},{"key":"ref6","doi-asserted-by":"publisher","DOI":"10.1145\/3603269.3604830"},{"key":"ref7","first-page":"663","article-title":"AlpaServe: Statistical multiplexing with model parallelism for deep learning serving","volume-title":"Proc. USENIX Symp. Operating Syst. Des. Implementation","author":"Li"},{"key":"ref8","doi-asserted-by":"publisher","DOI":"10.1145\/3600006.3613165"},{"key":"ref9","doi-asserted-by":"publisher","DOI":"10.1145\/3552326.3567508"},{"key":"ref10","first-page":"173","article-title":"Llumnix: Dynamic scheduling for large language model serving","volume-title":"Proc. USENIX Symp. Operating Syst. Des. Implementation","author":"Sun"},{"key":"ref11","doi-asserted-by":"publisher","DOI":"10.1145\/3620666.3651359"},{"key":"ref12","first-page":"46745","article-title":"DejaVu: KV-cache streaming for fast, fault-tolerant generative LLM serving","volume-title":"Proc. 41st Int. Conf. Mach. Learn.","author":"Strati"},{"key":"ref13","first-page":"709","article-title":"Characterization of large language model development in the datacenter","volume-title":"Proc. USENIX Symp. Netw. Syst. Des. Implementation","author":"Hu"},{"key":"ref14","first-page":"24829","article-title":"Piper: Multidimensional planner for DNN parallelization","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Tarnawski"},{"key":"ref15","doi-asserted-by":"publisher","DOI":"10.1145\/3437801.3441593"},{"key":"ref16","doi-asserted-by":"publisher","DOI":"10.1109\/SC41405.2020.00024"},{"key":"ref17","first-page":"307","article-title":"HetPipe: Enabling large DNN training on (whimpy) heterogeneous GPU clusters through integration of pipelined model parallelism and data parallelism","volume-title":"Proc. USENIX Conf. Usenix Annu. Tech. Conf.","author":"Park"},{"key":"ref18","doi-asserted-by":"publisher","DOI":"10.1109\/ICDCS60910.2024.00010"},{"key":"ref19","first-page":"559","article-title":"Alpa: Automating inter- and intra-operator parallelism for distributed deep learning","volume-title":"Proc. USENIX Symp. Operating Syst. Des. Implementation","author":"Zheng"},{"key":"ref20","doi-asserted-by":"publisher","DOI":"10.1145\/3341301.3359646"},{"key":"ref21","first-page":"787","article-title":"SHEPHERD: Serving DNNs in the wild","volume-title":"Proc. USENIX Symp. Netw. Syst. Des. Implementation","author":"Zhang"},{"key":"ref22","doi-asserted-by":"publisher","DOI":"10.1109\/SC41404.2022.00051"},{"key":"ref23","doi-asserted-by":"publisher","DOI":"10.1145\/3419111.3421285"},{"key":"ref24","doi-asserted-by":"publisher","DOI":"10.1145\/3620665.3640411"},{"key":"ref25","first-page":"613","article-title":"Clipper: A low-latency online prediction serving system","volume-title":"Proc. USENIX Symp. Netw. Syst. Des. Implementation","author":"Crankshaw"},{"article-title":"TensorFlow-serving: Flexible, high-performance ML serving","year":"2017","author":"Olston","key":"ref26"},{"key":"ref27","doi-asserted-by":"publisher","DOI":"10.1145\/3341301.3359658"},{"key":"ref28","first-page":"443","article-title":"Serving DNNs like clockwork: Performance predictability from the bottom up","volume-title":"Proc. USENIX Symp. Operating Syst. Des. Implementation","author":"Gujarati"},{"key":"ref29","doi-asserted-by":"publisher","DOI":"10.1145\/3620665.3640366"},{"key":"ref30","first-page":"103","article-title":"GPipe: Efficient training of giant neural networks using pipeline parallelism","volume-title":"Proc. 33rd Int. Conf. Neural Inf. Process. Syst.","author":"Huang"},{"article-title":"Scaling laws for neural language models","year":"2020","author":"Kaplan","key":"ref31"},{"article-title":"The Llama 3 herd of models","year":"2024","author":"Grattafiori","key":"ref32"},{"key":"ref33","first-page":"135","article-title":"ServerlessLLM: Low-latency serverless inference for large language models","volume-title":"Proc. USENIX Symp. Operating Syst. Des. Implementation","author":"Fu"},{"key":"ref34","first-page":"489","article-title":"PetS: A unified framework for parameter-efficient transformers serving","volume-title":"Proc. USENIX Conf. Usenix Annu. Tech. Conf.","author":"Zhou"},{"key":"ref35","first-page":"1877","article-title":"Language models are few-shot learners","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Brown"},{"key":"ref36","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v38i16.29810"},{"key":"ref37","doi-asserted-by":"publisher","DOI":"10.1145\/3698038.3698523"},{"key":"ref38","first-page":"205","article-title":"Serverless in the wild: Characterizing and optimizing the serverless workload at a large cloud provider","volume-title":"Proc. USENIX Conf. Usenix Annu. Tech. Conf.","author":"Shahrad"},{"key":"ref39","first-page":"1075","article-title":"Orca: Server-assisted multicast for datacenter networks","volume-title":"Proc. USENIX Symp. Netw. Syst. Des. Implementation","author":"Diab"},{"key":"ref40","doi-asserted-by":"publisher","DOI":"10.1145\/3503222.3507709"},{"key":"ref41","first-page":"473","article-title":"Tetris: Memory-efficient serverless inference through tensor sharing","volume-title":"Proc. USENIX Conf. Usenix Annu. Tech. Conf.","author":"Li"},{"key":"ref42","first-page":"533","article-title":"AntMan: Dynamic scaling on GPU clusters for deep learning","volume-title":"Proc. USENIX Symp. Operating Syst. Des. Implementation","author":"Xiao"},{"key":"ref43","first-page":"539","article-title":"Microsecond-scale preemption for concurrent GPU-accelerated DNN inferences","volume-title":"Proc. USENIX Symp. Operating Syst. Des. Implementation","author":"Han"},{"key":"ref44","first-page":"521","article-title":"Orca: A distributed serving system for transformer-based generative models","volume-title":"Proc. USENIX Symp. Operating Syst. Des. Implementation","author":"Yu"},{"key":"ref45","doi-asserted-by":"publisher","DOI":"10.1016\/j.future.2018.10.046"},{"key":"ref46","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2024.acl-long.623"},{"article-title":"Efficient interactive LLM serving with proxy model-based sequence length prediction","year":"2024","author":"Qiu","key":"ref47"},{"key":"ref48","doi-asserted-by":"publisher","DOI":"10.1145\/3302424.3303953"},{"key":"ref49","first-page":"381","article-title":"Fine-tuning giant neural networks on commodity hardware with automatic pipeline model parallelism","volume-title":"Proc. USENIX Conf. Usenix Annu. Tech. Conf.","author":"Eliad"},{"key":"ref50","doi-asserted-by":"publisher","DOI":"10.1109\/tits.2025.3557259"},{"key":"ref51","first-page":"473","article-title":"Prediction-based power oversubscription in cloud platforms","volume-title":"Proc. USENIX Conf. Usenix Annu. Tech. Conf.","author":"Kumbhare"},{"key":"ref52","doi-asserted-by":"publisher","DOI":"10.1109\/TPDS.2021.3100298"},{"key":"ref53","doi-asserted-by":"publisher","DOI":"10.1145\/3694715.3695970"},{"key":"ref54","doi-asserted-by":"publisher","DOI":"10.1016\/j.future.2018.10.046"},{"article-title":"SGLang: Efficient execution of structured language model programs","volume-title":"Proc. 38th Int. Conf. Neural Inf. Process. Syst.","author":"Zheng","key":"ref55"},{"key":"ref56","first-page":"753","article-title":"The CacheLib caching engine: Design and experiences at scale","volume-title":"Proc. USENIX Symp. Operating Syst. Des. Implementation","author":"Berg"},{"key":"ref57","doi-asserted-by":"publisher","DOI":"10.1109\/HPCA56546.2023.10070964"}],"container-title":["IEEE Transactions on Services Computing"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx8\/4629386\/11198176\/11143904.pdf?arnumber=11143904","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,10,10]],"date-time":"2025-10-10T22:30:50Z","timestamp":1760135450000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/11143904\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,9]]},"references-count":57,"journal-issue":{"issue":"5"},"URL":"https:\/\/doi.org\/10.1109\/tsc.2025.3603576","relation":{},"ISSN":["1939-1374","2372-0204"],"issn-type":[{"type":"electronic","value":"1939-1374"},{"type":"electronic","value":"2372-0204"}],"subject":[],"published":{"date-parts":[[2025,9]]}}}