{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,9,11]],"date-time":"2025-09-11T21:57:46Z","timestamp":1757627866667,"version":"3.44.0"},"reference-count":25,"publisher":"IEEE","license":[{"start":{"date-parts":[[2025,7,2]],"date-time":"2025-07-02T00:00:00Z","timestamp":1751414400000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2025,7,2]],"date-time":"2025-07-02T00:00:00Z","timestamp":1751414400000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2025,7,2]]},"DOI":"10.1109\/iwqos65803.2025.11143255","type":"proceedings-article","created":{"date-parts":[[2025,9,9]],"date-time":"2025-09-09T17:29:52Z","timestamp":1757438992000},"page":"1-10","source":"Crossref","is-referenced-by-count":0,"title":["Deploy Efficient Large Language Model Distributed Inference Pipeline for Heterogeneous GPUs"],"prefix":"10.1109","author":[{"given":"Junyang","family":"Zhang","sequence":"first","affiliation":[{"name":"University of Science and Technology of China,Hefei,China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Jiahui","family":"Hou","sequence":"additional","affiliation":[{"name":"University of Science and Technology of China,Hefei,China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Bowen","family":"Zhang","sequence":"additional","affiliation":[{"name":"University of Science and Technology of China,Hefei,China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Xiang-Yang","family":"Li","sequence":"additional","affiliation":[{"name":"University of Science and Technology of China,Hefei,China"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"263","reference":[{"key":"ref1","article-title":"A survey of large language models","author":"Zhao","year":"2023","journal-title":"arXiv preprint"},{"key":"ref2","article-title":"Prompting large language model for machine translation: A case study","author":"Zhang","year":"2023","journal-title":"arXiv preprint"},{"key":"ref3","doi-asserted-by":"publisher","DOI":"10.1126\/science.abq1158"},{"key":"ref4","article-title":"Llama: Open and efficient foundation language models","author":"Touvron","year":"2023","journal-title":"arXiv preprint"},{"key":"ref5","article-title":"Llm-pruner: On the structural pruning of large language models","author":"Ma","year":"2023","journal-title":"arXiv preprint"},{"key":"ref6","doi-asserted-by":"publisher","DOI":"10.1109\/SC41404.2022.00051"},{"key":"ref7","article-title":"Response length perception and sequence scheduling: An llm-empowered llm inference pipeline","author":"Zheng","year":"2023","journal-title":"arXiv preprint"},{"issue":"1","key":"ref8","first-page":"5232","article-title":"Switch transformers: Scaling to trillion parameter models with simple and efficient sparsity","volume":"23","author":"Fedus","year":"2022","journal-title":"The Journal of Machine Learning Research"},{"key":"ref9","article-title":"Palm: Scaling language modeling with pathways","author":"Chowdhery","year":"2022","journal-title":"arXiv preprint"},{"key":"ref10","article-title":"Fast distributed inference serving for large language models","author":"Wu","year":"2023","journal-title":"arXiv preprint"},{"key":"ref11","first-page":"443","article-title":"Serving \\{DNNs\\} like clockwork: Performance predictability from the bottom up","volume-title":"14th USENIX Symposium on Operating Systems Design and Implementation (OSDI 20)","author":"Gujarati","year":"2020"},{"key":"ref12","volume-title":"Multi-process service","year":"2022"},{"key":"ref13","author":"Jain","year":"2018","journal-title":"Dynamic space-time scheduling for gpu inference"},{"key":"ref14","doi-asserted-by":"publisher","DOI":"10.1109\/TPDS.2022.3195664"},{"key":"ref15","first-page":"199","article-title":"Serving heterogeneous machine learning models on \\{Multi-GPU\\} servers with \\{Spatio-Temporal\\} sharing","volume-title":"2022 USENIX Annual Technical Conference (USENIX ATC 22)","author":"Choi","year":"2022"},{"key":"ref16","doi-asserted-by":"publisher","DOI":"10.1145\/3419111.3421284"},{"key":"ref17","doi-asserted-by":"publisher","DOI":"10.48550\/arxiv.1811.06965"},{"key":"ref18","doi-asserted-by":"publisher","DOI":"10.1145\/3341301.3359646"},{"key":"ref19","doi-asserted-by":"publisher","DOI":"10.1145\/3494981"},{"key":"ref20","doi-asserted-by":"publisher","DOI":"10.1109\/TKDE.2022.3178211"},{"key":"ref21","first-page":"521","article-title":"Orca: A distributed serving system for \\{Transformer-Based\\} generative models","volume-title":"16th USENIX Symposium on Operating Systems Design and Implementation (OSDI 22)","author":"Yu","year":"2022"},{"key":"ref22","first-page":"20852","article-title":"Less is more: Task-aware layer-wise distillation for language model compression","volume-title":"International Conference on Machine Learning. PMLR","author":"Liang","year":"2023"},{"key":"ref23","doi-asserted-by":"publisher","DOI":"10.1145\/3437801.3441578"},{"key":"ref24","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2023.emnlp-industry.74"},{"key":"ref25","doi-asserted-by":"publisher","DOI":"10.1145\/3603269.3610856"}],"event":{"name":"2025 IEEE\/ACM International Symposium on Quality of Service (IWQoS)","location":"Gold Coast, Australia","start":{"date-parts":[[2025,7,2]]},"end":{"date-parts":[[2025,7,4]]}},"container-title":["2025 IEEE\/ACM 33rd International Symposium on Quality of Service (IWQoS)"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx8\/11143240\/11143247\/11143255.pdf?arnumber=11143255","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,9,10]],"date-time":"2025-09-10T05:04:36Z","timestamp":1757480676000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/11143255\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,7,2]]},"references-count":25,"URL":"https:\/\/doi.org\/10.1109\/iwqos65803.2025.11143255","relation":{},"subject":[],"published":{"date-parts":[[2025,7,2]]}}}