{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,23]],"date-time":"2026-10-23T20:31:31Z","timestamp":1792787491281,"version":"4.5.0"},"publisher-location":"New York, NY, USA","reference-count":44,"publisher":"ACM","license":[{"start":{"date-parts":[[2026,9,27]],"date-time":"2026-09-27T00:00:00Z","timestamp":1790467200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/legalcode"}],"funder":[{"name":"National Natural Science Foundation of China","award":["62372436"],"award-info":[{"award-number":["62372436"]}],"id":[{"id":"https:\/\/ror.org\/01h0zpd94","id-type":"ROR","asserted-by":"crossref"}]},{"name":"the Fundamental Research Funds for the Central Universities","award":["2024XKRC086"],"award-info":[{"award-number":["2024XKRC086"]}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2026,9,28]]},"DOI":"10.1145\/3832810.3832894","type":"proceedings-article","created":{"date-parts":[[2026,9,13]],"date-time":"2026-09-13T00:19:06Z","timestamp":1789258746000},"page":"381-391","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":0,"title":["Cross-Layer Performance Analysis of Single-GPU Large Language Model Inference"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0009-0007-1060-6785","authenticated-orcid":false,"given":"Zongxing","family":"Zhao","sequence":"first","affiliation":[{"name":"Beijing Jiaotong University, Beijing, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-7748-7967","authenticated-orcid":false,"given":"Xiaqing","family":"Li","sequence":"additional","affiliation":[{"name":"Beijing Jiaotong University, Beijing, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0009-7806-9108","authenticated-orcid":false,"given":"Zekai","family":"Meng","sequence":"additional","affiliation":[{"name":"Beijing Jiaotong University, Beijing, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0007-2261-0837","authenticated-orcid":false,"given":"Zhengjun","family":"Fu","sequence":"additional","affiliation":[{"name":"Beijing Institute of Technology, Beijing, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0007-4981-5875","authenticated-orcid":false,"given":"Qian","family":"Wang","sequence":"additional","affiliation":[{"name":"Beijing University of Technology, Beijing, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0005-4447-8860","authenticated-orcid":false,"given":"Yanling","family":"Zhang","sequence":"additional","affiliation":[{"name":"Beijing Jiaotong University, Beijing, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-2965-6196","authenticated-orcid":false,"given":"Yidong","family":"Li","sequence":"additional","affiliation":[{"name":"Beijing Jiaotong University, Beijing, China"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2026,9,27]]},"reference":[{"key":"e_1_3_3_1_2_2","first-page":"117","volume-title":"18th USENIX Symposium on Operating Systems Design and Implementation (OSDI 24)","author":"Agrawal Amey","year":"2024","unstructured":"Amey Agrawal, Nitin Kedia, Ashish Panwar, Jayashree Mohan, Nipun Kwatra, Bhargav Gulavani, Alexey Tumanov, and Ramachandran Ramjee. 2024. Taming Throughput-Latency Tradeoff in LLM Inference with Sarathi-Serve. In 18th USENIX Symposium on Operating Systems Design and Implementation (OSDI 24). USENIX Association, Santa Clara, CA, 117\u2013134. https:\/\/www.usenix.org\/conference\/osdi24\/presentation\/agrawal"},{"key":"e_1_3_3_1_3_2","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2023.emnlp-main.298"},{"key":"e_1_3_3_1_4_2","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/W19-5301"},{"key":"e_1_3_3_1_5_2","doi-asserted-by":"publisher","DOI":"10.1109\/ICWS62655.2024.00099"},{"key":"e_1_3_3_1_6_2","volume-title":"Proceedings of the 41st International Conference on Machine Learning","author":"Cai Tianle","year":"2024","unstructured":"Tianle Cai, Yuhong Li, Zhengyang Geng, Hongwu Peng, Jason\u00a0D. Lee, Deming Chen, and Tri Dao. 2024. Medusa: Simple LLM Inference Acceleration Framework with Multiple Decoding Heads. In Proceedings of the 41st International Conference on Machine Learning. PMLR. https:\/\/arxiv.org\/abs\/2401.10774"},{"key":"e_1_3_3_1_7_2","doi-asserted-by":"publisher","DOI":"10.1145\/3669940.3707267"},{"key":"e_1_3_3_1_8_2","doi-asserted-by":"publisher","DOI":"10.52202\/068431-1189"},{"key":"e_1_3_3_1_9_2","volume-title":"Proceedings of the 41st International Conference on Machine Learning","author":"Duan Jiangfei","year":"2024","unstructured":"Jiangfei Duan, Runyu Lu, Hao Duan, Xiaorui Jiang, Haoyu Zhang, Dahua Lin, and Hao Zhang. 2024. MuxServe: Flexible Spatial-Temporal Multiplexing for Multiple LLM Serving. In Proceedings of the 41st International Conference on Machine Learning. PMLR. https:\/\/arxiv.org\/abs\/2404.02015"},{"key":"e_1_3_3_1_10_2","unstructured":"Angela Fan Mike Lewis and Yann Dauphin. 2018. Hierarchical Neural Story Generation. https:\/\/arxiv.org\/abs\/1805.04833"},{"key":"e_1_3_3_1_11_2","first-page":"135","volume-title":"18th USENIX Symposium on Operating Systems Design and Implementation (OSDI 24)","author":"Fu Yao","year":"2024","unstructured":"Yao Fu, Leyang Xue, Yeqi Huang, Andrei-Octavian Brabete, Dmitrii Ustiugov, Yuvraj Patel, and Luo Mai. 2024. ServerlessLLM: Low-Latency Serverless Inference for Large Language Models. In 18th USENIX Symposium on Operating Systems Design and Implementation (OSDI 24). USENIX Association, Santa Clara, CA, 135\u2013153. https:\/\/www.usenix.org\/conference\/osdi24\/presentation\/fu"},{"key":"e_1_3_3_1_12_2","unstructured":"Gemma Team. 2024. Gemma 2: Improving Open Language Models at a Practical Size. https:\/\/arxiv.org\/abs\/2408.00118"},{"key":"e_1_3_3_1_13_2","unstructured":"GLM Team. 2024. ChatGLM: A Family of Large Language Models from GLM-130B to GLM-4 All Tools. https:\/\/arxiv.org\/abs\/2406.12793"},{"key":"e_1_3_3_1_14_2","unstructured":"Aaron Grattafiori Abhimanyu Dubey Abhinav Jauhri Abhinav Pandey Abhishek Kadian et\u00a0al. 2024. The Llama 3 Herd of Models. https:\/\/arxiv.org\/abs\/2407.21783"},{"key":"e_1_3_3_1_15_2","unstructured":"Shipeng Hu Guangyan Zhang and Weimin Zheng. 2026. Survey on KV Cache Compression Methods for Large Language Model Inference. Journal of Computer Research and Development 63 4 (2026) 1021\u20131038."},{"key":"e_1_3_3_1_16_2","doi-asserted-by":"crossref","unstructured":"Luyang Huang Shuyang Cao Nikolaus Parulian Heng Ji and Lu Wang. 2021. Efficient Attentions for Long Document Summarization. https:\/\/arxiv.org\/abs\/2104.02112","DOI":"10.18653\/v1\/2021.naacl-main.112"},{"key":"e_1_3_3_1_17_2","volume-title":"Proceedings of Machine Learning and Systems","volume":"7","author":"Jiang Youhe","year":"2025","unstructured":"Youhe Jiang, Fangcheng Fu, Xiaozhe Yao, Taiyi Wang, Bin Cui, Ana Klimovic, and Eiko Yoneki. 2025. ThunderServe: High-performance and Cost-efficient LLM Serving in Cloud Environments. In Proceedings of Machine Learning and Systems , M.\u00a0Zaharia, G.\u00a0Joshi, and Y.\u00a0Lin (Eds.), Vol.\u00a07. MLSys. https:\/\/proceedings.mlsys.org\/paper_files\/paper\/2025\/file\/c2a0e26dd9ee7d57e92bb1c24b39659a-Paper-Conference.pdf"},{"key":"e_1_3_3_1_18_2","unstructured":"Jared Kaplan Sam McCandlish Tom Henighan Tom\u00a0B. Brown Benjamin Chess Rewon Child Scott Gray Alec Radford Jeffrey Wu and Dario Amodei. 2020. Scaling Laws for Neural Language Models. arxiv:https:\/\/arXiv.org\/abs\/2001.08361\u00a0[cs.LG] https:\/\/arxiv.org\/abs\/2001.08361"},{"key":"e_1_3_3_1_19_2","doi-asserted-by":"publisher","DOI":"10.1145\/3600006.3613165"},{"key":"e_1_3_3_1_20_2","unstructured":"Jinhao Li Jiaming Xu Shan Huang Yonghua Chen Wen Li Jun Liu Yaoxiu Lian Jiayi Pan Li Ding Hao Zhou Yu Wang and Guohao Dai. 2025. Large Language Model Inference Acceleration: A Comprehensive Hardware Perspective. arxiv:https:\/\/arXiv.org\/abs\/2410.04466\u00a0[cs.AR] https:\/\/arxiv.org\/abs\/2410.04466"},{"key":"e_1_3_3_1_21_2","doi-asserted-by":"publisher","DOI":"10.1109\/ICPP.2016.15"},{"key":"e_1_3_3_1_22_2","volume-title":"Proceedings of the 41st International Conference on Machine Learning","author":"Li Yuhui","year":"2024","unstructured":"Yuhui Li, Fangyun Wei, Chao Zhang, and Hongyang Zhang. 2024. EAGLE: Speculative Sampling Requires Rethinking Feature Uncertainty. In Proceedings of the 41st International Conference on Machine Learning. PMLR. https:\/\/arxiv.org\/abs\/2401.15077"},{"key":"e_1_3_3_1_23_2","first-page":"663","volume-title":"17th USENIX Symposium on Operating Systems Design and Implementation (OSDI 23)","author":"Li Zhuohan","year":"2023","unstructured":"Zhuohan Li, Lianmin Zheng, Yinmin Zhong, Vincent Liu, Ying Sheng, Xin Jin, Yanping Huang, Zhifeng Chen, Hao Zhang, Joseph\u00a0E. Gonzalez, and Ion Stoica. 2023. AlpaServe: Statistical Multiplexing with Model Parallelism for Deep Learning Serving. In 17th USENIX Symposium on Operating Systems Design and Implementation (OSDI 23). USENIX Association, Boston, MA, 663\u2013679. https:\/\/www.usenix.org\/conference\/osdi23\/presentation\/li-zhouhan"},{"key":"e_1_3_3_1_24_2","volume-title":"Proceedings of Machine Learning and Systems","volume":"6","author":"Lin Ji","year":"2024","unstructured":"Ji Lin, Jiaming Tang, Haotian Tang, Shang Yang, Xingyu Dang, and Song Han. 2024. AWQ: Activation-aware Weight Quantization for LLM Compression and Acceleration. In Proceedings of Machine Learning and Systems , Vol.\u00a06. MLSys. https:\/\/proceedings.mlsys.org\/paper_files\/paper\/2024\/hash\/42a452cbafa9dd64e9ba4aa95cc1ef21-Abstract-Conference.html"},{"key":"e_1_3_3_1_25_2","doi-asserted-by":"publisher","DOI":"10.1145\/3651890.3672274"},{"key":"e_1_3_3_1_26_2","doi-asserted-by":"publisher","unstructured":"Xupeng Miao Gabriele Oliaro Zhihao Zhang Xinhao Cheng Hongyi Jin Tianqi Chen and Zhihao Jia. 2025. Towards Efficient Generative Large Language Model Serving: A Survey from Algorithms to Systems. ACM Comput. Surv. 58 1 Article 15 (Sept. 2025) 37\u00a0pages. 10.1145\/3754448","DOI":"10.1145\/3754448"},{"key":"e_1_3_3_1_27_2","doi-asserted-by":"publisher","DOI":"10.1145\/3620666.3651335"},{"key":"e_1_3_3_1_28_2","unstructured":"NVIDIA Corporation. [n. d.]. NVIDIA Nsight Compute. https:\/\/developer.nvidia.com\/nsight-compute. Version: 2023.2."},{"key":"e_1_3_3_1_29_2","unstructured":"Qwen Team. 2025. Qwen2.5 Technical Report. https:\/\/arxiv.org\/abs\/2412.15115"},{"key":"e_1_3_3_1_30_2","series-title":"Proceedings of Machine Learning Research","first-page":"18332","volume-title":"Proceedings of the 39th International Conference on Machine Learning","volume":"162","author":"Rajbhandari Samyam","year":"2022","unstructured":"Samyam Rajbhandari, Conglong Li, Zhewei Yao, Minjia Zhang, Reza\u00a0Yazdani Aminabadi, Ammar\u00a0Ahmad Awan, Jeff Rasley, and Yuxiong He. 2022. DeepSpeed-MoE: Advancing Mixture-of-Experts Inference and Training to Power Next-Generation AI Scale. In Proceedings of the 39th International Conference on Machine Learning(Proceedings of Machine Learning Research, Vol.\u00a0162), Kamalika Chaudhuri, Stefanie Jegelka, Le\u00a0Song, Csaba Szepesvari, Gang Niu, and Sivan Sabato (Eds.). PMLR, 18332\u201318346. https:\/\/proceedings.mlr.press\/v162\/rajbhandari22a.html"},{"key":"e_1_3_3_1_31_2","unstructured":"Pol\u00a0G. Recasens Ferran Agullo Yue Zhu Chen Wang Eun\u00a0Kyung Lee Olivier Tardieu Jordi Torres and Josep\u00a0Ll. Berral. 2025. Mind the Memory Gap: Unveiling GPU Bottlenecks in Large-Batch LLM Inference. arxiv:https:\/\/arXiv.org\/abs\/2503.08311\u00a0[cs.DC] https:\/\/arxiv.org\/abs\/2503.08311"},{"key":"e_1_3_3_1_32_2","unstructured":"Noam Shazeer. 2019. Fast Transformer Decoding: One Write-Head is All You Need. https:\/\/arxiv.org\/abs\/1911.02150"},{"key":"e_1_3_3_1_33_2","series-title":"Proceedings of Machine Learning Research","first-page":"31094","volume-title":"Proceedings of the 40th International Conference on Machine Learning","volume":"202","author":"Sheng Ying","year":"2023","unstructured":"Ying Sheng, Lianmin Zheng, Binhang Yuan, Zhuohan Li, Max Ryabinin, Daniel\u00a0Y. Fu, Zhiqiang Xie, Beidi Chen, Clark Barrett, Joseph\u00a0E. Gonzalez, Percy Liang, Christopher R\u00e9, Ion Stoica, and Ce Zhang. 2023. FlexGen: High-Throughput Generative Inference of Large Language Models with a Single GPU. In Proceedings of the 40th International Conference on Machine Learning(Proceedings of Machine Learning Research, Vol.\u00a0202). PMLR, 31094\u201331116. https:\/\/proceedings.mlr.press\/v202\/sheng23a.html"},{"key":"e_1_3_3_1_34_2","doi-asserted-by":"publisher","DOI":"10.1145\/3694715.3695964"},{"key":"e_1_3_3_1_35_2","first-page":"173","volume-title":"18th USENIX Symposium on Operating Systems Design and Implementation (OSDI 24)","author":"Sun Biao","year":"2024","unstructured":"Biao Sun, Ziming Huang, Hanyu Zhao, Wencong Xiao, Xinyi Zhang, Yong Li, and Wei Lin. 2024. Llumnix: Dynamic Scheduling for Large Language Model Serving. In 18th USENIX Symposium on Operating Systems Design and Implementation (OSDI 24). USENIX Association, Santa Clara, CA, 173\u2013191. https:\/\/www.usenix.org\/conference\/osdi24\/presentation\/sun-biao"},{"key":"e_1_3_3_1_36_2","volume-title":"Advances in Neural Information Processing Systems","author":"Vaswani Ashish","year":"2017","unstructured":"Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, Llion Jones, Aidan\u00a0N. Gomez, \u0141ukasz Kaiser, and Illia Polosukhin. 2017. Attention is All you Need. In Advances in Neural Information Processing Systems , I.\u00a0Guyon, U.\u00a0Von Luxburg, S.\u00a0Bengio, H.\u00a0Wallach, R.\u00a0Fergus, S.\u00a0Vishwanathan, and R.\u00a0Garnett (Eds.), Vol.\u00a030. Curran Associates, Inc.https:\/\/proceedings.neurips.cc\/paper_files\/paper\/2017\/file\/3f5ee243547dee91fbd053c1c4a845aa-Paper.pdf"},{"key":"e_1_3_3_1_37_2","unstructured":"Haonan Wang Xuxin Xiao Mingyu Yan Zhuoyuan Zhu Dengke Han Duo Wang Wenming Li Xiaochun Ye Cunchen Hu Hongyang Chen and Guangyu Sun. 2025. A Systematic Characterization of LLM Inference on GPUs. arxiv:https:\/\/arXiv.org\/abs\/2512.01644\u00a0[cs.AR] https:\/\/arxiv.org\/abs\/2512.01644"},{"key":"e_1_3_3_1_38_2","unstructured":"Tengyu Xu Eryk Helenowski Karthik\u00a0Abinav Sankararaman Di Jin Kaiyan Peng et\u00a0al. 2024. The Perfect Blend: Redefining RLHF with Mixture of Judges. https:\/\/arxiv.org\/abs\/2409.20370"},{"key":"e_1_3_3_1_39_2","unstructured":"An Yang Anfeng Li Baosong Yang Beichen Zhang Binyuan Hui Bo Zheng Bowen Yu et\u00a0al. 2025. Qwen3 Technical Report. https:\/\/arxiv.org\/abs\/2505.09388"},{"key":"e_1_3_3_1_40_2","first-page":"521","volume-title":"16th USENIX Symposium on Operating Systems Design and Implementation (OSDI 22)","author":"Yu Gyeong-In","year":"2022","unstructured":"Gyeong-In Yu, Joo\u00a0Seong Jeong, Geon-Woo Kim, Soojeong Kim, and Byung-Gon Chun. 2022. Orca: A Distributed Serving System for Transformer-Based Generative Models. In 16th USENIX Symposium on Operating Systems Design and Implementation (OSDI 22). USENIX Association, Carlsbad, CA, 521\u2013538. https:\/\/www.usenix.org\/conference\/osdi22\/presentation\/yu"},{"key":"e_1_3_3_1_41_2","doi-asserted-by":"publisher","unstructured":"Xinyuan Zhang Jiangtian Nie Yudong Huang Gaochang Xie Zehui Xiong Jiang Liu Dusit Niyato and Xuemin Shen. 2025. Beyond the Cloud: Edge Inference for Generative Large Language Models in Wireless Networks. IEEE Transactions on Wireless Communications 24 1 (2025) 643\u2013658. 10.1109\/TWC.2024.3497923","DOI":"10.1109\/TWC.2024.3497923"},{"key":"e_1_3_3_1_42_2","doi-asserted-by":"publisher","DOI":"10.52202\/079017-2000"},{"key":"e_1_3_3_1_43_2","unstructured":"Weimin Zheng. 2024. Application of distributed techniques in large language model training and inference. Big Data 10 5 (2024) 1\u201310."},{"key":"e_1_3_3_1_44_2","unstructured":"Weimin Zheng. 2024. Four Issues to Consider in Building Computer Systems Supporting Large Model Training. Big Data 10 01 (2024) 1\u20138."},{"key":"e_1_3_3_1_45_2","first-page":"193","volume-title":"18th USENIX Symposium on Operating Systems Design and Implementation (OSDI 24)","author":"Zhong Yinmin","year":"2024","unstructured":"Yinmin Zhong, Shengyu Liu, Junda Chen, Jianbo Hu, Yibo Zhu, Xuanzhe Liu, Xin Jin, and Hao Zhang. 2024. DistServe: Disaggregating Prefill and Decoding for Goodput-optimized Large Language Model Serving. In 18th USENIX Symposium on Operating Systems Design and Implementation (OSDI 24). USENIX Association, Santa Clara, CA, 193\u2013210. https:\/\/www.usenix.org\/conference\/osdi24\/presentation\/zhong-yinmin"}],"event":{"name":"ICPP '26: Proceedings of the 55th International Conference on Parallel Processing","location":"Singapore , Singapore","acronym":"ICPP '26"},"container-title":["Proceedings of the 55th International Conference on Parallel Processing"],"original-title":[],"deposited":{"date-parts":[[2026,9,16]],"date-time":"2026-09-16T18:00:22Z","timestamp":1789581622000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3832810.3832894"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026,9,27]]},"references-count":44,"alternative-id":["10.1145\/3832810.3832894","10.1145\/3832810"],"URL":"https:\/\/doi.org\/10.1145\/3832810.3832894","relation":{},"subject":[],"published":{"date-parts":[[2026,9,27]]},"assertion":[{"value":"2026-09-27","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}