{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,6,26]],"date-time":"2026-06-26T04:35:25Z","timestamp":1782448525678,"version":"3.54.5"},"reference-count":47,"publisher":"Institute of Electrical and Electronics Engineers (IEEE)","issue":"10","license":[{"start":{"date-parts":[[2025,10,1]],"date-time":"2025-10-01T00:00:00Z","timestamp":1759276800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/ieeexplore.ieee.org\/Xplorehelp\/downloads\/license-information\/IEEE.html"},{"start":{"date-parts":[[2025,10,1]],"date-time":"2025-10-01T00:00:00Z","timestamp":1759276800000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2025,10,1]],"date-time":"2025-10-01T00:00:00Z","timestamp":1759276800000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["IEEE Trans. Comput."],"published-print":{"date-parts":[[2025,10]]},"DOI":"10.1109\/tc.2025.3585339","type":"journal-article","created":{"date-parts":[[2025,7,2]],"date-time":"2025-07-02T13:43:50Z","timestamp":1751463830000},"page":"3263-3276","source":"Crossref","is-referenced-by-count":1,"title":["<i>FlashDecoding++Next<\/i>: High Throughput LLM Inference With Latency and Memory Optimization"],"prefix":"10.1109","volume":"74","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-8464-0130","authenticated-orcid":false,"given":"Guohao","family":"Dai","sequence":"first","affiliation":[{"name":"School of Artificial Intelligence, Shanghai Jiao Tong University, Shanghai, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-5768-6037","authenticated-orcid":false,"given":"Ke","family":"Hong","sequence":"additional","affiliation":[{"name":"Department of Electronic Engineering, Tsinghua University, Beijing, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0004-8777-2579","authenticated-orcid":false,"given":"Qiuli","family":"Mao","sequence":"additional","affiliation":[{"name":"Infinigence-AI, Shanghai, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-4896-121X","authenticated-orcid":false,"given":"Xiuhong","family":"Li","sequence":"additional","affiliation":[{"name":"Infinigence-AI, Shanghai, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0000-7000-6537","authenticated-orcid":false,"given":"Jiaming","family":"Xu","sequence":"additional","affiliation":[{"name":"Qing Yuan Research Institute, Shanghai Jiao Tong University, Shanghai, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0002-7253-1364","authenticated-orcid":false,"given":"Haofeng","family":"Huang","sequence":"additional","affiliation":[{"name":"Department of Electronic Engineering, Tsinghua University, Beijing, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0009-3340-2210","authenticated-orcid":false,"given":"Hongtu","family":"Xia","sequence":"additional","affiliation":[{"name":"Peking University, Beijing, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-2209-8312","authenticated-orcid":false,"given":"Xuefei","family":"Ning","sequence":"additional","affiliation":[{"name":"Department of Electronic Engineering, Tsinghua University, Beijing, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0005-3858-7972","authenticated-orcid":false,"given":"Shengen","family":"Yan","sequence":"additional","affiliation":[{"name":"Department of Electronic Engineering, Tsinghua University, Beijing, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-9076-7998","authenticated-orcid":false,"given":"Yun","family":"Liang","sequence":"additional","affiliation":[{"name":"Peking University, Beijing, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-6108-5157","authenticated-orcid":false,"given":"Yu","family":"Wang","sequence":"additional","affiliation":[{"name":"Department of Electronic Engineering, Tsinghua University, Beijing, China"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"263","reference":[{"key":"ref1","first-page":"38","article-title":"Transformers: State-of-the-art natural language processing","volume-title":"Proc. Conf. Empirical Methods Natural Lang. Process.: Syst. Demonstrations","author":"Wolf","year":"2020"},{"key":"ref2","doi-asserted-by":"publisher","DOI":"10.1145\/3600006.3613165"},{"key":"ref3","article-title":"Optimizing inference on large language models with NVIDIA TensorRT-LLM, now publicly available","author":"Vaidya","year":"2023"},{"key":"ref4","article-title":"Introducing meta Llama 3: The most capable openly available LLM to date","year":"2024"},{"key":"ref5","doi-asserted-by":"publisher","DOI":"10.1038\/s41591-023-02448-8"},{"key":"ref6","article-title":"PaLM 2 Technical Report","author":"Anil","year":"2023"},{"key":"ref7","doi-asserted-by":"publisher","DOI":"10.1038\/s43856-023-00370-1"},{"key":"ref8","doi-asserted-by":"publisher","DOI":"10.1109\/mits.2024.3381793"},{"key":"ref9","article-title":"OpenAI Pricing","year":"2023"},{"key":"ref10","article-title":"Up-to-Date ChatGPT statistics and user numbers","year":"2023"},{"key":"ref11","article-title":"The inference cost of search disruption - large language model cost analysis","author":"Dylan Patel","year":"2023"},{"key":"ref12","doi-asserted-by":"publisher","DOI":"10.1109\/SC41404.2022.00051"},{"key":"ref13","first-page":"31094","article-title":"FlexGen: High-throughput generative inference of large language models with a single GPU","volume-title":"Int. Conf. Mach. Learn.","author":"Sheng","year":"2023"},{"key":"ref14","article-title":"OpenPPL: A high-performance deep learning inference platform","year":"2023"},{"key":"ref15","first-page":"16344","article-title":"FlashAttention: Fast and memory-efficient exact attention with IO-awareness","volume":"35","author":"Dao","year":"2022","journal-title":"Proc. Adv. Neural Inf. Process. Syst."},{"key":"ref16","article-title":"FlashAttention-2: Faster attention with better parallelism and work partitioning","author":"Dao","year":"2023"},{"key":"ref17","article-title":"Flash-decoding for long-context inference","author":"Dao","year":"2023"},{"key":"ref18","article-title":"A light and fast inference service for LLM","year":"2023"},{"key":"ref19","article-title":"Text generation inference: Fast inference optimize for LLMs","year":"2023"},{"key":"ref20","article-title":"MLC LLM: Machine learning compilation for large language models","year":"2023"},{"key":"ref21","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/P19-1285"},{"key":"ref22","article-title":"A survey on long text modeling with transformers","author":"Dong","year":"2023"},{"key":"ref23","article-title":"Efficient streaming language models with attention sinks","author":"Xiao","year":"2023"},{"key":"ref24","article-title":"cuBLAS: Basic linear algebra on NVIDIA GPUs","year":"2017"},{"key":"ref25","article-title":"CUTLASS: CUDA templates for linear algebra subroutines"},{"key":"ref26","doi-asserted-by":"publisher","DOI":"10.48550\/ARXIV.1706.03762"},{"key":"ref27","doi-asserted-by":"publisher","DOI":"10.5555\/3104322.3104425"},{"key":"ref28","article-title":"Searching for activation functions","author":"Ramachandran","year":"2017"},{"key":"ref29","article-title":"Training stochastic model recognition algorithms as networks can lead to maximum mutual information estimation of parameters","volume":"2","author":"Bridle","year":"1989","journal-title":"Proc. Adv. Neural Inf. Process. Syst."},{"key":"ref30","article-title":"Pointer sentinel mixture models","author":"Merity","year":"2016"},{"key":"ref31","article-title":"MoA: Mixture of sparse attention for automatic large language model compression","author":"Fu","year":"2024"},{"key":"ref32","article-title":"NVIDIA tensor core","year":"2023"},{"key":"ref33","article-title":"FastGEMV: High-speed GEMV kernels","author":"Wang","year":"2023"},{"key":"ref34","article-title":"CUDA Toolkit","year":"2024"},{"key":"ref35","doi-asserted-by":"publisher","DOI":"10.5555\/3454287.3455008"},{"key":"ref36","article-title":"Qwen2 technical report","author":"Yang","year":"2024"},{"key":"ref37","article-title":"Llama 2: Open foundation and fine-tuned chat models","author":"Touvron","year":"2023"},{"key":"ref38","article-title":"OPT: Open pre-trained transformer language models","author":"Zhang","year":"2022"},{"key":"ref39","article-title":"xFormers: A modular and hackable transformer modelling library","author":"Lefaudeux","year":"2022"},{"key":"ref40","article-title":"Think you have solved question answering? Try ARC, the AI2 reasoning challenge","author":"Clark","year":"2018"},{"key":"ref41","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/p19-1472"},{"key":"ref42","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v34i05.6399"},{"key":"ref43","article-title":"Inference code for llama models"},{"key":"ref44","article-title":"Code Llama: Open foundation models for code","author":"Rozi\u00e8re","year":"2023"},{"key":"ref45","article-title":"Vicuna: An open-source chatbot impressing GPT-4 with 90%* ChatGPT quality","author":"Chiang","year":"2023"},{"key":"ref46","article-title":"FasterTransformer: About transformer related optimization, including BERT, GPT","author":"Nvidia","year":"2017"},{"key":"ref47","article-title":"NVIDIA TensorRT: An SDK for high-performance deep learning inference"}],"container-title":["IEEE Transactions on Computers"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx8\/12\/11164542\/11062854.pdf?arnumber=11062854","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,9,16]],"date-time":"2025-09-16T17:33:31Z","timestamp":1758044011000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/11062854\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,10]]},"references-count":47,"journal-issue":{"issue":"10"},"URL":"https:\/\/doi.org\/10.1109\/tc.2025.3585339","relation":{},"ISSN":["0018-9340","1557-9956","2326-3814"],"issn-type":[{"value":"0018-9340","type":"print"},{"value":"1557-9956","type":"electronic"},{"value":"2326-3814","type":"electronic"}],"subject":[],"published":{"date-parts":[[2025,10]]}}}