{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,24]],"date-time":"2026-07-24T15:14:15Z","timestamp":1784906055340,"version":"3.55.0"},"reference-count":36,"publisher":"Institute of Electrical and Electronics Engineers (IEEE)","license":[{"start":{"date-parts":[[2025,1,1]],"date-time":"2025-01-01T00:00:00Z","timestamp":1735689600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/legalcode"}],"funder":[{"DOI":"10.13039\/100021130","name":"Bundesministerium f?r Wirtschaft und Klimaschutz","doi-asserted-by":"publisher","award":["68GX21007J"],"award-info":[{"award-number":["68GX21007J"]}],"id":[{"id":"10.13039\/100021130","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100006114","name":"S?chsisches Staatsministerium f?r Wissenschaft und Kunst","doi-asserted-by":"publisher","id":[{"id":"10.13039\/501100006114","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100002347","name":"Bundesministerium f?r Bildung und Forschung","doi-asserted-by":"publisher","award":["SCADS22B"],"award-info":[{"award-number":["SCADS22B"]}],"id":[{"id":"10.13039\/501100002347","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["IEEE Access"],"published-print":{"date-parts":[[2025]]},"DOI":"10.1109\/access.2025.3576898","type":"journal-article","created":{"date-parts":[[2025,6,5]],"date-time":"2025-06-05T17:37:51Z","timestamp":1749145071000},"page":"98976-98989","source":"Crossref","is-referenced-by-count":2,"title":["BALI\u2014A Benchmark for Accelerated Language Model Inference"],"prefix":"10.1109","volume":"13","author":[{"ORCID":"https:\/\/orcid.org\/0009-0002-7332-5861","authenticated-orcid":false,"given":"Lena","family":"Jurkschat","sequence":"first","affiliation":[{"name":"Center for Interdisciplinary Digital Sciences (CIDS), ScaDS.AI Dresden\/Leipzig, Technische Universität Dresden, Dresden, Germany"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0001-4604-2009","authenticated-orcid":false,"given":"Preetam","family":"Gattogi","sequence":"additional","affiliation":[{"name":"Center for Interdisciplinary Digital Sciences (CIDS), ScaDS.AI Dresden\/Leipzig, Technische Universität Dresden, Dresden, Germany"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Sahar","family":"Vahdati","sequence":"additional","affiliation":[{"name":"Center for Interdisciplinary Digital Sciences (CIDS), ScaDS.AI Dresden\/Leipzig, Technische Universität Dresden, Dresden, Germany"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Jens","family":"Lehmann","sequence":"additional","affiliation":[{"name":"Center for Interdisciplinary Digital Sciences (CIDS), ScaDS.AI Dresden\/Leipzig, Technische Universität Dresden, Dresden, Germany"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"263","reference":[{"key":"ref1","article-title":"DeepSpeed-FastGen: High-throughput text generation for LLMs via MII and DeepSpeed-inference","author":"Holmes","year":"2024","journal-title":"arXiv:2401.08671"},{"key":"ref2","volume-title":"Striking Performance: Large Language Models Up to 4X Faster on RTX With TensorRT-LLM for Windows","author":"Clayton","year":"2024"},{"key":"ref3","volume-title":"OpenLLM: Operating LLMs in Production","author":"Pham"},{"key":"ref4","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2024.findings-acl.57"},{"key":"ref5","article-title":"HuggingFace\u2019s transformers: State-of-the-art natural language processing","author":"Wolf","year":"2019","journal-title":"arXiv:1910.03771"},{"key":"ref6","volume-title":"LLM Foundry-pretrain, Finetune, Evaluate and Deploy LLMs","author":"MosaicML","year":"2024"},{"key":"ref7","volume-title":"ModelTC LightLLM\u2014A Light and Fast Inference Service for LLM","year":"2024"},{"key":"ref8","volume-title":"Llama","author":"Gerganov","year":"2024"},{"key":"ref9","volume-title":"MOSEC: Model Serving Made Efficient in the Cloud","author":"Yang"},{"key":"ref10","article-title":"Megatron-LM: Training multi-billion parameter language models using model parallelism","author":"Shoeybi","year":"2019","journal-title":"arXiv:1909.08053"},{"key":"ref11","article-title":"FlashDecoding++: Faster large language model inference on GPUs","author":"Hong","year":"2023","journal-title":"arXiv:2311.01282"},{"key":"ref12","article-title":"SpecInfer: Accelerating generative large language model serving with tree-based speculative inference and verification","author":"Miao","year":"2023","journal-title":"arXiv:2305.09781"},{"key":"ref13","article-title":"Sparse fine-tuning for inference acceleration of large language models","author":"Kurtic","year":"2023","journal-title":"arXiv:2310.06927"},{"key":"ref14","article-title":"Accelerating LLaMA inference by enabling intermediate layer decoding via instruction tuning with LITE","author":"Varshney","year":"2023","journal-title":"arXiv:2310.18581"},{"key":"ref15","article-title":"Accelerating LLM inference with staged speculative decoding","author":"Spector","year":"2023","journal-title":"arXiv:2308.04623"},{"key":"ref16","article-title":"Inference with reference: Lossless acceleration of large language models","author":"Yang","year":"2023","journal-title":"arXiv:2304.04487"},{"key":"ref17","article-title":"MLPerf inference benchmark","author":"Reddi","year":"2019","journal-title":"arXiv:1911.02549"},{"key":"ref18","volume-title":"Ray LLMPerf\u2014A Tool for Evaulation the Performance of LLM APIs","year":"2024"},{"key":"ref19","volume-title":"GPU-Benchmarks-on-LLM-Inference","author":"Dai","year":"2025"},{"key":"ref20","article-title":"ReAct: Synergizing reasoning and acting in language models","author":"Yao","year":"2022","journal-title":"arXiv:2210.03629"},{"key":"ref21","article-title":"AutoGen: Enabling next-gen LLM applications via multi-agent conversation","author":"Wu","year":"2023","journal-title":"arXiv:2308.08155"},{"key":"ref22","doi-asserted-by":"publisher","DOI":"10.1162\/tacl_a_00671"},{"key":"ref23","article-title":"Gemma: Open models based on Gemini research and technology","author":"Team","year":"2024","journal-title":"arXiv:2403.08295"},{"key":"ref24","volume-title":"Introducing MPT-7B: A New Standard for Open-Source, Commercially Usable LLMs","author":"M","year":"2023"},{"key":"ref25","article-title":"OPT: Open pre-trained transformer language models","author":"Zhang","year":"2022","journal-title":"arXiv:2205.01068"},{"key":"ref26","article-title":"Llama 2: Open foundation and fine-tuned chat models","author":"Touvron","year":"2023","journal-title":"arXiv:2307.09288"},{"key":"ref27","article-title":"LLaMA: Open and efficient foundation language models","author":"Touvron","year":"2023","journal-title":"arXiv:2302.13971"},{"key":"ref28","article-title":"Mistral 7B","author":"Jiang","year":"2023","journal-title":"arXiv:2310.06825"},{"key":"ref29","article-title":"Textbooks are all you need II: Phi-1.5 technical report","volume-title":"arXiv:2309.05463","author":"Li","year":"2023"},{"key":"ref30","doi-asserted-by":"publisher","DOI":"10.1145\/3600006.3613165"},{"key":"ref31","article-title":"LLM-Inference-bench: Inference benchmarking of large language models on AI accelerators","author":"Teja Chitty-Venkata","year":"2024","journal-title":"arXiv:2411.00136"},{"key":"ref32","doi-asserted-by":"publisher","DOI":"10.1109\/HPEC58863.2023.10363447"},{"key":"ref33","volume-title":"Benchmarking LLM Inference Backends: VLLM, LMDeploy, MLC-LLM, TensorRT-LLM, and TGI","author":"Zhou","year":"2024"},{"key":"ref34","volume-title":"Benchmarking Large Language Models: Opportunities and Challenges","author":"Hodak"},{"key":"ref35","volume-title":"Serving LLMs on AMD MI300X: Best Practices"},{"key":"ref36","volume-title":"LMDeploy: A Toolkit for Compressing, Deploying, and Serving LLM","author":"Contributors","year":"2023"}],"container-title":["IEEE Access"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx8\/6287639\/10820123\/11026002.pdf?arnumber=11026002","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T04:51:05Z","timestamp":1750135865000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/11026002\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025]]},"references-count":36,"URL":"https:\/\/doi.org\/10.1109\/access.2025.3576898","relation":{},"ISSN":["2169-3536"],"issn-type":[{"value":"2169-3536","type":"electronic"}],"subject":[],"published":{"date-parts":[[2025]]}}}