{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,8,7]],"date-time":"2026-08-07T17:58:16Z","timestamp":1786125496081,"version":"3.56.0"},"reference-count":92,"publisher":"Institute of Electrical and Electronics Engineers (IEEE)","license":[{"start":{"date-parts":[[2024,1,1]],"date-time":"2024-01-01T00:00:00Z","timestamp":1704067200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by-nc-nd\/4.0\/"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["IEEE Access"],"published-print":{"date-parts":[[2024]]},"DOI":"10.1109\/access.2024.3424945","type":"journal-article","created":{"date-parts":[[2024,7,8]],"date-time":"2024-07-08T17:41:32Z","timestamp":1720460492000},"page":"96017-96050","source":"Crossref","is-referenced-by-count":35,"title":["Achieving Peak Performance for Large Language Models: A Systematic Review"],"prefix":"10.1109","volume":"12","author":[{"ORCID":"https:\/\/orcid.org\/0000-0001-9906-6883","authenticated-orcid":false,"given":"Zhyar Rzgar K.","family":"Rostam","sequence":"first","affiliation":[{"name":"Doctoral School of Applied Informatics and Applied Mathematics, Óbuda University, Budapest, Hungary"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-7292-0717","authenticated-orcid":false,"given":"S\u00e1ndor","family":"Sz\u00e9n\u00e1si","sequence":"additional","affiliation":[{"name":"John von Neumann Faculty of Informatics, Óbuda University, Budapest, Hungary"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-8845-8301","authenticated-orcid":false,"given":"G\u00e1bor","family":"Kert\u00e9sz","sequence":"additional","affiliation":[{"name":"John von Neumann Faculty of Informatics, Óbuda University, Budapest, Hungary"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"263","reference":[{"key":"ref1","doi-asserted-by":"publisher","DOI":"10.1145\/3458817.3476205"},{"key":"ref2","doi-asserted-by":"publisher","DOI":"10.1007\/s11431-020-1647-3"},{"key":"ref3","doi-asserted-by":"publisher","DOI":"10.48550\/arxiv.1811.06965"},{"key":"ref4","doi-asserted-by":"publisher","DOI":"10.1109\/IPDPS54959.2023.00042"},{"key":"ref5","doi-asserted-by":"publisher","DOI":"10.1109\/SC41404.2022.00051"},{"key":"ref6","article-title":"Multilevel large language models for everyone","author":"Gong","year":"2023","journal-title":"arXiv:2307.13221"},{"key":"ref7","article-title":"Accelerating LLM inference with staged speculative decoding","author":"Spector","year":"2023","journal-title":"arXiv:2308.04623"},{"key":"ref8","article-title":"BERT: Pre-training of deep bidirectional transformers for language understanding","author":"Devlin","year":"2018","journal-title":"arXiv:1810.04805"},{"key":"ref9","article-title":"Challenges and opportunities of using transformer-based multi-task learning in NLP through ML lifecycle: A survey","author":"Torbarina","year":"2023","journal-title":"arXiv:2308.08234"},{"key":"ref10","first-page":"29416","article-title":"Swarm parallelism: Training large models can be surprisingly communication-efficient","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Ryabinin"},{"key":"ref11","doi-asserted-by":"publisher","DOI":"10.1145\/3437801.3441578"},{"key":"ref12","article-title":"PaLM 2 technical report","volume-title":"arXiv:2305.10403","author":"Anil","year":"2023"},{"key":"ref13","doi-asserted-by":"publisher","DOI":"10.12700\/aph.20.5.2023.5.8"},{"key":"ref14","article-title":"LLaMA: Open and efficient foundation language models","author":"Touvron","year":"2023","journal-title":"arXiv:2302.13971"},{"key":"ref15","doi-asserted-by":"publisher","DOI":"10.1145\/3605573.3605613"},{"key":"ref16","first-page":"11117","article-title":"Cramming: Training a language model on a single gpu in one day","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Geiping"},{"key":"ref17","article-title":"FlexGen: High-throughput generative inference of large language models with a single GPU","author":"Sheng","year":"2023","journal-title":"arXiv:2303.06865"},{"key":"ref18","doi-asserted-by":"publisher","DOI":"10.1109\/SC41405.2020.00024"},{"key":"ref19","article-title":"Megatron-LM: Training multi-billion parameter language models using model parallelism","author":"Shoeybi","year":"2019","journal-title":"arXiv:1909.08053"},{"key":"ref20","first-page":"551","article-title":"ZeRO-offload: Democratizing billion-scale model training","volume-title":"Proc. USENIX Annu. Tech. Conf.","author":"Ren"},{"key":"ref21","article-title":"Training deep nets with sublinear memory cost","author":"Chen","year":"2016","journal-title":"arXiv:1604.06174"},{"key":"ref22","first-page":"25464","article-title":"Decentralized training of foundation models in heterogeneous environments","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","volume":"35","author":"Yuan"},{"key":"ref23","first-page":"3659","article-title":"Towards crowdsourced training of large neural networks using decentralized mixture-of-experts","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","volume":"33","author":"Ryabinin"},{"key":"ref24","article-title":"A survey of large language models","author":"Zhao","year":"2023","journal-title":"arXiv:2303.18223"},{"key":"ref25","doi-asserted-by":"publisher","DOI":"10.1155\/2021\/6641832"},{"key":"ref26","article-title":"A survey of large-scale deep learning serving system optimization: Challenges and opportunities","author":"Yu","year":"2021","journal-title":"arXiv:2111.14247"},{"key":"ref27","article-title":"Beyond efficiency: A systematic survey of resource-efficient large language models","author":"Bai","year":"2024","journal-title":"arXiv:2401.00625"},{"key":"ref28","doi-asserted-by":"publisher","DOI":"10.1109\/JIOT.2021.3111624"},{"key":"ref29","doi-asserted-by":"publisher","DOI":"10.1145\/3605943"},{"key":"ref30","article-title":"Efficient large language models: A survey","author":"Wan","year":"2023","journal-title":"arXiv:2312.03863"},{"key":"ref31","doi-asserted-by":"publisher","DOI":"10.29173\/jchla29699"},{"key":"ref32","doi-asserted-by":"publisher","DOI":"10.1186\/s13643-016-0384-4"},{"key":"ref33","article-title":"LoRA: Low-rank adaptation of large language models","author":"Hu","year":"2021","journal-title":"arXiv:2106.09685"},{"key":"ref34","doi-asserted-by":"publisher","DOI":"10.1145\/1034780.1034781"},{"key":"ref35","first-page":"258","article-title":"Faster and smaller N-gram language models","volume-title":"Proc. 49th Annu. Meeting Assoc. Comput. Linguistics, Hum. Lang. Technol.","author":"Pauls"},{"key":"ref36","doi-asserted-by":"publisher","DOI":"10.3115\/1034678.1034712"},{"key":"ref37","article-title":"Large language models: A comprehensive survey of its applications, challenges, limitations, and future prospects","author":"Hadi","year":"2023"},{"key":"ref38","doi-asserted-by":"publisher","DOI":"10.1186\/s40537-015-0029-9"},{"key":"ref39","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-33624-0_8"},{"key":"ref40","first-page":"2493","article-title":"Natural language processing (almost) from scratch","volume":"12","author":"Collobert","year":"2011","journal-title":"J. Mach. Learn. Res."},{"key":"ref41","doi-asserted-by":"publisher","DOI":"10.48550\/arXiv.1310.4546"},{"key":"ref42","article-title":"Efficient estimation of word representations in vector space","author":"Mikolov","year":"2013","journal-title":"arXiv:1301.3781"},{"key":"ref43","first-page":"1","article-title":"A neural probabilistic language model","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","volume":"13","author":"Bengio"},{"key":"ref44","doi-asserted-by":"publisher","DOI":"10.48550\/ARXIV.1706.03762"},{"key":"ref45","doi-asserted-by":"publisher","DOI":"10.1162\/coli_a_00492"},{"key":"ref46","article-title":"Emergent abilities of large language models","author":"Wei","year":"2022","journal-title":"arXiv:2206.07682"},{"issue":"8","key":"ref47","first-page":"9","article-title":"Language models are unsupervised multitask learners","volume":"1","author":"Radford","year":"2019","journal-title":"OpenAI blog"},{"key":"ref48","article-title":"BART: Denoising sequence-to-sequence pre-training for natural language generation, translation, and comprehension","author":"Lewis","year":"2019","journal-title":"arXiv:1910.13461"},{"key":"ref49","article-title":"Galactica: A large language model for science","author":"Taylor","year":"2022","journal-title":"arXiv:2211.09085"},{"key":"ref50","doi-asserted-by":"publisher","DOI":"10.1145\/3624724"},{"key":"ref51","article-title":"TensorFlow-serving: Flexible, high-performance ML serving","author":"Olston","year":"2017","journal-title":"arXiv:1712.06139"},{"key":"ref52","doi-asserted-by":"publisher","DOI":"10.1109\/SC41404.2022.00043"},{"key":"ref53","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2023.emnlp-demo.48"},{"key":"ref54","article-title":"Norm tweaking: High-performance low-bit quantization of large language models","author":"Li","year":"2023","journal-title":"arXiv:2309.02784"},{"key":"ref55","article-title":"ALBERT: A lite BERT for self-supervised learning of language representations","author":"Lan","year":"2019","journal-title":"arXiv:1909.11942"},{"key":"ref56","article-title":"Full parameter fine-tuning for large language models with limited resources","author":"Lv","year":"2023","journal-title":"arXiv:2306.09782"},{"key":"ref57","article-title":"AdaLomo: Low-memory optimization with adaptive learning rate","author":"Lv","year":"2023","journal-title":"arXiv:2310.10195"},{"key":"ref58","article-title":"NLP-fast: A fast, scalable, and flexible system to accelerate large-scale heterogeneous NLP models","author":"Kim","year":"2017","journal-title":"arXiv:1712.06139"},{"key":"ref59","first-page":"489","article-title":"PetS: A unified framework for parameter-efficient transformers serving","volume-title":"Proc. USENIX Annu. Tech. Conf.","author":"Zhou"},{"key":"ref60","article-title":"Petals: Collaborative inference and fine-tuning of large models","author":"Borzunov","year":"2022","journal-title":"arXiv:2209.01188"},{"key":"ref61","article-title":"LightSeq: A high performance inference library for transformers","author":"Wang","year":"2020","journal-title":"arXiv:2010.13887"},{"key":"ref62","article-title":"Easy and efficient transformer: Scalable inference solution for large NLP model","author":"Li","year":"2021","journal-title":"arXiv:2104.12470"},{"key":"ref63","article-title":"Splitwise: Efficient generative LLM inference using phase splitting","author":"Patel","year":"2023","journal-title":"arXiv:2311.18677"},{"key":"ref64","article-title":"A hardware evaluation framework for large language model inference","author":"Zhang","year":"2023","journal-title":"arXiv:2312.03134"},{"key":"ref65","article-title":"PowerInfer: Fast large language model serving with a consumer-grade GPU","author":"Song","year":"2023","journal-title":"arXiv:2312.12456"},{"key":"ref66","doi-asserted-by":"publisher","DOI":"10.48550\/ARXIV.1907.11692"},{"key":"ref67","doi-asserted-by":"publisher","DOI":"10.1145\/3600006.3613165"},{"key":"ref68","doi-asserted-by":"publisher","DOI":"10.1145\/3373376.3378530"},{"key":"ref69","first-page":"1","article-title":"Efficient large-scale language model training on GPU clusters using megatron-LM","volume-title":"Proc. Int. Conf. High Perform. Comput., Netw., Storage Anal.","author":"Narayanan"},{"key":"ref70","first-page":"10323","article-title":"Sparsegpt: Massive language models can be accurately pruned in one-shot","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Frantar"},{"key":"ref71","doi-asserted-by":"publisher","DOI":"10.1109\/HPCA.2017.29"},{"key":"ref72","article-title":"Sheared LLaMA: Accelerating language model pre-training via structured pruning","author":"Xia","year":"2023","journal-title":"arXiv:2310.06694"},{"key":"ref73","article-title":"GrowLength: Accelerating LLMs pretraining by progressively growing training length","author":"Jin","year":"2023","journal-title":"arXiv:2310.00576"},{"key":"ref74","article-title":"AlphaTuning: Quantization-aware parameter-efficient adaptation of large-scale pre-trained language models","author":"Jung Kwon","year":"2022","journal-title":"arXiv:2210.03858"},{"key":"ref75","article-title":"QFT: Quantized full-parameter tuning of LLMs with affordable resources","author":"Li","year":"2023","journal-title":"arXiv:2310.07147"},{"key":"ref76","article-title":"Scaling laws for neural language models","author":"Kaplan","year":"2020","journal-title":"arXiv:2001.08361"},{"key":"ref77","article-title":"QMoE: Practical sub-1-bit compression of trillion-parameter models","author":"Frantar","year":"2023","journal-title":"arXiv:2310.16795"},{"key":"ref78","article-title":"GPTQ: Accurate post-training quantization for generative pre-trained transformers","author":"Frantar","year":"2022","journal-title":"arXiv:2210.17323"},{"key":"ref79","article-title":"FPTQ: Fine-grained post-training quantization for large language models","author":"Li","year":"2023","journal-title":"arXiv:2308.15987"},{"key":"ref80","article-title":"BLOOM: A 176B-parameter open-access multilingual language model","author":"Scao","year":"2022","journal-title":"arXiv:2211.05100"},{"key":"ref81","article-title":"FineQuant: Unlocking efficiency with fine-grained weight-only quantization for LLMs","author":"Jin Kim","year":"2023","journal-title":"arXiv:2308.09723"},{"key":"ref82","article-title":"QuantEase: Optimization-based quantization for language models","author":"Behdin","year":"2023","journal-title":"arXiv:2309.01885"},{"key":"ref83","first-page":"21702","article-title":"LLM-Pruner: On the structural pruning of large language models","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","volume":"36","author":"Ma"},{"key":"ref84","article-title":"Instruction tuning with GPT-4","author":"Peng","year":"2023","journal-title":"arXiv:2304.03277"},{"key":"ref85","article-title":"GLM-130B: An open bilingual pre-trained model","author":"Zeng","year":"2022","journal-title":"arXiv:2210.02414"},{"key":"ref86","doi-asserted-by":"publisher","DOI":"10.1109\/MICRO56248.2022.00051"},{"key":"ref87","article-title":"Sequence parallelism: Long sequence training from system perspective","author":"Li","year":"2022","journal-title":"arXiv:2105.13120"},{"key":"ref88","first-page":"559","article-title":"Alpa: Automating inter-and intra-operator parallelism for distributed deep learning","volume-title":"Proc. 16th USENIX Symp. Operating Syst. Design Implement.","author":"Zheng"},{"key":"ref89","article-title":"Fast inference of mixture-of-experts language models with offloading","author":"Eliseev","year":"2023","journal-title":"arXiv:2312.17238"},{"key":"ref90","article-title":"FP8-LM: Training FP8 large language models","author":"Peng","year":"2023","journal-title":"arXiv:2310.18313"},{"key":"ref91","doi-asserted-by":"publisher","DOI":"10.1109\/FCCM51124.2021.00014"},{"key":"ref92","article-title":"Mixed precision training","author":"Micikevicius","year":"2017","journal-title":"arXiv:1710.03740"}],"container-title":["IEEE Access"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx8\/6287639\/10380310\/10589417.pdf?arnumber=10589417","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,7,19]],"date-time":"2024-07-19T05:01:19Z","timestamp":1721365279000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/10589417\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024]]},"references-count":92,"URL":"https:\/\/doi.org\/10.1109\/access.2024.3424945","relation":{},"ISSN":["2169-3536"],"issn-type":[{"value":"2169-3536","type":"electronic"}],"subject":[],"published":{"date-parts":[[2024]]}}}