{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,3,8]],"date-time":"2026-03-08T01:38:41Z","timestamp":1772933921759,"version":"3.50.1"},"reference-count":64,"publisher":"IEEE","license":[{"start":{"date-parts":[[2025,12,8]],"date-time":"2025-12-08T00:00:00Z","timestamp":1765152000000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2025,12,8]],"date-time":"2025-12-08T00:00:00Z","timestamp":1765152000000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2025,12,8]]},"DOI":"10.1109\/bigdata66926.2025.11401965","type":"proceedings-article","created":{"date-parts":[[2026,3,6]],"date-time":"2026-03-06T20:57:57Z","timestamp":1772830677000},"page":"6699-6706","source":"Crossref","is-referenced-by-count":0,"title":["Transformer Scalability Crisis: The First Comprehensive Empirical Analysis of Performance Walls in Modern Language Models"],"prefix":"10.1109","author":[{"given":"Mahdi Naser","family":"Moghadasi","sequence":"first","affiliation":[{"name":"BrightMind AI,Research Division,Seattle,WA"}]},{"given":"Faezeh","family":"Ghaderi","sequence":"additional","affiliation":[{"name":"University of Texas at Arlington,Arlington,TX"}]}],"member":"263","reference":[{"key":"ref1","doi-asserted-by":"publisher","DOI":"10.48550\/ARXIV.1706.03762"},{"key":"ref2","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/N19-1423"},{"key":"ref3","first-page":"1877","article-title":"Language models are few-shot learners","volume":"33","author":"Brown","year":"2020","journal-title":"Advances in neural information processing systems"},{"key":"ref4","article-title":"An image is worth 16x16 words: Transformers for image recognition at scale","volume-title":"International Conference on Learning Representations","author":"Dosovitskiy","year":"2021"},{"key":"ref5","author":"Beltagy","year":"2020","journal-title":"Longformer: The longdocument transformer"},{"key":"ref6","author":"Chen","year":"2021","journal-title":"Evaluating large language models trained on code"},{"key":"ref7","author":"Adiwardana","year":"2020","journal-title":"Towards a human-like open-domain chatbot"},{"key":"ref8","doi-asserted-by":"publisher","DOI":"10.1145\/3530811"},{"key":"ref9","doi-asserted-by":"publisher","DOI":"10.1007\/s11431-020-1647-3"},{"key":"ref10","author":"Child","year":"2019","journal-title":"Generating long sequences with sparse transformers"},{"key":"ref11","first-page":"5156","article-title":"Transformers are rnns: Fast autoregressive transformers with linear attention","volume-title":"International Conference on Machine Learning","author":"Katharopoulos","year":"2020"},{"key":"ref12","first-page":"17283","article-title":"Big bird: Transformers for longer sequences","volume":"33","author":"Zaheer","year":"2020","journal-title":"Advances in neural information processing systems"},{"key":"ref13","author":"Wang","year":"2020","journal-title":"Linformer: Self-attention with linear complexity"},{"key":"ref14","author":"Choromanski","year":"2020","journal-title":"Rethinking attention with performers"},{"key":"ref15","author":"Gu","year":"2023","journal-title":"Mamba: Linear-time sequence modeling with selective state spaces"},{"key":"ref16","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2023.findings-emnlp.936"},{"key":"ref17","author":"Sun","year":"2023","journal-title":"Retentive network: A successor to transformer for large language models"},{"key":"ref18","author":"Kaplan","year":"2020","journal-title":"Scaling laws for neural language models"},{"key":"ref19","author":"Hoffmann","year":"2022","journal-title":"Training compute-optimal large language models"},{"key":"ref20","author":"Tay","year":"2022","journal-title":"Scaling up models and data with t 5 x and seqio"},{"issue":"120","key":"ref21","first-page":"1","article-title":"Switch transformer: Scaling to trillion parameter models with simple and efficient sparsity","volume":"23","author":"Fedus","year":"2022","journal-title":"Journal of Machine Learning Research"},{"key":"ref22","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/W18-5446"},{"key":"ref23","article-title":"SuperGLUE: A stickier benchmark for general-purpose language understanding systems","volume":"32","author":"Wang","year":"2019","journal-title":"Advances in neural information processing systems"},{"key":"ref24","author":"Srivastava","year":"2022","journal-title":"Beyond the imitation game: Quantifying and extrapolating the capabilities of language models"},{"key":"ref25","author":"Liang","year":"2022","journal-title":"Holistic evaluation of language models"},{"key":"ref26","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/p19-1355"},{"issue":"8","key":"ref27","first-page":"9","article-title":"Language models are unsupervised multitask learners","volume":"1","author":"Radford","year":"2019","journal-title":"OpenAI blog"},{"key":"ref28","author":"Zhang","year":"2022","journal-title":"OPT: Open pre-trained transformer language models"},{"key":"ref29","author":"Scao","year":"2022","journal-title":"BLOOM: A 176b-parameter open-access multilingual language model"},{"key":"ref30","author":"Dey","year":"2023","journal-title":"Cerebras-GPT: Open compute-optimal language models trained on the cerebras wafer-scale cluster"},{"key":"ref31","first-page":"2397","article-title":"Pythia: A suite for analyzing large language models across training and scaling","volume-title":"International Conference on Machine Learning","author":"Biderman","year":"2023"},{"key":"ref32","author":"Jiang","year":"2023","journal-title":"Mistral 7B"},{"key":"ref33","author":"Almazrouei","year":"2023","journal-title":"The Falcon series of open language models"},{"key":"ref34","author":"Liu","year":"2019","journal-title":"RoBERTa: A robustly optimized BERT pretraining approach"},{"key":"ref35","article-title":"ALBERT: A lite BERT for self-supervised learning of language representations","volume-title":"International Conference on Learning Representations","author":"Lan","year":"2020"},{"key":"ref36","article-title":"ELECTRA: Pre-training text encoders as discriminators rather than generators","volume-title":"International Conference on Learning Representations","author":"Clark","year":"2020"},{"key":"ref37","article-title":"DeBERTa: Decoding-enhanced BERT with disentangled attention","volume-title":"International Conference on Learning Representations","author":"He","year":"2021"},{"key":"ref38","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D19-1371"},{"key":"ref39","author":"Araci","year":"2019","journal-title":"FinBERT: Financial sentiment analysis with pre-trained language models"},{"key":"ref40","doi-asserted-by":"publisher","DOI":"10.1093\/bioinformatics\/btz682"},{"key":"ref41","author":"Sanh","year":"2019","journal-title":"DistilBERT, a distilled version of BERT: smaller, faster, cheaper and lighter"},{"key":"ref42","author":"Gunasekar","year":"2023","journal-title":"Textbooks are all you need"},{"key":"ref43","author":"Zhang","year":"2024","journal-title":"TinyLlama: An open-source small language model"},{"key":"ref44","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.findings-emnlp.139"},{"key":"ref45","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.findings-emnlp.372"},{"key":"ref46","first-page":"9459","article-title":"Retrieval-augmented generation for knowledge-intensive nlp tasks","volume":"33","author":"Lewis","year":"2020","journal-title":"Advances in neural information processing systems"},{"key":"ref47","first-page":"2206","article-title":"Improving language models by retrieving from trillions of tokens","volume-title":"International Conference on Machine Learning","author":"Borgeaud","year":"2022"},{"key":"ref48","article-title":"Efficiently modeling long sequences with structured state spaces","volume-title":"International Conference on Learning Representations","author":"Gu","year":"2022"},{"key":"ref49","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.emnlp-main.19"},{"key":"ref50","article-title":"Compressive transformers for long-range sequence modelling","volume-title":"International Conference on Learning Representations","author":"Rae","year":"2020"},{"key":"ref51","article-title":"NVIDIA A100 tensor core GPU architecture","year":"2020","journal-title":"NVIDIA whitepaper"},{"key":"ref52","doi-asserted-by":"publisher","DOI":"10.1145\/3079856.3080246"},{"key":"ref53","doi-asserted-by":"publisher","DOI":"10.1162\/tacl_a_00349"},{"key":"ref54","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/N19-4009"},{"key":"ref55","article-title":"PyTorch: An imperative style, high-performance deep learning library","volume":"32","author":"Paszke","year":"2019","journal-title":"Advances in neural information processing systems"},{"key":"ref56","first-page":"265","article-title":"TensorFlow: A system for large-scale machine learning","volume-title":"12th USENIX symposium on operating systems design and implementation","author":"Abadi","year":"2016"},{"key":"ref57","author":"Bradbury","year":"2018","journal-title":"JAX: composable transformations of Python+NumPy programs"},{"key":"ref58","author":"Patterson","year":"2021","journal-title":"Carbon emissions and large neural network training"},{"key":"ref59","doi-asserted-by":"publisher","DOI":"10.21437\/Interspeech.2021-698"},{"key":"ref60","author":"Shoeybi","year":"2019","journal-title":"Megatron-LM: Training multi-billion parameter language models using model parallelism"},{"key":"ref61","doi-asserted-by":"publisher","DOI":"10.48550\/arxiv.1811.06965"},{"key":"ref62","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.emnlp-demos.6"},{"key":"ref63","author":"Chowdhery","year":"2022","journal-title":"PaLM: Scaling language modeling with pathways"},{"key":"ref64","doi-asserted-by":"publisher","DOI":"10.24818\/ida-ql\/2019.5"}],"event":{"name":"2025 IEEE International Conference on Big Data (BigData)","location":"Macau, China","start":{"date-parts":[[2025,12,8]]},"end":{"date-parts":[[2025,12,11]]}},"container-title":["2025 IEEE International Conference on Big Data (BigData)"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx8\/11400704\/11400712\/11401965.pdf?arnumber=11401965","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,3,7]],"date-time":"2026-03-07T06:55:21Z","timestamp":1772866521000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/11401965\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,12,8]]},"references-count":64,"URL":"https:\/\/doi.org\/10.1109\/bigdata66926.2025.11401965","relation":{},"subject":[],"published":{"date-parts":[[2025,12,8]]}}}