{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,6,4]],"date-time":"2026-06-04T12:09:44Z","timestamp":1780574984224,"version":"3.54.1"},"reference-count":27,"publisher":"IEEE","funder":[{"DOI":"10.13039\/100000015","name":"U.S. Department of Energy","doi-asserted-by":"publisher","award":["DE-AC05-000R22725"],"award-info":[{"award-number":["DE-AC05-000R22725"]}],"id":[{"id":"10.13039\/100000015","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2024,5]]},"DOI":"10.23919\/isc.2024.10528939","type":"proceedings-article","created":{"date-parts":[[2024,5,10]],"date-time":"2024-05-10T17:22:23Z","timestamp":1715361743000},"page":"1-11","source":"Crossref","is-referenced-by-count":15,"title":["Optimizing Distributed Training on Frontier for Large Language Models"],"prefix":"10.23919","author":[{"given":"Sajal","family":"Dash","sequence":"first","affiliation":[{"name":"Oak Ridge National Laboratory"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Isaac R","family":"Lyngaas","sequence":"additional","affiliation":[{"name":"Oak Ridge National Laboratory"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Junqi","family":"Yin","sequence":"additional","affiliation":[{"name":"Oak Ridge National Laboratory"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Xiao","family":"Wang","sequence":"additional","affiliation":[{"name":"Oak Ridge National Laboratory"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Romain","family":"Egele","sequence":"additional","affiliation":[{"name":"Universit\u00e9 Paris-Saclay"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"J. Austin","family":"Ellis","sequence":"additional","affiliation":[{"name":"AMD"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Matthias","family":"Maiterth","sequence":"additional","affiliation":[{"name":"Oak Ridge National Laboratory"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Guojing","family":"Cong","sequence":"additional","affiliation":[{"name":"Oak Ridge National Laboratory"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Feiyi","family":"Wang","sequence":"additional","affiliation":[{"name":"Oak Ridge National Laboratory"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Prasanna","family":"Balaprakash","sequence":"additional","affiliation":[{"name":"Oak Ridge National Laboratory"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"263","reference":[{"key":"ref1","article-title":"Scaling laws for neural language models","author":"Kaplan","year":"2020","journal-title":"arXiv preprint"},{"key":"ref2","first-page":"1877","article-title":"Language models are few-shot learners","volume":"33","author":"Brown","year":"2020","journal-title":"Advances in neural information processing systems"},{"key":"ref3","article-title":"Bloom: A 176b-parameter open-access multilingual language model","author":"Workshop","year":"2022","journal-title":"arXiv preprint"},{"key":"ref4","article-title":"Opt: Open pre-trained transformer language models","author":"Zhang","year":"2022","journal-title":"arXiv preprint"},{"key":"ref5","article-title":"Using deepspeed and megatron to train megatron-turing nlg 530b, a large-scale generative language model","author":"Smith","year":"2022","journal-title":"arXiv preprint"},{"key":"ref6","article-title":"Training compute-optimal large language models","author":"Hoffmann","year":"2022","journal-title":"arXiv preprint"},{"key":"ref7","doi-asserted-by":"publisher","DOI":"10.1145\/3581784.3613215"},{"key":"ref8","article-title":"Redpajama: an open dataset for training large languagemodels","author":"Computer","year":"2023"},{"key":"ref9","article-title":"Dolma: An Open Corpus of 3 Trillion Tokens for Language Model Pretraining Research","volume-title":"Allen Institute for AI","author":"Soldaini","year":"2023"},{"key":"ref10","article-title":"The Dolma Toolkit","author":"Luca","year":"2023","journal-title":"Apache 2.0 License, Version 0. 9. 0"},{"key":"ref11","doi-asserted-by":"publisher","DOI":"10.1007\/s11227-023-05479-7"},{"key":"ref12","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1145\/3458817.3476209","article-title":"Efficient large-scale language model training on gpu clusters using megatron-lm","volume-title":"Proceedings of the International Conference for High Performance Computing, Networking, Storage and Analysis","author":"Narayanan","year":"2021"},{"key":"ref13","doi-asserted-by":"publisher","DOI":"10.48550\/arxiv.1811.06965"},{"key":"ref14","doi-asserted-by":"publisher","DOI":"10.1145\/3341301.3359646"},{"key":"ref15","doi-asserted-by":"publisher","DOI":"10.1145\/3458817.3476145"},{"key":"ref17","doi-asserted-by":"publisher","DOI":"10.1109\/SC41405.2020.00024"},{"key":"ref18","doi-asserted-by":"publisher","DOI":"10.14778\/3611540.3611569"},{"key":"ref19","doi-asserted-by":"publisher","DOI":"10.48550\/ARXIV.1706.03762"},{"key":"ref20","article-title":"Bert: Pre-training of deep bidirectional transformers for language understanding","author":"Devlin","year":"2018","journal-title":"arXiv preprint"},{"key":"ref21","author":"Radford","year":"2018","journal-title":"Improving language understanding by generative pre-training"},{"key":"ref24","first-page":"16 344","article-title":"Flashattention: Fast and memory-efficient exact attention with io-awareness","volume":"35","author":"Dao","year":"2022","journal-title":"Advances in Neural Information Processing Systems"},{"key":"ref25","article-title":"Flashattention-2: Faster attention with better parallelism and work partitioning","author":"Dao","year":"2023","journal-title":"arXiv preprint"},{"key":"ref27","doi-asserted-by":"publisher","DOI":"10.1109\/HiPC.2018.00014"},{"key":"ref28","doi-asserted-by":"publisher","DOI":"10.1109\/SC41405.2020.00012"},{"key":"ref29","doi-asserted-by":"publisher","DOI":"10.1109\/ICPR56361.2022.9956231"},{"key":"ref30","doi-asserted-by":"publisher","DOI":"10.1109\/e-Science58273.2023.10254839"},{"key":"ref31","first-page":"4765","article-title":"A unified approach to interpreting model predictions","volume-title":"Advances in Neural Information Processing Systems 30","author":"Lundberg","year":"2017"}],"event":{"name":"ISC High Performance 2024 Research Paper Proceedings (39th International Conference)","location":"Hamburg, Germany","start":{"date-parts":[[2024,5,12]]},"end":{"date-parts":[[2024,5,16]]}},"container-title":["ISC High Performance 2024 Research Paper Proceedings (39th International Conference)"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx7\/10528919\/10528920\/10528939.pdf?arnumber=10528939","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,11,27]],"date-time":"2024-11-27T00:16:32Z","timestamp":1732666592000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/10528939\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,5]]},"references-count":27,"URL":"https:\/\/doi.org\/10.23919\/isc.2024.10528939","relation":{},"subject":[],"published":{"date-parts":[[2024,5]]}}}