{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,1,10]],"date-time":"2026-01-10T02:18:13Z","timestamp":1768011493818,"version":"3.49.0"},"publisher-location":"New York, NY, USA","reference-count":54,"publisher":"ACM","content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2025,11,16]]},"DOI":"10.1145\/3731599.3767699","type":"proceedings-article","created":{"date-parts":[[2025,11,7]],"date-time":"2025-11-07T16:13:44Z","timestamp":1762532024000},"page":"1512-1523","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":0,"title":["Pretraining LLMs at Scale: Tuning Strategies and Performance Portability."],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0001-7168-9050","authenticated-orcid":false,"given":"Adri\u00e1n","family":"P\u00e9rez Di\u00e9guez","sequence":"first","affiliation":[{"name":"Qualcomm Technologies, Inc., San Diego, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0009-0006-1469-0358","authenticated-orcid":false,"given":"\u00c0lex","family":"Batlle Casellas","sequence":"additional","affiliation":[{"name":"Qualcomm Europe, Inc., Barcelona, Spain"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0009-0007-2473-3590","authenticated-orcid":false,"given":"Aleix","family":"Torres-Camps","sequence":"additional","affiliation":[{"name":"Qualcomm Europe, Inc., Barcelona, Spain"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-9409-5674","authenticated-orcid":false,"given":"Harris","family":"Teague","sequence":"additional","affiliation":[{"name":"Qualcomm Technologies, Inc., San Diego, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-4450-609X","authenticated-orcid":false,"given":"Jordi","family":"Ros-Giralt","sequence":"additional","affiliation":[{"name":"Qualcomm Europe, Inc., Barcelona, Spain"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2025,11,15]]},"reference":[{"key":"e_1_3_3_2_2_2","unstructured":"Rishabh Agarwal Nino Vieillard Yongchao Zhou Piotr Stanczyk Sabela Ramos Matthieu Geist and Olivier Bachem. 2024. On-Policy Distillation of Language Models: Learning from Self-Generated Mistakes. arxiv:https:\/\/arXiv.org\/abs\/2306.13649\u00a0[cs.LG] https:\/\/arxiv.org\/abs\/2306.13649"},{"key":"e_1_3_3_2_3_2","doi-asserted-by":"publisher","unstructured":"Jason Ansel Shoaib Kamil Kalyan Veeramachaneni Jonathan Ragan-Kelley Jeffrey Bosboom Una-May O\u2019Reilly and Saman Amarasinghe. 2014. OpenTuner: An Extensible Framework for Program Autotuning. Parallel Architectures and Compilation Techniques - Conference Proceedings PACT (08 2014). 10.1145\/2628071.2628092","DOI":"10.1145\/2628071.2628092"},{"key":"e_1_3_3_2_4_2","doi-asserted-by":"publisher","DOI":"10.1109\/SCW63240.2024.00180"},{"key":"e_1_3_3_2_5_2","doi-asserted-by":"publisher","unstructured":"Prasanna Balaprakash Jack Dongarra Todd Gamblin Mary Hall Jeffrey\u00a0K. Hollingsworth Boyana Norris and Richard Vuduc. 2018. Autotuning in High-Performance Computing Applications. Proc. IEEE 106 11 (2018) 2068\u20132083. 10.1109\/JPROC.2018.2841200","DOI":"10.1109\/JPROC.2018.2841200"},{"key":"e_1_3_3_2_6_2","doi-asserted-by":"publisher","unstructured":"James Bergstra Nicolas Pinto and David Cox. 2012. Machine Learning for Predictive Auto-Tuning with Boosted Regression Trees. 2012 Innovative Parallel Computing InPar 2012 (05 2012). 10.1109\/InPar.2012.6339587","DOI":"10.1109\/InPar.2012.6339587"},{"key":"e_1_3_3_2_7_2","first-page":"253","volume-title":"ACM International Conference on Supercomputing 25th Anniversary Volume","author":"Bilmes Jeff","year":"1997","unstructured":"Jeff Bilmes, Krste Asanovic, Chee-Whye Chin, and Jim Demmel. 1997. Optimizing matrix multiply using PHiPAC: a portable, high-performance, ANSI C coding methodology. In ACM International Conference on Supercomputing 25th Anniversary Volume. 253\u2013260."},{"key":"e_1_3_3_2_8_2","doi-asserted-by":"publisher","unstructured":"Jeff Bilmes Krste Asanovic Chee-Whye Chin and Jim Demmel. 2014. Optimizing matrix multiply using PHiPAC: A portable high-performance ANSI C coding methodology. Proceedings of the International Conference on Supercomputing (06 2014). 10.1145\/2591635.2591656","DOI":"10.1145\/2591635.2591656"},{"key":"e_1_3_3_2_9_2","unstructured":"Eric Brochu Vlad Cora and Nando Freitas. 2010. A Tutorial on Bayesian Optimization of Expensive Cost Functions with Application to Active User Modeling and Hierarchical Reinforcement Learning. CoRR (2010)."},{"key":"e_1_3_3_2_10_2","doi-asserted-by":"publisher","DOI":"10.1109\/CLUSTR.2008.4663796"},{"key":"e_1_3_3_2_11_2","doi-asserted-by":"publisher","unstructured":"R. Clint Whaley Antoine Petitet and Jack\u00a0J. Dongarra. 2001. Automated empirical optimizations of software and the ATLAS project. Parallel Comput. 27 1 (2001) 3\u201335. 10.1016\/S0167-8191(00)00087-9New Trends in High Performance Computing.","DOI":"10.1016\/S0167-8191(00)00087-9"},{"key":"e_1_3_3_2_12_2","unstructured":"Tri Dao Daniel\u00a0Y. Fu Stefano Ermon Atri Rudra and Christopher R\u00e9. 2022. FlashAttention: Fast and Memory-Efficient Exact Attention with IO-Awareness. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2205.14135 (2022). https:\/\/arxiv.org\/abs\/2205.14135"},{"key":"e_1_3_3_2_13_2","doi-asserted-by":"publisher","DOI":"10.1109\/SC41406.2024.00039"},{"key":"e_1_3_3_2_14_2","doi-asserted-by":"publisher","DOI":"10.1109\/IPDPSW63119.2024.00143"},{"key":"e_1_3_3_2_15_2","doi-asserted-by":"publisher","DOI":"10.1109\/PMBS56514.2022.00006"},{"key":"e_1_3_3_2_16_2","doi-asserted-by":"publisher","DOI":"10.1109\/SC41406.2024.00071"},{"key":"e_1_3_3_2_17_2","unstructured":"Hugging Face. 2024. Accelerate: A simple way to train and run PyTorch models on any device or distributed setup. https:\/\/github.com\/huggingface\/accelerate. Accessed: 2025-08-06."},{"key":"e_1_3_3_2_18_2","doi-asserted-by":"publisher","DOI":"10.1109\/ipdpsw.2015.85"},{"key":"e_1_3_3_2_19_2","doi-asserted-by":"publisher","DOI":"10.1109\/ICASSP.1998.681704"},{"key":"e_1_3_3_2_20_2","doi-asserted-by":"publisher","DOI":"10.1109\/IPDPS.2009.5161004"},{"key":"e_1_3_3_2_21_2","doi-asserted-by":"publisher","unstructured":"Torsten Hoefler Duncan Roweth Keith Underwood Robert Alverson Mark Griswold Vahid Tabatabaee Mohan Kalkunte Surendra Anubolu Siyuan Shen Moray McLaren Abdul Kabbani and Steve Scott. 2023. Data Center Ethernet and Remote Direct Memory Access: Issues at Hyperscale. Computer 56 7 (2023) 67\u201377. 10.1109\/MC.2023.3261184","DOI":"10.1109\/MC.2023.3261184"},{"key":"e_1_3_3_2_22_2","doi-asserted-by":"publisher","DOI":"10.1145\/1555754.1555775"},{"key":"e_1_3_3_2_23_2","unstructured":"Linkedin Inc.2024. Triton: A Language and Compiler for Custom Deep Learning Primitives. https:\/\/github.com\/triton-lang\/triton. Accessed: 2025-08-06."},{"key":"e_1_3_3_2_24_2","doi-asserted-by":"publisher","DOI":"10.1145\/3581784.3607102"},{"key":"e_1_3_3_2_25_2","doi-asserted-by":"publisher","DOI":"10.1109\/PACT.2013.6618822"},{"key":"e_1_3_3_2_26_2","first-page":"1","volume-title":"Proceedings of Machine Learning and Systems","volume":"1","author":"Jia Zhihao","year":"2019","unstructured":"Zhihao Jia, Matei Zaharia, and Alex Aiken. 2019. Beyond Data and Model Parallelism for Deep Neural Networks.. In Proceedings of Machine Learning and Systems , A.\u00a0Talwalkar, V.\u00a0Smith, and M.\u00a0Zaharia (Eds.), Vol.\u00a01. 1\u201313. https:\/\/proceedings.mlsys.org\/paper_files\/paper\/2019\/file\/b422680f3db0986ddd7f8f126baaf0fa-Paper.pdf"},{"key":"e_1_3_3_2_27_2","doi-asserted-by":"publisher","DOI":"10.1109\/CLUSTER.2017.122"},{"key":"e_1_3_3_2_28_2","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-01970-8_89"},{"key":"e_1_3_3_2_29_2","unstructured":"Zongbiao Li Xiezhao Li Yinghao Cui Yijun Chen Zhixuan Gu Yuxuan Liu Wenbo Zhu Fei Jia Ke Liu Qifeng Li Junyao Zhan Jiangtao Zhou Chenxi Zhang and Qike Liu. 2024. Automatically Planning Optimal Parallel Strategy for Large Language Models. arxiv:https:\/\/arXiv.org\/abs\/2501.00254\u00a0[cs.AI] https:\/\/arxiv.org\/abs\/2501.00254"},{"key":"e_1_3_3_2_30_2","doi-asserted-by":"publisher","DOI":"10.1109\/HiPC56025.2022.00019"},{"key":"e_1_3_3_2_31_2","doi-asserted-by":"publisher","DOI":"10.1145\/3437801.3441621"},{"key":"e_1_3_3_2_32_2","doi-asserted-by":"publisher","DOI":"10.1109\/IPDPSW.2010.5470682"},{"key":"e_1_3_3_2_33_2","doi-asserted-by":"crossref","unstructured":"Riccardo Moriconi Marc\u00a0P. Deisenroth and K.\u00a0S.\u00a0Sesh Kumar. 2020. High-dimensional Bayesian optimization using low-dimensional feature spaces. Machine Learning (Springer) 109 (2020) 1925\u20131943.","DOI":"10.1007\/s10994-020-05899-z"},{"key":"e_1_3_3_2_34_2","doi-asserted-by":"publisher","DOI":"10.1145\/3470496.3533727"},{"key":"e_1_3_3_2_35_2","doi-asserted-by":"publisher","DOI":"10.1145\/1654059.1654090"},{"key":"e_1_3_3_2_36_2","doi-asserted-by":"publisher","DOI":"10.1145\/2038698.2038711"},{"key":"e_1_3_3_2_37_2","unstructured":"John Pennycook Jason Sewall and V. Lee. 2016. A Metric for Performance Portability. eprint arXiv:https:\/\/arXiv.org\/abs\/1611.07409 (11 2016) 7."},{"key":"e_1_3_3_2_38_2","doi-asserted-by":"crossref","unstructured":"Samyam Rajbhandari Jeff Rasley Olatunji Ruwase and Yuxiong He. 2020. ZeRO: Memory Optimizations Toward Training Trillion Parameter Models. ArXiv. https:\/\/www.microsoft.com\/en-us\/research\/publication\/zero-memory-optimizations-toward-training-trillion-parameter-models\/","DOI":"10.1109\/SC41405.2020.00024"},{"key":"e_1_3_3_2_39_2","doi-asserted-by":"publisher","DOI":"10.1109\/ISPASS48437.2020.00018"},{"key":"e_1_3_3_2_40_2","doi-asserted-by":"publisher","DOI":"10.1109\/IPDPS.2014.101"},{"key":"e_1_3_3_2_41_2","unstructured":"Mohammad Shoeybi Mostofa Patwary Raul Puri Patrick LeGresley Jared Casper and Bryan Catanzaro. 2020. Megatron-LM: Training Multi-Billion Parameter Language Models Using Model Parallelism. arxiv:https:\/\/arXiv.org\/abs\/1909.08053\u00a0[cs.CL] https:\/\/arxiv.org\/abs\/1909.08053"},{"key":"e_1_3_3_2_42_2","doi-asserted-by":"publisher","DOI":"10.1109\/SCW63240.2024.00179"},{"key":"e_1_3_3_2_43_2","unstructured":"Microsoft\u00a0DeepSpeed Team. 2024. DeepSpeed: Accelerating Deep Learning Training and Inference. https:\/\/github.com\/microsoft\/DeepSpeed. Accessed: 2025-08-06."},{"key":"e_1_3_3_2_44_2","doi-asserted-by":"crossref","unstructured":"Ben van Werkhoven Jason Maassen Henri\u00a0E. Bal and Frank\u00a0J. Seinstra. 2014. Optimizing Convolution Operations on GPUs Using Adaptive Tiling. Future Gener. Comput. Syst. 30 C (jan 2014) 14\u201326.","DOI":"10.1016\/j.future.2013.09.003"},{"key":"e_1_3_3_2_45_2","doi-asserted-by":"publisher","unstructured":"Richard Vuduc James\u00a0W Demmel and Katherine\u00a0A Yelick. 2005. OSKI: A library of automatically tuned sparse matrix kernels. Journal of Physics: Conference Series 16 (2005) 521\u2013530. 10.1088\/1742-6596\/16\/1\/071","DOI":"10.1088\/1742-6596\/16\/1\/071"},{"key":"e_1_3_3_2_46_2","first-page":"739","volume-title":"20th USENIX Symposium on Networked Systems Design and Implementation (NSDI 23)","author":"Wang Weiyang","year":"2023","unstructured":"Weiyang Wang, Moein Khazraee, Zhizhen Zhong, Manya Ghobadi, Zhihao Jia, Dheevatsa Mudigere, Ying Zhang, and Anthony Kewitsch. 2023. TopoOpt: Co-optimizing Network Topology and Parallelization Strategy for Distributed Training Jobs. In 20th USENIX Symposium on Networked Systems Design and Implementation (NSDI 23). USENIX Association, Boston, MA, 739\u2013767. https:\/\/www.usenix.org\/conference\/nsdi23\/presentation\/wang-weiyang"},{"key":"e_1_3_3_2_47_2","doi-asserted-by":"crossref","unstructured":"R.\u00a0Clinton Whaley Antoine Petitet and Jack\u00a0J. Dongarra. 2001. Automated empirical optimizations of software and the ATLAS project. Parallel Comput. 27 (2001) 3\u201335.","DOI":"10.1016\/S0167-8191(00)00087-9"},{"key":"e_1_3_3_2_48_2","doi-asserted-by":"publisher","DOI":"10.48550\/ARXIV.2111.14991"},{"key":"e_1_3_3_2_49_2","doi-asserted-by":"publisher","DOI":"10.1109\/ISPASS57527.2023.00035"},{"key":"e_1_3_3_2_50_2","doi-asserted-by":"publisher","DOI":"10.1109\/PMBS51919.2020.00012"},{"key":"e_1_3_3_2_51_2","doi-asserted-by":"publisher","DOI":"10.1109\/MASCOTS.2013.9"},{"key":"e_1_3_3_2_52_2","doi-asserted-by":"publisher","DOI":"10.1109\/IPDPS57955.2024.00056"},{"key":"e_1_3_3_2_53_2","doi-asserted-by":"publisher","DOI":"10.1145\/781131.781140"},{"key":"e_1_3_3_2_54_2","doi-asserted-by":"publisher","DOI":"10.1145\/2259016.2259037"},{"key":"e_1_3_3_2_55_2","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-38750-0_11"}],"event":{"name":"SC Workshops '25: Workshops of the International Conference for High Performance Computing, Networking, Storage and Analysis","location":"St Louis MO USA","acronym":"SC Workshops '25","sponsor":["SIGHPC ACM Special Interest Group on High Performance Computing, Special Interest Group on High Performance Computing"]},"container-title":["Proceedings of the SC '25 Workshops of the International Conference for High Performance Computing, Networking, Storage and Analysis"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3731599.3767699","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,1,9]],"date-time":"2026-01-09T19:29:40Z","timestamp":1767986980000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3731599.3767699"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,11,15]]},"references-count":54,"alternative-id":["10.1145\/3731599.3767699","10.1145\/3731599"],"URL":"https:\/\/doi.org\/10.1145\/3731599.3767699","relation":{},"subject":[],"published":{"date-parts":[[2025,11,15]]},"assertion":[{"value":"2025-11-15","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}