{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,8,5]],"date-time":"2026-08-05T06:45:23Z","timestamp":1785912323588,"version":"3.56.0"},"reference-count":36,"publisher":"IEEE","license":[{"start":{"date-parts":[[2026,6,1]],"date-time":"2026-06-01T00:00:00Z","timestamp":1780272000000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/ieeexplore.ieee.org\/Xplorehelp\/downloads\/license-information\/IEEE.html"},{"start":{"date-parts":[[2026,6,1]],"date-time":"2026-06-01T00:00:00Z","timestamp":1780272000000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2026,6,1]],"date-time":"2026-06-01T00:00:00Z","timestamp":1780272000000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"}],"funder":[{"name":"National Key Research and Development Program","award":["2024YFB4505604"],"award-info":[{"award-number":["2024YFB4505604"]}]},{"name":"National Natural Science Foundation of China","award":["62402025"],"award-info":[{"award-number":["62402025"]}]}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2026,6]]},"DOI":"10.1109\/2575-8411.2026.00049","type":"proceedings-article","created":{"date-parts":[[2026,8,4]],"date-time":"2026-08-04T19:15:39Z","timestamp":1785870939000},"page":"447-457","source":"Crossref","is-referenced-by-count":0,"title":["MEGATRACE: Troubleshooting Hang and Slowdown in Large-Scale LLM Training Clusters"],"prefix":"10.1109","author":[{"given":"Fangzheng","family":"Jiao","sequence":"first","affiliation":[{"name":"School of Software, Beihang University"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Menghao","family":"Zhang","sequence":"additional","affiliation":[{"name":"School of Software, Beihang University"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Bolin","family":"Chen","sequence":"additional","affiliation":[{"name":"Nanyang Technological University"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Jiaxun","family":"Huang","sequence":"additional","affiliation":[{"name":"School of Software, Beihang University"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Yanmin","family":"Jia","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Xiaohe","family":"Hu","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Bohua","family":"Xu","sequence":"additional","affiliation":[{"name":"China Unicom Research Institute"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Bowen","family":"Han","sequence":"additional","affiliation":[{"name":"China Unicom Research Institute"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Chunming","family":"Hu","sequence":"additional","affiliation":[{"name":"School of Software, Beihang University"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"263","reference":[{"key":"ref1","volume-title":"OpenAI","year":"2025"},{"key":"ref2","volume-title":"xAI","year":"2025"},{"key":"ref3","volume-title":"InfiniBand","year":"2025"},{"key":"ref4","author":"Trade Association","year":"2020","journal-title":"InfiniBand Architecture Specification Release 1.4 Annex A17: RoCEv2"},{"key":"ref5","year":"2024","journal-title":"Introducing Llama 3.1: Our most capable models to date"},{"key":"ref6","article-title":"Opt: Open pre-trained transformer language models","author":"Zhang","year":"2022","journal-title":"arXiv preprint"},{"key":"ref7","volume-title":"Nvidia gpumemory error management","year":"2022"},{"key":"ref8","volume-title":"Pytorch profiler","year":"2026"},{"key":"ref9","volume-title":"Nvidia nsight systems","year":"2024"},{"key":"ref10","doi-asserted-by":"publisher","DOI":"10.1145\/3627703.3650085"},{"key":"ref11","first-page":"505","article-title":"Minder: Faulty machine detection for large-scale distributed model training","author":"Deng","year":"2025","journal-title":"NSDI"},{"key":"ref12","first-page":"865","article-title":"Evolution of aegis: Fault diagnosis for {AI} model training service in production","author":"Dong","year":"2025","journal-title":"NSDI"},{"key":"ref13","first-page":"523","article-title":"Holmes: Localizing irregularities in {LLM} training with mega-scale {GPU} clusters","author":"Yao","year":"2025","journal-title":"NSDI"},{"key":"ref14","first-page":"731","article-title":"Greyhound: Hunting fail-slows in hybrid-parallel training at scale","author":"Wu","year":"2025","journal-title":"ATC"},{"key":"ref15","first-page":"745","article-title":"{MegaScale}: Scaling large language model training to more than 10,000 {GPUs}","author":"Jiang","year":"2024","journal-title":"NSDI"},{"key":"ref16","article-title":"Understanding stragglers in large model training using what-if analysis","author":"Lin","year":"2025","journal-title":"OSDI"},{"key":"ref17","doi-asserted-by":"publisher","DOI":"10.1145\/1460299.1460318"},{"key":"ref18","volume-title":"open source of megatrace","year":"2026"},{"key":"ref19","first-page":"1","article-title":"Efficient large-scale language model training on gpu clusters using megatron-lm","author":"Narayanan","year":"2021","journal-title":"SC"},{"key":"ref20","doi-asserted-by":"publisher","DOI":"10.1145\/3394486.3406703"},{"key":"ref21","first-page":"709","article-title":"Characterization of large language model development in the datacenter","author":"Hu","year":"2024","journal-title":"NSDI"},{"key":"ref22","volume-title":"Xid error","year":"2024"},{"key":"ref23","volume-title":"Nvidia cuda profiling tools interface","year":"2026"},{"key":"ref24","volume-title":"Dawnbench","year":"2018"},{"key":"ref25","volume-title":"Mlperf benchmarks","year":"2024"},{"key":"ref26","first-page":"835","article-title":"{SuperBench}: Improving cloud \\{AI\\} infrastructure reliability with proactive validation","author":"Xiong","year":"2024","journal-title":"ATC"},{"key":"ref27","volume-title":"Aicb","year":"2024"},{"key":"ref28","volume-title":"Torch distributed elastic","year":"2024"},{"key":"ref29","doi-asserted-by":"publisher","DOI":"10.1145\/3600006.3613145"},{"key":"ref30","first-page":"497","article-title":"Bamboo: Making preemptible instances resilient for affordable training of large {DNNs}","author":"Thorpe","year":"2023","journal-title":"NSDI"},{"key":"ref31","volume-title":"Dlrover","year":"2024"},{"key":"ref32","doi-asserted-by":"publisher","DOI":"10.1145\/3600006.3613152"},{"key":"ref33","doi-asserted-by":"publisher","DOI":"10.1145\/3694715.3695960"},{"key":"ref34","first-page":"203","article-title":"{CheckFreq}: Frequent, {Fine-Grained}\\{DNN} checkpointing","author":"Mohan","year":"2021","journal-title":"FAST"},{"key":"ref35","first-page":"929","article-title":"{Check-N-Run}: A checkpointing system for training deep learning recommendation models","author":"Eisenman","year":"2022","journal-title":"NSDI"},{"key":"ref36","article-title":"Boosting large-scale parallel training efficiency with c4: A communication-driven approach","author":"Dong","year":"2024","journal-title":"arXiv preprint"}],"event":{"name":"2026 IEEE 46th International Conference on Distributed Computing Systems (ICDCS)","location":"Seoul, Korea, Republic of","start":{"date-parts":[[2026,6,22]]},"end":{"date-parts":[[2026,6,25]]}},"container-title":["2026 IEEE 46th International Conference on Distributed Computing Systems (ICDCS)"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx8\/11618978\/11618981\/11619181.pdf?arnumber=11619181","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,8,5]],"date-time":"2026-08-05T05:53:20Z","timestamp":1785909200000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/11619181\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026,6]]},"references-count":36,"URL":"https:\/\/doi.org\/10.1109\/2575-8411.2026.00049","relation":{},"subject":[],"published":{"date-parts":[[2026,6]]}}}