{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,8]],"date-time":"2026-04-08T07:54:29Z","timestamp":1775634869264,"version":"3.50.1"},"publisher-location":"Singapore","reference-count":33,"publisher":"Springer Nature Singapore","isbn-type":[{"value":"9789819584017","type":"print"},{"value":"9789819584024","type":"electronic"}],"license":[{"start":{"date-parts":[[2026,1,1]],"date-time":"2026-01-01T00:00:00Z","timestamp":1767225600000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2026,1,1]],"date-time":"2026-01-01T00:00:00Z","timestamp":1767225600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2026]]},"DOI":"10.1007\/978-981-95-8402-4_13","type":"book-chapter","created":{"date-parts":[[2026,4,8]],"date-time":"2026-04-08T07:18:07Z","timestamp":1775632687000},"page":"246-266","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":0,"title":["SHPTA: Stable Hybrid Parallel Distributed Training Architecture in\u00a0Dual-Heterogeneous Environments"],"prefix":"10.1007","author":[{"given":"Yuxin","family":"Wang","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Chuantao","family":"Li","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Chunxiao","family":"Wang","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Fulai","family":"Liu","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Zhigang","family":"Zhao","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Jintao","family":"Li","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Guangdong","family":"Zhang","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2026,4,9]]},"reference":[{"key":"13_CR1","unstructured":"Achiam, et\u00a0al.: GPT-4 technical report. arXiv preprint arXiv:2303.08774 (2023)"},{"key":"13_CR2","doi-asserted-by":"crossref","unstructured":"Akta\u015f, et\u00a0al.: Straggler mitigation at scale. IEEE\/ACM Trans. Netw. 27(6), 2266\u20132279 (2019)","DOI":"10.1109\/TNET.2019.2946464"},{"key":"13_CR3","unstructured":"Anil, et\u00a0al.: Palm 2 technical report. arXiv preprint arXiv:2305.10403 (2023)"},{"key":"13_CR4","unstructured":"Benson, et\u00a0al.: Cephalo: harnessing heterogeneous GPU clusters for training transformer models, pp. arXiv\u20132411 (2024)"},{"key":"13_CR5","unstructured":"Chen, et\u00a0al.: EE-LLM: large-scale training and inference of early-exit large language models with 3d parallelism. arXiv preprint arXiv:2312.04916 (2023)"},{"key":"13_CR6","doi-asserted-by":"crossref","unstructured":"Duan, et\u00a0al.: HPH: hybrid parallelism on heterogeneous clusters for accelerating large-scale DNNs training. In: 2022 IEEE International Conference on Cluster Computing (CLUSTER), pp. 313\u2013323. IEEE (2022)","DOI":"10.1109\/CLUSTER51413.2022.00043"},{"key":"13_CR7","doi-asserted-by":"crossref","unstructured":"Fan, et\u00a0al.: Dapple: a pipelined data parallel approach for training large models. In: Proceedings of the 26th ACM SIGPLAN Symposium on Principles and Practice of Parallel Programming, pp. 431\u2013445 (2021)","DOI":"10.1145\/3437801.3441593"},{"key":"13_CR8","unstructured":"Grattafiori, et\u00a0al.: The llama 3 herd of models. arXiv e-prints, pp. arXiv\u20132407 (2024)"},{"key":"13_CR9","unstructured":"Griggs, et\u00a0al.: M\u00e9lange: cost efficient large language model serving by exploiting GPU heterogeneity. arXiv preprint arXiv:2404.14527 (2024)"},{"key":"13_CR10","unstructured":"Guo, et\u00a0al.: Deepseek-coder: when the large language model meets programming\u2013the rise of code intelligence. arXiv preprint arXiv:2401.14196 (2024)"},{"key":"13_CR11","unstructured":"Kim, et\u00a0al.: Bpipe: memory-balanced pipeline parallelism for training large language models. In: International Conference on Machine Learning, pp. 16639\u201316653. PMLR (2023)"},{"key":"13_CR12","doi-asserted-by":"crossref","unstructured":"Li, et\u00a0al.: Chimera: efficiently training large-scale neural networks with bidirectional pipelines. In: Proceedings of the International Conference for High Performance Computing, Networking, Storage and Analysis, pp. 1\u201314 (2021)","DOI":"10.1145\/3458817.3476145"},{"key":"13_CR13","doi-asserted-by":"crossref","unstructured":"Li, et\u00a0al.: Near-optimal sparse allreduce for distributed deep learning. In: Proceedings of the 27th ACM SIGPLAN Symposium on Principles and Practice of Parallel Programming, pp. 135\u2013149 (2022)","DOI":"10.1145\/3503221.3508399"},{"key":"13_CR14","doi-asserted-by":"crossref","unstructured":"Li, et\u00a0al.: Polygon training architecture for foundation models with network-and device-level heterogeneity. Inf. Fusion 123, 103264 (2025)","DOI":"10.1016\/j.inffus.2025.103264"},{"key":"13_CR15","unstructured":"Lian, et\u00a0al.: Asynchronous decentralized parallel stochastic gradient descent. In: International Conference on Machine Learning, pp. 3043\u20133052. PMLR (2018)"},{"key":"13_CR16","unstructured":"McCandlish, et\u00a0al.: An empirical model of large-batch training. arXiv preprint arXiv:1812.06162 (2018)"},{"key":"13_CR17","doi-asserted-by":"crossref","unstructured":"Miao, et\u00a0al.: Sdpipe: a semi-decentralized framework for heterogeneity-aware pipeline-parallel training. 16(9), 2354\u20132363 (2023)","DOI":"10.14778\/3598581.3598604"},{"key":"13_CR18","doi-asserted-by":"crossref","unstructured":"Narayanan, et\u00a0al.: Pipedream: generalized pipeline parallelism for DNN training. In: Proceedings of the 27th ACM Symposium on Operating Systems Principles, pp. 1\u201315 (2019)","DOI":"10.1145\/3341301.3359646"},{"key":"13_CR19","unstructured":"Park, et\u00a0al.: Hetpipe: enabling large $$\\{$$DNN$$\\}$$ training on (whimpy) heterogeneous $$\\{$$GPU$$\\}$$ clusters through integration of pipelined model parallelism and data parallelism. In: 2020 USENIX Annual Technical Conference (USENIX ATC 2020), pp. 307\u2013321 (2020)"},{"key":"13_CR20","doi-asserted-by":"crossref","unstructured":"Provatas, et\u00a0al.: A survey on parameter server architecture: approaches for optimizing distributed centralized learning. IEEE Access (2025)","DOI":"10.1109\/ACCESS.2025.3535085"},{"key":"13_CR21","doi-asserted-by":"crossref","unstructured":"Rajbhandari, et\u00a0al.: Zero: memory optimizations toward training trillion parameter models. In: SC20: International Conference for High Performance Computing, Networking, Storage and Analysis, pp. 1\u201316. IEEE (2020)","DOI":"10.1109\/SC41405.2020.00024"},{"key":"13_CR22","unstructured":"Ryabinin, et\u00a0al.: Swarm parallelism: training large models can be surprisingly communication-efficient. In: International Conference on Machine Learning, pp. 29416\u201329440. PMLR (2023)"},{"key":"13_CR23","doi-asserted-by":"crossref","unstructured":"Singh, et\u00a0al.: A hybrid tensor-expert-data parallelism approach to optimize mixture-of-experts training. In: Proceedings of the 37th International Conference on Supercomputing, pp. 203\u2013214 (2023)","DOI":"10.1145\/3577193.3593704"},{"key":"13_CR24","doi-asserted-by":"crossref","unstructured":"Song, et\u00a0al.: Optimus-CC: efficient large NLP model training with 3d parallelism aware communication compression. In: Proceedings of the 28th ACM International Conference on Architectural Support for Programming Languages and Operating Systems, vol. 2, pp. 560\u2013573 (2023)","DOI":"10.1145\/3575693.3575712"},{"key":"13_CR25","doi-asserted-by":"crossref","unstructured":"Sun, et\u00a0al.: Adapipe: optimizing pipeline parallelism with adaptive recomputation and partitioning. In: Proceedings of the 29th ACM International Conference on Architectural Support for Programming Languages and Operating Systems, vol. 3, pp. 86\u2013100 (2024)","DOI":"10.1145\/3620666.3651359"},{"key":"13_CR26","doi-asserted-by":"crossref","unstructured":"Xie, et\u00a0al.: Optimal distributed parallel algorithms for deep learning framework tensorflow. Appl. Intell. 52(4), 3880\u20133900 (2022)","DOI":"10.1007\/s10489-021-02588-9"},{"key":"13_CR27","doi-asserted-by":"crossref","unstructured":"Yu, et\u00a0al.: Decentralized parallel SGD with privacy preservation in vehicular networks. IEEE Trans. Veh. Technol. 70(6), 5211\u20135220 (2021)","DOI":"10.1109\/TVT.2021.3064877"},{"key":"13_CR28","doi-asserted-by":"crossref","unstructured":"Zhang, et\u00a0al.: Is network the bottleneck of distributed training? In: Proceedings of the Workshop on Network Meets AI & ML, pp. 8\u201313 (2020)","DOI":"10.1145\/3405671.3405810"},{"key":"13_CR29","doi-asserted-by":"crossref","unstructured":"Zhang, et\u00a0al.: Pipepar: enabling fast DNN pipeline parallel training in heterogeneous GPU clusters. Neurocomputing 555, 126661 (2023)","DOI":"10.1016\/j.neucom.2023.126661"},{"key":"13_CR30","unstructured":"Zhang, et\u00a0al.: Poplar: efficient scaling of distributed DNN training on heterogeneous GPU clusters. arXiv preprint arXiv:2408.12596 (2024)"},{"key":"13_CR31","doi-asserted-by":"crossref","unstructured":"Zhang, et\u00a0al.: Rethinking memory and communication costs for efficient data parallel training of large language models. In; Advances in Neural Information Processing Systems, vol. 37, pp. 28191\u201328218 (2024)","DOI":"10.52202\/079017-0884"},{"key":"13_CR32","doi-asserted-by":"crossref","unstructured":"Zhao, et\u00a0al.: Pytorch FSDP: experiences on scaling fully sharded data parallel. arXiv preprint arXiv:2304.11277 (2023)","DOI":"10.14778\/3611540.3611569"},{"key":"13_CR33","doi-asserted-by":"crossref","unstructured":"Zhou, et\u00a0al.: Falcon: addressing stragglers in heterogeneous parameter server via multiple parallelism. IEEE Trans. Comput. 70(1), 139\u2013155 (2020)","DOI":"10.1109\/TC.2020.2974461"}],"container-title":["Lecture Notes in Computer Science","Algorithms and Architectures for Parallel Processing"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-981-95-8402-4_13","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,4,8]],"date-time":"2026-04-08T07:18:19Z","timestamp":1775632699000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-981-95-8402-4_13"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026]]},"ISBN":["9789819584017","9789819584024"],"references-count":33,"URL":"https:\/\/doi.org\/10.1007\/978-981-95-8402-4_13","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"value":"0302-9743","type":"print"},{"value":"1611-3349","type":"electronic"}],"subject":[],"published":{"date-parts":[[2026]]},"assertion":[{"value":"9 April 2026","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"The authors declare that they have no known competing financial interests or personal relationships that could have appeared to influence the work reported in this work.","order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Disclosure of Funding"}},{"value":"ICA3PP","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"International Conference on Algorithms and Architectures for Parallel Processing","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Zhengzhou","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"China","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2025","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"30 October 2025","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2 November 2025","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"25","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"ica3pp2025","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"https:\/\/ieee-cybermatics.org\/2025\/ica3pp\/","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}}]}}