{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,12]],"date-time":"2026-07-12T06:08:05Z","timestamp":1783836485882,"version":"3.55.0"},"publisher-location":"Singapore","reference-count":23,"publisher":"Springer Nature Singapore","isbn-type":[{"value":"9789819234462","type":"print"},{"value":"9789819234479","type":"electronic"}],"license":[{"start":{"date-parts":[[2026,7,13]],"date-time":"2026-07-13T00:00:00Z","timestamp":1783900800000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2026,7,13]],"date-time":"2026-07-13T00:00:00Z","timestamp":1783900800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2027]]},"DOI":"10.1007\/978-981-92-3447-9_42","type":"book-chapter","created":{"date-parts":[[2026,7,12]],"date-time":"2026-07-12T05:41:08Z","timestamp":1783834868000},"page":"509-519","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":0,"title":["SelCo: Efficient Distributed Multimodal LLM Training with Selective Co-location"],"prefix":"10.1007","author":[{"ORCID":"https:\/\/orcid.org\/0009-0006-6139-6725","authenticated-orcid":false,"given":"Hao","family":"Yao","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-7659-1631","authenticated-orcid":false,"given":"Xi","family":"Wu","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-7221-1216","authenticated-orcid":false,"given":"Jing","family":"Peng","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-4472-2534","authenticated-orcid":false,"given":"Tiejun","family":"Wang","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-8602-3996","authenticated-orcid":false,"given":"Jiqing","family":"Gu","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2026,7,13]]},"reference":[{"key":"42_CR1","doi-asserted-by":"publisher","first-page":"23716","DOI":"10.52202\/068431-1723","volume":"35","author":"JB Alayrac","year":"2022","unstructured":"Alayrac, J.B., et al.: Flamingo: a visual language model for few-shot learning. Adv. Neural Inf. Proces. Syst. 35, 23716\u201323736 (2022)","journal-title":"Adv. Neural Inf. Proces. Syst."},{"key":"42_CR2","first-page":"1","volume-title":"SC22: International Conference for High Performance Computing, Networking, Storage and Analysis","author":"RY Aminabadi","year":"2022","unstructured":"Aminabadi, R.Y., et al.: DeepSpeed-inference: enabling efficient inference of transformer models at unprecedented scale. In: SC22: International Conference for High Performance Computing, Networking, Storage and Analysis, pp. 1\u201315 (2022)"},{"key":"42_CR3","unstructured":"Bai, J.: Qwen Technical Report (2023). arXiv preprint https:\/\/arxiv.org\/abs\/2309.16609 (2023)"},{"key":"42_CR4","unstructured":"Chen, T., Xu, B., Zhang, C., Guestrin, C.: Training Deep Nets with Sublinear Memory Cost (2016). arXiv preprint https:\/\/arxiv.org\/abs\/1604.06174 (2016)."},{"key":"42_CR5","doi-asserted-by":"publisher","first-page":"16344","DOI":"10.52202\/068431-1189","volume":"35","author":"T Dao","year":"2022","unstructured":"Dao, T., Fu, D., Ermon, S., Rudra, A., R\u00e9, C.: FlashAttention: fast and memory-efficient exact attention with IO-awareness. Adv. Neural Inf. Proces. Syst. 35, 16344\u201316359 (2022)","journal-title":"Adv. Neural Inf. Proces. Syst."},{"key":"42_CR6","unstructured":"Dosovitskiy, A., et al.: An Image is Worth 16x16 Words: Transformers for Image Recognition at Scale (2021). arXiv preprint https:\/\/arxiv.org\/abs\/2010.11929 (2021)."},{"key":"42_CR7","first-page":"161","volume-title":"2025 USENIX Annual Technical Conference (USENIX ATC 25)","author":"W Feng","year":"2025","unstructured":"Feng, W., Chen, Y., Wang, S., Peng, Y., Lin, H., Yu, M.: Optimus: accelerating large-scale multi-modal LLM training by bubble exploitation. In: 2025 USENIX Annual Technical Conference (USENIX ATC 25), pp. 161\u2013177 (2025)"},{"key":"42_CR8","first-page":"1157","volume-title":"21st USENIX Symposium on Networked Systems Design and Implementation (NSDI 24)","author":"J Huang","year":"2024","unstructured":"Huang, J., Zhang, Z., Zheng, S., Qin, F., Wang, Y.: DISTMM: accelerating distributed multimodal model training. In: 21st USENIX Symposium on Networked Systems Design and Implementation (NSDI 24), pp. 1157\u20131171 (2024)"},{"key":"42_CR9","volume-title":"Advances in Neural Information Processing Systems","author":"Y Huang","year":"2019","unstructured":"Huang, Y., et al.: GPipe: efficient training of Giant neural networks using pipeline parallelism. In: Advances in Neural Information Processing Systems, vol. 32. Curran Associates, Inc. (2019)"},{"key":"42_CR10","unstructured":"Jang, I., Lu, R., Bansal, N., Chen, A., Chowdhury, M.: Efficient Distributed MLLM Training with Cornstarch (2025). arXiv preprint https:\/\/arxiv.org\/abs\/2503.11367 (2025)"},{"key":"42_CR11","first-page":"19730","volume-title":"Proceedings of the 40th International Conference on Machine Learning","author":"J Li","year":"2023","unstructured":"Li, J., Li, D., Savarese, S., Hoi, S.: BLIP-2: bootstrapping language-image pre-training with frozen image encoders and large language models. In: Proceedings of the 40th International Conference on Machine Learning, pp. 19730\u201319742. PMLR (2023)"},{"key":"42_CR12","doi-asserted-by":"publisher","first-page":"34892","DOI":"10.52202\/075280-1516","volume":"36","author":"H Liu","year":"2023","unstructured":"Liu, H., Li, C., Wu, Q., Lee, Y.J.: Visual instruction tuning. Adv. Neural Inf. Proces. Syst. 36, 34892\u201334916 (2023)","journal-title":"Adv. Neural Inf. Proces. Syst."},{"key":"42_CR13","unstructured":"Micikevicius, P., et al.: Mixed Precision Training (2018). arXiv preprint https:\/\/arxiv.org\/abs\/1710.03740 (2018)"},{"key":"42_CR14","first-page":"1","volume-title":"Proceedings of the 27th ACM Symposium on Operating Systems Principles","author":"D Narayanan","year":"2019","unstructured":"Narayanan, D., et al.: PipeDream: generalized pipeline parallelism for DNN training. In: Proceedings of the 27th ACM Symposium on Operating Systems Principles, pp. 1\u201315. SOSP '19, Association for Computing Machinery, New York, NY, USA (2019)"},{"key":"42_CR15","first-page":"7937","volume-title":"Proceedings of the 38th International Conference on Machine Learning","author":"D Narayanan","year":"2021","unstructured":"Narayanan, D., Phanishayee, A., Shi, K., Chen, X., Zaharia, M.: Memory-efficient pipeline-parallel DNN training. In: Proceedings of the 38th International Conference on Machine Learning, pp. 7937\u20137947. PMLR (2021)"},{"key":"42_CR16","first-page":"8748","volume-title":"Proceedings of the 38th International Conference on Machine Learning","author":"A Radford","year":"2021","unstructured":"Radford, A., et al.: Learning transferable visual models from natural language supervision. In: Proceedings of the 38th International Conference on Machine Learning, pp. 8748\u20138763. PMLR (2021)"},{"key":"42_CR17","first-page":"1","volume-title":"SC20: International Conference for High Performance Computing, Networking, Storage and Analysis","author":"S Rajbhandari","year":"2020","unstructured":"Rajbhandari, S., Rasley, J., Ruwase, O., He, Y.: ZeRO: memory optimizations toward training trillion parameter models. In: SC20: International Conference for High Performance Computing, Networking, Storage and Analysis, pp. 1\u201316 (2020)"},{"key":"42_CR18","unstructured":"Shoeybi, M., Patwary, M., Puri, R., LeGresley, P., Casper, J., Catanzaro, B.: Megatron-LM: Training Multi-Billion Parameter Language Models Using Model Parallelism (2020).]. arXiv preprint https:\/\/arxiv.org\/abs\/1909.08053 (2020)"},{"key":"42_CR19","first-page":"101","volume":"6","author":"Y Tian","year":"2024","unstructured":"Tian, Y., Jia, Z., Luo, Z., Wang, Y., Wu, C.: DiffusionPipe: training large diffusion models with efficient pipelines. Proc. Mach. Learn. Syst. 6, 101\u2013113 (2024)","journal-title":"Proc. Mach. Learn. Syst."},{"key":"42_CR20","unstructured":"Wang, P., et al.: Qwen2-VL: Enhancing Vision-Language Model's Perception of the World at Any Resolution (2024).]. arXiv preprint https:\/\/arxiv.org\/abs\/2409.12191 (2024)"},{"key":"42_CR21","doi-asserted-by":"publisher","first-page":"24","DOI":"10.1145\/3718958.3750472","volume-title":"Proceedings of the ACM SIGCOMM 2025 Conference","author":"Z Zhang","year":"2025","unstructured":"Zhang, Z., et al.: DistTrain: addressing model and data heterogeneity with disaggregated training for multimodal large language models. In: Proceedings of the ACM SIGCOMM 2025 Conference, pp. 24\u201338. SIGCOMM '25, Association for Computing Machinery, New York, NY, USA (2025)"},{"key":"42_CR22","unstructured":"Zhao, Y., et al.: PyTorch FSDP: Experiences on Scaling Fully Sharded Data Parallel (2023). arXiv preprint https:\/\/arxiv.org\/abs\/2304.11277 (2023)."},{"key":"42_CR23","first-page":"559","volume-title":"16th USENIX Symposium on Operating Systems Design and Implementation (OSDI 22)","author":"L Zheng","year":"2022","unstructured":"Zheng, L., et al.: Alpa: automating inter-and intra-operator parallelism for distributed deep learning. In: 16th USENIX Symposium on Operating Systems Design and Implementation (OSDI 22), pp. 559\u2013578 (2022)"}],"container-title":["Lecture Notes in Computer Science","Advanced Intelligent Computing Technology and Applications"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-981-92-3447-9_42","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,7,12]],"date-time":"2026-07-12T05:41:14Z","timestamp":1783834874000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-981-92-3447-9_42"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026,7,13]]},"ISBN":["9789819234462","9789819234479"],"references-count":23,"URL":"https:\/\/doi.org\/10.1007\/978-981-92-3447-9_42","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"value":"0302-9743","type":"print"},{"value":"1611-3349","type":"electronic"}],"subject":[],"published":{"date-parts":[[2026,7,13]]},"assertion":[{"value":"13 July 2026","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"ICIC","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"International Conference on Intelligent Computing","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Toronto","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Canada","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2026","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"22 July 2026","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"26 July 2026","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"22","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"icic2026a","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"http:\/\/www.ic-icc.cn\/2026\/index.htm","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}}]}}