{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,8,24]],"date-time":"2025-08-24T00:03:21Z","timestamp":1755993801942,"version":"3.44.0"},"publisher-location":"New York, NY, USA","reference-count":31,"publisher":"ACM","license":[{"start":{"date-parts":[[2024,1,18]],"date-time":"2024-01-18T00:00:00Z","timestamp":1705536000000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"DOI":"10.13039\/501100006374","name":"Japan Society for the Promotion of Science","doi-asserted-by":"publisher","award":["20H04165"],"award-info":[{"award-number":["20H04165"]}],"id":[{"id":"10.13039\/501100006374","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2024,1,18]]},"DOI":"10.1145\/3635035.3635045","type":"proceedings-article","created":{"date-parts":[[2024,1,20]],"date-time":"2024-01-20T00:23:32Z","timestamp":1705710212000},"page":"117-126","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":0,"title":["AshPipe: Asynchronous Hybrid Pipeline Parallel for DNN Training"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-8435-7829","authenticated-orcid":false,"given":"Ryubu","family":"Hosoki","sequence":"first","affiliation":[{"name":"Tokyo Institute of Technology, Japan"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-7297-6211","authenticated-orcid":false,"given":"Toshio","family":"Endo","sequence":"additional","affiliation":[{"name":"Tokyo Institute of Technology, Japan"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-1253-6625","authenticated-orcid":false,"given":"Takahiro","family":"Hirofuchi","sequence":"additional","affiliation":[{"name":"National Institute of Advanced Industrial Science and Technology, Japan"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-2977-6390","authenticated-orcid":false,"given":"Tsutomu","family":"Ikegami","sequence":"additional","affiliation":[{"name":"National Institute of Advanced Industrial Science and Technology, Japan"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2024,1,19]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-50743-5_5"},{"key":"e_1_3_2_1_2_1","volume-title":"Language models are few-shot learners. Advances in neural information processing systems 33","author":"Brown Tom","year":"2020","unstructured":"Tom Brown, Benjamin Mann, Nick Ryder, Melanie Subbiah, Jared\u00a0D Kaplan, 2020. Language models are few-shot learners. Advances in neural information processing systems 33 (2020), 1877\u20131901."},{"key":"e_1_3_2_1_3_1","volume-title":"Efficient and robust parallel DNN training through model parallelism on multi-GPU platform. arXiv preprint arXiv:1809.02839","author":"Chen Chi-Chung","year":"2018","unstructured":"Chi-Chung Chen, Chia-Lin Yang, and Hsiang-Yun Cheng. 2018. Efficient and robust parallel DNN training through model parallelism on multi-GPU platform. arXiv preprint arXiv:1809.02839 (2018)."},{"key":"e_1_3_2_1_4_1","volume-title":"Training deep nets with sublinear memory cost. arXiv preprint arXiv:1604.06174","author":"Chen Tianqi","year":"2016","unstructured":"Tianqi Chen, Bing Xu, Chiyuan Zhang, and Carlos Guestrin. 2016. Training deep nets with sublinear memory cost. arXiv preprint arXiv:1604.06174 (2016)."},{"key":"e_1_3_2_1_5_1","volume-title":"An image is worth 16x16 words: Transformers for image recognition at scale. arXiv preprint arXiv:2010.11929","author":"Dosovitskiy Alexey","year":"2020","unstructured":"Alexey Dosovitskiy, Lucas Beyer, Alexander Kolesnikov, Dirk Weissenborn, Xiaohua Zhai, 2020. An image is worth 16x16 words: Transformers for image recognition at scale. arXiv preprint arXiv:2010.11929 (2020)."},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"publisher","DOI":"10.1145\/3437801.3441593"},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"publisher","DOI":"10.1145\/347837.347846"},{"key":"e_1_3_2_1_8_1","volume-title":"Pipedream: Fast and efficient pipeline parallel DNN training. arXiv preprint arXiv:1806.03377","author":"Harlap Aaron","year":"2018","unstructured":"Aaron Harlap, Deepak Narayanan, Amar Phanishayee, Vivek Seshadri, Nikhil Devanur, 2018. Pipedream: Fast and efficient pipeline parallel DNN training. arXiv preprint arXiv:1806.03377 (2018)."},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2016.90"},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2017.243"},{"key":"e_1_3_2_1_11_1","volume-title":"Gpipe: Efficient training of giant neural networks using pipeline parallelism. Advances in neural information processing systems 32","author":"Huang Yanping","year":"2019","unstructured":"Yanping Huang, Youlong Cheng, Ankur Bapna, Orhan Firat, Dehao Chen, 2019. Gpipe: Efficient training of giant neural networks using pipeline parallelism. Advances in neural information processing systems 32 (2019)."},{"key":"e_1_3_2_1_12_1","volume-title":"Beyond Data and Model Parallelism for Deep Neural Networks.Proceedings of Machine Learning and Systems 1","author":"Jia Zhihao","year":"2019","unstructured":"Zhihao Jia, Matei Zaharia, and Alex Aiken. 2019. Beyond Data and Model Parallelism for Deep Neural Networks.Proceedings of Machine Learning and Systems 1 (2019), 1\u201313."},{"key":"e_1_3_2_1_13_1","volume-title":"On large-batch training for deep learning: Generalization gap and sharp minima. arXiv preprint arXiv:1609.04836","author":"Keskar Nitish\u00a0Shirish","year":"2016","unstructured":"Nitish\u00a0Shirish Keskar, Dheevatsa Mudigere, Jorge Nocedal, Mikhail Smelyanskiy, and Ping Tak\u00a0Peter Tang. 2016. On large-batch training for deep learning: Generalization gap and sharp minima. arXiv preprint arXiv:1609.04836 (2016)."},{"key":"e_1_3_2_1_14_1","volume-title":"torchgpipe: On-the-fly Pipeline Parallelism for Training Giant Models. arXiv preprint arXiv:2004.09910","author":"Kim Chiheon","year":"2020","unstructured":"Chiheon Kim, Heungsub Lee, Myungryong Jeong, Woonhyuk Baek, Boogeon Yoon, 2020. torchgpipe: On-the-fly Pipeline Parallelism for Training Giant Models. arXiv preprint arXiv:2004.09910 (2020)."},{"key":"e_1_3_2_1_15_1","unstructured":"Alex Krizhevsky Geoffrey Hinton 2009. Learning multiple layers of features from tiny images. (2009)."},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.1145\/3065386"},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.1145\/3458817.3476145"},{"key":"e_1_3_2_1_18_1","volume-title":"International Conference on Machine Learning. PMLR, 6543\u20136552","author":"Li Zhuohan","year":"2021","unstructured":"Zhuohan Li, Siyuan Zhuang, Shiyuan Guo, Danyang Zhuo, Hao Zhang, 2021. Terapipe: Token-level pipeline parallelism for training large-scale language models. In International Conference on Machine Learning. PMLR, 6543\u20136552."},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-01216-8_12"},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.1145\/3341301.3359646"},{"key":"e_1_3_2_1_21_1","volume-title":"International Conference on Machine Learning. PMLR, 7937\u20137947","author":"Narayanan Deepak","year":"2021","unstructured":"Deepak Narayanan, Amar Phanishayee, Kaiyu Shi, Xie Chen, and Matei Zaharia. 2021. Memory-efficient pipeline-parallel DNN training. In International Conference on Machine Learning. PMLR, 7937\u20137947."},{"key":"e_1_3_2_1_22_1","volume-title":"2020 USENIX Annual Technical Conference (USENIX ATC 20)","author":"Park H","year":"2020","unstructured":"Jay\u00a0H Park, Gyeongchan Yun, M\u00a0Yi Chang, Nguyen\u00a0T Nguyen, Seungmin Lee, 2020. HetPipe: Enabling Large DNN Training on (Whimpy) Heterogeneous GPU Clusters through Integration of Pipelined Model Parallelism and Data Parallelism. In 2020 USENIX Annual Technical Conference (USENIX ATC 20). 307\u2013321."},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.1007\/s11263-015-0816-y"},{"key":"e_1_3_2_1_24_1","volume-title":"Green Cloud Supercomputer for Convergence of HPC, AI and Big-Data. Tsubame ESJ.: e-science journal 16","author":"Satoshi Matsuoka","year":"2017","unstructured":"Matsuoka Satoshi, Endo Toshio, Nukada Akira, Miura Shinichi, Nomura Akihiro, 2017. Overview of TSUBAME3. 0, Green Cloud Supercomputer for Convergence of HPC, AI and Big-Data. Tsubame ESJ.: e-science journal 16 (2017), 20\u201327."},{"key":"e_1_3_2_1_25_1","volume-title":"Megatron-LM: Training multi-billion parameter language models using model parallelism. arXiv preprint arXiv:1909.08053","author":"Shoeybi Mohammad","year":"2019","unstructured":"Mohammad Shoeybi, Mostofa Patwary, Raul Puri, Patrick LeGresley, Jared Casper, 2019. Megatron-LM: Training multi-billion parameter language models using model parallelism. arXiv preprint arXiv:1909.08053 (2019)."},{"key":"e_1_3_2_1_26_1","volume-title":"Very deep convolutional networks for large-scale image recognition. arXiv preprint arXiv:1409.1556","author":"Simonyan Karen","year":"2014","unstructured":"Karen Simonyan and Andrew Zisserman. 2014. Very deep convolutional networks for large-scale image recognition. arXiv preprint arXiv:1409.1556 (2014)."},{"key":"e_1_3_2_1_27_1","doi-asserted-by":"publisher","DOI":"10.1109\/CANDARW51189.2020.00036"},{"key":"e_1_3_2_1_28_1","volume-title":"International Conference on Machine Learning. PMLR, 10096\u201310106","author":"Tan Mingxing","year":"2021","unstructured":"Mingxing Tan and Quoc Le. 2021. Efficientnetv2: Smaller models and faster training. In International Conference on Machine Learning. PMLR, 10096\u201310106."},{"key":"e_1_3_2_1_29_1","first-page":"269","article-title":"Pipemare: Asynchronous pipeline parallel DNN training","volume":"3","author":"Yang Bowen","year":"2021","unstructured":"Bowen Yang, Jian Zhang, Jonathan Li, Christopher R\u00e9, Christopher Aberger, 2021. Pipemare: Asynchronous pipeline parallel DNN training. Proceedings of Machine Learning and Systems 3 (2021), 269\u2013296.","journal-title":"Proceedings of Machine Learning and Systems"},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"publisher","DOI":"10.1145\/3472456.3472497"},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"publisher","DOI":"10.1109\/TPDS.2021.3094364"}],"event":{"name":"HPCAsia 2024: International Conference on High Performance Computing in Asia-Pacific Region","acronym":"HPCAsia 2024","location":"Nagoya Japan"},"container-title":["Proceedings of the International Conference on High Performance Computing in Asia-Pacific Region"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3635035.3635045","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3635035.3635045","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,8,23]],"date-time":"2025-08-23T02:18:53Z","timestamp":1755915533000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3635035.3635045"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,1,18]]},"references-count":31,"alternative-id":["10.1145\/3635035.3635045","10.1145\/3635035"],"URL":"https:\/\/doi.org\/10.1145\/3635035.3635045","relation":{},"subject":[],"published":{"date-parts":[[2024,1,18]]},"assertion":[{"value":"2024-01-19","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}