{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,8,27]],"date-time":"2025-08-27T22:40:06Z","timestamp":1756334406935,"version":"3.44.0"},"publisher-location":"Cham","reference-count":21,"publisher":"Springer International Publishing","isbn-type":[{"type":"print","value":"9783030856649"},{"type":"electronic","value":"9783030856656"}],"license":[{"start":{"date-parts":[[2021,1,1]],"date-time":"2021-01-01T00:00:00Z","timestamp":1609459200000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"},{"start":{"date-parts":[[2021,1,1]],"date-time":"2021-01-01T00:00:00Z","timestamp":1609459200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2021]]},"DOI":"10.1007\/978-3-030-85665-6_12","type":"book-chapter","created":{"date-parts":[[2021,8,28]],"date-time":"2021-08-28T03:06:52Z","timestamp":1630120012000},"page":"183-198","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":2,"title":["Pipelined Model Parallelism: Complexity Results and Memory Considerations"],"prefix":"10.1007","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-2741-6228","authenticated-orcid":false,"given":"Olivier","family":"Beaumont","sequence":"first","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0003-2475-3309","authenticated-orcid":false,"given":"Lionel","family":"Eyraud-Dubois","sequence":"additional","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0002-1795-8421","authenticated-orcid":false,"given":"Alena","family":"Shilova","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2021,8,25]]},"reference":[{"key":"12_CR1","unstructured":"Abadi, M., et al.: TensorFlow: a system for large-scale machine learning. In: 12th USENIX Symposium on Operating Systems Design and Implementation, OSDI 2016, pp. 265\u2013283 (2016)"},{"key":"12_CR2","unstructured":"Beaumont, O., Eyraud-Dubois, L., Herrmann, J., Joly, A., Shilova, A.: Optimal checkpointing for heterogeneous chains: how to train deep neural networks with limited memory. Research Report RR-9302, Inria Bordeaux Sud-Ouest, November 2019"},{"key":"12_CR3","doi-asserted-by":"crossref","unstructured":"Beaumont, O., Eyraud-Dubois, L., Shilova, A.: Optimal GPU-CPU offloading strategies for deep neural network training. In: Proceeding of EuroPar 2020 (2020)","DOI":"10.1007\/978-3-030-57675-2_10"},{"issue":"26","key":"12_CR4","doi-asserted-by":"publisher","first-page":"2572","DOI":"10.1016\/j.tcs.2010.03.019","volume":"411","author":"J Boyar","year":"2010","unstructured":"Boyar, J., Epstein, L., Levin, A.: Tight results for next fit and worst fit with resource augmentation. Theor. Comput. Sci. 411(26), 2572\u20132580 (2010)","journal-title":"Theor. Comput. Sci."},{"key":"12_CR5","doi-asserted-by":"crossref","unstructured":"Chu, C.-H., Kousha, P., Awan, A.A., Khorassani, K.S., Subramoni, H., Panda, D.K.: NV-group: link-efficient reduction for distributed deep learning on modern dense GPU systems. In: Proceedings of the 34th ACM International Conference on Supercomputing, pp. 1\u201312 (2020)","DOI":"10.1145\/3392717.3392771"},{"key":"12_CR6","unstructured":"Dean, J., et al.: Large scale distributed deep networks. In: Advances in Neural Information Processing Systems, pp. 1223\u20131231 (2012)"},{"key":"12_CR7","doi-asserted-by":"crossref","unstructured":"Dryden, N., Maruyama, N., Benson, T., Moon, T., Snir, M., Van Essen, B.: Improving strong-scaling of CNN training by exploiting finer-grained parallelism. In: IEEE International Parallel and Distributed Processing Symposium. IEEE Press (2019)","DOI":"10.1109\/IPDPS.2019.00031"},{"key":"12_CR8","doi-asserted-by":"crossref","unstructured":"Dryden, N., Maruyama, N., Moon, T., Benson, T., Snir, M., Van Essen, B.: Channel and filter parallelism for large-scale CNN training. In: Proceedings of the International Conference for High Performance Computing, Networking, Storage and Analysis, p. 10. ACM (2019)","DOI":"10.1145\/3295500.3356207"},{"key":"12_CR9","volume-title":"Computers and Intractability","author":"MR Garey","year":"1979","unstructured":"Garey, M.R., Johnson, D.S.: Computers and Intractability, vol. 174. Freeman, San Francisco (1979)"},{"key":"12_CR10","unstructured":"Glorot, X., Bengio, Y.: Understanding the difficulty of training deep feedforward neural networks. In: Proceedings of the 13th International Conference on Artificial Intelligence and Statistics, pp. 249\u2013256 (2010)"},{"key":"12_CR11","unstructured":"Huang, Y., et al.: GPipe: efficient training of giant neural networks using pipeline parallelism. In: Advances in Neural Information Processing Systems, pp. 103\u2013112 (2019)"},{"key":"12_CR12","unstructured":"Jain, P., et al.: Checkmate: breaking the memory wall with optimal tensor rematerialization (2019)"},{"key":"12_CR13","unstructured":"Kusumoto, M., Inoue, T., Watanabe, G., Akiba, T., Koyama, M.: A graph theoretic framework of recomputation algorithms for memory-efficient backpropagation. arXiv preprint arXiv:1905.11722 (2019)"},{"issue":"1","key":"12_CR14","doi-asserted-by":"publisher","first-page":"42","DOI":"10.1109\/MM.2004.1268994","volume":"24","author":"J Liu","year":"2004","unstructured":"Liu, J., Yu, W., Wu, J., Buntinas, D., Panda, D.K., Wyckoff, P.: Microbenchmark performance comparison of high-speed cluster interconnects. IEEE Micro 24(1), 42\u201351 (2004)","journal-title":"IEEE Micro"},{"key":"12_CR15","doi-asserted-by":"crossref","unstructured":"Narayanan, D., et al.: PipeDream: generalized pipeline parallelism for DNN training. In: Proceedings of SOSP 2019, pp. 1\u201315 (2019)","DOI":"10.1145\/3341301.3359646"},{"key":"12_CR16","unstructured":"Narayanan, D., Phanishayee, A., Shi, K., Chen, X., Zaharia, M.: Memory-efficient pipeline-parallel DNN training. arXiv preprint arXiv:2006.09503 (2020)"},{"key":"12_CR17","unstructured":"Paszke, A., et al.: Automatic differentiation in PyTorch (2017)"},{"key":"12_CR18","doi-asserted-by":"crossref","unstructured":"Rajbhandari, S., Rasley, J., Ruwase, O., He, Y.: ZeRO: memory optimizations toward training trillion parameter models. In: Proceedings of the International Conference for High Performance Computing, Networking, Storage and Analysis, SC 2020. IEEE Press (2020)","DOI":"10.1109\/SC41405.2020.00024"},{"key":"12_CR19","doi-asserted-by":"crossref","unstructured":"You, Y., Zhang, Z., Hsieh, C.-J., Demmel, J., Keutzer, K.: ImageNet training in minutes. In: Proceedings of the 47th International Conference on Parallel Processing (New York, NY, USA, 2018), ICPP 2018. Association for Computing Machinery (2018)","DOI":"10.1145\/3225058.3225069"},{"key":"12_CR20","doi-asserted-by":"crossref","unstructured":"Zhan, J., Zhang, J.: Pipe-torch: pipeline-based distributed deep learning in a GPU cluster with heterogeneous networking. In: 2019 Seventh International Conference on Advanced Cloud and Big Data, pp. 55\u201360. IEEE (2019)","DOI":"10.1109\/CBD.2019.00020"},{"key":"12_CR21","unstructured":"Zinkevich, M., Weimer, M., Li, L., Smola, A.J.: Parallelized stochastic gradient descent. In: Advances in Neural Information Processing Systems, pp. 2595\u20132603 (2010)"}],"container-title":["Lecture Notes in Computer Science","Euro-Par 2021: Parallel Processing"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-030-85665-6_12","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,8,27]],"date-time":"2025-08-27T22:02:20Z","timestamp":1756332140000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-3-030-85665-6_12"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2021]]},"ISBN":["9783030856649","9783030856656"],"references-count":21,"URL":"https:\/\/doi.org\/10.1007\/978-3-030-85665-6_12","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"type":"print","value":"0302-9743"},{"type":"electronic","value":"1611-3349"}],"subject":[],"published":{"date-parts":[[2021]]},"assertion":[{"value":"25 August 2021","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"Euro-Par","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"European Conference on Parallel Processing","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Lisbon","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Portugal","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2021","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"1 September 2021","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"3 September 2021","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"27","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"europar2021","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"http:\/\/2021.euro-par.org\/","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Single-blind","order":1,"name":"type","label":"Type","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"EasyChair","order":2,"name":"conference_management_system","label":"Conference Management System","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"136","order":3,"name":"number_of_submissions_sent_for_review","label":"Number of Submissions Sent for Review","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"38","order":4,"name":"number_of_full_papers_accepted","label":"Number of Full Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"0","order":5,"name":"number_of_short_papers_accepted","label":"Number of Short Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"28% - The value is computed by the equation \"Number of Full Papers Accepted \/ Number of Submissions Sent for Review * 100\" and then rounded to a whole number.","order":6,"name":"acceptance_rate_of_full_papers","label":"Acceptance Rate of Full Papers","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"4","order":7,"name":"average_number_of_reviews_per_paper","label":"Average Number of Reviews per Paper","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"6","order":8,"name":"average_number_of_papers_per_reviewer","label":"Average Number of Papers per Reviewer","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"Yes","order":9,"name":"external_reviewers_involved","label":"External Reviewers Involved","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"The conference was held virtually due to the COVID-19 pandemic.","order":10,"name":"additional_info_on_review_process","label":"Additional Info on Review Process","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}}]}}