{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,5]],"date-time":"2026-07-05T21:51:48Z","timestamp":1783288308837,"version":"3.54.6"},"publisher-location":"New York, NY, USA","reference-count":50,"publisher":"ACM","license":[{"start":{"date-parts":[[2020,6,27]],"date-time":"2020-06-27T00:00:00Z","timestamp":1593216000000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2020,6,27]]},"DOI":"10.1145\/3377811.3380362","type":"proceedings-article","created":{"date-parts":[[2020,10,1]],"date-time":"2020-10-01T18:25:34Z","timestamp":1601576734000},"page":"1159-1170","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":83,"title":["An empirical study on program failures of deep learning jobs"],"prefix":"10.1145","author":[{"given":"Ru","family":"Zhang","sequence":"first","affiliation":[{"name":"Microsoft Research"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Wencong","family":"Xiao","sequence":"additional","affiliation":[{"name":"Alibaba Group"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Hongyu","family":"Zhang","sequence":"additional","affiliation":[{"name":"The University of Newcastle"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Yu","family":"Liu","sequence":"additional","affiliation":[{"name":"Microsoft Research"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Haoxiang","family":"Lin","sequence":"additional","affiliation":[{"name":"Microsoft Research"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Mao","family":"Yang","sequence":"additional","affiliation":[{"name":"Microsoft Research"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2020,10]]},"reference":[{"key":"e_1_3_2_1_1_1","unstructured":"2019. Amazon SageMaker. https:\/\/aws.amazon.com\/sagemaker."},{"key":"e_1_3_2_1_2_1","unstructured":"2019. Google Cloud AI. https:\/\/cloud.google.com\/products\/ai."},{"key":"e_1_3_2_1_3_1","unstructured":"2019. Microsoft Azure Machine Learning. https:\/\/azure.microsoft.com\/en-us\/services\/machine-learning-service."},{"key":"e_1_3_2_1_4_1","unstructured":"2019. The \"swap_memory\" argument in TensorFlow operator \"tf.nn.dynamic_rnns\" example. https:\/\/www.tensorflow.org\/versions\/r1.15\/api_docs\/python\/tf\/nn\/dynamic_rnn."},{"key":"e_1_3_2_1_5_1","unstructured":"2019. TensorBoard: TensorFlow's Visualization Toolkit. https:\/\/github.com\/tensorflow\/tensorboard."},{"key":"e_1_3_2_1_6_1","unstructured":"2019. TensorFlow Debugger. https:\/\/www.tensorflow.org\/guide\/debugger."},{"key":"e_1_3_2_1_7_1","volume-title":"Proceedings of the 12th USENIX Conference on Operating Systems Design and Implementation (OSDI '16)","author":"Abadi Mart\u00edn","unstructured":"Mart\u00edn Abadi, Paul Barham, Jianmin Chen, Zhifeng Chen, Andy Davis, Jeffrey Dean, Matthieu Devin, Sanjay Ghemawat, Geoffrey Irving, Michael Isard, and et al. 2016. TensorFlow: A System for Large-Scale Machine Learning. In Proceedings of the 12th USENIX Conference on Operating Systems Design and Implementation (OSDI '16). USENIX Association, USA, 265--283."},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.1145\/3097983.3098021"},{"key":"e_1_3_2_1_9_1","volume-title":"Workshop on ML Systems, NIPS.","author":"Boag Scott","year":"2017","unstructured":"Scott Boag, Parijat Dube, Benjamin Herta, Waldemar Hummer, Vatche Ishakian, K JAYARAM, Michael Kalantar, Vinod Muthusamy, Priya NAG-PURKAR, and Florian Rosenberg. 2017. Scalable multi-framework multi-tenant lifecycle management of deep learning training jobs. In Workshop on ML Systems, NIPS."},{"key":"e_1_3_2_1_10_1","volume-title":"MXNet: A Flexible and Efficient Machine Learning Library for Heterogeneous Distributed Systems. CoRR abs\/1512.01274","author":"Chen Tianqi","year":"2015","unstructured":"Tianqi Chen, Mu Li, Yutian Li, Min Lin, Naiyan Wang, Minjie Wang, Tianjun Xiao, Bing Xu, Chiyuan Zhang, and Zheng Zhang. 2015. MXNet: A Flexible and Efficient Machine Learning Library for Heterogeneous Distributed Systems. CoRR abs\/1512.01274 (2015). arXiv:1512.01274 http:\/\/arxiv.org\/abs\/1512.01274"},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.1145\/2988450.2988454"},{"key":"e_1_3_2_1_12_1","unstructured":"Fran\u00e7ois Chollet et al. 2015. Keras. https:\/\/github.com\/fchollet\/keras."},{"key":"e_1_3_2_1_13_1","volume-title":"Proceedings of the 2019 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, NAACL-HLT 2019","volume":"1","author":"Devlin Jacob","year":"2019","unstructured":"Jacob Devlin, Ming-Wei Chang, Kenton Lee, and Kristina Toutanova. 2019. BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding. In Proceedings of the 2019 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, NAACL-HLT 2019, Minneapolis, MN, USA, June 2-7, 2019, Volume 1 (Long and Short Papers). 4171--4186."},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.patrec.2005.10.010"},{"key":"e_1_3_2_1_15_1","volume-title":"Dauphin","author":"Gehring Jonas","year":"2017","unstructured":"Jonas Gehring, Michael Auli, David Grangier, Denis Yarats, and Yann N. Dauphin. 2017. Convolutional Sequence to Sequence Learning. CoRR abs\/1705.03122 (2017). arXiv:1705.03122 http:\/\/arxiv.org\/abs\/1705.03122"},{"key":"e_1_3_2_1_16_1","unstructured":"Ross Girshick Ilija Radosavovic Georgia Gkioxari Piotr Doll\u00e1r and Kaiming He. 2018. Detectron. https:\/\/github.com\/facebookresearch\/detectron."},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.1145\/3097983.3098043"},{"key":"e_1_3_2_1_18_1","volume-title":"Large Minibatch SGD: Training ImageNet in 1 Hour. CoRR abs\/1706.02677","author":"Goyal Priya","year":"2017","unstructured":"Priya Goyal, Piotr Doll\u00e1r, Ross B. Girshick, Pieter Noordhuis, Lukasz Wesolowski, Aapo Kyrola, Andrew Tulloch, Yangqing Jia, and Kaiming He. 2017. Accurate, Large Minibatch SGD: Training ImageNet in 1 Hour. CoRR abs\/1706.02677 (2017). arXiv:1706.02677 http:\/\/arxiv.org\/abs\/1706.02677"},{"key":"e_1_3_2_1_19_1","volume-title":"Proceedings of the 16th USENIX Conference on Networked Systems Design and Implementation (NSDI '19)","author":"Gu Juncheng","year":"2019","unstructured":"Juncheng Gu, Mosharaf Chowdhury, Kang G. Shin, Yibo Zhu, Myeongjae Jeon, Junjie Qian, Hongqiang Liu, and Chuanxiong Guo. 2019. Tiresias: A GPU Cluster Manager for Distributed Deep Learning. In Proceedings of the 16th USENIX Conference on Networked Systems Design and Implementation (NSDI '19). USENIX Association, USA, 485--500."},{"key":"e_1_3_2_1_20_1","volume-title":"Proceedings of the 8th USENIX Conference on Operating Systems Design and Implementation (OSDI '08)","author":"Guo Zhenyu","year":"2008","unstructured":"Zhenyu Guo, Xi Wang, Jian Tang, Xuezheng Liu, Zhilei Xu, Ming Wu, M. Frans Kaashoek, and Zheng Zhang. 2008. R2: An Application-Level Kernel for Record and Replay. In Proceedings of the 8th USENIX Conference on Operating Systems Design and Implementation (OSDI '08). USENIX Association, USA, 193--208."},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2017.322"},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.1145\/3338906.3338955"},{"key":"e_1_3_2_1_23_1","volume-title":"Proceedings of the 2019 USENIX Conference on Usenix Annual Technical Conference (USENIX ATC '19). USENIX Association, USA, 947--960","author":"Jeon Myeongjae","year":"2019","unstructured":"Myeongjae Jeon, Shivaram Venkataraman, Amar Phanishayee, unjie Qian, Wencong Xiao, and Fan Yang. 2019. Analysis of Large-Scale Multi-Tenant GPU Clusters for DNN Training Workloads. In Proceedings of the 2019 USENIX Conference on Usenix Annual Technical Conference (USENIX ATC '19). USENIX Association, USA, 947--960."},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"publisher","DOI":"10.1145\/360248.360252"},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.5555\/2486788.2486921"},{"key":"e_1_3_2_1_26_1","article-title":"Docker: Lightweight Linux Containers for Consistent Development and Deployment","volume":"2014","author":"Merkel Dirk","year":"2014","unstructured":"Dirk Merkel. 2014. Docker: Lightweight Linux Containers for Consistent Development and Deployment. Linux Journal 2014, 239, Article 2 (March 2014).","journal-title":"Linux Journal"},{"key":"e_1_3_2_1_27_1","volume-title":"Riedmiller","author":"Mnih Volodymyr","year":"2013","unstructured":"Volodymyr Mnih, Koray Kavukcuoglu, David Silver, Alex Graves, Ioannis Antonoglou, Daan Wierstra, and Martin A. Riedmiller. 2013. Playing Atari with Deep Reinforcement Learning. CoRR abs\/1312.5602 (2013). arXiv:1312.5602 http:\/\/arxiv.org\/abs\/1312.5602"},{"key":"e_1_3_2_1_28_1","volume-title":"A guide to NumPy","author":"Oliphant Travis E","unstructured":"Travis E Oliphant. 2006. A guide to NumPy. Vol. 1. Trelgol Publishing USA."},{"key":"e_1_3_2_1_29_1","volume-title":"Proceedings of the 3rd International Conference on Distributed Computing Systems. 22--30","author":"Ousterhout J.K.","year":"1982","unstructured":"J.K. Ousterhout. 1982. Scheduling techniques for concurrent systems. In Proceedings of the 3rd International Conference on Distributed Computing Systems. 22--30."},{"key":"e_1_3_2_1_30_1","volume-title":"PyTorch: An Imperative Style","author":"Paszke Adam","unstructured":"Adam Paszke, Sam Gross, Francisco Massa, Adam Lerer, James Bradbury, Gregory Chanan, Trevor Killeen, Zeming Lin, Natalia Gimelshein, Luca Antiga, Alban Desmaison, Andreas Kopf, Edward Yang, Zachary DeVito, Martin Raison, Alykhan Tejani, Sasank Chilamkurthy, Benoit Steiner, Lu Fang, Junjie Bai, and Soumith Chintala. 2019. PyTorch: An Imperative Style, High-Performance Deep Learning Library. In Advances in Neural Information Processing Systems 32. Curran Associates, Inc., 8024--8035."},{"key":"e_1_3_2_1_31_1","volume-title":"Matthew C. H. Lee","author":"Pawlowski Nick","year":"2017","unstructured":"Nick Pawlowski, Sofia Ira Ktena, Matthew C. H. Lee, Bernhard Kainz, Daniel Rueckert, Ben Glocker, and Martin Rajchl. 2017. DLTK: State of the Art Reference Implementations for Deep Learning on Medical Images. CoRR abs\/1711.06853 (2017). arXiv:1711.06853 http:\/\/arxiv.org\/abs\/1711.06853"},{"key":"e_1_3_2_1_32_1","doi-asserted-by":"publisher","DOI":"10.1145\/3132747.3132785"},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"publisher","DOI":"10.1145\/3190508.3190517"},{"key":"e_1_3_2_1_34_1","volume-title":"Proceedings of the International Conference on Learning Representations.","author":"Qi Hang","year":"2017","unstructured":"Hang Qi, Evan R. Sparks, and Ameet Talwalkar. 2017. Paleo: A Performance Model for Deep Neural Networks. In Proceedings of the International Conference on Learning Representations."},{"key":"e_1_3_2_1_35_1","doi-asserted-by":"publisher","DOI":"10.1145\/3135974.3135994"},{"key":"e_1_3_2_1_36_1","volume-title":"Keckler","author":"Rhu Minsoo","year":"2016","unstructured":"Minsoo Rhu, Natalia Gimelshein, Jason Clemons, Arslan Zulfiqar, and Stephen W. Keckler. 2016. VDNN: Virtualized Deep Neural Networks for Scalable, Memory-Efficient Neural Network Design. In The 49th Annual IEEE\/ACM International Symposium on Microarchitecture (MICRO-49). IEEE Press, Article 18, 13 pages."},{"key":"e_1_3_2_1_37_1","doi-asserted-by":"publisher","DOI":"10.1145\/2939672.2945397"},{"key":"e_1_3_2_1_38_1","doi-asserted-by":"publisher","DOI":"10.1109\/APSEC.2017.41"},{"key":"e_1_3_2_1_39_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2015.7298594"},{"key":"e_1_3_2_1_40_1","doi-asserted-by":"publisher","DOI":"10.1109\/ISSRE.2012.22"},{"key":"e_1_3_2_1_41_1","doi-asserted-by":"publisher","DOI":"10.1145\/3180155.3180220"},{"key":"e_1_3_2_1_42_1","doi-asserted-by":"publisher","DOI":"10.1162\/089120103321337458"},{"key":"e_1_3_2_1_43_1","doi-asserted-by":"publisher","DOI":"10.5555\/3295222.3295349"},{"key":"e_1_3_2_1_44_1","doi-asserted-by":"publisher","DOI":"10.1145\/3178487.3178491"},{"key":"e_1_3_2_1_45_1","doi-asserted-by":"publisher","DOI":"10.1109\/TVCG.2017.2744878"},{"key":"e_1_3_2_1_46_1","doi-asserted-by":"publisher","DOI":"10.1145\/2591062.2591177"},{"key":"e_1_3_2_1_47_1","volume-title":"Proceedings of the 12th USENIX Conference on Operating Systems Design and Implementation (OSDI '18)","author":"Xiao Wencong","unstructured":"Wencong Xiao, Romil Bhardwaj, Ramachandran Ramjee, Muthian Sivathanu, Nipun Kwatra, Zhenhua Han, Pratyush Patel, Xuan Peng, Hanyu Zhao, Quanlu Zhang, and et al. 2018. Gandiva: Introspective Cluster Scheduling for Deep Learning. In Proceedings of the 12th USENIX Conference on Operating Systems Design and Implementation (OSDI '18). USENIX Association, USA, 595--610."},{"key":"e_1_3_2_1_48_1","doi-asserted-by":"crossref","first-page":"2","DOI":"10.1109\/TSE.2007.70771","article-title":"On the Distribution of Software Faults","volume":"34","author":"Zhang Hongyu","year":"2008","unstructured":"Hongyu Zhang. 2008. On the Distribution of Software Faults. IEEE Transactions on Software Engineering 34, 2 (March 2008), 301--302.","journal-title":"IEEE Transactions on Software Engineering"},{"key":"e_1_3_2_1_49_1","doi-asserted-by":"publisher","DOI":"10.1145\/3213846.3213866"},{"key":"e_1_3_2_1_50_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE.2015.130"}],"event":{"name":"ICSE '20: 42nd International Conference on Software Engineering","location":"Seoul South Korea","acronym":"ICSE '20","sponsor":["SIGSOFT ACM Special Interest Group on Software Engineering","KIISE Korean Institute of Information Scientists and Engineers","IEEE CS"]},"container-title":["Proceedings of the ACM\/IEEE 42nd International Conference on Software Engineering"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3377811.3380362","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3377811.3380362","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T22:41:39Z","timestamp":1750200099000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3377811.3380362"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2020,6,27]]},"references-count":50,"alternative-id":["10.1145\/3377811.3380362","10.1145\/3377811"],"URL":"https:\/\/doi.org\/10.1145\/3377811.3380362","relation":{},"subject":[],"published":{"date-parts":[[2020,6,27]]},"assertion":[{"value":"2020-10-01","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}