{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,17]],"date-time":"2026-07-17T19:57:25Z","timestamp":1784318245984,"version":"3.55.0"},"publisher-location":"New York, NY, USA","reference-count":122,"publisher":"ACM","license":[{"start":{"date-parts":[[2020,12,21]],"date-time":"2020-12-21T00:00:00Z","timestamp":1608508800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"name":"National science foundation"},{"name":"Facebook Research Award"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2020,12,21]]},"DOI":"10.1145\/3324884.3416545","type":"proceedings-article","created":{"date-parts":[[2021,1,27]],"date-time":"2021-01-27T23:38:56Z","timestamp":1611790736000},"page":"771-783","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":82,"title":["Problems and opportunities in training deep learning software systems"],"prefix":"10.1145","author":[{"given":"Hung Viet","family":"Pham","sequence":"first","affiliation":[{"name":"University of Waterloo, Waterloo, ON, Canada"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Shangshu","family":"Qian","sequence":"additional","affiliation":[{"name":"Purdue University"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Jiannan","family":"Wang","sequence":"additional","affiliation":[{"name":"Purdue University"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Thibaud","family":"Lutellier","sequence":"additional","affiliation":[{"name":"University of Waterloo, Waterloo, ON, Canada"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Jonathan","family":"Rosenthal","sequence":"additional","affiliation":[{"name":"Purdue University"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Lin","family":"Tan","sequence":"additional","affiliation":[{"name":"Purdue University"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Yaoliang","family":"Yu","sequence":"additional","affiliation":[{"name":"University of Waterloo, Waterloo, ON, Canada"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Nachiappan","family":"Nagappan","sequence":"additional","affiliation":[{"name":"Microsoft Research"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2021,1,27]]},"reference":[{"key":"e_1_3_2_1_1_1","volume-title":"ASE '19: Proceedings of the 34th ACM\/IEEE International Conference on Automated Software Engineering.","unstructured":"2019. ASE '19: Proceedings of the 34th ACM\/IEEE International Conference on Automated Software Engineering."},{"key":"e_1_3_2_1_2_1","volume-title":"ASPLOS '19: Proceedings of the Twenty-Fourth International Conference on Architectural Support for Programming Languages and Operating Systems.","unstructured":"2019. ASPLOS '19: Proceedings of the Twenty-Fourth International Conference on Architectural Support for Programming Languages and Operating Systems."},{"key":"e_1_3_2_1_3_1","volume-title":"CVPR'19: Proceedings of The IEEE Conference on Computer Vision and Pattern Recognition.","unstructured":"2019. CVPR'19: Proceedings of The IEEE Conference on Computer Vision and Pattern Recognition."},{"key":"e_1_3_2_1_4_1","volume-title":"ESEC\/FSE '19: Proceedings of the 27th ACM Joint Meeting on European Software Engineering Conference and Symposium on the Foundations of Software Engineering.","unstructured":"2019. ESEC\/FSE '19: Proceedings of the 27th ACM Joint Meeting on European Software Engineering Conference and Symposium on the Foundations of Software Engineering."},{"key":"e_1_3_2_1_5_1","volume-title":"ICCV'19: Proceedings of The IEEE International Conference on Computer Vision. IEEE Press.","unstructured":"2019. ICCV'19: Proceedings of The IEEE International Conference on Computer Vision. IEEE Press."},{"key":"e_1_3_2_1_6_1","volume-title":"ICML'19: Proceedings of The International Conference on Machine Learning.","unstructured":"2019. ICML'19: Proceedings of The International Conference on Machine Learning."},{"key":"e_1_3_2_1_7_1","volume-title":"ICSE '19: Proceedings of the 41st International Conference on Software Engineering","unstructured":"2019. ICSE '19: Proceedings of the 41st International Conference on Software Engineering. IEEE Press."},{"key":"e_1_3_2_1_8_1","volume-title":"MLSys'19: Proceedings of Machine Learning and Systems.","unstructured":"2019. MLSys'19: Proceedings of Machine Learning and Systems."},{"key":"e_1_3_2_1_9_1","volume-title":"NIPS'19: Proceedings of the 33rd Conference on Neural Information Processing Systems.","unstructured":"2019. NIPS'19: Proceedings of the 33rd Conference on Neural Information Processing Systems."},{"key":"e_1_3_2_1_10_1","volume-title":"SOSP '19: Proceedings of the 27th ACM Symposium on Operating Systems Principles.","unstructured":"2019. SOSP '19: Proceedings of the 27th ACM Symposium on Operating Systems Principles."},{"key":"e_1_3_2_1_11_1","volume-title":"ICLR'20: Proceedings of The International Conference on Learning Representations.","unstructured":"2020. ICLR'20: Proceedings of The International Conference on Learning Representations."},{"key":"e_1_3_2_1_12_1","unstructured":"2020. Reproducibility Challenge @ NeurIPS 2019. https:\/\/reproducibilitychallenge.github.io\/neurips2019\/"},{"key":"e_1_3_2_1_13_1","unstructured":"Mart\u00edn Abadi Paul Barham Jianmin Chen Zhifeng Chen Andy Davis Jeffrey Dean Matthieu Devin Sanjay Ghemawat Geoffrey Irving Michael Isard et al. 2016. TensorFlow: a System for Large-Scale Machine Learning. In OSDI."},{"key":"e_1_3_2_1_14_1","volume-title":"Neural networks and deep learning","author":"Aggarwal Charu C","unstructured":"Charu C Aggarwal. 2018. Neural networks and deep learning. In Springer."},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"crossref","unstructured":"Andrea Arcuri and Lionel Briand. 2011. A practical guide for using statistical tests to assess randomized algorithms in software engineering. In ICSE.","DOI":"10.1145\/1985793.1985795"},{"key":"e_1_3_2_1_16_1","volume-title":"Theano: Deep learning on GPUs with Python. In NIPS.","author":"Bergstra James","year":"2011","unstructured":"James Bergstra, Fr\u00e9d\u00e9ric Bastien, Olivier Breuleux, Pascal Lamblin, Razvan Pascanu, Olivier Delalleau, Guillaume Desjardins, David Warde-Farley, Ian Goodfellow, Arnaud Bergeron, et al. 2011. Theano: Deep learning on GPUs with Python. In NIPS."},{"key":"e_1_3_2_1_17_1","unstructured":"James Bergstra and Yoshua Bengio. 2012. Random search for hyper-parameter optimization. In JMLR."},{"key":"e_1_3_2_1_18_1","unstructured":"Nghi D. Q. Bui Yijun Yu and Lingxiao Jiang. 2019. AutoFocus: Interpreting Attention-Based Neural Networks by Code Perturbation. In ASE."},{"key":"e_1_3_2_1_19_1","volume-title":"Deepdriving: Learning Affordance for Direct Perception in Autonomous Driving. In ICCV.","author":"Chen Chenyi","year":"2015","unstructured":"Chenyi Chen, Ari Seff, Alain Kornhauser, and Jianxiong Xiao. 2015. Deepdriving: Learning Affordance for Direct Perception in Autonomous Driving. In ICCV."},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"crossref","unstructured":"Jieshan Chen Chunyang Chen Zhenchang Xing Xiwei Xu Liming Zhu Guoqiang Li and Jinshui Wang. 2020. Unblind Your Apps: Predicting Natural-Language Labels for Mobile GUI Components by Deep Learning. In ICSE.","DOI":"10.1145\/3377811.3380327"},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"crossref","unstructured":"Jinyin Chen Keke Hu Yue Yu Zhuangzhi Chen Qi Xuan Yi Liu and Vladimir Filkov. 2020. Software Visualization and Deep Transfer Learning for Effective Software Defect Prediction. In ICSE.","DOI":"10.1145\/3377811.3380389"},{"key":"e_1_3_2_1_22_1","unstructured":"Liang-Chieh Chen Maxwell Collins Yukun Zhu George Papandreou Barret Zoph Florian Schroff Hartwig Adam and Jon Shlens. 2018. Searching for efficient multi-scale architectures for dense image prediction. In NIPS."},{"key":"e_1_3_2_1_23_1","unstructured":"Sharan Chetlur Cliff Woolley Philippe Vandermersch Jonathan Cohen John Tran Bryan Catanzaro and Evan Shelhamer. 2014. cuDNN: Efficient Primitives for Deep Learning."},{"key":"e_1_3_2_1_24_1","volume-title":"Xception: Deep learning with depthwise separable convolutions. In CVPR.","author":"Chollet Fran\u00e7ois","year":"2017","unstructured":"Fran\u00e7ois Chollet. 2017. Xception: Deep learning with depthwise separable convolutions. In CVPR."},{"key":"e_1_3_2_1_25_1","unstructured":"Fran\u00e7ois Chollet et al. 2015. Keras. https:\/\/keras.io."},{"key":"e_1_3_2_1_26_1","volume-title":"G\u00e9rard Ben Arous, and Yann LeCun","author":"Choromanska Anna","year":"2015","unstructured":"Anna Choromanska, Mikael Henaff, Michael Mathieu, G\u00e9rard Ben Arous, and Yann LeCun. 2015. The loss surfaces of multilayer networks. In AISTATS."},{"key":"e_1_3_2_1_27_1","volume-title":"Open Problem: The landscape of the loss surfaces of multilayer networks. In COLR.","author":"Choromanska Anna","year":"2015","unstructured":"Anna Choromanska, Yann LeCun, and G\u00e9rard Ben Arous. 2015. Open Problem: The landscape of the loss surfaces of multilayer networks. In COLR."},{"key":"e_1_3_2_1_28_1","unstructured":"J. Cohen. 1988. Statistical Power Analysis for the Behavioral Sciences."},{"key":"e_1_3_2_1_29_1","unstructured":"C\u00e9dric Colas Olivier Sigaud and Pierre-Yves Oudeyer. 2018. How Many Random Seeds? Statistical Power Analysis in Deep Reinforcement Learning Experiments. In CoRR."},{"key":"e_1_3_2_1_30_1","volume-title":"Dawnbench: An end-to-end deep learning benchmark and competition. In Training.","author":"Coleman Cody","year":"2017","unstructured":"Cody Coleman, Deepak Narayanan, Daniel Kang, Tian Zhao, Jian Zhang, Luigi Nardi, Peter Bailis, Kunle Olukotun, Chris R\u00e9, and Matei Zaharia. 2017. Dawnbench: An end-to-end deep learning benchmark and competition. In Training."},{"key":"e_1_3_2_1_31_1","volume-title":"SPEC CPU2006 Benchmarks. http:\/\/www.spec.org\/cpu2006\/.","author":"Standard Performance Evaluation Corporation","year":"2006","unstructured":"Standard Performance Evaluation Corporation. 2006. SPEC CPU2006 Benchmarks. http:\/\/www.spec.org\/cpu2006\/."},{"key":"e_1_3_2_1_32_1","volume-title":"Bryant","author":"Cui Heming","year":"2013","unstructured":"Heming Cui, Jiri Simsa, Yi-Hong Lin, Hao Li, Ben Blum, Xinan Xu, Junfeng Yang, Garth A. Gibson, and Randal E. Bryant. 2013. Parrot: a Practical Runtime for Deterministic, Stable, and Reliable Threads. In SOSP."},{"key":"e_1_3_2_1_33_1","unstructured":"Yann N Dauphin Razvan Pascanu Caglar Gulcehre Kyunghyun Cho Surya Ganguli and Yoshua Bengio. 2014. Identifying and attacking the saddle point problem in high-dimensional non-convex optimization. In NIPS."},{"key":"e_1_3_2_1_34_1","volume-title":"Smith","author":"Dodge Jesse","year":"2019","unstructured":"Jesse Dodge, Suchin Gururangan, Dallas Card, Roy Schwartz, and Noah A. Smith. 2019. Show Your Work: Improved Reporting of Experimental Results. In EMNLP-IJCNLP."},{"key":"e_1_3_2_1_35_1","unstructured":"Felix Draxler Kambis Veschgini Manfred Salmhofer and Fred Hamprecht. 2018. Essentially No Barriers in Neural Network Energy Landscape. In ICML."},{"key":"e_1_3_2_1_36_1","unstructured":"Simon S Du Jason D Lee Yuandong Tian Barnabas Poczos and Aarti Singh. 2017. Gradient descent learns one-hidden-layer cnn: Don't be afraid of spurious local minima. In arXiv preprint arXiv:1712.00779."},{"key":"e_1_3_2_1_37_1","unstructured":"Brian Everitt. 2002. The Cambridge dictionary of statistics."},{"key":"e_1_3_2_1_38_1","doi-asserted-by":"crossref","unstructured":"Hao-Shu Fang Guansong Lu Xiaolin Fang Jianwen Xie Yu-Wing Tai and Cewu Lu. 2018. Weakly and Semi Supervised Human Body Part Parsing via Pose-Guided Knowledge Transfer. In CVPR.","DOI":"10.1109\/CVPR.2018.00015"},{"key":"e_1_3_2_1_39_1","doi-asserted-by":"crossref","unstructured":"Xiang Gao Ripon Saha Mukul Prasad and Abhik Roychoudhury. 2020. Fuzz Testing based Data Augmentation to Improve Robustness of Deep Neural Networks. In ICSE.","DOI":"10.1145\/3377811.3380415"},{"key":"e_1_3_2_1_40_1","unstructured":"Rong Ge Furong Huang Chi Jin and Yang Yuan. 2015. Escaping from saddle points---online stochastic gradient for tensor decomposition. In COLT."},{"key":"e_1_3_2_1_41_1","doi-asserted-by":"crossref","unstructured":"Simos Gerasimou Hasan Ferit-Eniser Alper Sen and Alper \u00c7akan. 2020. Importance-Driven Deep Learning System Testing. In ICSE.","DOI":"10.1145\/3377811.3380391"},{"key":"e_1_3_2_1_42_1","unstructured":"Xavier Glorot and Yoshua Bengio. 2010. Understanding the difficulty of training deep feedforward neural networks. In AISTATS."},{"key":"e_1_3_2_1_43_1","doi-asserted-by":"crossref","unstructured":"David Goldberg. 1991. What Every Computer Scientist Should Know about Floating-Point Arithmetic. ACM Comput. Surv. (1991).","DOI":"10.1145\/103162.103163"},{"key":"e_1_3_2_1_44_1","unstructured":"Jes\u00fas M Gonz\u00e1lez-Barahona and Gregorio Robles. 2012. On the reproducibility of empirical software engineering studies based on data retrieved from development repositories. In ESE."},{"key":"e_1_3_2_1_45_1","volume-title":"Deep learning","author":"Goodfellow Ian","unstructured":"Ian Goodfellow, Yoshua Bengio, and Aaron Courville. 2016. Deep learning. MIT press."},{"key":"e_1_3_2_1_46_1","doi-asserted-by":"crossref","unstructured":"Divya Gopinath Hayes Converse Corina S. P\u0103s\u0103reanu and Ankur Taly. 2019. Property Inference for Deep Neural Networks. In ASE.","DOI":"10.1109\/ASE.2019.00079"},{"key":"e_1_3_2_1_47_1","volume-title":"Wootz: A Compiler-Based Framework for Fast CNN Pruning via Composability. In PLDI.","author":"Guan Hui","year":"2019","unstructured":"Hui Guan, Xipeng Shen, and Seung-Hwan Lim. 2019. Wootz: A Compiler-Based Framework for Fast CNN Pruning via Composability. In PLDI."},{"key":"e_1_3_2_1_48_1","unstructured":"Qianyu Guo Sen Chen Xiaofei Xie Lei Ma Qiang Hu Hongtao Liu Yang Liu Jianjun Zhao and Xiaohong Li. 2019. An empirical study towards characterizing deep learning development and deployment across different frameworks and platforms. In ASE."},{"key":"e_1_3_2_1_49_1","unstructured":"Jeff Haochen and Suvrit Sra. 2019. Random Shuffling Beats SGD after Finite Epochs. In ICML."},{"key":"e_1_3_2_1_50_1","unstructured":"Kaiming He Xiangyu Zhang Shaoqing Ren and Jian Sun. 2015. Delving deep into rectifiers: Surpassing human-level performance on imagenet classification. In ICCV."},{"key":"e_1_3_2_1_51_1","unstructured":"Kaiming He Xiangyu Zhang Shaoqing Ren and Jian Sun. 2016. Deep residual learning for image recognition. In CVPR."},{"key":"e_1_3_2_1_52_1","doi-asserted-by":"crossref","unstructured":"Vincent J. Hellendoorn Christian Bird Earl T. Barr and Miltiadis Allamanis. 2018. Deep Learning Type Inference. In ESEC\/FSE.","DOI":"10.1145\/3236024.3236051"},{"key":"e_1_3_2_1_53_1","doi-asserted-by":"crossref","unstructured":"Peter Henderson Riashat Islam Philip Bachman Joelle Pineau Doina Precup and David Meger. 2018. Deep Reinforcement Learning that Matters. In AAAI.","DOI":"10.1609\/aaai.v32i1.11694"},{"key":"e_1_3_2_1_54_1","doi-asserted-by":"crossref","unstructured":"Ilija Ilievski Taimoor Akhtar Jiashi Feng and Christine Annette Shoemaker. 2017. Efficient hyperparameter optimization for deep learning algorithms using deterministic rbf surrogates. In AAAI.","DOI":"10.1609\/aaai.v31i1.10647"},{"key":"e_1_3_2_1_55_1","volume-title":"Mike O'Connor, Joseph Devietti, and Tor M Aamodt.","author":"Jooybar Hadi","year":"2013","unstructured":"Hadi Jooybar, Wilson WL Fung, Mike O'Connor, Joseph Devietti, and Tor M Aamodt. 2013. GPUDet: a deterministic GPU architecture. In ASPLOS."},{"key":"e_1_3_2_1_56_1","unstructured":"Jinhan Kim Robert Feldt and Shin Yoo. 2019. Guiding Deep Learning System Testing Using Surprise Adequacy. In ICSE."},{"key":"e_1_3_2_1_57_1","doi-asserted-by":"crossref","unstructured":"YK Kim and JB Ra. 1991. Weight value initialization for improving training speed in the backpropagation network. In IJCNN.","DOI":"10.1109\/IJCNN.1991.170747"},{"key":"e_1_3_2_1_58_1","unstructured":"Bobby Kleinberg Yuanzhi Li and Yang Yuan. 2018. An Alternative View: When Does SGD Escape Local Minima?. In ICML."},{"key":"e_1_3_2_1_59_1","doi-asserted-by":"crossref","unstructured":"Yuriy Kochura Sergii Stirenko Oleg Alienin Michail Novotarskiy and Yuri Gordienko. 2018. Performance Analysis of Open Source Machine Learning Frameworks for Various Parameters in Single-Threaded and Multi-threaded Modes. In CSIT.","DOI":"10.1007\/978-3-319-70581-1_17"},{"key":"e_1_3_2_1_60_1","unstructured":"Vassili Kovalev Alexander Kalinovsky and Sergey Kovalev. 2016. Deep learning with theano torch caffe tensorflow and deeplearning4j: Which one is the best in speed and accuracy?"},{"key":"e_1_3_2_1_61_1","unstructured":"Alex Krizhevsky. 2009. Learning multiple layers of features from tiny images."},{"key":"e_1_3_2_1_62_1","unstructured":"Alex Krizhevsky Ilya Sutskever and Geoffrey E Hinton. 2012. Imagenet classification with deep convolutional neural networks. In NIPS."},{"key":"e_1_3_2_1_63_1","volume-title":"Graham Neubig, and Bogdan Vasilescu.","author":"Lacomis Jeremy","year":"2019","unstructured":"Jeremy Lacomis, Pengcheng Yin, Edward J. Schwartz, Miltiadis Allamanis, Claire Le Goues, Graham Neubig, and Bogdan Vasilescu. 2019. DIRE: A Neural Approach to Decompiled Identifier Naming. In ASE."},{"key":"e_1_3_2_1_64_1","volume-title":"Mozer","author":"Lamb Alex","year":"2019","unstructured":"Alex Lamb, Jonathan Binas, Anirudh Goyal, Sandeep Subramanian, Ioannis Mitliagkas, Denis Kazakov, Yoshua Bengio, and Michael C. Mozer. 2019. State-Reification Networks: Improving Generalization by Modeling the Distribution of Hidden Representations. In ICML."},{"key":"e_1_3_2_1_65_1","doi-asserted-by":"publisher","DOI":"10.1109\/5.726791"},{"key":"e_1_3_2_1_66_1","doi-asserted-by":"crossref","unstructured":"Yann A LeCun L\u00e9on Bottou Genevieve B Orr and Klaus-Robert M\u00fcller. 2012. Efficient backprop. In Neural networks: Tricks of the trade.","DOI":"10.1007\/978-3-642-35289-8_3"},{"key":"e_1_3_2_1_67_1","unstructured":"Jason D Lee Max Simchowitz Michael I Jordan and Benjamin Recht. 2016. Gradient descent converges to minimizers. In arXiv preprint arXiv:1602.04915."},{"key":"e_1_3_2_1_68_1","unstructured":"Hao Li Zheng Xu Gavin Taylor Christoph Studer and Tom Goldstein. 2018. Visualizing the Loss Landscape of Neural Nets. In NIPS."},{"key":"e_1_3_2_1_69_1","volume-title":"Nguyen","author":"Li Yi","year":"2020","unstructured":"Yi Li, Wang Shaohua, and Tien N. Nguyen. 2020. DLFix: Context-based Code Transformation Learning for Automated Program Repair. In ICSE."},{"key":"e_1_3_2_1_70_1","doi-asserted-by":"crossref","unstructured":"Yi Li Shaohua Wang Tien N Nguyen and Son Van Nguyen. 2019. Improving bug detection via context-based code representation learning and attention-based neural networks. In PACMPL.","DOI":"10.1145\/3360588"},{"key":"e_1_3_2_1_71_1","unstructured":"Zenan Li Xiaoxing Ma Chang Xu Chun Cao Jingwei Xu and Jian L\u00fc. 2019. Boosting Operational DNN Testing Efficiency through Conditioning. In ESEC\/FSE."},{"key":"e_1_3_2_1_72_1","doi-asserted-by":"crossref","unstructured":"Ling Liu Yanzhao Wu Wenqi Wei Wenqi Cao Semih Sahin and Qi Zhang. 2018. Benchmarking deep learning frameworks: Design considerations metrics and beyond. In ICDCS.","DOI":"10.1109\/ICDCS.2018.00125"},{"key":"e_1_3_2_1_73_1","doi-asserted-by":"crossref","unstructured":"Lei Ma Felix Juefei-Xu Fuyuan Zhang Jiyuan Sun Minhui Xue Bo Li Chunyang Chen Ting Su Li Li Yang Liu Jianjun Zhao and Yadong Wang. 2018. DeepGauge: Multi-Granularity Testing Criteria for Deep Learning Systems. In ASE.","DOI":"10.1145\/3238147.3238202"},{"key":"e_1_3_2_1_74_1","volume-title":"MODE: Automated Neural Network Model Debugging via State Differential Analysis and Input Selection. In ESEC\/FSE.","author":"Ma Shiqing","year":"2018","unstructured":"Shiqing Ma, Yingqi Liu, Wen-Chuan Lee, Xiangyu Zhang, and Ananth Grama. 2018. MODE: Automated Neural Network Model Debugging via State Differential Analysis and Input Selection. In ESEC\/FSE."},{"key":"e_1_3_2_1_75_1","volume-title":"Peter CR Lane, and Jean Petri\u0107","author":"Mahmood Zaheed","year":"2018","unstructured":"Zaheed Mahmood, David Bowes, Tracy Hall, Peter CR Lane, and Jean Petri\u0107. 2018. Reproducibility and replicability of software defect prediction studies. ISE (2018)."},{"key":"e_1_3_2_1_76_1","doi-asserted-by":"crossref","unstructured":"Andrii Maksai and Pascal Fua. 2019. Eliminating exposure bias and metric mismatch in multiple object tracking. In CVPR.","DOI":"10.1109\/CVPR.2019.00477"},{"key":"e_1_3_2_1_77_1","unstructured":"Luke Metz Niru Maheswaranathan Jeremy Nixon C Daniel Freeman and Jascha Sohl-Dickstein. 2019. Understanding and correcting pathologies in the training of learned optimizers. In ICML."},{"key":"e_1_3_2_1_78_1","volume-title":"van der Walt","author":"Mhaskar Hrushikesh N.","year":"2017","unstructured":"Hrushikesh N. Mhaskar, Sergei V. Pereverzyev, and Maria D. van der Walt. 2017. A Deep Learning Approach to Diabetic Blood Glucose Prediction. In Front. Appl. Math. Stat."},{"key":"e_1_3_2_1_79_1","doi-asserted-by":"crossref","unstructured":"Todd Mytkowicz Amer Diwan Matthias Hauswirth and Peter Sweeney. 2009. Producing wrong data without doing anything obviously wrong!. In ACM SIGARCH Computer Architecture News.","DOI":"10.1145\/1508244.1508275"},{"key":"e_1_3_2_1_80_1","volume-title":"Deterministic Implementations for Reproducibility in Deep Reinforcement Learning. In AAAI Workshop on Reproducible AI.","author":"Nagarajan Prabhat","year":"2019","unstructured":"Prabhat Nagarajan, Garrett Warnell, and Peter Stone. 2019. Deterministic Implementations for Reproducibility in Deep Reinforcement Learning. In AAAI Workshop on Reproducible AI."},{"key":"e_1_3_2_1_81_1","volume-title":"Reproducibility in Machine Learning Workshop, ICML.","author":"Olorisade Babatunde K","year":"2017","unstructured":"Babatunde K Olorisade, Pearl Brereton, and Peter Andras. 2017. Reproducibility in machine Learning-Based studies: An example of text mining. In Reproducibility in Machine Learning Workshop, ICML."},{"key":"e_1_3_2_1_82_1","volume-title":"PyTorch: An Imperative Style","author":"Paszke Adam","unstructured":"Adam Paszke, Sam Gross, Francisco Massa, Adam Lerer, James Bradbury, Gregory Chanan, Trevor Killeen, Zeming Lin, Natalia Gimelshein, Luca Antiga, Alban Desmaison, Andreas Kopf, Edward Yang, Zachary DeVito, Martin Raison, Alykhan Tejani, Sasank Chilamkurthy, Benoit Steiner, Lu Fang, Junjie Bai, and Soumith Chintala. 2019. PyTorch: An Imperative Style, High-Performance Deep Learning Library. In NIPS."},{"key":"e_1_3_2_1_83_1","doi-asserted-by":"crossref","unstructured":"Brandon Paulsen Jingbo Wang and Chao Wang. 2020. ReluDiff: Differential Verification of Deep Neural Networks. In ICSE.","DOI":"10.1145\/3377811.3380337"},{"key":"e_1_3_2_1_84_1","doi-asserted-by":"crossref","unstructured":"Kexin Pei Yinzhi Cao Junfeng Yang and Suman Jana. 2019. DeepXplore: Automated Whitebox Testing of Deep Learning Systems. In Commun. ACM.","DOI":"10.1145\/3361566"},{"key":"e_1_3_2_1_85_1","volume-title":"CRADLE: Cross-Backend Validation to Detect and Localize Bugs in Deep Learning Libraries. In ICSE.","author":"Pham Hung Viet","year":"2019","unstructured":"Hung Viet Pham, Thibaud Lutellier, Weizhen Qi, and Lin Tan. 2019. CRADLE: Cross-Backend Validation to Detect and Localize Bugs in Deep Learning Libraries. In ICSE."},{"key":"e_1_3_2_1_86_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.jbi.2018.04.007"},{"key":"e_1_3_2_1_87_1","volume-title":"Bug Synthesis: Challenging Bug-Finding Tools with Deep Faults. In ESEC\/FSE.","author":"Roy Subhajit","year":"2018","unstructured":"Subhajit Roy, Awanish Pandey, Brendan Dolan-Gavitt, and Yu Hu. 2018. Bug Synthesis: Challenging Bug-Finding Tools with Deep Faults. In ESEC\/FSE."},{"key":"e_1_3_2_1_88_1","unstructured":"Tim Salimans and Durk P Kingma. 2016. Weight normalization: A simple reparameterization to accelerate training of deep neural networks. In NIPS."},{"key":"e_1_3_2_1_89_1","volume-title":"Grissom","author":"Sawilowsky Shlomo","year":"2011","unstructured":"Shlomo Sawilowsky, Jack Sawilowsky, and Robert J. Grissom. 2011. Effect Size."},{"key":"e_1_3_2_1_90_1","unstructured":"Andrew M. Saxe James L. Mcclelland and Surya Ganguli. 2014. Exact solutions to the nonlinear dynamics of learning in deep linear neural network. In ICLR."},{"key":"e_1_3_2_1_91_1","doi-asserted-by":"publisher","DOI":"10.1145\/2939672.2945397"},{"key":"e_1_3_2_1_92_1","doi-asserted-by":"crossref","unstructured":"Shayan Shams Richard Platania Kisung Lee and Seung-Jong Park. 2017. Evaluation of deep learning frameworks over different HPC architectures. In ICDCS.","DOI":"10.1109\/ICDCS.2017.259"},{"key":"e_1_3_2_1_93_1","doi-asserted-by":"crossref","unstructured":"Ali Shatnawi Ghadeer Al-Bdour Raffi Al-Qurran and Mahmoud Al-Ayyoub. 2018. A comparative study of open source deep learning frameworks. In ICICS.","DOI":"10.1109\/IACS.2018.8355444"},{"key":"e_1_3_2_1_94_1","volume-title":"Dsod: Learning deeply supervised object detectors from scratch. In ICCV.","author":"Shen Zhiqiang","year":"2017","unstructured":"Zhiqiang Shen, Zhuang Liu, Jianguo Li, Yu-Gang Jiang, Yurong Chen, and Xiangyang Xue. 2017. Dsod: Learning deeply supervised object detectors from scratch. In ICCV."},{"key":"e_1_3_2_1_95_1","unstructured":"Shaohuai Shi Qiang Wang Pengfei Xu and Xiaowen Chu. 2016. Benchmarking state-of-the-art deep learning software tools. In CCBD."},{"key":"e_1_3_2_1_96_1","volume-title":"Khoshgoftaar","author":"Shorten Connor","year":"2019","unstructured":"Connor Shorten and Taghi M. Khoshgoftaar. 2019. A survey on Image Data Augmentation for Deep Learning. In Journal of Big Data."},{"key":"e_1_3_2_1_97_1","unstructured":"Karen Simonyan and Andrew Zisserman. 2014. Very deep convolutional networks for large-scale image recognition. In arXiv preprint arXiv:1409.1556."},{"key":"e_1_3_2_1_98_1","doi-asserted-by":"crossref","unstructured":"Xinhang Song Luis Herranz and Shuqiang Jiang. 2017. Depth cnns for rgb-d scene recognition: Learning from scratch better than transferring from rgb-cnns. In AAAI.","DOI":"10.1609\/aaai.v31i1.11226"},{"key":"e_1_3_2_1_99_1","unstructured":"Nitish Srivastava Geoffrey Hinton Alex Krizhevsky Ilya Sutskever and Ruslan Salakhutdinov. 2014. Dropout: a simple way to prevent neural networks from overfitting. In JMLR."},{"key":"e_1_3_2_1_100_1","unstructured":"Youcheng Sun Min Wu Wenjie Ruan Xiaowei Huang Marta Kwiatkowska and Daniel Kroening. 2018. Concolic Testing for Deep Neural Networks. In ASE."},{"key":"e_1_3_2_1_101_1","unstructured":"Ilya Sutskever James Martens George Dahl and Geoffrey Hinton. 2013. On the importance of initialization and momentum in deep learning. In ICML."},{"key":"e_1_3_2_1_102_1","doi-asserted-by":"crossref","unstructured":"Christian Szegedy Vincent Vanhoucke Sergey Ioffe Jon Shlens and Zbigniew Wojna. 2016. Rethinking the inception architecture for computer vision. In CVPR.","DOI":"10.1109\/CVPR.2016.308"},{"key":"e_1_3_2_1_103_1","doi-asserted-by":"publisher","DOI":"10.1109\/TMI.2016.2535302"},{"key":"e_1_3_2_1_104_1","unstructured":"Kok Keong Teo Lipo Wang and Zhiping Lin. 2001. Wavelet packet multi-layer perceptron for chaotic time series prediction: effects of weight initialization. In ICCS."},{"key":"e_1_3_2_1_105_1","doi-asserted-by":"crossref","unstructured":"Yuchi Tian Kexin Pei Suman Jana and Baishakhi Ray. 2018. DeepTest: Automated Testing of Deep-Neural-Network-Driven Autonomous Cars. In ICSE.","DOI":"10.1145\/3180155.3180220"},{"key":"e_1_3_2_1_106_1","doi-asserted-by":"crossref","unstructured":"Yuchi Tian Ziyuan Zhong Vicente Ordonez Gail Kaiser and Baishakhi Ray. 2020. Testing DNN Image Classifier for Confusion & Bias Errors. In ICSE.","DOI":"10.1145\/3377811.3380400"},{"key":"e_1_3_2_1_107_1","doi-asserted-by":"crossref","unstructured":"Fabian Trautsch Steffen Herbold Philip Makedonski and Jens Grabowski. 2018. Addressing problems with replicability and validity of repository mining studies through a smart data platform. In ESE.","DOI":"10.1007\/s10664-017-9537-x"},{"key":"e_1_3_2_1_108_1","doi-asserted-by":"crossref","unstructured":"Nam Vo Lu Jiang Chen Sun Kevin Murphy Li-Jia Li Li Fei-Fei and James Hays. 2019. Composing text and image for image retrieval-an empirical odyssey. In CVPR.","DOI":"10.1109\/CVPR.2019.00660"},{"key":"e_1_3_2_1_109_1","doi-asserted-by":"publisher","DOI":"10.1145\/3377811.3380379"},{"key":"e_1_3_2_1_110_1","doi-asserted-by":"crossref","unstructured":"Jingyi Wang Guoliang Dong Jun Sun Xinyu Wang and Peixin Zhang. 2019. Adversarial Sample Detection for Deep Neural Network through Model Mutation Testing. In ICSE.","DOI":"10.1109\/ICSE.2019.00126"},{"key":"e_1_3_2_1_111_1","doi-asserted-by":"crossref","unstructured":"Steven R Young Derek C Rose Thomas P Karnowski Seung-Hwan Lim and Robert M Patton. 2015. Optimizing deep learning hyper-parameters through an evolutionary algorithm. In MLHPC.","DOI":"10.1145\/2834892.2834896"},{"key":"e_1_3_2_1_112_1","doi-asserted-by":"crossref","unstructured":"Sergey Zagoruyko and Nikos Komodakis. 2016. Wide Residual Networks. In BMVC.","DOI":"10.5244\/C.30.87"},{"key":"e_1_3_2_1_113_1","volume-title":"Apricot: A Weight-Adaptation Approach to Fixing Deep Learning Models. In ASE.","author":"Zhang Hao","year":"2019","unstructured":"Hao Zhang and W. K. Chan. 2019. Apricot: A Weight-Adaptation Approach to Fixing Deep Learning Models. In ASE."},{"key":"e_1_3_2_1_114_1","doi-asserted-by":"crossref","unstructured":"Jian Zhang Xu Wang Hongyu Zhang Hailong Sun and Xudong Liu. 2020. Retrieval-based Neural Source Code Summarization. In ICSE.","DOI":"10.1145\/3377811.3380383"},{"key":"e_1_3_2_1_115_1","doi-asserted-by":"crossref","unstructured":"Mengshi Zhang Yuqun Zhang Lingming Zhang Cong Liu and Sarfraz Khurshid. 2018. DeepRoad: GAN-Based Metamorphic Testing and Input Validation Framework for Autonomous Driving Systems. In ASE.","DOI":"10.1145\/3238147.3238187"},{"key":"e_1_3_2_1_116_1","doi-asserted-by":"crossref","unstructured":"Xiyue Zhang Xiaofei Xie Lei Ma Xiaoning Du Qiang Hu Yang Liu Jianjun Zhao and Meng Sun. 2020. Towards Characterizing Adversarial Defects of Deep Learning Software from the Lens of Uncertainty. In ICSE.","DOI":"10.1145\/3377811.3380368"},{"key":"e_1_3_2_1_117_1","doi-asserted-by":"crossref","unstructured":"Gang Zhao and Jeff Huang. 2018. DeepSim: Deep Learning Code Functional Similarity. In ESEC\/FSE.","DOI":"10.1145\/3236024.3236068"},{"key":"e_1_3_2_1_118_1","unstructured":"Hengshuang Zhao Jianping Shi Xiaojuan Qi Xiaogang Wang and Jiaya Jia. 2017. Pyramid scene parsing network. In CVPR."},{"key":"e_1_3_2_1_119_1","volume-title":"Wuji: Automatic Online Combat Game Testing Using Evolutionary Deep Reinforcement Learning. In ASE.","author":"Zheng Yan","year":"2019","unstructured":"Yan Zheng, Xiaofei Xie, Ting Su, Lei Ma, Jianye Hao, Zhaopeng Meng, Yang Liu, Ruimin Shen, Yingfeng Chen, and Changjie Fan. 2019. Wuji: Automatic Online Combat Game Testing Using Evolutionary Deep Reinforcement Learning. In ASE."},{"key":"e_1_3_2_1_120_1","doi-asserted-by":"crossref","unstructured":"Husheng Zhou Wei Li Zelun Kong Junfeng Guo Yuqun Zhang Lingming Zhang Bei Yu and Cong Liu. 2020. DeepBillboard: Systematic Physical-World Testing of Autonomous Driving Systems. In ICSE.","DOI":"10.1145\/3377811.3380422"},{"key":"e_1_3_2_1_121_1","doi-asserted-by":"crossref","unstructured":"H. Zhu M. Akrout B. Zheng A. Pelegris A. Jayarajan A. Phanishayee B. Schroeder and G. Pekhimenko. 2018. Benchmarking and Analyzing Deep Neural Network Training. In IISWC.","DOI":"10.1109\/IISWC.2018.8573476"},{"key":"e_1_3_2_1_122_1","doi-asserted-by":"crossref","unstructured":"Rui Zhu Shifeng Zhang Xiaobo Wang Longyin Wen Hailin Shi Liefeng Bo and Tao Mei. 2019. ScratchDet: Training single-shot object detectors from scratch. In CVPR.","DOI":"10.1109\/CVPR.2019.00237"}],"event":{"name":"ASE '20: 35th IEEE\/ACM International Conference on Automated Software Engineering","location":"Virtual Event Australia","acronym":"ASE '20","sponsor":["SIGAI ACM Special Interest Group on Artificial Intelligence","SIGSOFT ACM Special Interest Group on Software Engineering","IEEE CS"]},"container-title":["Proceedings of the 35th IEEE\/ACM International Conference on Automated Software Engineering"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3324884.3416545","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3324884.3416545","content-type":"application\/pdf","content-version":"vor","intended-application":"syndication"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3324884.3416545","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T22:01:37Z","timestamp":1750197697000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3324884.3416545"}},"subtitle":["an analysis of variance"],"short-title":[],"issued":{"date-parts":[[2020,12,21]]},"references-count":122,"alternative-id":["10.1145\/3324884.3416545","10.1145\/3324884"],"URL":"https:\/\/doi.org\/10.1145\/3324884.3416545","relation":{},"subject":[],"published":{"date-parts":[[2020,12,21]]},"assertion":[{"value":"2021-01-27","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}