{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,3,18]],"date-time":"2026-03-18T13:23:17Z","timestamp":1773840197042,"version":"3.50.1"},"publisher-location":"New York, NY, USA","reference-count":71,"publisher":"ACM","license":[{"start":{"date-parts":[[2020,11,8]],"date-time":"2020-11-08T00:00:00Z","timestamp":1604793600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"DOI":"10.13039\/501100012659","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["61872263,U1836214,61802275"],"award-info":[{"award-number":["61872263,U1836214,61802275"]}],"id":[{"id":"10.13039\/501100012659","id-type":"DOI","asserted-by":"publisher"}]},{"name":"Intelligent Manufacturing Special Fund of Tianjin","award":["20191012,20193155"],"award-info":[{"award-number":["20191012,20193155"]}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2020,11,8]]},"DOI":"10.1145\/3368089.3409761","type":"proceedings-article","created":{"date-parts":[[2020,11,8]],"date-time":"2020-11-08T06:03:47Z","timestamp":1604815427000},"page":"788-799","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":143,"title":["Deep learning library testing via effective model generation"],"prefix":"10.1145","author":[{"given":"Zan","family":"Wang","sequence":"first","affiliation":[{"name":"Tianjin University, China"}]},{"given":"Ming","family":"Yan","sequence":"additional","affiliation":[{"name":"Tianjin University, China"}]},{"given":"Junjie","family":"Chen","sequence":"additional","affiliation":[{"name":"Tianjin University, China"}]},{"given":"Shuang","family":"Liu","sequence":"additional","affiliation":[{"name":"Tianjin University, China"}]},{"given":"Dongdi","family":"Zhang","sequence":"additional","affiliation":[{"name":"Tianjin University, China"}]}],"member":"320","published-online":{"date-parts":[[2020,11,8]]},"reference":[{"key":"e_1_3_2_2_1_1","unstructured":"Accessed : 2020. CNTK. https:\/\/docs.microsoft.com\/cognitive-toolkit\/.  Accessed : 2020. CNTK. https:\/\/docs.microsoft.com\/cognitive-toolkit\/."},{"key":"e_1_3_2_2_2_1","unstructured":"Accessed : 2020. Keras. http:\/\/keras.io.  Accessed : 2020. Keras. http:\/\/keras.io."},{"key":"e_1_3_2_2_3_1","unstructured":"Accessed : 2020. LEMON. https:\/\/github.com\/Jacob-yen\/LEMON.  Accessed : 2020. LEMON. https:\/\/github.com\/Jacob-yen\/LEMON."},{"key":"e_1_3_2_2_4_1","unstructured":"Accessed : 2020. MXNet. http:\/\/mxnet.incubator.apache.org.  Accessed : 2020. MXNet. http:\/\/mxnet.incubator.apache.org."},{"key":"e_1_3_2_2_5_1","unstructured":"Accessed : 2020. News. https:\/\/www.vice.com\/en_us\/article\/9kga85\/uber-isgiving-up-on-self-driving-cars-in-california-after-deadly-crash.  Accessed : 2020. News. https:\/\/www.vice.com\/en_us\/article\/9kga85\/uber-isgiving-up-on-self-driving-cars-in-california-after-deadly-crash."},{"key":"e_1_3_2_2_6_1","unstructured":"Accessed : 2020. News. https:\/\/en.wikipedia.org\/wiki\/List_of_self-driving_car_fatalities#cite_note-15.  Accessed : 2020. News. https:\/\/en.wikipedia.org\/wiki\/List_of_self-driving_car_fatalities#cite_note-15."},{"key":"e_1_3_2_2_7_1","unstructured":"Accessed : 2020. TensorFlow. https:\/\/www.tensorflow.org.  Accessed : 2020. TensorFlow. https:\/\/www.tensorflow.org."},{"key":"e_1_3_2_2_8_1","unstructured":"Accessed : 2020. Theano. http:\/\/deeplearning.net\/software\/theano\/.  Accessed : 2020. Theano. http:\/\/deeplearning.net\/software\/theano\/."},{"key":"e_1_3_2_2_9_1","doi-asserted-by":"publisher","DOI":"10.1145\/2610384.2610403"},{"key":"e_1_3_2_2_10_1","volume-title":"DeepEvolution: A Search-Based Testing Approach for Deep Neural Networks. In 2019 IEEE International Conference on Software Maintenance and Evolution (ICSME). IEEE, 454-458","author":"Braiek Houssem Ben","year":"2019"},{"key":"e_1_3_2_2_11_1","doi-asserted-by":"publisher","DOI":"10.1093\/jamia\/ocy017"},{"key":"e_1_3_2_2_12_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2015.312"},{"key":"e_1_3_2_2_13_1","doi-asserted-by":"publisher","DOI":"10.1145\/2970276.2970366"},{"key":"e_1_3_2_2_14_1","volume-title":"Proceedings of the ACM Joint Meeting on European Software Engineering Conference and Symposium on the Foundations of Software Engineering. 223-234","author":"Chen Junjie","year":"2019"},{"key":"e_1_3_2_2_15_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE-SEIP.2019.00020"},{"key":"e_1_3_2_2_16_1","doi-asserted-by":"publisher","DOI":"10.1109\/ASE.2019.00042"},{"key":"e_1_3_2_2_17_1","doi-asserted-by":"publisher","DOI":"10.1145\/2884781.2884878"},{"key":"e_1_3_2_2_18_1","doi-asserted-by":"publisher","DOI":"10.1145\/3324884.3416570"},{"key":"e_1_3_2_2_19_1","doi-asserted-by":"crossref","unstructured":"Junjie Chen Jibesh Patra Michael Pradel Yingfei Xiong Hongyu Zhang Dan Hao and Lu Zhang. 2020. A Survey of Compiler Testing. Comput. Surveys 53 (02 2020 ) 1-36.  Junjie Chen Jibesh Patra Michael Pradel Yingfei Xiong Hongyu Zhang Dan Hao and Lu Zhang. 2020. A Survey of Compiler Testing. Comput. Surveys 53 (02 2020 ) 1-36.","DOI":"10.1145\/3363562"},{"key":"e_1_3_2_2_20_1","doi-asserted-by":"crossref","unstructured":"Junjie Chen Zhuo Wu Zan Wang Hanmo You Lingming Zhang and Ming Yan. 2020. Practical Accuracy Estimation for Eficient Deep Neural Network Testing. ACM Transactions on Software Engineering and Methodology ( 2020 ). to appear.  Junjie Chen Zhuo Wu Zan Wang Hanmo You Lingming Zhang and Ming Yan. 2020. Practical Accuracy Estimation for Eficient Deep Neural Network Testing. ACM Transactions on Software Engineering and Methodology ( 2020 ). to appear.","DOI":"10.1145\/3394112"},{"key":"e_1_3_2_2_21_1","volume-title":"Characterizing and Prioritizing Incidents for LargeScale Online Service Systems. In The 35th IEEE\/ACM International Conference on Automated Software Engineering.","author":"Chen Junjie","year":"2020"},{"key":"e_1_3_2_2_22_1","article-title":"Metamorphic Testing","volume":"51","author":"Chen Tsong Yueh","year":"2018","journal-title":"A Review of Challenges and Opportunities. Comput. Surveys"},{"key":"e_1_3_2_2_23_1","first-page":"85","article-title":"Coverage-directed Diferential Testing of JVM Implementations","author":"Chen Yuting","year":"2016","journal-title":"PLDI."},{"key":"e_1_3_2_2_24_1","volume-title":"Proceedings of the 25th International Conference on Neural Information Processing Systems-Volume 1. 1223-1231","author":"Dean Jefrey"},{"key":"e_1_3_2_2_25_1","volume-title":"The Oxford dictionary of statistical terms","author":"Dodge Yadolah"},{"key":"e_1_3_2_2_26_1","doi-asserted-by":"publisher","DOI":"10.1145\/3236024.3236057"},{"key":"e_1_3_2_2_27_1","doi-asserted-by":"publisher","DOI":"10.1145\/3213846.3213858"},{"key":"e_1_3_2_2_28_1","doi-asserted-by":"crossref","unstructured":"David Goldberg. 1991. What every computer scientist should know about floatingpoint arithmetic. Comput. Surveys 23 1 ( 1991 ) 5-48.  David Goldberg. 1991. What every computer scientist should know about floatingpoint arithmetic. Comput. Surveys 23 1 ( 1991 ) 5-48.","DOI":"10.1145\/103162.103163"},{"key":"e_1_3_2_2_29_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICASSP.2013.6638947"},{"key":"e_1_3_2_2_30_1","volume-title":"Proceedings of the 34th International Conference on Automated Software Engineering.","author":"Guo Qianyu","year":"2019"},{"key":"e_1_3_2_2_31_1","unstructured":"Fitash Ul Haq Donghwan Shin Shiva Nejati and Lionel Briand. 2019. Comparing Ofline and Online Testing of Deep Neural Networks: An Autonomous Car Case Study. arXiv preprint arXiv:1912. 00805 ( 2019 ).  Fitash Ul Haq Donghwan Shin Shiva Nejati and Lionel Briand. 2019. Comparing Ofline and Online Testing of Deep Neural Networks: An Autonomous Car Case Study. arXiv preprint arXiv:1912. 00805 ( 2019 )."},{"key":"e_1_3_2_2_32_1","volume-title":"A Comprehensive Study on Deep Learning Bug Characteristics. arXiv preprint arXiv","author":"Islam Md Johirul","year":"1906"},{"key":"e_1_3_2_2_33_1","doi-asserted-by":"publisher","DOI":"10.1145\/3106237.3106244"},{"key":"e_1_3_2_2_34_1","doi-asserted-by":"crossref","unstructured":"Yue Jia and Mark Harman. 2009. Higher order mutation testing. Information and Software Technology 51 10 ( 2009 ) 1379-1393.  Yue Jia and Mark Harman. 2009. Higher order mutation testing. Information and Software Technology 51 10 ( 2009 ) 1379-1393.","DOI":"10.1016\/j.infsof.2009.04.016"},{"key":"e_1_3_2_2_35_1","volume-title":"An analysis and survey of the development of mutation testing","author":"Jia Yue","year":"2010"},{"key":"e_1_3_2_2_36_1","doi-asserted-by":"publisher","DOI":"10.1109\/DASC.2016.7778091"},{"key":"e_1_3_2_2_37_1","doi-asserted-by":"publisher","DOI":"10.5555\/3155562.3155636"},{"key":"e_1_3_2_2_38_1","volume-title":"Neal","author":"Kass Robert E.","year":"1998"},{"key":"e_1_3_2_2_39_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE.2019.00108"},{"key":"e_1_3_2_2_40_1","unstructured":"Alexey Kurakin Ian Goodfellow and Samy Bengio. 2016. Adversarial examples in the physical world. arXiv preprint arXiv:1607.02533 ( 2016 ).  Alexey Kurakin Ian Goodfellow and Samy Bengio. 2016. Adversarial examples in the physical world. arXiv preprint arXiv:1607.02533 ( 2016 )."},{"key":"e_1_3_2_2_41_1","doi-asserted-by":"publisher","DOI":"10.1145\/3107411.3107451"},{"key":"e_1_3_2_2_42_1","doi-asserted-by":"publisher","DOI":"10.1145\/3293882.3330574"},{"key":"e_1_3_2_2_43_1","doi-asserted-by":"publisher","DOI":"10.1145\/3338906.3338930"},{"key":"e_1_3_2_2_44_1","doi-asserted-by":"crossref","unstructured":"Adam Lipowski and Dorota Lipowska. 2012. Roulette-wheel selection via stochastic acceptance. Physica A: Statistical Mechanics and its Applications 391 6 ( 2012 ) 2193-2196.  Adam Lipowski and Dorota Lipowska. 2012. Roulette-wheel selection via stochastic acceptance. Physica A: Statistical Mechanics and its Applications 391 6 ( 2012 ) 2193-2196.","DOI":"10.1016\/j.physa.2011.12.004"},{"key":"e_1_3_2_2_45_1","doi-asserted-by":"publisher","DOI":"10.1145\/3238147.3238202"},{"key":"e_1_3_2_2_46_1","volume-title":"DeepMutation: Mutation Testing of Deep Learning Systems. In 29th IEEE International Symposium on Software Reliability Engineering. 100-111","author":"Ma Lei","year":"2018"},{"key":"e_1_3_2_2_47_1","doi-asserted-by":"publisher","DOI":"10.1145\/3236024.3236082"},{"key":"e_1_3_2_2_48_1","volume-title":"NIC: Detecting Adversarial Samples with Neural Network Invariant Checking. In 26th Annual Network and Distributed System Security Symposium.","author":"Ma Shiqing","year":"2019"},{"key":"e_1_3_2_2_49_1","article-title":"Diferential testing for software","volume":"10","author":"McKeeman William M","year":"1998","journal-title":"Digital Technical Journal"},{"key":"e_1_3_2_2_50_1","article-title":"Guided mutation testing for javascript web applications","volume":"41","author":"Mirshokraie Shabnam","year":"2014","journal-title":"IEEE Transactions on Software Engineering"},{"key":"e_1_3_2_2_51_1","doi-asserted-by":"crossref","unstructured":"Martin Monperrus. 2018. Automatic Software Repair: A Bibliography. Comput. Surveys 51 1 ( 2018 ) 17 : 1-17 : 24.  Martin Monperrus. 2018. Automatic Software Repair: A Bibliography. Comput. Surveys 51 1 ( 2018 ) 17 : 1-17 : 24.","DOI":"10.1145\/3105906"},{"key":"e_1_3_2_2_52_1","volume-title":"Tensorfuzz: Debugging neural networks with coverage-guided fuzzing. arXiv preprint arXiv","author":"Odena Augustus","year":"2018"},{"key":"e_1_3_2_2_53_1","doi-asserted-by":"publisher","DOI":"10.1145\/2931037.2931040"},{"key":"e_1_3_2_2_54_1","doi-asserted-by":"publisher","DOI":"10.1016\/bs.adcom.2018.03.015"},{"key":"e_1_3_2_2_55_1","doi-asserted-by":"publisher","DOI":"10.1145\/3132747.3132785"},{"key":"e_1_3_2_2_56_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE.2019.00107"},{"key":"e_1_3_2_2_57_1","volume-title":"Testing Machine Learning Algorithms for Balanced Data Usage. In 12th IEEE Conference on Software Testing, Validation and Verification. 125-135","author":"Sharma Arnab","year":"2019"},{"key":"e_1_3_2_2_58_1","volume-title":"Multiple-Implementation Testing of Supervised Learning Software. In The Workshops of the The Thirty-Second AAAI Conference on Artificial Intelligence. 384-391","author":"Srisakaokul Siwakorn","year":"2018"},{"key":"e_1_3_2_2_59_1","unstructured":"Yi Sun Yuheng Chen Xiaogang Wang and Xiaoou Tang. 2014. Deep learning face representation by joint identification-verification. In Advances in neural information processing systems. 1988-1996.  Yi Sun Yuheng Chen Xiaogang Wang and Xiaoou Tang. 2014. Deep learning face representation by joint identification-verification. In Advances in neural information processing systems. 1988-1996."},{"key":"e_1_3_2_2_60_1","doi-asserted-by":"publisher","DOI":"10.1145\/3238147.3238172"},{"key":"e_1_3_2_2_61_1","doi-asserted-by":"crossref","unstructured":"Kim-Han Thung Pew-Thian Yap and Dinggang Shen. 2017. Multi-stage diagnosis of Alzheimer's disease with incomplete multimodal data via multi-task deep learning. In Deep learning in medical image analysis and multimodal learning for clinical decision support. 160-168.  Kim-Han Thung Pew-Thian Yap and Dinggang Shen. 2017. Multi-stage diagnosis of Alzheimer's disease with incomplete multimodal data via multi-task deep learning. In Deep learning in medical image analysis and multimodal learning for clinical decision support. 160-168.","DOI":"10.1007\/978-3-319-67558-9_19"},{"key":"e_1_3_2_2_62_1","doi-asserted-by":"publisher","DOI":"10.1145\/3180155.3180220"},{"key":"e_1_3_2_2_63_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE.2019.00126"},{"key":"e_1_3_2_2_64_1","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2016.2521368"},{"key":"e_1_3_2_2_65_1","doi-asserted-by":"publisher","DOI":"10.1109\/DSN.2019.00026"},{"key":"e_1_3_2_2_66_1","doi-asserted-by":"publisher","DOI":"10.1145\/3293882.3330579"},{"key":"e_1_3_2_2_67_1","volume-title":"Machine Learning Testing: Survey, Landscapes and Horizons. arXiv preprint arXiv","author":"Zhang Jie M","year":"1906"},{"key":"e_1_3_2_2_68_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSM.2010.5609672"},{"key":"e_1_3_2_2_69_1","doi-asserted-by":"publisher","DOI":"10.1145\/3238147.3238187"},{"key":"e_1_3_2_2_70_1","doi-asserted-by":"publisher","DOI":"10.1145\/3338906.3338931"},{"key":"e_1_3_2_2_71_1","doi-asserted-by":"publisher","DOI":"10.1145\/3213846.3213866"}],"event":{"name":"ESEC\/FSE '20: 28th ACM Joint European Software Engineering Conference and Symposium on the Foundations of Software Engineering","location":"Virtual Event USA","acronym":"ESEC\/FSE '20","sponsor":["SIGSOFT ACM Special Interest Group on Software Engineering"]},"container-title":["Proceedings of the 28th ACM Joint Meeting on European Software Engineering Conference and Symposium on the Foundations of Software Engineering"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3368089.3409761","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3368089.3409761","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T22:01:58Z","timestamp":1750197718000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3368089.3409761"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2020,11,8]]},"references-count":71,"alternative-id":["10.1145\/3368089.3409761","10.1145\/3368089"],"URL":"https:\/\/doi.org\/10.1145\/3368089.3409761","relation":{},"subject":[],"published":{"date-parts":[[2020,11,8]]},"assertion":[{"value":"2020-11-08","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}