{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,8,18]],"date-time":"2026-08-18T15:47:30Z","timestamp":1787068050291,"version":"build-2736575974"},"publisher-location":"New York, NY, USA","reference-count":74,"publisher":"ACM","license":[{"start":{"date-parts":[[2024,9,11]],"date-time":"2024-09-11T00:00:00Z","timestamp":1726012800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"name":"RGC GRF","award":["16214723"],"award-info":[{"award-number":["16214723"]}]},{"name":"RGC CRF","award":["C6015-23G"],"award-info":[{"award-number":["C6015-23G"]}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2024,9,11]]},"DOI":"10.1145\/3650212.3680385","type":"proceedings-article","created":{"date-parts":[[2024,9,11]],"date-time":"2024-09-11T07:44:25Z","timestamp":1726040665000},"page":"1605-1617","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":4,"title":["See the Forest, not Trees: Unveiling and Escaping the Pitfalls of Error-Triggering Inputs in Neural Network Testing"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-3053-8923","authenticated-orcid":false,"given":"Yuanyuan","family":"Yuan","sequence":"first","affiliation":[{"name":"Hong Kong University of Science and Technology, Hong Kong, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-0866-0308","authenticated-orcid":false,"given":"Shuai","family":"Wang","sequence":"additional","affiliation":[{"name":"Hong Kong University of Science and Technology, Hong Kong, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-2970-1391","authenticated-orcid":false,"given":"Zhendong","family":"Su","sequence":"additional","affiliation":[{"name":"ETH Zurich, Zurich, Switzerland"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2024,9,11]]},"reference":[{"key":"e_1_3_2_1_1_1","unstructured":"[n. d.]. Amazon Mechanical Turk. https:\/\/www.mturk.com\/"},{"key":"e_1_3_2_1_2_1","unstructured":"[n. d.]. Artifact. https:\/\/github.com\/Yuanyuan-Yuan\/EP-DNN-testing"},{"key":"e_1_3_2_1_3_1","unstructured":"[n. d.]. Detectron2: Facebook AI Research\u2019s platform for object detection and semantic segmentation. https:\/\/ai.facebook.com\/tools\/detectron2\/"},{"key":"e_1_3_2_1_4_1","unstructured":"OpenAI (2023). 2023. GPT-4 Technical Report."},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1145\/3550271"},{"key":"e_1_3_2_1_6_1","volume-title":"Large Scale GAN Training for High Fidelity Natural Image Synthesis. In International Conference on Learning Representations.","author":"Brock Andrew","year":"2018","unstructured":"Andrew Brock, Jeff Donahue, and Karen Simonyan. 2018. Large Scale GAN Training for High Fidelity Natural Image Synthesis. In International Conference on Learning Representations."},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"publisher","DOI":"10.1145\/3377816.3381734"},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR42600.2020.00821"},{"key":"e_1_3_2_1_10_1","volume-title":"The theory of max-min and its application to weapons allocation problems. 5","author":"Danskin John M","unstructured":"John M Danskin. 2012. The theory of max-min and its application to weapons allocation problems. 5, Springer Science & Business Media."},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"crossref","unstructured":"J. Deng W. Dong R. Socher L.-J. Li K. Li and L. Fei-Fei. 2009. ImageNet: A Large-Scale Hierarchical Image Database. In CVPR09.","DOI":"10.1109\/CVPR.2009.5206848"},{"key":"e_1_3_2_1_12_1","volume-title":"2021 IEEE\/ACM 43rd International Conference on Software Engineering (ICSE). 226\u2013237","author":"Dola Swaroopa","year":"2021","unstructured":"Swaroopa Dola, Matthew B Dwyer, and Mary Lou Soffa. 2021. Distribution-aware testing of neural networks using generative models. In 2021 IEEE\/ACM 43rd International Conference on Software Engineering (ICSE). 226\u2013237."},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1145\/3460319.3464801"},{"key":"e_1_3_2_1_14_1","volume-title":"International conference on machine learning. 1802\u20131811","author":"Engstrom Logan","year":"2019","unstructured":"Logan Engstrom, Brandon Tran, Dimitris Tsipras, Ludwig Schmidt, and Aleksander Madry. 2019. Exploring the landscape of spatial robustness. In International conference on machine learning. 1802\u20131811."},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.1145\/3395363.3397357"},{"key":"e_1_3_2_1_16_1","volume-title":"Measuring nominal scale agreement among many raters.. Psychological bulletin, 76, 5","author":"Fleiss Joseph L","year":"1971","unstructured":"Joseph L Fleiss. 1971. Measuring nominal scale agreement among many raters.. Psychological bulletin, 76, 5 (1971), 378."},{"key":"e_1_3_2_1_17_1","volume-title":"International Conference on Learning Representations.","author":"Geirhos Robert","year":"2019","unstructured":"Robert Geirhos, Patricia Rubisch, Claudio Michaelis, Matthias Bethge, Felix A. Wichmann, and Wieland Brendel. 2019. ImageNet-trained CNNs are biased towards texture; increasing shape bias improves accuracy and robustness.. In International Conference on Learning Representations."},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1145\/3377811.3380391"},{"key":"e_1_3_2_1_19_1","volume-title":"Singular value decomposition and least squares solutions. Linear algebra, 2","author":"Golub Gene H","year":"1971","unstructured":"Gene H Golub and Christian Reinsch. 1971. Singular value decomposition and least squares solutions. Linear algebra, 2 (1971), 134\u2013151."},{"key":"e_1_3_2_1_20_1","unstructured":"Ian Goodfellow Jean Pouget-Abadie Mehdi Mirza Bing Xu David Warde-Farley Sherjil Ozair Aaron Courville and Yoshua Bengio. 2014. Generative adversarial nets. In Advances in neural information processing systems. 2672\u20132680."},{"key":"e_1_3_2_1_21_1","volume-title":"International Conference on Machine Learning. 2484\u20132493","author":"Guo Chuan","year":"2019","unstructured":"Chuan Guo, Jacob Gardner, Yurong You, Andrew Gordon Wilson, and Kilian Weinberger. 2019. Simple black-box adversarial attacks. In International Conference on Machine Learning. 2484\u20132493."},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.1145\/3236024.3264835"},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.1145\/3368089.3409754"},{"key":"e_1_3_2_1_24_1","unstructured":"Kaiming He Xiangyu Zhang Shaoqing Ren and Jian Sun. 2016. Deep residual learning for image recognition. In CVPR. 770\u2013778."},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.1145\/3377811.3380339"},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"crossref","unstructured":"James F Hemphill. 2003. Interpreting the magnitudes of correlation coefficients..","DOI":"10.1037\/0003-066X.58.1.78"},{"key":"e_1_3_2_1_27_1","volume-title":"Mobilenets: Efficient convolutional neural networks for mobile vision applications. arXiv preprint arXiv:1704.04861.","author":"Howard Andrew G","year":"2017","unstructured":"Andrew G Howard, Menglong Zhu, Bo Chen, Dmitry Kalenichenko, Weijun Wang, Tobias Weyand, Marco Andreetto, and Hartwig Adam. 2017. Mobilenets: Efficient convolutional neural networks for mobile vision applications. arXiv preprint arXiv:1704.04861."},{"key":"e_1_3_2_1_28_1","volume-title":"Laurens Van Der Maaten, and Kilian Q Weinberger","author":"Huang Gao","year":"2017","unstructured":"Gao Huang, Zhuang Liu, Laurens Van Der Maaten, and Kilian Q Weinberger. 2017. Densely connected convolutional networks. In CVPR."},{"key":"e_1_3_2_1_29_1","volume-title":"2023 IEEE\/ACM 45th International Conference on Software Engineering (ICSE). 1788\u20131800","author":"Ji Zhenlan","year":"2023","unstructured":"Zhenlan Ji, Pingchuan Ma, Yuanyuan Yuan, and Shuai Wang. 2023. CC: Causality-aware coverage criterion for deep neural networks. In 2023 IEEE\/ACM 45th International Conference on Software Engineering (ICSE). 1788\u20131800."},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"crossref","unstructured":"Sungmin Kang Robert Feldt and Shin Yoo. 2023. Deceiving Humans and Machines Alike: Search-based Test Input Generation for DNNs using Variational Autoencoders. ACM Transactions on Software Engineering and Methodology.","DOI":"10.1145\/3635706"},{"key":"e_1_3_2_1_31_1","unstructured":"Hamid Karimi Tyler Derr and Jiliang Tang. 2019. Characterizing the decision boundary of deep neural networks. arXiv preprint arXiv:1912.11460."},{"key":"e_1_3_2_1_32_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2019.00453"},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE.2019.00108"},{"key":"e_1_3_2_1_34_1","doi-asserted-by":"publisher","DOI":"10.1145\/3368089.3417065"},{"key":"e_1_3_2_1_35_1","doi-asserted-by":"publisher","DOI":"10.1145\/3065386"},{"key":"e_1_3_2_1_36_1","volume-title":"Shuvendu K Lahiri, and Siddhartha Sen.","author":"Lemieux Caroline","year":"2023","unstructured":"Caroline Lemieux, Jeevana Priya Inala, Shuvendu K Lahiri, and Siddhartha Sen. 2023. CODAMOSA: Escaping Coverage Plateaus in Test Generation with Pre-trained Large Language Models. In 45th International Conference on Software Engineering, ser. ICSE."},{"key":"e_1_3_2_1_37_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2015.425"},{"key":"e_1_3_2_1_38_1","doi-asserted-by":"publisher","DOI":"10.1145\/3238147.3238202"},{"key":"e_1_3_2_1_39_1","doi-asserted-by":"publisher","DOI":"10.1080\/01621459.1949.10483310"},{"key":"e_1_3_2_1_40_1","doi-asserted-by":"publisher","DOI":"10.1145\/3533767.3534388"},{"key":"e_1_3_2_1_41_1","doi-asserted-by":"crossref","unstructured":"Qi Pang Yuanyuan Yuan and Shuai Wang. 2024. MPCDiff: Testing and Repairing MPC-Hardened Deep Learning Models.","DOI":"10.14722\/ndss.2024.23380"},{"key":"e_1_3_2_1_42_1","doi-asserted-by":"publisher","DOI":"10.1145\/3132747.3132785"},{"key":"e_1_3_2_1_43_1","unstructured":"Alec Radford Karthik Narasimhan Tim Salimans and Ilya Sutskever. 2018. Improving language understanding by generative pre-training."},{"key":"e_1_3_2_1_44_1","doi-asserted-by":"publisher","DOI":"10.1145\/2939672.2939778"},{"key":"e_1_3_2_1_45_1","volume-title":"2023 IEEE\/ACM 45th International Conference on Software Engineering (ICSE). 1161\u20131173","author":"Riccio Vincenzo","year":"2023","unstructured":"Vincenzo Riccio and Paolo Tonella. 2023. When and why test generators for deep learning produce invalid inputs: an empirical study. In 2023 IEEE\/ACM 45th International Conference on Software Engineering (ICSE). 1161\u20131173."},{"key":"e_1_3_2_1_46_1","doi-asserted-by":"publisher","DOI":"10.1007\/s11263-015-0816-y"},{"key":"e_1_3_2_1_47_1","volume-title":"International Conference on Learning Representations.","author":"Schott Lukas","year":"2018","unstructured":"Lukas Schott, Jonas Rauber, Matthias Bethge, and Wieland Brendel. 2018. Towards the first adversarially robust neural network model on MNIST. In International Conference on Learning Representations."},{"key":"e_1_3_2_1_48_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2015.7298682"},{"key":"e_1_3_2_1_49_1","unstructured":"Julius Sim and Chris C Wright. 2005. The kappa statistic in reliability studies: use interpretation and sample size requirements. Physical therapy."},{"key":"e_1_3_2_1_50_1","unstructured":"Karen Simonyan and Andrew Zisserman. 2014. Very deep convolutional networks for large-scale image recognition. arXiv preprint arXiv:1409.1556."},{"key":"e_1_3_2_1_51_1","doi-asserted-by":"crossref","unstructured":"Christian Szegedy Vincent Vanhoucke Sergey Ioffe Jon Shlens and Zbigniew Wojna. 2016. Rethinking the inception architecture for computer vision. In CVPR.","DOI":"10.1109\/CVPR.2016.308"},{"key":"e_1_3_2_1_52_1","volume-title":"International conference on machine learning. 6105\u20136114","author":"Tan Mingxing","year":"2019","unstructured":"Mingxing Tan and Quoc Le. 2019. Efficientnet: Rethinking model scaling for convolutional neural networks. In International conference on machine learning. 6105\u20136114."},{"key":"e_1_3_2_1_53_1","doi-asserted-by":"publisher","DOI":"10.1177\/875647939000600106"},{"key":"e_1_3_2_1_54_1","doi-asserted-by":"publisher","DOI":"10.1145\/3180155.3180220"},{"key":"e_1_3_2_1_55_1","volume-title":"Adversarial training and robustness for multiple perturbations. Advances in neural information processing systems, 32","author":"Tramer Florian","year":"2019","unstructured":"Florian Tramer and Dan Boneh. 2019. Adversarial training and robustness for multiple perturbations. Advances in neural information processing systems, 32 (2019)."},{"key":"e_1_3_2_1_56_1","volume-title":"Theoretical analysis of adversarial learning: A minimax approach. Advances in Neural Information Processing Systems, 32","author":"Tu Zhuozhuo","year":"2019","unstructured":"Zhuozhuo Tu, Jingwei Zhang, and Dacheng Tao. 2019. Theoretical analysis of adversarial learning: A minimax approach. Advances in Neural Information Processing Systems, 32 (2019)."},{"key":"e_1_3_2_1_57_1","volume-title":"International Conference on Machine Learning. 5025\u20135034","author":"Uesato Jonathan","year":"2018","unstructured":"Jonathan Uesato, Brendan O\u2019donoghue, Pushmeet Kohli, and Aaron Oord. 2018. Adversarial risk and the dangers of evaluating against weak attacks. In International Conference on Machine Learning. 5025\u20135034."},{"key":"e_1_3_2_1_58_1","volume-title":"2021 IEEE\/ACM 43rd International Conference on Software Engineering (ICSE). 300\u2013311","author":"Wang Jingyi","year":"2021","unstructured":"Jingyi Wang, Jialuo Chen, Youcheng Sun, Xingjun Ma, Dongxia Wang, Jun Sun, and Peng Cheng. 2021. Robot: Robustness-oriented testing for deep learning systems. In 2021 IEEE\/ACM 43rd International Conference on Software Engineering (ICSE). 300\u2013311."},{"key":"e_1_3_2_1_59_1","volume-title":"Proceedings of the 35th IEEE\/ACM International Conference on Automated Software Engineering. 1053\u20131065","author":"Wang Shuai","year":"2020","unstructured":"Shuai Wang and Zhendong Su. 2020. Metamorphic object insertion for testing object detection systems. In Proceedings of the 35th IEEE\/ACM International Conference on Automated Software Engineering. 1053\u20131065."},{"key":"e_1_3_2_1_60_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE43902.2021.00046"},{"key":"e_1_3_2_1_61_1","volume-title":"Proceedings of the IEEE\/CVF International Conference on Computer Vision. 6373\u20136382","author":"Wu Yi-Lun","year":"2021","unstructured":"Yi-Lun Wu, Hong-Han Shuai, Zhi-Rui Tam, and Hong-Yu Chiu. 2021. Gradient normalization for generative adversarial networks. In Proceedings of the IEEE\/CVF International Conference on Computer Vision. 6373\u20136382."},{"key":"e_1_3_2_1_62_1","doi-asserted-by":"publisher","DOI":"10.1145\/3293882.3330579"},{"key":"e_1_3_2_1_63_1","volume-title":"Boosting the Revealing of Detected Violations in Deep Learning Testing: A Diversity-Guided Method. In ASE","author":"Xie Xiaoyuan","year":"2022","unstructured":"Xiaoyuan Xie, Pengbo Yin, and Songqiang Chen. 2022. Boosting the Revealing of Detected Violations in Deep Learning Testing: A Diversity-Guided Method. In ASE 2022."},{"key":"e_1_3_2_1_64_1","doi-asserted-by":"publisher","DOI":"10.1145\/3554729"},{"key":"e_1_3_2_1_65_1","doi-asserted-by":"publisher","DOI":"10.1145\/3533767.3534389"},{"key":"e_1_3_2_1_66_1","volume-title":"Lsun: Construction of a large-scale image dataset using deep learning with humans in the loop. arXiv preprint arXiv:1506.03365.","author":"Yu Fisher","year":"2015","unstructured":"Fisher Yu, Ari Seff, Yinda Zhang, Shuran Song, Thomas Funkhouser, and Jianxiong Xiao. 2015. Lsun: Construction of a large-scale image dataset using deep learning with humans in the loop. arXiv preprint arXiv:1506.03365."},{"key":"e_1_3_2_1_67_1","volume-title":"Unveiling Hidden DNN Defects with Decision-Based Metamorphic Testing. In 37th IEEE\/ACM International Conference on Automated Software Engineering. 1\u201313","author":"Yuan Yuanyuan","year":"2022","unstructured":"Yuanyuan Yuan, Qi Pang, and Shuai Wang. 2022. Unveiling Hidden DNN Defects with Decision-Based Metamorphic Testing. In 37th IEEE\/ACM International Conference on Automated Software Engineering. 1\u201313."},{"key":"e_1_3_2_1_68_1","volume-title":"2023 IEEE\/ACM 45th International Conference on Software Engineering (ICSE). 1200\u20131212","author":"Yuan Yuanyuan","year":"2023","unstructured":"Yuanyuan Yuan, Qi Pang, and Shuai Wang. 2023. Revisiting neuron coverage for dnn testing: A layer-wise and distribution-aware criterion. In 2023 IEEE\/ACM 45th International Conference on Software Engineering (ICSE). 1200\u20131212."},{"key":"e_1_3_2_1_69_1","article-title":"Provably Valid and Diverse Mutations of Real-World Media Data for DNN Testing","author":"Yuan Yuanyuan","year":"2024","unstructured":"Yuanyuan Yuan, Qi Pang, and Shuai Wang. 2024. Provably Valid and Diverse Mutations of Real-World Media Data for DNN Testing. IEEE Transactions on Software Engineering.","journal-title":"IEEE Transactions on Software Engineering."},{"key":"e_1_3_2_1_70_1","volume-title":"Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition. 16908\u201316917","author":"Yuan Yuanyuan","year":"2021","unstructured":"Yuanyuan Yuan, Shuai Wang, Mingyue Jiang, and Tsong Yueh Chen. 2021. Perception matters: Detecting perception failures of vqa models using metamorphic testing. In Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition. 16908\u201316917."},{"key":"e_1_3_2_1_71_1","doi-asserted-by":"publisher","DOI":"10.1145\/3238147.3238187"},{"key":"e_1_3_2_1_72_1","volume-title":"International Conference on Machine Learning. 7584\u20137593","author":"Zhou Zhiming","year":"2019","unstructured":"Zhiming Zhou, Jiadong Liang, Yuxuan Song, Lantao Yu, Hongwei Wang, Weinan Zhang, Yong Yu, and Zhihua Zhang. 2019. Lipschitz generative adversarial nets. In International Conference on Machine Learning. 7584\u20137593."},{"key":"e_1_3_2_1_73_1","first-page":"16648","article-title":"Low-rank subspaces in gans","volume":"34","author":"Zhu Jiapeng","year":"2021","unstructured":"Jiapeng Zhu, Ruili Feng, Yujun Shen, Deli Zhao, Zheng-Jun Zha, Jingren Zhou, and Qifeng Chen. 2021. Low-rank subspaces in gans. Advances in Neural Information Processing Systems, 34 (2021), 16648\u201316658.","journal-title":"Advances in Neural Information Processing Systems"},{"key":"e_1_3_2_1_74_1","volume-title":"Region-Based Semantic Factorization in GANs. In International Conference on Machine Learning. 27612\u201327632","author":"Zhu Jiapeng","year":"2022","unstructured":"Jiapeng Zhu, Yujun Shen, Yinghao Xu, Deli Zhao, and Qifeng Chen. 2022. Region-Based Semantic Factorization in GANs. In International Conference on Machine Learning. 27612\u201327632."},{"key":"e_1_3_2_1_75_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2017.244"}],"event":{"name":"ISSTA '24: 33rd ACM SIGSOFT International Symposium on Software Testing and Analysis","location":"Vienna Austria","acronym":"ISSTA '24","sponsor":["SIGSOFT ACM Special Interest Group on Software Engineering","AITO"]},"container-title":["Proceedings of the 33rd ACM SIGSOFT International Symposium on Software Testing and Analysis"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3650212.3680385","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3650212.3680385","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,18]],"date-time":"2025-06-18T20:58:09Z","timestamp":1750280289000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3650212.3680385"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,9,11]]},"references-count":74,"alternative-id":["10.1145\/3650212.3680385","10.1145\/3650212"],"URL":"https:\/\/doi.org\/10.1145\/3650212.3680385","relation":{},"subject":[],"published":{"date-parts":[[2024,9,11]]},"assertion":[{"value":"2024-09-11","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}