{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,11]],"date-time":"2026-04-11T02:13:49Z","timestamp":1775873629057,"version":"3.50.1"},"publisher-location":"New York, NY, USA","reference-count":87,"publisher":"ACM","license":[{"start":{"date-parts":[[2020,12,21]],"date-time":"2020-12-21T00:00:00Z","timestamp":1608508800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2020,12,21]]},"DOI":"10.1145\/3324884.3416584","type":"proceedings-article","created":{"date-parts":[[2021,1,27]],"date-time":"2021-01-27T23:39:02Z","timestamp":1611790742000},"page":"1053-1065","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":57,"title":["Metamorphic object insertion for testing object detection systems"],"prefix":"10.1145","author":[{"given":"Shuai","family":"Wang","sequence":"first","affiliation":[{"name":"The Hong Kong University of Science and Technology"}]},{"given":"Zhendong","family":"Su","sequence":"additional","affiliation":[{"name":"ETH Zurich"}]}],"member":"320","published-online":{"date-parts":[[2021,1,27]]},"reference":[{"key":"e_1_3_2_1_1_1","unstructured":"2018. Amazon Rekognition. https:\/\/aws.amazon.com\/rekognition\/."},{"key":"e_1_3_2_1_2_1","unstructured":"2018. Google Cloud Detecting Objects. https:\/\/cloud.google.com\/vision\/docs\/detecting-objects."},{"key":"e_1_3_2_1_3_1","unstructured":"2018. Tensorflow Object Detection API. https:\/\/cloud.google.com\/solutions\/creating-object-detection-application-tensorflow."},{"key":"e_1_3_2_1_4_1","unstructured":"2019. Azure Computer Vision API. https:\/\/docs.microsoft.com\/en-us\/azure\/cognitive-services\/computer-vision\/."},{"key":"e_1_3_2_1_5_1","unstructured":"2019. berkeley DeepDrive. https:\/\/bdd-data.berkeley.edu\/."},{"key":"e_1_3_2_1_6_1","unstructured":"2019. IBM Vision API. https:\/\/developer.ibm.com\/exchanges\/models\/all\/max-object-detector\/."},{"key":"e_1_3_2_1_7_1","unstructured":"2019. YOLACT. https:\/\/github.com\/dbolya\/yolact."},{"key":"e_1_3_2_1_8_1","unstructured":"2020. Dropbox folder of all erroneous detection results found by MetaOD. https:\/\/www.dropbox.com\/sh\/ivpprt96nycokd9\/AAAZ_3D8TVAF25L90Z9AqpRya?dl=0."},{"key":"e_1_3_2_1_9_1","unstructured":"2020. MetaOD Codebase. https:\/\/github.com\/MetaOD\/MetaOD."},{"key":"e_1_3_2_1_10_1","volume-title":"Proceedings of the 12th USENIX Conference on Operating Systems Design and Implementation (OSDI'16)","author":"Abadi Mart\u00edn","year":"2016","unstructured":"Mart\u00edn Abadi, Paul Barham, Jianmin Chen, Zhifeng Chen, Andy Davis, Jeffrey Dean, Matthieu Devin, Sanjay Ghemawat, Geoffrey Irving, Michael Isard, Manjunath Kudlur, Josh Levenberg, Rajat Monga, Sherry Moore, Derek G. Murray, Benoit Steiner, Paul Tucker, Vijay Vasudevan, Pete Warden, Martin Wicke, Yuan Yu, and Xiaoqiang Zheng. 2016. TensorFlow: A System for Large-scale Machine Learning. In Proceedings of the 12th USENIX Conference on Operating Systems Design and Implementation (OSDI'16). USENIX Association, Berkeley, CA, USA, 265--283. http:\/\/dl.acm.org\/citation.cfm?id=3026877.3026899"},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.1145\/3238147.3238192"},{"key":"e_1_3_2_1_12_1","volume-title":"Neural Networks Are Easily Fooled by Strange Poses of Familiar Objects (CVPR","author":"Alcorn Michael A.","year":"2019","unstructured":"Michael A. Alcorn, Qi Li, Zhitao Gong, Chengfei Wang, Long Mai, Wei-Shinn Ku, and Anh Nguyen. 2019. Strike (with) a Pose: Neural Networks Are Easily Fooled by Strange Poses of Familiar Objects (CVPR 2019)."},{"key":"e_1_3_2_1_13_1","volume-title":"In Joint IEEE Int. Workshop on Visual Surveillance and Performance Evaluation of Tracking and Surveillance (VS-PETS. 125--132","author":"Black James","year":"2003","unstructured":"James Black, Tim Ellis, and Paul Rosin. 2003. A Novel Method for Video Tracking Performance Evaluation. In In Joint IEEE Int. Workshop on Visual Surveillance and Performance Evaluation of Tracking and Surveillance (VS-PETS. 125--132."},{"key":"e_1_3_2_1_14_1","volume-title":"YOLACT: Realtime Instance Segmentation. arXiv preprint arXiv:1904.02689","author":"Bolya Daniel","year":"2019","unstructured":"Daniel Bolya, Chong Zhou, Fanyi Xiao, and Yong Jae Lee. 2019. YOLACT: Realtime Instance Segmentation. arXiv preprint arXiv:1904.02689 (2019)."},{"key":"e_1_3_2_1_15_1","unstructured":"Neal E. Boudette. 2017. Tesla's Self-Driving System Cleared in Deadly Crash. https:\/\/www.nytimes.com\/2017\/01\/19\/business\/tesla-model-s-autopilot-fatal-crash.html."},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.1145\/3143561"},{"key":"e_1_3_2_1_18_1","volume-title":"R-fcn: Object detection via region-based fully convolutional networks. In Advances in neural information processing systems. 379--387.","author":"Dai Jifeng","year":"2016","unstructured":"Jifeng Dai, Yi Li, Kaiming He, and Jian Sun. 2016. R-fcn: Object detection via region-based fully convolutional networks. In Advances in neural information processing systems. 379--387."},{"key":"e_1_3_2_1_19_1","unstructured":"Navneet Dalal and Bill Triggs. 2005. Histograms of oriented gradients for human detection."},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.1145\/3338906.3338954"},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.1145\/3236024.3236057"},{"key":"e_1_3_2_1_22_1","volume-title":"Proceedings of the 27th ACM SIGSOFT International Symposium on Software Testing and Analysis (ISSTA","author":"Dwarakanath Anurag","year":"2018","unstructured":"Anurag Dwarakanath, Manish Ahuja, Samarth Sikand, Raghotham M. Rao, R. P. Jagadeesh Chandra Bose, Neville Dubash, and Sanjay Podder. 2018. Identifying Implementation Bugs in Machine Learning Based Image Classifiers Using Metamorphic Testing. In Proceedings of the 27th ACM SIGSOFT International Symposium on Software Testing and Analysis (ISSTA 2018). 118--128."},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2017.146"},{"key":"e_1_3_2_1_24_1","volume-title":"Christopher KI Williams, John Winn, and Andrew Zisserman.","author":"Everingham Mark","year":"2010","unstructured":"Mark Everingham, Luc Van Gool, Christopher KI Williams, John Winn, and Andrew Zisserman. 2010. The pascal visual object classes (voc) challenge. International journal of computer vision 88, 2 (2010), 303--338."},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.1109\/ISBI.2018.8363576"},{"key":"e_1_3_2_1_26_1","volume-title":"Proceedings of the IEEE conference on computer vision and pattern recognition. 4340--4349","author":"Gaidon Adrien","year":"2016","unstructured":"Adrien Gaidon, Qiao Wang, Yohann Cabon, and Eleonora Vig. 2016. Virtual worlds as proxy for multi-object tracking analysis. In Proceedings of the IEEE conference on computer vision and pattern recognition. 4340--4349."},{"key":"e_1_3_2_1_27_1","doi-asserted-by":"crossref","unstructured":"Sainyam Galhotra Yuriy Brun and Alexandra Meliou. 2017. Fairness testing: testing software for discrimination. In ACM ESEC\/FSE. ACM 498--510.","DOI":"10.1145\/3106237.3106277"},{"key":"e_1_3_2_1_28_1","volume-title":"Synthesizing training data for object detection in indoor scenes. arXiv preprint arXiv:1702.07836","author":"Georgakis Georgios","year":"2017","unstructured":"Georgios Georgakis, Arsalan Mousavian, Alexander C Berg, and Jana Kosecka. 2017. Synthesizing training data for object detection in indoor scenes. arXiv preprint arXiv:1702.07836 (2017)."},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"crossref","unstructured":"Ross B. Girshick. 2015. Fast R-CNN. ICCV.","DOI":"10.1109\/ICCV.2015.169"},{"key":"e_1_3_2_1_30_1","volume-title":"Rich feature hierarchies for accurate object detection and semantic segmentation. CVPR","author":"Girshick Ross B.","year":"2014","unstructured":"Ross B. Girshick, Jeff Donahue, Trevor Darrell, and Jitendra Malik. 2014. Rich feature hierarchies for accurate object detection and semantic segmentation. CVPR (2014)."},{"key":"e_1_3_2_1_31_1","volume-title":"Symbolic execution for deep neural networks. arXiv preprint arXiv:1807.10439","author":"Gopinath Divya","year":"2018","unstructured":"Divya Gopinath, Kaiyuan Wang, Mengshi Zhang, Corina S Pasareanu, and Sarfraz Khurshid. 2018. Symbolic execution for deep neural networks. arXiv preprint arXiv:1807.10439 (2018)."},{"key":"e_1_3_2_1_32_1","unstructured":"David Grossman. 2018. Uber Self-Driving Car Kills Pedestrian in Arizona. https:\/\/www.popularmechanics.com\/technology\/infrastructure\/a19482100\/uber-self-driving-car-kills-pedestrian-in-arizona\/."},{"key":"e_1_3_2_1_33_1","volume-title":"Mask R-CNN. In Proceedings of the IEEE international conference on computer vision. 2961--2969","author":"He Kaiming","year":"2017","unstructured":"Kaiming He, Georgia Gkioxari, Piotr Doll\u00e1r, and Ross Girshick. 2017. Mask R-CNN. In Proceedings of the IEEE international conference on computer vision. 2961--2969."},{"key":"e_1_3_2_1_34_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2016.90"},{"key":"e_1_3_2_1_35_1","unstructured":"Pinjia He Clara Meister and Zhendong Su. 2020. Structure-invariant testing for machine translation. (2020)."},{"key":"e_1_3_2_1_36_1","volume-title":"Proceedings of the European Conference on Computer Vision (ECCV). 0--0.","author":"Hinterstoisser Stefan","year":"2018","unstructured":"Stefan Hinterstoisser, Vincent Lepetit, Paul Wohlhart, and Kurt Konolige. 2018. On pre-trained image features and synthetic images for deep learning. In Proceedings of the European Conference on Computer Vision (ECCV). 0--0."},{"key":"e_1_3_2_1_37_1","volume-title":"Mobilenets: Efficient convolutional neural networks for mobile vision applications. arXiv preprint arXiv:1704.04861","author":"Howard Andrew G","year":"2017","unstructured":"Andrew G Howard, Menglong Zhu, Bo Chen, Dmitry Kalenichenko, Weijun Wang, Tobias Weyand, Marco Andreetto, and Hartwig Adam. 2017. Mobilenets: Efficient convolutional neural networks for mobile vision applications. arXiv preprint arXiv:1704.04861 (2017)."},{"key":"e_1_3_2_1_38_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.1999.786990"},{"key":"e_1_3_2_1_39_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE.2019.00108"},{"key":"e_1_3_2_1_40_1","unstructured":"Neal Krawetz. 2011. Average Hash. http:\/\/www.hackerfactor.com\/blog\/index.php?\/archives\/432-Looks-Like-It.html."},{"key":"e_1_3_2_1_41_1","doi-asserted-by":"publisher","DOI":"10.1145\/3065386"},{"key":"e_1_3_2_1_42_1","unstructured":"Alina Kuznetsova Hassan Rom Neil Alldrin Jasper Uijlings Ivan Krasin Jordi Pont-Tuset Shahab Kamali Stefan Popov Matteo Malloci Tom Duerig et al. 2018. The open images dataset v4: Unified image classification object detection and visual relationship detection at scale. arXiv preprint arXiv:1811.00982 (2018)."},{"key":"e_1_3_2_1_43_1","unstructured":"Damon Lavrinc. 2018. This Is How Bad Self-Driving Cars Suck In The Rain. https:\/\/jalopnik.com\/this-is-how-bad-self-driving-cars-suck-in-the-rain-1666268433."},{"key":"e_1_3_2_1_44_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2017.472"},{"key":"e_1_3_2_1_45_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-10602-1_48"},{"key":"e_1_3_2_1_46_1","volume-title":"Deep Learning for Generic Object Detection: A Survey. CoRR abs\/1809.02165","author":"Liu Li","year":"2018","unstructured":"Li Liu, Wanli Ouyang, Xiaogang Wang, Paul W. Fieguth, Jie Chen, Xinwang Liu, and Matti Pietik\u00e4inen. 2018. Deep Learning for Generic Object Detection: A Survey. CoRR abs\/1809.02165 (2018)."},{"key":"e_1_3_2_1_47_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-46448-0_2"},{"key":"e_1_3_2_1_48_1","doi-asserted-by":"publisher","DOI":"10.1145\/3238147.3238202"},{"key":"e_1_3_2_1_49_1","doi-asserted-by":"publisher","DOI":"10.1109\/ISSRE.2018.00021"},{"key":"e_1_3_2_1_50_1","doi-asserted-by":"publisher","DOI":"10.24963\/ijcai.2020\/64"},{"key":"e_1_3_2_1_51_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2015.7299155"},{"key":"e_1_3_2_1_52_1","doi-asserted-by":"publisher","DOI":"10.1007\/s11263-016-0911-8"},{"key":"e_1_3_2_1_53_1","volume-title":"Tensorfuzz: Debugging neural networks with coverage-guided fuzzing. arXiv preprint arXiv:1807.10875","author":"Odena Augustus","year":"2018","unstructured":"Augustus Odena and Ian Goodfellow. 2018. Tensorfuzz: Debugging neural networks with coverage-guided fuzzing. arXiv preprint arXiv:1807.10875 (2018)."},{"key":"e_1_3_2_1_54_1","doi-asserted-by":"publisher","DOI":"10.1109\/TKDE.2009.191"},{"key":"e_1_3_2_1_55_1","unstructured":"Adam Paszke Sam Gross Soumith Chintala Gregory Chanan Edward Yang Zachary DeVito Zeming Lin Alban Desmaison Luca Antiga and Adam Lerer. 2017. Automatic differentiation in PyTorch. In NIPS-W."},{"key":"e_1_3_2_1_56_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.procs.2016.02.108"},{"key":"e_1_3_2_1_57_1","doi-asserted-by":"publisher","DOI":"10.1145\/3132747.3132785"},{"key":"e_1_3_2_1_58_1","doi-asserted-by":"publisher","DOI":"10.1145\/3132747.3132785"},{"key":"e_1_3_2_1_59_1","volume-title":"Exploring Invariances in Deep Convolutional Neural Networks Using Synthetic Images. CoRR abs\/1412.7122","author":"Peng Xingchao","year":"2014","unstructured":"Xingchao Peng, Baochen Sun, Karim Ali, and Kate Saenko. 2014. Exploring Invariances in Deep Convolutional Neural Networks Using Synthetic Images. CoRR abs\/1412.7122 (2014). arXiv:1412.7122 http:\/\/arxiv.org\/abs\/1412.7122"},{"key":"e_1_3_2_1_60_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-15561-1_11"},{"key":"e_1_3_2_1_61_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE.2019.00107"},{"key":"e_1_3_2_1_62_1","volume-title":"Omer Shapira, and Stan Birchfield.","author":"Prakash Aayush","year":"2018","unstructured":"Aayush Prakash, Shaad Boochoon, Mark Brophy, David Acuna, Eric Cameracci, Gavriel State, Omer Shapira, and Stan Birchfield. 2018. Structured Domain Randomization: Bridging the Reality Gap by Context-Aware Synthetic Data. arXiv preprint arXiv:1810.10093 (2018)."},{"key":"e_1_3_2_1_63_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2018.00618"},{"key":"e_1_3_2_1_64_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2016.91"},{"key":"e_1_3_2_1_65_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2017.690"},{"key":"e_1_3_2_1_66_1","volume-title":"YOLOv3: An Incremental Improvement. CoRR abs\/1804.02767","author":"Redmon Joseph","year":"2018","unstructured":"Joseph Redmon and Ali Farhadi. 2018. YOLOv3: An Incremental Improvement. CoRR abs\/1804.02767 (2018)."},{"key":"e_1_3_2_1_67_1","unstructured":"Shaoqing Ren Kaiming He Ross Girshick and Jian Sun. 2015. Faster r-cnn: Towards real-time object detection with region proposal networks. In Advances in neural information processing systems. 91--99."},{"key":"e_1_3_2_1_68_1","volume-title":"On Rendering Synthetic Images for Training an Object Detector. CoRR abs\/1411.7911","author":"Rozantsev Artem","year":"2014","unstructured":"Artem Rozantsev, Vincent Lepetit, and Pascal Fua. 2014. On Rendering Synthetic Images for Training an Object Detector. CoRR abs\/1411.7911 (2014). arXiv:1411.7911 http:\/\/arxiv.org\/abs\/1411.7911"},{"key":"e_1_3_2_1_69_1","doi-asserted-by":"crossref","unstructured":"Olga Russakovsky Jia Deng Hao Su Jonathan Krause Sanjeev Satheesh Sean Ma Zhiheng Huang Andrej Karpathy Aditya Khosla Michael Bernstein et al. 2015. Imagenet large scale visual recognition challenge. International journal of computer vision 115 3 (2015) 211--252.","DOI":"10.1007\/s11263-015-0816-y"},{"key":"e_1_3_2_1_70_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE-Companion.2019.00051"},{"key":"e_1_3_2_1_71_1","doi-asserted-by":"publisher","DOI":"10.1145\/3238147.3238172"},{"key":"e_1_3_2_1_72_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2016.308"},{"key":"e_1_3_2_1_73_1","doi-asserted-by":"publisher","DOI":"10.1145\/3180155.3180220"},{"key":"e_1_3_2_1_74_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPRW.2018.00143"},{"key":"e_1_3_2_1_75_1","doi-asserted-by":"publisher","DOI":"10.1145\/3238147.3238165"},{"key":"e_1_3_2_1_76_1","doi-asserted-by":"publisher","DOI":"10.1109\/IVS.2018.8500621"},{"key":"e_1_3_2_1_77_1","doi-asserted-by":"publisher","DOI":"10.1007\/s11263-013-0620-5"},{"key":"e_1_3_2_1_78_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE.2019.00126"},{"key":"e_1_3_2_1_79_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2009.5459207"},{"key":"e_1_3_2_1_80_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2017.324"},{"key":"e_1_3_2_1_81_1","volume-title":"Coverage-guided fuzzing for deep neural networks. arXiv preprint arXiv:1809.01266","author":"Xie Xiaofei","year":"2018","unstructured":"Xiaofei Xie, Lei Ma, Felix Juefei-Xu, Hongxu Chen, Minhui Xue, Bo Li, Yang Liu, Jianjun Zhao, Jianxiong Yin, and Simon See. 2018. Coverage-guided fuzzing for deep neural networks. arXiv preprint arXiv:1809.01266 (2018)."},{"key":"e_1_3_2_1_82_1","volume-title":"BDD100K: A Diverse Driving Video Database with Scalable Annotation Tooling. CoRR","author":"Yu Fisher","year":"2018","unstructured":"Fisher Yu, Wenqi Xian, Yingying Chen, Fangchen Liu, Mike Liao, Vashisht Madhavan, and Trevor Darrell. 2018. BDD100K: A Diverse Driving Video Database with Scalable Annotation Tooling. CoRR (2018)."},{"key":"e_1_3_2_1_83_1","doi-asserted-by":"publisher","DOI":"10.5555\/318773.318946"},{"key":"e_1_3_2_1_84_1","doi-asserted-by":"publisher","DOI":"10.1145\/3238147.3238187"},{"key":"e_1_3_2_1_85_1","doi-asserted-by":"publisher","DOI":"10.1145\/3213846.3213866"},{"key":"e_1_3_2_1_86_1","volume-title":"Object Detection with Deep Learning: A Review. CoRR","author":"Zhao Zhong-Qiu","year":"2018","unstructured":"Zhong-Qiu Zhao, Peng Zheng, Shou-tao Xu, and Xindong Wu. 2018. Object Detection with Deep Learning: A Review. CoRR (2018)."},{"key":"e_1_3_2_1_87_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2017.405"},{"key":"e_1_3_2_1_88_1","doi-asserted-by":"publisher","DOI":"10.1145\/3377811.3380422"}],"event":{"name":"ASE '20: 35th IEEE\/ACM International Conference on Automated Software Engineering","location":"Virtual Event Australia","acronym":"ASE '20","sponsor":["SIGAI ACM Special Interest Group on Artificial Intelligence","SIGSOFT ACM Special Interest Group on Software Engineering","IEEE CS"]},"container-title":["Proceedings of the 35th IEEE\/ACM International Conference on Automated Software Engineering"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3324884.3416584","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3324884.3416584","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T22:01:38Z","timestamp":1750197698000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3324884.3416584"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2020,12,21]]},"references-count":87,"alternative-id":["10.1145\/3324884.3416584","10.1145\/3324884"],"URL":"https:\/\/doi.org\/10.1145\/3324884.3416584","relation":{},"subject":[],"published":{"date-parts":[[2020,12,21]]},"assertion":[{"value":"2021-01-27","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}