{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,12,15]],"date-time":"2025-12-15T04:21:15Z","timestamp":1765772475308,"version":"3.40.5"},"reference-count":50,"publisher":"Springer Science and Business Media LLC","issue":"6","license":[{"start":{"date-parts":[[2024,9,23]],"date-time":"2024-09-23T00:00:00Z","timestamp":1727049600000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2024,9,23]],"date-time":"2024-09-23T00:00:00Z","timestamp":1727049600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Empir Software Eng"],"published-print":{"date-parts":[[2024,11]]},"DOI":"10.1007\/s10664-024-10539-4","type":"journal-article","created":{"date-parts":[[2024,9,24]],"date-time":"2024-09-24T13:17:15Z","timestamp":1727183835000},"update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":1,"title":["Seeing the invisible: test prioritization for object detection system"],"prefix":"10.1007","volume":"29","author":[{"given":"Shihao","family":"Weng","sequence":"first","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0002-7477-3642","authenticated-orcid":false,"given":"Yang","family":"Feng","sequence":"additional","affiliation":[]},{"given":"Yining","family":"Yin","sequence":"additional","affiliation":[]},{"given":"Yuxuan","family":"Dai","sequence":"additional","affiliation":[]},{"given":"Jia","family":"Liu","sequence":"additional","affiliation":[]},{"given":"Zhihong","family":"Zhao","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2024,9,23]]},"reference":[{"key":"10539_CR1","unstructured":"VOC (2012) Pascal voc. http:\/\/host.robots.ox.ac.uk\/pascal\/VOC\/"},{"key":"10539_CR2","unstructured":"Python 3.9.15 (2022a). https:\/\/www.python.org\/downloads\/release\/python-3915\/"},{"key":"10539_CR3","unstructured":"Pytorch 1.12.0 (2022b). https:\/\/pytorch.org\/get-started\/previous-versions\/"},{"issue":"9","key":"10539_CR4","doi-asserted-by":"publisher","first-page":"865","DOI":"10.1145\/358234.381162","volume":"27","author":"J Bentley","year":"1984","unstructured":"Bentley J (1984) Programming pearls: algorithm design techniques. Commun ACM 27(9):865\u2013873","journal-title":"Commun ACM"},{"key":"10539_CR5","doi-asserted-by":"crossref","unstructured":"Brust CA, K\u00e4ding C, Denzler J (2018) Active learning for deep object detection. arXiv:1809.09875","DOI":"10.5220\/0007248600002108"},{"key":"10539_CR6","unstructured":"Chen TY, Cheung SC, Yiu SM (2020) Metamorphic testing: a new approach for generating next test cases. arXiv:2002.12543"},{"key":"10539_CR7","unstructured":"Cloud G (2022) Ai platform data labeling service pricing. https:\/\/cloud.google.com\/ai-platform\/data-labeling\/pricing"},{"key":"10539_CR8","doi-asserted-by":"crossref","unstructured":"Coates A, Ng AY (2010) Multi-camera object detection for robotics. In: 2010 IEEE International conference on robotics and automation. IEEE, pp 412\u2013419","DOI":"10.1109\/ROBOT.2010.5509644"},{"key":"10539_CR9","unstructured":"Dai J, Li Y, He K, Sun J (2016) R-fcn: Object detection via region-based fully convolutional networks. Adv Neural Inf Process Syst 29"},{"issue":"2","key":"10539_CR10","doi-asserted-by":"publisher","first-page":"159","DOI":"10.1109\/32.988497","volume":"28","author":"S Elbaum","year":"2002","unstructured":"Elbaum S, Malishevsky AG, Rothermel G (2002) Test case prioritization: a family of empirical studies. IEEE Trans Softw Eng 28(2):159\u2013182","journal-title":"IEEE Trans Softw Eng"},{"issue":"9","key":"10539_CR11","doi-asserted-by":"publisher","first-page":"1627","DOI":"10.1109\/TPAMI.2009.167","volume":"32","author":"PF Felzenszwalb","year":"2009","unstructured":"Felzenszwalb PF, Girshick RB, McAllester D, Ramanan D (2009) Object detection with discriminatively trained part-based models. IEEE Trans Pattern Anal Mach Intell 32(9):1627\u20131645","journal-title":"IEEE Trans Pattern Anal Mach Intell"},{"key":"10539_CR12","doi-asserted-by":"publisher","unstructured":"Feng Y, Shi Q, Gao X, Wan J, Fang C, Chen Z (2020) Deepgini: prioritizing massive tests to enhance the robustness of deep neural networks. In: Khurshid S, Pasareanu CS (eds) ISSTA \u201920: 29th ACM SIGSOFT International Symposium on Software Testing and Analysis, Virtual Event, USA, July 18-22, 2020. ACM, pp 177\u2013188. https:\/\/doi.org\/10.1145\/3395363.3397357","DOI":"10.1145\/3395363.3397357"},{"key":"10539_CR13","unstructured":"Feng Z, Zhang S, Takezoe R, Hu W, Chandraker M, Li LJ, Narayanan VK, Wang X (2022) Albench: A framework for evaluating active learning in object detection. arXiv:2207.13339"},{"key":"10539_CR14","doi-asserted-by":"crossref","unstructured":"Gao X, Feng Y, Yin Y, Liu Z, Chen Z, Xu B (2022) Adaptive test selection for deep neural networks. In: Proceedings of the 44th international conference on software engineering, pp 73\u201385","DOI":"10.1145\/3510003.3510232"},{"key":"10539_CR15","doi-asserted-by":"crossref","unstructured":"Geiger A, Lenz P, Stiller C, Urtasun R (2013) Vision meets robotics: The kitti dataset. International Journal of Robotics Research (IJRR)","DOI":"10.1177\/0278364913491297"},{"key":"10539_CR16","doi-asserted-by":"crossref","unstructured":"Haussmann E, Fenzi M, Chitta K, Ivanecky J, Xu H, Roy D, Mittel A, Koumchatzky N, Farabet C, Alvarez JM (2020) Scalable active learning for object detection. In: 2020 IEEE intelligent vehicles symposium (iv). IEEE, pp 1430\u20131435","DOI":"10.1109\/IV47402.2020.9304793"},{"key":"10539_CR17","unstructured":"Hawkins AJ (2019) Tesla\u2019s autopilot was engaged when model 3 crashed into truck, report states. https:\/\/www.theverge.com\/2019\/5\/16\/18627766\/tesla-autopilot-fatal-crash-delray-florida-ntsb-model-3"},{"issue":"9","key":"10539_CR18","doi-asserted-by":"publisher","first-page":"1904","DOI":"10.1109\/TPAMI.2015.2389824","volume":"37","author":"K He","year":"2015","unstructured":"He K, Zhang X, Ren S, Sun J (2015) Spatial pyramid pooling in deep convolutional networks for visual recognition. IEEE Trans Pattern Anal Mach Intell 37(9):1904\u20131916","journal-title":"IEEE Trans Pattern Anal Mach Intell"},{"key":"10539_CR19","doi-asserted-by":"crossref","unstructured":"Hoiem D, Chodpathumwan Y, Dai Q (2012) Diagnosing error in object detectors. In: European conference on computer vision, Springer, pp 340\u2013353","DOI":"10.1007\/978-3-642-33712-3_25"},{"key":"10539_CR20","doi-asserted-by":"crossref","unstructured":"Huang J, Rathod V, Sun C, Zhu M, Korattikara A, Fathi A, Fischer I, Wojna Z, Song Y, Guadarrama S et\u00a0al (2017) Speed\/accuracy trade-offs for modern convolutional object detectors. In: Proceedings of the IEEE conference on computer vision and pattern recognition, pp 7310\u20137311","DOI":"10.1109\/CVPR.2017.351"},{"key":"10539_CR21","doi-asserted-by":"crossref","unstructured":"Kang D, Arechiga N, Pillai S, Bailis PD, Zaharia M (2022) Finding label and model errors in perception data with learned observation assertions. In: Proceedings of the 2022 international conference on management of data, pp 496\u2013505","DOI":"10.1145\/3514221.3517907"},{"key":"10539_CR22","doi-asserted-by":"crossref","unstructured":"Kao CC, Lee TY, Sen P, Liu MY (2018) Localization-aware active learning for object detection. In: Asian conference on computer vision. Springer, pp 506\u2013522","DOI":"10.1007\/978-3-030-20876-9_32"},{"key":"10539_CR23","doi-asserted-by":"publisher","unstructured":"Kim J, Feldt R, Yoo S (2019) Guiding deep learning system testing using surprise adequacy. In: Atlee JM, Bultan T, Whittle J (eds) Proceedings of the 41st International Conference on Software Engineering, ICSE 2019, Montreal, QC, Canada, May 25-31, 2019. IEEE \/ ACM, pp 1039\u20131049. https:\/\/doi.org\/10.1109\/ICSE.2019.00108","DOI":"10.1109\/ICSE.2019.00108"},{"key":"10539_CR24","doi-asserted-by":"crossref","unstructured":"LeCun Y, Bengio Y, Hinton GE (2015) Deep learning. Nature","DOI":"10.1038\/nature14539"},{"key":"10539_CR25","doi-asserted-by":"crossref","unstructured":"Lin TY, Goyal P, Girshick R, He K, Doll\u00e1r P (2017) Focal loss for dense object detection. In: Proceedings of the IEEE international conference on computer vision, pp 2980\u20132988","DOI":"10.1109\/ICCV.2017.324"},{"key":"10539_CR26","doi-asserted-by":"crossref","unstructured":"Liu W, Anguelov D, Erhan D, Szegedy C, Reed S, Fu CY, Berg AC (2016) Ssd: Single shot multibox detector. In: European conference on computer vision, Springer, pp 21\u201337","DOI":"10.1007\/978-3-319-46448-0_2"},{"key":"10539_CR27","doi-asserted-by":"publisher","unstructured":"Liu Z, Feng Y, Yin Y, Chen Z (2022) Deepstate: Selecting test suites to enhance the robustness of recurrent neural networks. In: 44th IEEE\/ACM 44th International conference on software engineering, ICSE 2022, Pittsburgh, PA, USA, May 25-27, 2022. ACM, pp 598\u2013609. https:\/\/doi.org\/10.1145\/3510003.3510231","DOI":"10.1145\/3510003.3510231"},{"key":"10539_CR28","doi-asserted-by":"crossref","unstructured":"Ma J, Ushiku Y, Sagara M (2022) The effect of improving annotation quality on object detection datasets: A preliminary study. In: Proceedings Of The IEEE\/CVF conference on computer vision and pattern recognition, pp 4850\u20134859","DOI":"10.1109\/CVPRW56347.2022.00532"},{"key":"10539_CR29","unstructured":"McFarland M (2020) Uber self-driving car operator charged in pedestrian death. https:\/\/www.cnn.com\/2020\/09\/18\/cars\/uber-vasquez-charged\/index.html"},{"key":"10539_CR30","unstructured":"Microsoft (2017) Ms coco. https:\/\/cocodataset.org\/"},{"key":"10539_CR31","doi-asserted-by":"crossref","unstructured":"Padilla R, Netto SL, Da\u00a0Silva EA (2020) A survey on performance metrics for object-detection algorithms. In: 2020 international conference on systems, signals and image processing (IWSSIP). IEEE, pp 237\u2013242","DOI":"10.1109\/IWSSIP48289.2020.9145130"},{"issue":"2","key":"10539_CR32","doi-asserted-by":"publisher","first-page":"29","DOI":"10.1007\/s10664-021-10066-6","volume":"27","author":"R Pan","year":"2022","unstructured":"Pan R, Bagherzadeh M, Ghaleb TA, Briand L (2022) Test case selection and prioritization using machine learning: a systematic literature review. Empir Softw Eng 27(2):29","journal-title":"Empir Softw Eng"},{"key":"10539_CR33","doi-asserted-by":"crossref","unstructured":"Papadopoulos DP, Uijlings J, Keller F, Ferrari V (2016) We don\u2019t need no bounding-boxes: Training object class detectors using only human verification. computer vision and pattern recognition","DOI":"10.1109\/CVPR.2016.99"},{"key":"10539_CR34","doi-asserted-by":"crossref","unstructured":"Qu X, Cohen MB, Woolf KM (2007) Combinatorial interaction regression testing: A study of test case generation and prioritization. In: 2007 IEEE international conference on software maintenance. IEEE, pp 255\u2013264","DOI":"10.1109\/ICSM.2007.4362638"},{"key":"10539_CR35","doi-asserted-by":"crossref","unstructured":"Real E, Shlens J, Mazzocchi S, Pan X, Vanhoucke V (2017) Youtube-boundingboxes: A large high-precision human-annotated data set for object detection in video. In: Proceedings of the IEEE conference on computer vision and pattern recognition, pp 5296\u20135305","DOI":"10.1109\/CVPR.2017.789"},{"key":"10539_CR36","doi-asserted-by":"crossref","unstructured":"Redmon J, Divvala S, Girshick R, Farhadi A (2016) You only look once: Unified, real-time object detection. In: Proceedings of the IEEE conference on computer vision and pattern recognition, pp 779\u2013788","DOI":"10.1109\/CVPR.2016.91"},{"key":"10539_CR37","unstructured":"Ren S, He K, Girshick R, Sun J (2015) Faster r-cnn: Towards real-time object detection with region proposal networks. Adv Neural Inf Process Syst 28"},{"issue":"10","key":"10539_CR38","doi-asserted-by":"publisher","first-page":"929","DOI":"10.1109\/32.962562","volume":"27","author":"G Rothermel","year":"2001","unstructured":"Rothermel G, Untch RH, Chu C, Harrold MJ (2001) Prioritizing test cases for regression testing. IEEE Trans Softw Eng 27(10):929\u2013948","journal-title":"IEEE Trans Softw Eng"},{"key":"10539_CR39","unstructured":"Roy S, Unmesh A, Namboodiri VP (2018) Deep active learning for object detection. In: BMVC, p\u00a091"},{"key":"10539_CR40","unstructured":"Su H, Deng J, Fei-Fei L (2012) Crowdsourcing annotations for visual object detection. In: Workshops at the Twenty-Sixth AAAI Conference on Artificial Intelligence"},{"key":"10539_CR41","doi-asserted-by":"publisher","first-page":"191","DOI":"10.1016\/S1571-0661(04)00313-5","volume":"61","author":"T Takaoka","year":"2002","unstructured":"Takaoka T (2002) Efficient algorithms for the maximum subarray problem by distance matrix multiplication. Electron Notes Theor Comput Sci 61:191\u2013200","journal-title":"Electron Notes Theor Comput Sci"},{"key":"10539_CR42","doi-asserted-by":"crossref","unstructured":"Tian Z, Shen C, Chen H, He T (2019) Fcos: Fully convolutional one-stage object detection. In: Proceedings of the IEEE\/CVF international conference on computer vision, pp 9627\u20139636","DOI":"10.1109\/ICCV.2019.00972"},{"key":"10539_CR43","doi-asserted-by":"publisher","unstructured":"Wang Z, You H, Chen J, Zhang Y, Dong X, Zhang W (2021) Prioritizing test inputs for deep neural networks via mutation analysis. In: 43rd IEEE\/ACM International Conference on Software Engineering, ICSE 2021, Madrid, Spain, 22-30 May 2021. IEEE, pp 397\u2013409. https:\/\/doi.org\/10.1109\/ICSE43902.2021.00046","DOI":"10.1109\/ICSE43902.2021.00046"},{"key":"10539_CR44","doi-asserted-by":"crossref","unstructured":"Weng S, Feng Y, Yin Y, Liu J (2023) Prioritizing testing instances to enhance the robustness of object detection systems. In: Proceedings of the 14th asia-pacific symposium on Internetware, pp 194\u2013204","DOI":"10.1145\/3609437.3609446"},{"key":"10539_CR45","doi-asserted-by":"crossref","unstructured":"Wu J, Chen J, Huang D (2022) Entropy-based active learning for object detection with progressive diversity constraint. In: Proceedings of the IEEE\/CVF conference on computer vision and pattern recognition, pp 9397\u20139406","DOI":"10.1109\/CVPR52688.2022.00918"},{"key":"10539_CR46","doi-asserted-by":"crossref","unstructured":"Yu W, Zhu S, Yang T, Chen C (2022) Consistency-based active learning for object detection. In: Proceedings of the IEEE\/CVF conference on computer vision and pattern recognition, pp 3951\u20133960","DOI":"10.1109\/CVPRW56347.2022.00440"},{"key":"10539_CR47","doi-asserted-by":"publisher","first-page":"58443","DOI":"10.1109\/ACCESS.2020.2983149","volume":"8","author":"E Yurtsever","year":"2020","unstructured":"Yurtsever E, Lambert J, Carballo A, Takeda K (2020) A survey of autonomous driving: common practices and emerging technologies. IEEE Access 8:58443\u201358469. https:\/\/doi.org\/10.1109\/ACCESS.2020.2983149","journal-title":"IEEE Access"},{"issue":"2","key":"10539_CR48","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1109\/TSE.2019.2962027","volume":"48","author":"JM Zhang","year":"2022","unstructured":"Zhang JM, Harman M, Ma L, Liu Y (2022) Machine learning testing: survey, landscapes and horizons. IEEE Trans Software Eng 48(2):1\u20133. https:\/\/doi.org\/10.1109\/TSE.2019.2962027","journal-title":"IEEE Trans Software Eng"},{"issue":"4","key":"10539_CR49","doi-asserted-by":"publisher","first-page":"399","DOI":"10.1145\/954339.954342","volume":"35","author":"W Zhao","year":"2003","unstructured":"Zhao W, Chellappa R, Phillips PJ, Rosenfeld A (2003) Face recognition: a literature survey. ACM Comput Surv 35(4):399\u2013458. https:\/\/doi.org\/10.1145\/954339.954342","journal-title":"ACM Comput Surv"},{"key":"10539_CR50","unstructured":"Zou Z, Shi Z, Guo Y, Ye J (2019) Object detection in 20 years: a survey. Computer Vision and Pattern Recognition"}],"container-title":["Empirical Software Engineering"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10664-024-10539-4.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s10664-024-10539-4\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10664-024-10539-4.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,11,6]],"date-time":"2024-11-06T04:53:18Z","timestamp":1730868798000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s10664-024-10539-4"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,9,23]]},"references-count":50,"journal-issue":{"issue":"6","published-print":{"date-parts":[[2024,11]]}},"alternative-id":["10539"],"URL":"https:\/\/doi.org\/10.1007\/s10664-024-10539-4","relation":{},"ISSN":["1382-3256","1573-7616"],"issn-type":[{"type":"print","value":"1382-3256"},{"type":"electronic","value":"1573-7616"}],"subject":[],"published":{"date-parts":[[2024,9,23]]},"assertion":[{"value":"12 August 2024","order":1,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"23 September 2024","order":2,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Declarations"}},{"value":"Each of the authors confirms that this manuscript has not been previously published and is not currently under consideration by any other venues. Additionally, all the authors have approved the contents of this paper and have agreed to the Empirical Software Engineering Journal\u2019s submission policies. The manuscript is prepared following the instructions to authors. Each named author has substantially contributed to conducting the underlying research and drafting this manuscript. To the best of our knowledge, all the named authors have no conflict of interest, financial or otherwise.","order":2,"name":"Ethics","group":{"name":"EthicsHeading","label":"Conflict of Interests"}}],"article-number":"154"}}