{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,6,6]],"date-time":"2026-06-06T23:28:15Z","timestamp":1780788495640,"version":"3.54.1"},"reference-count":50,"publisher":"Springer Science and Business Media LLC","issue":"4","license":[{"start":{"date-parts":[[2021,1,11]],"date-time":"2021-01-11T00:00:00Z","timestamp":1610323200000},"content-version":"tdm","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"},{"start":{"date-parts":[[2021,1,11]],"date-time":"2021-01-11T00:00:00Z","timestamp":1610323200000},"content-version":"vor","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Int J Comput Vis"],"published-print":{"date-parts":[[2021,4]]},"DOI":"10.1007\/s11263-020-01423-x","type":"journal-article","created":{"date-parts":[[2021,1,11]],"date-time":"2021-01-11T08:39:42Z","timestamp":1610354382000},"page":"1185-1201","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":23,"title":["Label-Free Robustness Estimation of Object Detection CNNs for Autonomous Driving Applications"],"prefix":"10.1007","volume":"129","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-5853-5310","authenticated-orcid":false,"given":"Arvind Kumar","family":"Shekar","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Liang","family":"Gou","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Liu","family":"Ren","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Axel","family":"Wendt","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2021,1,11]]},"reference":[{"key":"1423_CR1","unstructured":"Bastani, O., Ioannou, Y., Lampropoulos, L., Vytiniotis, D., Nori, A., & Criminisi, A. Measuring neural net robustness with constraints. In Advances in neural information processing systems (pp. 2613\u20132621) (2016)."},{"key":"1423_CR2","unstructured":"Brach, K., Sick, B., & D\u00fcrr, O. (2020). Single shot mc dropout approximation. arXiv preprint arXiv:2007.03293."},{"key":"1423_CR3","unstructured":"Carlini, N., Athalye, A., Papernot, N., Brendel, W., Rauber, J., Tsipras, D., GoodfellowI., Madry, A., & Kurakin A. (2019). On evaluating adversarial robustness. arXiv preprint arXiv:1902.06705."},{"key":"1423_CR4","doi-asserted-by":"crossref","unstructured":"Cordts, M., Omran, M., Ramos, S., Rehfeld, T., Enzweiler, M., Benenson, R., Franke, U., Roth, S., & Schiele, B. (2016). The cityscapes dataset for semantic urban scene understanding. In Proceedings of the IEEE conference on computer vision and pattern recognition (pp. 3213\u20133223).","DOI":"10.1109\/CVPR.2016.350"},{"key":"1423_CR5","doi-asserted-by":"crossref","unstructured":"Deng, J., Dong, W., Socher, R., Li, L.-J., Li, K., & Fei-Fei, L. (2009). Imagenet: A large-scale hierarchical image database. In 2009 IEEE conference on computer vision and pattern recognition (pp. 248\u2013255) IEEE.","DOI":"10.1109\/CVPR.2009.5206848"},{"key":"1423_CR6","doi-asserted-by":"crossref","unstructured":"Dodge, S., & Karam, L. (2016). Understanding how image quality affects deep neural networks. In 2016 eighth international conference on quality of multimedia experience (QoMEX) (pp. 1\u20136). IEEE.","DOI":"10.1109\/QoMEX.2016.7498955"},{"issue":"4","key":"1423_CR7","doi-asserted-by":"publisher","first-page":"376","DOI":"10.14778\/3297753.3297758","volume":"12","author":"M Dolatshah","year":"2018","unstructured":"Dolatshah, M., Teoh, M., Wang, J., & Pei, J. (2018). Cleaning crowdsourced labels using oracles for statistical classification. Proceedings of the VLDB Endowment, 12(4), 376\u2013389.","journal-title":"Proceedings of the VLDB Endowment"},{"key":"1423_CR8","doi-asserted-by":"crossref","unstructured":"Feng, D., Rosenbaum, L., & Dietmayer, K. (2018).Towards safe autonomous driving: Capture uncertainty in the deep neural network for lidar 3d vehicle detection. In 2018 21st international conference on intelligent transportation systems (ITSC) (pp. 3266\u20133273). IEEE.","DOI":"10.1109\/ITSC.2018.8569814"},{"key":"1423_CR9","doi-asserted-by":"crossref","unstructured":"Fregin, A., M\u00fcller, J., Kre$$\\beta $$el, U., & Dietmayer, K. (2018). The driveu traffic light dataset: Introduction and comparison with existing datasets. In 2018 IEEE international conference on robotics and automation (ICRA) (pp. 3376\u20133383). IEEE.","DOI":"10.1109\/ICRA.2018.8460737"},{"key":"1423_CR10","unstructured":"Gal, Y., & Ghahramani, Z. (2016). Dropout as a bayesian approximation: Representing model uncertainty in deep learning. In International conference on machine learning (pp. 1050\u20131059)."},{"key":"1423_CR11","unstructured":"Goodfellow, I. J., Shlens, J., & Szegedy, C. (2014). Explaining and harnessing adversarial examples. arXiv preprint arXiv:1412.6572."},{"key":"1423_CR12","unstructured":"Google cloud data labeling service: Pricing. https:\/\/cloud.google.com\/ai-platform\/data-labeling\/pricing. Accessed: 2020-02-13."},{"key":"1423_CR13","doi-asserted-by":"crossref","unstructured":"Gopinath, D., Katz, G., P\u0103s\u0103reanu, C. S., & Barrett, C. (2018). Deepsafe: A data-driven approach for assessing robustness of neural networks. In International symposium on automated technology for verification and analysis (pp. 3\u201319). Springer.","DOI":"10.1007\/978-3-030-01090-4_1"},{"key":"1423_CR14","unstructured":"Guo, C., Pleiss, G., Sun, Y., & Weinberger, K. Q. (2017). On calibration of modern neural networks. In Proceedings of the 34th International Conference on Machine Learning - Volume 70, ICML\u201917 (pp. 1321\u20131330). JMLR.org."},{"key":"1423_CR15","unstructured":"Hein, M., & Andriushchenko, M. (2017). Formal guarantees on the robustness of a classifier against adversarial manipulation. In Advances in neural information processing systems (pp. 2266\u20132276)."},{"key":"1423_CR16","unstructured":"Hendrycks, D., & Dietterich, T. (2019). Benchmarking neural network robustness to common corruptions and perturbations. arXiv preprint arXiv:1903.12261"},{"key":"1423_CR17","doi-asserted-by":"crossref","unstructured":"Hosseini, H., Xiao, B., & Poovendran, R. (2017). Google\u2019s cloud vision api is not robust to noise. In 2017 16th IEEE international conference on machine learning and applications (ICMLA) (pp. 101\u2013105). IEEE.","DOI":"10.1109\/ICMLA.2017.0-172"},{"key":"1423_CR18","doi-asserted-by":"crossref","unstructured":"Huang, X., Kwiatkowska, M., Wang, S., & Wu, M. (2017). Safety verification of deep neural networks. In International conference on computer aided verification (pp. 3\u201329). Springer.","DOI":"10.1007\/978-3-319-63387-9_1"},{"key":"1423_CR19","doi-asserted-by":"crossref","unstructured":"Johanson, M., Belenki, S., Jalminger, J., Fant, M., & Gjertz, M. (2014). Big automotive data: Leveraging large volumes of data for knowledge-driven product development. In 2014 IEEE international conference on big data (Big Data) (pp. 736\u2013741).","DOI":"10.1109\/BigData.2014.7004298"},{"key":"1423_CR20","unstructured":"Kapishnikov, A., Bolukbasi, T., Vi\u00e9gas, F. B., & Terry, M. (2019). Segment integrated gradients: Better attributions through regions. CoRR. arXiv:1906.02825."},{"key":"1423_CR21","doi-asserted-by":"crossref","unstructured":"Katz, G., Barrett, C., Dill, D. L., Julian, K., & Kochenderfer, M. J. (2017). Reluplex: An efficient smt solver for verifying deep neural networks. In International conference on computer aided verification (pp. 97\u2013117). Springer.","DOI":"10.1007\/978-3-319-63387-9_5"},{"key":"1423_CR22","unstructured":"Katz, G., Barrett, C., Dill, D. L., Julian, K., & Kochenderfer, M. J. (2017). Towards proving the adversarial robustness of deep neural networks. arXiv preprint arXiv:1709.02802."},{"key":"1423_CR23","unstructured":"Krizhevsky, A., Hinton, G. et\u00a0al. (2009). Learning multiple layers of features from tiny images."},{"key":"1423_CR24","unstructured":"Lakshminarayanan, B., Pritzel, A., & Blundell, C. (2017). Simple and scalable predictive uncertainty estimation using deep ensembles. In Advances in neural information processing systems (pp. 6402\u20136413)."},{"issue":"4","key":"1423_CR25","doi-asserted-by":"publisher","first-page":"745","DOI":"10.1109\/TASLP.2014.2304637","volume":"22","author":"J Li","year":"2014","unstructured":"Li, J., Deng, L., Gong, Y., & Haeb-Umbach, R. (2014). An overview of noise-robust automatic speech recognition. IEEE\/ACM Transactions on Audio, Speech, and Language Processing, 22(4), 745\u2013777.","journal-title":"IEEE\/ACM Transactions on Audio, Speech, and Language Processing"},{"issue":"1","key":"1423_CR26","doi-asserted-by":"publisher","first-page":"145","DOI":"10.1109\/18.61115","volume":"37","author":"J Lin","year":"1991","unstructured":"Lin, J. (1991). Divergence measures based on the shannon entropy. IEEE Transactions on Information theory, 37(1), 145\u2013151.","journal-title":"IEEE Transactions on Information theory"},{"key":"1423_CR27","doi-asserted-by":"crossref","unstructured":"Lin, T.-Y., Maire, M., Belongie, S. J., Bourdev, L. D., Girshick, R. B., Hays, J., Perona, P., Ramanan, D., Doll\u00e1r, P., & Lawrence Zitnick, C. (2014). Microsoft COCO: Common objects in context. CoRR, abs\/1405.0312,","DOI":"10.1007\/978-3-319-10602-1_48"},{"key":"1423_CR28","unstructured":"Maglogiannis, I. G. (2007). Emerging artificial intelligence applications in computer engineering: Real word AI systems with applications in ehealth, hci, information retrieval and pervasive technologies (Vol. 160). Ios Press."},{"key":"1423_CR29","doi-asserted-by":"crossref","unstructured":"Maier-Hein, L., Mersmann, S., Kondermann, D., Stock, C., Kenngott, H. G., Sanchez, A., Wagner, M., Preukschas, A., Wekerle, A.-L., Helfert, S., et\u00a0al. (2014). Crowdsourcing for reference correspondence generation in endoscopic images. In International conference on medical image computing and computer-assisted intervention (pp. 349\u2013356). Springer.","DOI":"10.1007\/978-3-319-10470-6_44"},{"key":"1423_CR30","unstructured":"Mangal, R., Nori, A. V., Orso, A. (2019). Robustness of neural networks: A probabilistic and practical approach. In 2019 IEEE\/ACM 41st International Conference on Software Engineering: New Ideas and Emerging Results (ICSE-NIER) (pp. 93\u201396). IEEE."},{"key":"1423_CR31","doi-asserted-by":"crossref","unstructured":"Moosavi-Dezfooli, S.-M., Fawzi, A., Fawzi, O., & Frossard, P. (2017). Universal adversarial perturbations. In Proceedings of the IEEE conference on computer vision and pattern recognition (pp. 1765\u20131773).","DOI":"10.1109\/CVPR.2017.17"},{"key":"1423_CR32","doi-asserted-by":"crossref","unstructured":"Nguyen, A., Yosinski, J., & Clune, J. (2015). Deep neural networks are easily fooled: High confidence predictions for unrecognizable images. In Proceedings of the IEEE conference on computer vision and pattern recognition (pp. 427\u2013436).","DOI":"10.1109\/CVPR.2015.7298640"},{"key":"1423_CR33","doi-asserted-by":"crossref","unstructured":"Papernot, N., McDaniel, P., Goodfellow, I., Jha, S., Celik, Z. B., & Swami, A. (2017). Practical black-box attacks against machine learning. In Proceedings of the 2017 ACM on Asia conference on computer and communications security (pp. 506\u2013519).","DOI":"10.1145\/3052973.3053009"},{"key":"1423_CR34","doi-asserted-by":"crossref","unstructured":"Sandler, M., Howard, A., Zhu, M., Zhmoginov, A., & Chen, L.-C. (2018). Mobilenetv2: Inverted residuals and linear bottlenecks. In Proceedings of the IEEE conference on computer vision and pattern recognition (pp. 4510\u20134520).","DOI":"10.1109\/CVPR.2018.00474"},{"key":"1423_CR35","unstructured":"Schmidt, L., Santurkar, S., Tsipras, D., Talwar, K., & Madry, A. (2018). Adversarially robust generalization requires more data. In Advances in neural information processing systems (pp. 5014\u20135026)."},{"issue":"1","key":"1423_CR36","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1016\/j.infsof.2012.06.002","volume":"55","author":"A Shahrokni","year":"2013","unstructured":"Shahrokni, A., & Feldt, R. (2013). A systematic review of software robustness. Information and Software Technology, 55(1), 1\u201317.","journal-title":"Information and Software Technology"},{"key":"1423_CR37","doi-asserted-by":"crossref","unstructured":"Shekar, A. K., Bocklisch, T., S\u00e1nchez, P. I., Straehle, C. N., & M\u00fcller, E. (2017). Including multi-feature interactions and redundancy for feature ranking in mixed datasets. In Joint European conference on machine learning and knowledge discovery in databases (pp. 239\u2013255). Springer.","DOI":"10.1007\/978-3-319-71249-9_15"},{"key":"1423_CR38","unstructured":"Strong, A. I. (2016). Applications of artificial intelligence & associated technologies. Science [ETEBMS-2016], 5(6)."},{"key":"1423_CR39","doi-asserted-by":"crossref","unstructured":"Szegedy, C., Vanhoucke, V., Ioffe, S., Shlens, J., & Wojna, Z. (2016). Rethinking the inception architecture for computer vision. In Proceedings of the IEEE conference on computer vision and pattern recognition (pp. 2818\u20132826).","DOI":"10.1109\/CVPR.2016.308"},{"key":"1423_CR40","unstructured":"Szegedy, C., Zaremba, W., Sutskever, I., Bruna, J., Erhan, D., Goodfellow, I., & Fergus, R. (2013). Intriguing properties of neural networks. arXiv preprint arXiv:1312.6199."},{"key":"1423_CR41","doi-asserted-by":"crossref","unstructured":"Temel, D., Lee, J., & AlRegib, G. (2018). Cure-or: Challenging unreal and real environments for object recognition. In 2018 17th IEEE international conference on machine learning and applications (ICMLA) (pp. 137\u2013144) IEEE.","DOI":"10.1109\/ICMLA.2018.00028"},{"key":"1423_CR42","unstructured":"Tensorflow deeplab model zoo. https:\/\/github.com\/tensorflow\/models\/blob\/master\/research\/deeplab\/g3doc\/model_zoo.md. Accessed: 2020-28-23."},{"key":"1423_CR43","unstructured":"Tensorflow detection model zoo. https:\/\/github.com\/tensorflow\/models\/blob\/master\/research\/object_detection\/g3doc\/detection_model_zoo.md. Accessed 2020-02-20."},{"key":"1423_CR44","unstructured":"Tram\u00e9r, F., Behrmann, J., Carlini, N., Papernot, N., & Jacobsen, J.-H. (2020). Fundamental tradeoffs between invariance and sensitivity to adversarial perturbations. arXiv preprint arXiv:2002.04599."},{"key":"1423_CR45","unstructured":"Vijayanarasimhan, S., Grauman, K. (2009). What\u2019s it going to cost you?: Predicting effort vs. informativeness for multi-label image annotations. In 2009 IEEE conference on computer vision and pattern recognition (pp. 2262\u20132269). IEEE."},{"key":"1423_CR46","doi-asserted-by":"crossref","unstructured":"Von\u00a0Ahn, L., & Dabbish, L. (2004). Labeling images with a computer game. In Proceedings of the SIGCHI conference on Human factors in computing systems (pp. 319\u2013326).","DOI":"10.1145\/985692.985733"},{"key":"1423_CR47","unstructured":"Yu, F., Qin, Z., Liu, C., Zhao, L., Wang, Y., & Chen, X. (2019). Interpreting and evaluating neural network robustness. arXiv preprint arXiv:1905.04270."},{"key":"1423_CR48","unstructured":"Zhang, H., Weng, T.-W., Chen, P.-Y., Hsieh, C.-J., & Daniel, L. (2018). Efficient neural network robustness certification with general activation functions. In Advances in neural information processing systems (pp. 4939\u20134948)."},{"key":"1423_CR49","unstructured":"Zhang, J. M., Harman, M., Ma, L., & Liu, Y. (2020). Machine learning testing: Survey, landscapes and horizons. CoRR. arXiv:1906.10742."},{"issue":"11","key":"1423_CR50","doi-asserted-by":"publisher","first-page":"3212","DOI":"10.1109\/TNNLS.2018.2876865","volume":"30","author":"Z-Q Zhao","year":"2019","unstructured":"Zhao, Z.-Q., Zheng, P., Shou-tao, X., & Xindong, W. (2019). Object detection with deep learning: A review. IEEE Transactions on Neural Networks and Learning Systems, 30(11), 3212\u20133232.","journal-title":"IEEE Transactions on Neural Networks and Learning Systems"}],"container-title":["International Journal of Computer Vision"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s11263-020-01423-x.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/article\/10.1007\/s11263-020-01423-x\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s11263-020-01423-x.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2021,3,22]],"date-time":"2021-03-22T18:17:10Z","timestamp":1616437030000},"score":1,"resource":{"primary":{"URL":"http:\/\/link.springer.com\/10.1007\/s11263-020-01423-x"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2021,1,11]]},"references-count":50,"journal-issue":{"issue":"4","published-print":{"date-parts":[[2021,4]]}},"alternative-id":["1423"],"URL":"https:\/\/doi.org\/10.1007\/s11263-020-01423-x","relation":{},"ISSN":["0920-5691","1573-1405"],"issn-type":[{"value":"0920-5691","type":"print"},{"value":"1573-1405","type":"electronic"}],"subject":[],"published":{"date-parts":[[2021,1,11]]},"assertion":[{"value":"12 April 2020","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"23 December 2020","order":2,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"11 January 2021","order":3,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}}]}}