{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,6,5]],"date-time":"2026-06-05T07:32:31Z","timestamp":1780644751856,"version":"3.54.1"},"reference-count":52,"publisher":"Springer Science and Business Media LLC","issue":"10","license":[{"start":{"date-parts":[[2021,7,6]],"date-time":"2021-07-06T00:00:00Z","timestamp":1625529600000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2021,7,6]],"date-time":"2021-07-06T00:00:00Z","timestamp":1625529600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"funder":[{"name":"Government of the Russian Federation","award":["08-08"],"award-info":[{"award-number":["08-08"]}]}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Mach Learn"],"published-print":{"date-parts":[[2023,10]]},"DOI":"10.1007\/s10994-021-05994-9","type":"journal-article","created":{"date-parts":[[2021,7,6]],"date-time":"2021-07-06T15:04:29Z","timestamp":1625583869000},"page":"3977-4012","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":15,"title":["Metrics and methods for robustness evaluation of neural networks with generative models"],"prefix":"10.1007","volume":"112","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-3713-6051","authenticated-orcid":false,"given":"Igor","family":"Buzhinsky","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Arseny","family":"Nerinovsky","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Stavros","family":"Tripakis","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2021,7,6]]},"reference":[{"key":"5994_CR1","doi-asserted-by":"publisher","first-page":"14410","DOI":"10.1109\/ACCESS.2018.2807385","volume":"6","author":"N Akhtar","year":"2018","unstructured":"Akhtar, N., & Mian, A. (2018). Threat of adversarial attacks on deep learning in computer vision: A survey. IEEE Access, 6, 14410\u201314430.","journal-title":"IEEE Access"},{"key":"5994_CR2","unstructured":"Amadou\u00a0Dia, O., Barshan, E., & Babanezhad, R. (2019). Semantics preserving adversarial attacks. arXiv preprint arXiv:190303905v5."},{"key":"5994_CR3","doi-asserted-by":"crossref","unstructured":"Anderson, G., Pailoor, S., Dillig, I., & Chaudhuri, S. (2019). Optimization and abstraction: A synergistic approach for analyzing neural network robustness. In 40th ACM SIGPLAN Conference on Programming Language Design and Implementation, ACM, pp. 731\u2013744.","DOI":"10.1145\/3314221.3314614"},{"key":"5994_CR4","unstructured":"Arjovsky, M., Chintala, S., & Bottou, L. (2017). Wasserstein generative adversarial networks. In International Conference on Machine Learning, pp. 214\u2013223."},{"key":"5994_CR5","first-page":"2613","volume-title":"Advances in Neural Information Processing Systems","author":"O Bastani","year":"2016","unstructured":"Bastani, O., Ioannou, Y., Lampropoulos, L., Vytiniotis, D., Nori, A., & Criminisi, A. (2016). Measuring neural net robustness with constraints. In D. Lee, M. Sugiyama, U. Luxburg, I. Guyon, & R. Garnett (Eds.), Advances in Neural Information Processing Systems (pp. 2613\u20132621). Curran Associates, Inc."},{"key":"5994_CR6","doi-asserted-by":"crossref","unstructured":"Bau, D., Zhu, J.-Y., Wulff, J., Peebles, W., Strobelt, H., Zhou, B., & Torralba, A. (2019). Seeing what a GAN cannot generate. In IEEE International Conference on Computer Vision, pp. 4502\u20134511.","DOI":"10.1109\/ICCV.2019.00460"},{"key":"5994_CR7","doi-asserted-by":"publisher","first-page":"3240","DOI":"10.1609\/aaai.v33i01.33013240","volume":"33","author":"A Boopathy","year":"2019","unstructured":"Boopathy, A., Weng, T.-W., Chen, P.-Y., Liu, S., & Daniel, L. (2019). CNN-Cert: An efficient framework for certifying robustness of convolutional neural networks. AAAI Conference on Artificial Intelligence, 33, 3240\u20133247.","journal-title":"AAAI Conference on Artificial Intelligence"},{"key":"5994_CR8","unstructured":"Brock, A., Donahue, J., & Simonyan, K. (2018). Large scale GAN training for high fidelity natural image synthesis. In International Conference on Learning Representations."},{"key":"5994_CR9","doi-asserted-by":"crossref","unstructured":"Dalvi, N., Domingos, P., Sanghai, S., & Verma, D., et\u00a0al. (2004). Adversarial classification. In 10th ACM SIGKDD International Conference on Knowledge Discovery and Data Mining, ACM, pp. 99\u2013108.","DOI":"10.1145\/1014052.1014066"},{"key":"5994_CR10","doi-asserted-by":"crossref","unstructured":"Dreossi, T., Jha, S., & Seshia, S. A. (2018). Semantic adversarial deep learning. In International Conference on Computer Aided Verification, Springer, pp. 3\u201326.","DOI":"10.1007\/978-3-319-96145-3_1"},{"key":"5994_CR11","doi-asserted-by":"crossref","unstructured":"Dutta, S., Jha, S., Sanakaranarayanan, S., & Tiwari, A. (2017). Output range analysis for deep neural networks. arXiv preprint arXiv:170909130.","DOI":"10.1007\/978-3-319-77935-5_9"},{"key":"5994_CR12","doi-asserted-by":"crossref","unstructured":"Elboher, Y.\u00a0Y., Gottschlich, J., & Katz, G. (2020). An abstraction-based framework for neural network verification. In International Conference on Computer Aided Verification,  Springer, pp. 43\u201365.","DOI":"10.1007\/978-3-030-53288-8_3"},{"key":"5994_CR13","unstructured":"Engstrom, L., Tran, B., Tsipras, D., Schmidt, L., & Madry, A. (2019). Exploring the landscape of spatial robustness. In International Conference on Machine Learning, pp. 1802\u20131811."},{"issue":"3","key":"5994_CR14","doi-asserted-by":"publisher","first-page":"481","DOI":"10.1007\/s10994-017-5663-3","volume":"107","author":"A Fawzi","year":"2018","unstructured":"Fawzi, A., Fawzi, O., & Frossard, P. (2018). Analysis of classifiers\u2019 robustness to adversarial perturbations. Machine Learning, 107(3), 481\u2013508.","journal-title":"Machine Learning"},{"key":"5994_CR15","unstructured":"Gilmer, J., Adams, R.\u00a0P., Goodfellow, I., Andersen, D., & Dahl, G.\u00a0E. (2018). Motivating the rules of the game for adversarial example research. arXiv preprint arXiv:180706732."},{"key":"5994_CR16","unstructured":"Gilmer, J., Ford, N., Carlini, N., & Cubuk, E. (2019). Adversarial examples are a natural consequence of test error in noise. In 36th International Conference on Machine Learning, pp. 2280\u20132289."},{"key":"5994_CR17","doi-asserted-by":"crossref","unstructured":"Globerson, A., & Roweis, S. (2006). Nightmare at test time: Robust learning by feature deletion. In International Conference on Machine Learning, ACM, pp.  353\u2013360.","DOI":"10.1145\/1143844.1143889"},{"key":"5994_CR18","unstructured":"Goodfellow, I. (2018). Gradient masking causes CLEVER to overestimate adversarial perturbation size. arXiv preprint arXiv:180407870."},{"key":"5994_CR19","first-page":"2672","volume-title":"Advances in Neural Information Processing Systems","author":"I Goodfellow","year":"2014","unstructured":"Goodfellow, I., Pouget-Abadie, J., Mirza, M., Xu, B., Warde-Farley, D., Ozair, S., Courville, A., & Bengio, Y. (2014). Generative adversarial nets. In Z. Ghahramani, M. Welling, C. Cortes, N. Lawrence, K. Q. Weinberger (Eds.), Advances in Neural Information Processing Systems (pp. 2672\u20132680). Curran Associates, Inc."},{"key":"5994_CR20","doi-asserted-by":"crossref","unstructured":"Gopinath, D., Katz, G., Pasareanu, C.\u00a0S., & Barrett, C. (2017). DeepSafe: A data-driven approach for checking adversarial robustness in neural networks. arXiv preprint arXiv:171000486.","DOI":"10.1007\/978-3-030-01090-4_1"},{"key":"5994_CR21","unstructured":"Gu, K., Yang, B., Ngiam, J., Le, Q., & Shlens, J. (2019). Using videos to evaluate image model robustness. In 7th International Conference on Learning Representations."},{"key":"5994_CR22","doi-asserted-by":"crossref","unstructured":"Heljakka, A., Solin, A., & Kannala, J. (2018). Pioneer networks: Progressively growing generative autoencoder. In Asian Conference on Computer Vision, Springer, pp. 22\u201338.","DOI":"10.1007\/978-3-030-20887-5_2"},{"key":"5994_CR23","doi-asserted-by":"crossref","unstructured":"Heljakka, A., Solin, A., & Kannala, J. (2020). Towards photographic image manipulation with balanced growing of generative autoencoders. In IEEE Winter Conference on Applications of Computer Vision (WACV).","DOI":"10.1109\/WACV45572.2020.9093375"},{"key":"5994_CR24","unstructured":"Hendrycks, D., & Dietterich, T. (2019). Benchmarking neural network robustness to common corruptions and perturbations. In 7th International Conference on Learning Representations."},{"key":"5994_CR25","unstructured":"Hendrycks, D., Zhao, K., Basart, S., Steinhardt, J., & Song, D. (2019). Natural adversarial examples. arXiv preprint arXiv:190707174."},{"key":"5994_CR26","first-page":"6626","volume-title":"Advances in Neural Information Processing Systems","author":"M Heusel","year":"2017","unstructured":"Heusel, M., Ramsauer, H., Unterthiner, T., Nessler, B., & Hochreiter, S. (2017). GANs trained by a two time-scale update rule converge to a local Nash equilibrium. In I. Guyon, U. V. Luxburg, S. Bengio, H. Wallach, R. Fergus, S. Vishwanathan, R. Garnett (Eds.), Advances in Neural Information Processing Systems (pp. 6626\u20136637). Curran Associates, Inc."},{"key":"5994_CR27","doi-asserted-by":"crossref","unstructured":"Huang, X., Kwiatkowska, M., Wang, S., & Wu, M. (2017). Safety verification of deep neural networks. In International Conference on Computer Aided Verification, Springer, pp. 3\u201329.","DOI":"10.1007\/978-3-319-63387-9_1"},{"key":"5994_CR28","unstructured":"Ilyas, A., Santurkar, S., Tsipras, D., Engstrom, L., Tran, B., & Madry, A. (2019). Adversarial examples are not bugs, they are features. In Advances in Neural Information Processing Systems, pp. 125\u2013136."},{"key":"5994_CR29","unstructured":"Jalal, A., Ilyas, A., Asteri, E., Daskalakis, C., & Dimakis, A.\u00a0G. (2019). The robust manifold defense: Adversarial training using generative models. arXiv preprint arXiv:171209196."},{"key":"5994_CR30","doi-asserted-by":"crossref","unstructured":"Katz, G., Barrett, C., Dill, D.\u00a0L., Julian, K., & Kochenderfer, M.\u00a0J. (2017). Reluplex: An efficient SMT solver for verifying deep neural networks. In International Conference on Computer Aided Verification, Springer, pp. 97\u2013117.","DOI":"10.1007\/978-3-319-63387-9_5"},{"key":"5994_CR31","doi-asserted-by":"crossref","unstructured":"Katz, G., Huang, D.\u00a0A., Ibeling, D., Julian, K., Lazarus, C., Lim, R., Shah, P., Thakoor, S., Wu, H., & Zelji\u0107, A., et\u00a0al. (2019). The Marabou framework for verification and analysis of deep neural networks. In International Conference on Computer Aided Verification, Springer, pp. 443\u2013452.","DOI":"10.1007\/978-3-030-25540-4_26"},{"key":"5994_CR32","unstructured":"LeCun, Y. (1998). The MNIST database of handwritten digits."},{"key":"5994_CR33","doi-asserted-by":"crossref","unstructured":"Liu, Z., Luo, P., Wang, X., & Tang, X. (2015). Deep learning face attributes in the wild. In IEEE International Conference on Computer Vision, pp. 3730\u20133738.","DOI":"10.1109\/ICCV.2015.425"},{"key":"5994_CR34","unstructured":"Madry, A., Makelov, A., Schmidt, L., Tsipras, D., & Vladu, A. (2018). Towards deep learning models resistant to adversarial attacks. In 6th International Conference on Learning Representations."},{"key":"5994_CR35","unstructured":"Makhzani, A., Shlens, J., Jaitly, N., Goodfellow, I., & Frey, B. (2015). Adversarial autoencoders. arXiv preprint arXiv:151105644."},{"key":"5994_CR36","doi-asserted-by":"crossref","unstructured":"Mirman, M., Gehr, T., & Vechev, M. (2020). Robustness certification of generative models. arXiv preprint arXiv:200414756.","DOI":"10.1145\/3453483.3454100"},{"key":"5994_CR37","doi-asserted-by":"crossref","unstructured":"Moosavi-Dezfooli, S.-M., Fawzi, A., & Frossard, P. (2016). DeepFool: A simple and accurate method to fool deep neural networks. In IEEE Conference on Computer Vision and Pattern Recognition, pp. 2574\u20132582.","DOI":"10.1109\/CVPR.2016.282"},{"key":"5994_CR38","unstructured":"Odena, A., Olah, C., & Shlens, J. (2017). Conditional image synthesis with auxiliary classifier GANs. In 34th International Conference on Machine Learning, JMLR.org, pp. 2642\u20132651"},{"key":"5994_CR39","doi-asserted-by":"crossref","unstructured":"Ruan, W., Huang, X., & Kwiatkowska, M. (2018). Reachability analysis of deep neural networks with provable guarantees. arXiv preprint arXiv:180502242.","DOI":"10.24963\/ijcai.2018\/368"},{"issue":"3","key":"5994_CR40","doi-asserted-by":"publisher","first-page":"211","DOI":"10.1007\/s11263-015-0816-y","volume":"115","author":"O Russakovsky","year":"2015","unstructured":"Russakovsky, O., Deng, J., Su, H., Krause, J., Satheesh, S., Ma, S., et al. (2015). ImageNet large scale visual recognition challenge. International Journal of Computer Vision (IJCV), 115(3), 211\u2013252.","journal-title":"International Journal of Computer Vision (IJCV)"},{"key":"5994_CR41","unstructured":"Samangouei, P., Kabkab, M., & Chellappa, R. (2018). Defense-GAN: Protecting classifiers against adversarial attacks using generative models. arXiv preprint arXiv:180506605."},{"key":"5994_CR42","unstructured":"Santurkar, S., Ilyas, A., Tsipras, D., Engstrom, L., Tran, B., & Madry, A. (2019). Image synthesis with a single (robust) classifier. In Advances in Neural Information Processing Systems, 1260\u20131271."},{"key":"5994_CR43","doi-asserted-by":"crossref","unstructured":"Singh, G., Gehr, T., P\u00fcschel, M., & Vechev, M. (2019). An abstract domain for certifying neural networks. In Proceedings of the ACM on Programming Languages 3(POPL):41.","DOI":"10.1145\/3290354"},{"key":"5994_CR44","unstructured":"Song, Y., Kim, T., Nowozin, S., Ermon, S., & Kushman, N. (2018). PixelDefend: Leveraging generative models to understand and defend against adversarial examples. In 6th International Conference on Learning Representations."},{"key":"5994_CR45","first-page":"8312","volume-title":"Advances in Neural Information Processing Systems","author":"Y Song","year":"2018","unstructured":"Song, Y., Shu, R., Kushman, N., & Ermon, S. (2018). Constructing unrestricted adversarial examples with generative models. In S. Bengio, H. Wallach, H. Larochelle, K. Grauman, N. Cesa-Bianchi, & R. Garnett (Eds.), Advances in Neural Information Processing Systems (pp. 8312\u20138323). Curran Associates, Inc"},{"key":"5994_CR46","unstructured":"Szegedy, C., Zaremba, W., Sutskever, I., Estrach, J.\u00a0B., Erhan, D., Goodfellow, I., & Fergus, R. (2014). Intriguing properties of neural networks. In 2nd International Conference on Learning Representations."},{"key":"5994_CR47","unstructured":"Tsipras, D., Santurkar, S., Engstrom, L., Turner, A., & Madry, A. (2018). Robustness may be at odds with accuracy. In 6th International Conference on Learning Representations."},{"key":"5994_CR48","volume-title":"The nature of statistical learning theory","author":"V Vapnik","year":"2013","unstructured":"Vapnik, V. (2013). The nature of statistical learning theory. Springer Science & Business Media."},{"key":"5994_CR49","unstructured":"Weng, T.-W., Zhang, H., Chen, P.-Y., Yi, J., Su, D., Gao, Y., Hsieh, C.-J., Daniel, L. (2018). Evaluating the robustness of neural networks: An extreme value theory approach. arXiv preprint arXiv:180110578."},{"key":"5994_CR50","unstructured":"Yu, F., Seff, A., Zhang, Y., Song, S., Funkhouser, T., &  Xiao, J. (2015). LSUN: Construction of a large-scale image dataset using deep learning with humans in the loop. arXiv preprint arXiv:150603365."},{"key":"5994_CR51","doi-asserted-by":"crossref","unstructured":"Yu, F., Qin, Z., Liu, C., Zhao, L., Wang, Y., & Chen, X. (2019). Interpreting and evaluating neural network robustness. arXiv preprint arXiv:190504270.","DOI":"10.24963\/ijcai.2019\/583"},{"key":"5994_CR52","unstructured":"Zhao, Z., Dua, D., & Singh, S. (2017). Generating natural adversarial examples. arXiv preprint arXiv:171011342."}],"container-title":["Machine Learning"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10994-021-05994-9.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s10994-021-05994-9\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10994-021-05994-9.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2023,9,7]],"date-time":"2023-09-07T22:04:03Z","timestamp":1694124243000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s10994-021-05994-9"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2021,7,6]]},"references-count":52,"journal-issue":{"issue":"10","published-print":{"date-parts":[[2023,10]]}},"alternative-id":["5994"],"URL":"https:\/\/doi.org\/10.1007\/s10994-021-05994-9","relation":{},"ISSN":["0885-6125","1573-0565"],"issn-type":[{"value":"0885-6125","type":"print"},{"value":"1573-0565","type":"electronic"}],"subject":[],"published":{"date-parts":[[2021,7,6]]},"assertion":[{"value":"16 March 2020","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"20 January 2021","order":2,"name":"revised","label":"Revised","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"10 May 2021","order":3,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"6 July 2021","order":4,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"This content has been made available to all.","name":"free","label":"Free to read"}]}}