{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,11,11]],"date-time":"2025-11-11T13:29:38Z","timestamp":1762867778724,"version":"3.37.3"},"reference-count":43,"publisher":"Springer Science and Business Media LLC","issue":"2","license":[{"start":{"date-parts":[[2019,4,12]],"date-time":"2019-04-12T00:00:00Z","timestamp":1555027200000},"content-version":"tdm","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Prog Artif Intell"],"published-print":{"date-parts":[[2019,6]]},"DOI":"10.1007\/s13748-019-00179-x","type":"journal-article","created":{"date-parts":[[2019,4,12]],"date-time":"2019-04-12T11:04:07Z","timestamp":1555067047000},"page":"273-285","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":17,"title":["Improving transparency of deep neural inference process"],"prefix":"10.1007","volume":"8","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-0731-8057","authenticated-orcid":false,"given":"Hiroshi","family":"Kuwajima","sequence":"first","affiliation":[]},{"given":"Masayuki","family":"Tanaka","sequence":"additional","affiliation":[]},{"given":"Masatoshi","family":"Okutomi","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2019,4,12]]},"reference":[{"issue":"7","key":"179_CR1","doi-asserted-by":"publisher","first-page":"e0130,140","DOI":"10.1371\/journal.pone.0130140","volume":"10","author":"S Bach","year":"2015","unstructured":"Bach, S., Binder, A., Montavon, G., Klauschen, F., M\u00fcller, K.R., Samek, W.: On pixel-wise explanations for non-linear classifier decisions by layer-wise relevance propagation. PLoS ONE 10(7), e0130,140 (2015). \n                    https:\/\/doi.org\/10.1371\/journal.pone.0130140","journal-title":"PLoS ONE"},{"key":"179_CR2","doi-asserted-by":"crossref","unstructured":"Bau, D., Zhou, B., Khosla, A., Oliva, A., Torralba, A.: Network dissection: quantifying interpretability of deep visual representations. In: Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition (2017)","DOI":"10.1109\/CVPR.2017.354"},{"key":"179_CR3","doi-asserted-by":"publisher","unstructured":"Binder, A., Montavon, G., Lapuschkin, S., M\u00fcller, K., Samek, W.: Layer-wise relevance propagation for neural networks with local renormalization layers. In: Villa, A. E. P., Masulli, P., Rivero, A. J. P. (eds.) Artificial Neural Networks and Machine Learning\u2014ICANN 2016\u201425th International Conference on Artificial Neural Networks, Barcelona, Spain, 6\u20139 Sept, 2016, Proceedings, Part II, Springer, Lecture Notes in Computer Science, vol 9887, pp. 63\u201371 (2016) \n                    https:\/\/doi.org\/10.1007\/978-3-319-44781-0_8","DOI":"10.1007\/978-3-319-44781-0_8"},{"key":"179_CR4","unstructured":"Bojarski, M., Testa, D.D., Dworakowski, D., Firner, B., Flepp, B., Goyal, P., Jackel, L.D., Monfort, M., Muller, U., Zhang, J., Zhang, X., Zhao, J., Zieba, K.: End to end learning for self-driving cars. CoRR (2016) \n                    arXiv:1604.07316"},{"key":"179_CR5","doi-asserted-by":"crossref","unstructured":"Bojarski, M., Choromanska, A., Choromanski, K., Firner, B., Ackel, L.J., Muller, U., Yeres, P., Zieba, K.: Visualbackprop: Efficient visualization of cnns for autonomous driving. In: 2018 IEEE International Conference on Robotics and Automation (ICRA), pp. 1\u20138 (2018)","DOI":"10.1109\/ICRA.2018.8461053"},{"issue":"4","key":"179_CR6","doi-asserted-by":"publisher","first-page":"84","DOI":"10.1109\/MCG.2018.042731661","volume":"38","author":"J Choo","year":"2018","unstructured":"Choo, J., Liu, S.: Visual analytics for explainable deep learning. IEEE Comput. Graph. Appl. 38(4), 84\u201392 (2018). \n                    https:\/\/doi.org\/10.1109\/MCG.2018.042731661","journal-title":"IEEE Comput. Graph. Appl."},{"key":"179_CR7","doi-asserted-by":"publisher","unstructured":"Dam, H.K., Tran, T., Ghose, A.: Explainable software analytics. In: Zisman, A., Apel, S. (eds.) Proceedings of the 40th International Conference on Software Engineering: New Ideas and Emerging Results, ICSE (NIER) 2018, Gothenburg, Sweden, May 27\u2013June 03, 2018, pp. 53\u201356. ACM (2018). \n                    https:\/\/doi.org\/10.1145\/3183399.3183424","DOI":"10.1145\/3183399.3183424"},{"key":"179_CR8","unstructured":"Ding, W., Wang, R., Mao, F., Taylor, G.: Theano-based large-scale visual recognition with multiple gpus (2014) arXiv preprint \n                    arXiv:1412.2302"},{"key":"179_CR9","doi-asserted-by":"crossref","unstructured":"Do\u0161ilovi\u0107, F.K., Br\u010di\u0107, M., Hlupi\u0107, N.: Explainable artificial intelligence: a survey. In: 2018 41st International Convention on Information and Communication Technology, Electronics and Microelectronics (MIPRO), pp. 0210\u20130215 (2018)","DOI":"10.23919\/MIPRO.2018.8400040"},{"key":"179_CR10","doi-asserted-by":"publisher","unstructured":"Escorcia, V., Niebles, J.C., Ghanem, B.: On the relationship between visual attributes and convolutional networks. In: IEEE Conference on Computer Vision and Pattern Recognition, CVPR 2015, Boston, MA, USA, 7\u201312 June, 2015, pp. 1256\u20131264 (2015). \n                    https:\/\/doi.org\/10.1109\/CVPR.2015.7298730","DOI":"10.1109\/CVPR.2015.7298730"},{"issue":"6","key":"179_CR11","doi-asserted-by":"publisher","first-page":"455","DOI":"10.1016\/0031-3203(82)90024-3","volume":"15","author":"K Fukushima","year":"1982","unstructured":"Fukushima, K., Miyake, S.: Neocognitron: a new algorithm for pattern recognition tolerant of deformations and shifts in position. Pattern Recognit. 15(6), 455\u2013469 (1982). \n                    https:\/\/doi.org\/10.1016\/0031-3203(82)90024-3","journal-title":"Pattern Recognit."},{"key":"179_CR12","unstructured":"Ganesan, K.: Computing precision and recall for multi-class classification problems. \n                    http:\/\/text-analytics101.rxnlp.com\/2014\/10\/computing-precision-and-recall-for.html\n                    \n                   (2014)"},{"key":"179_CR13","doi-asserted-by":"publisher","unstructured":"Graves, A., Jaitly, N., Mohamed, A.: Hybrid speech recognition with deep bidirectional LSTM. In: 2013 IEEE Workshop on Automatic Speech Recognition and Understanding, Olomouc, Czech Republic, 8\u201312 Dec, 2013, pp. 273\u2013278. IEEE (2013) \n                    https:\/\/doi.org\/10.1109\/ASRU.2013.6707742","DOI":"10.1109\/ASRU.2013.6707742"},{"key":"179_CR14","unstructured":"Gr\u00fcn, F., Rupprecht, C., Navab, N., Federico, T.: A taxonomy and library for visualizing learned features in convolutional neural networks. In: ICML Workshop on Visualization for Deep Learning (ICML-W) (2016)"},{"key":"179_CR15","unstructured":"Gunning, D.: (2016) Explainable artificial intelligence (xai). \n                    https:\/\/www.darpa.mil\/program\/explainable-artificial-intelligence"},{"key":"179_CR16","doi-asserted-by":"publisher","unstructured":"He, K., Zhang, X., Ren, S., Sun, J.: Delving deep into rectifiers: Surpassing human-level performance on imagenet classification. In: Proceedings of the 2015 IEEE International Conference on Computer Vision (ICCV), IEEE Computer Society, Washington, DC, USA, ICCV \u201915, pp. 1026\u20131034 (2015). \n                    https:\/\/doi.org\/10.1109\/ICCV.2015.123","DOI":"10.1109\/ICCV.2015.123"},{"key":"179_CR17","doi-asserted-by":"publisher","unstructured":"Hendricks, L.A., Akata, Z., Rohrbach, M., Donahue, J., Schiele, B., Darrell, T.: Generating visual explanations. In: Leibe, B., Matas, J., Sebe, N., Welling, M. (eds.) Computer Vision\u2014ECCV 2016\u201414th European Conference, Amsterdam, The Netherlands, 11\u201314 Oct, 2016, Proceedings, Part IV, Springer, Lecture Notes in Computer Science, vol. 9908, pp. 3\u201319 (2016). \n                    https:\/\/doi.org\/10.1007\/978-3-319-46493-0_1","DOI":"10.1007\/978-3-319-46493-0_1"},{"key":"179_CR18","doi-asserted-by":"publisher","unstructured":"Hohman, F.M., Kahng, M., Pienta, R., Chau, D.H.: Visual analytics in deep learning: An interrogative survey for the next frontiers. In: IEEE Transactions on Visualization & Computer Graphics (2018). \n                    https:\/\/doi.org\/10.1109\/TVCG.2018.2843369","DOI":"10.1109\/TVCG.2018.2843369"},{"issue":"(2016\u2014-01\u2014-0128","key":"179_CR19","doi-asserted-by":"publisher","first-page":"15","DOI":"10.4271\/2016-01-0128","volume":"4","author":"P Koopman","year":"2016","unstructured":"Koopman, P., Wagner, M.: Challenges in autonomous vehicle testing and validation. SAE Int J Transp Saf 4((2016\u2014-01\u2014-0128)), 15\u201324 (2016)","journal-title":"SAE Int J Transp Saf"},{"key":"179_CR20","unstructured":"Krizhevsky, A., Sutskever, I., Hinton, G.E.: Imagenet classification with deep convolutional neural networks. In: Pereira, F., Burges, C.J.C., Bottou, L., Weinberger, K.Q. (eds.) Advances in Neural Information Processing Systems, vol. 25, pp. 1097\u20131105. Curran Associates, Inc. (2012)"},{"key":"179_CR21","unstructured":"Kuwajima, H., Tanaka, M.: Network analysis for explanation. In: Transparent and interpretable Machine Learning in Safety Critical Environments (NIPS2017 Workshop) (2017)"},{"key":"179_CR22","unstructured":"Kuwajima, H., Yasuoka, H., Nakae, T.: Open problems in engineering and quality assurance of safety critical machine learning systems. In: Joint Workshop Between ICML, AAMAS and IJCAI on Deep (or Machine) Learning for Safety-Critical Applications in Engineering (2018)"},{"key":"179_CR23","unstructured":"Lin, M., Chen, Q., Yan, S.: Network in network. CoRR (2013). \n                    arXiv:1312.4400"},{"key":"179_CR24","unstructured":"Lundberg, S.M., Lee, S.I.: A unified approach to interpreting model predictions. In: Guyon, I., Luxburg, U.V., Bengio, S., Wallach, H., Fergus, R., Vishwanathan, S., Garnett, R. (eds) Advances in Neural Information Processing Systems, vol. 30, pp. 4765\u20134774. Curran Associates, Inc. (2017)"},{"key":"179_CR25","unstructured":"Mikolov, T., Sutskever, I., Chen, K., Corrado, GS., Dean, J.: Distributed representations of words and phrases and their compositionality. In: Burges, C.J.C, Bottou, L., Welling, M., Ghahramani, Z., Weinberger, K.Q. (eds.) Advances in Neural Information Processing Systems, vol. 26, pp. 3111\u20133119. Curran Associates, Inc. (2013)"},{"issue":"4","key":"179_CR26","doi-asserted-by":"publisher","first-page":"235","DOI":"10.1093\/ijl\/3.4.235","volume":"3","author":"GA Miller","year":"1990","unstructured":"Miller, G.A., Beckwith, R., Fellbaum, C., Gross, D., Miller, K.J.: Introduction to WordNet: an on-line lexical database. Int. J. Lexicogr. 3(4), 235\u2013244 (1990)","journal-title":"Int. J. Lexicogr."},{"key":"179_CR27","doi-asserted-by":"publisher","first-page":"211","DOI":"10.1016\/j.patcog.2016.11.008","volume":"65","author":"G Montavon","year":"2017","unstructured":"Montavon, G., Lapuschkin, S., Binder, A., Samek, W., M\u00fcller, K.: Explaining nonlinear classification decisions with deep taylor decomposition. Pattern Recognit. 65, 211\u2013222 (2017). \n                    https:\/\/doi.org\/10.1016\/j.patcog.2016.11.008","journal-title":"Pattern Recognit."},{"key":"179_CR28","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1016\/j.dsp.2017.10.011","volume":"73","author":"G Montavon","year":"2018","unstructured":"Montavon, G., Samek, W., M\u00fcller, K.: Methods for interpreting and understanding deep neural networks. Digital Signal Process. 73, 1\u201315 (2018)","journal-title":"Digital Signal Process."},{"key":"179_CR29","unstructured":"Park, D.H., Hendricks, L.A., Akata, Z., Schiele, B., Darrell, T., Rohrbach, M.: Attentive explanations: justifying decisions and pointing to the evidence. CoRR (2016). \n                    arXiv:1612.04757"},{"key":"179_CR30","doi-asserted-by":"crossref","unstructured":"Pennington, J., Socher, R., Manning, C.D.: Glove: Global vectors for word representation. In: Empirical Methods in Natural Language Processing (EMNLP), pp. 1532\u20131543 (2014)","DOI":"10.3115\/v1\/D14-1162"},{"issue":"1","key":"179_CR31","first-page":"37","volume":"2","author":"DMW Powers","year":"2011","unstructured":"Powers, D.M.W.: Evaluation: From precision, recall and f-measure to roc., informedness, markedness & correlation. J. Mach. Learn. Technol. 2(1), 37\u201363 (2011)","journal-title":"J. Mach. Learn. Technol."},{"key":"179_CR32","unstructured":"Ribeiro, M.T., Singh, S., Guestrin, C.: \u201dwhy should I trust you?\u201d: Explaining the predictions of any classifier. In: Proceedings of the 22nd ACM SIGKDD International Conference on Knowledge Discovery and Data Mining, San Francisco, CA, USA, 13\u201317 Aug, 2016, pp. 1135\u20131144 (2016)"},{"issue":"3","key":"179_CR33","doi-asserted-by":"publisher","first-page":"211","DOI":"10.1007\/s11263-015-0816-y","volume":"115","author":"O Russakovsky","year":"2015","unstructured":"Russakovsky, O., Deng, J., Su, H., Krause, J., Satheesh, S., Ma, S., Huang, Z., Karpathy, A., Khosla, A., Bernstein, M., Berg, A.C., Fei-Fei, L.: ImageNet large scale visual recognition challenge. Int. J. Comput. Vis. (IJCV) 115(3), 211\u2013252 (2015). \n                    https:\/\/doi.org\/10.1007\/s11263-015-0816-y","journal-title":"Int. J. Comput. Vis. (IJCV)"},{"key":"179_CR34","doi-asserted-by":"crossref","unstructured":"Selvaraju, R.R., Cogswell, M., Das, A., Vedantam, R., Parikh, D., Batra, D.: Grad-cam: visual explanations from deep networks via gradient-based localization. In: The IEEE International Conference on Computer Vision (ICCV) (2017)","DOI":"10.1109\/ICCV.2017.74"},{"key":"179_CR35","unstructured":"Shrikumar, A., Greenside, P., Shcherbina, A., Kundaje, A.: Not just a black box: learning important features through propagating activation differences. CoRR (2013). \n                    arXiv:1605.01713"},{"key":"179_CR36","unstructured":"Shrikumar, A., Greenside, P., Kundaje, A.: Learning important features through propagating activation differences. In: Precup, D., Teh, Y.W. (eds.) Proceedings of the 34th International Conference on Machine Learning, PMLR, International Convention Centre, Sydney, Australia, Proceedings of Machine Learning Research, vol.\u00a070, pp. 3145\u20133153 (2017)"},{"key":"179_CR37","unstructured":"Simonyan, K., Vedaldi, A., Zisserman, A.: Deep inside convolutional networks: visualising image classification models and saliency maps. In: Proceedings of the International Conference on Learning Representations (ICLR) (2014)"},{"key":"179_CR38","doi-asserted-by":"publisher","first-page":"197","DOI":"10.1016\/j.neunet.2018.05.002","volume":"105","author":"K Uchida","year":"2018","unstructured":"Uchida, K., Tanaka, M., Okutomi, M.: Coupled convolution layer for convolutional neural network. Neural Netw. 105, 197\u2013205 (2018)","journal-title":"Neural Netw."},{"key":"179_CR39","doi-asserted-by":"publisher","unstructured":"Vinyals, O., Toshev, A., Bengio, S., Erhan, D.: Show and tell: A neural image caption generator. In: IEEE Conference on Computer Vision and Pattern Recognition, CVPR 2015, Boston, MA, USA, 7\u201312 June, 2015, pp. 3156\u20133164 (2015). \n                    https:\/\/doi.org\/10.1109\/CVPR.2015.7298935","DOI":"10.1109\/CVPR.2015.7298935"},{"key":"179_CR40","unstructured":"Xu, K., Ba, J., Kiros, R., Cho, K., Courville, A.C., Salakhutdinov, R., Zemel, R.S., Bengio, Y.: Show, attend and tell: neural image caption generation with visual attention. In: Bach, F.R., Blei, D.M. (eds) Proceedings of the 32nd International Conference on Machine Learning, ICML 2015, Lille, France, 6\u201311 July 2015, JMLR.org, JMLR Workshop and Conference Proceedings, vol.\u00a037, pp. 2048\u20132057 (2015)"},{"key":"179_CR41","doi-asserted-by":"publisher","first-page":"55","DOI":"10.1109\/MCI.2018.2840738","volume":"13","author":"T Young","year":"2018","unstructured":"Young, T., Hazarika, D., Poria, S., Cambria, E.: Recent trends in deep learning based natural language processing. IEEE Comput. Intell. Mag. 13, 55\u201375 (2018). \n                    https:\/\/doi.org\/10.1109\/MCI.2018.2840738","journal-title":"IEEE Comput. Intell. Mag."},{"key":"179_CR42","doi-asserted-by":"crossref","first-page":"818","DOI":"10.1007\/978-3-319-10590-1_53","volume-title":"Computer Vision\u2014ECCV 2014","author":"MD Zeiler","year":"2014","unstructured":"Zeiler, M.D., Fergus, R.: Visualizing and understanding convolutional networks. In: Fleet, D., Pajdla, T., Schiele, B., Tuytelaars, T. (eds.) Computer Vision\u2014ECCV 2014, pp. 818\u2013833. Springer International Publishing, Cham (2014)"},{"key":"179_CR43","doi-asserted-by":"crossref","unstructured":"Zhou, B., Khosla, A., Lapedriza, A., Oliva, A., Torralba, A.: Learning deep features for discriminative localization. In: IEEE Conference on Computer Vision and Pattern Recognition (CVPR) (2016)","DOI":"10.1109\/CVPR.2016.319"}],"container-title":["Progress in Artificial Intelligence"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s13748-019-00179-x.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/article\/10.1007\/s13748-019-00179-x\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s13748-019-00179-x.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2020,4,10]],"date-time":"2020-04-10T23:16:13Z","timestamp":1586560573000},"score":1,"resource":{"primary":{"URL":"http:\/\/link.springer.com\/10.1007\/s13748-019-00179-x"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2019,4,12]]},"references-count":43,"journal-issue":{"issue":"2","published-print":{"date-parts":[[2019,6]]}},"alternative-id":["179"],"URL":"https:\/\/doi.org\/10.1007\/s13748-019-00179-x","relation":{},"ISSN":["2192-6352","2192-6360"],"issn-type":[{"type":"print","value":"2192-6352"},{"type":"electronic","value":"2192-6360"}],"subject":[],"published":{"date-parts":[[2019,4,12]]},"assertion":[{"value":"6 September 2018","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"26 February 2019","order":2,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"12 April 2019","order":3,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}}]}}