{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,3,10]],"date-time":"2026-03-10T15:23:42Z","timestamp":1773156222051,"version":"3.50.1"},"reference-count":42,"publisher":"Springer Science and Business Media LLC","issue":"1","license":[{"start":{"date-parts":[[2019,11,1]],"date-time":"2019-11-01T00:00:00Z","timestamp":1572566400000},"content-version":"tdm","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"},{"start":{"date-parts":[[2019,11,1]],"date-time":"2019-11-01T00:00:00Z","timestamp":1572566400000},"content-version":"vor","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"}],"funder":[{"name":"National NSF of China","award":["61702359"],"award-info":[{"award-number":["61702359"]}]}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["J Vis"],"published-print":{"date-parts":[[2020,2]]},"DOI":"10.1007\/s12650-019-00607-z","type":"journal-article","created":{"date-parts":[[2019,11,1]],"date-time":"2019-11-01T13:03:17Z","timestamp":1572613397000},"page":"141-156","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":34,"title":["Visualizing surrogate decision trees of convolutional neural networks"],"prefix":"10.1007","volume":"23","author":[{"ORCID":"https:\/\/orcid.org\/0000-0001-6876-9180","authenticated-orcid":false,"given":"Shichao","family":"Jia","sequence":"first","affiliation":[]},{"given":"Peiwen","family":"Lin","sequence":"additional","affiliation":[]},{"given":"Zeyu","family":"Li","sequence":"additional","affiliation":[]},{"given":"Jiawan","family":"Zhang","sequence":"additional","affiliation":[]},{"given":"Shixia","family":"Liu","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2019,11,1]]},"reference":[{"key":"607_CR1","doi-asserted-by":"publisher","unstructured":"Asahi T, Turo D, Shneiderman B (1995) Visual decision-making: using treemaps for the analytic hierarchy process. In: Bederson BB, B Shneiderman (eds) The craft of information visualization. Elsevier, pp 405\u2013406. https:\/\/doi.org\/10.1016\/B978-1-55860-915-0.X5000-8","DOI":"10.1016\/B978-1-55860-915-0.X5000-8"},{"key":"607_CR2","doi-asserted-by":"crossref","unstructured":"Bau D, Zhou B, Khosla A, Oliva A, Torralba A (2017) Network dissection: quantifying interpretability of deep visual representations. In: CVPR","DOI":"10.1109\/CVPR.2017.354"},{"key":"607_CR3","volume-title":"Classification and regression trees","author":"L Breiman","year":"1984","unstructured":"Breiman L, Friedman J, Stone CJ, Olshen RA (1984) Classification and regression trees. CRC Press, Boca Raton"},{"key":"607_CR4","doi-asserted-by":"publisher","first-page":"84","DOI":"10.1109\/MCG.2018.042731661","volume":"38","author":"J Choo","year":"2018","unstructured":"Choo J, Liu S (2018) Visual analytics for explainable deep learning. IEEE Comput Graph Appl 38:84\u201392","journal-title":"IEEE Comput Graph Appl"},{"key":"607_CR5","doi-asserted-by":"publisher","first-page":"1009,1016","DOI":"10.1109\/TVCG.2009.122","volume":"15","author":"C Collins","year":"2009","unstructured":"Collins C, Penn G, Carpendale S (2009) Bubble sets: revealing set relations with isocontours over existing visualizations. IEEE Trans Vis Comput Graph 15:1009,1016\u20131009,1016","journal-title":"IEEE Trans Vis Comput Graph"},{"key":"607_CR6","doi-asserted-by":"crossref","unstructured":"Craven M, Shavlik JW (1994) Using sampling and queries to extract rules from trained neural networks. In: International joint conference on neural networks, pp 37\u201345","DOI":"10.1016\/B978-1-55860-335-6.50013-1"},{"key":"607_CR7","unstructured":"Craven M, Shavlik JW (1996) Extracting tree-structured representations of trained networks. In: Advances in neural information processing systems, pp 24\u201330"},{"key":"607_CR8","first-page":"6967","volume":"2017","author":"P Dabkowski","year":"2017","unstructured":"Dabkowski P, Yarin G (2017) Real time image saliency for black box classifiers. Adv Neural Inf Process Syst 2017:6967\u20136976","journal-title":"Adv Neural Inf Process Syst"},{"key":"607_CR9","doi-asserted-by":"publisher","first-page":"127","DOI":"10.1007\/978-3-642-18638-7_6","volume-title":"Graph drawing software","author":"J Ellson","year":"2004","unstructured":"Ellson J, Gansner ER, Koutsofios E, North SC, Woodhull G (2004) Graphviz and dynagraph\u2014static and dynamic graph drawing tools. In: Graph drawing software. Springer, Berlin, Heidelberg, pp 127\u2013148"},{"key":"607_CR10","unstructured":"Haipeng Z (2017) Towards better understanding of deep learning with visualization. Foundations of Science"},{"key":"607_CR11","series-title":"AKDD 2001. Lecture notes in computer science","doi-asserted-by":"publisher","first-page":"575","DOI":"10.1007\/3-540-45357-1_61","volume-title":"Advances in knowledge discovery and data mining, Pe","author":"J Han","year":"2001","unstructured":"Han J, Cercone N (2001) Interactive construction of decision trees. In: Cheung D, Williams GJ, Li Q (eds) Advances in knowledge discovery and data mining, PAKDD 2001. Lecture notes in computer science, vol 2035. Springer, Berlin, Heidelberg, pp 575\u2013580. https:\/\/doi.org\/10.1007\/3-540-45357-1_61"},{"key":"607_CR12","doi-asserted-by":"crossref","unstructured":"Hohman FM, Kahng M, Pienta R, Chau DH (2018) Visual analytics in deep learning: an interrogative survey for the next frontiers. IEEE Trans Vis Comput Graph","DOI":"10.1109\/TVCG.2018.2843369"},{"key":"607_CR13","doi-asserted-by":"publisher","DOI":"10.2139\/ssrn.961358","volume-title":"Using rule extraction to improve the comprehensibility of predictive models","author":"J Huysmans","year":"2006","unstructured":"Huysmans J, Baesens B, Vanthienen J (2006) Using rule extraction to improve the comprehensibility of predictive models. Social science Electronic Publishing, Rochester"},{"key":"607_CR14","first-page":"1","volume":"24","author":"M Kahng","year":"2017","unstructured":"Kahng M, Andrews PY, Kalro A, Polo DC (2017) ACTIVIS: visual exploration of industry-scale deep neural network models. IEEE Trans Vis Comput Graph 24:1\u20131","journal-title":"IEEE Trans Vis Comput Graph"},{"key":"607_CR15","doi-asserted-by":"crossref","unstructured":"Karpathy A, Toderici G, Shetty S, Leung T, Sukthankar R, Fei-Fei L (2014) Large-scale video classification with convolutional neural networks. In: Proceedings of the IEEE conference on computer vision and pattern recognition, pp 1725\u20131732","DOI":"10.1109\/CVPR.2014.223"},{"key":"607_CR16","unstructured":"Krizhevsky A, Hinton G (2009) Learning multiple layers of features from tiny images"},{"key":"607_CR17","first-page":"1097","volume-title":"Proceedings of the 25th international conference on neural information processing systems","author":"A Krizhevsky","year":"2012","unstructured":"Krizhevsky A, Sutskever I, Hinton GE (2012) Imagenet classification with deep convolutional neural networks. In: Proceedings of the 25th international conference on neural information processing systems, vol 1. Curran Associates Inc, Lake Tahoe, Nevada, pp 1097\u20131105"},{"key":"607_CR18","doi-asserted-by":"crossref","unstructured":"Lakkaraju H, Bach SH, Leskovec J (2016) Interpretable decision sets: a joint framework for description and prediction. In: Proceedings of the 22nd ACM SIGKDD international conference on knowledge discovery and data mining pp 1675\u20131684","DOI":"10.1145\/2939672.2939874"},{"key":"607_CR19","doi-asserted-by":"publisher","first-page":"95","DOI":"10.1016\/j.ijhcs.2006.07.005","volume":"65","author":"Y Liu","year":"2007","unstructured":"Liu Y, Salvendy G (2007) Design and evaluation of visualization support to facilitate decision trees classification. Int J Hum Comput Stud 65:95\u2013110","journal-title":"Int J Hum Comput Stud"},{"key":"607_CR20","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1109\/TVCG.2016.2598432","volume":"24","author":"M Liu","year":"2017","unstructured":"Liu M, Shi J, Cao K, Zhu J, Liu S (2017) Analyzing the training processes of deep generative models. IEEE Trans Vis Comput Graph 24:1\u20131","journal-title":"IEEE Trans Vis Comput Graph"},{"key":"607_CR21","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1109\/TVCG.2016.2598432","volume":"24","author":"S Liu","year":"2017","unstructured":"Liu S, Xiao J, Liu J, Wang X, Jing W, Zhu J (2017) Visual diagnosis of tree boosting methods. IEEE Trans Vis Comput Graph 24:1\u20131","journal-title":"IEEE Trans Vis Comput Graph"},{"key":"607_CR22","first-page":"2579","volume":"9","author":"Maaten Lvd","year":"2008","unstructured":"Lvd Maaten, Hinton G (2008) Visualizing data using t-SNE. J Mach Learn Res 9:2579\u20132605","journal-title":"J Mach Learn Res"},{"key":"607_CR23","doi-asserted-by":"crossref","unstructured":"Mansmann F, G\u00f6bel T, Cheswick W (2012) Visual analysis of complex firewall configurations. In: Proceedings of the ninth international symposium on visualization for cyber security, pp 1\u20138","DOI":"10.1145\/2379690.2379691"},{"key":"607_CR24","first-page":"1","volume":"24","author":"T Muhlbacher","year":"2017","unstructured":"Muhlbacher T, Linhardt L, Moller T, Piringer H (2017) TreePOD: sensitivity-aware selection of pareto-optimal decision trees. IEEE Trans Vis Comput Graph 24:1\u20131","journal-title":"IEEE Trans Vis Comput Graph"},{"key":"607_CR25","first-page":"2825","volume":"12","author":"F Pedregosa","year":"2011","unstructured":"Pedregosa F et al (2011) Scikit-learn: machine learning in Python. J Mach Learn Res 12:2825\u20132830","journal-title":"J Mach Learn Res"},{"key":"607_CR26","first-page":"1","volume":"24","author":"N Pezzotti","year":"2017","unstructured":"Pezzotti N, H\u00f6llt T, Gemert JV, Lelieveldt BPF, Eisemann E, Vilanova A (2017) DeepEyes: progressive visual analytics for designing deep neural networks. IEEE Trans Vis Comput Graph 24:1\u20131","journal-title":"IEEE Trans Vis Comput Graph"},{"key":"607_CR27","doi-asserted-by":"publisher","first-page":"101","DOI":"10.1109\/TVCG.2016.2598838","volume":"23","author":"Paulo E Rauber","year":"2017","unstructured":"Rauber Paulo E, Fadel Samuel G, Falcao Alexandre X, Telea Alexandru C (2017) Visualizing the hidden activity of artificial neural networks. IEEE Trans Vis Comput Graph 23:101\u2013110. https:\/\/doi.org\/10.1109\/TVCG.2016.2598838","journal-title":"IEEE Trans Vis Comput Graph"},{"key":"607_CR28","doi-asserted-by":"publisher","first-page":"223","DOI":"10.1109\/TSE.1981.234519","volume":"2","author":"EM Reingold","year":"1981","unstructured":"Reingold EM, Tilford JS (1981) Tidier drawings of trees. IEEE Trans Softw Eng 2:223\u2013228","journal-title":"IEEE Trans Softw Eng"},{"key":"607_CR29","doi-asserted-by":"crossref","unstructured":"Ribeiro MT, Singh S, Guestrin C (2016) Why should I trust you?: Explaining the predictions of any classifier. In: Proceedings of the 22nd ACM SIGKDD international conference on knowledge discovery and data mining pp 1135\u20131144","DOI":"10.1145\/2939672.2939778"},{"key":"607_CR30","doi-asserted-by":"publisher","first-page":"385","DOI":"10.1109\/TVCG.2018.2864838","volume":"25","author":"D Sacha","year":"2019","unstructured":"Sacha D, Kraus M, Keim DA, Chen M (2019) VIS4ML: an ontology for visual analytics assisted machine learning. IEEE Trans Vis Comput Graph 25:385\u2013395","journal-title":"IEEE Trans Vis Comput Graph"},{"key":"607_CR31","doi-asserted-by":"crossref","unstructured":"Sato M, Tsukimoto H (2001) Rule extraction from neural networks via decision tree induction. In: International joint conference on neural networks. Proceedings (Cat. No. 01CH37222), vol 3. IEEE, pp 1870\u20131875","DOI":"10.1109\/IJCNN.2001.938448"},{"key":"607_CR32","doi-asserted-by":"publisher","first-page":"48","DOI":"10.1016\/j.visinf.2017.01.006","volume":"1","author":"Liu Shixia","year":"2017","unstructured":"Shixia Liu, Xiting Wang, Liu Mengchen, Jun Zhu (2017) Towards better analysis of machine learning models: a visual analytics perspective. Vis Inform 1:48\u201356","journal-title":"Vis Inform"},{"key":"607_CR33","unstructured":"Simonyan K, Zisserman A (2013) Very deep convolutional networks for large-scale image recognition. In: ICLR 2015"},{"key":"607_CR34","unstructured":"Tzeng FY, Ma KL (2005) Opening the black box\u2014data driven visualization of neural networks. In: VIS 05. IEEE Visualization, vol 2005. IEEE, pp 383\u2013390"},{"key":"607_CR35","doi-asserted-by":"crossref","unstructured":"Van Den Elzen S, van Wijk JJ (2011) Baobabview: interactive construction and analysis of decision trees. In: IEEE conference on visual analytics science and technology, pp 151\u2013160","DOI":"10.1109\/VAST.2011.6102453"},{"key":"607_CR36","doi-asserted-by":"publisher","first-page":"2168","DOI":"10.1109\/TVCG.2019.2903943","volume":"25","author":"J Wang","year":"2019","unstructured":"Wang J, Gou L, Zhang W, Yang H, Shen H-W (2019) DeepVID: deep visual interpretation and diagnosis for image classifiers via knowledge distillation. IEEE Trans Vis Comput Graph 25:2168\u20132180","journal-title":"IEEE Trans Vis Comput Graph"},{"key":"607_CR37","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1109\/TVCG.2017.2744878","volume":"24","author":"K Wongsuphasawat","year":"2017","unstructured":"Wongsuphasawat K et al (2017) Visualizing dataflow graphs of deep learning models in tensorflow. IEEE Trans Vis Comput Graph 24:1\u20131","journal-title":"IEEE Trans Vis Comput Graph"},{"key":"607_CR38","doi-asserted-by":"crossref","unstructured":"Wu M, Hughes MC, Parbhoo S, Zazzi M, Roth V, Doshi-Velez F (2017) Beyond Sparsity: tree regularization of deep models for interpretability. IIPS TIML workshop","DOI":"10.1609\/aaai.v32i1.11501"},{"key":"607_CR39","doi-asserted-by":"publisher","first-page":"342","DOI":"10.1109\/TVCG.2018.2864812","volume":"25","author":"HQ Yao Ming","year":"2019","unstructured":"Yao Ming HQ, Bertini E (2019) RuleMatrix: visualizing and understanding classifiers with rules. IEEE Trans Vis Comput Graph 25:342\u2013352","journal-title":"IEEE Trans Vis Comput Graph"},{"key":"607_CR40","doi-asserted-by":"publisher","first-page":"818","DOI":"10.1007\/978-3-319-10590-1_53","volume-title":"Computer Vision \u2013 ECCV 2014","author":"Matthew D. Zeiler","year":"2014","unstructured":"Zeiler MD, Fergus R (2014) Visualizing and understanding convolutional networks. In: European conference on computer vision, pp 818\u2013833"},{"key":"607_CR41","doi-asserted-by":"publisher","first-page":"27","DOI":"10.1631\/FITEE.1700808","volume":"19","author":"Q-s Zhang","year":"2018","unstructured":"Zhang Q-s, Zhu S-C (2018) Visual interpretability for deep learning: a survey. Front Inf Technol Electron Eng 19:27\u201339","journal-title":"Front Inf Technol Electron Eng"},{"key":"607_CR42","unstructured":"Zhang Q, Yang Y, Wu YN, Zhu S-C (2018) Interpreting CNNs via Decision Trees CoRR. arXiv:abs\/1802.00121"}],"container-title":["Journal of Visualization"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s12650-019-00607-z.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/article\/10.1007\/s12650-019-00607-z\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s12650-019-00607-z.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2022,10,3]],"date-time":"2022-10-03T07:09:43Z","timestamp":1664780983000},"score":1,"resource":{"primary":{"URL":"http:\/\/link.springer.com\/10.1007\/s12650-019-00607-z"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2019,11,1]]},"references-count":42,"journal-issue":{"issue":"1","published-print":{"date-parts":[[2020,2]]}},"alternative-id":["607"],"URL":"https:\/\/doi.org\/10.1007\/s12650-019-00607-z","relation":{},"ISSN":["1343-8875","1875-8975"],"issn-type":[{"value":"1343-8875","type":"print"},{"value":"1875-8975","type":"electronic"}],"subject":[],"published":{"date-parts":[[2019,11,1]]},"assertion":[{"value":"6 July 2019","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"3 September 2019","order":2,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"1 November 2019","order":3,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}}]}}