{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,1,2]],"date-time":"2026-01-02T07:51:19Z","timestamp":1767340279339,"version":"3.40.3"},"publisher-location":"Cham","reference-count":38,"publisher":"Springer International Publishing","isbn-type":[{"type":"print","value":"9783030695347"},{"type":"electronic","value":"9783030695354"}],"license":[{"start":{"date-parts":[[2021,1,1]],"date-time":"2021-01-01T00:00:00Z","timestamp":1609459200000},"content-version":"tdm","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"},{"start":{"date-parts":[[2021,1,1]],"date-time":"2021-01-01T00:00:00Z","timestamp":1609459200000},"content-version":"vor","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2021]]},"DOI":"10.1007\/978-3-030-69535-4_13","type":"book-chapter","created":{"date-parts":[[2021,2,24]],"date-time":"2021-02-24T15:13:11Z","timestamp":1614179591000},"page":"206-222","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":5,"title":["ERIC: Extracting Relations Inferred from Convolutions"],"prefix":"10.1007","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-5478-0028","authenticated-orcid":false,"given":"Joe","family":"Townsend","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-2008-5817","authenticated-orcid":false,"given":"Theodoros","family":"Kasioumis","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-4405-8952","authenticated-orcid":false,"given":"Hiroya","family":"Inakoshi","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2021,2,25]]},"reference":[{"key":"13_CR1","doi-asserted-by":"crossref","unstructured":"Ribeiro, M.T., Singh, S., Guestrin, C.: Why should I trust you?: explaining the predictions of any classifier. In: Proceedings of the 22nd ACM SIGKDD International Conference on Knowledge Discovery and Data Mining, pp. 1135\u20131144. ACM (2016)","DOI":"10.1145\/2939672.2939778"},{"key":"13_CR2","doi-asserted-by":"crossref","unstructured":"Gilpin, L.H., Bau, D., Yuan, B.Z., Bajwa, A., Specter, M., Kagal, L.: Explaining explanations: an overview of interpretability of machine learning. In: 2018 IEEE 5th International Conference on Data Science and Advanced Analytics (DSAA), pp. 80\u201389. IEEE (2018)","DOI":"10.1109\/DSAA.2018.00018"},{"key":"13_CR3","first-page":"93","volume":"51","author":"R Guidotti","year":"2018","unstructured":"Guidotti, R., Monreale, A., Ruggieri, S., Turini, F., Giannotti, F., Pedreschi, D.: A survey of methods for explaining black box models. ACM Comput. Surv. (CSUR) 51, 93 (2018)","journal-title":"ACM Comput. Surv. (CSUR)"},{"key":"13_CR4","doi-asserted-by":"publisher","first-page":"373","DOI":"10.1016\/0950-7051(96)81920-4","volume":"8","author":"R Andrews","year":"1995","unstructured":"Andrews, R., Diederich, J., Tickle, A.B.: Survey and critique of techniques for extracting rules from trained artificial neural networks. Knowl.-Based Syst. 8, 373\u2013389 (1995)","journal-title":"Knowl.-Based Syst."},{"key":"13_CR5","doi-asserted-by":"publisher","first-page":"1223","DOI":"10.1162\/0899766053630350","volume":"17","author":"H Jacobsson","year":"2005","unstructured":"Jacobsson, H.: Rule extraction from recurrent neural networks: a taxonomy and review. Neural Comput. 17, 1223\u20131263 (2005)","journal-title":"Neural Comput."},{"key":"13_CR6","doi-asserted-by":"publisher","first-page":"3456","DOI":"10.1109\/TNNLS.2019.2944672","volume":"31","author":"J Townsend","year":"2019","unstructured":"Townsend, J., Chaton, T., Monteiro, J.M.: Extracting relational explanations from deep neural networks: a survey from a neural-symbolic perspective. IEEE Trans. Neural Networks Learn. Syst. 31, 3456 (2019)","journal-title":"IEEE Trans. Neural Networks Learn. Syst."},{"key":"13_CR7","doi-asserted-by":"publisher","first-page":"27","DOI":"10.1631\/FITEE.1700808","volume":"19","author":"Q Zhang","year":"2018","unstructured":"Zhang, Q., Zhu, S.: Visual interpretability for deep learning: a survey. Front. Inf. Technol. Electron. Eng. 19, 27\u201339 (2018)","journal-title":"Front. Inf. Technol. Electron. Eng."},{"key":"13_CR8","doi-asserted-by":"crossref","unstructured":"Lamb, L., Garcez, A., Gori, M., Prates, M., Avelar, P., Vardi, M.: Graph neural networks meet neural-symbolic computing: a survey and perspective. arXiv preprint arXiv:2003.00330 (2020)","DOI":"10.24963\/ijcai.2020\/679"},{"key":"13_CR9","unstructured":"Simonyan, K., Vedaldi, A., Zisserman, A.: Deep inside convolutional networks: visualising image classification models and saliency maps. arXiv preprint arXiv:1312.6034 (2013)"},{"key":"13_CR10","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"818","DOI":"10.1007\/978-3-319-10590-1_53","volume-title":"Computer Vision \u2013 ECCV 2014","author":"MD Zeiler","year":"2014","unstructured":"Zeiler, M.D., Fergus, R.: Visualizing and understanding convolutional networks. In: Fleet, D., Pajdla, T., Schiele, B., Tuytelaars, T. (eds.) ECCV 2014. LNCS, vol. 8689, pp. 818\u2013833. Springer, Cham (2014). https:\/\/doi.org\/10.1007\/978-3-319-10590-1_53"},{"key":"13_CR11","unstructured":"Springenberg, J.T., Dosovitskiy, A., Brox, T., Riedmiller, M.: Striving for simplicity: the all convolutional net. arXiv preprint arXiv:1412.6806 (2014)"},{"key":"13_CR12","unstructured":"Bojarski, M., et al.: Visualbackprop: efficient visualization of CNNs. arXiv preprint arXiv:1611.05418 (2016)"},{"key":"13_CR13","doi-asserted-by":"publisher","first-page":"e0130140","DOI":"10.1371\/journal.pone.0130140","volume":"10","author":"S Bach","year":"2015","unstructured":"Bach, S., Binder, A., Montavon, G., Klauschen, F., M\u00fcller, K., Samek, W.: On pixel-wise explanations for non-linear classifier decisions by layer-wise relevance propagation. PloS One 10, e0130140 (2015)","journal-title":"PloS One"},{"key":"13_CR14","doi-asserted-by":"publisher","first-page":"2660","DOI":"10.1109\/TNNLS.2016.2599820","volume":"28","author":"W Samek","year":"2017","unstructured":"Samek, W., Binder, A., Montavon, G., Lapuschkin, S., M\u00fcller, K.: Evaluating the visualization of what a deep neural network has learned. IEEE Trans. Neural Networks Learn. Syst. 28, 2660\u20132673 (2017)","journal-title":"IEEE Trans. Neural Networks Learn. Syst."},{"key":"13_CR15","unstructured":"Shrikumar, A., Greenside, P., Kundaje, A.: Learning important features through propagating activation differences. arXiv preprint arXiv:1704.02685 (2017)"},{"key":"13_CR16","unstructured":"Frosst, N., Hinton, G.: Distilling a neural network into a soft decision tree. arXiv preprint arXiv:1711.09784 (2017)"},{"key":"13_CR17","unstructured":"Chen, C., Li, O., Tao, D., Barnett, A., Rudin, C., Su, J.K.: This looks like that: deep learning for interpretable image recognition. In: Advances in Neural Information Processing Systems, pp. 8930\u20138941 (2019)"},{"key":"13_CR18","doi-asserted-by":"publisher","first-page":"990","DOI":"10.3390\/electronics9060990","volume":"9","author":"G Bologna","year":"2020","unstructured":"Bologna, G., Fossati, S.: A two-step rule-extraction technique for a CNN. Electronics 9, 990 (2020)","journal-title":"Electronics"},{"key":"13_CR19","doi-asserted-by":"crossref","unstructured":"Zhang, Q., Cao, R., Wu, Y.N., Zhu, S.: Growing interpretable part graphs on convnets via multi-shot learning. In: Thirty-First AAAI Conference on Artificial Intelligence (2017)","DOI":"10.1609\/aaai.v31i1.10924"},{"key":"13_CR20","doi-asserted-by":"crossref","unstructured":"Zhang, Q., Cao, R., Shi, F., Wu, Y.N., Zhu, S.: Interpreting CNN knowledge via an explanatory graph. In: Thirty-Second AAAI Conference on Artificial Intelligence (2018)","DOI":"10.1609\/aaai.v32i1.11819"},{"key":"13_CR21","doi-asserted-by":"crossref","unstructured":"Zhang, Q., Yang, Y., Ma, H., Wu, Y.N.: Interpreting CNNs via decision trees. In: Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition, pp. 6261\u20136270 (2019)","DOI":"10.1109\/CVPR.2019.00642"},{"key":"13_CR22","unstructured":"Zhou, B., Khosla, A., Lapedriza, A., Oliva, A., Torralba, A.: Object detectors emerge in deep scene CNNs. arXiv preprint arXiv:1412.6856 (2014)"},{"key":"13_CR23","doi-asserted-by":"crossref","unstructured":"Zhang, Q., Nian Wu, Y., Zhu, S.: Interpretable convolutional neural networks. In: Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition, pp. 8827\u20138836 (2018)","DOI":"10.1109\/CVPR.2018.00920"},{"key":"13_CR24","first-page":"974","volume":"285","author":"C Percy","year":"2016","unstructured":"Percy, C., d\u2019Avila Garcez, A.S., Dragicevic, S., Fran\u00e7a, M.V., Slabaugh, G.G., Weyde, T.: The need for knowledge extraction: understanding harmful gambling behavior with neural networks. Front. Artif. Intell. Appl. 285, 974\u2013981 (2016)","journal-title":"Front. Artif. Intell. Appl."},{"key":"13_CR25","doi-asserted-by":"crossref","unstructured":"Ribeiro, M.T., Singh, S., Guestrin, C.: Anchors: High-precision model-agnostic explanations. In: AAAI Conference on Artificial Intelligence (2018)","DOI":"10.1609\/aaai.v32i1.11491"},{"key":"13_CR26","series-title":"Lecture Notes in Computer Science (Lecture Notes in Artificial Intelligence)","doi-asserted-by":"publisher","first-page":"457","DOI":"10.1007\/978-3-319-46307-0_29","volume-title":"Discovery Science","author":"JR Zilke","year":"2016","unstructured":"Zilke, J.R., Loza Menc\u00eda, E., Janssen, F.: DeepRED \u2013 rule extraction from deep neural networks. In: Calders, T., Ceci, M., Malerba, D. (eds.) DS 2016. LNCS (LNAI), vol. 9956, pp. 457\u2013473. Springer, Cham (2016). https:\/\/doi.org\/10.1007\/978-3-319-46307-0_29"},{"key":"13_CR27","doi-asserted-by":"crossref","unstructured":"Schaaf, N., Huber, M.F.: Enhancing decision tree based interpretation of deep neural networks through l1-orthogonal regularization. arXiv preprint arXiv:1904.05394 (2019)","DOI":"10.1109\/ICMLA.2019.00016"},{"key":"13_CR28","unstructured":"Nguyen, T.D., Kasmarik, K.E., Abbass, H.A.: Towards interpretable deep neural networks: an exact transformation to multi-class multivariate decision trees. arXiv preprint arXiv:2003.04675 (2020)"},{"key":"13_CR29","unstructured":"Murdoch, W.J., Szlam, A.: Automatic rule extraction from long short term memory networks. arXiv preprint arXiv:1702.02540 (2017)"},{"key":"13_CR30","doi-asserted-by":"publisher","first-page":"246","DOI":"10.1109\/TNNLS.2016.2603784","volume":"29","author":"SN Tran","year":"2016","unstructured":"Tran, S.N., d\u2019Avila Garcez, A.S.: Deep logic networks: inserting and extracting knowledge from deep belief networks. IEEE Trans. Neural Networks Learn. Syst. 29, 246 (2016)","journal-title":"IEEE Trans. Neural Networks Learn. Syst."},{"key":"13_CR31","doi-asserted-by":"crossref","unstructured":"Zhou, B., Khosla, A., Lapedriza, A., Oliva, A., Torralba, A.: Learning deep features for discriminative localization. In: 2016 IEEE Conference on Computer Vision and Pattern Recognition (CVPR), pp. 2921\u20132929. IEEE (2016)","DOI":"10.1109\/CVPR.2016.319"},{"key":"13_CR32","unstructured":"Denil, M., Demiraj, A., De Freitas, N.: Extraction of salient sentences from labelled documents. arXiv preprint arXiv:1412.6815 (2014)"},{"key":"13_CR33","doi-asserted-by":"crossref","unstructured":"Selvaraju, R.R., Cogswell, M., Das, A., Vedantam, R., Parikh, D., Batra, D.: Grad-cam: visual explanations from deep networks via gradient-based localization. In: Proceedings of the IEEE international conference on computer vision, pp. 618\u2013626 (2017)","DOI":"10.1109\/ICCV.2017.74"},{"key":"13_CR34","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"162","DOI":"10.1007\/978-3-319-16808-1_12","volume-title":"Computer Vision \u2013 ACCV 2014","author":"M Simon","year":"2015","unstructured":"Simon, M., Rodner, E., Denzler, J.: Part detector discovery in deep convolutional neural networks. In: Cremers, D., Reid, I., Saito, H., Yang, M.-H. (eds.) ACCV 2014. LNCS, vol. 9004, pp. 162\u2013177. Springer, Cham (2015). https:\/\/doi.org\/10.1007\/978-3-319-16808-1_12"},{"key":"13_CR35","unstructured":"Xie, N., Sarker, M.K., Doran, D., Hitzler, P., Raymer, M.: Relating input concepts to convolutional neural network decisions. arXiv preprint arXiv:1711.08006 (2017)"},{"key":"13_CR36","doi-asserted-by":"publisher","first-page":"1452","DOI":"10.1109\/TPAMI.2017.2723009","volume":"40","author":"B Zhou","year":"2017","unstructured":"Zhou, B., Lapedriza, A., Khosla, A., Oliva, A., Torralba, A.: Places: a 10 million image database for scene recognition. IEEE Trans. Pattern Anal. Mach. Intell. 40, 1452 (2017)","journal-title":"IEEE Trans. Pattern Anal. Mach. Intell."},{"key":"13_CR37","first-page":"38","volume-title":"The Morgan Kaufmann Series in Machine Learning","author":"JR Quinlan","year":"1993","unstructured":"Quinlan, J.R.: C4.5: programming for machine learning. The Morgan Kaufmann Series in Machine Learning, pp. 38\u201348. Morgan Kaufmann, San Mateo, CA (1993)"},{"key":"13_CR38","unstructured":"Chollet, F., et al.: Keras (2015). https:\/\/github.com\/fchollet\/keras"}],"container-title":["Lecture Notes in Computer Science","Computer Vision \u2013 ACCV 2020"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-030-69535-4_13","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2022,12,18]],"date-time":"2022-12-18T21:26:38Z","timestamp":1671398798000},"score":1,"resource":{"primary":{"URL":"http:\/\/link.springer.com\/10.1007\/978-3-030-69535-4_13"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2021]]},"ISBN":["9783030695347","9783030695354"],"references-count":38,"URL":"https:\/\/doi.org\/10.1007\/978-3-030-69535-4_13","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"type":"print","value":"0302-9743"},{"type":"electronic","value":"1611-3349"}],"subject":[],"published":{"date-parts":[[2021]]},"assertion":[{"value":"25 February 2021","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"ACCV","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Asian Conference on Computer Vision","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Kyoto","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Japan","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2020","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"30 November 2020","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"4 December 2020","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"15","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"accv2020","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"http:\/\/accv2020.kyoto\/","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Double-blind","order":1,"name":"type","label":"Type","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"Microsoft CMT","order":2,"name":"conference_management_system","label":"Conference Management System","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"768","order":3,"name":"number_of_submissions_sent_for_review","label":"Number of Submissions Sent for Review","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"254","order":4,"name":"number_of_full_papers_accepted","label":"Number of Full Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"0","order":5,"name":"number_of_short_papers_accepted","label":"Number of Short Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"33% - The value is computed by the equation \"Number of Full Papers Accepted \/ Number of Submissions Sent for Review * 100\" and then rounded to a whole number.","order":6,"name":"acceptance_rate_of_full_papers","label":"Acceptance Rate of Full Papers","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"3","order":7,"name":"average_number_of_reviews_per_paper","label":"Average Number of Reviews per Paper","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"3","order":8,"name":"average_number_of_papers_per_reviewer","label":"Average Number of Papers per Reviewer","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"Yes","order":9,"name":"external_reviewers_involved","label":"External Reviewers Involved","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"The conference was held virtually.","order":10,"name":"additional_info_on_review_process","label":"Additional Info on Review Process","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}}]}}