{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,11,20]],"date-time":"2025-11-20T18:33:57Z","timestamp":1763663637312,"version":"3.45.0"},"publisher-location":"Cham","reference-count":51,"publisher":"Springer Nature Switzerland","isbn-type":[{"value":"9783032090867","type":"print"},{"value":"9783032090874","type":"electronic"}],"license":[{"start":{"date-parts":[[2025,11,21]],"date-time":"2025-11-21T00:00:00Z","timestamp":1763683200000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2025,11,21]],"date-time":"2025-11-21T00:00:00Z","timestamp":1763683200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2026]]},"DOI":"10.1007\/978-3-032-09087-4_17","type":"book-chapter","created":{"date-parts":[[2025,11,20]],"date-time":"2025-11-20T18:21:18Z","timestamp":1763662878000},"page":"238-249","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":0,"title":["Benchmarking in\u00a0Neuro-Symbolic AI"],"prefix":"10.1007","author":[{"given":"Robin","family":"Manhaeve","sequence":"first","affiliation":[]},{"given":"Francesco","family":"Giannini","sequence":"additional","affiliation":[]},{"given":"Mehdi","family":"Ali","sequence":"additional","affiliation":[]},{"given":"Damiano","family":"Azzolini","sequence":"additional","affiliation":[]},{"given":"Alice","family":"Bizzarri","sequence":"additional","affiliation":[]},{"given":"Andrea","family":"Borghesi","sequence":"additional","affiliation":[]},{"given":"Samuele","family":"Bortolotti","sequence":"additional","affiliation":[]},{"given":"Luc","family":"De Raedt","sequence":"additional","affiliation":[]},{"given":"Devendra","family":"Dhami","sequence":"additional","affiliation":[]},{"given":"Michelangelo","family":"Diligenti","sequence":"additional","affiliation":[]},{"given":"Sebastijan","family":"Duman\u010di\u0107","sequence":"additional","affiliation":[]},{"given":"Boi","family":"Faltings","sequence":"additional","affiliation":[]},{"given":"Elisabetta","family":"Gentili","sequence":"additional","affiliation":[]},{"given":"Alfonso","family":"Gerevini","sequence":"additional","affiliation":[]},{"given":"Marco","family":"Gori","sequence":"additional","affiliation":[]},{"given":"Tias","family":"Guns","sequence":"additional","affiliation":[]},{"given":"Martin","family":"Homola","sequence":"additional","affiliation":[]},{"given":"Kristian","family":"Kersting","sequence":"additional","affiliation":[]},{"given":"Jens","family":"Lehmann","sequence":"additional","affiliation":[]},{"given":"Michele","family":"Lombardi","sequence":"additional","affiliation":[]},{"given":"Luca","family":"Lorello","sequence":"additional","affiliation":[]},{"given":"Emanuele","family":"Marconato","sequence":"additional","affiliation":[]},{"given":"Stefano","family":"Melacci","sequence":"additional","affiliation":[]},{"given":"Andrea","family":"Passerini","sequence":"additional","affiliation":[]},{"given":"Debjit","family":"Paul","sequence":"additional","affiliation":[]},{"given":"Fabrizio","family":"Riguzzi","sequence":"additional","affiliation":[]},{"given":"Stefano","family":"Teso","sequence":"additional","affiliation":[]},{"given":"Neil","family":"Yorke-Smith","sequence":"additional","affiliation":[]},{"given":"Marco","family":"Lippi","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2025,11,21]]},"reference":[{"key":"17_CR1","unstructured":"Adamski, D.M., Potoniec, J.: Reasonable embeddings: learning concept embeddings with a transferable neural reasoner. Semantic Web (Preprint), 1\u201333 (2023)"},{"key":"17_CR2","unstructured":"Anthony, P., et al.: Explainable malware detection with tailored logic explained networks. arXiv preprint arXiv:2405.03009 (2024)"},{"key":"17_CR3","doi-asserted-by":"crossref","unstructured":"Badreddine, S., Garcez, A.d., Serafini, L., Spranger, M.: Logic tensor networks. Artif. Intell. 303, 103649 (2022)","DOI":"10.1016\/j.artint.2021.103649"},{"key":"17_CR4","unstructured":"Barbiero, P., et al.: Interpretable neural-symbolic concept reasoning. In: International Conference on Machine Learning, pp. 1801\u20131825. PMLR (2023)"},{"key":"17_CR5","doi-asserted-by":"crossref","unstructured":"Barbiero, P., Giannini, F., Ciravegna, G., Diligenti, M., Marra, G.: Relational concept based models. arXiv preprint arXiv:2308.11991 (2023)","DOI":"10.52202\/079017-2468"},{"key":"17_CR6","doi-asserted-by":"crossref","unstructured":"Chiari, M., Gerevini, A.E., Percassi, F., Putelli, L., Serina, I., Olivato, M.: Goal recognition as a deep learning task: the grnet approach. In: Proceedings of the International Conference on Automated Planning and Scheduling, vol.\u00a033, pp. 560\u2013568 (2023)","DOI":"10.1609\/icaps.v33i1.27237"},{"key":"17_CR7","doi-asserted-by":"publisher","DOI":"10.1016\/j.artint.2022.103822","volume":"314","author":"G Ciravegna","year":"2023","unstructured":"Ciravegna, G.: Logic explained networks. Artif. Intell. 314, 103822 (2023)","journal-title":"Artif. Intell."},{"key":"17_CR8","doi-asserted-by":"publisher","first-page":"285","DOI":"10.1613\/jair.1.11944","volume":"67","author":"WW Cohen","year":"2020","unstructured":"Cohen, W.W., Yang, F., Mazaitis, K.: Tensorlog: a probabilistic database implemented using deep-learning infrastructure. J. Artif. Intell. Res. 67, 285\u2013325 (2020)","journal-title":"J. Artif. Intell. Res."},{"issue":"2","key":"17_CR9","doi-asserted-by":"publisher","first-page":"515","DOI":"10.1007\/s10994-022-06278-6","volume":"112","author":"D Cunnington","year":"2023","unstructured":"Cunnington, D., Law, M., Lobo, J., Russo, A.: FFNSL: feed-forward neural-symbolic learner. Mach. Learn. 112(2), 515\u2013569 (2023)","journal-title":"Mach. Learn."},{"key":"17_CR10","doi-asserted-by":"crossref","unstructured":"Diligenti, M., Giannini, F., Fioravanti, S., Graziani, C., Falaschi, M., Marra, G.: Enhancing embedding representations of biomedical data using logic knowledge. In: 2023 International Joint Conference on Neural Networks (IJCNN), pp.\u00a01\u20138. IEEE (2023)","DOI":"10.1109\/IJCNN54540.2023.10191706"},{"key":"17_CR11","doi-asserted-by":"publisher","first-page":"143","DOI":"10.1016\/j.artint.2015.08.011","volume":"244","author":"M Diligenti","year":"2017","unstructured":"Diligenti, M., Gori, M., Sacca, C.: Semantic-based regularization for learning and inference. Artif. Intell. 244, 143\u2013165 (2017)","journal-title":"Artif. Intell."},{"key":"17_CR12","first-page":"21400","volume":"35","author":"M Espinosa Zarlenga","year":"2022","unstructured":"Espinosa Zarlenga, M., et al.: Concept embedding models: beyond the accuracy-explainability trade-off. Adv. Neural. Inf. Process. Syst. 35, 21400\u201321413 (2022)","journal-title":"Adv. Neural. Inf. Process. Syst."},{"key":"17_CR13","doi-asserted-by":"crossref","unstructured":"Francobaldi, M., et al.: Tinderai: support system for matching AI algorithms and embedded devices. In: The International FLAIRS Conference Proceedings, vol.\u00a036 (2023)","DOI":"10.32473\/flairs.36.133100"},{"key":"17_CR14","doi-asserted-by":"crossref","unstructured":"Gal\u00e1rraga, L.A., Teflioudi, C., Hose, K., Suchanek, F.: Amie: association rule mining under incomplete evidence in ontological knowledge bases. In: Proceedings of the 22nd International Conference on World Wide Web, pp. 413\u2013422 (2013)","DOI":"10.1145\/2488388.2488425"},{"key":"17_CR15","unstructured":"Galassi, A., Lippi, M., Torroni, P.: Investigating logic tensor networks for neural-symbolic argument mining. In: Proceedings of 1st International Joint Conference on Learning, Reasoning, pp.\u00a01\u20137 (2021)"},{"key":"17_CR16","unstructured":"Giannini, F., et al.: Interpretable graph networks formulate universal algebra conjectures. Adv. Neural Inf. Process. Syst. 36 (2024)"},{"key":"17_CR17","doi-asserted-by":"crossref","unstructured":"Giles, C.L., Bollacker, K.D., Lawrence, S.: Citeseer: an automatic citation indexing system. In: Proceedings of the Third ACM Conference on Digital Libraries, pp. 89\u201398 (1998)","DOI":"10.1145\/276675.276685"},{"key":"17_CR18","doi-asserted-by":"crossref","unstructured":"Guo, S., Wang, Q., Wang, L., Wang, B., Guo, L.: Knowledge graph embedding with iterative guidance from soft rules. In: Proceedings of the AAAI Conference on Artificial Intelligence, vol.\u00a032 (2018)","DOI":"10.1609\/aaai.v32i1.11918"},{"key":"17_CR19","unstructured":"Hamilton, K., Nayak, A., Bo\u017ei\u0107, B., Longo, L.: Is neuro-symbolic AI meeting its promises in natural language processing? A structured review. Semantic Web (Preprint), 1\u201342 (2022)"},{"key":"17_CR20","doi-asserted-by":"crossref","unstructured":"van Krieken, E., Badreddine, S., Manhaeve, R., Giunchiglia, E.: Uller: a unified language for learning and reasoning. arXiv preprint arXiv:2405.00532 (2024)","DOI":"10.1007\/978-3-031-71167-1_12"},{"key":"17_CR21","doi-asserted-by":"crossref","unstructured":"Li, Z., Huang, J., Naik, M.: Scallop: a language for neurosymbolic programming. Proc. ACM Program. Lang. 7(PLDI), 1463\u20131487 (2023)","DOI":"10.1145\/3591280"},{"key":"17_CR22","unstructured":"Lorello, L.S., Lippi, M.: The challenge of learning symbolic representations. In: d\u2019Avila Garcez, A.S., Besold, T.R., Gori, M., Jim\u00e9nez-Ruiz, E. (eds.) Proceedings of the 17th International Workshop on Neural-Symbolic Learning and Reasoning, La Certosa di Pontignano, Siena, Italy, 3\u20135 July 2023. CEUR Workshop Proceedings, vol.\u00a03432, pp. 44\u201361. CEUR-WS.org (2023). https:\/\/ceur-ws.org\/Vol-3432\/paper4.pdf"},{"key":"17_CR23","unstructured":"Maene, J., De Raedt, L.: Soft-unification in deep probabilistic logic. Adv. Neural Inf. Process. Syst. 36 (2024)"},{"key":"17_CR24","unstructured":"Manhaeve, R., Dumancic, S., Kimmig, A., Demeester, T., De Raedt, L.: Deepproblog: neural probabilistic logic programming. Adv. Neural Inf. Process. Syst. 31 (2018)"},{"key":"17_CR25","unstructured":"Marconato, E., Bontempo, G., Ficarra, E., Calderara, S., Passerini, A., Teso, S.: Neuro-symbolic continual learning: knowledge, reasoning shortcuts and concept rehearsal. In: Proceedings of the 40th International Conference on Machine Learning, pp. 23915\u201323936 (2023)"},{"key":"17_CR26","unstructured":"Marra, G., Diligenti, M., Giannini, F.: Relational reasoning networks. arXiv preprint arXiv:2106.00393 (2021)"},{"key":"17_CR27","doi-asserted-by":"publisher","DOI":"10.1016\/J.ARTINT.2023.104062","volume":"328","author":"G Marra","year":"2024","unstructured":"Marra, G., Dumancic, S., Manhaeve, R., De Raedt, L.: From statistical relational to neurosymbolic artificial intelligence: a survey. Artif. Intell. 328, 104062 (2024). https:\/\/doi.org\/10.1016\/J.ARTINT.2023.104062","journal-title":"Artif. Intell."},{"key":"17_CR28","series-title":"Lecture Notes in Computer Science (Lecture Notes in Artificial Intelligence)","doi-asserted-by":"publisher","first-page":"517","DOI":"10.1007\/978-3-030-46147-8_31","volume-title":"Machine Learning and Knowledge Discovery in Databases","author":"G Marra","year":"2020","unstructured":"Marra, G., Giannini, F., Diligenti, M., Gori, M.: Integrating learning and reasoning with deep logic models. In: Brefeld, U., Fromont, E., Hotho, A., Knobbe, A., Maathuis, M., Robardet, C. (eds.) ECML PKDD 2019. LNCS (LNAI), vol. 11907, pp. 517\u2013532. Springer, Cham (2020). https:\/\/doi.org\/10.1007\/978-3-030-46147-8_31"},{"key":"17_CR29","unstructured":"Marra, G., Ku\u017eelka, O.: Neural markov logic networks. In: Uncertainty in Artificial Intelligence, pp. 908\u2013917. PMLR (2021)"},{"key":"17_CR30","doi-asserted-by":"crossref","unstructured":"Minervini, P., Bosnjak, M., Rockt\u00e4schel, T., Riedel, S., Grefenstette, E.: Differentiable reasoning on large knowledge bases and natural language. In: AAAI, pp. 5182\u20135190. AAAI Press (2020)","DOI":"10.1609\/aaai.v34i04.5962"},{"key":"17_CR31","doi-asserted-by":"publisher","DOI":"10.1016\/j.artint.2021.103546","volume":"300","author":"H M\u00fcller","year":"2021","unstructured":"M\u00fcller, H., Holzinger, A.: Kandinsky patterns. Artif. Intell. 300, 103546 (2021)","journal-title":"Artif. Intell."},{"key":"17_CR32","doi-asserted-by":"crossref","unstructured":"Pryor, C., Dickens, C., Augustine, E., Albalak, A., Wang, W.Y., Getoor, L.: Neupsl: neural probabilistic soft logic. In: IJCAI, pp. 4145\u20134153. ijcai.org (2023)","DOI":"10.24963\/ijcai.2023\/461"},{"key":"17_CR33","unstructured":"Rockt\u00e4schel, T., Riedel, S.: End-to-end differentiable proving. Adv. Neural Inf. Process. Syst. 30 (2017)"},{"key":"17_CR34","unstructured":"Sadeghian, A., Armandpour, M., Ding, P., Wang, D.Z.: Drum: end-to-end differentiable rule mining on knowledge graphs. Adv. Neural Inf. Process. Syst. 32 (2019)"},{"issue":"5","key":"17_CR35","doi-asserted-by":"publisher","first-page":"1465","DOI":"10.1007\/s10994-023-06320-1","volume":"112","author":"H Shindo","year":"2023","unstructured":"Shindo, H., Pfanschilling, V., Dhami, D.S., Kersting, K.: $$\\alpha $$ilp: thinking visual scenes as differentiable logic programs. Mach. Learn. 112(5), 1465\u20131497 (2023)","journal-title":"Mach. Learn."},{"key":"17_CR36","doi-asserted-by":"crossref","unstructured":"Skryagin, A., Stammer, W., Ochs, D., Dhami, D.S., Kersting, K.: Neural-probabilistic answer set programming. In: KR (2022)","DOI":"10.24963\/kr.2022\/48"},{"key":"17_CR37","doi-asserted-by":"publisher","first-page":"69","DOI":"10.1613\/jair.1.11203","volume":"62","author":"G Sourek","year":"2018","unstructured":"Sourek, G., Aschenbrenner, V., Zelezny, F., Schockaert, S., Kuzelka, O.: Lifted relational neural networks: efficient learning of latent relational structures. J. Artif. Intell. Res. 62, 69\u2013100 (2018)","journal-title":"J. Artif. Intell. Res."},{"key":"17_CR38","doi-asserted-by":"crossref","unstructured":"Spillo, G., De\u00a0Filippo, A., Musto, C., Milano, M., Semeraro, G.: Towards sustainability-aware recommender systems: analyzing the trade-off between algorithms performance and carbon footprint. In: Proceedings of the 17th ACM Conference on Recommender Systems, pp. 856\u2013862 (2023)","DOI":"10.1145\/3604915.3608840"},{"key":"17_CR39","doi-asserted-by":"crossref","unstructured":"Stammer, W., Schramowski, P., Kersting, K.: Right for the right concept: revising neuro-symbolic concepts by interacting with their explanations. In: Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition, pp. 3619\u20133629 (2021)","DOI":"10.1109\/CVPR46437.2021.00362"},{"key":"17_CR40","unstructured":"Stoian, M.C., Giunchiglia, E., Lukasiewicz, T.: Exploiting t-norms for deep learning in autonomous driving. arXiv preprint arXiv:2402.11362 (2024)"},{"key":"17_CR41","unstructured":"\u0160vec, P., Balogh, \u0160., Homola, M., Kl\u2019uka, J., Bist\u00e1k, T.: Semantic data representation for explainable windows malware detection models. arXiv preprint arXiv:2403.11669 (2024)"},{"key":"17_CR42","doi-asserted-by":"publisher","unstructured":"Tsamoura, E., Hospedales, T.M., Michael, L.: Neural-symbolic integration: a compositional perspective. In: 35th Conference on Artificial Intelligence, AAAI 2021, Virtual Event, 2\u20139 February 2021, pp. 5051\u20135060. AAAI Press (2021). https:\/\/doi.org\/10.1609\/aaai.v35i6.16639","DOI":"10.1609\/aaai.v35i6.16639"},{"key":"17_CR43","unstructured":"Varbella, A., Amara, K., Gjorgiev, B., Sansavini, G.: Powergraph: a power grid benchmark dataset for graph neural networks. In: NeurIPS 2023 Workshop: New Frontiers in Graph Learning (2023)"},{"key":"17_CR44","doi-asserted-by":"publisher","unstructured":"Vermeulen, A., Manhaeve, R., Marra, G.: An experimental overview of neural-symbolic systems. In: International Conference on Inductive Logic Programming, pp. 124\u2013138. Springer, Heidelberg (2023). https:\/\/doi.org\/10.1007\/978-3-031-49299-0_9","DOI":"10.1007\/978-3-031-49299-0_9"},{"key":"17_CR45","doi-asserted-by":"crossref","unstructured":"Weber, L., Minervini, P., M\u00fcnchmeyer, J., Leser, U., Rockt\u00e4schel, T.: Nlprolog: reasoning with weak unification for question answering in natural language. In: ACL (1), pp. 6151\u20136161. Association for Computational Linguistics (2019)","DOI":"10.18653\/v1\/P19-1618"},{"key":"17_CR46","doi-asserted-by":"crossref","unstructured":"Winters, T., Marra, G., Manhaeve, R., De\u00a0Raedt, L.: Deepstochlog: neural stochastic logic programming. In: Proceedings of the AAAI Conference on Artificial Intelligence, vol.\u00a036, pp. 10090\u201310100 (2022)","DOI":"10.1609\/aaai.v36i9.21248"},{"key":"17_CR47","unstructured":"Yang, F., Yang, Z., Cohen, W.W.: Differentiable learning of logical rules for knowledge base reasoning. Adv. Neural Inf. Process. Syst. 30 (2017)"},{"key":"17_CR48","doi-asserted-by":"crossref","unstructured":"Yang, W.C., Marra, G., Rens, G., De Raedt, L.: Safe reinforcement learning via probabilistic logic shields. In: Proceedings IJCAI (2023)","DOI":"10.24963\/ijcai.2023\/637"},{"key":"17_CR49","doi-asserted-by":"crossref","unstructured":"Yang, Z., Ishay, A., Lee, J.: Neurasp: embracing neural networks into answer set programming. In: IJCAI, pp. 1755\u20131762. ijcai.org (2020)","DOI":"10.24963\/ijcai.2020\/243"},{"key":"17_CR50","doi-asserted-by":"crossref","unstructured":"Zeinalipour, K., Iaquinta, T., Angelini, G., Rigutini, L., Maggini, M., Gori, M.: Building bridges of knowledge: innovating education with automated crossword generation. In: 2023 International Conference on Machine Learning and Applications (ICMLA), pp. 1228\u20131236. IEEE (2023)","DOI":"10.1109\/ICMLA58977.2023.00185"},{"key":"17_CR51","doi-asserted-by":"crossref","unstructured":"Zheng, S., et al.: Pharmkg: a dedicated knowledge graph benchmark for biomedical data mining. Brief. Bioinf. 22(4), bbaa344 (2021)","DOI":"10.1093\/bib\/bbaa344"}],"container-title":["Lecture Notes in Computer Science","Learning and Reasoning"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-032-09087-4_17","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,11,20]],"date-time":"2025-11-20T18:21:24Z","timestamp":1763662884000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-3-032-09087-4_17"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,11,21]]},"ISBN":["9783032090867","9783032090874"],"references-count":51,"URL":"https:\/\/doi.org\/10.1007\/978-3-032-09087-4_17","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"value":"0302-9743","type":"print"},{"value":"1611-3349","type":"electronic"}],"subject":[],"published":{"date-parts":[[2025,11,21]]},"assertion":[{"value":"21 November 2025","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"ILP","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"International Joint Conference on Learning and Reasoning","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Nanjing","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"China","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2024","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"20 September 2024","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"22 September 2024","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"33","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"ilp2024","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"https:\/\/www.lamda.nju.edu.cn\/ijclr24\/","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}}]}}