{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,2,21]],"date-time":"2026-02-21T18:42:59Z","timestamp":1771699379336,"version":"3.50.1"},"reference-count":125,"publisher":"Institute of Electrical and Electronics Engineers (IEEE)","issue":"8","license":[{"start":{"date-parts":[[2024,8,1]],"date-time":"2024-08-01T00:00:00Z","timestamp":1722470400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/ieeexplore.ieee.org\/Xplorehelp\/downloads\/license-information\/IEEE.html"},{"start":{"date-parts":[[2024,8,1]],"date-time":"2024-08-01T00:00:00Z","timestamp":1722470400000},"content-version":"am","delay-in-days":0,"URL":"https:\/\/ieeexplore.ieee.org\/Xplorehelp\/downloads\/license-information\/IEEE.html"},{"start":{"date-parts":[[2024,8,1]],"date-time":"2024-08-01T00:00:00Z","timestamp":1722470400000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2024,8,1]],"date-time":"2024-08-01T00:00:00Z","timestamp":1722470400000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"}],"funder":[{"DOI":"10.13039\/100000001","name":"U.S. National Science Foundation","doi-asserted-by":"publisher","award":["2309760"],"award-info":[{"award-number":["2309760"]}],"id":[{"id":"10.13039\/100000001","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000001","name":"U.S. National Science Foundation","doi-asserted-by":"publisher","award":["2317117"],"award-info":[{"award-number":["2317117"]}],"id":[{"id":"10.13039\/100000001","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["IEEE Trans. Artif. Intell."],"published-print":{"date-parts":[[2024,8]]},"DOI":"10.1109\/tai.2024.3351798","type":"journal-article","created":{"date-parts":[[2024,1,9]],"date-time":"2024-01-09T15:57:58Z","timestamp":1704815878000},"page":"3765-3779","source":"Crossref","is-referenced-by-count":18,"title":["A Survey on Verification and Validation, Testing and Evaluations of Neurosymbolic Artificial Intelligence"],"prefix":"10.1109","volume":"5","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-7021-734X","authenticated-orcid":false,"given":"Justus","family":"Renkhoff","sequence":"first","affiliation":[{"name":"Security and Optimization for Networked Globe Laboratory (SONG Lab), Department of Information Systems, University of Maryland, Baltimore, MD, USA"}]},{"ORCID":"https:\/\/orcid.org\/0009-0006-5514-6818","authenticated-orcid":false,"given":"Ke","family":"Feng","sequence":"additional","affiliation":[{"name":"Department of Electrical Engineering and Computer Science, Embry-Riddle Aeronautical University, Daytona Beach, FL, USA"}]},{"ORCID":"https:\/\/orcid.org\/0009-0005-9171-5646","authenticated-orcid":false,"given":"Marc","family":"Meier-Doernberg","sequence":"additional","affiliation":[{"name":"Department of Electrical Engineering and Computer Science, Embry-Riddle Aeronautical University, Daytona Beach, FL, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-6757-105X","authenticated-orcid":false,"given":"Alvaro","family":"Velasquez","sequence":"additional","affiliation":[{"name":"Department of Computer Science, University of Colorado, Boulder, CO, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-2631-9223","authenticated-orcid":false,"given":"Houbing Herbert","family":"Song","sequence":"additional","affiliation":[{"name":"Security and Optimization for Networked Globe Laboratory (SONG Lab), Department of Information Systems, University of Maryland, Baltimore, MD, USA"}]}],"member":"263","reference":[{"key":"ref1","doi-asserted-by":"publisher","DOI":"10.1007\/s10462-023-10448-w"},{"key":"ref2","doi-asserted-by":"publisher","DOI":"10.1063\/1.1144830"},{"key":"ref3","doi-asserted-by":"publisher","DOI":"10.1016\/j.cobeha.2018.12.010"},{"key":"ref4","doi-asserted-by":"publisher","DOI":"10.1109\/tai.2023.3311428"},{"key":"ref5","article-title":"Transfer from imprecise and abstract models to autonomous technologies (TIAMAT)","volume-title":"Defense Advanced Research Projects Agency (DARPA) Program Solicitation","author":"Velasquez","year":"2023"},{"key":"ref6","article-title":"Executive order on the safe, secure, and trustworthy development and use of artificial intelligence","volume-title":"The White House, Oct 2023","author":"Biden","year":"2024"},{"key":"ref7","doi-asserted-by":"publisher","DOI":"10.1145\/3459086.3459636"},{"key":"ref8","doi-asserted-by":"crossref","DOI":"10.1016\/j.cosrev.2020.100270","article-title":"A survey of safety and trustworthiness of deep neural networks: Verification, testing, adversarial attack and defence, and interpretability","volume":"37","author":"Huang","year":"2020","journal-title":"Comput. Sci. Rev."},{"key":"ref9","article-title":"Software verification and validation: Its role in computer assurance and its relationship with software project management standards","author":"Wallace","year":"1989"},{"key":"ref10","doi-asserted-by":"publisher","DOI":"10.1145\/318123.318150"},{"key":"ref11","doi-asserted-by":"publisher","DOI":"10.1109\/52.28119"},{"issue":"1","key":"ref12","first-page":"93","article-title":"The third AI summer: AAAI Robert S. Engelmore memorial lecture","volume":"43","author":"Kautz","year":"2022","journal-title":"AI Mag."},{"key":"ref13","doi-asserted-by":"crossref","first-page":"105","DOI":"10.1016\/j.neunet.2023.06.028","article-title":"A survey on neural-symbolic learning systems","volume":"166","author":"Yu","year":"2023","journal-title":"Neural Netw."},{"key":"ref14","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2019.2962027"},{"key":"ref15","doi-asserted-by":"crossref","DOI":"10.1016\/j.jss.2020.110542","article-title":"On testing machine learning programs","volume":"164","author":"Braiek","year":"2020","journal-title":"J. Syst. Softw."},{"key":"ref16","doi-asserted-by":"publisher","DOI":"10.1007\/s43154-021-00058-1"},{"issue":"1","key":"ref17","first-page":"327","article-title":"Neural-symbolic learning and reasoning: A survey and interpretation","volume":"342","author":"Garcez","year":"2022","journal-title":"Neuro-Symbolic Artif. Intell.: State Art"},{"key":"ref18","doi-asserted-by":"publisher","DOI":"10.1109\/MIS.2023.3268724"},{"key":"ref19","doi-asserted-by":"publisher","DOI":"10.1109\/MIC.2022.3179759"},{"key":"ref20","volume-title":"Neuro-Symbolic Artificial Intelligence: The State of the Art","author":"Hitzler","year":"2022"},{"key":"ref21","doi-asserted-by":"publisher","DOI":"10.3233\/AIC-210084"},{"key":"ref22","article-title":"Neuro-symbolic AI: An emerging class of AI workloads and their characterization","author":"Susskind","year":"2021"},{"key":"ref23","article-title":"Towards data-and knowledge-driven artificial intelligence: A survey on neuro-symbolic computing","author":"Wang","year":"2022"},{"key":"ref24","article-title":"Neurosymbolic AI and its taxonomy: A survey","author":"Gibaut","year":"2023"},{"key":"ref25","article-title":"Neurosymbolic AI for reasoning on graph structures: A survey","author":"DeLong","year":"2023"},{"key":"ref26","article-title":"Neurosymbolic AI for reasoning on biomedical knowledge graphs","author":"DeLong","year":"2023"},{"key":"ref27","doi-asserted-by":"publisher","DOI":"10.3233\/SW-223228"},{"key":"ref28","doi-asserted-by":"publisher","DOI":"10.1017\/S1351324916000334"},{"key":"ref29","doi-asserted-by":"publisher","DOI":"10.3115\/v1\/D14-1162"},{"key":"ref30","doi-asserted-by":"publisher","DOI":"10.1038\/nature24270"},{"key":"ref31","article-title":"The neuro-symbolic concept learner: Interpreting scenes, words, and sentences from natural supervision","author":"Mao"},{"key":"ref32","first-page":"3749","article-title":"DeepProbLog: Neural probabilistic logic programming","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","volume":"31","author":"Manhaeve","year":"2018"},{"key":"ref33","article-title":"Deep learning for symbolic mathematics","author":"Lample","year":"2019"},{"key":"ref34","doi-asserted-by":"publisher","DOI":"10.1145\/3019612.3019642"},{"key":"ref35","article-title":"Basic reasoning with tensor product representations","author":"Smolensky","year":"2016"},{"key":"ref36","volume-title":"Thinking, Fast and Slow","author":"Kahneman","year":"2011"},{"key":"ref37","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v32i1.11797"},{"key":"ref38","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v32i1.11797"},{"key":"ref39","doi-asserted-by":"publisher","DOI":"10.1080\/01445340.2011.621702"},{"key":"ref40","volume-title":"Semantic Entailment and Formal Derivability","author":"Beth","year":"1955"},{"key":"ref41","article-title":"GitHub\u2014wo\/tpg: Tree Proof Generator\u2014github.com","author":"Schwarz","year":"2022"},{"key":"ref42","article-title":"Knowledge graph validation","author":"Huaman"},{"key":"ref43","first-page":"201","article-title":"COPAAL\u2014An interface for explaining facts using corroborative paths","volume-title":"Proc. ISWC (Satellites)","author":"Syed","year":"2019"},{"key":"ref44","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-35176-1_20"},{"key":"ref45","doi-asserted-by":"publisher","DOI":"10.1145\/3308560.3316524"},{"key":"ref46","doi-asserted-by":"publisher","DOI":"10.1145\/3269206.3269308"},{"key":"ref47","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-21348-0_4"},{"key":"ref48","doi-asserted-by":"publisher","DOI":"10.1016\/j.websem.2018.12.006"},{"key":"ref49","article-title":"SURFACE: Semantically rich fact validation with explanations","author":"Padia","year":"2018"},{"key":"ref50","doi-asserted-by":"publisher","DOI":"10.1145\/2063576.2063587"},{"key":"ref51","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-46547-0_19"},{"key":"ref52","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v31i1.11164"},{"key":"ref53","doi-asserted-by":"publisher","DOI":"10.1609\/aimag.v31i3.2303"},{"key":"ref54","doi-asserted-by":"publisher","DOI":"10.1145\/219717.219748"},{"key":"ref55","article-title":"A systematic review of robustness in deep learning for computer vision: Mind the gap?,\u201d","author":"Drenkow","year":"2022"},{"key":"ref56","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-89960-2_22"},{"key":"ref57","doi-asserted-by":"publisher","DOI":"10.24963\/ijcai.2018\/368"},{"key":"ref58","doi-asserted-by":"crossref","first-page":"298","DOI":"10.1016\/j.tcs.2019.05.046","article-title":"A game-based approximate verification of deep neural networks with provable guarantees","volume":"807","author":"Wu","year":"2020","journal-title":"Theor. Comput. Sci."},{"key":"ref59","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-63387-9_5"},{"key":"ref60","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-68167-2_19"},{"key":"ref61","first-page":"4790","article-title":"A unified view of piecewise linear neural network verification","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","volume":"31","author":"Bunel","year":"2018"},{"key":"ref62","article-title":"An approach to reachability analysis for feed-forward ReLU neural networks","author":"Lomuscio","year":"2017"},{"key":"ref63","doi-asserted-by":"publisher","DOI":"10.1109\/TNNLS.2018.2808470"},{"key":"ref64","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-68167-2_18"},{"key":"ref65","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v32i1.12206"},{"key":"ref66","doi-asserted-by":"publisher","DOI":"10.24963\/ijcai.2018\/811"},{"key":"ref67","first-page":"3","article-title":"AI2: Safety and robustness certification of neural networks with abstract interpretation [paper presentation]","volume-title":"Proc. IEEE Symp. Secur. Privacy (SP)","author":"Gehr","year":"2018"},{"key":"ref68","first-page":"1599","article-title":"Formal security analysis of neural networks using symbolic intervals","volume-title":"Proc. 27th USENIX Secur. Symp. (USENIX Secur.)","author":"Wang","year":"2018"},{"key":"ref69","article-title":"Certified defenses against adversarial examples","volume-title":"Proc. Int. Conf. Learn. Represent.","author":"Raghunathan","year":"2018"},{"key":"ref70","doi-asserted-by":"publisher","DOI":"10.24963\/ijcai.2019\/824"},{"key":"ref71","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-63387-9_1"},{"key":"ref72","article-title":"Output range analysis for deep neural networks","author":"Dutta","year":"2017"},{"key":"ref73","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-14295-6_24"},{"key":"ref74","first-page":"5286","article-title":"Provable defenses against adversarial examples via the convex outer adversarial polytope","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Wong","year":"2018"},{"key":"ref75","first-page":"3578","article-title":"Differentiable abstract interpretation for provably robust neural networks","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Mirman","year":"2018"},{"key":"ref76","doi-asserted-by":"publisher","DOI":"10.1067\/mod.2000.109032"},{"key":"ref77","doi-asserted-by":"publisher","DOI":"10.1201\/9780429246593"},{"key":"ref78","first-page":"6","article-title":"Why question machine learning evaluation methods","volume-title":"Proc. AAAI Workshop Eval. Methods Mach. Learn.","author":"Japkowicz","year":"2006"},{"key":"ref79","doi-asserted-by":"publisher","DOI":"10.1145\/3183713.3196934"},{"key":"ref80","doi-asserted-by":"publisher","DOI":"10.1145\/3077257.3077271"},{"key":"ref81","doi-asserted-by":"publisher","DOI":"10.1109\/MET.2017.2"},{"key":"ref82","first-page":"174","article-title":"Generalized oracle for testing machine learning computer programs","volume-title":"Proc. Softw. Eng. Formal Methods (SEFM) Collocated Workshops: DataMod, FAACS, MSE, CoSim-CPS, FOCLASA","author":"Nakajima","year":"2017"},{"key":"ref83","doi-asserted-by":"publisher","DOI":"10.1145\/3213846.3213858"},{"key":"ref84","doi-asserted-by":"publisher","DOI":"10.1145\/3193977.3193980"},{"key":"ref85","doi-asserted-by":"publisher","DOI":"10.1109\/BigData.2017.8258038"},{"key":"ref86","doi-asserted-by":"publisher","DOI":"10.1145\/3236024.3236082"},{"issue":"5","key":"ref87","article-title":"The data linter: Lightweight, automated sanity checking for ml data sets","volume-title":"Proc. NIPS MLSys Workshop","volume":"1","author":"Hynes","year":"2017"},{"key":"ref88","doi-asserted-by":"publisher","DOI":"10.14778\/3229863.3229867"},{"key":"ref89","doi-asserted-by":"publisher","DOI":"10.1145\/3132747.3132785"},{"key":"ref90","doi-asserted-by":"publisher","DOI":"10.1145\/3293882.3330579"},{"key":"ref91","doi-asserted-by":"publisher","DOI":"10.1109\/TNSE.2020.2997359"},{"key":"ref92","doi-asserted-by":"publisher","DOI":"10.1145\/3133956.3134057"},{"key":"ref93","doi-asserted-by":"publisher","DOI":"10.1109\/FUZZ52849.2023.10309766"},{"key":"ref94","doi-asserted-by":"crossref","first-page":"278","DOI":"10.1016\/j.neucom.2022.04.020","article-title":"Adversarial attack and defense technologies in natural language processing: A survey","volume":"492","author":"Qiu","year":"2022","journal-title":"Neurocomputing"},{"key":"ref95","doi-asserted-by":"publisher","DOI":"10.1109\/ACCESS.2020.2974752"},{"issue":"1","key":"ref96","first-page":"8","article-title":"Explanation and justification in machine learning: A survey","volume-title":"Proc. IJCAI Workshop Explainable AI (XAI)","volume":"8","author":"Biran","year":"2017"},{"key":"ref97","volume-title":"Interpretable Machine Learning","author":"Molnar","year":"2022"},{"key":"ref98","doi-asserted-by":"publisher","DOI":"10.1177\/000312240607100202"},{"key":"ref99","doi-asserted-by":"publisher","DOI":"10.2307\/1268249"},{"key":"ref100","first-page":"6172","article-title":"Neurosymbolic reinforcement learning with formally verified exploration","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","volume":"33","author":"Anderson","year":"2020"},{"key":"ref101","doi-asserted-by":"publisher","DOI":"10.32657\/10356\/90191"},{"key":"ref102","first-page":"22","article-title":"Constrained policy optimization","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Achiam","year":"2017"},{"key":"ref103","article-title":"A review of safe reinforcement learning: Methods, theory and applications","author":"Gu","year":"2022"},{"key":"ref104","article-title":"Verifiable reinforcement learning via policy extraction","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","volume":"31","author":"Bastani","year":"2018"},{"key":"ref105","article-title":"Learning to infer and execute 3D shape programs","author":"Tian","year":"2019"},{"key":"ref106","first-page":"25146","article-title":"Learning to synthesize programs as interpretable and generalizable policies","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","volume":"34","author":"Trivedi","year":"2021"},{"key":"ref107","article-title":"LASER: Neuro-symbolic learning of semantic video representations","author":"Huang","year":"2023"},{"key":"ref108","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2018.00717"},{"key":"ref109","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v24i1.7519"},{"key":"ref110","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2013.178"},{"key":"ref111","doi-asserted-by":"publisher","DOI":"10.1007\/s11263-015-0816-y"},{"key":"ref112","doi-asserted-by":"publisher","DOI":"10.1145\/3292500.3330905"},{"issue":"43","key":"ref113","first-page":"1","article-title":"Benchmarking graph neural networks","volume":"24","author":"Dwivedi","year":"2023","journal-title":"J. Mach. Learn. Res."},{"key":"ref114","doi-asserted-by":"publisher","DOI":"10.1038\/s41524-021-00554-0"},{"key":"ref115","doi-asserted-by":"publisher","DOI":"10.1080\/01621459.1949.10483310"},{"key":"ref116","first-page":"3749","article-title":"DeepProbLog: Neural probabilistic logic programming","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","volume":"31","author":"Manhaeve","year":"2018"},{"key":"ref117","doi-asserted-by":"publisher","DOI":"10.1145\/2593882.2593900"},{"key":"ref118","doi-asserted-by":"publisher","DOI":"10.1145\/3236024.3264835"},{"key":"ref119","first-page":"547","article-title":"Programming with a differentiable forth interpreter","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Bosnjak","year":"2017"},{"key":"ref120","first-page":"4901","article-title":"TensorFuzz: Debugging neural networks with coverage-guided fuzzing","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Odena","year":"2019"},{"key":"ref121","first-page":"819","article-title":"SDD: A new canonical representation of propositional knowledge bases","volume-title":"Proc. 22nd Int. Joint Conf. Artif. Intell.","author":"Darwiche","year":"2011"},{"key":"ref122","doi-asserted-by":"crossref","first-page":"199","DOI":"10.1016\/j.future.2018.02.044","article-title":"Sustainable software products\u2014Towards assessment criteria for resource and energy efficiency","volume":"86","author":"Kern","year":"2018","journal-title":"Future Gener. Comput. Syst."},{"key":"ref123","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-18311-9_11"},{"key":"ref124","doi-asserted-by":"publisher","DOI":"10.1109\/icsc59802.2024.00034"},{"key":"ref125","doi-asserted-by":"publisher","DOI":"10.1145\/3591280"}],"container-title":["IEEE Transactions on Artificial Intelligence"],"original-title":[],"link":[{"URL":"https:\/\/ieeexplore.ieee.org\/ielam\/9078688\/10635096\/10385139-aam.pdf","content-type":"application\/pdf","content-version":"am","intended-application":"syndication"},{"URL":"http:\/\/xplorestaging.ieee.org\/ielx7\/9078688\/10635096\/10385139.pdf?arnumber=10385139","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,8,23]],"date-time":"2025-08-23T01:09:11Z","timestamp":1755911351000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/10385139\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,8]]},"references-count":125,"journal-issue":{"issue":"8"},"URL":"https:\/\/doi.org\/10.1109\/tai.2024.3351798","relation":{},"ISSN":["2691-4581"],"issn-type":[{"value":"2691-4581","type":"electronic"}],"subject":[],"published":{"date-parts":[[2024,8]]}}}