{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,5,20]],"date-time":"2026-05-20T16:09:46Z","timestamp":1779293386411,"version":"3.51.4"},"reference-count":183,"publisher":"Wiley","issue":"2","license":[{"start":{"date-parts":[[2025,4,20]],"date-time":"2025-04-20T00:00:00Z","timestamp":1745107200000},"content-version":"vor","delay-in-days":19,"URL":"http:\/\/creativecommons.org\/licenses\/by-nc-nd\/4.0\/"}],"funder":[{"DOI":"10.13039\/100010661","name":"Horizon 2020 Framework Programme","doi-asserted-by":"publisher","award":["MSCA\u2010ETN SAS"],"award-info":[{"award-number":["MSCA\u2010ETN SAS"]}],"id":[{"id":"10.13039\/100010661","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100010661","name":"Horizon 2020 Framework Programme","doi-asserted-by":"publisher","award":["grant agreement No 812.788"],"award-info":[{"award-number":["grant agreement No 812.788"]}],"id":[{"id":"10.13039\/100010661","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["onlinelibrary.wiley.com"],"crossmark-restriction":true},"short-container-title":["Computational Intelligence"],"published-print":{"date-parts":[[2025,4]]},"abstract":"<jats:title>ABSTRACT<\/jats:title><jats:p>Machine Learning (ML) models, such as deep neural networks, are widely applied in autonomous systems to perform complex perception tasks. New dependability challenges arise when ML predictions are used in safety\u2010critical applications, like autonomous cars and surgical robots. Thus, the use of fault tolerance mechanisms, such as safety monitors, is essential to ensure the safe behavior of the system despite the occurrence of faults. This paper presents an extensive literature review on safety monitoring of perception functions using ML in a safety\u2010critical context. In this review, we structure the existing literature to highlight key factors to consider when designing such monitors: threat identification, requirements elicitation, detection of failure, reaction, and evaluation. We also highlight the ongoing challenges associated with safety monitoring and suggest directions for future research.<\/jats:p>","DOI":"10.1111\/coin.70032","type":"journal-article","created":{"date-parts":[[2025,4,21]],"date-time":"2025-04-21T04:36:36Z","timestamp":1745210196000},"update-policy":"https:\/\/doi.org\/10.1002\/crossmark_policy","source":"Crossref","is-referenced-by-count":5,"title":["Safety Monitoring of Machine Learning Perception Functions: A Survey"],"prefix":"10.1111","volume":"41","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-4629-8821","authenticated-orcid":false,"given":"Raul Sena","family":"Ferreira","sequence":"first","affiliation":[{"name":"LAAS\/CNRS  Toulouse France"},{"name":"Universit\u00e9 de Toulouse  Toulouse France"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Joris","family":"Gu\u00e9rin","sequence":"additional","affiliation":[{"name":"LAAS\/CNRS  Toulouse France"},{"name":"Universit\u00e9 de Toulouse  Toulouse France"},{"name":"Espace\u2010Dev, IRD, Universit\u00e9 de Montpellier  Montpellier France"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Kevin","family":"Delmas","sequence":"additional","affiliation":[{"name":"ONERA  Toulouse France"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"J\u00e9r\u00e9mie","family":"Guiochet","sequence":"additional","affiliation":[{"name":"LAAS\/CNRS  Toulouse France"},{"name":"Universit\u00e9 de Toulouse  Toulouse France"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"H\u00e9l\u00e8ne","family":"Waeselynck","sequence":"additional","affiliation":[{"name":"LAAS\/CNRS  Toulouse France"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"311","published-online":{"date-parts":[[2025,4,20]]},"reference":[{"key":"e_1_2_14_2_1","volume-title":"Runtime Monitoring of Cyber\u2010Physical Systems Using Data\u2010Driven Models","author":"Calvi M. G.","year":"2019"},{"key":"e_1_2_14_3_1","doi-asserted-by":"publisher","DOI":"10.1109\/TMRB.2019.2913282"},{"key":"e_1_2_14_4_1","doi-asserted-by":"publisher","DOI":"10.1109\/DSN-W52860.2021.00020"},{"key":"e_1_2_14_5_1","volume-title":"Intelligent Robotic Perception Systems. In: Hurtado Efren Gorrostieta, Ed","author":"Premebida C.","year":"2018"},{"key":"e_1_2_14_6_1","first-page":"1329","volume-title":"International Conference on Machine Learning","author":"Duan Y.","year":"2016"},{"key":"e_1_2_14_7_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.neucom.2018.01.092"},{"key":"e_1_2_14_8_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICRA46639.2022.9811924"},{"key":"e_1_2_14_9_1","doi-asserted-by":"publisher","DOI":"10.1089\/big.2016.0051"},{"key":"e_1_2_14_10_1","first-page":"6","volume-title":"Proceedings of the 26th Safety\u2010Critical Systems Symposium, York, UK","author":"Faria J. M.","year":"2018"},{"key":"e_1_2_14_11_1","first-page":"162","volume-title":"Proceedings of the Workshop on Artificial Intelligence Safety","author":"Mohseni S.","year":"2020"},{"key":"e_1_2_14_12_1","first-page":"1050","volume-title":"International Conference on Machine Learning (ICML), new York, United States","author":"Gal Y.","year":"2016"},{"key":"e_1_2_14_13_1","doi-asserted-by":"publisher","DOI":"10.1109\/TDSC.2004.2"},{"key":"e_1_2_14_14_1","doi-asserted-by":"publisher","DOI":"10.1109\/TSMC.2016.2633291"},{"key":"e_1_2_14_15_1","first-page":"210","article-title":"Kernels for Safety","volume":"13","author":"Rushby J.","year":"1989","journal-title":"Safe and Secure Computing Systems"},{"key":"e_1_2_14_16_1","volume-title":"International Symposium on Automation and Robotics in Construction","author":"Pace C.","year":"2000"},{"key":"e_1_2_14_17_1","doi-asserted-by":"publisher","DOI":"10.1109\/MIS.2004.53"},{"key":"e_1_2_14_18_1","doi-asserted-by":"publisher","DOI":"10.1109\/IROS.2004.1389549"},{"key":"e_1_2_14_19_1","article-title":"Safe and Sound","volume":"307","author":"Fox J.","year":"2000","journal-title":"Artificial Intelligence in Hazardous Applications"},{"key":"e_1_2_14_20_1","first-page":"1","volume-title":"Operational Expert System Applications in Europe","author":"Klein P.","year":"1991"},{"key":"e_1_2_14_21_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-19457-3_16"},{"key":"e_1_2_14_22_1","volume-title":"Safety Monitoring for Autonomous Systems: Interactive Elicitation of Safety Rules","author":"Masson L.","year":"2019"},{"key":"e_1_2_14_23_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.ress.2021.107530"},{"key":"e_1_2_14_24_1","doi-asserted-by":"publisher","DOI":"10.1109\/TITS.2020.3032227"},{"key":"e_1_2_14_25_1","doi-asserted-by":"publisher","DOI":"10.3390\/app11020821"},{"key":"e_1_2_14_26_1","doi-asserted-by":"publisher","DOI":"10.1109\/JSEN.2023.3262134"},{"key":"e_1_2_14_27_1","doi-asserted-by":"publisher","DOI":"10.1145\/3551385"},{"key":"e_1_2_14_28_1","doi-asserted-by":"publisher","DOI":"10.1109\/ACCESS.2021.3055015"},{"key":"e_1_2_14_29_1","doi-asserted-by":"publisher","DOI":"10.1109\/JPROC.2023.3238024"},{"key":"e_1_2_14_30_1","volume-title":"The 37th AAAI Conference on Artificial Intelligence (2023)","author":"Guerin J.","year":"2023"},{"issue":"16","key":"e_1_2_14_31_1","first-page":"4102","article-title":"Review of Data Preprocessing Techniques in Data Mining","volume":"12","author":"Alasadi S.","year":"2017","journal-title":"Journal of Engineering and Applied Sciences"},{"key":"e_1_2_14_32_1","first-page":"2682","volume-title":"IEEE International Conference on Acoustics, Speech and Signal Processing, China","author":"Bekker A. J.","year":"2016"},{"key":"e_1_2_14_33_1","doi-asserted-by":"publisher","DOI":"10.1109\/BigData.2017.8258038"},{"key":"e_1_2_14_34_1","first-page":"2503","volume-title":"Advances in Neural Information Processing Systems (NeurIPS), Montreal, Canada","author":"Sculley D.","year":"2015"},{"key":"e_1_2_14_35_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.sigpro.2013.12.026"},{"key":"e_1_2_14_36_1","unstructured":"A.Chakraborty M.Alam V.Dey A.Chattopadhyay andD.Mukhopadhyay \u201cAdversarial Attacks and Defences: A Survey \u201d2018arXiv preprint arXiv:1810.00069."},{"key":"e_1_2_14_37_1","first-page":"5669","article-title":"DOCTOR: A Simple Method for Detecting Misclassification Errors","volume":"34","author":"Granese F.","year":"2021","journal-title":"Advances in Neural Information Processing Systems"},{"key":"e_1_2_14_38_1","doi-asserted-by":"publisher","DOI":"10.1109\/PRDC53464.2021.00012"},{"key":"e_1_2_14_39_1","unstructured":"Z.Shen J.Liu Y.He et al. \u201cTowards Out\u2010of\u2010Distribution Generalization: A Survey \u201d(2021) arXiv preprint arXiv:2108.13624."},{"key":"e_1_2_14_40_1","doi-asserted-by":"publisher","DOI":"10.1109\/PRDC55274.2022.00021"},{"key":"e_1_2_14_41_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2016.350"},{"key":"e_1_2_14_42_1","volume-title":"European Conference on Computer Vision (ECCV)","author":"Yuan Y.","year":"2021"},{"key":"e_1_2_14_43_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2009.5206848"},{"key":"e_1_2_14_44_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-10602-1_48"},{"key":"e_1_2_14_45_1","doi-asserted-by":"crossref","unstructured":"Z.Liu H.Hu Y.Lin et al. \u201cSwin Transformer V2: Scaling Up Capacity and Resolution \u201d2021arXiv Preprint arXiv:2111.09883.","DOI":"10.1109\/CVPR52688.2022.01170"},{"key":"e_1_2_14_46_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-99229-7_39"},{"key":"e_1_2_14_47_1","unstructured":"J.Yang K.Zhou Y.Li andZ.Liu \u201cGeneralized Out\u2010of\u2010Distribution Detection: A Survey \u201d2021Preprint arXiv:2110.11334."},{"key":"e_1_2_14_48_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICCVW.2019.00294"},{"key":"e_1_2_14_49_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.patcog.2016.10.011"},{"key":"e_1_2_14_50_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.ins.2019.03.025"},{"key":"e_1_2_14_51_1","volume-title":"International Conference on Learning Representations","author":"Hendrycks D.","year":"2019"},{"key":"e_1_2_14_52_1","first-page":"15","volume-title":"Proceedings of the 2013 International Conference on Autonomous Agents and Multi\u2010Agent Systems","author":"Khalastchi E.","year":"2013"},{"key":"e_1_2_14_53_1","doi-asserted-by":"publisher","DOI":"10.1109\/TASE.2009.2017735"},{"key":"e_1_2_14_54_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICOEI.2018.8553680"},{"key":"e_1_2_14_55_1","doi-asserted-by":"publisher","DOI":"10.1038\/s41597-021-01083-7"},{"key":"e_1_2_14_56_1","first-page":"27","volume-title":"Proceedings of GSPX","author":"Motwani M. C.","year":"2004"},{"key":"e_1_2_14_57_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.neunet.2020.07.025"},{"key":"e_1_2_14_58_1","volume-title":"Proceedings of the European Conference on Computer Vision (ECCV) Workshops","author":"Rasouli A.","year":"2018"},{"key":"e_1_2_14_59_1","unstructured":"R.Stumpf \u201cAutopilot Blamed for Tesla's Crash Into Overturned Truck \u201d2020."},{"key":"e_1_2_14_60_1","doi-asserted-by":"publisher","DOI":"10.1109\/TMM.2020.3020691"},{"key":"e_1_2_14_61_1","doi-asserted-by":"publisher","DOI":"10.1109\/ACCESS.2018.2807385"},{"key":"e_1_2_14_62_1","doi-asserted-by":"publisher","DOI":"10.1201\/9781351251389-8"},{"key":"e_1_2_14_63_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.sysarc.2020.101766"},{"key":"e_1_2_14_64_1","doi-asserted-by":"publisher","DOI":"10.1109\/HASE.2010.10"},{"key":"e_1_2_14_65_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10817-018-09509-5"},{"key":"e_1_2_14_66_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-25540-4_25"},{"key":"e_1_2_14_67_1","doi-asserted-by":"publisher","DOI":"10.1109\/ISSRE.2019.00013"},{"key":"e_1_2_14_68_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-01246-5_32"},{"key":"e_1_2_14_69_1","doi-asserted-by":"publisher","DOI":"10.1109\/ITSC.2019.8917207"},{"key":"e_1_2_14_70_1","doi-asserted-by":"publisher","DOI":"10.1109\/TRO.2020.3001674"},{"key":"e_1_2_14_71_1","unstructured":"G.Chhablani A.Sharma H.Pandey andT.Dash \u201cSuperpixel\u2010Based Domain\u2010Knowledge Infusion in Computer Vision \u201d2021arXiv Preprint arXiv:2105.09448."},{"key":"e_1_2_14_72_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2015.7298713"},{"key":"e_1_2_14_73_1","first-page":"5502","volume-title":"International Conference on Machine Learning","author":"Xu J.","year":"2018"},{"key":"e_1_2_14_74_1","doi-asserted-by":"publisher","DOI":"10.24963\/ijcai.2017\/221"},{"key":"e_1_2_14_75_1","first-page":"1660","article-title":"Boosting With Abstention","volume":"29","author":"Cortes C.","year":"2016","journal-title":"Advances in Neural Information Processing Systems"},{"key":"e_1_2_14_76_1","first-page":"2151","volume-title":"International Conference on Machine Learning","author":"Geifman Y.","year":"2019"},{"key":"e_1_2_14_77_1","doi-asserted-by":"publisher","DOI":"10.1109\/TITS.2019.2909066"},{"key":"e_1_2_14_78_1","doi-asserted-by":"publisher","DOI":"10.1109\/LRA.2018.2857402"},{"key":"e_1_2_14_79_1","unstructured":"D.HendrycksandK.Gimpel \u201cA Baseline for Detecting Misclassified and Out\u2010of\u2010Distribution Examples in Neural Networks \u201d2016arXiv Preprint arXiv:1610.02136."},{"key":"e_1_2_14_80_1","volume-title":"International Conference on Learning Representations","author":"Liang S.","year":"2018"},{"key":"e_1_2_14_81_1","first-page":"10951","volume-title":"Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition","author":"Hsu Y.\u2010C.","year":"2020"},{"key":"e_1_2_14_82_1","volume-title":"NeurIPS MLSys Workshop","author":"Kang D.","year":"2018"},{"key":"e_1_2_14_83_1","unstructured":"C.Harper G.Chance A.Ghobrial S.Alam T.Pipe andK.Eder \u201cSafety Validation of Autonomous Vehicles Using Assertion\u2010Based Oracles \u201d2021arXiv Preprint arXiv:2111.04611."},{"key":"e_1_2_14_84_1","doi-asserted-by":"publisher","DOI":"10.1109\/IROS51168.2021.9636713"},{"key":"e_1_2_14_85_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICMLA51294.2020.00113"},{"key":"e_1_2_14_86_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.asoc.2019.105613"},{"key":"e_1_2_14_87_1","volume-title":"British Machine Vision Conference","author":"Roitberg A.","year":"2018"},{"key":"e_1_2_14_88_1","unstructured":"Y.Kantaros T.Carpenter S.Park et al. \u201cVisionGuard: Runtime Detection of Adversarial Inputs to Perception Systems \u201d2020arXiv Preprint arXiv:2002.09792."},{"key":"e_1_2_14_89_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE.2019.00126"},{"key":"e_1_2_14_90_1","unstructured":"J.Liu L.Chen A.Mine andJ.Wang \u201cInput Validation for Neural Networks via Runtime Local Robustness Verification \u201d2020arXiv Preprint arXiv:2002.03339."},{"key":"e_1_2_14_91_1","doi-asserted-by":"publisher","DOI":"10.1109\/IROS40897.2019.8968525"},{"key":"e_1_2_14_92_1","first-page":"144","volume-title":"Advances in Neural Information Processing Systems","author":"Sun Y.","year":"2021"},{"key":"e_1_2_14_93_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-88494-9_3"},{"key":"e_1_2_14_94_1","doi-asserted-by":"publisher","DOI":"10.23919\/DATE.2019.8714971"},{"key":"e_1_2_14_95_1","doi-asserted-by":"publisher","DOI":"10.3233\/FAIA200375"},{"key":"e_1_2_14_96_1","volume-title":"26th European Conference on Artificial Intelligence\u2010ECAI 2023","author":"Ferreira R. S.","year":"2023"},{"key":"e_1_2_14_97_1","unstructured":"C.Wu Y.Falcone andS.Bensalem \u201cCustomizable Reference Runtime Monitoring of Neural Networks Using Resolution Boxes \u201d2021arXiv Preprint arXiv:2104.14435."},{"key":"e_1_2_14_98_1","doi-asserted-by":"publisher","DOI":"10.1145\/3377811.3380379"},{"key":"e_1_2_14_99_1","doi-asserted-by":"publisher","DOI":"10.1109\/ITSC45102.2020.9294226"},{"key":"e_1_2_14_100_1","first-page":"11060","article-title":"A Simple Unified Framework for Detecting Out\u2010of\u2010Distribution Samples and Adversarial Attacks","volume":"31","author":"Lee K.","year":"2018","journal-title":"Advances in Neural Information Processing Systems"},{"key":"e_1_2_14_101_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICRA40945.2020.9196720"},{"key":"e_1_2_14_102_1","first-page":"32","volume-title":"Proc. 2011 3rd International Conference on Evolving Internet","author":"Ndong J.","year":"2011"},{"key":"e_1_2_14_103_1","doi-asserted-by":"publisher","DOI":"10.1109\/JSAC.2006.877215"},{"key":"e_1_2_14_104_1","doi-asserted-by":"publisher","DOI":"10.3923\/itj.2013.569.583"},{"key":"e_1_2_14_105_1","first-page":"3379","volume-title":"Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition","author":"Sabokrou M.","year":"2018"},{"key":"e_1_2_14_106_1","unstructured":"T.Denouden R.Salay K.Czarnecki V.Abdelzad B.Phan andS.Vernekar \u201cImproving Reconstruction Autoencoder Out\u2010of\u2010Distribution Detection With Mahalanobis Distance \u201d(2018) arXiv Preprint arXiv:1812.02765."},{"key":"e_1_2_14_107_1","doi-asserted-by":"publisher","DOI":"10.1145\/3377811.3380353"},{"key":"e_1_2_14_108_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICCPS48487.2020.00024"},{"key":"e_1_2_14_109_1","doi-asserted-by":"publisher","DOI":"10.1109\/LRA.2020.2974682"},{"key":"e_1_2_14_110_1","doi-asserted-by":"publisher","DOI":"10.1109\/DASC50938.2020.9256581"},{"key":"e_1_2_14_111_1","unstructured":"C.Buerkle F.Geissler M.Paulitsch andK.\u2010U.Scholl \u201cFault\u2010Tolerant Perception for Automated Driving A Lightweight Monitoring Approach \u201d2021arXiv Preprint arXiv:2111.12360."},{"key":"e_1_2_14_112_1","doi-asserted-by":"publisher","DOI":"10.1145\/3477140"},{"key":"e_1_2_14_113_1","unstructured":"J.Gawlikowski C. R. N.Tassi M.Ali et al. \u201cA Survey of Uncertainty in Deep Neural Networks \u201d2021arXiv Preprint arXiv:2107.03342."},{"key":"e_1_2_14_114_1","unstructured":"D.Ulmer \u201cA Survey on Evidential Deep Learning for Single\u2010Pass Uncertainty Estimation \u201d2021arXiv Preprint arXiv:2110.03051."},{"key":"e_1_2_14_115_1","first-page":"3183","volume-title":"Proceedings of the 32nd International Conference on Neural Information Processing Systems","author":"Sensoy M.","year":"2018"},{"key":"e_1_2_14_116_1","volume-title":"Bayesian Modelling in Machine Learning: A Tutorial Review","author":"Seeger M.","year":"2006"},{"key":"e_1_2_14_117_1","volume-title":"Bayesian Inference in Statistical Analysis","author":"Box G. E. P.","year":"2011"},{"key":"e_1_2_14_118_1","unstructured":"W.Chen Y.Shen H.Jin andW.Wang \u201cA Variational Dirichlet Framework for Out\u2010of\u2010Distribution Detection \u201d2018arXiv Preprint arXiv:1811.07308."},{"key":"e_1_2_14_119_1","article-title":"Dirichlet\u2010Based Gaussian Processes for Large\u2010Scale Calibrated Classification","volume":"31","author":"Milios D.","year":"2018","journal-title":"Advances in Neural Information Processing Systems"},{"key":"e_1_2_14_120_1","article-title":"Predictive Uncertainty Estimation via Prior Networks","volume":"31","author":"Malinin A.","year":"2018","journal-title":"Advances in Neural Information Processing Systems"},{"key":"e_1_2_14_121_1","first-page":"5487","volume-title":"International Conference on Machine Learning","author":"Rossi S.","year":"2019"},{"key":"e_1_2_14_122_1","first-page":"681","volume-title":"Proceedings of the 28th International Conference on Machine Learning (ICML\u201011)","author":"Welling M.","year":"2011"},{"key":"e_1_2_14_123_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-42553-1_3"},{"key":"e_1_2_14_124_1","doi-asserted-by":"publisher","DOI":"10.1109\/TITS.2021.3096854"},{"key":"e_1_2_14_125_1","unstructured":"J.MukhotiandY.Gal \u201cEvaluating Bayesian Deep Learning Methods for Semantic Segmentation \u201d2018arXiv Preprint arXiv:1811.12709."},{"key":"e_1_2_14_126_1","article-title":"Concrete Dropout","volume":"30","author":"Gal Y.","year":"2017","journal-title":"Advances in Neural Information Processing Systems"},{"key":"e_1_2_14_127_1","unstructured":"T.Dash S.Chitlangia A.Ahuja andA.Srinivasan \u201cIncorporating Domain Knowledge Into Deep Neural Networks \u201d2021arXiv Preprint arXiv:2103.00180."},{"key":"e_1_2_14_128_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2007.383272"},{"key":"e_1_2_14_129_1","doi-asserted-by":"publisher","DOI":"10.1007\/3-540-45665-1_6"},{"key":"e_1_2_14_130_1","doi-asserted-by":"publisher","DOI":"10.1109\/TSSC.1970.300339"},{"key":"e_1_2_14_131_1","unstructured":"K.Hendrickx L.Perini D.Plas W.Meert andJ.Davis \u201cMachine Learning With a Reject Option: A Survey \u201d2021arXiv Preprint arXiv:2107.11277."},{"key":"e_1_2_14_132_1","doi-asserted-by":"publisher","DOI":"10.1109\/SPW50608.2020.00050"},{"key":"e_1_2_14_133_1","doi-asserted-by":"publisher","DOI":"10.1109\/ISSREW51248.2020.00073"},{"key":"e_1_2_14_134_1","doi-asserted-by":"publisher","DOI":"10.1145\/3450267.3452000"},{"key":"e_1_2_14_135_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.neucom.2020.10.068"},{"key":"e_1_2_14_136_1","doi-asserted-by":"publisher","DOI":"10.1177\/0278364917730603"},{"key":"e_1_2_14_137_1","doi-asserted-by":"publisher","DOI":"10.1109\/IV47402.2020.9304844"},{"key":"e_1_2_14_138_1","unstructured":"R.Gontijo\u2010Lopes Y.Dauphin andE. D.Cubuk \u201cNo One Representation to Rule Them All: Overlapping Features of Training Methods \u201d2021arXiv Preprint arXiv:2110.12899."},{"key":"e_1_2_14_139_1","first-page":"1905","volume-title":"Proceedings of the IEEE\/CVF International Conference on Computer Vision","author":"Wang X.","year":"2021"},{"key":"e_1_2_14_140_1","first-page":"4921","volume-title":"Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition","author":"Wang H.","year":"2022"},{"key":"e_1_2_14_141_1","first-page":"21464","article-title":"Energy\u2010Based Out\u2010of\u2010Distribution Detection","volume":"33","author":"Liu W.","year":"2020","journal-title":"Advances in Neural Information Processing Systems"},{"key":"e_1_2_14_142_1","doi-asserted-by":"publisher","DOI":"10.1109\/IROS40897.2019.8968176"},{"key":"e_1_2_14_143_1","volume-title":"Machine Learning: A Bayesian and Optimization Perspective","author":"Theodoridis S.","year":"2015"},{"key":"e_1_2_14_144_1","volume-title":"AAAI 2022 Workshop on Designing Artificial Intelligence for Open Worlds","author":"Roy A.","year":"2022"},{"key":"e_1_2_14_145_1","doi-asserted-by":"publisher","DOI":"10.1145\/3238147.3238187"},{"key":"e_1_2_14_146_1","doi-asserted-by":"publisher","DOI":"10.1109\/LRA.2022.3151970"},{"key":"e_1_2_14_147_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-55754-6_6"},{"key":"e_1_2_14_148_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICDMW.2016.0041"},{"key":"e_1_2_14_149_1","doi-asserted-by":"publisher","DOI":"10.1007\/s00521-020-05020-4"},{"key":"e_1_2_14_150_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.patcog.2021.108342"},{"key":"e_1_2_14_151_1","first-page":"1","article-title":"Dual\u2010Attention\u2010Guided Network for Ghost\u2010Free High Dynamic Range Imaging","author":"Yan Q.","year":"2021","journal-title":"International Journal of Computer Vision"},{"key":"e_1_2_14_152_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPRW53098.2021.00057"},{"key":"e_1_2_14_153_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2018.00263"},{"key":"e_1_2_14_154_1","first-page":"13074","volume-title":"Proceedings of the AAAI Conference on Artificial Intelligence","author":"Zhou Y.","year":"2020"},{"key":"e_1_2_14_155_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICCVW54120.2021.00017"},{"key":"e_1_2_14_156_1","doi-asserted-by":"publisher","DOI":"10.1109\/TPAMI.2020.2982166"},{"key":"e_1_2_14_157_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2016.207"},{"key":"e_1_2_14_158_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2017.19"},{"key":"e_1_2_14_159_1","volume-title":"NeurIPS 2021 Workshop on Deep Generative Models and Downstream Applications","author":"Marinescu R.","year":"2021"},{"key":"e_1_2_14_160_1","volume-title":"NeurIPS 2021 Workshop on Deep Generative Models and Downstream Applications","author":"Saharia C.","year":"2021"},{"key":"e_1_2_14_161_1","doi-asserted-by":"publisher","DOI":"10.1109\/TIP.2020.2975978"},{"key":"e_1_2_14_162_1","doi-asserted-by":"publisher","DOI":"10.1109\/TIP.2021.3067166"},{"key":"e_1_2_14_163_1","volume-title":"Workshop on Adversarial Machine Learning in Real\u2010World Computer Vision Systems and Online Challenges (AML\u2010CV)","author":"Al\u2010Afandi J.","year":"2021"},{"key":"e_1_2_14_164_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICIP42928.2021.9506451"},{"key":"e_1_2_14_165_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICGCIoT.2015.7380451"},{"key":"e_1_2_14_166_1","unstructured":"K.Andrej \u201cTesla Autonomous Driving Talk at CVPR 2021 \u201d2021."},{"key":"e_1_2_14_167_1","unstructured":"G.VenandA. S.Tolias \u201cThree Scenarios for Continual Learning \u201d2019arXiv Preprint arXiv:1904.07734."},{"key":"e_1_2_14_168_1","first-page":"2544","volume-title":"Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition","author":"Liu Y.","year":"2021"},{"key":"e_1_2_14_169_1","doi-asserted-by":"publisher","DOI":"10.1109\/MSP.2012.2211477"},{"key":"e_1_2_14_170_1","volume-title":"Learning Multiple Layers of Features From Tiny Images","author":"Krizhevsky A.","year":"2009"},{"key":"e_1_2_14_171_1","volume-title":"30th British Machine Vision Conference","author":"Shafaei A.","year":"2019"},{"key":"e_1_2_14_172_1","first-page":"1","volume-title":"Proceedings of the 1st Annual Conference on Robot Learning","author":"Dosovitskiy A.","year":"2017"},{"key":"e_1_2_14_173_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICST46399.2020.00019"},{"key":"e_1_2_14_174_1","doi-asserted-by":"publisher","DOI":"10.1109\/ISSRE55969.2022.00047"},{"key":"e_1_2_14_175_1","volume-title":"Evaluating the Simulation Gap for Training off\u2010Road Self\u2010Driving","author":"Zandbergen L.","year":"2021"},{"key":"e_1_2_14_176_1","doi-asserted-by":"publisher","DOI":"10.2991\/jase.d.190131.001"},{"key":"e_1_2_14_177_1","doi-asserted-by":"publisher","DOI":"10.1145\/3092703.3098235"},{"key":"e_1_2_14_178_1","article-title":"Failing Loudly: An Empirical Study of Methods for Detecting Dataset Shift","volume":"32","author":"Rabanser S.","year":"2019","journal-title":"Advances in Neural Information Processing Systems"},{"key":"e_1_2_14_179_1","unstructured":"R.MyersandZ.Saigol \u201cPass\u2010Fail Criteria for Scenario\u2010Based Testing of Automated Driving Systems \u201d2020arXiv Preprint arXiv:2005.09417."},{"key":"e_1_2_14_180_1","unstructured":"E.De GelderandC. O. O.Den \u201cProcedure for the Safety Assessment of an Autonomous Vehicle Using Real\u2010World Scenarios \u201d2020arXiv Preprint arXiv:2012.00643."},{"key":"e_1_2_14_181_1","doi-asserted-by":"publisher","DOI":"10.1109\/ACCESS.2021.3136585"},{"key":"e_1_2_14_182_1","unstructured":"Joint Authorities for Rulemaking of Unmanned Systems (JARUS) \u201cJARUS Guidelines on Specific Operations Risk Assessment (SORA) v2.0. \u201d(2019) Guidelines: EASA."},{"key":"e_1_2_14_183_1","doi-asserted-by":"publisher","DOI":"10.1109\/INES52918.2021.9512897"},{"key":"e_1_2_14_184_1","first-page":"481","article-title":"Model Assertions for Monitoring and Improving ML Models","volume":"2","author":"Kang D.","year":"2020","journal-title":"Proceedings of Machine Learning and Systems"}],"container-title":["Computational Intelligence"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/onlinelibrary.wiley.com\/doi\/pdf\/10.1111\/coin.70032","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,4,21]],"date-time":"2025-04-21T14:18:23Z","timestamp":1745245103000},"score":1,"resource":{"primary":{"URL":"https:\/\/onlinelibrary.wiley.com\/doi\/10.1111\/coin.70032"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,4]]},"references-count":183,"journal-issue":{"issue":"2","published-print":{"date-parts":[[2025,4]]}},"alternative-id":["10.1111\/coin.70032"],"URL":"https:\/\/doi.org\/10.1111\/coin.70032","archive":["Portico"],"relation":{},"ISSN":["0824-7935","1467-8640"],"issn-type":[{"value":"0824-7935","type":"print"},{"value":"1467-8640","type":"electronic"}],"subject":[],"published":{"date-parts":[[2025,4]]},"assertion":[{"value":"2022-07-13","order":0,"name":"received","label":"Received","group":{"name":"publication_history","label":"Publication History"}},{"value":"2025-01-31","order":2,"name":"accepted","label":"Accepted","group":{"name":"publication_history","label":"Publication History"}},{"value":"2025-04-20","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}],"article-number":"e70032"}}