{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,5,8]],"date-time":"2026-05-08T20:16:10Z","timestamp":1778271370653,"version":"3.51.4"},"reference-count":85,"publisher":"Institute of Electrical and Electronics Engineers (IEEE)","license":[{"start":{"date-parts":[[2026,1,1]],"date-time":"2026-01-01T00:00:00Z","timestamp":1767225600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/legalcode"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["IEEE Access"],"published-print":{"date-parts":[[2026]]},"DOI":"10.1109\/access.2026.3688958","type":"journal-article","created":{"date-parts":[[2026,4,30]],"date-time":"2026-04-30T19:48:46Z","timestamp":1777578526000},"page":"66955-66970","source":"Crossref","is-referenced-by-count":0,"title":["ULTRA: A Standardized Certification Metric for Machine Unlearning"],"prefix":"10.1109","volume":"14","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-2573-273X","authenticated-orcid":false,"given":"Seyfullah","family":"Arslan","sequence":"first","affiliation":[{"name":"Department of Computer Engineering, Kütahya Dumlupınar Üniversitesi, Kütahya, Türkiye"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-8965-1429","authenticated-orcid":false,"given":"F\u0131rat","family":"Aydemir","sequence":"additional","affiliation":[{"name":"Department of Computer Engineering, Kütahya Dumlupınar Üniversitesi, Kütahya, Türkiye"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"263","reference":[{"key":"ref1","doi-asserted-by":"publisher","DOI":"10.1109\/tetci.2024.3379240"},{"key":"ref2","doi-asserted-by":"publisher","DOI":"10.1145\/3448016.3457239"},{"key":"ref3","doi-asserted-by":"publisher","DOI":"10.1109\/tifs.2023.3265506"},{"key":"ref4","doi-asserted-by":"publisher","DOI":"10.1109\/tnnls.2024.3486109"},{"key":"ref5","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-57959-7"},{"key":"ref6","doi-asserted-by":"publisher","DOI":"10.2139\/ssrn.3211013"},{"key":"ref7","doi-asserted-by":"publisher","DOI":"10.1145\/3749987"},{"key":"ref8","doi-asserted-by":"publisher","DOI":"10.52202\/079017-3754"},{"key":"ref9","doi-asserted-by":"publisher","DOI":"10.1145\/3460120.3484756"},{"key":"ref10","doi-asserted-by":"publisher","DOI":"10.1109\/satml64287.2025.00034"},{"key":"ref11","doi-asserted-by":"publisher","DOI":"10.1109\/tdsc.2025.3542092"},{"key":"ref12","article-title":"An adversarial perspective on machine unlearning for AI safety","author":"\u0141ucki","year":"2024","journal-title":"arXiv:2409.18025"},{"key":"ref13","doi-asserted-by":"publisher","DOI":"10.1109\/tnnls.2025.3530988"},{"key":"ref14","doi-asserted-by":"publisher","DOI":"10.1038\/s42256-025-00985-0"},{"key":"ref15","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-72664-4_16"},{"key":"ref16","first-page":"5545","article-title":"Rectifying privacy and efficacy measurements in machine unlearning: A new inference attack perspective","volume-title":"Proc. 34th USENIX Secur. Symp. (USENIX Secur.)","author":"Naderloui"},{"key":"ref17","article-title":"Towards scalable exact machine unlearning using parameter-efficient fine-tuning","author":"Chowdhury","year":"2024","journal-title":"arXiv:2406.16257"},{"key":"ref18","doi-asserted-by":"publisher","DOI":"10.52202\/075280-2246"},{"key":"ref19","first-page":"10355","article-title":"DeltaGrad: Rapid retraining of machine learning models","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Wu"},{"key":"ref20","doi-asserted-by":"publisher","DOI":"10.24963\/ijcai.2022\/556"},{"key":"ref21","doi-asserted-by":"publisher","DOI":"10.1145\/3548606.3559352"},{"key":"ref22","doi-asserted-by":"publisher","DOI":"10.1109\/sp40001.2021.00019"},{"key":"ref23","doi-asserted-by":"publisher","DOI":"10.1109\/EuroSP53844.2022.00027"},{"key":"ref24","first-page":"931","article-title":"Descent-to-delete: Gradient-based methods for machine unlearning","volume-title":"Proc. Algorithmic Learn. Theory","author":"Neel"},{"key":"ref25","article-title":"A unified gradient-based framework for task-agnostic continual learning-unlearning","author":"Huang","year":"2025","journal-title":"arXiv:2505.15178"},{"key":"ref26","doi-asserted-by":"publisher","DOI":"10.52202\/079017-0831"},{"key":"ref27","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v35i13.17371"},{"key":"ref28","article-title":"Delta-influence: Unlearning poisons via influence functions","author":"Li","year":"2024","journal-title":"arXiv:2411.13731"},{"key":"ref29","doi-asserted-by":"publisher","DOI":"10.1145\/3701763"},{"key":"ref30","doi-asserted-by":"publisher","DOI":"10.1145\/3543507.3583521"},{"key":"ref31","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v38i11.29092"},{"key":"ref32","article-title":"Unlearning with Fisher masking","author":"Liu","year":"2023","journal-title":"arXiv:2310.05331"},{"key":"ref33","doi-asserted-by":"publisher","DOI":"10.1109\/tnnls.2023.3266233"},{"key":"ref34","doi-asserted-by":"publisher","DOI":"10.1016\/j.inffus.2025.103796"},{"key":"ref35","doi-asserted-by":"publisher","DOI":"10.1109\/TNNLS.2025.3607995"},{"key":"ref36","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52734.2025.01895"},{"key":"ref37","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v38i19.30118"},{"key":"ref38","doi-asserted-by":"publisher","DOI":"10.1109\/TIFS.2025.3557671"},{"key":"ref39","doi-asserted-by":"publisher","DOI":"10.1109\/TDSC.2024.3520614"},{"key":"ref40","doi-asserted-by":"publisher","DOI":"10.1109\/IWQOS52092.2021.9521274"},{"key":"ref41","doi-asserted-by":"publisher","DOI":"10.14778\/3641204.3641220"},{"key":"ref42","article-title":"In-context unlearning: Language models as few shot unlearners","author":"Pawelczyk","year":"2023","journal-title":"arXiv:2310.07579"},{"key":"ref43","article-title":"Editing models with task arithmetic","author":"Ilharco","year":"2022","journal-title":"arXiv:2212.04089"},{"key":"ref44","doi-asserted-by":"publisher","DOI":"10.1145\/3589334.3645669"},{"key":"ref45","article-title":"Verification of machine unlearning is fragile","author":"Zhang","year":"2024","journal-title":"arXiv:2408.00929"},{"key":"ref46","doi-asserted-by":"publisher","DOI":"10.1007\/s10994-022-06178-9"},{"key":"ref47","doi-asserted-by":"publisher","DOI":"10.1109\/tifs.2025.3571666"},{"key":"ref48","doi-asserted-by":"publisher","DOI":"10.1109\/tifs.2023.3297905"},{"key":"ref49","doi-asserted-by":"publisher","DOI":"10.1109\/tdsc.2022.3194884"},{"key":"ref50","doi-asserted-by":"publisher","DOI":"10.1007\/s10994-025-06806-0"},{"key":"ref51","article-title":"Label-agnostic forgetting: A supervision-free unlearning in deep models","author":"Shen","year":"2024","journal-title":"arXiv:2404.00506"},{"key":"ref52","doi-asserted-by":"publisher","DOI":"10.1109\/tifs.2023.3331239"},{"key":"ref53","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-73232-4_1"},{"key":"ref54","doi-asserted-by":"publisher","DOI":"10.1016\/j.icte.2025.06.007"},{"key":"ref55","doi-asserted-by":"publisher","DOI":"10.52202\/075280-0095"},{"key":"ref56","doi-asserted-by":"publisher","DOI":"10.1109\/SP54263.2024.00248"},{"key":"ref57","article-title":"Do unlearning methods remove information from language model weights?","author":"Deeb","year":"2024","journal-title":"arXiv:2410.08827"},{"key":"ref58","first-page":"1","article-title":"Machine unlearning in recommendation systems","volume-title":"Proc. 2nd Int. Conf. Trends Eng. Syst. Technol. (ICTEST)","author":"Devan"},{"key":"ref59","doi-asserted-by":"publisher","DOI":"10.1007\/978-981-96-4506-0_20"},{"key":"ref60","article-title":"TOFU: A task of fictitious unlearning for LLMs","author":"Maini","year":"2024","journal-title":"arXiv:2401.06121"},{"key":"ref61","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v37i6.25879"},{"key":"ref62","article-title":"SSSE: Efficiently erasing samples from trained machine learning models","author":"Peste","year":"2021","journal-title":"arXiv:2107.03860"},{"key":"ref63","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52734.2025.00939"},{"key":"ref64","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR46437.2021.00085"},{"key":"ref65","doi-asserted-by":"publisher","DOI":"10.1109\/WACV61041.2025.00317"},{"key":"ref66","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v40i26.39373"},{"key":"ref67","doi-asserted-by":"publisher","DOI":"10.1109\/SP46214.2022.9833649"},{"key":"ref68","doi-asserted-by":"publisher","DOI":"10.56553\/popets-2022-0072"},{"key":"ref69","article-title":"WaterDrum: Watermarking for data-centric unlearning metric","author":"Lu","year":"2025","journal-title":"arXiv:2505.05064"},{"key":"ref70","doi-asserted-by":"publisher","DOI":"10.1109\/SaTML64287.2025.00033"},{"key":"ref71","doi-asserted-by":"publisher","DOI":"10.1109\/TKDE.2025.3528551"},{"key":"ref72","doi-asserted-by":"publisher","DOI":"10.1145\/3734436.3734454"},{"key":"ref73","doi-asserted-by":"publisher","DOI":"10.1145\/3746252.3761627"},{"key":"ref74","doi-asserted-by":"publisher","DOI":"10.1109\/eurosp63326.2025.00058"},{"key":"ref75","article-title":"Understanding intermediate layers using linear classifier probes","author":"Alain","year":"2016","journal-title":"arXiv:1610.01644"},{"key":"ref76","doi-asserted-by":"publisher","DOI":"10.3390\/computers14090351"},{"key":"ref77","doi-asserted-by":"publisher","DOI":"10.1007\/s42488-024-00119-y"},{"key":"ref78","doi-asserted-by":"publisher","DOI":"10.3390\/app151910857"},{"key":"ref79","doi-asserted-by":"publisher","DOI":"10.3390\/computers14040150"},{"key":"ref80","doi-asserted-by":"publisher","DOI":"10.1016\/j.neunet.2025.107879"},{"key":"ref81","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR42600.2020.00932"},{"key":"ref82","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-58526-6_23"},{"key":"ref83","first-page":"1273","article-title":"Communication-efficient learning of deep networks from decentralized data","volume-title":"Proc. Artif. Intell. Statist.","author":"McMahan"},{"key":"ref84","doi-asserted-by":"publisher","DOI":"10.1145\/3679014"},{"key":"ref85","doi-asserted-by":"publisher","DOI":"10.1109\/tvt.2025.3597870"}],"container-title":["IEEE Access"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx8\/6287639\/11323511\/11501543.pdf?arnumber=11501543","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,5,8]],"date-time":"2026-05-08T19:41:01Z","timestamp":1778269261000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/11501543\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026]]},"references-count":85,"URL":"https:\/\/doi.org\/10.1109\/access.2026.3688958","relation":{},"ISSN":["2169-3536"],"issn-type":[{"value":"2169-3536","type":"electronic"}],"subject":[],"published":{"date-parts":[[2026]]}}}