{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,2,21]],"date-time":"2026-02-21T19:34:20Z","timestamp":1771702460206,"version":"3.50.1"},"reference-count":84,"publisher":"Institute of Electrical and Electronics Engineers (IEEE)","license":[{"start":{"date-parts":[[2025,1,1]],"date-time":"2025-01-01T00:00:00Z","timestamp":1735689600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/legalcode"},{"start":{"date-parts":[[2025,1,1]],"date-time":"2025-01-01T00:00:00Z","timestamp":1735689600000},"content-version":"am","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/legalcode"}],"funder":[{"DOI":"10.13039\/100000001","name":"National Science Foundation","doi-asserted-by":"publisher","award":["2233893"],"award-info":[{"award-number":["2233893"]}],"id":[{"id":"10.13039\/100000001","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["IEEE Access"],"published-print":{"date-parts":[[2025]]},"DOI":"10.1109\/access.2025.3540399","type":"journal-article","created":{"date-parts":[[2025,2,10]],"date-time":"2025-02-10T18:35:11Z","timestamp":1739212511000},"page":"27908-27927","source":"Crossref","is-referenced-by-count":5,"title":["Optimizing Vision Transformers: Unveiling \u2018Focus and Forget\u2019 for Enhanced Computational Efficiency"],"prefix":"10.1109","volume":"13","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-3735-9191","authenticated-orcid":false,"given":"Banafsheh Saber","family":"Latibari","sequence":"first","affiliation":[{"name":"Electrical and Computer Engineering, University of California at Davis, Davis, CA, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-8904-4699","authenticated-orcid":false,"given":"Houman","family":"Homayoun","sequence":"additional","affiliation":[{"name":"Electrical and Computer Engineering, University of California at Davis, Davis, CA, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-4052-8075","authenticated-orcid":false,"given":"Avesta","family":"Sasan","sequence":"additional","affiliation":[{"name":"Electrical and Computer Engineering, University of California at Davis, Davis, CA, USA"}]}],"member":"263","reference":[{"key":"ref1","volume-title":"Pytorch Wavelets","year":"2024"},{"key":"ref2","first-page":"63117","article-title":"Fast attention requires bounded entries","volume-title":"Advances in Neural Information Processing Systems","volume":"36","author":"Alman","year":"2023"},{"key":"ref3","article-title":"Multi-exit vision transformer for dynamic inference","author":"Bakhtiarnia","year":"2021","journal-title":"arXiv:2106.15183"},{"key":"ref4","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-58452-8_13"},{"key":"ref5","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52733.2024.01776"},{"key":"ref6","article-title":"Rethinking attention with performers","author":"Choromanski","year":"2020","journal-title":"arXiv:2009.14794"},{"key":"ref7","first-page":"1","article-title":"Patch n\u2019pack: Navit, a vision transformer for any aspect ratio and resolution","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","volume":"36","author":"Dehghani"},{"key":"ref8","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2009.5206848"},{"key":"ref9","article-title":"BERT: Pre-training of deep bidirectional transformers for language understanding","volume-title":"Proc. NAACL-HLT","author":"Devlin"},{"key":"ref10","article-title":"An image is worth 16\u00d716 words: Transformers for image recognition at scale","author":"Dosovitskiy","year":"2020","journal-title":"arXiv:2010.11929"},{"key":"ref11","article-title":"Depth-adaptive transformer","author":"Elbayad","year":"2019","journal-title":"arXiv:1910.10073"},{"key":"ref12","doi-asserted-by":"publisher","DOI":"10.1145\/3653019"},{"key":"ref13","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV48922.2021.00675"},{"key":"ref14","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-20083-0_24"},{"key":"ref15","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52688.2022.01190"},{"key":"ref16","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV51070.2023.00548"},{"key":"ref17","first-page":"15908","article-title":"Transformer in transformer","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","author":"Han"},{"key":"ref18","article-title":"FasterViT: Fast vision transformers with hierarchical attention","author":"Hatamizadeh","year":"2023","journal-title":"arXiv:2306.06189"},{"key":"ref19","doi-asserted-by":"publisher","DOI":"10.1109\/ICCVW60793.2023.00091"},{"key":"ref20","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2016.90"},{"key":"ref21","article-title":"Magic pyramid: Accelerating inference with early exiting and token pruning","author":"He","year":"2021","journal-title":"arXiv:2111.00230"},{"key":"ref22","first-page":"9099","article-title":"Transformer quality in linear time","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Hua"},{"key":"ref23","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV51070.2023.00881"},{"key":"ref24","first-page":"5156","article-title":"Transformers are RNNs: Fast autoregressive transformers with linear attention","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Katharopoulos"},{"key":"ref25","doi-asserted-by":"publisher","DOI":"10.1109\/WACV57701.2024.00141"},{"key":"ref26","article-title":"Learned token pruning for transformers","author":"Kim","year":"2021","journal-title":"arXiv:2107.00910"},{"key":"ref27","article-title":"SimA: Simple softmax-free attention for vision transformers","author":"Abbasi Koohpayegani","year":"2022","journal-title":"arXiv:2206.08898"},{"key":"ref28","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52733.2024.01490"},{"key":"ref29","doi-asserted-by":"publisher","DOI":"10.1038\/s41598-024-60579-5"},{"key":"ref30","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2021.acl-long.16"},{"key":"ref31","article-title":"Not all patches are what you need: Expediting vision transformers via token reorganizations","author":"Liang","year":"2022","journal-title":"arXiv:2202.07800"},{"key":"ref32","first-page":"8501","article-title":"MCUFormer: Deploying vision tranformers on microcontrollers with limited memory","volume-title":"Advances in Neural Information Processing Systems","volume":"36","author":"Liang","year":"2023"},{"key":"ref33","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2021.naacl-main.162"},{"key":"ref34","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV51070.2023.00553"},{"key":"ref35","article-title":"ConSmax: Hardware-friendly alternative softmax with learnable parameters","author":"Liu","year":"2024","journal-title":"arXiv:2402.10930"},{"key":"ref36","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52729.2023.01386"},{"key":"ref37","article-title":"PatchDropout: Economizing vision transformers using patch dropout","author":"Liu","year":"2022","journal-title":"arXiv:2208.07220"},{"key":"ref38","doi-asserted-by":"publisher","DOI":"10.1109\/WACV57701.2024.00184"},{"key":"ref39","doi-asserted-by":"publisher","DOI":"10.1109\/WACV57701.2024.00264"},{"key":"ref40","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV48922.2021.00986"},{"key":"ref41","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52688.2022.01199"},{"key":"ref42","doi-asserted-by":"publisher","DOI":"10.1016\/j.compeleceng.2024.109209"},{"key":"ref43","first-page":"29449","article-title":"FMMformer: Efficient and flexible transformer via decomposed near-field and far-field attention","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","author":"Nguyen"},{"key":"ref44","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52733.2024.01781"},{"key":"ref45","article-title":"Fast vision transformers with HiLo attention","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","author":"Pan"},{"key":"ref46","article-title":"CosFormer: Rethinking softmax in attention","author":"Qin","year":"2022","journal-title":"arXiv:2202.08791"},{"key":"ref47","first-page":"1526","article-title":"Multi-scale hierarchical vision transformer with cascaded attention decoding for medical image segmentation","volume":"227","author":"Rahman","year":"2024","journal-title":"Medical Imaging With Deep Learning"},{"key":"ref48","first-page":"13937","article-title":"DynamicViT: Efficient vision transformers with dynamic token sparsification","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","author":"Rao"},{"key":"ref49","doi-asserted-by":"publisher","DOI":"10.1109\/WACV45572.2020.9093580"},{"key":"ref50","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52733.2024.00305"},{"key":"ref51","doi-asserted-by":"publisher","DOI":"10.1109\/DAC18074.2021.9586134"},{"key":"ref52","article-title":"Early exiting with ensemble internal classifiers","author":"Sun","year":"2021","journal-title":"arXiv:2105.13792"},{"key":"ref53","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52729.2023.01038"},{"key":"ref54","first-page":"10183","article-title":"Synthesizer: Rethinking self-attention for transformer models","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Tay"},{"key":"ref55","first-page":"10347","article-title":"Training data-efficient image transformers & distillation through attention","volume-title":"Proc. Int. conf. Mach. Learn.","author":"Touvron"},{"key":"ref56","doi-asserted-by":"publisher","DOI":"10.48550\/ARXIV.1706.03762"},{"key":"ref57","article-title":"Efficient softmax approximation for deep neural networks with attention mechanism","author":"Vasyltsov","year":"2021","journal-title":"arXiv:2111.10770"},{"key":"ref58","first-page":"21665","article-title":"Fast transformers with clustered attention","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","author":"Vyas"},{"key":"ref59","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52733.2024.01521"},{"key":"ref60","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV48922.2021.00061"},{"key":"ref61","doi-asserted-by":"publisher","DOI":"10.1007\/s41095-022-0274-8"},{"key":"ref62","doi-asserted-by":"publisher","DOI":"10.1109\/TPAMI.2023.3341806"},{"key":"ref63","first-page":"11960","article-title":"Not all images are worth 16\u00d716 words: Dynamic transformers for efficient image recognition","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","author":"Wang"},{"key":"ref64","volume-title":"Pytorch Image Models","author":"Wightman","year":"2019"},{"key":"ref65","article-title":"Replacing softmax with ReLU in vision transformers","author":"Wortsman","year":"2023","journal-title":"arXiv:2309.08586"},{"key":"ref66","article-title":"Fastformer: Additive attention can be all you need","author":"Wu","year":"2021","journal-title":"arXiv:2108.09084"},{"key":"ref67","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52733.2024.01039"},{"key":"ref68","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52733.2024.00525"},{"key":"ref69","doi-asserted-by":"publisher","DOI":"10.24963\/ijcai.2019\/735"},{"key":"ref70","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.acl-main.204"},{"key":"ref71","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2021.eacl-main.8"},{"key":"ref72","doi-asserted-by":"publisher","DOI":"10.1145\/3581783.3611762"},{"key":"ref73","doi-asserted-by":"publisher","DOI":"10.1007\/978-981-99-8388-9_3"},{"key":"ref74","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v36i3.20202"},{"key":"ref75","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-19806-9_19"},{"key":"ref76","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52688.2022.01054"},{"key":"ref77","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52729.2023.01387"},{"key":"ref78","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52733.2024.00550"},{"key":"ref79","doi-asserted-by":"publisher","DOI":"10.1109\/TPAMI.2023.3347693"},{"key":"ref80","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV51070.2023.00472"},{"key":"ref81","first-page":"1","article-title":"MG-ViT: A multi-granularity method for compact and efficient vision transformers","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","volume":"36","author":"Zhang"},{"key":"ref82","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v36i3.20252"},{"key":"ref83","first-page":"18330","article-title":"BERT loses patience: Fast and robust inference with early exit","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","author":"Zhou"},{"key":"ref84","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2021.acl-long.231"}],"container-title":["IEEE Access"],"original-title":[],"link":[{"URL":"https:\/\/ieeexplore.ieee.org\/ielam\/6287639\/10820123\/10879005-aam.pdf","content-type":"application\/pdf","content-version":"am","intended-application":"syndication"},{"URL":"http:\/\/xplorestaging.ieee.org\/ielx8\/6287639\/10820123\/10879005.pdf?arnumber=10879005","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,2,19]],"date-time":"2025-02-19T18:58:38Z","timestamp":1739991518000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/10879005\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025]]},"references-count":84,"URL":"https:\/\/doi.org\/10.1109\/access.2025.3540399","relation":{},"ISSN":["2169-3536"],"issn-type":[{"value":"2169-3536","type":"electronic"}],"subject":[],"published":{"date-parts":[[2025]]}}}