{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,12]],"date-time":"2026-07-12T16:26:09Z","timestamp":1783873569277,"version":"3.55.0"},"reference-count":85,"publisher":"Institute of Electrical and Electronics Engineers (IEEE)","issue":"5","license":[{"start":{"date-parts":[[2024,5,1]],"date-time":"2024-05-01T00:00:00Z","timestamp":1714521600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/ieeexplore.ieee.org\/Xplorehelp\/downloads\/license-information\/IEEE.html"},{"start":{"date-parts":[[2024,5,1]],"date-time":"2024-05-01T00:00:00Z","timestamp":1714521600000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2024,5,1]],"date-time":"2024-05-01T00:00:00Z","timestamp":1714521600000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["IEEE Trans. Pattern Anal. Mach. Intell."],"published-print":{"date-parts":[[2024,5]]},"DOI":"10.1109\/tpami.2024.3355890","type":"journal-article","created":{"date-parts":[[2024,1,19]],"date-time":"2024-01-19T18:43:03Z","timestamp":1705689783000},"page":"3910-3922","source":"Crossref","is-referenced-by-count":38,"title":["Pruning Self-Attentions Into Convolutional Layers in Single Path"],"prefix":"10.1109","volume":"46","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-1505-0284","authenticated-orcid":false,"given":"Haoyu","family":"He","sequence":"first","affiliation":[{"name":"Department of Data Science and AI, Faculty of IT, Monash University, Clayton, VIC, Australia"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-9444-3763","authenticated-orcid":false,"given":"Jianfei","family":"Cai","sequence":"additional","affiliation":[{"name":"Department of Data Science and AI, Faculty of IT, Monash University, Clayton, VIC, Australia"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-6745-3050","authenticated-orcid":false,"given":"Jing","family":"Liu","sequence":"additional","affiliation":[{"name":"Department of Data Science and AI, Faculty of IT, Monash University, Clayton, VIC, Australia"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-1717-7844","authenticated-orcid":false,"given":"Zizheng","family":"Pan","sequence":"additional","affiliation":[{"name":"Department of Data Science and AI, Faculty of IT, Monash University, Clayton, VIC, Australia"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-6595-7661","authenticated-orcid":false,"given":"Jing","family":"Zhang","sequence":"additional","affiliation":[{"name":"Faculty of Engineering, The University of Sydney, Camperdown, NSW, Australia"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-7225-5449","authenticated-orcid":false,"given":"Dacheng","family":"Tao","sequence":"additional","affiliation":[{"name":"School of Computer Science and Engineering, Nanyang Technological University, Singapore"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-0074-0303","authenticated-orcid":false,"given":"Bohan","family":"Zhuang","sequence":"additional","affiliation":[{"name":"Department of Data Science and AI, Faculty of IT, Monash University, Clayton, VIC, Australia"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"263","reference":[{"key":"ref1","article-title":"An image is worth 16x16 words: Transformers for image recognition at scale","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Dosovitskiy"},{"key":"ref2","doi-asserted-by":"publisher","DOI":"10.1109\/iccv48922.2021.00043"},{"key":"ref3","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV48922.2021.00986"},{"key":"ref4","first-page":"10347","article-title":"Training data-efficient image transformers & distillation through attention","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Touvron"},{"key":"ref5","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-19806-9_27"},{"key":"ref6","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR46437.2021.00681"},{"key":"ref7","first-page":"17864","article-title":"Per-pixel classification is not all you need for semantic segmentation","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Cheng"},{"key":"ref8","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR46437.2021.00542"},{"key":"ref9","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-58452-8_13"},{"key":"ref10","article-title":"Deformable {detr}: Deformable transformers for end-to-end object detection","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Zhu"},{"key":"ref11","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV48922.2021.00360"},{"key":"ref12","doi-asserted-by":"publisher","DOI":"10.48550\/ARXIV.1706.03762"},{"key":"ref13","article-title":"Visual transformer pruning","volume-title":"Proc. Korea Dig. Dis. Week","author":"Zhu"},{"key":"ref14","first-page":"18547","article-title":"NViT: Vision transformer compression and parameter redistribution","volume-title":"Proc. Conf. Comput. Vis. Pattern Recognit.","author":"Yang"},{"key":"ref15","doi-asserted-by":"publisher","DOI":"10.1109\/iros55552.2023.10342025"},{"key":"ref16","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52688.2022.01186"},{"key":"ref17","first-page":"30392","article-title":"Early convolutions help transformers see better","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Xiao"},{"key":"ref18","first-page":"28522","article-title":"ViTAE: Vision transformer advanced by exploring intrinsic inductive bias","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Xu"},{"key":"ref19","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV48922.2021.00008"},{"key":"ref20","doi-asserted-by":"publisher","DOI":"10.1145\/3447548.3467262"},{"key":"ref21","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV48922.2021.01206"},{"key":"ref22","article-title":"On the relationship between self-attention and convolutional layers","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Cordonnier"},{"key":"ref23","article-title":"Unified visual transformer compression","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Yu"},{"key":"ref24","first-page":"19974","article-title":"Chasing sparsity in vision transformers: An end-to-end exploration","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Chen"},{"key":"ref25","article-title":"Pruning convolutional neural networks for resource efficient inference","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Molchanov"},{"key":"ref26","doi-asserted-by":"publisher","DOI":"10.1007\/s11263-015-0816-y"},{"key":"ref27","first-page":"24898","article-title":"IA-RED$^{2}$2: Interpretability-aware redundancy reduction for vision transformers","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Pan","year":"2021"},{"key":"ref28","first-page":"13937","article-title":"DynamicViT: Efficient vision transformers with dynamic token sparsification","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Rao"},{"key":"ref29","article-title":"Token merging: Your ViT but faster","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Bolya"},{"key":"ref30","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52729.2023.00208"},{"key":"ref31","first-page":"14014","article-title":"Are sixteen heads really better than one?","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","volume":"32","author":"Michel"},{"key":"ref32","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.emnlp-main.211"},{"key":"ref33","doi-asserted-by":"publisher","DOI":"10.1145\/3446640"},{"key":"ref34","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.findings-emnlp.286"},{"key":"ref35","first-page":"20852","article-title":"The generalization-stability tradeoff in neural network pruning","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Bartoldson"},{"key":"ref36","first-page":"15834","article-title":"The lottery ticket hypothesis for pre-trained BERT networks","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Chen"},{"key":"ref37","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.emnlp-main.259"},{"key":"ref38","doi-asserted-by":"publisher","DOI":"10.1109\/ICME52920.2022.9859786"},{"key":"ref39","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52729.2023.02333"},{"key":"ref40","first-page":"2943","article-title":"Rigging the lottery: Making all tickets winners","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Evci"},{"key":"ref41","article-title":"Distilling the knowledge in a neural network","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst. W","author":"Hinton"},{"key":"ref42","article-title":"ImageNet-trained CNNs are biased towards texture; increasing shape bias improves accuracy and robustness","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Geirhos"},{"key":"ref43","article-title":"Approximating CNNs with bag-of-local-features models works surprisingly well on ImageNet","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Brendel"},{"key":"ref44","first-page":"23296","article-title":"Intriguing properties of vision transformers","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Naseer"},{"key":"ref45","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV48922.2021.00951"},{"key":"ref46","article-title":"MobileViT: Light-weight, general-purpose, and mobile-friendly vision transformer","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Mehta"},{"key":"ref47","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52729.2023.01778"},{"key":"ref48","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52729.2023.01777"},{"key":"ref49","first-page":"35632","article-title":"CONVMAE: Masked convolution meets masked autoencoders","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Gao"},{"key":"ref50","article-title":"Augmenting convolutional networks with attention-based aggregation","author":"Touvron","year":"2021"},{"key":"ref51","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV48922.2021.01204"},{"key":"ref52","article-title":"FastViT: A fast hybrid vision transformer using structural reparameterization","volume-title":"Proc. Int. Conf. Comput. Vis.","author":"Vasu"},{"key":"ref53","first-page":"14663","article-title":"Bridging the gap between vision transformers and convolutional neural networks on small datasets","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Lu"},{"key":"ref54","article-title":"X-volution: On the unification of convolution and self-attention","author":"Chen","year":"2021"},{"key":"ref55","doi-asserted-by":"publisher","DOI":"10.1088\/1742-5468\/ac9830"},{"key":"ref56","article-title":"Transformed CNNs: Recasting pre-trained convolutional layers with self-attention","author":"d\u2019Ascoli","year":"2021"},{"key":"ref57","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-46147-8_29"},{"key":"ref58","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-58517-4_32"},{"key":"ref59","article-title":"Once-for-all: Train one network and specialize it for efficient deployment","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Cai"},{"key":"ref60","doi-asserted-by":"publisher","DOI":"10.48550\/ARXIV.1706.03762"},{"key":"ref61","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2016.90"},{"key":"ref62","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v36i2.20099"},{"key":"ref63","doi-asserted-by":"publisher","DOI":"10.5555\/3104322.3104425"},{"key":"ref64","doi-asserted-by":"publisher","DOI":"10.5555\/3045118.3045167"},{"key":"ref65","first-page":"12116","article-title":"Do vision transformers see like convolutional neural networks?","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Raghu"},{"key":"ref66","article-title":"Elastic architecture search for diverse tasks with different resources","author":"Liu","year":"2021"},{"key":"ref67","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-58542-6_35"},{"key":"ref68","article-title":"Layer normalization","author":"Ba","year":"2016"},{"key":"ref69","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV48922.2021.01205"},{"key":"ref70","article-title":"Searching for efficient multi-stage vision transformers","author":"Liao","year":"2021"},{"key":"ref71","article-title":"Gaussian error linear units (GELUs)","author":"Hendrycks","year":"2016"},{"key":"ref72","article-title":"Not all patches are what you need: Expediting vision transformers via token reorganizations","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Liang"},{"key":"ref73","doi-asserted-by":"publisher","DOI":"10.1162\/tacl_a_00436"},{"key":"ref74","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV48922.2021.00061"},{"key":"ref75","doi-asserted-by":"publisher","DOI":"10.1109\/TPAMI.2023.3275159"},{"key":"ref76","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR46437.2021.00850"},{"key":"ref77","article-title":"Estimating or propagating gradients through stochastic neurons for conditional computation","author":"Bengio","year":"2013"},{"key":"ref78","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR42600.2020.01044"},{"key":"ref79","article-title":"Large memory layers with product keys","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Lample"},{"key":"ref80","first-page":"2793","article-title":"Attention is not all you need: Pure attention loses rank doubly exponentially with depth","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Dong"},{"key":"ref81","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV48922.2021.00010"},{"key":"ref82","article-title":"DeepViT: Towards deeper vision transformer","author":"Zhou","year":"2021"},{"key":"ref83","first-page":"3690","article-title":"Power-BERT: Accelerating BERT inference via progressive word-vector elimination","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Goyal"},{"key":"ref84","article-title":"MobileNets: Efficient convolutional neural networks for mobile vision applications","author":"Howard","year":"2017"},{"key":"ref85","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2018.00716"}],"container-title":["IEEE Transactions on Pattern Analysis and Machine Intelligence"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx7\/34\/10490207\/10409620.pdf?arnumber=10409620","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,4,9]],"date-time":"2024-04-09T19:38:36Z","timestamp":1712691516000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/10409620\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,5]]},"references-count":85,"journal-issue":{"issue":"5"},"URL":"https:\/\/doi.org\/10.1109\/tpami.2024.3355890","relation":{},"ISSN":["0162-8828","2160-9292","1939-3539"],"issn-type":[{"value":"0162-8828","type":"print"},{"value":"2160-9292","type":"electronic"},{"value":"1939-3539","type":"electronic"}],"subject":[],"published":{"date-parts":[[2024,5]]}}}