{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,18]],"date-time":"2026-07-18T15:40:18Z","timestamp":1784389218363,"version":"3.55.0"},"reference-count":207,"publisher":"Institute of Electrical and Electronics Engineers (IEEE)","issue":"6","license":[{"start":{"date-parts":[[2024,6,1]],"date-time":"2024-06-01T00:00:00Z","timestamp":1717200000000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/ieeexplore.ieee.org\/Xplorehelp\/downloads\/license-information\/IEEE.html"},{"start":{"date-parts":[[2024,6,1]],"date-time":"2024-06-01T00:00:00Z","timestamp":1717200000000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2024,6,1]],"date-time":"2024-06-01T00:00:00Z","timestamp":1717200000000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["IEEE Trans. Neural Netw. Learning Syst."],"published-print":{"date-parts":[[2024,6]]},"DOI":"10.1109\/tnnls.2022.3227717","type":"journal-article","created":{"date-parts":[[2023,3,30]],"date-time":"2023-03-30T17:35:06Z","timestamp":1680197706000},"page":"7478-7498","source":"Crossref","is-referenced-by-count":386,"title":["A Survey of Visual Transformers"],"prefix":"10.1109","volume":"35","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-5348-7807","authenticated-orcid":false,"given":"Yang","family":"Liu","sequence":"first","affiliation":[{"name":"Institute of Computing Technology, Chinese Academy of Sciences, Beijing, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-8759-4811","authenticated-orcid":false,"given":"Yao","family":"Zhang","sequence":"additional","affiliation":[{"name":"Institute of Computing Technology, Chinese Academy of Sciences, Beijing, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-8062-0765","authenticated-orcid":false,"given":"Yixin","family":"Wang","sequence":"additional","affiliation":[{"name":"School of Engineering, Stanford University, Palo Alto, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-2587-720X","authenticated-orcid":false,"given":"Feng","family":"Hou","sequence":"additional","affiliation":[{"name":"Institute of Computing Technology, Chinese Academy of Sciences, Beijing, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-9954-0693","authenticated-orcid":false,"given":"Jin","family":"Yuan","sequence":"additional","affiliation":[{"name":"School of Computer Science and Engineering, Southeast University, Nanjing, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-1299-8747","authenticated-orcid":false,"given":"Jiang","family":"Tian","sequence":"additional","affiliation":[{"name":"AI Lab, Lenovo Research, Beijing, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-7260-5098","authenticated-orcid":false,"given":"Yang","family":"Zhang","sequence":"additional","affiliation":[{"name":"AI Lab, Lenovo Research, Beijing, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-5216-3827","authenticated-orcid":false,"given":"Zhongchao","family":"Shi","sequence":"additional","affiliation":[{"name":"AI Lab, Lenovo Research, Beijing, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-2290-1785","authenticated-orcid":false,"given":"Jianping","family":"Fan","sequence":"additional","affiliation":[{"name":"AI Lab, Lenovo Research, Beijing, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-4730-0521","authenticated-orcid":false,"given":"Zhiqiang","family":"He","sequence":"additional","affiliation":[{"name":"Institute of Computing Technology, Chinese Academy of Sciences, Beijing, China"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"263","reference":[{"key":"ref1","doi-asserted-by":"publisher","DOI":"10.48550\/ARXIV.1706.03762"},{"key":"ref2","volume-title":"Improving language understanding by generative pre-training","author":"Radford","year":"2018"},{"key":"ref3","volume-title":"Language models are unsupervised multitask learners","author":"Radford","year":"2019"},{"key":"ref4","first-page":"1877","article-title":"Language models are few-shot learners","volume-title":"Proc. NeurIPS","author":"Brown"},{"key":"ref5","first-page":"4171","article-title":"BERT: Pre-training of deep bidirectional transformers for language understanding","volume-title":"Proc. NAACL","author":"Devlin"},{"key":"ref6","doi-asserted-by":"publisher","DOI":"10.48550\/ARXIV.1907.11692"},{"key":"ref7","first-page":"1","article-title":"ALBERT: A lite BERT for self-supervised learning of language representations","volume-title":"Proc. ICLR","author":"Lan"},{"key":"ref8","first-page":"5753","article-title":"XLNet: Generalized autoregressive pretraining for language understanding","volume-title":"Proc. NeurIPS","author":"Yang"},{"key":"ref9","doi-asserted-by":"publisher","DOI":"10.1109\/TNNLS.2020.2979670"},{"key":"ref10","first-page":"1097","article-title":"Imagenet classification with deep convolutional neural networks","volume-title":"Proc. NIPS","author":"Krizhevsky"},{"key":"ref11","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2016.90"},{"key":"ref12","first-page":"6105","article-title":"EfficientNet: Rethinking model scaling for convolutional neural networks","volume-title":"Proc. ICML","author":"Tan"},{"key":"ref13","doi-asserted-by":"publisher","DOI":"10.1109\/TNNLS.2020.3019893"},{"key":"ref14","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2018.00813"},{"key":"ref15","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2019.00069"},{"key":"ref16","doi-asserted-by":"publisher","DOI":"10.1109\/ICCVW.2019.00246"},{"key":"ref17","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2018.00745"},{"key":"ref18","doi-asserted-by":"publisher","DOI":"10.48550\/ARXIV.1807.06521"},{"key":"ref19","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR42600.2020.01155"},{"key":"ref20","first-page":"4055","article-title":"Image transformer","volume-title":"Proc. ICML","author":"Parmar"},{"key":"ref21","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2018.00378"},{"key":"ref22","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2019.00356"},{"key":"ref23","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2019.00338"},{"key":"ref24","first-page":"68","article-title":"Stand-alone self-attention in vision models","volume-title":"Proc. NeurIPS","author":"Ramachandran"},{"key":"ref25","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR42600.2020.01009"},{"key":"ref26","doi-asserted-by":"publisher","DOI":"10.1109\/TNNLS.2020.2978613"},{"key":"ref27","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR46437.2021.01270"},{"key":"ref28","first-page":"1","article-title":"On the relationship between self-attention and convolutional layers","volume-title":"Proc. ICLR","author":"Cordonnier"},{"key":"ref29","first-page":"1","article-title":"An image is worth 16\u00d716 words: Transformers for image recognition at scale","volume-title":"Proc. ICLR","author":"Dosovitskiy"},{"key":"ref30","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-58452-8_13"},{"key":"ref31","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR46437.2021.00542"},{"key":"ref32","first-page":"17864","article-title":"Per-pixel classification is not all you need for semantic segmentation","volume-title":"Proc. NeurIPS","author":"Cheng"},{"key":"ref33","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR46437.2021.00803"},{"key":"ref34","article-title":"TransGAN: Two pure transformers can make one strong GAN, and that can scale up","author":"Jiang","year":"2021","journal-title":"arXiv:2102.07074"},{"key":"ref35","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV48922.2021.00986"},{"key":"ref36","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV48922.2021.00009"},{"key":"ref37","article-title":"Refiner: Refining self-attention for vision transformers","author":"Zhou","year":"2021","journal-title":"arXiv:2106.03714"},{"key":"ref38","article-title":"Scaling vision transformers","author":"Zhai","year":"2021","journal-title":"arXiv:2106.04560"},{"key":"ref39","first-page":"3965","article-title":"CoAtNet: Marrying convolution and attention for all data sizes","volume-title":"Proc. NeurIPS","author":"Dai"},{"key":"ref40","first-page":"10347","article-title":"Training data-efficient image transformers & distillation through attention","volume-title":"Proc. ICLR","author":"Touvron"},{"key":"ref41","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV48922.2021.00061"},{"key":"ref42","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV48922.2021.00010"},{"key":"ref43","doi-asserted-by":"publisher","DOI":"10.48550\/ARXIV.2104.10858"},{"key":"ref44","article-title":"VOLO: Vision outlooker for visual recognition","author":"Yuan","year":"2021","journal-title":"arXiv:2106.13112"},{"issue":"6","key":"ref45","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1145\/3530811","article-title":"Efficient transformers: A survey","volume":"55","author":"Tay","year":"2022","journal-title":"ACM Comput. Surv."},{"key":"ref46","doi-asserted-by":"publisher","DOI":"10.1145\/3505244"},{"key":"ref47","doi-asserted-by":"publisher","DOI":"10.1109\/TPAMI.2022.3152247"},{"key":"ref48","doi-asserted-by":"publisher","DOI":"10.1016\/j.aiopen.2022.10.001"},{"key":"ref49","first-page":"3104","article-title":"Sequence to sequence learning with neural networks","volume-title":"Proc. NeurIPS","author":"Sutskever"},{"key":"ref50","doi-asserted-by":"publisher","DOI":"10.1109\/TNNLS.2016.2582924"},{"key":"ref51","article-title":"Visual transformers: Token-based image representation and processing for computer vision","author":"Wu","year":"2020","journal-title":"arXiv:2006.03677"},{"key":"ref52","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR46437.2021.01625"},{"key":"ref53","first-page":"2286","article-title":"ConViT: Improving vision transformers with soft convolutional inductive biases","volume-title":"Proc. ICLR","author":"D\u2019Ascoli"},{"key":"ref54","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV48922.2021.00062"},{"key":"ref55","article-title":"LocalViT: Bringing locality to vision transformers","author":"Li","year":"2021","journal-title":"arXiv:2104.05707"},{"key":"ref56","doi-asserted-by":"publisher","DOI":"10.48550\/arXiv.2102.10882"},{"key":"ref57","first-page":"15475","article-title":"ResT: An efficient transformer for visual recognition","volume-title":"Proc. NeurIPS","author":"Zhang"},{"key":"ref58","first-page":"30392","article-title":"Early convolutions help transformers see better","volume-title":"Proc. NeruIPS","author":"Xiao"},{"key":"ref59","first-page":"15908","article-title":"Transformer in transformer","volume-title":"Proc. NeurIPS","author":"Han"},{"key":"ref60","first-page":"9355","article-title":"Twins: Revisiting the design of spatial attention in vision transformers","volume-title":"Proc. NeurIPS","author":"Chu"},{"key":"ref61","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV48922.2021.00299"},{"key":"ref62","article-title":"Focal self-attention for local-global interactions in vision transformers","author":"Yang","year":"2021","journal-title":"arXiv:2107.00641"},{"key":"ref63","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV48922.2021.00060"},{"key":"ref64","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV48922.2021.01172"},{"key":"ref65","article-title":"PVT v2: Improved baselines with pyramid vision transformer","author":"Wang","year":"2021","journal-title":"arXiv:2106.13797"},{"key":"ref66","article-title":"DeepViT: Towards deeper vision transformer","author":"Zhou","year":"2021","journal-title":"arXiv:2103.11886"},{"key":"ref67","article-title":"Vision transformers with patch diversification","author":"Gong","year":"2021","journal-title":"arXiv:2104.12753"},{"key":"ref68","first-page":"1691","article-title":"Generative pretraining from pixels","volume-title":"Proc. ICML","author":"Chen"},{"key":"ref69","first-page":"13165","article-title":"MST: Masked self-supervised transformer for visual representation","volume-title":"Proc. NeurIPS","author":"Li"},{"key":"ref70","first-page":"1","article-title":"BEiT: BERT pre-training of image transformers","volume-title":"Proc. ICLR","author":"Bao"},{"key":"ref71","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52688.2022.01553"},{"key":"ref72","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV48922.2021.00950"},{"key":"ref73","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV48922.2021.00951"},{"key":"ref74","article-title":"Self-supervised learning with Swin transformers","author":"Xie","year":"2021","journal-title":"arXiv:2105.04553"},{"key":"ref75","first-page":"1","article-title":"Pix2seq: A language modeling framework for object detection","volume-title":"Proc. ICLR","author":"Chen"},{"key":"ref76","first-page":"1","article-title":"Deformable DETR: Deformable transformers for end-to-end object detection","volume-title":"Proc. ICLR","author":"Zhu"},{"key":"ref77","article-title":"End-to-End object detection with adaptive clustering transformer","author":"Zheng","year":"2020","journal-title":"arXiv:2011.09315"},{"key":"ref78","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV48922.2021.00462"},{"key":"ref79","first-page":"1","article-title":"Sparse DETR: Efficient end-to-end object detection with learnable sparsity","volume-title":"Proc. ICLR","author":"Roh"},{"key":"ref80","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV48922.2021.00360"},{"key":"ref81","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV48922.2021.00363"},{"key":"ref82","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v36i3.20158"},{"key":"ref83","first-page":"1","article-title":"DAB-DETR: Dynamic anchor boxes are better queries for DETR","volume-title":"Proc. ICLR","author":"Liu"},{"key":"ref84","article-title":"SAP-DETR: Bridging the gap between salient points and queries-based transformer detector for fast model convergency","author":"Liu","year":"2022","journal-title":"arXiv:2211.02006"},{"key":"ref85","article-title":"Efficient DETR: Improving end-to-end object detector with dense prior","author":"Yao","year":"2021","journal-title":"arXiv:2104.01318"},{"key":"ref86","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV48922.2021.00298"},{"key":"ref87","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV48922.2021.00359"},{"key":"ref88","first-page":"26183","article-title":"You only look at one sequence: Rethinking transformer in vision through object detection","volume-title":"Proc. NeurIPS","author":"Fang"},{"key":"ref89","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR46437.2021.00165"},{"key":"ref90","first-page":"1","article-title":"FP-DETR: Detection transformer advanced by fully pre-training","volume-title":"Proc. ICLR","author":"Wang"},{"key":"ref91","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52688.2022.01325"},{"key":"ref92","article-title":"DINO: DETR with improved denoising anchor boxes for end-to-end object detection","author":"Zhang","year":"2022","journal-title":"arXiv:2203.03605"},{"key":"ref93","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-58604-1_20"},{"key":"ref94","first-page":"7281","article-title":"HRFormer: High-resolution vision transformer for dense predict","volume-title":"Proc. NeurIPS","author":"Yuan"},{"key":"ref95","article-title":"HRViT: Multi-scale high-resolution vision transformer","author":"Gu","year":"2021","journal-title":"arXiv:2111.01236"},{"key":"ref96","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR46437.2021.00681"},{"key":"ref97","doi-asserted-by":"publisher","DOI":"10.48550\/arXiv.2102.04306"},{"key":"ref98","first-page":"12077","article-title":"SegFormer: Simple and efficient design for semantic segmentation with transformers","volume-title":"Proc. NeurIPS","author":"Xie"},{"key":"ref99","doi-asserted-by":"publisher","DOI":"10.1109\/BIBM49941.2020.9313305"},{"key":"ref100","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR46437.2021.00863"},{"key":"ref101","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV48922.2021.00683"},{"key":"ref102","article-title":"ISTR: End-to-end instance segmentation with transformers","author":"Hu","year":"2021","journal-title":"arXiv:2105.00637"},{"key":"ref103","first-page":"21898","article-title":"SOLQ: Segmenting objects by learning queries","volume-title":"Proc. NeurIPS","author":"Dong"},{"key":"ref104","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV48922.2021.00717"},{"key":"ref105","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV48922.2021.01595"},{"key":"ref106","doi-asserted-by":"publisher","DOI":"10.1007\/s41095-021-0229-5"},{"key":"ref107","doi-asserted-by":"publisher","DOI":"10.1109\/TITS.2022.3198836"},{"key":"ref108","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52688.2022.01644"},{"key":"ref109","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR46437.2021.00738"},{"key":"ref110","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52688.2022.00827"},{"key":"ref111","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV48922.2021.00315"},{"key":"ref112","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52688.2022.00823"},{"key":"ref113","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52688.2022.01871"},{"key":"ref114","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-20086-1_35"},{"key":"ref115","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-20086-1_38"},{"key":"ref116","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV48922.2021.00290"},{"key":"ref117","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV48922.2021.00294"},{"key":"ref118","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV48922.2021.00274"},{"key":"ref119","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52688.2022.00398"},{"key":"ref120","article-title":"MonoDETR: Depth-guided transformer for monocular 3D object detection","author":"Zhang","year":"2022","journal-title":"arXiv:2203.13310"},{"key":"ref121","first-page":"180","article-title":"DETR3D: 3D object detection from multi-view images via 3D-to-2D queries","volume-title":"Proc. 5th Conf. Robot Learn.","author":"Wang"},{"key":"ref122","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52688.2022.00116"},{"key":"ref123","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV48922.2021.01227"},{"key":"ref124","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV48922.2021.00545"},{"key":"ref125","article-title":"Deep point cloud reconstruction","author":"Choe","year":"2021","journal-title":"arXiv:2111.11704"},{"key":"ref126","article-title":"MVT: Multi-view vision transformer for 3D object recognition","author":"Chen","year":"2021","journal-title":"arXiv:2110.13083"},{"key":"ref127","doi-asserted-by":"publisher","DOI":"10.1145\/3474085.3475310"},{"key":"ref128","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR46437.2021.00700"},{"key":"ref129","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV48922.2021.00615"},{"key":"ref130","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV48922.2021.00567"},{"key":"ref131","article-title":"FUTR3D: A unified sensor fusion framework for 3D detection","author":"Chen","year":"2022","journal-title":"arXiv:2203.10642"},{"key":"ref132","first-page":"1403","article-title":"TransformerFusion: Monocular RGB scene reconstruction using transformers","volume-title":"Proc. NeurIPS","author":"Bozic"},{"key":"ref133","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-16443-9_11"},{"key":"ref134","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-87199-4_10"},{"key":"ref135","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR46437.2021.00816"},{"key":"ref136","doi-asserted-by":"publisher","DOI":"10.1109\/TMI.2021.3077484"},{"key":"ref137","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2019.00756"},{"key":"ref138","first-page":"13","article-title":"ViLBERT: Pretraining task-agnostic visiolinguistic representations for vision-and-language tasks","volume-title":"Proc. NeurIPS","author":"Lu"},{"key":"ref139","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D19-1514"},{"key":"ref140","article-title":"VisualBERT: A simple and performant baseline for vision and language","author":"Harold Li","year":"2019","journal-title":"arXiv:1908.03557"},{"key":"ref141","article-title":"VL-BERT: Pre-training of generic visual-linguistic representations","author":"Su","year":"2019","journal-title":"arXiv:1908.08530"},{"key":"ref142","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-58577-8_7"},{"key":"ref143","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-58577-8_8"},{"key":"ref144","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v34i07.7005"},{"key":"ref145","first-page":"5583","article-title":"ViLT: Vision-and-language transformer without convolution or region supervision","volume-title":"Proc. ICML","author":"Kim"},{"key":"ref146","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR46437.2021.00553"},{"key":"ref147","first-page":"8748","article-title":"Learning transferable visual models from natural language supervision","volume-title":"Proc. ICML","author":"Radford"},{"key":"ref148","first-page":"8821","article-title":"Zero-shot text-to-image generation","volume-title":"Proc. ICML","author":"Ramesh"},{"key":"ref149","first-page":"4904","article-title":"Scaling up visual and vision-language representation learning with noisy text supervision","volume-title":"Proc. ICML","author":"Jia"},{"key":"ref150","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV48922.2021.00147"},{"key":"ref151","first-page":"1","article-title":"SimVLM: Simple visual language model pretraining with weak supervision","volume-title":"Proc. ICLR","author":"Wang"},{"key":"ref152","article-title":"Data2vec: A general framework for self-supervised learning in speech, vision and language","author":"Baevski","year":"2022","journal-title":"arXiv:2202.03555"},{"key":"ref153","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV48922.2021.00180"},{"key":"ref154","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV48922.2021.00179"},{"key":"ref155","doi-asserted-by":"publisher","DOI":"10.1109\/ICME52920.2022.9859880"},{"key":"ref156","first-page":"19652","article-title":"Referring transformer: A one-step approach to multi-task visual grounding","volume-title":"Proc. NeurIPS","author":"Li"},{"key":"ref157","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52688.2022.01507"},{"key":"ref158","first-page":"1046","article-title":"LanguageRefer: Spatial-language model for 3D visual grounding","volume-title":"Proc. 5th Conf. Robot Learn.","author":"Roh"},{"key":"ref159","doi-asserted-by":"publisher","DOI":"10.1145\/3474085.3475397"},{"key":"ref160","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52688.2022.01508"},{"key":"ref161","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52688.2022.01595"},{"key":"ref162","article-title":"Layer normalization","author":"Lei Ba","year":"2016","journal-title":"arXiv:1607.06450"},{"key":"ref163","doi-asserted-by":"publisher","DOI":"10.1109\/TNNLS.2015.2496947"},{"key":"ref164","first-page":"352","article-title":"A2-Nets: Double attention networks","volume-title":"Proc. NeurIPS","author":"Chen"},{"key":"ref165","volume-title":"Learning multiple layers of features from tiny images","author":"Krizhevsky","year":"2009"},{"key":"ref166","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2017.97"},{"key":"ref167","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2009.5206848"},{"key":"ref168","first-page":"2793","article-title":"Attention is not all you need: Pure attention loses rank doubly exponentially with depth","volume-title":"Proc. ICLR","author":"Dong"},{"key":"ref169","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/N18-2074"},{"key":"ref170","article-title":"Relational inductive biases, deep learning, and graph networks","author":"Battaglia","year":"2018","journal-title":"arXiv:1806.01261"},{"key":"ref171","article-title":"Transferring inductive biases through knowledge distillation","author":"Abnar","year":"2020","journal-title":"arXiv:2006.00555"},{"key":"ref172","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2018.00474"},{"key":"ref173","first-page":"1","article-title":"How much position information do convolutional neural networks encode?","volume-title":"Proc. ICLR","author":"Islam"},{"key":"ref174","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2019.00718"},{"key":"ref175","doi-asserted-by":"publisher","DOI":"10.1109\/TNNLS.2017.2676130"},{"key":"ref176","first-page":"1","article-title":"Representation degeneration problem in training natural language generation models","volume-title":"Proc. ICLR","author":"Gao"},{"key":"ref177","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2019.00612"},{"key":"ref178","first-page":"9912","article-title":"Unsupervised learning of visual features by contrasting cluster assignments","volume-title":"Proc. NeurIPS","author":"Caron"},{"key":"ref179","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV48922.2021.00041"},{"key":"ref180","doi-asserted-by":"publisher","DOI":"10.1109\/TPAMI.2016.2577031"},{"key":"ref181","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2016.91"},{"key":"ref182","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2017.324"},{"key":"ref183","doi-asserted-by":"publisher","DOI":"10.1109\/TNNLS.2018.2876865"},{"key":"ref184","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2017.89"},{"key":"ref185","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2017.106"},{"key":"ref186","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2019.00972"},{"key":"ref187","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2017.322"},{"key":"ref188","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2018.00644"},{"key":"ref189","first-page":"9934","article-title":"What makes for end-to-end object detection?","volume-title":"Proc. ICML","author":"Sun"},{"key":"ref190","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2019.00584"},{"key":"ref191","doi-asserted-by":"publisher","DOI":"10.1109\/TPAMI.2017.2699184"},{"key":"ref192","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-24574-4_28"},{"key":"ref193","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR46437.2021.01422"},{"key":"ref194","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-01228-1_26"},{"key":"ref195","first-page":"1","article-title":"Searching the search space of vision transformer","volume-title":"Proc. NeurIPS","author":"Chen"},{"key":"ref196","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR42600.2020.00860"},{"key":"ref197","first-page":"17721","article-title":"SOLOv2: Dynamic and fast instance segmentation","volume-title":"Proc. NeurIPS","author":"Wang"},{"key":"ref198","first-page":"11960","article-title":"Not all images are worth 16\u00d716 words: Dynamic transformers for efficient image recognition","volume-title":"Proc. NeurIPS","author":"Wang"},{"key":"ref199","first-page":"1","article-title":"PointNet++: Deep hierarchical feature learning on point sets in a metric space","volume-title":"Proc. NeurIPS","author":"Qi"},{"key":"ref200","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2018.00636"},{"key":"ref201","doi-asserted-by":"publisher","DOI":"10.1007\/s11263-016-0981-7"},{"key":"ref202","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/P18-1238"},{"key":"ref203","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2015.279"},{"key":"ref204","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2015.7298935"},{"key":"ref205","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-58452-8_25"},{"key":"ref206","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52688.2022.01055"},{"key":"ref207","first-page":"1","article-title":"How do vision transformers work?","volume-title":"Proc. ICLR","author":"Park"}],"container-title":["IEEE Transactions on Neural Networks and Learning Systems"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx7\/5962385\/10547160\/10088164.pdf?arnumber=10088164","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,12,11]],"date-time":"2024-12-11T01:48:06Z","timestamp":1733881686000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/10088164\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,6]]},"references-count":207,"journal-issue":{"issue":"6"},"URL":"https:\/\/doi.org\/10.1109\/tnnls.2022.3227717","relation":{},"ISSN":["2162-237X","2162-2388"],"issn-type":[{"value":"2162-237X","type":"print"},{"value":"2162-2388","type":"electronic"}],"subject":[],"published":{"date-parts":[[2024,6]]}}}