{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,12,1]],"date-time":"2025-12-01T11:31:05Z","timestamp":1764588665040,"version":"build-2065373602"},"reference-count":111,"publisher":"Institute of Electrical and Electronics Engineers (IEEE)","issue":"11","license":[{"start":{"date-parts":[[2025,11,1]],"date-time":"2025-11-01T00:00:00Z","timestamp":1761955200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/ieeexplore.ieee.org\/Xplorehelp\/downloads\/license-information\/IEEE.html"},{"start":{"date-parts":[[2025,11,1]],"date-time":"2025-11-01T00:00:00Z","timestamp":1761955200000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2025,11,1]],"date-time":"2025-11-01T00:00:00Z","timestamp":1761955200000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"}],"funder":[{"name":"National Key R&D Program of China","award":["2022ZD0161300","2022ZD0160100"],"award-info":[{"award-number":["2022ZD0161300","2022ZD0160100"]}]},{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["U24A20325","62376134","62321005"],"award-info":[{"award-number":["U24A20325","62376134","62321005"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100007219","name":"Natural Science Foundation of Shanghai","doi-asserted-by":"publisher","award":["25ZR1402268"],"award-info":[{"award-number":["25ZR1402268"]}],"id":[{"id":"10.13039\/100007219","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["IEEE Trans. Pattern Anal. Mach. Intell."],"published-print":{"date-parts":[[2025,11]]},"DOI":"10.1109\/tpami.2025.3593283","type":"journal-article","created":{"date-parts":[[2025,7,28]],"date-time":"2025-07-28T19:49:49Z","timestamp":1753732189000},"page":"10142-10159","source":"Crossref","is-referenced-by-count":1,"title":["Parameter-Inverted Image Pyramid Networks for Visual Perception and Multimodal Understanding"],"prefix":"10.1109","volume":"47","author":[{"ORCID":"https:\/\/orcid.org\/0009-0003-3331-4022","authenticated-orcid":false,"given":"Zhaokai","family":"Wang","sequence":"first","affiliation":[{"name":"Shanghai Jiao Tong University, Shanghai, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Xizhou","family":"Zhu","sequence":"additional","affiliation":[{"name":"Tsinghua University, Beijing, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-7084-9101","authenticated-orcid":false,"given":"Xue","family":"Yang","sequence":"additional","affiliation":[{"name":"Shanghai Jiao Tong University, Shanghai, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Gen","family":"Luo","sequence":"additional","affiliation":[{"name":"Shanghai Artificial Intelligence Laboratory, Shanghai, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Hao","family":"Li","sequence":"additional","affiliation":[{"name":"The Chinese University of Hong Kong, Hong Kong"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-3285-4671","authenticated-orcid":false,"given":"Changyao","family":"Tian","sequence":"additional","affiliation":[{"name":"The Chinese University of Hong Kong, Hong Kong"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0009-0002-6398-0512","authenticated-orcid":false,"given":"Wenhan","family":"Dou","sequence":"additional","affiliation":[{"name":"Tsinghua University, Beijing, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Junqi","family":"Ge","sequence":"additional","affiliation":[{"name":"Tsinghua University, Beijing, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0009-0009-9809-3818","authenticated-orcid":false,"given":"Lewei","family":"Lu","sequence":"additional","affiliation":[{"name":"Sensetime, Hong Kong"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-1889-2567","authenticated-orcid":false,"given":"Yu","family":"Qiao","sequence":"additional","affiliation":[{"name":"Shanghai Artificial Intelligence Laboratory, Shanghai, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-6785-0785","authenticated-orcid":false,"given":"Jifeng","family":"Dai","sequence":"additional","affiliation":[{"name":"Tsinghua University, Beijing, China"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"263","reference":[{"key":"ref1","first-page":"132267","article-title":"Parameter-inverted image pyramid networks","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Zhu"},{"key":"ref2","first-page":"10347","article-title":"Training data-efficient image transformers and distillation through attention","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Touvron"},{"key":"ref3","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-20053-3_30"},{"article-title":"How to train your ViT? data, augmentation, and regularization in vision transformers","year":"2021","author":"Steiner","key":"ref4"},{"key":"ref5","first-page":"8748","article-title":"Learning transferable visual models from natural language supervision","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Radford"},{"key":"ref6","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2018.00644"},{"article-title":"Deformable DETR: Deformable transformers for end-to-end object detection","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Zhu","key":"ref7"},{"key":"ref8","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2019.00832"},{"article-title":"DINO: DETR with improved deNoising anchor boxes for end-to-end object detection","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Zhang","key":"ref9"},{"key":"ref10","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2017.322"},{"key":"ref11","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-01228-1_26"},{"article-title":"SNIPER: Efficient multi-scale training","volume-title":"Proc. IEEE\/CVF Int. Conf. Neural Inf. Process. Syst.","author":"Singh","key":"ref12"},{"key":"ref13","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2019.00984"},{"key":"ref14","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2017.106"},{"key":"ref15","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2019.00720"},{"key":"ref16","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR42600.2020.01079"},{"key":"ref17","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52729.2023.01385"},{"key":"ref18","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52729.2023.01855"},{"key":"ref19","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV51070.2023.00621"},{"article-title":"Vision transformer adapter for dense predictions","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Chen","key":"ref20"},{"key":"ref21","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-10602-1_48"},{"article-title":"The dawn of LMMs: Preliminary explorations with GPT-4V","year":"2023","author":"Yang","key":"ref22"},{"key":"ref23","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52733.2024.02484"},{"article-title":"Expanding performance boundaries of open-source multimodal models with model, data, and test-time scaling","year":"2024","author":"Chen","key":"ref24"},{"key":"ref25","doi-asserted-by":"publisher","DOI":"10.1109\/cvpr52734.2025.02324"},{"article-title":"Qwen-vl: A frontier large vision-language model with versatile abilities","year":"2023","author":"Bai","key":"ref26"},{"article-title":"Feast your eyes: Mixture-of-resolution adaptation for multimodal large language models","year":"2024","author":"Luo","key":"ref27"},{"article-title":"Mini-Gemini: Mining the potential of multi-modality vision language models","year":"2024","author":"Li","key":"ref28"},{"article-title":"MG-LLaVA: Towards multi-granularity visual instruction tuning","year":"2024","author":"Zhao","key":"ref29"},{"article-title":"Llava-next: Stronger llms supercharge multimodal capabilities in the wild","year":"2024","author":"Li","key":"ref30"},{"key":"ref31","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-73010-8_23"},{"article-title":"LLaVA-UHD v2: An MLLM integrating high-resolution feature pyramid via hierarchical window transformer","year":"2024","author":"Zhang","key":"ref32"},{"article-title":"An image is worth 16x16 words: Transformers for image recognition at scale","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Dosovitskiy","key":"ref33"},{"key":"ref34","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52688.2022.01167"},{"key":"ref35","doi-asserted-by":"publisher","DOI":"10.1109\/cvpr52733.2024.02283"},{"key":"ref36","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2017.544"},{"key":"ref37","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2018.00377"},{"key":"ref38","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV48922.2021.00041"},{"key":"ref39","doi-asserted-by":"publisher","DOI":"10.1109\/TPAMI.2020.2983686"},{"key":"ref40","first-page":"11653","article-title":"CBNet: A composite backbone network architecture for object detection","volume-title":"Proc. AAAI Conf. Artif. Intell.","author":"Liang"},{"key":"ref41","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52733.2024.00525"},{"key":"ref42","first-page":"7281","article-title":"HRFormer: High-resolution vision transformer for dense prediction","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Yuan"},{"article-title":"Hrvit: Multi-scale high-resolution vision transformer","year":"2021","author":"Gu","key":"ref43"},{"key":"ref44","doi-asserted-by":"publisher","DOI":"10.1109\/cvpr52733.2024.01354"},{"year":"2023","key":"ref45","article-title":"GPT-4 technical report"},{"key":"ref46","article-title":"The claude 3 model family: Opus, sonnet, haiku","author":"Anthropic","year":"2024","journal-title":"Claude-3 Model Card"},{"article-title":"Gemini: A family of highly capable multimodal models","year":"2023","author":"Team","key":"ref47"},{"key":"ref48","doi-asserted-by":"publisher","DOI":"10.1007\/s11432-024-4231-5"},{"key":"ref49","first-page":"13937","article-title":"DynamicViT: Efficient vision transformers with dynamic token sparsification","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Rao"},{"key":"ref50","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52688.2022.01199"},{"article-title":"Not all patches are what you need: Expediting vision transformers via token reorganizations","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Liang","key":"ref51"},{"key":"ref52","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v36i3.20202"},{"key":"ref53","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV51070.2023.01587"},{"key":"ref54","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV48922.2021.00986"},{"key":"ref55","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV48922.2021.00061"},{"article-title":"Layer normalization","year":"2016","author":"Ba","key":"ref56"},{"key":"ref57","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-20077-9_17"},{"key":"ref58","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-01261-8_1"},{"key":"ref59","first-page":"34892","article-title":"Visual instruction tuning","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Liu"},{"article-title":"AutoAugment: Learning augmentation policies from data","year":"2018","author":"Cubuk","key":"ref60"},{"article-title":"Benchmarking detection transfer learning with vision transformers","year":"2021","author":"Li","key":"ref61"},{"article-title":"Shuffle transformer: Rethinking spatial shuffle for vision transformer","year":"2021","author":"Huang","key":"ref62"},{"key":"ref63","doi-asserted-by":"publisher","DOI":"10.1109\/cvpr52688.2022.01166"},{"key":"ref64","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2009.5206848"},{"key":"ref65","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52688.2022.01553"},{"article-title":"Decoupled weight decay regularization","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Loshchilov","key":"ref66"},{"key":"ref67","doi-asserted-by":"publisher","DOI":"10.1145\/3394486.3406703"},{"key":"ref68","doi-asserted-by":"publisher","DOI":"10.1109\/SC41405.2020.00024"},{"article-title":"BEiT: BERT pre-training of image transformers","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Bao","key":"ref69"},{"key":"ref70","first-page":"25278","article-title":"LAION-5B: An open large-scale dataset for training next generation image-text models","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Schuhmann"},{"article-title":"MMDetection: Open MMlab detection toolbox and benchmark","year":"2019","author":"Chen","key":"ref71"},{"key":"ref72","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52688.2022.01630"},{"article-title":"DINOv2: Learning robust visual features without supervision","year":"2023","author":"Oquab","key":"ref73"},{"article-title":"BEiT v2: Masked image modeling with vector-quantized visual tokenizers","year":"2022","author":"Peng","key":"ref74"},{"article-title":"More ConvNets in the 2020s: Scaling up kernels beyond 51x51 using sparsity","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Liu","key":"ref75"},{"key":"ref76","first-page":"49250","article-title":"InstructBLIP: Towards general-purpose vision-language models with instruction tuning","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Dai"},{"key":"ref77","first-page":"19730","article-title":"BLIP-2: Bootstrapping language-image pre-training with frozen image encoders and large language models","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Li"},{"article-title":"Shikra: Unleashing multimodal LLM\u2019s referential dialogue magic","year":"2023","author":"Chen","key":"ref78"},{"key":"ref79","doi-asserted-by":"publisher","DOI":"10.1109\/cvpr52733.2024.01239"},{"article-title":"SPHINX-X: Scaling data and parameters for a family of multi-modal large language models","year":"2024","author":"Liu","key":"ref80"},{"key":"ref81","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-72775-7_2"},{"article-title":"InternLM-XComposer: A vision-language large model for advanced text-image comprehension and composition","year":"2023","author":"Zhang","key":"ref82"},{"key":"ref83","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-73397-0_18"},{"article-title":"MMSegmentation: Openmmlab semantic segmentation toolbox and benchmark","year":"2020","author":"Contributors","key":"ref84"},{"article-title":"Vicuna: An open-source chatbot impressing GPT-4 with 90% chatgpt quality","year":"2023","author":"Chiang","key":"ref85"},{"key":"ref86","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-72643-9_22"},{"year":"2023","key":"ref87","article-title":"Laion-gpt-4v"},{"article-title":"ALLaVA: Harnessing GPT4V-synthesized data for a lite vision-language model","year":"2024","author":"Chen","key":"ref88"},{"key":"ref89","first-page":"55006","article-title":"LIMA: Less is More for Alignment","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Zhou"},{"key":"ref90","first-page":"47669","article-title":"Openassistant conversations-democratizing large language model alignment","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"K\u00f6pf"},{"article-title":"Dynamic prompt learning via policy gradient for semi-structured mathematical reasoning","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Lu","key":"ref91"},{"article-title":"MetaMath: Bootstrap your own mathematical questions for large language models","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Yu","key":"ref92"},{"key":"ref93","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v33i01.33018876"},{"key":"ref94","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2021.acl-long.528"},{"key":"ref95","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2019.00439"},{"key":"ref96","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2022.findings-acl.177"},{"key":"ref97","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2018.00592"},{"key":"ref98","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-46493-0_15"},{"key":"ref99","first-page":"28541","article-title":"LLaVA-Med: Training a large language-and-vision assistant for biomedicine in one day","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Li"},{"key":"ref100","doi-asserted-by":"publisher","DOI":"10.1109\/WACV51458.2022.00264"},{"key":"ref101","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2024.findings-emnlp.268"},{"key":"ref102","first-page":"2507","article-title":"Learn to explain: Multimodal reasoning via thought chains for science question answering","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Lu"},{"key":"ref103","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-72658-3_13"},{"key":"ref104","first-page":"57730","article-title":"MM-Vet: Evaluating large multimodal models for integrated capabilities","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Yu"},{"key":"ref105","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2019.00851"},{"key":"ref106","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2019.00686"},{"key":"ref107","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2017.670"},{"article-title":"SEED-Bench: Benchmarking multimodal llms with generative comprehension","year":"2023","author":"Li","key":"ref108"},{"key":"ref109","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2023.emnlp-main.20"},{"key":"ref110","doi-asserted-by":"publisher","DOI":"10.48550\/ARXIV.1706.03762"},{"article-title":"MobileNets: Efficient convolutional neural networks for mobile vision applications","year":"2017","author":"Howard","key":"ref111"}],"container-title":["IEEE Transactions on Pattern Analysis and Machine Intelligence"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx8\/34\/11192800\/11098674.pdf?arnumber=11098674","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,10,6]],"date-time":"2025-10-06T17:36:43Z","timestamp":1759772203000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/11098674\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,11]]},"references-count":111,"journal-issue":{"issue":"11"},"URL":"https:\/\/doi.org\/10.1109\/tpami.2025.3593283","relation":{},"ISSN":["0162-8828","2160-9292","1939-3539"],"issn-type":[{"type":"print","value":"0162-8828"},{"type":"electronic","value":"2160-9292"},{"type":"electronic","value":"1939-3539"}],"subject":[],"published":{"date-parts":[[2025,11]]}}}