{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,7]],"date-time":"2026-07-07T15:59:23Z","timestamp":1783439963021,"version":"3.54.6"},"reference-count":100,"publisher":"Institute of Electrical and Electronics Engineers (IEEE)","issue":"9","license":[{"start":{"date-parts":[[2025,9,1]],"date-time":"2025-09-01T00:00:00Z","timestamp":1756684800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/ieeexplore.ieee.org\/Xplorehelp\/downloads\/license-information\/IEEE.html"},{"start":{"date-parts":[[2025,9,1]],"date-time":"2025-09-01T00:00:00Z","timestamp":1756684800000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2025,9,1]],"date-time":"2025-09-01T00:00:00Z","timestamp":1756684800000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["IEEE Trans. Pattern Anal. Mach. Intell."],"published-print":{"date-parts":[[2025,9]]},"DOI":"10.1109\/tpami.2025.3571946","type":"journal-article","created":{"date-parts":[[2025,5,20]],"date-time":"2025-05-20T13:15:03Z","timestamp":1747746903000},"page":"7543-7557","source":"Crossref","is-referenced-by-count":89,"title":["Otter: A Multi-Modal Model With In-Context Instruction Tuning"],"prefix":"10.1109","volume":"47","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-8447-0928","authenticated-orcid":false,"given":"Bo","family":"Li","sequence":"first","affiliation":[{"name":"S-Lab, Nanyang Technological University, Singapore"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-9063-7886","authenticated-orcid":false,"given":"Yuanhan","family":"Zhang","sequence":"additional","affiliation":[{"name":"S-Lab, Nanyang Technological University, Singapore"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-5181-8457","authenticated-orcid":false,"given":"Liangyu","family":"Chen","sequence":"additional","affiliation":[{"name":"S-Lab, Nanyang Technological University, Singapore"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0009-5919-5328","authenticated-orcid":false,"given":"Jinghao","family":"Wang","sequence":"additional","affiliation":[{"name":"S-Lab, Nanyang Technological University, Singapore"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Fanyi","family":"Pu","sequence":"additional","affiliation":[{"name":"S-Lab, Nanyang Technological University, Singapore"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Joshua Adrian","family":"Cahyono","sequence":"additional","affiliation":[{"name":"S-Lab, Nanyang Technological University, Singapore"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-2923-3137","authenticated-orcid":false,"given":"Jingkang","family":"Yang","sequence":"additional","affiliation":[{"name":"S-Lab, Nanyang Technological University, Singapore"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Chunyuan","family":"Li","sequence":"additional","affiliation":[{"name":"Microsoft Research, Redmond, WA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-4220-5958","authenticated-orcid":false,"given":"Ziwei","family":"Liu","sequence":"additional","affiliation":[{"name":"S-Lab, Nanyang Technological University, Singapore"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"263","reference":[{"key":"ref2","article-title":"What learning algorithm is in-context learning? investigations with linear models","author":"Aky\u00fcrek","year":"2022"},{"key":"ref3","first-page":"23716","article-title":"Flamingo: A visual language model for few-shot learning","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","author":"Alayrac"},{"key":"ref4","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2015.279"},{"key":"ref5","article-title":"\u201cOpenFlamingo: An open-source framework for training large autoregressive vision-language models","author":"Awadalla","year":"2023"},{"key":"ref6","article-title":"OpenFlamingo: An open-source framework for training large autoregressive vision-language models","author":"Awadalla","year":"2023"},{"key":"ref7","article-title":"Qwen-vl: A versatile vision-language model for understanding, localization, text reading, and beyond","author":"Bai","year":"2023"},{"key":"ref8","article-title":"Introducing our multimodal models","author":"Bavishi","year":"2023","journal-title":"Adept AI Blog"},{"key":"ref9","first-page":"1877","article-title":"Language models are few-shot learners","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","author":"Brown"},{"key":"ref10","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2015.7298698"},{"key":"ref11","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR46437.2021.00356"},{"key":"ref12","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-58565-5_13"},{"key":"ref13","article-title":"Shikra: Unleashing multimodal llm\u2019s referential dialogue magic","author":"Chen","year":"2023"},{"key":"ref14","article-title":"Making your first choice: To address cold start problem in vision active learning","author":"Chen","year":"2022"},{"key":"ref15","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-72643-9_22"},{"key":"ref16","doi-asserted-by":"publisher","DOI":"10.1109\/cvpr52733.2024.01368"},{"key":"ref17","article-title":"PaLI-3 vision language models: Smaller, faster, stronger","author":"Chen","year":"2023"},{"key":"ref18","article-title":"Microsoft COCO captions: Data collection and evaluation server","author":"Chen","year":"2015"},{"key":"ref19","article-title":"Vicuna: An open-source chatbot impressing GPT-4 with 90%* chatgpt quality","author":"Chiang","year":"2023","journal-title":"LMSYS Blog"},{"key":"ref20","article-title":"PALM: Scaling language modeling with pathways","author":"Chowdhery","year":"2022"},{"key":"ref21","article-title":"Scaling instruction-finetuned language models","author":"WonC","year":"2022"},{"key":"ref22","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2017.261"},{"key":"ref23","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2023.findings-acl.247"},{"key":"ref24","article-title":"InstructBLIP: Towards general-purpose vision-language models with instruction tuning","author":"Dai","year":"2023"},{"key":"ref25","article-title":"InstructBLIP: Towards general-purpose vision-language models with instruction tuning","author":"Dai","year":"2023"},{"key":"ref26","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2009.5206848"},{"key":"ref27","article-title":"A comprehensive evaluation benchmark for multimodal large language models","author":"Fu","year":"2023"},{"key":"ref28","article-title":"LLaMA-adapter V2: Parameter-efficient visual instruction model","author":"Gao","year":"2023"},{"key":"ref29","first-page":"30583","article-title":"What can transformers learn in-context? A case study of simple function classes","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","author":"Garg"},{"key":"ref30","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2017.670"},{"key":"ref31","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2023.acl-long.806"},{"key":"ref32","first-page":"1233","article-title":"Visual storytelling","volume-title":"Proc. 15th Annu. Conf. North Amer. Chapter Assoc. Comput. Linguistics","author":"Ting-Hao"},{"key":"ref33","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2019.00686"},{"key":"ref34","article-title":"Face classification and detection","author":"Hyeon","year":"2023"},{"key":"ref35","first-page":"4024","volume-title":"Proc. Conf. Empirical Methods Natural Lang. Process.","author":"Jhamtani"},{"key":"ref36","doi-asserted-by":"publisher","DOI":"10.3115\/v1\/D14-1086"},{"key":"ref37","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2017.356"},{"key":"ref38","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2017.83"},{"key":"ref39","doi-asserted-by":"publisher","DOI":"10.1007\/s11263-016-0981-7"},{"key":"ref40","doi-asserted-by":"publisher","DOI":"10.1007\/s11263-016-0981-7"},{"key":"ref41","first-page":"71683","article-title":"OBELICS: An open web-scale filtered dataset of interleaved image-text documents","volume-title":"Proc. 27th Conf. Neural Inf. Process. Syst. Datasets Benchmarks Track","author":"Lauren\u00e7on"},{"key":"ref42","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-58589-1_27"},{"key":"ref43","article-title":"A high-resolution multi-modality model","author":"Li","year":"2023"},{"key":"ref44","article-title":"BLIP-2: Bootstrapping language-image pre-training with frozen image encoders and large language models","author":"Li","year":"2023"},{"key":"ref45","article-title":"VideoChat: Chat-centric video understanding","author":"Li","year":"2023"},{"key":"ref46","article-title":"M3IT: A large-scale dataset towards multi-modal multilingual instruction tuning","author":"Li","year":"2023"},{"key":"ref47","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2023.emnlp-main.20"},{"key":"ref48","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-10602-1_48"},{"key":"ref49","article-title":"Aligning large multi-modal model with robust instruction tuning","author":"Liu","year":"2023"},{"key":"ref50","article-title":"Mitigating hallucination in large multi-modal models via robust instruction tuning","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Liu","year":"2024"},{"key":"ref51","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52733.2024.02484"},{"key":"ref52","article-title":"Visual instruction tuning","author":"Liu","year":"2023"},{"key":"ref53","article-title":"Prismer: A vision-language model with an ensemble of experts","author":"Liu","year":"2023"},{"key":"ref54","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-72658-3_13"},{"key":"ref55","article-title":"Evaluating mathematical reasoning of foundation models in visual contexts","author":"Lu","year":"2023"},{"key":"ref56","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2019.00331"},{"key":"ref57","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2022.emnlp-main.759"},{"key":"ref58","doi-asserted-by":"publisher","DOI":"10.1109\/ICDAR.2019.00156"},{"key":"ref59","article-title":"Video-chatGPT: Towards detailed video understanding via large vision and language models","author":"Khan","year":"2023"},{"key":"ref61","first-page":"1143","article-title":"Im2Text: Describing images using 1 million captioned photographs","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","author":"Ordonez"},{"key":"ref62","first-page":"27730","article-title":"Training language models to follow instructions with human feedback","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","author":"Ouyang"},{"key":"ref63","article-title":"Instruction tuning with GPT-4","author":"Peng","year":"2023"},{"key":"ref64","first-page":"8748","article-title":"Learning transferable visual models from natural language supervision","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Radford"},{"key":"ref65","first-page":"8748","article-title":"Learning transferable visual models from natural language supervision","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Radford"},{"issue":"8","key":"ref66","article-title":"Language models are unsupervised multitask learners","volume":"1","author":"Radford","year":"2019","journal-title":"OpenAI blog"},{"key":"ref67","article-title":"Multitask prompted training enables zero-shot task generalization","author":"Sanh","year":"2021"},{"key":"ref68","first-page":"25278","article-title":"Laion-5B: An open large-scale dataset for training next generation image-text models","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","author":"Schuhmann"},{"key":"ref69","article-title":"Laion-400m: Open dataset of clip-filtered 400 million image-text pairs","author":"Schuhmann","year":"2021"},{"key":"ref70","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-20074-8_9"},{"key":"ref71","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/P18-1238"},{"key":"ref72","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2019.00851"},{"key":"ref73","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2023.acl-long.401"},{"key":"ref74","article-title":"Stanford Alpaca: An instruction-following llama model","author":"Taori","year":"2023"},{"key":"ref75","article-title":"Introducing MPT-7B: A new standard for open-source, commercially usable LLMs","year":"2023","journal-title":"MosaicML Blog"},{"key":"ref76","article-title":"LLaMA: Open and efficient foundation language models","author":"Touvron","year":"2023"},{"key":"ref77","article-title":"LLaMA: Open and efficient foundation language models","author":"Touvron","year":"2023"},{"key":"ref78","first-page":"35151","article-title":"Transformers learn in-context by gradient descent","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Von"},{"key":"ref79","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2023.acl-long.754"},{"key":"ref80","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2022.emnlp-main.340"},{"key":"ref81","article-title":"Finetuned language models are zero-shot learners","volume-title":"Proc. Int. Conf. Learn. Representations","author":"Wei"},{"key":"ref82","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-73232-4_3"},{"key":"ref83","doi-asserted-by":"publisher","DOI":"10.1145\/3123266.3123427"},{"key":"ref84","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2023.acl-long.641"},{"key":"ref85","article-title":"mPLUG-Owl: Modularization empowers large language models with multimodality","author":"Ye","year":"2023"},{"key":"ref86","article-title":"In-context instruction learning","author":"Ye","year":"2023"},{"key":"ref87","first-page":"26650","article-title":"Language-assisted multi-modal instruction-tuning dataset, framework, and benchmark","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","author":"Yin"},{"key":"ref88","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-46475-6_5"},{"key":"ref89","article-title":"MM-Vet: Evaluating large multimodal models for integrated capabilities","author":"Yu","year":"2023"},{"key":"ref90","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v33i01.33019127"},{"key":"ref91","article-title":"LLaMA-adapter: Efficient fine-tuning of language models with zero-init attention","author":"Zhang","year":"2023"},{"key":"ref92","article-title":"LLAMA-adapter: Efficient fine-tuning of language models with zero-init attention","author":"Zhang","year":"2023"},{"key":"ref93","article-title":"LLAMA-adapter: Efficient fine-tuning of language models with zero-init attention","author":"Zhang","year":"2023"},{"key":"ref94","article-title":"Open pre-trained transformer language models","author":"Zhang","year":"2022"},{"key":"ref95","article-title":"Enhanced visual instruction tuning for text-rich image understanding","author":"Zhang","year":"2023"},{"key":"ref96","article-title":"SVIT: Scaling up visual instruction tuning","author":"Zhao","year":"2023"},{"key":"ref97","article-title":"MMICL: Empowering vision-language model with multi-modal in-context learning","author":"Zhao","year":"2023"},{"key":"ref98","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52688.2022.01631"},{"key":"ref99","doi-asserted-by":"publisher","DOI":"10.1007\/s11263-022-01653-1"},{"key":"ref100","article-title":"MiniGPT-4: Enhancing vision-language understanding with advanced large language models","author":"Zhu","year":"2023"},{"key":"ref101","article-title":"Multimodal C4: An open, billion-scale corpus of images interleaved with text","author":"Zhu","year":"2023"},{"key":"ref102","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2016.540"}],"container-title":["IEEE Transactions on Pattern Analysis and Machine Intelligence"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx8\/34\/11118328\/11007678.pdf?arnumber=11007678","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,8,7]],"date-time":"2025-08-07T17:44:22Z","timestamp":1754588662000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/11007678\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,9]]},"references-count":100,"journal-issue":{"issue":"9"},"URL":"https:\/\/doi.org\/10.1109\/tpami.2025.3571946","relation":{},"ISSN":["0162-8828","2160-9292","1939-3539"],"issn-type":[{"value":"0162-8828","type":"print"},{"value":"2160-9292","type":"electronic"},{"value":"1939-3539","type":"electronic"}],"subject":[],"published":{"date-parts":[[2025,9]]}}}