{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,9,11]],"date-time":"2025-09-11T16:38:12Z","timestamp":1757608692205,"version":"3.44.0"},"reference-count":75,"publisher":"IEEE","license":[{"start":{"date-parts":[[2025,5,19]],"date-time":"2025-05-19T00:00:00Z","timestamp":1747612800000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2025,5,19]],"date-time":"2025-05-19T00:00:00Z","timestamp":1747612800000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2025,5,19]]},"DOI":"10.1109\/icra55743.2025.11128326","type":"proceedings-article","created":{"date-parts":[[2025,9,2]],"date-time":"2025-09-02T17:28:56Z","timestamp":1756834136000},"page":"8234-8241","source":"Crossref","is-referenced-by-count":0,"title":["Towards Robust Autonomous Driving: Conditional Multimodal Large Language Models for Fine-Grained Perception"],"prefix":"10.1109","author":[{"given":"Fengzhao","family":"Sun","sequence":"first","affiliation":[{"name":"University of Science and Technology of China,Hefei,China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Jun","family":"Yu","sequence":"additional","affiliation":[{"name":"University of Science and Technology of China,Hefei,China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Yunxiang","family":"Zhang","sequence":"additional","affiliation":[{"name":"University of Science and Technology of China,Hefei,China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Jiaming","family":"Hou","sequence":"additional","affiliation":[{"name":"Harbin Institute of Technology,Harbin,China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Xilong","family":"Lu","sequence":"additional","affiliation":[{"name":"University of Science and Technology of China,Hefei,China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Heng","family":"Song","sequence":"additional","affiliation":[{"name":"China Railway No.4 Engineering Group Co., Ltd"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Fang","family":"Gao","sequence":"additional","affiliation":[{"name":"Guangxi University,Nanning,China"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"263","reference":[{"key":"ref1","first-page":"1877","article-title":"Language models are few-shot learners","volume":"33","author":"Brown","year":"2020","journal-title":"Advances in Neural Information Processing Systems"},{"key":"ref2","first-page":"27730","article-title":"Training language models to follow instructions with human feedback","volume":"35","author":"Ouyang","year":"2022","journal-title":"Advances in Neural Information Processing Systems"},{"issue":"3","key":"ref3","first-page":"6","volume-title":"Vicuna: An open-source chatbot impressing gpt-4 with 90%* chatgpt quality","volume":"2","author":"Chiang","year":"2023"},{"issue":"240","key":"ref4","first-page":"1","article-title":"Palm: Scaling language modeling with pathways","volume":"24","author":"Chowdhery","year":"2023","journal-title":"Journal of Machine Learning Research"},{"key":"ref5","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2022.acl-long.26"},{"volume-title":"Chatgpt","year":"2022","key":"ref6"},{"journal-title":"Gpt-4 technical report","year":"2023","author":"Achiam","key":"ref7"},{"journal-title":"Qwen technical report","year":"2023","author":"Bai","key":"ref8"},{"issue":"140","key":"ref9","first-page":"1","article-title":"Exploring the limits of transfer learning with a unified text-to-text transformer","volume":"21","author":"Raffel","year":"2020","journal-title":"Journal of Machine Learning Research"},{"volume-title":"Internlm: A multilingual language model with progres-sively enhanced capabilities","year":"2023","author":"Team","key":"ref10"},{"journal-title":"Llama: Open and efficient foundation language models","year":"2023","author":"Touvron","key":"ref11"},{"journal-title":"Llama 2: Open foundation and fine-tuned chat models","year":"2023","author":"Touvron","key":"ref12"},{"key":"ref13","first-page":"23716","article-title":"Flamingo: a visual language model for few-shot learning","volume":"35","author":"Alayrac","year":"2022","journal-title":"Advances in Neural Information Processing Systems"},{"journal-title":"Shikra: Unleashing multimodal Ilm\u2019s referential dialogue magic","year":"2023","author":"Chen","key":"ref14"},{"key":"ref15","article-title":"PaLI: A jointly-scaled multilingual language-image model","volume-title":"The Eleventh International Conference on Learning Representations","author":"Chen","year":"2023"},{"key":"ref16","article-title":"Visual instruction tuning","volume":"36","author":"Liu","year":"2024","journal-title":"Advances in Neural Information Processing Systems"},{"key":"ref17","article-title":"Kosmos-2: Grounding multimodal large language models to the world","volume-title":"The Twelfth International Conference on Learning Representations","author":"Peng","year":"2024"},{"key":"ref18","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52733.2024.02484"},{"journal-title":"Intern Im-xcomposer: A vision-language large model for advanced text-image comprehension and composition","year":"2023","author":"Zhang","key":"ref19"},{"key":"ref20","doi-asserted-by":"publisher","DOI":"10.1007\/s11432-024-4231-5"},{"journal-title":"Qwen-vl: A frontier large vision-language model with versatile abilities","year":"2023","author":"Bai","key":"ref21"},{"journal-title":"Qwen2-audio technical report","year":"2024","author":"Chu","key":"ref22"},{"key":"ref23","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v38i21.30570"},{"journal-title":"LIama-omni: Seamless speech interaction with large language models","year":"2024","author":"Fang","key":"ref24"},{"key":"ref25","article-title":"Llm4drive: A survey of large language models for autonomous driving","volume-title":"NeurIPS 2024 Workshop on Open- World Agents","author":"Yang","year":"2023"},{"key":"ref26","doi-asserted-by":"publisher","DOI":"10.1109\/WACVW60836.2024.00106"},{"journal-title":"Mme-realworld: Could your multimodal Ilm challenge high-resolution real-world scenarios that are difficult for humans?","year":"2024","author":"Zhang","key":"ref27"},{"key":"ref28","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2024.acl-long.360"},{"journal-title":"Baichuan 2: Open large-scale language models","year":"2023","author":"Yang","key":"ref29"},{"key":"ref30","first-page":"79155","article-title":"The refinedweb dataset for falcon Ilm: Outperforming curated corpora with web data only","volume":"36","author":"Penedo","year":"2023","journal-title":"Advances in Neural Information Processing Systems"},{"issue":"1","key":"ref31","first-page":"1","volume":"9","author":"Yang","year":"2023","journal-title":"The dawn of Imms: Preliminary explorations with gpt-4v (ision)"},{"key":"ref32","volume":"1","author":"Anil","year":"2023","journal-title":"Gemini: A family of highly capable multimodal models"},{"journal-title":"Gemini 1.5: Unlocking multimodal understanding across millions of tokens of context","year":"2024","author":"Reid","key":"ref33"},{"key":"ref34","first-page":"8748","article-title":"Learning transferable visual models from natural language supervision","volume-title":"International Conference on Machine Learning","author":"Radford","year":"2021"},{"key":"ref35","first-page":"19730","article-title":"Blip-2: Bootstrapping language-image pre-training with frozen image encoders and large language models","volume-title":"International Conference on Machine Learning","author":"Li","year":"2023"},{"key":"ref36","first-page":"49250","article-title":"Instructblip: Towards general-purpose vision-language models with instruction tuning","volume":"36","author":"Dai","year":"2023","journal-title":"Advances in Neural Information Processing Systems"},{"journal-title":"Minigpt-4: Enhancing vision-language understanding with advanced large language models","year":"2023","author":"Zhu","key":"ref37"},{"key":"ref38","first-page":"61501","article-title":"Visionllm: Large language model is also an open-ended decoder for vision-centric tasks","volume":"36","author":"Wang","year":"2023","journal-title":"Advances in Neural Information Processing Systems"},{"key":"ref39","first-page":"121475","article-title":"Cogvlm: Visual expert for pretrained language models","volume":"37","author":"Wang","year":"2025","journal-title":"Advances in Neural Information Processing Systems"},{"journal-title":"xgen-mm (blip-): A family of open large multimodal models","year":"2024","author":"Xue","key":"ref40"},{"journal-title":"Ferret: Refer and ground anything anywhere at any granularity","year":"2023","author":"You","key":"ref41"},{"journal-title":"Ferret-v2: An improved baseline for referring and grounding with large language models","year":"2024","author":"Zhang","key":"ref42"},{"key":"ref43","doi-asserted-by":"publisher","DOI":"10.1109\/ICRA57147.2024.10611018"},{"key":"ref44","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v38i5.28253"},{"key":"ref45","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D19-1215"},{"key":"ref46","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v39i8.32902"},{"key":"ref47","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR42600.2020.01164"},{"key":"ref48","doi-asserted-by":"publisher","DOI":"10.1109\/LRA.2024.3440097"},{"key":"ref49","doi-asserted-by":"publisher","DOI":"10.1109\/ICRA57147.2024.10611485"},{"journal-title":"Drivemlm: Aligning multi-modal large language models with behavioral planning states for autonomous driving","year":"2023","author":"Wang","key":"ref50"},{"key":"ref51","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-72943-0_15"},{"key":"ref52","article-title":"Automated evaluation of large vision-language models on self-driving corner cases","volume-title":"Proceedings of the IEEE\/CVF Winter Conference on Applications of Computer Vision","author":"Li","year":"2025"},{"key":"ref53","article-title":"DriveVLM: The convergence of autonomous driving and large vision-language models","volume-title":"Proceedings of the 8th Annual Conference on Robot Learning","author":"Tian","year":"2024"},{"key":"ref54","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2023.findings-emnlp.187"},{"key":"ref55","doi-asserted-by":"publisher","DOI":"10.1109\/TMM.2022.3160060"},{"key":"ref56","doi-asserted-by":"publisher","DOI":"10.1109\/TPAMI.2016.2577031"},{"key":"ref57","doi-asserted-by":"publisher","DOI":"10.1007\/s11263-016-0981-7"},{"key":"ref58","first-page":"770","article-title":"Deep residual learning for imagerecognition","volume-title":"Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition","author":"He","year":"2016"},{"key":"ref59","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2017.322"},{"key":"ref60","first-page":"25278","article-title":"Laion-5b: An open large-scale dataset for training next generation image-text models","volume":"35","author":"Schuhmann","year":"2022","journal-title":"Advances in Neural Information Processing Systems"},{"key":"ref61","article-title":"Coyo-700M: Image-Text Pair Dataset","author":"Byeon","year":"2022","journal-title":"Tech. Rep."},{"key":"ref62","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2019.00852"},{"key":"ref63","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2016.9"},{"journal-title":"The all-seeing project: Towards panoptic visual recognition and understanding of the open world","year":"2023","author":"Wang","key":"ref64"},{"key":"ref65","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-72643-9_22"},{"journal-title":"Allava: Harnessing gpt4v-synthesized data for a lite vision-language model","year":"2024","author":"Chen","key":"ref66"},{"journal-title":"Svit: Scaling up visual instruction tuning","year":"2023","author":"Zhao","key":"ref67"},{"key":"ref68","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2019.00851"},{"key":"ref69","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2017.670"},{"key":"ref70","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2019.00686"},{"key":"ref71","article-title":"Decoupled weight decay regularization","volume-title":"Proceedings of the International Conference on Learning Representations","author":"Loshchilov","year":"2019"},{"key":"ref72","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-72658-3_13"},{"key":"ref73","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52733.2024.01263"},{"key":"ref74","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-46475-6_5"},{"key":"ref75","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2019.00550"}],"event":{"name":"2025 IEEE International Conference on Robotics and Automation (ICRA)","start":{"date-parts":[[2025,5,19]]},"location":"Atlanta, GA, USA","end":{"date-parts":[[2025,5,23]]}},"container-title":["2025 IEEE International Conference on Robotics and Automation (ICRA)"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx8\/11127273\/11127223\/11128326.pdf?arnumber=11128326","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,9,3]],"date-time":"2025-09-03T06:11:50Z","timestamp":1756879910000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/11128326\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,5,19]]},"references-count":75,"URL":"https:\/\/doi.org\/10.1109\/icra55743.2025.11128326","relation":{},"subject":[],"published":{"date-parts":[[2025,5,19]]}}}