{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,5,1]],"date-time":"2026-05-01T17:03:36Z","timestamp":1777655016874,"version":"3.51.4"},"publisher-location":"New York, NY, USA","reference-count":44,"publisher":"ACM","license":[{"start":{"date-parts":[[2024,10,28]],"date-time":"2024-10-28T00:00:00Z","timestamp":1730073600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2024,10,28]]},"DOI":"10.1145\/3664647.3689171","type":"proceedings-article","created":{"date-parts":[[2024,10,26]],"date-time":"2024-10-26T06:59:41Z","timestamp":1729925981000},"page":"11289-11291","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":13,"title":["From Multimodal LLM to Human-level AI: Modality, Instruction, Reasoning and Beyond"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-3026-6347","authenticated-orcid":false,"given":"Hao","family":"Fei","sequence":"first","affiliation":[{"name":"National University of Singapore, Singapore, Singapore"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-0550-8247","authenticated-orcid":false,"given":"Xiangtai","family":"Li","sequence":"additional","affiliation":[{"name":"Nanyang Technological University, Singapore, Singapore"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-7939-9242","authenticated-orcid":false,"given":"Haotian","family":"Liu","sequence":"additional","affiliation":[{"name":"University of Wisconsin-Madison, Wisconsin, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0009-0009-4662-2810","authenticated-orcid":false,"given":"Fuxiao","family":"Liu","sequence":"additional","affiliation":[{"name":"University of Maryland, College Park, College Park, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-4183-3645","authenticated-orcid":false,"given":"Zhuosheng","family":"Zhang","sequence":"additional","affiliation":[{"name":"Shanghai Jiao Tong University, Shanghai, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-7374-8739","authenticated-orcid":false,"given":"Hanwang","family":"Zhang","sequence":"additional","affiliation":[{"name":"Nanyang Technological University, Singapore, Singapore"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-8906-3777","authenticated-orcid":false,"given":"Shuicheng","family":"Yan","sequence":"additional","affiliation":[{"name":"Skywork AI, Singapore, Singapore"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2024,10,28]]},"reference":[{"key":"e_1_3_2_1_1_1","volume-title":"Proceedings of the NeurIPS.","author":"Alayrac Jean-Baptiste","year":"2022","unstructured":"Jean-Baptiste Alayrac, Jeff Donahue, Pauline Luc, Antoine Miech, and Kar\u00e9n Simonyan. 2022. Flamingo: a Visual Language Model for Few-Shot Learning. In Proceedings of the NeurIPS."},{"key":"e_1_3_2_1_2_1","first-page":"23716","article-title":"Flamingo: a visual language model for few-shot learning","volume":"35","author":"Alayrac Jean-Baptiste","year":"2022","unstructured":"Jean-Baptiste Alayrac, Jeff Donahue, Pauline Luc, Antoine Miech, Iain Barr, Yana Hasson, Karel Lenc, Arthur Mensch, Katherine Millican, Malcolm Reynolds, et al. 2022. Flamingo: a visual language model for few-shot learning. Advances in Neural Information Processing Systems, Vol. 35 (2022), 23716--23736.","journal-title":"Advances in Neural Information Processing Systems"},{"key":"e_1_3_2_1_3_1","volume-title":"Proceedings of the 2019 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies","volume":"1","author":"Devlin Jacob","year":"2019","unstructured":"Jacob Devlin, Ming-Wei Chang, Kenton Lee, and Kristina Toutanova. 2019. BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding. In Proceedings of the 2019 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, Volume 1 (Long and Short Papers). 4171--4186."},{"key":"e_1_3_2_1_4_1","volume-title":"Corey Lynch, Aakanksha Chowdhery, Brian Ichter, Ayzaan Wahid, Jonathan Tompson, Quan Vuong, Tianhe Yu, et al.","author":"Driess Danny","year":"2023","unstructured":"Danny Driess, Fei Xia, Mehdi SM Sajjadi, Corey Lynch, Aakanksha Chowdhery, Brian Ichter, Ayzaan Wahid, Jonathan Tompson, Quan Vuong, Tianhe Yu, et al. 2023. Palm-e: An embodied multimodal language model. arXiv preprint arXiv:2303.03378 (2023)."},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2023.acl-short.101"},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52733.2024.00730"},{"key":"e_1_3_2_1_7_1","volume-title":"Proceedings of the International Conference on Machine Learning, ICML. 6373--6391","author":"Fei Hao","year":"2024","unstructured":"Hao Fei, Shengqiong Wu, Wei Ji, Hanwang Zhang, Meishan Zhang, Mong Li Lee, and Wynne Hsu. 2024. Video-of-Thought: Step-by-Step Video Reasoning from Perception to Cognition. In Proceedings of the International Conference on Machine Learning, ICML. 6373--6391."},{"key":"e_1_3_2_1_8_1","volume-title":"international conference on machine learning. PMLR, 6373--6391","author":"Fei Hao","year":"2022","unstructured":"Hao Fei, Shengqiong Wu, Yafeng Ren, and Meishan Zhang. 2022. Matching structure for dual learning. In international conference on machine learning. PMLR, 6373--6391."},{"key":"e_1_3_2_1_9_1","volume-title":"VITRON: A Unified Pixel-level Vision LLM for Understanding, Generating, Segmenting, Editing. CoRR","author":"Fei Hao","year":"2024","unstructured":"Hao Fei, Shengqiong Wu, Hanwang Zhang, Tat-Seng Chua, and Shuicheng Yan. 2024. VITRON: A Unified Pixel-level Vision LLM for Understanding, Generating, Segmenting, Editing. CoRR (2024)."},{"key":"e_1_3_2_1_10_1","volume-title":"2024 d. Enhancing Video-Language Representations with Structural Spatio-Temporal Alignment","author":"Fei Hao","year":"2024","unstructured":"Hao Fei, Shengqiong Wu, Meishan Zhang, Min Zhang, Tat-Seng Chua, and Shuicheng Yan. 2024 d. Enhancing Video-Language Representations with Structural Spatio-Temporal Alignment. IEEE Transactions on Pattern Analysis and Machine Intelligence (2024)."},{"key":"e_1_3_2_1_11_1","unstructured":"Jinyi Hu Yuan Yao Chongyi Wang Shan Wang Yinxu Pan Qianyu Chen Tianyu Yu Hanghao Wu Yue Zhao Haoye Zhang et al. 2023. Large Multilingual Models Pivot Zero-Shot Multimodal Learning across Languages. arXiv preprint arXiv:2308.12038 (2023)."},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2017.93"},{"key":"e_1_3_2_1_13_1","volume-title":"Proceedings of the ICML. 19730--19742","author":"Li Junnan","unstructured":"Junnan Li, Dongxu Li, Silvio Savarese, and Steven C. H. Hoi. 2023. BLIP-2: Bootstrapping Language-Image Pre-training with Frozen Image Encoders and Large Language Models. In Proceedings of the ICML. 19730--19742."},{"key":"e_1_3_2_1_14_1","unstructured":"Xiangtai Li Haobo Yuan Wei Li Henghui Ding Size Wu Wenwei Zhang Yining Li Kai Chen and Chen Change Loy. 2024. OMG-Seg: Is One Model Good Enough For All Segmentation?. In CVPR."},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52729.2023.02156"},{"key":"e_1_3_2_1_16_1","volume-title":"LLaVA-1.5, and Other Multi-modality Models. arXiv preprint arXiv:2310.14566","author":"Liu Fuxiao","year":"2023","unstructured":"Fuxiao Liu, Tianrui Guan, Zongxia Li, Lichang Chen, Yaser Yacoob, Dinesh Manocha, and Tianyi Zhou. 2023. HallusionBench: You See What You Think? Or You Think What You See? An Image-Context Reasoning Benchmark Challenging for GPT-4V (ision), LLaVA-1.5, and Other Multi-modality Models. arXiv preprint arXiv:2310.14566 (2023)."},{"key":"e_1_3_2_1_17_1","volume-title":"2023 d. Aligning Large Multi-Modal Model with Robust Instruction Tuning. arXiv preprint arXiv:2306.14565","author":"Liu Fuxiao","year":"2023","unstructured":"Fuxiao Liu, Kevin Lin, Linjie Li, Jianfeng Wang, Yaser Yacoob, and Lijuan Wang. 2023 d. Aligning Large Multi-Modal Model with Robust Instruction Tuning. arXiv preprint arXiv:2306.14565 (2023)."},{"key":"e_1_3_2_1_18_1","unstructured":"Fuxiao Liu Kevin Lin Linjie Li Jianfeng Wang Yaser Yacoob and Lijuan Wang. 2023 e. Mitigating Hallucination in Large Multi-Modal Models via Robust Instruction Tuning. arxiv: cs.CV\/2306.14565"},{"key":"e_1_3_2_1_19_1","volume-title":"2023 f. MMC: Advancing Multimodal Chart Understanding with Large-scale Instruction Tuning. arXiv preprint arXiv:2311.10774","author":"Liu Fuxiao","year":"2023","unstructured":"Fuxiao Liu, Xiaoyang Wang, Wenlin Yao, Jianshu Chen, Kaiqiang Song, Sangwoo Cho, Yaser Yacoob, and Dong Yu. 2023 f. MMC: Advancing Multimodal Chart Understanding with Large-scale Instruction Tuning. arXiv preprint arXiv:2311.10774 (2023)."},{"key":"e_1_3_2_1_20_1","volume-title":"Improved baselines with visual instruction tuning. arXiv preprint arXiv:2310.03744","author":"Liu Haotian","year":"2023","unstructured":"Haotian Liu, Chunyuan Li, Yuheng Li, and Yong Jae Lee. 2023. Improved baselines with visual instruction tuning. arXiv preprint arXiv:2310.03744 (2023)."},{"key":"e_1_3_2_1_21_1","volume-title":"Visual Instruction Tuning. CoRR","author":"Liu Haotian","year":"2023","unstructured":"Haotian Liu, Chunyuan Li, Qingyang Wu, and Yong Jae Lee. 2023. Visual Instruction Tuning. CoRR, Vol. abs\/2304.08485 (2023)."},{"key":"e_1_3_2_1_22_1","first-page":"2507","article-title":"Learn to explain: Multimodal reasoning via thought chains for science question answering","volume":"35","author":"Lu Pan","year":"2022","unstructured":"Pan Lu, Swaroop Mishra, Tanglin Xia, Liang Qiu, Kai-Wei Chang, Song-Chun Zhu, Oyvind Tafjord, Peter Clark, and Ashwin Kalyan. 2022. Learn to explain: Multimodal reasoning via thought chains for science question answering. Advances in Neural Information Processing Systems, Vol. 35 (2022), 2507--2521.","journal-title":"Advances in Neural Information Processing Systems"},{"key":"e_1_3_2_1_23_1","volume-title":"Song-Chun Zhu, and Jianfeng Gao.","author":"Lu Pan","year":"2023","unstructured":"Pan Lu, Baolin Peng, Hao Cheng, Michel Galley, Kai-Wei Chang, Ying Nian Wu, Song-Chun Zhu, and Jianfeng Gao. 2023. Chameleon: Plug-and-play compositional reasoning with large language models. arXiv preprint arXiv:2304.09842 (2023)."},{"key":"e_1_3_2_1_24_1","unstructured":"OpenAI. 2022. Gpt-4 technical report. (2022)."},{"key":"e_1_3_2_1_25_1","unstructured":"OpenAI. 2022. Introducing chatgpt. (2022)."},{"key":"e_1_3_2_1_26_1","volume-title":"International conference on machine learning. PMLR, 8748--8763","author":"Radford Alec","year":"2021","unstructured":"Alec Radford, Jong Wook Kim, Chris Hallacy, Aditya Ramesh, Gabriel Goh, Sandhini Agarwal, Girish Sastry, Amanda Askell, Pamela Mishkin, Jack Clark, et al. 2021. Learning transferable visual models from natural language supervision. In International conference on machine learning. PMLR, 8748--8763."},{"key":"e_1_3_2_1_27_1","volume-title":"PandaGPT: One Model To Instruction-Follow Them All. CoRR","author":"Su Yixuan","year":"2023","unstructured":"Yixuan Su, Tian Lan, Huayang Li, Jialu Xu, Yan Wang, and Deng Cai. 2023. PandaGPT: One Model To Instruction-Follow Them All. CoRR, Vol. abs\/2305.16355 (2023)."},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2023.acl-long.147"},{"key":"e_1_3_2_1_29_1","first-page":"24824","article-title":"Chain-of-thought prompting elicits reasoning in large language models","volume":"35","author":"Wei Jason","year":"2022","unstructured":"Jason Wei, Xuezhi Wang, Dale Schuurmans, Maarten Bosma, Fei Xia, Ed Chi, Quoc V Le, Denny Zhou, et al. 2022. Chain-of-thought prompting elicits reasoning in large language models. Advances in Neural Information Processing Systems, Vol. 35 (2022), 24824--24837.","journal-title":"Advances in Neural Information Processing Systems"},{"key":"e_1_3_2_1_30_1","volume-title":"Towards Semantic Equivalence of Tokenization in Multimodal LLM. arXiv preprint arXiv:2406.05127","author":"Wu Shengqiong","year":"2024","unstructured":"Shengqiong Wu, Hao Fei, Xiangtai Li, Jiayi Ji, Hanwang Zhang, Tat-Seng Chua, and Shuicheng Yan. 2024. Towards Semantic Equivalence of Tokenization in Multimodal LLM. arXiv preprint arXiv:2406.05127 (2024)."},{"key":"e_1_3_2_1_31_1","unstructured":"Shengqiong Wu Hao Fei Leigang Qu Wei Ji and Tat-Seng Chua. 2024. NExT-GPT: Any-to-Any Multimodal LLM. (2024)."},{"key":"e_1_3_2_1_32_1","volume-title":"Faithful Logical Reasoning via Symbolic Chain-of-Thought. arXiv preprint arXiv:2405.18357","author":"Xu Jundong","year":"2024","unstructured":"Jundong Xu, Hao Fei, Liangming Pan, Qian Liu, Mong-Li Lee, and Wynne Hsu. 2024. Faithful Logical Reasoning via Symbolic Chain-of-Thought. arXiv preprint arXiv:2405.18357 (2024)."},{"key":"e_1_3_2_1_33_1","volume-title":"Mm-react: Prompting chatgpt for multimodal reasoning and action. arXiv preprint arXiv:2303.11381","author":"Yang Zhengyuan","year":"2023","unstructured":"Zhengyuan Yang, Linjie Li, Jianfeng Wang, Kevin Lin, Ehsan Azarnasab, Faisal Ahmed, Zicheng Liu, Ce Liu, Michael Zeng, and Lijuan Wang. 2023. Mm-react: Prompting chatgpt for multimodal reasoning and action. arXiv preprint arXiv:2303.11381 (2023)."},{"key":"e_1_3_2_1_34_1","volume-title":"LAMM: Language-Assisted Multi-Modal Instruction-Tuning Dataset, Framework, and Benchmark. CoRR","author":"Yin Zhenfei","year":"2023","unstructured":"Zhenfei Yin, Jiong Wang, Jianjian Cao, Zhelun Shi, Dingning Liu, Mukai Li, Lu Sheng, Lei Bai, Xiaoshui Huang, Zhiyong Wang, Jing Shao, and Wanli Ouyang. 2023. LAMM: Language-Assisted Multi-Modal Instruction-Tuning Dataset, Framework, and Benchmark. CoRR, Vol. abs\/2306.06687 (2023)."},{"key":"e_1_3_2_1_35_1","volume-title":"Proceedings of the NeurIPS","author":"Zhang Ao","year":"2023","unstructured":"Ao Zhang, Hao Fei, Yuan Yao, Wei Ji, Li Li, Zhiyuan Liu, and Tat-Seng Chua. 2023. Transfer visual prompt generator across llms. Proceedings of the NeurIPS (2023)."},{"key":"e_1_3_2_1_36_1","volume-title":"2023 d. SpeechGPT: Empowering Large Language Models with Intrinsic Cross-Modal Conversational Abilities. CoRR","author":"Zhang Dong","year":"2023","unstructured":"Dong Zhang, Shimin Li, Xin Zhang, Jun Zhan, Pengyu Wang, Yaqian Zhou, and Xipeng Qiu. 2023 d. SpeechGPT: Empowering Large Language Models with Intrinsic Cross-Modal Conversational Abilities. CoRR, Vol. abs\/2305.11000 (2023)."},{"key":"e_1_3_2_1_37_1","volume-title":"Video-LLaMA: An Instruction-tuned Audio-Visual Language Model for Video Understanding. CoRR","author":"Zhang Hang","year":"2023","unstructured":"Hang Zhang, Xin Li, and Lidong Bing. 2023. Video-LLaMA: An Instruction-tuned Audio-Visual Language Model for Video Understanding. CoRR, Vol. abs\/2306.02858 (2023)."},{"key":"e_1_3_2_1_38_1","volume-title":"Chen Change Loy, and Shuicheng Yan","author":"Zhang Tao","year":"2024","unstructured":"Tao Zhang, Xiangtai Li, Hao Fei, Haobo Yuan, Shengqiong Wu, Shunping Ji, Chen Change Loy, and Shuicheng Yan. 2024. Omg-llava: Bridging image-level, object-level, pixel-level reasoning and understanding. arXiv preprint arXiv:2406.19389 (2024)."},{"key":"e_1_3_2_1_39_1","doi-asserted-by":"publisher","DOI":"10.1109\/TPAMI.2023.3234170"},{"key":"e_1_3_2_1_40_1","volume-title":"Findings of ACL","author":"Zhang Zhuosheng","year":"2023","unstructured":"Zhuosheng Zhang and Aston Zhang. 2023. You Only Look at Screens: Multimodal Chain-of-Action Agents. In Findings of ACL 2024."},{"key":"e_1_3_2_1_41_1","volume-title":"Automatic Chain of Thought Prompting in Large Language Models. In The Eleventh International Conference on Learning Representations (ICLR).","author":"Zhang Zhuosheng","year":"2023","unstructured":"Zhuosheng Zhang, Aston Zhang, Mu Li, and Alex Smola. 2023 e. Automatic Chain of Thought Prompting in Large Language Models. In The Eleventh International Conference on Learning Representations (ICLR)."},{"key":"e_1_3_2_1_42_1","volume-title":"Multimodal chain-of-thought reasoning in language models. Transactions on Machine Learning Research","author":"Zhang Zhuosheng","year":"2024","unstructured":"Zhuosheng Zhang, Aston Zhang, Mu Li, Hai Zhao, George Karypis, and Alex Smola. 2024. Multimodal chain-of-thought reasoning in language models. Transactions on Machine Learning Research (2024)."},{"key":"e_1_3_2_1_43_1","volume-title":"MMICL: Empowering Vision-language Model with Multi-Modal In-Context Learning. arXiv preprint arXiv:2309.07915","author":"Zhao Haozhe","year":"2023","unstructured":"Haozhe Zhao, Zefan Cai, Shuzheng Si, Xiaojian Ma, Kaikai An, Liang Chen, Zixuan Liu, Sheng Wang, Wenjuan Han, and Baobao Chang. 2023. MMICL: Empowering Vision-language Model with Multi-Modal In-Context Learning. arXiv preprint arXiv:2309.07915 (2023)."},{"key":"e_1_3_2_1_44_1","volume-title":"MiniGPT-4: Enhancing Vision-Language Understanding with Advanced Large Language Models. CoRR","author":"Zhu Deyao","year":"2023","unstructured":"Deyao Zhu, Jun Chen, Xiaoqian Shen, Xiang Li, and Mohamed Elhoseiny. 2023. MiniGPT-4: Enhancing Vision-Language Understanding with Advanced Large Language Models. CoRR, Vol. abs\/2304.10592 (2023)."}],"event":{"name":"MM '24: The 32nd ACM International Conference on Multimedia","location":"Melbourne VIC Australia","acronym":"MM '24","sponsor":["SIGMM ACM Special Interest Group on Multimedia"]},"container-title":["Proceedings of the 32nd ACM International Conference on Multimedia"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3664647.3689171","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3664647.3689171","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,19]],"date-time":"2025-06-19T01:17:29Z","timestamp":1750295849000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3664647.3689171"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,10,28]]},"references-count":44,"alternative-id":["10.1145\/3664647.3689171","10.1145\/3664647"],"URL":"https:\/\/doi.org\/10.1145\/3664647.3689171","relation":{},"subject":[],"published":{"date-parts":[[2024,10,28]]},"assertion":[{"value":"2024-10-28","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}