{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,28]],"date-time":"2026-10-28T03:18:08Z","timestamp":1793157488459,"version":"4.5.0"},"reference-count":325,"publisher":"Tech Science Press","issue":"2","content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["CMC"],"published-print":{"date-parts":[[2024]]},"DOI":"10.32604\/cmc.2024.052618","type":"journal-article","created":{"date-parts":[[2024,8,15]],"date-time":"2024-08-15T09:00:14Z","timestamp":1723712414000},"page":"1753-1808","source":"Crossref","is-referenced-by-count":96,"title":["Evolution and Prospects of Foundation Models: From Large Language Models to Large Multimodal Models"],"prefix":"10.32604","volume":"80","author":[{"given":"Zheyi","family":"Chen","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Liuchang","family":"Xu","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Hongting","family":"Zheng","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Luyao","family":"Chen","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Amr","family":"Tolba","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Liang","family":"Zhao","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Keping","family":"Yu","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Hailin","family":"Feng","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"17807","published-online":{"date-parts":[[2024]]},"reference":[{"key":"ref1","article-title":"A comprehensive overview of large language models","author":"Naveed","year":"2023","journal-title":"arXiv preprint arXiv:2307.06435"},{"key":"ref2","first-page":"34","volume":"1","author":"Pinker","year":"2003","journal-title":"The Language Instinct: How the Mind Creates Language"},{"key":"ref3","doi-asserted-by":"crossref","first-page":"1569","DOI":"10.1126\/science.298.5598.1569","article-title":"The faculty of language: What is it, who has it, and how did it evolve?","volume":"298","author":"Hauser","year":"2002","journal-title":"Science"},{"key":"ref4","first-page":"437","article-title":"Learning machines","author":"Turing","year":"2009","journal-title":"Computing Machinery and Intelligence"},{"key":"ref5","article-title":"A survey of large language models","author":"Zhao","year":"2023","journal-title":"arXiv preprint arXiv:2303.18223"},{"key":"ref6","first-page":"297","author":"Jelinek","year":"1998","journal-title":"Statistical Methods for Speech Recognition"},{"key":"ref7","doi-asserted-by":"crossref","first-page":"1270","DOI":"10.1109\/5.880083","article-title":"Two decades of statistical language modeling: Where do we go from here?","volume":"88","author":"Rosenfeld","year":"2000","journal-title":"Proc. IEEE"},{"key":"ref8","first-page":"1","article-title":"Introduction to the special issue on word sense disambiguation: The state of the art","volume":"24","author":"Ide","year":"1998","journal-title":"Comput. Linguist."},{"key":"ref9","series-title":"Proc. 7th Int. Conf. Spoken Lang. Process. (ICSLP 2002)","first-page":"901","article-title":"SRILM\u2013an extensible language modeling toolkit","author":"Stolcke","year":"USA, 2002"},{"key":"ref10","series-title":"Proc. 37th Annu. Meet. Assoc. Comput. Linguist.","first-page":"175","article-title":"A second-order hidden Markov model for part-of-speech tagging","author":"Thede","year":"1999"},{"key":"ref11","doi-asserted-by":"crossref","first-page":"137","DOI":"10.1561\/1500000008","article-title":"Statistical language models for information retrieval a critical review","volume":"2","author":"Zhai","year":"2008","journal-title":"Found. Trends\u00ae Inf. Retr."},{"key":"ref12","article-title":"BERT: Pretraining of deep bidirectional transformers for language understanding","author":"Devlin","year":"2018","journal-title":"arXiv preprint arXiv:1810.04805"},{"key":"ref13","first-page":"5485","article-title":"Exploring the limits of transfer learning with a unified text-to-text transformer","volume":"21","author":"Raffel","year":"2020","journal-title":"J. Mach. Learn. Res."},{"key":"ref14","first-page":"7","article-title":"What do we mean by GenAI? a systematic mapping of the evolution, trends, and techniques involved in generative AI","volume":"8","author":"Garc\u00eda-Pe\u00f1alvo","year":"2023","journal-title":"Int. J. Interact. Multimed. Artif. Intell."},{"key":"ref15","first-page":"23716","volume":"35","author":"Alayrac","year":"2022","journal-title":"Advances in Neural Information Processing Systems"},{"key":"ref16","article-title":"Visual instruction tuning","author":"Liu","year":"2023","journal-title":"arXiv preprint arXiv:2304.08485"},{"key":"ref17","article-title":"VideoChat: Chat-centric video understanding","author":"Li","year":"2023","journal-title":"arXiv preprint arXiv:2305.06355"},{"key":"ref18","article-title":"Video- LLaMA: An instruction-tuned audio-visual language model for video understanding","author":"Zhang","year":"2023","journal-title":"arXiv preprint arXiv:2306.02858"},{"key":"ref19","article-title":"WavCaps: A ChatGPT-assisted weaklylabelled audio captioning dataset for audio-language multimodal research","author":"Mei","year":"2023","journal-title":"arXiv preprint arXiv:2303.17395"},{"key":"ref20","article-title":"LLM: Multi-modal language modeling with image, audio, video, and text integration","author":"Lyu","year":"2023","journal-title":"arXiv preprint arXiv:2306.09093"},{"key":"ref21","article-title":"Language is not all you need: Aligning perception with language models","author":"Huang","year":"2023","journal-title":"arXiv preprint arXiv:2302.14045"},{"key":"ref22","article-title":"A comprehensive survey of AI-generated content (AIGC): A history of generative ai from GAN to ChatGPT","author":"Cao","year":"2023","journal-title":"arXiv preprint arXiv:2303.04226"},{"key":"ref23","article-title":"PaLM-E: An embodied multimodal language model","author":"Driess","year":"2023","journal-title":"arXiv preprint arXiv:2303.03378"},{"key":"ref24","article-title":"Visual ChatGPT: Talking, drawing and editing with visual foundation models","author":"Wu","year":"2023","journal-title":"arXiv preprint arXiv:2303.04671"},{"key":"ref25","article-title":"GPT-4 technical report","author":"Achiam","year":"2023","journal-title":"arXiv preprint arXiv:2303.08774"},{"key":"ref26","article-title":"Aligning large language models with human: A survey","author":"Wang","year":"2023","journal-title":"arXiv preprint arXiv:2307.12966"},{"key":"ref27","first-page":"1877","article-title":"Language models are few-shot learners","volume":"33","author":"Brown","year":"2020","journal-title":"Adv. Neural Inf. Process Syst."},{"key":"ref28","first-page":"1","article-title":"PaLM: Scaling language modeling with pathways","volume":"24","author":"Chowdhery","year":"2023","journal-title":"J. Mach. Learn. Res."},{"key":"ref29","article-title":"Galactica: A large language model for science","author":"Taylor","year":"2022","journal-title":"arXiv preprint arXiv:2211.09085"},{"key":"ref30","article-title":"LLaMA: Open and efficient foundation language models","author":"Touvron","year":"2023","journal-title":"arXiv preprint arXiv:2302.13971"},{"key":"ref31","first-page":"24824","article-title":"Chain-of-thought prompting elicits reasoning in large language models","volume":"35","author":"Wei","year":"2022","journal-title":"Adv. Neural Inf. Process. Syst."},{"key":"ref32","first-page":"5998","article-title":"Attention is all you need","volume":"30","author":"Vaswani","year":"2017","journal-title":"Adv. Neural Inf. Process Syst."},{"key":"ref33","article-title":"Generating long sequences with sparse transformers","author":"Child","year":"2019","journal-title":"arXiv preprint arXiv:1904.10509"},{"key":"ref34","article-title":"Fast transformer decoding: One write-head is all you need","author":"Shazeer","year":"2019","journal-title":"arXiv preprint arXiv:1911.02150"},{"key":"ref35","first-page":"16344","article-title":"FlashAttention: Fast and memory-efficient exact attention with IO-awareness","volume":"35","author":"Dao","year":"2022","journal-title":"Adv. Neural Inf. Process. Syst."},{"key":"ref36","first-page":"8026","article-title":"PyTorch: An imperative style, high-performance deep learning library","volume":"32","author":"Paszke","year":"2019","journal-title":"Adv. Neural Inf. Process. Syst."},{"key":"ref37","series-title":"Proc. 26th ACM SIGKDD Int. Conf. Knowl. Discov. Data Min.","first-page":"3505","article-title":"DeepSpeed: System optimizations enable training deep learning models with over 100 billion parameters","author":"Rasley","year":"2020"},{"key":"ref38","article-title":"Megatron-LM: Training multi-billion parameter language models using model parallelism","author":"Shoeybi","year":"2019","journal-title":"arXiv preprint arXiv:1909.08053"},{"key":"ref39","article-title":"FlAshattention-2: Faster attention with better parallelism and work partitioning","author":"Dao","year":"2023","journal-title":"arXiv preprint arXiv:2307.08691"},{"key":"ref40","series-title":"Proc. 29th Symp. Oper. Syst. Princ.","first-page":"611","article-title":"Efficient memory management for large language model serving with pagedattention","author":"Kwon","year":"2023"},{"key":"ref41","doi-asserted-by":"crossref","first-page":"5412","DOI":"10.1109\/TNNLS.2020.2967597","article-title":"Cross-modal attention with semantic consistence for image\u2013 text matching","volume":"31","author":"Xu","year":"2020","journal-title":"IEEE Trans. Neural Netw. Learn. Syst."},{"key":"ref42","doi-asserted-by":"crossref","first-page":"123587","DOI":"10.1016\/j.eswa.2024.123587","article-title":"A novel graph-attention based multimodal fusion network for joint classification of hyperspectral image and lidar data","volume":"249","author":"Cai","year":"2024","journal-title":"Expert. Syst. Appl."},{"key":"ref43","article-title":"CLIP-Nav: Using CLIP for zero-shot vision-andlanguage navigation","author":"Dorbala","year":"2022","journal-title":"arXiv preprint arXiv:2211.16649"},{"key":"ref44","article-title":"RoboCLIP: One demonstration is enough to learn robot policies","author":"Sontakke","year":"2023","journal-title":"arXiv preprint arXiv:2310.07899"},{"key":"ref45","series-title":"Proc. IEEE\/CVF Conf. Comput. Vis. Pattern Recognit.","first-page":"26296","article-title":"Improved baselines with visual instruction tuning","author":"Liu","year":"2024"},{"key":"ref46","article-title":"ERNIE 3.0: Large-scale knowledge enhanced pre-training for language understanding and generation","author":"Sun","year":"2021","journal-title":"arXiv preprint arXiv:2107.02137"},{"key":"ref47","article-title":"PanGu-\u03b1: Large-scale autoregressive pretrained chinese language models with auto-parallel computation","author":"Zeng","year":"2021","journal-title":"arXiv preprint arXiv:2104.12369"},{"key":"ref48","doi-asserted-by":"crossref","first-page":"216","DOI":"10.1016\/j.aiopen.2021.12.003","article-title":"CPM-2: Large-scale cost-effective pre-trained language models","volume":"2","author":"Zhang","year":"2021","journal-title":"AI Open"},{"key":"ref49","article-title":"ERNIE 3.0 Titan: Exploring large-scale knowledge enhanced pre-training for language understanding and generation","author":"Wang","year":"2021","journal-title":"arXiv preprint arXiv:2112.12731"},{"key":"ref50","article-title":"GPT-NeoX-20B: An open-source autoregressive language model","author":"Black","year":"2022","journal-title":"arXiv preprint arXiv:2204.06745"},{"key":"ref51","article-title":"BLOOM: A 176B-parameter open-access multilingual language model","author":"Workshop","year":"2022","journal-title":"arXiv preprint arXiv:2211.05100"},{"key":"ref52","series-title":"Int. Conf. Mach. Learn.","first-page":"5547","article-title":"GLaM: Efficient scaling of language models with mixture-of-experts","author":"Du","year":"2022"},{"key":"ref53","article-title":"LaMDA: Language models for dialog applications","author":"Thoppilan","year":"2022","journal-title":"arXiv preprint arXiv:2201.08239"},{"key":"ref54","series-title":"Int. Conf. Learn. Represent.","article-title":"UL2: Unifying language learning paradigms","author":"Tay","year":"2022"},{"key":"ref55","article-title":"GLM-130B: An open bilingual pre-trained model","author":"Zeng","year":"2022","journal-title":"arXiv preprint arXiv:2210.02414"},{"key":"ref56","first-page":"9","article-title":"Jurassic-1: Technical details and evaluation","volume":"1","author":"Lieber","year":"2021","journal-title":"White Paper, AI21 Labs"},{"key":"ref57","article-title":"What changes can large-scale language models bring? intensive study on HyperCLOVA: Billions scale Korean generative pretrained transformers","author":"Kim","year":"2021","journal-title":"arXiv preprint arXiv:2109.04650"},{"key":"ref58","article-title":"Yuan 1.0: Large-scale pre-trained language model in zero-shot and few-shot learning","author":"Wu","year":"2021","journal-title":"arXiv preprint arXiv:2110.04725"},{"key":"ref59","article-title":"PanGu-\u03a3: Towards trillion parameter language model with sparse heterogeneous computing","author":"Ren","year":"2023","journal-title":"arXiv preprint arXiv:2303.10845"},{"key":"ref60","series-title":"Proc. 32nd ACM Int. Conf. Inf. Knowl. Manag.","first-page":"4435","article-title":"XinYuan 2.0: A large chinese financial chat model with hundreds of billions parameters","author":"Zhang","year":"2023"},{"key":"ref61","article-title":"OPT: Open pre-trained transformer language models","author":"Zhang","year":"2022","journal-title":"arXiv preprint arXiv:2205.01068"},{"key":"ref62","article-title":"Scaling language models: Methods, analysis & insights from training gopher","author":"Rae","year":"2021","journal-title":"arXiv preprint arXiv:2112.11446"},{"key":"ref63","article-title":"Training compute-optimal large language models","author":"Hoffmann","year":"2022","journal-title":"arXiv preprint arXiv:2203.15556"},{"key":"ref64","doi-asserted-by":"crossref","first-page":"249","DOI":"10.1162\/tacl_a_00266","article-title":"CoQA: A conversational question answering challenge","volume":"7","author":"Reddy","year":"2019","journal-title":"Trans. Assoc. Comput. Linguist."},{"key":"ref65","article-title":"Transcending scaling laws with 0.1% extra compute","author":"Tay","year":"2022","journal-title":"arXiv preprint arXiv:2210.11399"},{"key":"ref66","article-title":"AlexaTM 20B: Few-shot learning using a large-scale multilingual seq2seq model","author":"Soltan","year":"2022","journal-title":"arXiv preprint arXiv:2208.01448"},{"key":"ref67","article-title":"BloombergGPT: A large language model for finance","author":"Wu","year":"2023","journal-title":"arXiv preprint arXiv:2303.17564"},{"key":"ref68","first-page":"11934","article-title":"mT5: A massively multilingual pretrained text-to-text transformer","author":"Xue","year":"2010","journal-title":"arXiv preprint arXiv:2010.11934"},{"key":"ref69","doi-asserted-by":"crossref","first-page":"1092","DOI":"10.1126\/science.abq1158","article-title":"Competitionlevel code generation with AlphaCode","volume":"378","author":"Li","year":"2022","journal-title":"Science"},{"key":"ref70","article-title":"CodeGen: An open large language model for code with multi-turn program synthesis","author":"Nijkamp","year":"2022","journal-title":"arXiv preprint arXiv:2203.13474"},{"key":"ref71","article-title":"CodeT5+: Open code large language models for code understanding and generation","author":"Wang","year":"2023","journal-title":"arXiv preprint arXiv:2305.07922"},{"key":"ref72","first-page":"16","article-title":"The ethical implications of DALL-E: Opportunities and challenges","volume":"2023","author":"Zhou","year":"2023","journal-title":"Mesopo. J. Comput. Sci."},{"key":"ref73","first-page":"3","article-title":"Hierarchical text-conditional image generation with clip latents","volume":"1","author":"Ramesh","year":"2022","journal-title":"arXiv preprint arXiv:2204.06125"},{"key":"ref74","first-page":"36479","article-title":"Photorealistic text-to-image diffusion models with deep language understanding","volume":"35","author":"Saharia","year":"2022","journal-title":"Adv. Neural Inf. Process. Syst."},{"key":"ref75","first-page":"5","article-title":"Scaling autoregressive models for content-rich text-to-image generation","volume":"2","author":"Yu","year":"2022","journal-title":"arXiv preprint arXiv:2206.10789"},{"key":"ref76","article-title":"MiniGPT-4: Enhancing vision-language understanding with advanced large language models","author":"Zhu","year":"2023","journal-title":"arXiv preprint arXiv:2304.10592"},{"key":"ref77","series-title":"Int. Conf. Mach. Learn.","first-page":"4904","article-title":"Scaling up visual and visionlanguage representation learning with noisy text supervision","author":"Jia","year":"2021"},{"key":"ref78","series-title":"Int. Conf. Mach. Learn.","first-page":"8748","article-title":"Learning transferable visual models from natural language supervision","author":"Radford","year":"2021"},{"key":"ref79","article-title":"BLIP-2: Bootstrapping language-image pre-training with frozen image encoders and large language models","author":"Li","year":"2023","journal-title":"arXiv preprint arXiv:2301.12597"},{"key":"ref80","article-title":"CogVLM: Visual expert for pretrained language models","author":"Wang","year":"2023","journal-title":"arXiv preprint arXiv:2311.03079"},{"key":"ref81","article-title":"PaLM 2 technical report","author":"Anil","year":"2023","journal-title":"arXiv preprint arXiv:2305.10403"},{"key":"ref82","article-title":"WizardCoder: Empowering code large language models with evol-instruct","author":"Luo","year":"2023","journal-title":"arXiv preprint arXiv:2306.08568"},{"key":"ref83","article-title":"Qwen-VL: A frontier large vision-language model with versatile abilities","author":"Bai","year":"2023","journal-title":"arXiv preprint arXiv:2308.12966"},{"key":"ref84","article-title":"MoE-LLaVA: Mixture of experts for large vision-language models","author":"Lin","year":"2024","journal-title":"arXiv preprint arXiv:2401.15947"},{"key":"ref85","first-page":"3104","article-title":"Sequence to sequence learning with neural networks","volume":"27","author":"Sutskever","year":"2014","journal-title":"Adv. Neural Inf. Process. Syst."},{"key":"ref86","unstructured":"A. Radford and K. Narasimhan, \u201cImproving language understanding by generative pre-training,\u201d 2018."},{"key":"ref87","article-title":"Generating wikipedia by summarizing long sequences","author":"Liu","year":"2018","journal-title":"arXiv preprint arXiv:1801.10198"},{"key":"ref88","article-title":"BART: Denoising sequence-to-sequence pre-training for natural language generation, translation, and comprehension","author":"Lewis","year":"2019","journal-title":"arXiv preprint arXiv:1910.13461"},{"key":"ref89","article-title":"RoBERTa: A robustly optimized BERT pretraining approach","author":"Liu","year":"2019","journal-title":"arXiv preprint arXiv:1907.11692"},{"key":"ref90","article-title":"ALBERT: A lite BERT for self-supervised learning of language representations","author":"Lan","year":"2019","journal-title":"arXiv preprint arXiv:1909.11942"},{"key":"ref91","first-page":"5753","article-title":"XLNet: Generalized autoregressive pretraining for language understanding","volume":"32","author":"Yang","year":"2019","journal-title":"Adv. Neural Inf. Process Syst."},{"key":"ref92","article-title":"Transformer-XL: Attentive language models beyond a fixedlength context","author":"Dai","year":"2019","journal-title":"arXiv preprint arXiv:1901.02860"},{"key":"ref93","article-title":"ELECTRA: Pretraining text encoders as discriminators rather than generators","author":"Clark","year":"2020","journal-title":"arXiv preprint arXiv:2003.10555"},{"key":"ref94","article-title":"MM-LLMs: Recent advances in multimodal large language models","author":"Zhang","year":"2024","journal-title":"arXiv preprint arXiv:2401.13601"},{"key":"ref95","series-title":"Proc. 38th Int. Conf. Mach. Learn.","first-page":"1059","article-title":"High-performance large-scale image recognition without normalization","volume":"139","author":"Brock","year":"Jul. 2021"},{"key":"ref96","article-title":"An image is worth 16x16 words: Transformers for image recognition at scale","author":"Dosovitskiy","year":"2021","journal-title":"arXiv preprint arXiv:2010.11929"},{"key":"ref97","series-title":"Proc. IEEE\/CVF Conf. Comput. Vis. Pattern Recognit.","first-page":"19358","article-title":"Exploring the limits of masked visual representation learning at scale","author":"Fang","year":"2023"},{"key":"ref98","article-title":"A survey on generative ai and LLM for video generation, understanding, and streaming","author":"Zhou","year":"2024","journal-title":"arXiv preprint arXiv:2404.16038"},{"key":"ref99","article-title":"X-LLM: Bootstrapping advanced large language models by treating multi-modalities as foreign languages","author":"Chen","year":"2023","journal-title":"arXiv preprint arXiv:2305.04160"},{"key":"ref100","doi-asserted-by":"crossref","first-page":"3451","DOI":"10.1109\/TASLP.2021.3122291","article-title":"HuBERT: Self-supervised speech representation learning by masked prediction of hidden units","volume":"29","author":"Hsu","year":"2021","journal-title":"IEEE\/ACM Trans. Audio, Speech, Lang. Process."},{"key":"ref101","article-title":"BEATs: Audio pre-training with acoustic tokenizers","author":"Chen","year":"2022","journal-title":"arXiv preprint arXiv:2212.09058"},{"key":"ref102","series-title":"Int. Conf. Mach. Learn.","first-page":"28492","article-title":"Robust speech recognition via large-scale weak supervision","author":"Radford","year":"2023"},{"key":"ref103","article-title":"Scaling instructionfinetuned language models","author":"Chung","year":"2022","journal-title":"arXiv preprint arXiv:2210.11416"},{"key":"ref104","unstructured":"W. -L. Chiang et al., \u201cVicuna: An open-source chatbot impressing GPT-4 with 90%* ChatGPT quality,\u201d vol. 2, no. 3, pp. 6. 2023. Accessed: Aug. 14, 2023. [Online]. Available: https:\/\/lmsys.org\/blog\/2023-03-30-vicuna\/"},{"key":"ref105","first-page":"3609","article-title":"EGSDE: Unpaired image-to-image translation via energy-guided stochastic differential equations","volume":"35","author":"Zhao","year":"2022","journal-title":"Adv. Neural Inf. Process Syst."},{"key":"ref106","series-title":"Proc. IEEE\/CVF Conf. Comput. Vis. Pattern Recognit.","first-page":"10684","article-title":"High-resolution image synthesis with latent diffusion models","author":"Rombach","year":"2022"},{"key":"ref107","series-title":"Proc. IEEE\/CVF Int. Conf. Comput. Vis.","first-page":"15954","article-title":"Text2Video-Zero: Text-to-image diffusion models are zero-shot video generators","author":"Khachatryan","year":"2023"},{"key":"ref108","article-title":"AudioLDM: Text-to-audio generation with latent diffusion models","author":"Liu","year":"2023","journal-title":"arXiv preprint arXiv:2301.12503"},{"key":"ref109","series-title":"Proc. 26th Australas. Comput. Educ. Conf.","first-page":"29","article-title":"More robots are coming: Large multimodal models (ChatGPT) can solve visually diverse images of parsons problems","author":"Hou","year":"2024"},{"key":"ref110","first-page":"27730","article-title":"Training language models to follow instructions with human feedback","volume":"35","author":"Ouyang","year":"2022","journal-title":"Adv. Neural Inf. Process. Syst."},{"key":"ref111","article-title":"OPT-IML: Scaling language model instruction meta learning through the lens of generalization","author":"Iyer","year":"2022","journal-title":"arXiv preprint arXiv:2212.12017"},{"key":"ref112","article-title":"mPLUG-Owl: Modularization empowers large language models with multimodality","author":"Ye","year":"2023","journal-title":"arXiv preprint arXiv:2304.14178"},{"key":"ref113","article-title":"InstructBLIP: Towards general-purpose vision-language models with instruction tuning","author":"Dai","year":"2023","journal-title":"arXiv preprint arXiv:2305.06500"},{"key":"ref114","article-title":"VisionLLM: Large language model is also an open-ended decoder for vision-centric tasks","author":"Wang","year":"2023","journal-title":"arXiv preprint arXiv:2305.11175"},{"key":"ref115","article-title":"Multiinstruct: Improving multimodal zero-shot learning via instruction tuning","author":"Xu","year":"2022","journal-title":"arXiv preprint arXiv:2212.10773"},{"key":"ref116","article-title":"A survey on multimodal large language models","author":"Yin","year":"2023","journal-title":"arXiv preprint arXiv:2306.13549"},{"key":"ref117","series-title":"Proc. IEEE\/CVF Conf. Comput. Vis. Pattern Recognit.","first-page":"16399","article-title":"Towards general purpose vision systems: An end-to-end task-agnostic vision-language architecture","author":"Gupta","year":"2022"},{"key":"ref118","article-title":"ChatBridge: Bridging modalities with large language model as a language catalyst","author":"Zhao","year":"2023","journal-title":"arXiv preprint arXiv:2305.16103"},{"key":"ref119","author":"Li","year":"2023","journal-title":"arXiv preprint arXiv:2306.04387"},{"key":"ref120","article-title":"Self-instruct: Aligning language model with self-generated instructions","author":"Wang","year":"2022","journal-title":"arXiv preprint arXiv:2212.10560"},{"key":"ref121","article-title":"GPT4Tools: Teaching large language model to use tools via selfinstruction","author":"Yang","year":"2023","journal-title":"arXiv preprint arXiv:2305.18752"},{"key":"ref122","article-title":"DetGPT: Detect what you need via reasoning","author":"Pi","year":"2023","journal-title":"arXiv preprint arXiv:2305.14167"},{"key":"ref123","article-title":"Cheap and quick: Efficient vision-language instruction tuning for large language models","author":"Luo","year":"2023","journal-title":"arXiv preprint arXiv:2305.15023"},{"key":"ref124","article-title":"LLaMA-Adapter: Efficient fine-tuning of language models with zero-init attention","author":"Zhang","year":"2023","journal-title":"arXiv preprint arXiv:2303.16199"},{"key":"ref125","article-title":"Finetuned language models are zero-shot learners","author":"Wei","year":"2021","journal-title":"arXiv preprint arXiv:2109.01652"},{"key":"ref126","article-title":"Chain of thought prompt tuning in vision language models","author":"Ge","year":"2023","journal-title":"arXiv preprint arXiv:2304.07919"},{"key":"ref127","article-title":"MM-REACT: Prompting ChatGPTfor multimodal reasoning and action","author":"Yang","year":"2023","journal-title":"arXiv preprint arXiv:2303.11381"},{"key":"ref128","series-title":"Proc. IEEE Int. Conf. Comput. Vis.","first-page":"19","article-title":"Aligning books and movies: Towards storylike visual explanations by watching movies and reading books","author":"Zhu","year":"2015"},{"key":"ref129","unstructured":"M.S. Hart et al., \u201cProject Gutenberg,\u201d Accessed: Jan. 06, 2024. [Online]. Available: https:\/\/www.gutenberg.org\/"},{"key":"ref130","article-title":"Training a helpful and harmless assistant with reinforcement learning from human feedback","author":"Bai","year":"2022","journal-title":"arXiv preprint arXiv:2204.05862"},{"key":"ref131","article-title":"Introducing the world\u2019s first truly open instruction-tuned LLM","author":"Dolly","year":"2023","journal-title":"Databricks"},{"key":"ref132","article-title":"How close is ChatGPT to human experts? comparison corpus, evaluation, and detection","author":"Guo","year":"2023","journal-title":"arXiv preprint arXiv:2301.07597"},{"key":"ref133","article-title":"Open Assistant conversations-democratizing large language model alignment","author":"K\u00f6pf","year":"2023","journal-title":"arXiv preprint arXiv.2304.07327"},{"key":"ref134","unstructured":"S. Tey et al., \u201cShareGPT,\u201d 2023. Accessed: Jan. 06, 2023. [Online]. Available: https:\/\/sharegpt.com\/"},{"key":"ref135","doi-asserted-by":"crossref","first-page":"485","DOI":"10.1007\/978-1-4842-4470-8_38","article-title":"Google BigQuery","author":"Bisong","year":"2019","journal-title":"Building Machine Learning and Deep Learning Models on Google Cloud Platform: A Comprehensive Guide for Beginners"},{"key":"ref136","article-title":"A simple method for commonsense reasoning","author":"Trinh","year":"2018","journal-title":"arXiv preprint arXiv:1806.02847"},{"key":"ref137","first-page":"9054","article-title":"Defending against neural fake news","volume":"32","author":"Zellers","year":"2019","journal-title":"Adv. Neural Inf. Process. Syst."},{"key":"ref138","article-title":"MVP: Multi-task supervised pre-training for natural language generation","author":"Tang","year":"2022","journal-title":"arXiv preprint arXiv:2206.12131"},{"key":"ref139","article-title":"Cross-task generalization via natural language crowdsourcing instructions","author":"Mishra","year":"2021","journal-title":"arXiv preprint arXiv:2104.08773"},{"key":"ref140","unstructured":"C. Schuhmann et al., \u201cThe OIG dataset,\u201d 2023. Accessed: Apr. 07, 2024. [Online]. Available: https:\/\/laion.ai\/blog\/oig-dataset\/"},{"key":"ref141","article-title":"PromptSource: An integrated development environment and repository for natural language prompts","author":"Bach","year":"2022","journal-title":"arXiv preprint arXiv:2202.01279"},{"key":"ref142","article-title":"Super-Naturalinstructions: Generalization via declarative instructions on 1600+ NLP tasks","author":"Wang","year":"2022","journal-title":"arXiv preprint arXiv:2204.07705"},{"key":"ref143","article-title":"Crosslingual generalization through multitask finetuning","author":"Muennighoff","year":"2022","journal-title":"arXiv preprint arXiv:2211.01786"},{"key":"ref144","unstructured":"A. Gokalan et al., \u201cOpenWebText corpus,\u201d 2023. Accessed: Apr. 07, 2024. [Online]. Available: https:\/\/skylion007.github.io\/OpenWebTextCorpus\/"},{"key":"ref145","series-title":"Proc. Int. AAAI Conf. Web Social Media","first-page":"830","article-title":"The pushshift reddit dataset","volume":"14","author":"Baumgartner","year":"2020"},{"key":"ref146","unstructured":"R. Taori et al., \u201cStanford Alpaca: An instruction-following Llama model,\u201d vol. 1, no. 9,2023.Accessed: Apr. 07, 2024.https:\/\/github.com\/tatsu-lab\/stanford_alpaca\/"},{"key":"ref147","article-title":"Baize: An open-source chat model with parameter-efficient tuning on self-chat data","author":"Xu","year":"2023","journal-title":"arXiv preprint arXiv:2304.01196"},{"key":"ref148","article-title":"Towards better instruction following language models for Chinese: Investigating the impact of training data and evaluation","author":"Ji","year":"2023","journal-title":"arXiv preprint arXiv:2304.07854"},{"key":"ref149","unstructured":"C. Josephus, \u201cGuanaco-generative universal assistant for natural-language adaptive context-aware omnilingual outputs.,\u201d 2023. Accessed: Apr. 07, 2024. [Online]. Available: https:\/\/guanaco-model.github.io\/"},{"key":"ref150","article-title":"The refinedweb dataset for falcon LLM: Outperforming curated corpora with web data, and web data only","author":"Penedo","year":"2023","journal-title":"arXiv preprint arXiv:2306.01116"},{"key":"ref151","unstructured":"J. Wales et al., \u201cI. positive and wikipedia,\u201d 2001. Accessed: Apr. 06, 2024. [Online]. Available: https:\/\/en.wikipedia.org\/wiki\/"},{"key":"ref152","unstructured":"M. Weber et al., \u201cRedPajama-data: An open-source recipe to reproduce llama training dataset,\u201d 2023. Accessed: Apr. 04, 2024. [Online]. Available: https:\/\/github.com\/open-models-platform\/openmodels.RedPajama-Data\/"},{"key":"ref153","first-page":"31809","article-title":"The bigscience ROOTS corpus: A 1.6TB composite multilingual dataset","volume":"35","author":"Lauren\u00e7on","year":"2022","journal-title":"Adv. Neural Inf. Process. Syst."},{"key":"ref154","article-title":"The Pile: An 800GB dataset of diverse text for language modeling","author":"Gao","year":"2020","journal-title":"arXiv preprint arXiv:2101.00027"},{"key":"ref155","article-title":"The Stack: 3 TB of permissively licensed source code","author":"Kocetkov","year":"2022","journal-title":"arXiv preprint arXiv:2211.15533"},{"key":"ref156","article-title":"LIMA: Less is more for alignment","author":"Zhou","year":"2023","journal-title":"arXiv preprint arXiv:2305.11206"},{"key":"ref157","article-title":"A large annotated corpus for learning natural language inference","author":"Bowman","year":"2015","journal-title":"arXiv preprint arXiv:1508.05326"},{"key":"ref158","article-title":"Know what you don\u2019t know: Unanswerable questions for squad","author":"Rajpurkar","year":"2018","journal-title":"arXiv preprint arXiv:1806.03822"},{"key":"ref159","article-title":"Red teaming language models to reduce harms: Methods, scaling behaviors, and lessons learned","author":"Ganguli","year":"2022","journal-title":"arXiv preprint arXiv:2209.07858"},{"key":"ref160","article-title":"CValues: Measuring the values of chinese large language models from safety to responsibility","author":"Xu","year":"2023","journal-title":"arXiv preprint arXiv:2307.09705"},{"key":"ref161","article-title":"Safe RLHF: Safe reinforcement learning from human feedback","author":"Dai","year":"2023","journal-title":"arXiv preprint arXiv:2310.12773"},{"key":"ref162","article-title":"Training socially aligned language models in simulated human society","author":"Liu","year":"2023","journal-title":"arXiv preprint arXiv:2305.16960"},{"key":"ref163","series-title":"Int. Conf. Mach. Learn.","first-page":"5988","author":"Ethayarajh","year":"2022"},{"key":"ref164","unstructured":"N. Lambert, L. Tunstall, N. Rajani, and T. Thrush, \u201cHuggingFace H4 stack exchange preference dataset,\u201d 2023. Accessed: Apr. 04, 2024. [Online]. Available: https:\/\/huggingface.co\/datasets\/HuggingFaceH4\/stack-exchange-preferences\/"},{"key":"ref165","first-page":"3008","article-title":"Learning to summarize with human feedback","volume":"33","author":"Stiennon","year":"2020","journal-title":"Adv. Neural Inf. Process Syst."},{"key":"ref166","article-title":"WebGPT: Browser-assisted question-answering with human feedback","author":"Nakano","year":"2021","journal-title":"arXiv preprint arXiv:2112.09332"},{"key":"ref167","unstructured":"M. Byeon, B. Park, H. Kim, S. Lee, W. Baek and S. Kim, \u201cCOYO-700M: Image-text pair dataset,\u201d 2022. Accessed: Apr. 04, 2024. [Online]. Available: https:\/\/github.com\/kakaobrain\/coyo-dataset\/"},{"key":"ref168","series-title":"Proc. IEEE\/CVF Conf. Comput. Vis. Pattern Recognit.","first-page":"3558","article-title":"Conceptual 12M: Pushing web-scale image-text pre-training to recognize long-tail visual concepts","author":"Changpinyo","year":"2021"},{"key":"ref169","series-title":"Proc. 56th Annu. Meet. Assoc. Comput. Linguist.","first-page":"2556","article-title":"Conceptual captions: A cleaned, hypernymed, image ALT-text dataset for automatic image captioning","author":"Sharma","year":"2018"},{"key":"ref170","series-title":"Proc. IEEE Int. Conf. Comput. Vis.","first-page":"843","article-title":"Revisiting unreasonable effectiveness of data in deep learning era","author":"Sun","year":"2017"},{"key":"ref171","series-title":"Proc. IEEE\/CVF Conf. Comput. Vis. Pattern Recognit.","first-page":"12104","article-title":"Scaling vision transformers","author":"Zhai","year":"2022"},{"key":"ref172","series-title":"Comput. Vis.\u2013ECCV 2014: 13th Eur. Conf.","first-page":"740","article-title":"Microsoft COCO: Common objects in context","author":"Lin","year":"2014"},{"key":"ref173","article-title":"Microsoft COCO captions: Data collection and evaluation server","author":"Chen","year":"2015","journal-title":"arXiv preprint arXiv:1504.00325"},{"key":"ref174","series-title":"Proc. IEEE\/CVF Conf. Comput. Vis. Pattern Recognit.","first-page":"6700","article-title":"GQA: A new dataset for realworld visual reasoning and compositional question answering","author":"Hudson","year":"2019"},{"key":"ref175","series-title":"2019 Int. Conf. Document Anal. Recognit. (ICDAR)","first-page":"947","article-title":"OCR-VQA: Visual question answering by reading text in images","author":"Mishra","year":"2019"},{"key":"ref176","doi-asserted-by":"crossref","first-page":"1886","DOI":"10.1109\/JBHI.2023.3294249","article-title":"K-PathVQA: Knowledge-aware multimodal representation for pathology visual question answering","volume":"28","author":"Naseem","year":"2023","journal-title":"IEEE J. Biomed. Health Inform."},{"key":"ref177","article-title":"QuAC: Question answering in context","author":"Choi","year":"2018","journal-title":"arXiv preprint arXiv:1808.07036"},{"key":"ref178","series-title":"Proc. 2014 Conf. Empirical Methods Natural Lang. Process. (EMNLP)","first-page":"787","article-title":"ReferItGame: Referring to objects in photographs of natural scenes","author":"Kazemzadeh","year":"2014"},{"key":"ref179","series-title":"Comput. Vis.-ECCV 2016: 14th Euro. Conf.","first-page":"69","article-title":"Modeling context in referring expressions","author":"Yu","year":"2016"},{"key":"ref180","series-title":"2021 IEEE 18th Int. Symp. Biomed. Imag. (ISBI)","first-page":"1650","article-title":"Slake: A semantically-labeled knowledgeenhanced dataset for medical visual question answering","author":"Liu","year":"2021"},{"key":"ref181","series-title":"Proc. IEEE\/CVF Conf. Comput. Vis. Pattern Recognit.","first-page":"16548","article-title":"LaTr: Layout-aware transformer for scene-text VQA","author":"Biten","year":"2022"},{"key":"ref182","series-title":"Proc. IEEE\/CVF Conf. Comput. Vis. Pattern Recognit.","first-page":"8317","article-title":"Towards VQA models that can read","author":"Singh","year":"2019"},{"key":"ref183","doi-asserted-by":"crossref","first-page":"32","DOI":"10.1007\/s11263-016-0981-7","article-title":"Visual Genome: Connecting language and vision using crowdsourced dense image annotations","volume":"123","author":"Krishna","year":"2017","journal-title":"Int. J. Comput. Vis."},{"key":"ref184","series-title":"Proc. IEEE Conf. Comput. Visi. Pattern Recognit.","first-page":"4995","article-title":"Visual7W: Grounded question answering in images","author":"Zhu","year":"2016"},{"key":"ref185","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1038\/sdata.2018.251","article-title":"A dataset of clinically generated visual questions and answers about radiology images","volume":"5","author":"Lau","year":"2018","journal-title":"Sci. Data"},{"key":"ref186","series-title":"Proc. IEEE Conf. Comput. Vis. Pattern Recognit.","first-page":"6904","article-title":"Making the V in VQA matter: Elevating the role of image understanding in visual question answering","author":"Goyal","year":"2017"},{"key":"ref187","series-title":"Euro. Conf. Comput. Vis.","first-page":"146","article-title":"A-OKVQA: A benchmark for visual question answering using world knowledge","author":"Schwenk","year":"2022"},{"key":"ref188","article-title":"DataComp: In search of the next generation of multimodal datasets","author":"Yitzhak Gadre","year":"2023","journal-title":"arXiv preprint arXiv:2304.14108"},{"key":"ref189","series-title":"Proc. IEEE\/CVF Winter Conf. Appl. Comput. Vis.","first-page":"2200","article-title":"DocVQA: A dataset for VQA on document images","author":"Mathew","year":"2021"},{"key":"ref190","series-title":"Proc. IEEE Conf. Comput. Vis. Pattern Recognit.","first-page":"5648","article-title":"DVQA: Understanding data visualizations via question answering","author":"Kafle","year":"2018"},{"key":"ref191","article-title":"RedCaps: Webcurated image-text data created by the people","author":"Desai","year":"2021","journal-title":"arXiv preprint arXiv:2111.11431"},{"key":"ref192","first-page":"1143","article-title":"Im2Text: Describing images using 1 million captioned photographs","volume":"24","author":"Ordonez","year":"2011","journal-title":"Adv. Neural Inf. Process. Syst."},{"key":"ref193","series-title":"Proc. 44th Int. ACM SIGIR Con. Res. Develop. Inform. Retr.","first-page":"2443","article-title":"WIT: Wikipedia-based image text dataset for multimodal multilingual machine learning","author":"Srinivasan","year":"2021"},{"key":"ref194","doi-asserted-by":"crossref","first-page":"64","DOI":"10.1145\/2812802","article-title":"YFCC100M: The new data in multimedia research","volume":"59","author":"Thomee","year":"2016","journal-title":"Commun. ACM"},{"key":"ref195","article-title":"AI challenger: A large-scale dataset for going deeper in image understanding","author":"Wu","year":"2017","journal-title":"arXiv preprint arXiv:1711.06475"},{"key":"ref196","doi-asserted-by":"crossref","first-page":"67","DOI":"10.1162\/tacl_a_00166","article-title":"From image descriptions to visual denotations: New similarity metrics for semantic inference over event descriptions","volume":"2","author":"Young","year":"2014","journal-title":"Trans. Assoc. Computat. Linguist."},{"key":"ref197","series-title":"Proc. IEEE Int. Conf. Comput. Vis.","first-page":"2641","article-title":"Flickr30k entities: Collecting regionto-phrase correspondences for richer image-to-sentence models","author":"Plummer","year":"2015"},{"key":"ref198","series-title":"Proc. IEEE\/CVF Conf. Comput. Vis. Pattern Recognit.","first-page":"804","article-title":"Revisiting weakly supervised pre-training of visual perception models","author":"Singh","year":"2022"},{"key":"ref199","doi-asserted-by":"crossref","first-page":"84","DOI":"10.1145\/3065386","article-title":"ImageNet classification with deep convolutional neural networks","volume":"60","author":"Krizhevsky","year":"2017","journal-title":"Commun. ACM"},{"key":"ref200","doi-asserted-by":"crossref","first-page":"211","DOI":"10.1007\/s11263-015-0816-y","article-title":"ImageNet large scale visual recognition challenge","volume":"115","author":"Russakovsky","year":"2015","journal-title":"Int. J. Comput. Vis."},{"key":"ref201","article-title":"ImageNet-21K pretraining for the masses","author":"Ridnik","year":"2021","journal-title":"arXiv preprint arXiv:2104.10972"},{"key":"ref202","first-page":"25278","article-title":"LAION-5B: An open large-scale dataset for training next generation image-text models","volume":"35","author":"Schuhmann","year":"2022","journal-title":"Adv. in Neural Inf. Process. Syst."},{"key":"ref203","article-title":"LAION-400M: Open dataset of clip-filtered 400 million image-text pairs","author":"Schuhmann","year":"2021","journal-title":"arXiv preprint arXiv:2111.02114"},{"key":"ref204","unstructured":"C. Schuhmann, A. K\u00f6pf, R. Vencu, T. Coombes, and R. Beaumont, \u201cLAION COCO: 600m synthetic captions from LAION 2B-EN,\u201d Accessed: Apr. 07, 2024. [Online]. Available: https:\/\/laion.ai\/blog\/laion-coco\/"},{"key":"ref205","series-title":"Proc. IEEE\/CVF Int. Conf. Comput. Vis.","first-page":"8948","article-title":"nocaps: Novel object captioning at scale","author":"Agrawal","year":"2019"},{"key":"ref206","unstructured":"B. -L Lu and W. -L Schuhmann, \u201cSeed dataset,\u201d Accessed: Apr. 07, 2024. [Online]. Available: https:\/\/bcmi.sjtu.edu.cn\/home\/seed\/"},{"key":"ref207","series-title":"Proc. IEEE\/CVF Int. Conf. Comput. Vis.","first-page":"1728","article-title":"Frozen in time: A joint video and image encoder for end-to-end retrieval","author":"Bain","year":"2021"},{"key":"ref208","unstructured":"P. Lu et al., \u201cLearn to explain: Multimodal reasoning via thought chains for science question answering,\u201d 2022. Accessed: Apr. 07, 2024. [Online]. Available: https:\/\/scienceqa.github.io\/"},{"key":"ref209","article-title":"OBELISC: An open web-scale filtered dataset of interleaved image-text documents","author":"Lauren\u00e7on","year":"2023","journal-title":"arXiv preprint arXiv:2306.16527"},{"key":"ref210","series-title":"Proc. IEEE Conf. Comput. Vis. Pattern Recognit.","first-page":"5288","article-title":"MSR-VTT: A large video description dataset for bridging video and language","author":"Xu","year":"2016"},{"key":"ref211","series-title":"Comput. Vis.\u2013ECCV 2020: 16th Euro. Conf., Glasgow, UK","first-page":"742","article-title":"TextCaps: A dataset for image captioning with reading comprehension","author":"Sidorov","year":"2020"},{"key":"ref212","article-title":"PaLI: A jointly-scaled multilingual language-image model","author":"Chen","year":"2022","journal-title":"arXiv preprint arXiv:2209.06794"},{"key":"ref213","first-page":"26418","article-title":"Wukong: A 100 million largescale Chinese cross-modal pre-training benchmark","volume":"35","author":"Gu","year":"2022","journal-title":"Adv. Neural Inf. Process. Syst."},{"key":"ref214","article-title":"PaLI-X: On scaling up a multilingual vision and language model","author":"Chen","year":"2023","journal-title":"arXiv preprint arXiv:2305.18565"},{"key":"ref215","article-title":"Multimodal C4: An open, billion-scale corpus of images interleaved with text","author":"Zhu","year":"2023","journal-title":"arXiv preprint ArXiv:2304.06939"},{"key":"ref216","article-title":"AISHELL-2: Transforming mandarin ASR research into industrial scale","author":"Du","year":"2018","journal-title":"arXiv preprint arXiv:1808.10583"},{"key":"ref217","article-title":"MOFI: Learning image representations from noisy entity annotated images","author":"Wu","year":"2023","journal-title":"arXiv preprint arXiv:2306.07952"},{"key":"ref218","article-title":"Documenting large webtext corpora: A case study on the colossal clean crawled corpus","author":"Dodge","year":"2021","journal-title":"arXiv preprint arXiv:2104.08758"},{"key":"ref219","series-title":"2009 IEEE Conf. Comput. Vis. Pattern Recognit.","first-page":"248","article-title":"ImageNet: A large-scale hierarchical image database","author":"Deng","year":"2009"},{"key":"ref220","article-title":"Gemini: A family of highly capable multimodal models","author":"Team","year":"2023","journal-title":"arXiv preprint arXiv:2312.11805"},{"key":"ref221","article-title":"Measuring massive multitask language understanding","author":"Hendrycks","year":"2020","journal-title":"arXiv preprint arXiv:2009.03300"},{"key":"ref222","article-title":"C-Eval: A multi-level multi-discipline Chinese evaluation suite for foundation models","author":"Huang","year":"2023","journal-title":"arXiv preprint arXiv:2305.08322"},{"key":"ref223","article-title":"Training verifiers to solve math word problems","author":"Cobbe","year":"2021","journal-title":"arXiv preprintarXiv: 2110. 14168"},{"key":"ref224","article-title":"Challenging big-bench tasks and whether chain-of-thought can solve them","author":"M.Suzgun","year":"2022","journal-title":"arXiv preprint arXiv:2210.09261"},{"key":"ref225","article-title":"Measuring coding challenge competence with apps","author":"Hendrycks","year":"2021","journal-title":"arXiv preprint arXiv:2105.09938"},{"key":"ref226","article-title":"Video-MME: The first-ever comprehensive evaluation benchmark of multi-modal LLMs in video analysis","author":"Fu","year":"2024","journal-title":"arXiv preprint arXiv:2405.21075"},{"key":"ref227","article-title":"LAMM: Language-assisted multi-modal instruction-tuning dataset, framework, and benchmark","author":"Yin","year":"2023","journal-title":"arXiv preprint arXiv:2306.06687"},{"key":"ref228","article-title":"Video-ChatGPT: Towards detailed video understanding via large vision and language models","author":"Maaz","year":"2023","journal-title":"arXiv preprint arXiv:2306.05424"},{"key":"ref229","article-title":"Holistic evaluation of language models","author":"Liang","year":"2022","journal-title":"arXiv preprint arXiv:2306.05685"},{"key":"ref230","article-title":"Judging LLM-as-a-judge with MT-bench and chatbot arena","author":"Zheng","year":"2023","journal-title":"arXiv preprint arXiv:2306.05685"},{"key":"ref231","unstructured":"Y. Dubois et al., \u201cAlpacaFarm: A simulation framework for methods that learn from human feedback,\u201d 2024, Accessed: Apr. 06, 2024. [Online]. Available: https:\/\/arxiv.org\/abs\/2305.14387\/"},{"key":"ref232","unstructured":"Y. Zhao, M. Khalman, R. Joshi, S. Narayan, M. Saleh and P. J. Liu, \u201cCalibrating sequence likelihood improves conditional language generation,\u201d arXiv preprint arXiv:2210.00045, 2022."},{"key":"ref233","article-title":"Emergent abilities of large language models","author":"Wei","year":"2022","journal-title":"arXiv preprint arXiv:2206.07682"},{"key":"ref234","doi-asserted-by":"crossref","first-page":"155","DOI":"10.1016\/0004-3702(87)90033-6","article-title":"Phase transitions in artificial intelligence systems","volume":"33","author":"Huberman","year":"1987","journal-title":"Artif. Intell."},{"key":"ref235","article-title":"Multitask prompted training enables zero-shot task generalization","author":"Sanh","year":"2021","journal-title":"arXiv preprint arXiv:2110.08207"},{"key":"ref236","article-title":"Least-to-most prompting enables complex reasoning in large language models","author":"Zhou","year":"2022","journal-title":"arXiv preprint arXiv:2205.10625"},{"key":"ref237","article-title":"How does GPT obtain its ability? Tracing emergent abilities of language models to their sources","author":"Fu","year":"2022","journal-title":"Yao Fu\u2019s Notion"},{"key":"ref238","article-title":"Extending context window of large language models via positional interpolation","author":"Chen","year":"2023","journal-title":"arXiv preprint arXiv:2306.15595"},{"key":"ref239","article-title":"DoLa: Decoding by contrasting layers improves factuality in large language models","author":"Chuang","year":"2023","journal-title":"arXiv preprint arXiv:2309.03883"},{"key":"ref240","article-title":"Few-shot learning with retrieval augmented language models","author":"Izacard","year":"2022","journal-title":"arXiv preprint arXiv:2208.03299"},{"key":"ref241","series-title":"Int. Conf. Mach. Learn.","first-page":"3929","article-title":"Retrieval augmented language model pre-training","author":"Guu","year":"2020"},{"key":"ref242","first-page":"9459","article-title":"Retrieval-augmented generation for knowledge-intensive NLP tasks","volume":"33","author":"Lewis","year":"2020","journal-title":"Adv. Neural Inf. Process. Syst."},{"key":"ref243","doi-asserted-by":"crossref","first-page":"11196","DOI":"10.1109\/TKDE.2022.3223858","article-title":"Complex knowledge base question answering: A survey","volume":"35","author":"Lan","year":"2022","journal-title":"IEEE Trans. Knowl. Data Eng."},{"key":"ref244","article-title":"Check your facts and try again: Improving large language models with external knowledge and automated feedback","author":"Peng","year":"2023","journal-title":"arXiv preprint arXiv:2302.12813"},{"key":"ref245","doi-asserted-by":"crossref","first-page":"103510","DOI":"10.1016\/j.ipm.2023.103510","article-title":"What is the limitation of multimodal LLMs? A deeper look into multimodal LLMs through prompt probing","volume":"60","author":"Qi","year":"2023","journal-title":"Inform. Process. Manage."},{"key":"ref246","series-title":"Proc. IEEE\/CVF Conf. Comput. Vis. Pattern Recognit.","first-page":"19113","article-title":"MaPLe: Multi-modal prompt learning","author":"Khattak","year":"2023"},{"key":"ref247","doi-asserted-by":"crossref","first-page":"100211","DOI":"10.1016\/j.hcc.2024.100211","article-title":"A survey on large language model (LLM) security and privacy: The good, the bad, and the ugly","author":"Yao","year":"2024","journal-title":"High-Confid. Comput."},{"key":"ref248","article-title":"Compositional chain-of-thought prompting for large multimodal models","author":"Mitra","year":"2023","journal-title":"arXiv preprint arXiv:2311.17076"},{"key":"ref249","article-title":"BenchLMM: Benchmarking cross-style visual capability of large multimodal models","author":"Cai","year":"2023","journal-title":"arXiv preprint arXiv:2312.02896"},{"key":"ref250","article-title":"PixeLLM: Pixel reasoning with large multimodal model","author":"Ren","year":"2023","journal-title":"arXiv preprint arXiv:2312.02228"},{"key":"ref251","series-title":"Int. Conf. Mach. Learn.","first-page":"22137","article-title":"Deja Vu: Contextual sparsity for efficient LLMs at inference time","author":"Liu","year":"2023"},{"key":"ref252","article-title":"SelfcheckGPT: Zeroresource black-box hallucination detection for generative large language models","author":"Manakul","year":"2023","journal-title":"arXiv preprint arXiv:2303.08896"},{"key":"ref253","article-title":"Chainpoll: A high efficacy method for LLM hallucination detection","author":"Friel","year":"2023","journal-title":"arXiv preprint arXiv:2310.18344"},{"key":"ref254","article-title":"GPTscore: Evaluate as you desire","author":"Fu","year":"2023","journal-title":"arXiv preprint arXiv:2302. 04166"},{"key":"ref255","article-title":"G-Eval: NLG evaluation using GPT-4 with better human alignment","author":"Liu","year":"May 2023","journal-title":"arXiv preprint arXiv:2303.16634"},{"key":"ref256","article-title":"True: Re-evaluating factual consistency evaluation","author":"Honovich","year":"2022","journal-title":"arXiv preprint arXiv:2204.04991"},{"key":"ref257","series-title":"Proc. IEEE\/CVF Conf. Comput. Vis. Pattern Recognit.","first-page":"10553","article-title":"Audio-visual generalised zero-shot learning with cross-modal attention and language","author":"Mercea","year":"2022"},{"key":"ref258","series-title":"Proc. IEEE\/CVF Conf. Comput. Vis. Pattern Recognit.","first-page":"3042","article-title":"Unified multi-modal transformers for joint video moment retrieval and highlight detection","author":"Liu","year":"2022"},{"key":"ref259","series-title":"Proc. IEEE\/CVF Conf. Comput. Vis. Pattern Recognit.","first-page":"5227","article-title":"VL-adapter: Parameter efficient transfer learning for vision-and-language tasks","author":"Sung","year":"2022"},{"key":"ref260","article-title":"Aligning large multimodal models with factually augmented RLHF","author":"Sun","year":"2023","journal-title":"arXiv preprint arXiv:2309.14525"},{"key":"ref261","article-title":"An empirical study of scaling instruct-tuned large multimodal models","author":"Lu","year":"2023","journal-title":"arXiv preprint arXiv:2309.09958"},{"key":"ref262","article-title":"Zero-shot faithful factual error correction","author":"Huang","year":"2023","journal-title":"arXiv preprint arXiv:2305.07982"},{"key":"ref263","article-title":"Med-HALT: Medical domain hallucination test for large language models","author":"Umapathi","year":"2023","journal-title":"arXiv preprint arXiv:2307.15343"},{"key":"ref264","article-title":"Inference-time intervention: Eliciting truthful answers from a language model","author":"Li","year":"2023","journal-title":"arXiv preprint arXiv:2306.03341"},{"key":"ref265","article-title":"CommonsenseQA: A question answering challenge targeting commonsense knowledge","author":"Talmor","year":"2018","journal-title":"arXiv preprint arXiv:1811.00937"},{"key":"ref266","doi-asserted-by":"crossref","first-page":"289","DOI":"10.1007\/s00799-022-00329-y","article-title":"ScienceQA: A novel resource for question answering on scholarly articles","volume":"23","author":"Saikh","year":"2022","journal-title":"Int. J. Digit. Libr."},{"key":"ref267","series-title":"Proc. IEEE\/CVF Conf. Comput. Vis. Pattern Recognit.","first-page":"4938","article-title":"SuperGlue: Learning feature matching with graph neural networks","author":"Sarlin","year":"2020"},{"key":"ref268","article-title":"ChatGPT is a knowledgeable but inexperienced solver: An investigation of commonsense problem in large language models","author":"Bian","year":"2023","journal-title":"arXiv preprint arXiv:2303.16421"},{"key":"ref269","article-title":"ChatCoT: Tool-augmented chain-of-thought reasoning on chat-based large language models","author":"Chen","year":"2023","journal-title":"arXiv preprint arXiv:2305.14323"},{"key":"ref270","article-title":"Mind meets machine: Unravelling GPT-4\u2019s cognitive psychology","author":"Dhingra","year":"2023","journal-title":"arXiv preprint arXiv:2303.11436"},{"key":"ref271","article-title":"On the advance of making language models better reasoners","author":"Li","year":"2022","journal-title":"arXiv preprint arXiv:2206.02336"},{"key":"ref272","article-title":"KCTS: Knowledge-constrained tree search decoding with token-level hallucination detection","author":"Choi","year":"2023","journal-title":"arXiv preprint arXiv:2310.09044"},{"key":"ref273","first-page":"22199","article-title":"Large language models are zero-shot reasoners","volume":"35","author":"Kojima","year":"2022","journal-title":"Adv. Neural Inf. Process. Syst."},{"key":"ref274","doi-asserted-by":"crossref","first-page":"e46939","DOI":"10.2196\/46939","article-title":"Putting ChatGPT\u2019s medical advice to the (turing) test","volume":"10","author":"Nov","year":"2023","journal-title":"JMIR Med. Educ."},{"key":"ref275","article-title":"On the evaluations of ChatGPT and emotion-enhanced prompting for mental health analysis","author":"Yang","year":"2023","journal-title":"arXiv preprint arXiv:2304.03347"},{"key":"ref276","doi-asserted-by":"crossref","first-page":"1233","DOI":"10.1056\/NEJMsr2214184","article-title":"Benefits, limits, and risks of GPT-4 as an ai chatbot for medicine","volume":"388","author":"Lee","year":"2023","journal-title":"New Engl. J. Med."},{"key":"ref277","year":"2015","journal-title":"Tracking Universal Health Coverage: First Global Monitoring Report"},{"key":"ref278","article-title":"Large language models encode clinical knowledge","author":"Singhal","year":"2022","journal-title":"arXiv preprint arXiv:2212.13138"},{"key":"ref279","article-title":"Zhongjing: Enhancing the Chinese medical capabilities of large language model through expert feedback and real-world multiturn dialogue","author":"Yang","year":"2023","journal-title":"arXiv preprint arXiv:2308.03549"},{"key":"ref280","article-title":"Towards expertlevel medical question answering with large language models","author":"Singhal","year":"2023","journal-title":"arXiv preprint arXiv:2305.09617"},{"key":"ref281","doi-asserted-by":"crossref","first-page":"866","DOI":"10.1001\/jama.2023.14217","article-title":"Creation and adoption of large language models in medicine","volume":"330","author":"Shah","year":"2023","journal-title":"JAMA"},{"key":"ref282","doi-asserted-by":"crossref","first-page":"6655","DOI":"10.3390\/su15086655","article-title":"A review on innovation in healthcare sector (telehealth) through artificial intelligence","volume":"15","author":"Amjad","year":"2023","journal-title":"Sustainability"},{"key":"ref283","article-title":"Does synthetic data generation of LLMs help clinical text mining?","author":"Tang","year":"2023","journal-title":"arXiv preprint arXiv:2303.04360"},{"key":"ref284","doi-asserted-by":"crossref","first-page":"141","DOI":"10.1038\/s43856-023-00370-1","article-title":"The future landscape of large language models in medicine","volume":"3","author":"Clusmann","year":"2023","journal-title":"Commun. Med."},{"key":"ref285","article-title":"ChatCAD: Interactive computer-aided diagnosis on medical image using large language models","author":"Wang","year":"2023","journal-title":"arXiv preprint arXiv:2302.07257"},{"key":"ref286","article-title":"Med-MMHL: A multimodal dataset for detecting human-and LLM-generated misinformation in the medical domain","author":"Sun","year":"2023","journal-title":"arXiv preprint arXiv:2306.08871"},{"key":"ref287","article-title":"Augmenting black-box LLMs with medical textbooks for clinical question answering","author":"Wang","year":"2023","journal-title":"arXiv preprint arXiv:2309.02233"},{"key":"ref288","doi-asserted-by":"crossref","first-page":"1549","DOI":"10.1007\/s00259-023-06172-w","article-title":"Large language models (LLM) and ChatGPT: What will the impact on nuclear medicine be?","volume":"50","author":"Alberts","year":"2023","journal-title":"Eur. J. Nucl. Med. Mol. Imag."},{"key":"ref289","article-title":"FinBERT: A pretrained language model for financial communications","author":"Yang","year":"2020","journal-title":"arXiv preprint arXiv:2006.08097"},{"key":"ref290","article-title":"Beyond classification: Financial reasoning in state-of-the-art language models","author":"Son","year":"2023","journal-title":"arXiv preprint arXiv:2305.01505"},{"key":"ref291","series-title":"Proc. Fourth ACM Int. Conf. AI in Finance","first-page":"349","article-title":"Enhancing financial sentiment analysis via retrieval augmented large language models","author":"Zhang","year":"2023"},{"key":"ref292","article-title":"FinGPT: Open-source financial large language models","author":"Yang","year":"2023","journal-title":"arXiv preprint arXiv:2306.06031"},{"key":"ref293","article-title":"Temporal data meets LLM-explainable financial time series forecasting","author":"Yu","year":"2023","journal-title":"arXiv preprint arXiv:2306.11025"},{"key":"ref294","article-title":"The wall street neophyte: A zero-shot analysis of chatgpt over multimodal stock movement prediction challenges","author":"Xie","year":"2023","journal-title":"arXiv preprint arXiv:2304.05351"},{"key":"ref295","article-title":"LLM-BRAIn: AI-driven fast generation of robot behaviour tree based on large language model","author":"Lykov","year":"2023","journal-title":"arXiv preprint arXiv:2305.19352"},{"key":"ref296","doi-asserted-by":"crossref","first-page":"100131","DOI":"10.1016\/j.birob.2023.100131","article-title":"Large language models for human-robot interaction: A review","volume":"3","author":"Zhang","year":"2023","journal-title":"Biomimetic Intell. Robot."},{"key":"ref297","article-title":"Task and motion planning with large language models for object rearrangement","author":"Ding","year":"2023","journal-title":"arXiv preprint arXiv:2303.06247"},{"key":"ref298","article-title":"Large language models as zero-shot human models for human-robot interaction","author":"Zhang","year":"2023","journal-title":"arXiv preprint arXiv:2303.03548"},{"key":"ref299","series-title":"2023 IEEE Int. Conf. Robot. Automat. (ICRA)","first-page":"11523","article-title":"ProgPrompt: Generating situated robot task plans using large language models","author":"Singh","year":"2023"},{"key":"ref300","article-title":"TidyBot: Personalized robot assistance with large language models","author":"Wu","year":"2023","journal-title":"arXiv preprint arXiv:2305.05658"},{"key":"ref301","article-title":"REAL: Resilience and adaptation using large language models on autonomous aerial robots","author":"Tagliabue","year":"2023","journal-title":"arXiv preprint arXiv:2311.01403"},{"key":"ref302","doi-asserted-by":"crossref","first-page":"201","DOI":"10.3390\/drones6080201","article-title":"Multimodal fusion of voice and gesture data for UAV control","volume":"6","author":"Xiang","year":"2022","journal-title":"Drones"},{"key":"ref303","doi-asserted-by":"crossref","first-page":"55748","DOI":"10.1109\/ACCESS.2023.3282111","article-title":"Improved trust in human-robot collaboration with ChatGPT","volume":"11","author":"Ye","year":"2023","journal-title":"IEEE Access"},{"key":"ref304","article-title":"Robot task planning based on large language model representing knowledge with directed graph structures","author":"Zhen","year":"2023","journal-title":"arXiv preprint arXiv:2306.05171"},{"key":"ref305","article-title":"OpenAGI: When LLM meets domain experts","author":"Ge","year":"2023","journal-title":"arXiv preprint arXiv:2304.04370"},{"key":"ref306","doi-asserted-by":"crossref","first-page":"58","DOI":"10.1007\/s00138-023-01408-z","article-title":"Text-driven object affordance for guiding grasp-type recognition in multimodal robot teaching","volume":"34","author":"Wake","year":"2023","journal-title":"Mach. Vision Appl."},{"key":"ref307","doi-asserted-by":"crossref","first-page":"265","DOI":"10.1146\/annurev-control-091420-084139","article-title":"Integrated task and motion planning, annual review of control","volume":"4","author":"Garrett","year":"2021","journal-title":"Robot. Auton. Syst."},{"key":"ref308","article-title":"Exploring the impacts of ChatGPT on future scientific work","author":"Irons","year":"2023","journal-title":"SocArXiv"},{"key":"ref309","first-page":"93","article-title":"Using generative AI for literature searches and scholarly writing: Is the integrity of the scientific discourse in jeopardy?","volume":"71","author":"Schmidt","year":"2023","journal-title":"Notice Am. Math. Soc."},{"key":"ref310","article-title":"PubMedQA: A dataset for biomedical research question answering","author":"Jin","year":"2019","journal-title":"arXiv preprint arXiv:1909.06146"},{"key":"ref311","series-title":"Proc. 2023 Conf. Innovat. Technol. Comput. Sci. Educ.","first-page":"47","article-title":"On the educational impact of ChatGPT: Is artificial intelligence ready to obtain a university degree?","volume":"1","author":"Malinka","year":"2023"},{"key":"ref312","article-title":"ChatGPT: The end of online exam integrity?","author":"Susnjak","year":"2022","journal-title":"arXiv preprint arXiv: 2212. 09292"},{"key":"ref313","doi-asserted-by":"crossref","first-page":"102274","DOI":"10.1016\/j.lindif.2023.102274","article-title":"ChatGPT for good? on opportunities and challenges of large language models for education","volume":"103","author":"Kasneci","year":"2023","journal-title":"Learn. Individ. Differ."},{"key":"ref314","doi-asserted-by":"crossref","first-page":"1","DOI":"10.48185\/tts.v5i1.1000","article-title":"Enhancing the quality of teaching and learning through ChatGPT and similar large language models: Challenges, future prospects, and ethical considerations in education","volume":"5","author":"Rane","year":"2024","journal-title":"TESOL Technol. Studies"},{"key":"ref315","article-title":"Are large multimodal foundation models all we need? On opportunities and challenges of these models in education","author":"K\u00fcchemann","year":"2024","journal-title":"EdArXiv"},{"key":"ref316","first-page":"pp. 265","article-title":"Is e-learning the solution for individual learning?","volume":"2","author":"Tavangarian","year":"2004","journal-title":"Electron. J. E-Learn."},{"key":"ref317","article-title":"PG-Video-LLaVA: Pixel grounding large video-language models","author":"Munasinghe","year":"2023","journal-title":"arXiv preprint arXiv:2311.13435"},{"key":"ref318","doi-asserted-by":"crossref","first-page":"1879","DOI":"10.1007\/s11676-020-01245-0","article-title":"Tree species classification using deep learning and RGB optical images obtained by an unmanned aerial vehicle","volume":"32","author":"Zhang","year":"2021","journal-title":"J. For. Res."},{"key":"ref319","doi-asserted-by":"crossref","first-page":"1471","DOI":"10.3390\/f13091471","article-title":"Comparison of multiple machine learning models for estimating the forest growing stock in large-scale forests using multi-source data","volume":"13","author":"Huang","year":"2022","journal-title":"Forests"},{"key":"ref320","doi-asserted-by":"crossref","first-page":"1310","DOI":"10.3390\/f14071310","article-title":"An improved sap flow prediction model based on CNN-GRU-BiLSTM and factor analysis of historical environmental variables","volume":"14","author":"Li","year":"2023","journal-title":"Forests"},{"key":"ref321","doi-asserted-by":"crossref","first-page":"499","DOI":"10.3390\/ani14030499","article-title":"DiffusionFR: Species recognition of fish in blurry scenarios via diffusion and attention","volume":"14","author":"Wang","year":"2024","journal-title":"Animals"},{"key":"ref322","doi-asserted-by":"crossref","first-page":"1302361","DOI":"10.3389\/fpls.2024.1302361","article-title":"Pine wilt disease detection algorithm based on improved YOLOv5","volume":"15","author":"Du","year":"2024","journal-title":"Front. Plant Sci."},{"key":"ref323","doi-asserted-by":"crossref","first-page":"196","DOI":"10.3390\/ijgi12050196","article-title":"A deep transfer learning toponym extraction and geospatial clustering framework for investigating scenic spots as cognitive regions","volume":"12","author":"Zhang","year":"2023","journal-title":"ISPRS Int. J. Geo Inf."},{"key":"ref324","doi-asserted-by":"crossref","first-page":"394","DOI":"10.3390\/ijgi12100394","article-title":"ChineseCTRE: A model for geographical named entity recognition and correction based on deep neural networks and the BERT model","volume":"12","author":"Zhang","year":"2023","journal-title":"ISPRS Int. J. Geo Inf."},{"key":"ref325","first-page":"1","article-title":"Beyond extraction accuracy: Addressing the quality of geographical named entity through advanced recognition and correction models using a modified BERT framework","author":"Xu","year":"2024","journal-title":"Geo-Spatial Inf. Sci."}],"container-title":["Computers, Materials & Continua"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/www.techscience.com\/files\/cmc\/2024\/TSP_CMC-80-2\/TSP_CMC_52618\/TSP_CMC_52618.pdf","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,3,6]],"date-time":"2025-03-06T12:20:45Z","timestamp":1741263645000},"score":1,"resource":{"primary":{"URL":"https:\/\/www.techscience.com\/cmc\/v80n2\/57626"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024]]},"references-count":325,"journal-issue":{"issue":"2","published-online":{"date-parts":[[2024]]},"published-print":{"date-parts":[[2024]]}},"URL":"https:\/\/doi.org\/10.32604\/cmc.2024.052618","relation":{},"ISSN":["1546-2226"],"issn-type":[{"value":"1546-2226","type":"electronic"}],"subject":[],"published":{"date-parts":[[2024]]}}}