{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,4]],"date-time":"2026-07-04T18:25:05Z","timestamp":1783189505814,"version":"3.54.6"},"reference-count":235,"publisher":"Springer Science and Business Media LLC","issue":"5","license":[{"start":{"date-parts":[[2024,9,30]],"date-time":"2024-09-30T00:00:00Z","timestamp":1727654400000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2024,9,30]],"date-time":"2024-09-30T00:00:00Z","timestamp":1727654400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Vis Comput"],"published-print":{"date-parts":[[2025,3]]},"DOI":"10.1007\/s00371-024-03608-8","type":"journal-article","created":{"date-parts":[[2024,9,30]],"date-time":"2024-09-30T16:02:56Z","timestamp":1727712176000},"page":"3371-3412","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":22,"title":["Toward the unification of generative and discriminative visual foundation model: a survey"],"prefix":"10.1007","volume":"41","author":[{"given":"Xu","family":"Liu","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Tong","family":"Zhou","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Chong","family":"Wang","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Yuping","family":"Wang","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Yuanxin","family":"Wang","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Qinjingwen","family":"Cao","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Weizhi","family":"Du","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Yonghuan","family":"Yang","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Junjun","family":"He","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Yu","family":"Qiao","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Yiqing","family":"Shen","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2024,9,30]]},"reference":[{"key":"3608_CR1","unstructured":"Bommasani, R., Hudson, D.A., Adeli, E., Altman, R., Arora, S., Arx, S., Bernstein, M.S., Bohg, J., Bosselut, A., Brunskill, E., et al.: On the opportunities and risks of foundation models (2022). arXiv preprint arXiv:2108.07258"},{"key":"3608_CR2","unstructured":"Devlin, J., Chang, M.-W., Lee, K., Toutanova, K.: BERT: pre-training of deep bidirectional transformers for language understanding (2019). arXiv preprint arXiv:1810.04805 [cs.CL]"},{"issue":"140","key":"3608_CR3","first-page":"1","volume":"21","author":"C Raffel","year":"2020","unstructured":"Raffel, C., Shazeer, N., Roberts, A., Lee, K., Narang, S., Matena, M., Zhou, Y., Li, W., Liu, P.J.: Exploring the limits of transfer learning with a unified text-to-text transformer. J. Mach. Learn. Res. 21(140), 1\u201367 (2020)","journal-title":"J. Mach. Learn. Res."},{"key":"3608_CR4","unstructured":"Ouyang, L., Wu, J., Jiang, X., Almeida, D., Wainwright, C.L., Mishkin, P., Zhang, C., Agarwal, S., Slama, K., Ray, A., et al.: Training language models to follow instructions with human feedback (2022). arXiv preprint arXiv:2203.02155 [cs.CL]"},{"key":"3608_CR5","doi-asserted-by":"crossref","unstructured":"Esser, P., Rombach, R., Ommer, B.: Taming transformers for high-resolution image synthesis. In: Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition, pp. 12873\u201312883 (2021)","DOI":"10.1109\/CVPR46437.2021.01268"},{"key":"3608_CR6","unstructured":"Yu, J., Xu, Y., Koh, J.Y., Luong, T., Baid, G., Wang, Z., Vasudevan, V., Ku, A., Yang, Y., Ayan, B.K.: Scaling autoregressive models for content-rich text-to-image generation, vol. 2, no. 3, pp. 5 (2022). arXiv preprint arXiv:2206.10789"},{"key":"3608_CR7","doi-asserted-by":"crossref","unstructured":"Zhang, H., Xu, T., Li, H., Zhang, S., Wang, X., Huang, X., Metaxas, D.N.: StackGAN: text to photo-realistic image synthesis with stacked generative adversarial networks. In: Proceedings of the IEEE International Conference on Computer Vision, pp. 5907\u20135915 (2017)","DOI":"10.1109\/ICCV.2017.629"},{"key":"3608_CR8","doi-asserted-by":"crossref","unstructured":"Nilsback, M.-E., Zisserman, A.: Automated flower classification over a large number of classes. In: 2008 Sixth Indian Conference on Computer Vision, Graphics & Image Processing, pp. 722\u2013729. IEEE (2008)","DOI":"10.1109\/ICVGIP.2008.47"},{"key":"3608_CR9","unstructured":"Schuhmann, C., Vencu, R., Beaumont, R., Kaczmarczyk, R., Mullis, C., Katta, A., Coombes, T., Jitsev, J., Borgeaud, A.: Laion-400m: open dataset of clip-filtered 400 million image-text pairs (2021). arXiv preprint arXiv:2111.02114 [cs.CV]"},{"key":"3608_CR10","unstructured":"Zhang, W., Pang, J., Chen, K., Loy, C.C.: K-Net: towards unified image segmentation. In: Advances in Neural Information Processing Systems, vol. 34, pp. 20417\u201320428 (2021)"},{"key":"3608_CR11","doi-asserted-by":"crossref","unstructured":"Kirillov, A., Mintun, E., Ravi, N., Mao, H., Rolland, C., Gustafson, L., Xiao, T., Whitehead, S., Berg, A.C., Lo, W.-Y., Doll\u00e1r, P., Girshick, R.: Segment anything (2023). arXiv preprint arXiv:2304.02643 [cs.CV]","DOI":"10.1109\/ICCV51070.2023.00371"},{"key":"3608_CR12","unstructured":"Reed, S., Akata, Z., Yan, X., Logeswaran, L., Schiele, B., Lee, H.: Generative adversarial text to image synthesis. In: Proceedings of the 33rd International Conference on Machine Learning. Proceedings of Machine Learning Research, vol. 48, pp. 1060\u20131069 (2016)"},{"key":"3608_CR13","doi-asserted-by":"crossref","unstructured":"Karras, T., Laine, S., Aila, T.: A style-based generator architecture for generative adversarial networks. In: Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition, pp. 4401\u20134410 (2019)","DOI":"10.1109\/CVPR.2019.00453"},{"key":"3608_CR14","unstructured":"Ramesh, A., Pavlov, M., Goh, G., Gray, S., Voss, C., Radford, A., Chen, M., Sutskever, I.: Zero-shot text-to-image generation. In: International Conference on Machine Learning, pp. 8821\u20138831. PMLE (2021)"},{"key":"3608_CR15","first-page":"19822","volume":"34","author":"M Ding","year":"2021","unstructured":"Ding, M., Yang, Z., Hong, W., Zheng, W., Zhou, C., Yin, D., Lin, J., Zou, X., Shao, Z., Yang, H.: Cogview: mastering text-to-image generation via transformers. Adv. Neural Inf. Process. Syst. 34, 19822\u201319835 (2021)","journal-title":"Adv. Neural Inf. Process. Syst."},{"key":"3608_CR16","doi-asserted-by":"crossref","unstructured":"Gafni, O., Polyak, A., Ashual, O., Sheynin, S., Parikh, D., Taigman, Y.: Make-a-scene: scene-based text-to-image generation with human priors. In: European Conference on Computer Vision, pp. 89\u2013106. Springer (2022)","DOI":"10.1007\/978-3-031-19784-0_6"},{"key":"3608_CR17","unstructured":"Ramesh, A., Dhariwal, P., Nichol, A., Chu, C., Chen, M.: Hierarchical text-conditional image generation with clip Latents, vol. 1, no. 2, p. 3 (2022). arXiv preprint arXiv:2204.06125"},{"key":"3608_CR18","first-page":"36479","volume":"35","author":"C Saharia","year":"2022","unstructured":"Saharia, C., Chan, W., Saxena, S., Li, L., Whang, J., Denton, E.L., Ghasemipour, K., Gontijo Lopes, R., Karagol Ayan, B., Salimans, T.: Photorealistic text-to-image diffusion models with deep language understanding. Adv. Neural Inf. Process. Syst. 35, 36479\u201336494 (2022)","journal-title":"Adv. Neural Inf. Process. Syst."},{"key":"3608_CR19","unstructured":"Nichol, A., Dhariwal, P., Ramesh, A., Shyam, P., Mishkin, P., McGrew, B., Sutskever, I., Chen, M.: GLIDE: towards photorealistic image generation and editing with text-guided diffusion models (2021). arXiv preprint arXiv:2112.10741"},{"key":"3608_CR20","doi-asserted-by":"crossref","unstructured":"Rombach, R., Blattmann, A., Lorenz, D., Esser, P., Ommer, B.: High-resolution image synthesis with latent diffusion models. In: Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition, pp. 10684\u201310695 (2022)","DOI":"10.1109\/CVPR52688.2022.01042"},{"key":"3608_CR21","doi-asserted-by":"crossref","unstructured":"Zhai, X., Kolesnikov, A., Houlsby, N., Beyer, L.: Scaling vision transformers (2022). arXiv preprint arXiv:2106.04560 [cs.CV]","DOI":"10.1109\/CVPR52688.2022.01179"},{"key":"3608_CR22","unstructured":"Dehghani, M., Djolonga, J., Mustafa, B., Padlewski, P., Heek, J., Gilmer, J., Steiner, A., Caron, M., Geirhos, R., Alabdulmohsin, I.M., et al.: Scaling vision transformers to 22 billion parameters (2023). arXiv preprint arXiv:2302.05442 [cs.CV]"},{"key":"3608_CR23","doi-asserted-by":"crossref","unstructured":"Liu, Z., Hu, H., Lin, Y., Yao, Z., Xie, Z., Wei, Y., Ning, J., Cao, Y., Zhang, Z., Dong, L., Wei, F., Guo, B.: Swin transformer V2: scaling up capacity and resolution (2022). arXiv preprint arXiv:2111.09883 [cs.CV]","DOI":"10.1109\/CVPR52688.2022.01170"},{"key":"3608_CR24","doi-asserted-by":"crossref","unstructured":"Wang, L., Huang, B., Zhao, Z., Tong, Z., He, Y., Wang, Y., Wang, Y., Qiao, Y.: VideoMAE V2: scaling video masked autoencoders with dual masking (2023). arXiv preprint arXiv:2303.16727 [cs.CV]","DOI":"10.1109\/CVPR52729.2023.01398"},{"key":"3608_CR25","unstructured":"Radford, A., Kim, J.W., Hallacy, C., Ramesh, A., Goh, G., Agarwal, S., Sastry, G., Askell, A., Mishkin, P., Clark, J.: Learning transferable visual models from natural language supervision. In: International Conference on Machine Learning, pp. 8748\u20138763. PMLR (2021)"},{"key":"3608_CR26","unstructured":"Jia, C., Yang, Y., Xia, Y., Chen, Y.-T., Parekh, Z., Pham, H., Le, Q., Sung, Y.-H., Li, Z., Duerig, T.: Scaling up visual and vision-language representation learning with noisy text supervision. In: International Conference on Machine Learning, pp. 4904\u20134916. PMLR (2021)"},{"key":"3608_CR27","unstructured":"Ng, A.Y., Jordan, M.I.: On discriminative vs. generative classifiers: a comparison of logistic regression and Naive Bayes. In: Advances in Neural Information Processing Systems, vol. 14, pp. 841\u2013848. MIT Press (2001)"},{"key":"3608_CR28","doi-asserted-by":"publisher","unstructured":"Hinton, G.E.: To recognize shapes, first learn to generate images. In: Computational Neuroscience: Theoretical Insights Into Brain Function. Progress in Brain Research, vol. 165, pp. 535\u2013547. Elsevier (2007). https:\/\/doi.org\/10.1016\/S0079-6123(06)65034-6","DOI":"10.1016\/S0079-6123(06)65034-6"},{"key":"3608_CR29","doi-asserted-by":"crossref","unstructured":"Li, D., Yang, J., Kreis, K., Torralba, A., Fidler, S.: Semantic segmentation with generative models: semi-supervised learning and strong out-of-domain generalization. In: Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition, pp. 8300\u20138311 (2021)","DOI":"10.1109\/CVPR46437.2021.00820"},{"key":"3608_CR30","unstructured":"Amit, T., Shaharbany, T., Nachmani, E., Wolf, L.: SegDIFF: image segmentation with diffusion probabilistic models (2022). arXiv preprint arXiv:2112.00390 [cs.CV]"},{"key":"3608_CR31","unstructured":"Baranchuk, D., Rubachev, I., Voynov, A., Khrulkov, V., Babenko, A.: Label-efficient semantic segmentation with diffusion models. In: International Conference on Learning Representations (2022)"},{"key":"3608_CR32","doi-asserted-by":"crossref","unstructured":"Chen, T., Li, L., Saxena, S., Hinton, G., Fleet, D.J.: A generalist framework for panoptic segmentation of images and videos (2023). arXiv preprint arXiv:2210.06366 [cs.CV]","DOI":"10.1109\/ICCV51070.2023.00090"},{"key":"3608_CR33","unstructured":"Wolleb, J., Sandk\u00fchler, R., Bieder, F., Valmaggia, P., Cattin, P.C.: Diffusion models for implicit image segmentation ensembles (2021). arXiv preprint arXiv:2112.03145 [cs.CV]"},{"key":"3608_CR34","doi-asserted-by":"crossref","unstructured":"Chen, S., Sun, P., Song, Y., Luo, P.: DiffusionDet: diffusion model for object detection (2023). arXiv preprint arXiv:2211.09788 [cs.CV]","DOI":"10.1109\/ICCV51070.2023.01816"},{"key":"3608_CR35","unstructured":"Zhou, Y., Shimada, N.: Vision + language applications: a survey (2023). arXiv preprint arXiv:2305.14598 [cs.CV]"},{"key":"3608_CR36","unstructured":"Zhang, C., Liu, L., Cui, Y., Huang, G., Lin, W., Yang, Y., Hu, Y.: A comprehensive survey on segment anything model for vision and beyond (2023). arXiv preprint arXiv:2305.08196 [cs.CV]"},{"key":"3608_CR37","unstructured":"Awais, M., Naseer, M., Khan, S., Anwer, R.M., Cholakkal, H., Shah, M., Yang, M.-H., Khan, F.S.: Foundational models defining a new era in vision: a survey and outlook (2023). arXiv preprint arXiv:2307.13721 [cs.CV]"},{"issue":"8","key":"3608_CR38","first-page":"9","volume":"1","author":"A Radford","year":"2019","unstructured":"Radford, A., Wu, J., Child, R., Luan, D., Amodei, D., Sutskever, I.: Language models are unsupervised multitask learners. OpenAI Blog 1(8), 9 (2019)","journal-title":"OpenAI Blog"},{"key":"3608_CR39","unstructured":"Brown, T.B., Mann, B., Ryder, N., Subbiah, M., Kaplan, J., Dhariwal, P., Neelakantan, A., Shyam, P., Sastry, G., Askell, A., et al.: Language models are few-shot learners (2020). arXiv preprint arXiv:2005.14165 [cs.CL]"},{"key":"3608_CR40","doi-asserted-by":"crossref","unstructured":"Kombrink, S., Mikolov, T., Karafi\u00e1t, M., Burget, L.: Recurrent neural network based language modeling in meeting recognition. In: Interspeech, vol. 11, pp. 2877\u20132880 (2011)","DOI":"10.21437\/Interspeech.2011-720"},{"key":"3608_CR41","doi-asserted-by":"crossref","unstructured":"Mikolov, T., Karafi\u00e1t, M., Burget, L., Cernock\u1ef3, J., Khudanpur, S.: Recurrent neural network based language model. In: Interspeech, vol. 2, pp. 1045\u20131048 (2010)","DOI":"10.21437\/Interspeech.2010-343"},{"key":"3608_CR42","unstructured":"Mikolov, T., Chen, K., Corrado, G., Dean, J.: Efficient estimation of word representations in vector space (2013). arXiv preprint arXiv:1301.3781 [cs.CL]"},{"key":"3608_CR43","doi-asserted-by":"crossref","unstructured":"Liu, P., Zhang, L., Gulla, J.A.: Pre-train, prompt and recommendation: a comprehensive survey of language modelling paradigm adaptations in recommender systems (2023). arXiv preprint arXiv:2302.03735 [cs.IR]","DOI":"10.1162\/tacl_a_00619"},{"key":"3608_CR44","doi-asserted-by":"crossref","unstructured":"Lewis, M., Liu, Y., Goyal, N., Ghazvininejad, M., Mohamed, A., Levy, O., Stoyanov, V., Zettlemoyer, L.: BART: denoising sequence-to-sequence pre-training for natural language generation, translation, and comprehension. In: Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics, pp. 7871\u20137880. Association for Computational Linguistics (2020)","DOI":"10.18653\/v1\/2020.acl-main.703"},{"key":"3608_CR45","unstructured":"Liu, Y., Ott, M., Goyal, N., Du, J., Joshi, M., Chen, D., Levy, O., Lewis, M., Zettlemoyer, L., Stoyanov, V.: RoBERTa: a robustly optimized BERT pretraining approach (2019). arXiv preprint arXiv:1907.11692 [cs.CL]"},{"key":"3608_CR46","unstructured":"Sanh, V., Webson, A., Raffel, C., Bach, S.H., Sutawika, L., Alyafeai, Z., Chaffin, A., Stiegler, A., Scao, T.L., Raja, A., Dey, M., Bari, M.S., Xu, C., Thakker, U., Sharma, S.S., Szczechla, E., Kim, T., Chhablani, G., Nayak, N.V., Datta, D., Chang, J., Jiang, M.T.-J., Wang, H., Manica, M., Shen, S., Yong, Z.X., Pandey, H., Bawden, R., Wang, T., Neeraj, T., Rozen, J., Sharma, A., Santilli, A., Fevry, T., Fries, J.A., Teehan, R., Biderman, S., Gao, L., Bers, T., Wolf, T., Rush, A.M.: Multitask prompted training enables zero-shot task generalization (2022). arXiv preprint arXiv:2110.08207 [cs.LG]"},{"key":"3608_CR47","unstructured":"Wang, T., Roberts, A., Hesslow, D., Scao, T.L., Chung, H.W., Beltagy, I., Launay, J., Raffel, C.: What language model architecture and pretraining objective work best for zero-shot generalization? (2022). arXiv preprint arXiv:2204.05832 [cs.CL]"},{"key":"3608_CR48","unstructured":"Kaplan, J., McCandlish, S., Henighan, T., Brown, T.B., Chess, B., Child, R., Gray, S., Radford, A., Wu, J., Amodei, D.: Scaling laws for neural language models (2020). arXiv preprint arXiv:2001.08361 [cs.LG]"},{"key":"3608_CR49","unstructured":"Zhao, W.X., Zhou, K., Li, J., Tang, T., Wang, X., Hou, Y., Min, Y., Zhang, B., Zhang, J., Dong, Z., Du, Y., Chen, Y., Chen, Z., Jiang, J., Ren, R., Li, Y., Tang, X., Liu, Z., Liu, P., Nie, J., Wen, J.-R.: A survey of large language models (2023). arXiv preprint arXiv:2303.18223"},{"key":"3608_CR50","doi-asserted-by":"crossref","unstructured":"Li, J., Tang, T., Zhao, W.X., Wen, J.-R.: Pretrained language models for text generation: a survey (2021). arXiv preprint arXiv:2105.10311 [cs.CL]","DOI":"10.24963\/ijcai.2021\/612"},{"key":"3608_CR51","unstructured":"Bahdanau, D., Cho, K., Bengio, Y.: Neural machine translation by jointly learning to align and translate (2015). arXiv preprint arXiv:1409.0473"},{"key":"3608_CR52","doi-asserted-by":"crossref","unstructured":"Rush, A.M., Chopra, S., Weston, J.: A neural attention model for abstractive sentence summarization. In: Proceedings of the 2015 Conference on Empirical Methods in Natural Language Processing, pp. 379\u2013389 (2015)","DOI":"10.18653\/v1\/D15-1044"},{"key":"3608_CR53","doi-asserted-by":"crossref","unstructured":"Chen, D., Fisch, A., Weston, J., Bordes, A.: Reading Wikipedia to answer open-domain questions. In: Proceedings of the 55th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers), pp. 1870\u20131879 (2017)","DOI":"10.18653\/v1\/P17-1171"},{"key":"3608_CR54","unstructured":"Zhou, D., Sch\u00e4rli, N., Hou, L., Wei, J., Scales, N., Wang, X., Schuurmans, D., Cui, C., Bousquet, O., Le, Q., Chi, E.: Least-to-most prompting enables complex reasoning in large language models (2023). arXiv preprint arXiv:2205.10625 [cs.AI]"},{"issue":"140","key":"3608_CR55","first-page":"1","volume":"21","author":"C Raffel","year":"2020","unstructured":"Raffel, C., Shazeer, N., Roberts, A., Lee, K., Narang, S., Matena, M., Zhou, Y., Li, W., Liu, P.J.: Exploring the limits of transfer learning with a unified text-to-text transformer. J. Mach. Learn. Res. 21(140), 1\u201367 (2020)","journal-title":"J. Mach. Learn. Res."},{"key":"3608_CR56","unstructured":"Amatriain, X., Sankar, A., Bing, J., Bodigutla, P.K., Hazen, T.J., Kazi, M.: Transformer models: an introduction and catalog (2023). arXiv preprint arXiv:2302.07730 [cs.CL]"},{"key":"3608_CR57","unstructured":"Soltan, S., Ananthakrishnan, S., FitzGerald, J., Gupta, R., Hamza, W., Khan, H., Peris, C., Rawls, S., Rosenbaum, A., Rumshisky, A., Satya\u00a0Prakash, C., Sridhar, M., Triefenbach, F., Verma, A., Tur, G., Natarajan, P.: Alexatm 20b: few-shot learning using a large-scale multilingual seq2seq model (2022). arXiv preprint arXiv:2208.01448 [cs.CL]"},{"key":"3608_CR58","doi-asserted-by":"crossref","unstructured":"Wang, Z., Li, Y., Chen, X., Lim, S.-N., Torralba, A., Zhao, H., Wang, S.: Detecting everything in the open world: towards universal object detection (2023). arXiv preprint arXiv:2303.11749 [cs.CV]","DOI":"10.1109\/CVPR52729.2023.01100"},{"key":"3608_CR59","unstructured":"Yuan, L., Chen, D., Chen, Y.-L., Codella, N., Dai, X., Gao, J., Hu, H., Huang, X., Li, B., Li, C., et al.: Florence: a new foundation model for computer vision (2021). arXiv preprint arXiv:2111.11432"},{"key":"3608_CR60","unstructured":"Su, W., Zhu, X., Cao, Y., Li, B., Lu, L., Wei, F., Dai, J.: VL-BERT: pre-training of generic visual-linguistic representations. In: International Conference on Learning Representations (2020)"},{"key":"3608_CR61","doi-asserted-by":"crossref","unstructured":"Cho, J., Lu, J., Schwenk, D., Hajishirzi, H., Kembhavi, A.: X-LXMERT: paint, caption and answer questions with multi-modal transformers. In: Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing (EMNLP), pp. 8785\u20138805 (2020)","DOI":"10.18653\/v1\/2020.emnlp-main.707"},{"key":"3608_CR62","unstructured":"Mirza, M., Osindero, S.: Conditional generative adversarial nets (2014). arXiv preprint arXiv:1411.1784"},{"key":"3608_CR63","unstructured":"Singer, U., Polyak, A., Hayes, T., Yin, X., An, J., Zhang, S., Hu, Q., Yang, H., Ashual, O., Gafni, O., Parikh, D., Gupta, S., Taigman, Y.: Make-a-video: text-to-video generation without text-video data (2022) arXiv:2209.14792 [cs.CV]"},{"key":"3608_CR64","unstructured":"Ho, J., Chan, W., Saharia, C., Whang, J., Gao, R., Gritsenko, A., Kingma, D.P., Poole, B., Norouzi, M., Fleet, D.J., Salimans, T.: Imagen video: high definition video generation with diffusion models (2022) arXiv:2210.02303 [cs.CV]"},{"key":"3608_CR65","unstructured":"Tumanyan, N., Geyer, M., Bagon, S., Dekel, T.: Plug-and-play diffusion features for text-driven image-to-image translation. In: Advances in Neural Information Processing Systems, vol. 35, pp. 29248\u201329262 (2022)"},{"key":"3608_CR66","doi-asserted-by":"crossref","unstructured":"Isola, P., Zhu, J.-Y., Zhou, T., Efros, A.A.: Image-to-image translation with conditional adversarial networks. In: Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition, pp. 1125\u20131134 (2017)","DOI":"10.1109\/CVPR.2017.632"},{"key":"3608_CR67","doi-asserted-by":"crossref","unstructured":"Richardson, E., Alaluf, Y., Patashnik, O., Nitzan, Y., Azar, Y., Shapiro, S., Cohen-Or, D.: Encoding in style: a StyleGAN encoder for image-to-image translation. In: Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition (CVPR), pp. 2287\u20132296 (2021)","DOI":"10.1109\/CVPR46437.2021.00232"},{"key":"3608_CR68","first-page":"1","volume":"32","author":"A Razavi","year":"2019","unstructured":"Razavi, A., Oord, A., Vinyals, O.: Generating diverse high-fidelity images with VQ-VAE-2. Adv. Neural Inf. Process. Syst. 32, 1\u201311 (2019)","journal-title":"Adv. Neural Inf. Process. Syst."},{"key":"3608_CR69","unstructured":"Dhariwal, P., Nichol, A.: Diffusion models beat GANs on image synthesis. In: Advances in Neural Information Processing Systems, vol. 34, pp. 8780\u20138794. Curran Associates, Inc. (2021)"},{"key":"3608_CR70","doi-asserted-by":"crossref","unstructured":"Lugmayr, A., Danelljan, M., Romero, A., Yu, F., Timofte, R., Van\u00a0Gool, L.: Repaint: inpainting using denoising diffusion probabilistic models. In: Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition, pp. 11451\u201311461 (2022)","DOI":"10.1109\/CVPR52688.2022.01117"},{"key":"3608_CR71","doi-asserted-by":"crossref","unstructured":"Li, H., Yang, Y., Chang, M., Feng, H., Xu, Z., Li, Q., Chen, Y.: SRDiff: single image super-resolution with diffusion probabilistic models (2021). arXiv preprint arXiv:2104.14951 [cs.CV]","DOI":"10.1016\/j.neucom.2022.01.029"},{"key":"3608_CR72","doi-asserted-by":"crossref","unstructured":"Saharia, C., Ho, J., Chan, W., Salimans, T., Fleet, D.J., Norouzi, M.: Image super-resolution via iterative refinement (2021). arXiv preprint arXiv:2104.07636 [eess.IV]","DOI":"10.1109\/TPAMI.2022.3204461"},{"key":"3608_CR73","unstructured":"Chandramouli, P., Gandikota, K.V.: LDEdit: towards generalized text guided image manipulation via latent diffusion models (2022). arXiv preprint arXiv:2210.02249 [cs.CV]"},{"key":"3608_CR74","unstructured":"Couairon, G., Verbeek, J., Schwenk, H., Cord, M.: DiffEdit: diffusion-based semantic image editing with mask guidance (2023)"},{"key":"3608_CR75","unstructured":"Kingma, D.P., Welling, M.: Auto-encoding variational Bayes (2013) arXiv:1312.6114 [cs.ML]"},{"key":"3608_CR76","unstructured":"Goodfellow, I., Pouget-Abadie, J., Mirza, M., Xu, B., Warde-Farley, D., Ozair, S., Courville, A., Bengio, Y.: Generative adversarial nets. In: Advances in Neural Information Processing Systems, vol. 27 (2014)"},{"key":"3608_CR77","unstructured":"Sohl-Dickstein, J., Weiss, E., Maheswaranathan, N., Ganguli, S.: Deep unsupervised learning using nonequilibrium thermodynamics. In: International Conference on Machine Learning, pp. 2256\u20132265. PMLR (2015)"},{"key":"3608_CR78","first-page":"6840","volume":"33","author":"J Ho","year":"2020","unstructured":"Ho, J., Jain, A., Abbeel, P.: Denoising diffusion probabilistic models. Adv. Neural Inf. Process. Syst. 33, 6840\u20136851 (2020)","journal-title":"Adv. Neural Inf. Process. Syst."},{"key":"3608_CR79","unstructured":"Song, J., Meng, C., Ermon, S.: Denoising diffusion implicit models (2020). arXiv preprint arXiv:2010.02502"},{"key":"3608_CR80","unstructured":"Song, Y., Sohl-Dickstein, J., Kingma, D.P., Kumar, A., Ermon, S., Poole, B.: Score-based generative modeling through stochastic differential equations (2020). arXiv preprint arXiv:2011.13456"},{"key":"3608_CR81","unstructured":"Jolicoeur-Martineau, A., Pich\u00e9-Taillefer, R., Combes, R.T.D., Mitliagkas, I.: Adversarial score matching and improved sampling for image generation (2020). arXiv preprint arXiv:2009.05475"},{"key":"3608_CR82","unstructured":"Van Den\u00a0Oord, A., Vinyals, O., et al.: Neural discrete representation learning. Adv. Neural Inf. Process. Syst. 30, 1\u201310(2017)"},{"key":"3608_CR83","unstructured":"Lucas, J., Tucker, G., Grosse, R.B., Norouzi, M.: Understanding posterior collapse in generative latent variable models. In: International Conference on Learning Representations (2019)"},{"key":"3608_CR84","doi-asserted-by":"crossref","unstructured":"Gu, S., Chen, D., Bao, J., Wen, F., Zhang, B., Chen, D., et al.: Vector quantized diffusion model for text-to-image synthesis (2022) arXiv:2111.14822 [cs.CV]","DOI":"10.1109\/CVPR52688.2022.01043"},{"key":"3608_CR85","unstructured":"Child, R.: Very deep VAEs generalize autoregressive models and can outperform them on images (2020). arXiv preprint arXiv:2011.10650"},{"key":"3608_CR86","unstructured":"Pandey, K., Mukherjee, A., Rai, P., Kumar, A.: DiffuseVAE: efficient, controllable and high-fidelity generation from low-dimensional Latents (2022). arXiv preprint arXiv:2201.00308"},{"key":"3608_CR87","unstructured":"Odena, A., Olah, C., Shlens, J.: Conditional image synthesis with auxiliary classifier GANs. In: Proceedings of the 34th International Conference on Machine Learning, pp. 2642\u20132651 (2017)"},{"issue":"8","key":"3608_CR88","doi-asserted-by":"publisher","first-page":"1947","DOI":"10.1109\/TPAMI.2018.2856256","volume":"41","author":"H Zhang","year":"2019","unstructured":"Zhang, H., Xu, T., Li, H., Zhang, S., Wang, X., Huang, X., Metaxas, D.N.: StackGan++: realistic image synthesis with stacked generative adversarial networks. IEEE Trans. Pattern Anal. Mach. Intell. 41(8), 1947\u20131962 (2019). https:\/\/doi.org\/10.1109\/TPAMI.2018.2856256","journal-title":"IEEE Trans. Pattern Anal. Mach. Intell."},{"key":"3608_CR89","doi-asserted-by":"crossref","unstructured":"Zhang, Z., Xie, Y., Yang, L.: Photographic text-to-image synthesis with a hierarchically-nested adversarial network. In: Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition, pp. 6199\u20136208 (2018)","DOI":"10.1109\/CVPR.2018.00649"},{"key":"3608_CR90","doi-asserted-by":"crossref","unstructured":"Wang, T.-C., Liu, M.-Y., Zhu, J.-Y., Tao, A., Kautz, J., Catanzaro, B.: High-resolution image synthesis and semantic manipulation with conditional GANs. In: Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition, pp. 8798\u20138807 (2018)","DOI":"10.1109\/CVPR.2018.00917"},{"key":"3608_CR91","doi-asserted-by":"crossref","unstructured":"Xu, T., Zhang, P., Huang, Q., Zhang, H., Gan, Z., Huang, X., He, X.: AttnGAN: fine-grained text to image generation with attentional generative adversarial networks. In: Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition, pp. 1316\u20131324 (2018)","DOI":"10.1109\/CVPR.2018.00143"},{"key":"3608_CR92","doi-asserted-by":"crossref","unstructured":"Karras, T., Laine, S., Aittala, M., Hellsten, J., Lehtinen, J., Aila, T.: Analyzing and improving the image quality of StyleGAN. In: Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition, pp. 8110\u20138119 (2020)","DOI":"10.1109\/CVPR42600.2020.00813"},{"key":"3608_CR93","unstructured":"Brock, A.: Large scale GAN training for high fidelity natural image synthesis (2018). arXiv preprint arXiv:1809.11096"},{"key":"3608_CR94","doi-asserted-by":"crossref","unstructured":"Zhu, M., Pan, P., Chen, W., Yang, Y.: DM-GAN: dynamic memory generative adversarial networks for text-to-image synthesis. In: Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition, pp. 5802\u20135810 (2019)","DOI":"10.1109\/CVPR.2019.00595"},{"key":"3608_CR95","doi-asserted-by":"crossref","unstructured":"Li, W., Zhang, P., Zhang, L., Huang, Q., He, X., Lyu, S., Gao, J.: Object-driven text-to-image synthesis via adversarial training. In: Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition, pp. 12174\u201312182 (2019)","DOI":"10.1109\/CVPR.2019.01245"},{"key":"3608_CR96","unstructured":"Li, B., Qi, X., Lukasiewicz, T., Torr, P.H.S.: Controllable text-to-image generation. In: Advances in Neural Information Processing Systems, pp. 2063\u20132073 (2019)"},{"key":"3608_CR97","unstructured":"Ho, J., Salimans, T.: Classifier-free diffusion guidance (2022). arXiv preprint arXiv:2207.12598"},{"key":"3608_CR98","unstructured":"Li, W., Xu, X., Xiao, X., Liu, J., Yang, H., Li, G., Wang, Z., Feng, Z., She, Q., Lyu, Y., et al.: UPainting: unified text-to-image diffusion generation with cross-modal guidance (2022). arXiv preprint arXiv:2210.16031"},{"key":"3608_CR99","doi-asserted-by":"crossref","unstructured":"Avrahami, O., Lischinski, D., Fried, O.: Blended diffusion for text-driven editing of natural images. In: Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition, pp. 18208\u201318218 (2022)","DOI":"10.1109\/CVPR52688.2022.01767"},{"key":"3608_CR100","doi-asserted-by":"crossref","unstructured":"Fan, W.-C., Chen, Y.-C., Chen, D., Cheng, Y., Yuan, L., Wang, Y.-C.F.: Frido: Feature pyramid diffusion for complex scene image synthesis. In: Proceedings of the AAAI Conference on Artificial Intelligence, vol. 37, pp. 579\u2013587 (2023)","DOI":"10.1609\/aaai.v37i1.25133"},{"key":"3608_CR101","doi-asserted-by":"crossref","unstructured":"Xu, X., Wang, Z., Zhang, G., Wang, K., Shi, H.: Versatile diffusion: text, images and variations all in one diffusion model (2023). In: Proceedings of the IEEE\/CVF International Conference on Computer Vision, pp. 7754\u20137765","DOI":"10.1109\/ICCV51070.2023.00713"},{"key":"3608_CR102","unstructured":"Bao, F., Nie, S., Xue, K., Li, C., Pu, S., Wang, Y., Yue, G., Cao, Y., Su, H., Zhu, J.: One transformer fits all distributions in multi-modal diffusion at scale. In: International Conference on Machine Learning, pp. 1692\u20131717. PMLR (2023)"},{"key":"3608_CR103","unstructured":"Sheynin, S., Ashual, O., Polyak, A., Singer, U., Gafni, O., Nachmani, E., Taigman, Y.: KNN-diffusion: image generation via large-scale retrieval (2022). arXiv preprint arXiv:2204.02849"},{"key":"3608_CR104","unstructured":"Valevski, D., Kalman, M., Matias, Y., Leviathan, Y.: UniTune: text-driven image editing by fine tuning an image generation model on a single imagen, vol.2, no.3, p. 5 (2022). arXiv preprint arXiv:2210.09477"},{"key":"3608_CR105","doi-asserted-by":"crossref","unstructured":"Kim, G., Kwon, T., Ye, J.C.: DiffusionCLIP: text-guided diffusion models for robust image manipulation. In: Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition, pp. 2426\u20132435 (2022)","DOI":"10.1109\/CVPR52688.2022.00246"},{"key":"3608_CR106","doi-asserted-by":"crossref","unstructured":"Kawar, B., Zada, S., Lang, O., Tov, O., Chang, H., Dekel, T., Mosseri, I., Irani, M.: Imagic: text-based real image editing with diffusion models (2023). In: Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition, pp. 6007\u20136017","DOI":"10.1109\/CVPR52729.2023.00582"},{"key":"3608_CR107","doi-asserted-by":"crossref","unstructured":"Ruiz, N., Li, Y., Jampani, V., Pritch, Y., Rubinstein, M., Aberman, K.: DreamBooth: fine tuning text-to-image diffusion models for subject-driven generation. In: Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition, pp. 22500\u201322510 (2023)","DOI":"10.1109\/CVPR52729.2023.02155"},{"key":"3608_CR108","unstructured":"Balaji, Y., Nah, S., Huang, X., Vahdat, A., Song, J., Zhang, Q., Kreis, K., Aittala, M., Aila, T., Laine, S., et al.: eDiff-I: text-to-image diffusion models with an ensemble of expert denoisers (2022). arXiv preprint arXiv:2211.01324"},{"key":"3608_CR109","doi-asserted-by":"crossref","unstructured":"Feng, Z., Zhang, Z., Yu, X., Fang, Y., Li, L., Chen, X., Lu, Y., Liu, J., Yin, W., Feng, S.: ERNIE-ViLG 2.0: improving text-to-image diffusion model with knowledge-enhanced mixture-of-denoising-experts. In: Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition, pp. 10135\u201310145 (2023)","DOI":"10.1109\/CVPR52729.2023.00977"},{"key":"3608_CR110","unstructured":"Welinder, P., Branson, S., Mita, T., Wah, C., Schroff, F., Belongie, S., et al.: Caltech-UCSD Birds 200. Technical Report CNS-TR-201, Caltech (2010)"},{"key":"3608_CR111","doi-asserted-by":"crossref","unstructured":"Lin, T.-Y., Maire, M., Belongie, S., Hays, J., Perona, P., Ramanan, D., Doll\u00e1r, P., Zitnick, C.L.: Microsoft coco: common objects in context. In: Computer Vision\u2013ECCV 2014: 13th European Conference, Zurich, Switzerland, September 6\u201312, 2014, Proceedings, Part V 13, pp. 740\u2013755. Springer (2014)","DOI":"10.1007\/978-3-319-10602-1_48"},{"key":"3608_CR112","doi-asserted-by":"crossref","unstructured":"Cho, J., Zala, A., Bansal, M.: DALL-Eval: probing the reasoning skills and social biases of text-to-image generation models. In: Proceedings of the IEEE\/CVF International Conference on Computer Vision, pp. 3043\u20133054 (2023)","DOI":"10.1109\/ICCV51070.2023.00283"},{"key":"3608_CR113","unstructured":"Chen, W., Hu, H., Saharia, C., Cohen, W.W.: Re-imagen: Retrieval-augmented text-to-image generator (2022). arXiv preprint arXiv:2209.14491"},{"key":"3608_CR114","unstructured":"Petsiuk, V., Siemenn, A.E., Surbehera, S., Chin, Z., Tyser, K., Hunter, G., Raghavan, A., Hicke, Y., Plummer, B.A., Kerret, O., et al.: Human evaluation of text-to-image models on a multi-task benchmark. arXiv preprint arXiv:2211.12112 (2022)"},{"key":"3608_CR115","unstructured":"Liao, P., Li, X., Liu, X., Keutzer, K.: The artbench dataset: benchmarking generative models with artworks (2022). arXiv preprint arXiv:2206.11404"},{"key":"3608_CR116","doi-asserted-by":"crossref","unstructured":"Sharma, P., Ding, N., Goodman, S., Soricut, R.: Conceptual captions: a cleaned, hypernymed, image alt-text dataset for automatic image captioning (2018). In: Proceedings of the 56th Annual Meeting of the Association for Computational Linguistics, pp. 2556\u20132565","DOI":"10.18653\/v1\/P18-1238"},{"key":"3608_CR117","doi-asserted-by":"crossref","unstructured":"Changpinyo, S., Sharma, P., Ding, N., Soricut, R.: Conceptual 12m: pushing web-scale image-text pre-training to recognize long-tail visual concepts (2021). arXiv preprint arXiv:2102.08981 [cs.CV]","DOI":"10.1109\/CVPR46437.2021.00356"},{"key":"3608_CR118","unstructured":"Wu, X., Hao, Y., Sun, K., Chen, Y., Zhu, F., Zhao, R., Li, H.: Human preference score v2: a solid benchmark for evaluating human preferences of text-to-image synthesis (2023) arXiv:2306.09341 [cs.CV]"},{"key":"3608_CR119","unstructured":"Heusel, M., Ramsauer, H., Unterthiner, T., Nessler, B., Hochreiter, S.: GANs trained by a two time-scale update rule converge to a local Nash equilibrium. Adv. Neural Inf. Process. Syst. 30, 1\u201312 (2017)"},{"key":"3608_CR120","unstructured":"Salimans, T., Goodfellow, I., Zaremba, W., Cheung, V., Radford, A., Chen, X.: Improved techniques for training GANs. Adv. Neural Inf. Process. Syst. 29, 1\u20139 (2016)"},{"key":"3608_CR121","unstructured":"Sajjadi, M.S.M., Bachem, O., Lucic, M., Bousquet, O., Gelly, S.: Assessing generative models via precision and recall. In: Advances in Neural Information Processing Systems, pp. 5228\u20135237 (2018)"},{"key":"3608_CR122","unstructured":"Kynk\u00e4\u00e4nniemi, T., Karras, T., Laine, S., Lehtinen, J., Aila, T.: Improved precision and recall metric for assessing generative models, pp. 3927\u20133936 (2019)"},{"key":"3608_CR123","doi-asserted-by":"crossref","unstructured":"Hessel, J., Holtzman, A., Forbes, M., Le\u00a0Bras, R., Choi, Y.: CLIPScore: a reference-free evaluation metric for image captioning (2021). arXiv preprint arXiv:2104.08718 [cs.CV]","DOI":"10.18653\/v1\/2021.emnlp-main.595"},{"key":"3608_CR124","unstructured":"DALL E 2. https:\/\/openai.com\/product\/dall-e-2"},{"key":"3608_CR125","unstructured":"Midjourney model versions. https:\/\/docs.midjourney.com\/docs\/model-versions"},{"key":"3608_CR126","unstructured":"Stable Diffusion Launch Announcement. https:\/\/stability.ai\/blog\/stable-diffusion-announcement (2022). Accessed 2024 Mar 30"},{"key":"3608_CR127","unstructured":"Stable Diffusion XL. https:\/\/ja.stability.ai\/stable-diffusion"},{"key":"3608_CR128","doi-asserted-by":"crossref","unstructured":"Esser, P., Chiu, J., Atighehchian, P., Granskog, J., Germanidis, A.: Structure and content-guided video synthesis with diffusion models (2023). arXiv preprint arXiv:2302.03011 [cs.CV]","DOI":"10.1109\/ICCV51070.2023.00675"},{"key":"3608_CR129","doi-asserted-by":"crossref","unstructured":"Pan, Y., Qiu, Z., Yao, T., Li, H., Mei, T.: To create what you tell: generating videos from captions. In: Proceedings of the 25th ACM international conference on Multimedia (2017)","DOI":"10.1145\/3123266.3127905"},{"key":"3608_CR130","unstructured":"Poole, B., Jain, A., Barron, J.T., Mildenhall, B.: DreamFusion: text-to-3D using 2D diffusion (2022) arXiv:2209.14988 [cs.CV]"},{"key":"3608_CR131","doi-asserted-by":"crossref","unstructured":"Luo, Z., Chen, D., Zhang, Y., Huang, Y., Wang, L., Shen, Y., Zhao, D., Zhou, J., Tan, T.: VideoFusion: decomposed diffusion models for high-quality video generation. In: Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition (2023)","DOI":"10.1109\/CVPR52729.2023.10308948"},{"key":"3608_CR132","doi-asserted-by":"crossref","unstructured":"Lin, C.-H., Gao, J., Tang, L., Takikawa, T., Zeng, X., Huang, X., Kreis, K., Fidler, S., Liu, M.-Y., Lin, T.-Y.: Magic3D: high-resolution text-to-3d content creation. In: Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition (2023)","DOI":"10.1109\/CVPR52729.2023.00037"},{"key":"3608_CR133","doi-asserted-by":"crossref","unstructured":"Chen, Y., Pan, Y., Li, Y., Yao, T., Mei, T.: Control3D: towards controllable text-to-3D generation. In: Proceedings of the 31st ACM International Conference on Multimedia (2023)","DOI":"10.1145\/3581783.3612489"},{"key":"3608_CR134","doi-asserted-by":"crossref","unstructured":"Yang, H., Chen, Y., Pan, Y., Yao, T., Chen, Z., Mei, T.: 3DStyle-diffusion: pursuing fine-grained text-driven 3D stylization with 2D diffusion models (2023)","DOI":"10.1145\/3581783.3612363"},{"key":"3608_CR135","doi-asserted-by":"crossref","unstructured":"Zhang, Z., Long, F., Pan, Y., Qiu, Z., Yao, T., Cao, Y., Mei, T.: Trip: temporal residual learning with image noise prior for image-to-video diffusion models (2024) arXiv:2403.17005 [cs.CV]","DOI":"10.1109\/CVPR52733.2024.00828"},{"key":"3608_CR136","doi-asserted-by":"crossref","unstructured":"Deng, J., Dong, W., Socher, R., Li, L.-J., Li, K., Fei-Fei, L.: Imagenet: a large-scale hierarchical image database. In: 2009 IEEE Conference on Computer Vision and Pattern Recognition, pp. 248\u2013255. IEEE (2009)","DOI":"10.1109\/CVPR.2009.5206848"},{"issue":"1","key":"3608_CR137","doi-asserted-by":"publisher","first-page":"74","DOI":"10.1007\/s11263-016-0965-7","volume":"123","author":"BA Plummer","year":"2017","unstructured":"Plummer, B.A., Wang, L., Cervantes, C.M., Caicedo, J.C., Hockenmaier, J., Lazebnik, S.: Flickr30k entities: collecting region-to-phrase correspondences for richer image-to-sentence models. Int. J. Comput. Vis. 123(1), 74\u201393 (2017)","journal-title":"Int. J. Comput. Vis."},{"key":"3608_CR138","doi-asserted-by":"crossref","unstructured":"Long, J., Shelhamer, E., Darrell, T.: Fully convolutional networks for semantic segmentation. In: Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition, pp. 3431\u20133440 (2015)","DOI":"10.1109\/CVPR.2015.7298965"},{"issue":"11","key":"3608_CR139","first-page":"3692","volume":"43","author":"A Kirillov","year":"2021","unstructured":"Kirillov, A., He, K., Girshick, R., Rother, C., Doll\u00e1r, P.: Panoptic segmentation. IEEE Trans. Pattern Anal. Mach. Intell. 43(11), 3692\u20133711 (2021)","journal-title":"IEEE Trans. Pattern Anal. Mach. Intell."},{"key":"3608_CR140","unstructured":"Krizhevsky, A., Sutskever, I., Hinton, G.E.: Imagenet classification with deep convolutional neural networks. In: Advances in Neural Information Processing Systems, vol. 25, pp. 1097\u20131105 (2012)"},{"key":"3608_CR141","doi-asserted-by":"crossref","unstructured":"He, K., Zhang, X., Ren, S., Sun, J.: Deep residual learning for image recognition (2015). arXiv preprint arXiv:1512.03385 [cs.CV]","DOI":"10.1109\/CVPR.2016.90"},{"key":"3608_CR142","unstructured":"Dosovitskiy, A., Beyer, L., Kolesnikov, A., Weissenborn, D., Zhai, X., Unterthiner, T., Dehghani, M., Minderer, M., Heigold, G., Gelly, S., Uszkoreit, J., Houlsby, N.: An image is worth 16 x 16 words: transformers for image recognition at scale. In: International Conference on Learning Representations (2021)"},{"key":"3608_CR143","doi-asserted-by":"crossref","unstructured":"Li, C., Gan, Z., Yang, Z., Yang, J., Li, L., Wang, L., Gao, J.: Multimodal foundation models: from specialists to general-purpose assistants (2023). arXiv preprint arXiv:2309.10020 [cs.CV]","DOI":"10.1561\/9781638283379"},{"key":"3608_CR144","unstructured":"Zou, X., Yang, J., Zhang, H., Li, F., Li, L., Wang, J., Gao, J., Lee, Y.J.: Segment everything everywhere all at once (2023). arXiv preprint arXiv:2304.06718 [cs.CV]"},{"key":"3608_CR145","doi-asserted-by":"crossref","unstructured":"Liu, Z., Lin, Y., Cao, Y., Hu, H., Wei, Y., Zhang, Z., Lin, S., Guo, B.: Swin transformer: hierarchical vision transformer using shifted windows. In: Proceedings of the IEEE\/CVF International Conference on Computer Vision, pp. 10012\u201310022 (2021)","DOI":"10.1109\/ICCV48922.2021.00986"},{"key":"3608_CR146","doi-asserted-by":"crossref","unstructured":"Carion, N., Massa, F., Synnaeve, G., Usunier, N., Kirillov, A., Zagoruyko, S.: End-to-end object detection with transformers. In: European Conference on Computer Vision, pp. 213\u2013229. Springer (2020)","DOI":"10.1007\/978-3-030-58452-8_13"},{"key":"3608_CR147","unstructured":"Zhu, X., Su, W., Lu, L., Li, B., Wang, X., Dai, J.: Deformable DETR: deformable transformers for end-to-end object detection. In: International Conference on Learning Representations (2021)"},{"key":"3608_CR148","doi-asserted-by":"crossref","unstructured":"Zheng, S., Lu, J., Zhao, H., Zhu, X., Luo, Z., Wang, Y., Fu, Y., Feng, J., Xiang, T., Torr, P.H.S., Zhang, L.: Rethinking semantic segmentation from a sequence-to-sequence perspective with transformers. In: Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition, pp. 6881\u20136890 (2021)","DOI":"10.1109\/CVPR46437.2021.00681"},{"key":"3608_CR149","unstructured":"Xie, E., Wang, W., Yu, Z., Anandkumar, A., Alvarez, J.M., Luo, P.: SegFormer: simple and efficient design for semantic segmentation with transformers. In: Advances in Neural Information Processing Systems, vol. 34, pp. 1236\u20131248 (2021)"},{"key":"3608_CR150","doi-asserted-by":"crossref","unstructured":"Mu, N., Kirillov, A., Wagner, D., Xie, S.: Slip: self-supervision meets language-image pre-training. In: European Conference on Computer Vision, pp. 772\u2013788. Springer (2022)","DOI":"10.1007\/978-3-031-19809-0_30"},{"key":"3608_CR151","doi-asserted-by":"crossref","unstructured":"He, K., Fan, H., Wu, Y., Xie, S., Girshick, R.: Momentum contrast for unsupervised visual representation learning. In: Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition, pp. 9729\u20139738 (2020)","DOI":"10.1109\/CVPR42600.2020.00975"},{"key":"3608_CR152","unstructured":"Chen, T., Kornblith, S., Norouzi, M., Hinton, G.: A simple framework for contrastive learning of visual representations. In: Proceedings of the 37th International Conference on Machine Learning. Proceedings of Machine Learning Research, vol. 119, pp. 1597\u20131607. PMLR (2020)"},{"key":"3608_CR153","doi-asserted-by":"crossref","unstructured":"Pathak, D., Krahenbuhl, P., Donahue, J., Darrell, T., Efros, A.A.: Context encoders: feature learning by inpainting. In: Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition, pp. 2536\u20132544 (2016)","DOI":"10.1109\/CVPR.2016.278"},{"key":"3608_CR154","doi-asserted-by":"crossref","unstructured":"He, K., Chen, X., Xie, S., Li, Y., Doll\u00e1r, P., Girshick, R.: Masked autoencoders are scalable vision learners. In: Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition, pp. 16000\u201316009 (2022)","DOI":"10.1109\/CVPR52688.2022.01553"},{"key":"3608_CR155","unstructured":"Bao, H., Dong, L., Piao, S., Wei, F.: Beit: Bert pre-training of image transformers. In: International Conference on Learning Representations (2022)"},{"key":"3608_CR156","doi-asserted-by":"crossref","unstructured":"Xie, Z., Zhang, Z., Cao, Y., Lin, Y., Bao, J., Yao, Z., Dai, Q., Hu, H.: SimMIM: a simple framework for masked image modeling. In: Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition, pp. 9653\u20139663 (2022)","DOI":"10.1109\/CVPR52688.2022.00943"},{"key":"3608_CR157","unstructured":"Li, J., Li, D., Xiong, C., Hoi, S.: Blip: bootstrapping language-image pre-training for unified vision-language understanding and generation. In: International Conference on Machine Learning, pp. 12888\u201312900. PMLR (2022)"},{"key":"3608_CR158","unstructured":"Li, J., Li, D., Savarese, S., Hoi, S.: Blip-2: bootstrapping language-image pre-training with frozen image encoders and large language models. In: International Conference on Machine Learning, pp. 19730\u201319742. PMLR (2023)"},{"issue":"4","key":"3608_CR159","doi-asserted-by":"publisher","first-page":"834","DOI":"10.1109\/TPAMI.2017.2699184","volume":"40","author":"L-C Chen","year":"2018","unstructured":"Chen, L.-C., Papandreou, G., Kokkinos, I., Murphy, K., Yuille, A.L.: Semantic image segmentation with deep convolutional nets and fully connected CRFs. IEEE Trans. Pattern Anal. Mach. Intell. 40(4), 834\u2013848 (2018)","journal-title":"IEEE Trans. Pattern Anal. Mach. Intell."},{"key":"3608_CR160","unstructured":"Chen, L.-C., Papandreou, G., Schroff, F., Adam, H.: Rethinking atrous convolution for semantic image segmentation. In: Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition, pp. 1251\u20131258 (2017)"},{"key":"3608_CR161","doi-asserted-by":"crossref","unstructured":"Chen, L.-C., Zhu, Y., Papandreou, G., Schroff, F., Adam, H.: Encoder\u2013decoder with atrous separable convolution for semantic image segmentation. In: Proceedings of the European Conference on Computer Vision (ECCV), pp. 801\u2013818 (2018)","DOI":"10.1007\/978-3-030-01234-2_49"},{"issue":"3","key":"3608_CR162","doi-asserted-by":"publisher","first-page":"171","DOI":"10.1007\/s13735-020-00195-x","volume":"9","author":"AM Hafiz","year":"2020","unstructured":"Hafiz, A.M., Bhat, G.M.: A survey on instance segmentation: state of the art. Int. J. Multimed. Inf. Retr. 9(3), 171\u2013189 (2020)","journal-title":"Int. J. Multimed. Inf. Retr."},{"key":"3608_CR163","doi-asserted-by":"crossref","unstructured":"Liu, S., Qi, L., Qin, H., Shi, J., Jia, J.: Path aggregation network for instance segmentation. In: Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition, pp. 8759\u20138768 (2018)","DOI":"10.1109\/CVPR.2018.00913"},{"key":"3608_CR164","doi-asserted-by":"crossref","unstructured":"Bolya, D., Zhou, C., Xiao, F., Lee, Y.J.: YOLACT: real-time instance segmentation. In: Proceedings of the IEEE\/CVF International Conference on Computer Vision, pp. 9157\u20139166 (2019)","DOI":"10.1109\/ICCV.2019.00925"},{"issue":"5","key":"3608_CR165","doi-asserted-by":"publisher","first-page":"898","DOI":"10.1109\/TPAMI.2010.161","volume":"33","author":"P Arbel\u00e1ez","year":"2011","unstructured":"Arbel\u00e1ez, P., Maire, M., Fowlkes, C., Malik, J.: Contour detection and hierarchical image segmentation. IEEE Trans. Pattern Anal. Mach. Intell. 33(5), 898\u2013916 (2011). https:\/\/doi.org\/10.1109\/TPAMI.2010.161","journal-title":"IEEE Trans. Pattern Anal. Mach. Intell."},{"key":"3608_CR166","doi-asserted-by":"publisher","unstructured":"Ren, X., Malik, J.: Learning a classification model for segmentation. In: Proceedings Ninth IEEE International Conference on Computer Vision, vol. 1, pp. 10\u201317 (2003). https:\/\/doi.org\/10.1109\/ICCV.2003.1238308","DOI":"10.1109\/ICCV.2003.1238308"},{"key":"3608_CR167","doi-asserted-by":"publisher","unstructured":"Alexe, B., Deselaers, T., Ferrari, V.: What is an object? In: 2010 IEEE Computer Society Conference on Computer Vision and Pattern Recognition, pp. 73\u201380 (2010). https:\/\/doi.org\/10.1109\/CVPR.2010.5540226","DOI":"10.1109\/CVPR.2010.5540226"},{"key":"3608_CR168","doi-asserted-by":"publisher","unstructured":"Stauffer, C., Grimson, W.E.L.: Adaptive background mixture models for real-time tracking. In: Proceedings. 1999 IEEE Computer Society Conference on Computer Vision and Pattern Recognition (Cat. No PR00149), vol. 2, pp. 246\u2013252 (1999). https:\/\/doi.org\/10.1109\/CVPR.1999.784637","DOI":"10.1109\/CVPR.1999.784637"},{"issue":"2","key":"3608_CR169","doi-asserted-by":"publisher","first-page":"318","DOI":"10.1109\/TPAMI.2018.2858826","volume":"42","author":"T-Y Lin","year":"2020","unstructured":"Lin, T.-Y., Goyal, P., Girshick, R., He, K., Doll\u00e1r, P.: Focal loss for dense object detection. IEEE Trans. Pattern Anal. Mach. Intell. 42(2), 318\u2013327 (2020)","journal-title":"IEEE Trans. Pattern Anal. Mach. Intell."},{"key":"3608_CR170","doi-asserted-by":"crossref","unstructured":"Milletari, F., Navab, N., Ahmadi, S.-A.: V-Net: fully convolutional neural networks for volumetric medical image segmentation. In: 2016 Fourth International Conference on 3D Vision (3DV), pp. 565\u2013571. IEEE (2016)","DOI":"10.1109\/3DV.2016.79"},{"key":"3608_CR171","doi-asserted-by":"crossref","unstructured":"Wang, X., Zhang, X., Cao, Y., Wang, W., Shen, C., Huang, T.: SegGPT: segmenting everything in context (2023). arXiv preprint arXiv:2304.03284 [cs.CV]","DOI":"10.1109\/ICCV51070.2023.00110"},{"key":"3608_CR172","doi-asserted-by":"crossref","unstructured":"Shao, S., Li, Z., Zhang, T., Peng, C., Yu, G., Zhang, X., Li, J., Sun, J.: Objects365: a large-scale, high-quality dataset for object detection. In: Proceedings of the IEEE\/CVF International Conference on Computer Vision, pp. 8430\u20138439 (2019)","DOI":"10.1109\/ICCV.2019.00852"},{"key":"3608_CR173","doi-asserted-by":"crossref","unstructured":"Kamath, A., Singh, M., LeCun, Y., Synnaeve, G., Misra, I., Carion, N.: MDETR\u2014modulated detection for end-to-end multi-modal understanding. In: Proceedings of the IEEE\/CVF International Conference on Computer Vision, pp. 1780\u20131790 (2021)","DOI":"10.1109\/ICCV48922.2021.00180"},{"key":"3608_CR174","doi-asserted-by":"crossref","unstructured":"Bansal, A., Sikka, K., Sharma, G., Chellappa, R., Divakaran, A.: Zero-shot object detection. In: European Conference on Computer Vision, pp. 384\u2013400 (2018)","DOI":"10.1007\/978-3-030-01246-5_24"},{"key":"3608_CR175","doi-asserted-by":"crossref","unstructured":"Rahman, S., Khan, S., Barnes, N.: Improved visual-semantic alignment for zero-shot object detection. In: Proceedings of the AAAI Conference on Artificial Intelligence, vol. 34, pp. 11932\u201311939 (2020)","DOI":"10.1609\/aaai.v34i07.6868"},{"key":"3608_CR176","doi-asserted-by":"crossref","unstructured":"Zhu, P., Wang, H., Saligrama, V.: Don\u2019t even look once: synthesizing features for zero-shot detection. In: Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition, pp. 11693\u201311702 (2020)","DOI":"10.1109\/CVPR42600.2020.01171"},{"key":"3608_CR177","doi-asserted-by":"crossref","unstructured":"Du, Y., Wei, F., Zhang, Z., Shi, M., Gao, Y., Li, G.: Learning to prompt for open-vocabulary object detection with vision-language model. In: Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition, pp. 16096\u201316106 (2022)","DOI":"10.1109\/CVPR52688.2022.01369"},{"key":"3608_CR178","doi-asserted-by":"crossref","unstructured":"Feng, C., Zhong, Y., Jie, Z., Chu, X., Ren, H., Wei, X., Xie, W., Ma, L.: Promptdet: towards open-vocabulary detection using uncurated images. In: European Conference on Computer Vision, pp. 107\u2013124. Springer (2022)","DOI":"10.1007\/978-3-031-20077-9_41"},{"key":"3608_CR179","doi-asserted-by":"crossref","unstructured":"Zhong, Y., Yang, J., Zhang, P., Li, C., Codella, N., Li, L.H., Zhou, L., Dai, X., Yuan, L., Li, Y., Gao, J.: RegionCLIP: region-based language-image pretraining (2021). arXiv preprint arXiv:2112.09106 [cs.CV]","DOI":"10.1109\/CVPR52688.2022.01629"},{"key":"3608_CR180","unstructured":"Chen, X., Fang, H., Lin, T.-Y., Vedantam, R., Gupta, S., Doll\u00e1r, P., Zitnick, C.L.: Microsoft coco captions: data collection and evaluation server (2015). arXiv preprint arXiv:1504.00325 [cs.CV]"},{"key":"3608_CR181","doi-asserted-by":"publisher","first-page":"32","DOI":"10.1007\/s11263-016-0981-7","volume":"123","author":"R Krishna","year":"2017","unstructured":"Krishna, R., Zhu, Y., Groth, O., Johnson, J., Hata, K., Kravitz, J., Chen, S., Kalantidis, Y., Li, L.-J., Shamma, D.A.: Visual genome: connecting language and vision using crowdsourced dense image annotations. Int. J. Comput. Vis. 123, 32\u201373 (2017). https:\/\/doi.org\/10.1007\/s11263-016-0981-7","journal-title":"Int. J. Comput. Vis."},{"key":"3608_CR182","doi-asserted-by":"crossref","unstructured":"Srinivasan, K., Raman, K., Chen, J., Bendersky, M., Najork, M.: WIT: wikipedia-based image text dataset for multimodal multilingual machine learning. In: Proceedings of the 44th International ACM SIGIR Conference on Research and Development in Information Retrieval, pp. 2443\u20132449 (2021)","DOI":"10.1145\/3404835.3463257"},{"key":"3608_CR183","unstructured":"Huo, Y., Zhang, M., Liu, G., Lu, H., Gao, Y., Yang, G., Wen, J., Zhang, H., Xu, B., Zheng, W., et al.: WenLan: bridging vision and language by large-scale multi-modal pre-training. arXiv preprint arXiv:2103.06561 (2021) [cs.CV]"},{"key":"3608_CR184","unstructured":"Desai, K., Kaul, G., Aysola, Z., Johnson, J.: RedCaps: web-curated image-text data created by the people, for the people (2021). arXiv preprint arXiv:2111.11431 [cs.CV]"},{"key":"3608_CR185","unstructured":"Schuhmann, C., Beaumont, R., Vencu, R., Gordon, C., Wightman, R., Cherti, M., Coombes, T., Katta, A., Mullis, C., Wortsman, M.: Laion-5b: an open large-scale dataset for training next generation image-text models. In: Advances in Neural Information Processing Systems, vol. 35, pp. 25278\u201325294 (2022)"},{"key":"3608_CR186","doi-asserted-by":"publisher","unstructured":"Parkhi, O.M., Vedaldi, A., Zisserman, A., Jawahar, C.V.: Cats and dogs. In: 2012 IEEE Conference on Computer Vision and Pattern Recognition, pp. 3498\u20133505 (2012). https:\/\/doi.org\/10.1109\/CVPR.2012.6248092","DOI":"10.1109\/CVPR.2012.6248092"},{"key":"3608_CR187","doi-asserted-by":"crossref","unstructured":"Bossard, L., Guillaumin, M., Van\u00a0Gool, L.: Food-101\u2014mining discriminative components with random forests. In: European Conference on Computer Vision, pp. 446\u2013461. Springer (2014)","DOI":"10.1007\/978-3-319-10599-4_29"},{"key":"3608_CR188","unstructured":"Moran, S.: Learning to Hash for Large-Scale Image Retrieval. PhD thesis, University of Edinburgh (2016)"},{"key":"3608_CR189","doi-asserted-by":"publisher","unstructured":"Xiao, J., Hays, J., Ehinger, K.A., Oliva, A., Torralba, A.: Sun database: large-scale scene recognition from abbey to zoo. In: 2010 IEEE Computer Society Conference on Computer Vision and Pattern Recognition, pp. 3485\u20133492 (2010). https:\/\/doi.org\/10.1109\/CVPR.2010.5539970","DOI":"10.1109\/CVPR.2010.5539970"},{"key":"3608_CR190","unstructured":"Krause, J., Deng, J., Stark, M., Fei-Fei, L.: Collecting a large-scale dataset of fine-grained cars. In: Second Workshop on Fine-Grained Visual Categorization (FGVC2) (2013)"},{"key":"3608_CR191","unstructured":"Maji, S., Kannala, J., Rahtu, E., Blaschko, M., Vedaldi, A.: Fine-Grained Visual Classification of Aircraft. Technical Report (2013)"},{"issue":"2","key":"3608_CR192","doi-asserted-by":"publisher","first-page":"303","DOI":"10.1007\/s11263-009-0275-4","volume":"88","author":"M Everingham","year":"2010","unstructured":"Everingham, M., Van Gool, L., Williams, C.K.I., Winn, J., Zisserman, A.: The pascal visual object classes (VOC) challenge. Int. J. Comput. Vis. 88(2), 303\u2013338 (2010). https:\/\/doi.org\/10.1007\/s11263-009-0275-4","journal-title":"Int. J. Comput. Vis."},{"key":"3608_CR193","doi-asserted-by":"publisher","unstructured":"Cimpoi, M., Maji, S., Kokkinos, I., Mohamed, S., Vedaldi, A.: Describing textures in the wild. In: 2014 IEEE Conference on Computer Vision and Pattern Recognition, pp. 3606\u20133613 (2014). https:\/\/doi.org\/10.1109\/CVPR.2014.461","DOI":"10.1109\/CVPR.2014.461"},{"key":"3608_CR194","doi-asserted-by":"publisher","unstructured":"Fei-Fei, L., Fergus, R., Perona, P.: Learning generative visual models from few training examples: an incremental Bayesian approach tested on 101 object categories. In: 2004 Conference on Computer Vision and Pattern Recognition Workshop, p. 178 (2004). https:\/\/doi.org\/10.1109\/CVPR.2004.383","DOI":"10.1109\/CVPR.2004.383"},{"key":"3608_CR195","doi-asserted-by":"crossref","unstructured":"Mottaghi, R., Chen, X., Liu, X., Cho, N.-G., Lee, S.-W., Fidler, S., Urtasun, R., Yuille, A.: The role of context for object detection and semantic segmentation in the wild. In: 2014 IEEE Conference on Computer Vision and Pattern Recognition, pp. 891\u2013898. IEEE (2014)","DOI":"10.1109\/CVPR.2014.119"},{"issue":"3","key":"3608_CR196","doi-asserted-by":"publisher","first-page":"302","DOI":"10.1007\/s11263-018-1140-0","volume":"127","author":"B Zhou","year":"2019","unstructured":"Zhou, B., Zhao, H., Puig, X., Xiao, T., Fidler, S., Barriuso, A., Torralba, A.: Semantic understanding of scenes through the ade20k dataset. Int. J. Comput. Vis. 127(3), 302\u2013321 (2019). https:\/\/doi.org\/10.1007\/s11263-018-1140-0","journal-title":"Int. J. Comput. Vis."},{"key":"3608_CR197","doi-asserted-by":"crossref","unstructured":"Cordts, M., Omran, M., Ramos, S., Rehfeld, T., Enzweiler, M., Benenson, R., Franke, U., Roth, S., Schiele, B.: The cityscapes dataset for semantic urban scene understanding (2016). arXiv preprint arXiv:1604.01685 [cs.CV]","DOI":"10.1109\/CVPR.2016.350"},{"key":"3608_CR198","doi-asserted-by":"publisher","first-page":"80","DOI":"10.1016\/j.patrec.2015.10.013","volume":"81","author":"M Minervini","year":"2016","unstructured":"Minervini, M., Fischbach, A., Scharr, H., Tsaftaris, S.A.: Finely-grained annotated datasets for image-based plant phenotyping. Pattern Recognit. Lett. 81, 80\u201389 (2016). https:\/\/doi.org\/10.1016\/j.patrec.2015.10.013","journal-title":"Pattern Recognit. Lett."},{"issue":"12","key":"3608_CR199","doi-asserted-by":"publisher","first-page":"1247","DOI":"10.1038\/s41592-019-0612-7","volume":"16","author":"JC Caicedo","year":"2019","unstructured":"Caicedo, J.C., Goodman, A., Karhohs, K.W., Cimini, B.A., Ackerman, J., Haghighi, M., Heng, C., Becker, T., Doan, M., McQuin, C.: Nucleus segmentation across imaging experiments: the 2018 data science bowl. Nat. Methods 16(12), 1247\u20131253 (2019). https:\/\/doi.org\/10.1038\/s41592-019-0612-7","journal-title":"Nat. Methods"},{"key":"3608_CR200","doi-asserted-by":"crossref","unstructured":"Fortin, J.-M., Gamache, O., Grondin, V., Pomerleau, F., Gigu\u00e8re, P.: Instance segmentation for autonomous log grasping in forestry operations. In: 2022 IEEE\/RSJ International Conference on Intelligent Robots and Systems (IROS), pp. 6064\u20136071. IEEE (2022)","DOI":"10.1109\/IROS47612.2022.9982286"},{"key":"3608_CR201","unstructured":"Trotter, C., Atkinson, G., Sharpe, M., Richardson, K., McGough, A.S., Wright, N., Burville, B., Berggren, P.: Ndd20: a large-scale few-shot dolphin dataset for coarse and fine-grained categorisation (2020). arXiv preprint arXiv:2005.13359 [cs.CV]"},{"key":"3608_CR202","unstructured":"Snyder, C., Do, M.: Streets: a novel camera network dataset for traffic flow. In: Advances in Neural Information Processing Systems, vol. 32 (2019)"},{"key":"3608_CR203","unstructured":"Hong, J., Fulton, M., Sattar, J.: TrashCan: a semantically-segmented dataset towards visual detection of marine debris (2020). arXiv preprint arXiv:2007.08097 [cs.CV]"},{"issue":"7","key":"3608_CR204","doi-asserted-by":"publisher","first-page":"1956","DOI":"10.1007\/s11263-020-01316-z","volume":"128","author":"A Kuznetsova","year":"2020","unstructured":"Kuznetsova, A., Rom, H., Alldrin, N., Uijlings, J., Krasin, I., Pont-Tuset, J., Kamali, S., Popov, S., Malloci, M., Kolesnikov, A., Duerig, T., Ferrari, V.: The open images dataset v4: unified image classification, object detection, and visual relationship detection at scale. Int. J. Comput. Vis. 128(7), 1956\u20131981 (2020). https:\/\/doi.org\/10.1007\/s11263-020-01316-z","journal-title":"Int. J. Comput. Vis."},{"issue":"2","key":"3608_CR205","first-page":"479","volume":"43","author":"S Minaee","year":"2021","unstructured":"Minaee, S., Boykov, Y., Porikli, F., Plaza, A., Kehtarnavaz, N., Terzopoulos, D.: Image segmentation using deep learning: a survey. IEEE Trans. Pattern Anal. Mach. Intell. 43(2), 479\u2013500 (2021)","journal-title":"IEEE Trans. Pattern Anal. Mach. Intell."},{"key":"3608_CR206","doi-asserted-by":"crossref","unstructured":"Chen, X., Ding, M., Wang, X., Xin, Y., Mo, S., Wang, Y., Han, S., Luo, P., Zeng, G., Wang, J.: Context autoencoder for self-supervised representation learning (2022). arXiv preprint arXiv:2202.03026 [cs.CV]","DOI":"10.1007\/s11263-023-01852-4"},{"key":"3608_CR207","doi-asserted-by":"crossref","unstructured":"Xi, T., Sun, Y., Yu, D., Li, B., Peng, N., Zhang, G., Ling, Y., Zhao, W., Zeng, Z., Yu, N.: UFO: unified feature optimization. In: European Conference on Computer Vision, pp. 359\u2013376. Springer (2022)","DOI":"10.1007\/978-3-031-19809-0_27"},{"key":"3608_CR208","unstructured":"Jiang, J., Min, S., Kong, W., Gong, D., Wang, H., Li, Z., Guo, J., Xu, C., Jiang, J., Mao, X.: Tencent text-video retrieval: hierarchical cross-modal interactions with multi-level representations (2022). arXiv preprint arXiv:2204.03382 [cs.CV]"},{"key":"3608_CR209","doi-asserted-by":"crossref","unstructured":"Xu, J., Mei, T., Yao, T., Rui, Y.: MSR-VTT: a large video description dataset for bridging video and language. In: Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition, pp. 5288\u20135296 (2016)","DOI":"10.1109\/CVPR.2016.571"},{"key":"3608_CR210","doi-asserted-by":"crossref","unstructured":"Wu, Z., Yao, T., Fu, Y., Jiang, Y.-G.: Deep learning for video classification and captioning. In: Frontiers of Multimedia Research, pp. 3\u201329. ACM, New York, NY, USA (2017)","DOI":"10.1145\/3122865.3122867"},{"key":"3608_CR211","doi-asserted-by":"crossref","unstructured":"Hendricks, L.A., Wang, O., Shechtman, E., Sivic, J., Darrell, T., Russell, B.: Localizing moments in video with natural language. In: Proceedings of the IEEE International Conference on Computer Vision, pp. 5803\u20135812 (2017)","DOI":"10.1109\/ICCV.2017.618"},{"key":"3608_CR212","doi-asserted-by":"crossref","unstructured":"L\u00fcddecke, T., Ecker, A.S.: Image segmentation using text and image prompts (2022). arXiv preprint arXiv:2112.10003 [cs.CV]","DOI":"10.1109\/CVPR52688.2022.00695"},{"key":"3608_CR213","doi-asserted-by":"crossref","unstructured":"Wei, L., Xie, L., Zhou, W., Li, H., Tian, Q.: MVP: multimodality-guided visual pre-training. In: European Conference on Computer Vision (2022)","DOI":"10.1007\/978-3-031-20056-4_20"},{"key":"3608_CR214","doi-asserted-by":"crossref","unstructured":"Ni, M., Huang, H., Su, L., Cui, E., Bharti, T., Wang, L., Zhang, D., Duan, N.: M3p: learning universal representations via multitask multilingual multimodal pre-training. In: Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition, pp. 3977\u20133986 (2021)","DOI":"10.1109\/CVPR46437.2021.00397"},{"key":"3608_CR215","unstructured":"Razavi, A., Oord, A., Vinyals, O.: Generating diverse high-fidelity images with VQ-VAE-2. In: Advances in Neural Information Processing Systems, vol. 32 (2019)"},{"key":"3608_CR216","unstructured":"Radford, A., Kim, J.W., Hallacy, C., Ramesh, A., Goh, G., Agarwal, S., Sastry, G., Askell, A., Mishkin, P., Clark, J.: Learning transferable visual models from natural language supervision. In: International Conference on Machine Learning, pp. 8748\u20138763. PMLR (2021)"},{"key":"3608_CR217","doi-asserted-by":"crossref","unstructured":"Wang, F., Liu, H.: Understanding the behaviour of contrastive loss (2021). In: Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition, pp. 2495\u20132504","DOI":"10.1109\/CVPR46437.2021.00252"},{"key":"3608_CR218","doi-asserted-by":"crossref","unstructured":"Chen, Z., Zhang, Y., Rosenberg, A., Ramabhadran, B., Moreno, P., Bapna, A., Zen, H.: Maestro: Matched speech text representations through modality matching (2022). arXiv preprint arXiv:2204.03409 [cs.CL]","DOI":"10.21437\/Interspeech.2022-10937"},{"key":"3608_CR219","doi-asserted-by":"crossref","unstructured":"Wang, B., Ma, L., Zhang, W., Liu, W.: Reconstruction network for video captioning. In: Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition, pp. 7622\u20137631 (2018)","DOI":"10.1109\/CVPR.2018.00795"},{"key":"3608_CR220","doi-asserted-by":"crossref","unstructured":"Li, W., Zhu, X., Gong, S.: Person re-identification by deep joint learning of multi-loss classification. In: Proceedings of the IEEE International Conference on Computer Vision, pp. 2194\u20132203 (2017)","DOI":"10.24963\/ijcai.2017\/305"},{"issue":"14","key":"3608_CR221","doi-asserted-by":"publisher","first-page":"4623","DOI":"10.3390\/s21144623","volume":"21","author":"S Barton","year":"2021","unstructured":"Barton, S., Alakkari, S., O\u2019Dwyer, K., Ward, T., Hennelly, B.: Convolution network with custom loss function for the denoising of low SNR Raman spectra. Sensors 21(14), 4623 (2021). https:\/\/doi.org\/10.3390\/s21144623","journal-title":"Sensors"},{"key":"3608_CR222","unstructured":"Mao, J., Xu, W., Yang, Y., Wang, J., Yuille, A.L.: Explain images with multimodal recurrent neural networks (2014). arXiv preprint arXiv:1410.1090 [cs.CV]"},{"key":"3608_CR223","doi-asserted-by":"publisher","unstructured":"Young, P., Lai, A., Hodosh, M., Hockenmaier, J.: From image descriptions to visual denotations: new similarity metrics for semantic inference over event descriptions. Trans. Assoc. Comput. Linguist. 2, 67\u201378 (2014). https:\/\/doi.org\/10.1162\/tacl_a_00166","DOI":"10.1162\/tacl_a_00166"},{"key":"3608_CR224","doi-asserted-by":"crossref","unstructured":"Sharma, P., Ding, N., Goodman, S., Soricut, R.: Conceptual captions: a cleaned, hypernymed, image alt-text dataset for automatic image captioning. In: Proceedings of the 56th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers), pp. 2556\u20132565 (2018)","DOI":"10.18653\/v1\/P18-1238"},{"key":"3608_CR225","doi-asserted-by":"crossref","unstructured":"Dong, X., Zhan, X., Wu, Y., Wei, Y., Kampffmeyer, M.C., Wei, X., Lu, M., Wang, Y., Liang, X.: M5product: self-harmonized contrastive learning for e-commercial multi-modal pretraining. In: Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition, pp. 21252\u201321262 (2022)","DOI":"10.1109\/CVPR52688.2022.02057"},{"key":"3608_CR226","unstructured":"Wu, C., Yin, S., Qi, W., Wang, X., Tang, Z., Duan, N.: Visual ChatGPT: talking, drawing and editing with visual foundation models (2023). arXiv preprint arXiv:2303.04671 [cs.CV]"},{"key":"3608_CR227","unstructured":"Su, Y., Lan, T., Li, H., Xu, J., Wang, Y., Cai, D.: PandaGPT: one model to instruction-follow them all (2023). arXiv preprint arXiv:2305.16355 [cs.CL]"},{"key":"3608_CR228","unstructured":"Wu, S., Fei, H., Qu, L., Ji, W., Chua, T.-S.: NExT-GPT: any-to-any multimodal LLM (2023). arXiv preprint arXiv:2309.05519 [cs.AI]"},{"key":"3608_CR229","doi-asserted-by":"crossref","unstructured":"Huang, R., Han, J., Lu, G., Liang, X., Zeng, Y., Zhang, W., Xu, H.: DiffDis: empowering generative diffusion model with cross-modal discrimination capability. In: Proceedings of the IEEE\/CVF International Conference on Computer Vision, pp. 20391\u201320401 (2023)","DOI":"10.1109\/ICCV51070.2023.01440"},{"key":"3608_CR230","unstructured":"Ge, Y., Xu, J., Zhao, B.N., Joshi, N., Itti, L., Vineet, V.: Beyond generation: harnessing text to image models for object detection and segmentation (2023). arXiv preprint arXiv:2309.05956 [cs.CV]"},{"key":"3608_CR231","unstructured":"Gu, Z., Chen, H., Xu, Z., Lan, J., Meng, C., Wang, W.: DiffusionInst: diffusion model for instance segmentation (2022). arXiv preprint arXiv:2212.02773 [cs.CV]"},{"key":"3608_CR232","unstructured":"Ni, M., Zhang, Y., Feng, K., Li, X., Guo, Y., Zuo, W.: Ref-Diff: zero-shot referring image segmentation with generative models (2023). arXiv:2308.16777 [cs.CV]"},{"key":"3608_CR233","doi-asserted-by":"crossref","unstructured":"Li, A.C., Prabhudesai, M., Duggal, S., Brown, E., Pathak, D.: Your diffusion model is secretly a zero-shot classifier (2023). arXiv:2303.16203 [cs.LG]","DOI":"10.1109\/ICCV51070.2023.00210"},{"key":"3608_CR234","doi-asserted-by":"crossref","unstructured":"Xiang, W., Yang, H., Huang, D., Wang, Y.: Denoising diffusion autoencoders are unified self-supervised learners. In: Proceedings of the IEEE\/CVF International Conference on Computer Vision (2023)","DOI":"10.1109\/ICCV51070.2023.01448"},{"key":"3608_CR235","doi-asserted-by":"crossref","unstructured":"Zhu, R., Pan, Y., Li, Y., Yao, T., Sun, Z., Mei, T., Chen, C.W.: SD-DiT: unleashing the power of self-supervised discrimination in diffusion transformer (2024). arXiv:2403.17004 [cs.CV]","DOI":"10.1109\/CVPR52733.2024.00806"}],"container-title":["The Visual Computer"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s00371-024-03608-8.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s00371-024-03608-8\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s00371-024-03608-8.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,3,10]],"date-time":"2025-03-10T09:12:32Z","timestamp":1741597952000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s00371-024-03608-8"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,9,30]]},"references-count":235,"journal-issue":{"issue":"5","published-print":{"date-parts":[[2025,3]]}},"alternative-id":["3608"],"URL":"https:\/\/doi.org\/10.1007\/s00371-024-03608-8","relation":{},"ISSN":["0178-2789","1432-2315"],"issn-type":[{"value":"0178-2789","type":"print"},{"value":"1432-2315","type":"electronic"}],"subject":[],"published":{"date-parts":[[2024,9,30]]},"assertion":[{"value":"9 August 2024","order":1,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"30 September 2024","order":2,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Declarations"}},{"value":"The authors declare no competing interests.","order":2,"name":"Ethics","group":{"name":"EthicsHeading","label":"Competing interests"}}]}}