{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,2,8]],"date-time":"2026-02-08T09:05:58Z","timestamp":1770541558017,"version":"3.49.0"},"publisher-location":"Cham","reference-count":68,"publisher":"Springer Nature Switzerland","isbn-type":[{"value":"9783031729911","type":"print"},{"value":"9783031729928","type":"electronic"}],"license":[{"start":{"date-parts":[[2024,10,30]],"date-time":"2024-10-30T00:00:00Z","timestamp":1730246400000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2024,10,30]],"date-time":"2024-10-30T00:00:00Z","timestamp":1730246400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2025]]},"DOI":"10.1007\/978-3-031-72992-8_1","type":"book-chapter","created":{"date-parts":[[2024,10,29]],"date-time":"2024-10-29T08:29:02Z","timestamp":1730190542000},"page":"1-18","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":16,"title":["HowToCaption: Prompting LLMs to\u00a0Transform Video Annotations at\u00a0Scale"],"prefix":"10.1007","author":[{"given":"Nina","family":"Shvetsova","sequence":"first","affiliation":[]},{"given":"Anna","family":"Kukleva","sequence":"additional","affiliation":[]},{"given":"Xudong","family":"Hong","sequence":"additional","affiliation":[]},{"given":"Christian","family":"Rupprecht","sequence":"additional","affiliation":[]},{"given":"Bernt","family":"Schiele","sequence":"additional","affiliation":[]},{"given":"Hilde","family":"Kuehne","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2024,10,30]]},"reference":[{"key":"1_CR1","unstructured":"Abu-El-Haija, S., et al.: YouTube-8m: a large-scale video classification benchmark. arXiv preprint arXiv:1609.08675 (2016)"},{"key":"1_CR2","unstructured":"Afouras, T., Mavroudi, E., Nagarajan, T., Wang, H., Torresani, L.: HT-step: aligning instructional articles with how-to videos. In: NeurIPS, vol. 36 (2024)"},{"key":"1_CR3","doi-asserted-by":"crossref","unstructured":"Amrani, E., Ben-Ari, R., Rotman, D., Bronstein, A.: Noise estimation using density estimation for self-supervised multimodal learning. In: AAAI (2021)","DOI":"10.1609\/aaai.v35i8.16822"},{"key":"1_CR4","doi-asserted-by":"crossref","unstructured":"Bain, M., Nagrani, A., Varol, G., Zisserman, A.: Frozen in time: a joint video and image encoder for end-to-end retrieval. In: ICCV (2021)","DOI":"10.1109\/ICCV48922.2021.00175"},{"key":"1_CR5","unstructured":"Chang, T.A., Bergen, B.K.: Language model behavior: a comprehensive survey. arXiv preprint arXiv:2303.11504 (2023)"},{"key":"1_CR6","doi-asserted-by":"crossref","unstructured":"Changpinyo, S., Sharma, P., Ding, N., Soricut, R.: Conceptual 12m: pushing web-scale image-text pre-training to recognize long-tail visual concepts. In: CVPR (2021)","DOI":"10.1109\/CVPR46437.2021.00356"},{"key":"1_CR7","doi-asserted-by":"crossref","unstructured":"Chen, B., et\u00a0al.: Multimodal clustering networks for self-supervised learning from unlabeled videos. In: ICCV (2021)","DOI":"10.1109\/ICCV48922.2021.00791"},{"key":"1_CR8","unstructured":"Chen, D., Dolan, W.B.: Collecting highly parallel data for paraphrase evaluation. In: ACL (2011)"},{"key":"1_CR9","unstructured":"Chen, S., et al.: Vast: a vision-audio-subtitle-text omni-modality foundation model and dataset. In: NeurIPS, vol. 36 (2023)"},{"key":"1_CR10","unstructured":"Chiang, W.L., et al.: Vicuna: an open-source chatbot impressing GPT-4 with 90%* ChatGPT quality. Large Model Syst. Organ. (2023)"},{"key":"1_CR11","unstructured":"Cho, J., Lei, J., Tan, H., Bansal, M.: Unifying vision-and-language tasks via text generation. In: ICML (2021)"},{"key":"1_CR12","doi-asserted-by":"crossref","unstructured":"Desai, K., Johnson, J.: Virtex: learning visual representations from textual annotations. In: CVPR (2021)","DOI":"10.1109\/CVPR46437.2021.01101"},{"key":"1_CR13","unstructured":"Devlin, J., Chang, M.W., Lee, K., Toutanova, K.: BERT: pre-training of deep bidirectional transformers for language understanding. In: NAACL (2019)"},{"key":"1_CR14","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"214","DOI":"10.1007\/978-3-030-58548-8_13","volume-title":"Computer Vision \u2013 ECCV 2020","author":"V Gabeur","year":"2020","unstructured":"Gabeur, V., Sun, C., Alahari, K., Schmid, C.: Multi-modal transformer for video retrieval. In: Vedaldi, A., Bischof, H., Brox, T., Frahm, J.-M. (eds.) ECCV 2020. LNCS, vol. 12349, pp. 214\u2013229. Springer, Cham (2020). https:\/\/doi.org\/10.1007\/978-3-030-58548-8_13"},{"key":"1_CR15","doi-asserted-by":"crossref","unstructured":"Ghadiyaram, D., Tran, D., Mahajan, D.: Large-scale weakly-supervised pre-training for video action recognition. In: CVPR (2019)","DOI":"10.1109\/CVPR.2019.01232"},{"key":"1_CR16","doi-asserted-by":"crossref","unstructured":"Han, T., Xie, W., Zisserman, A.: Temporal alignment networks for long-term video. In: CVPR (2022)","DOI":"10.1109\/CVPR52688.2022.00292"},{"key":"1_CR17","doi-asserted-by":"crossref","unstructured":"Hou, S., Pan, X., Loy, C.C., Wang, Z., Lin, D.: Learning a unified classifier incrementally via rebalancing. In: CVPR (2019)","DOI":"10.1109\/CVPR.2019.00092"},{"key":"1_CR18","unstructured":"Jia, C., et al.: Scaling up visual and vision-language representation learning with noisy text supervision. In: ICML (2021)"},{"key":"1_CR19","unstructured":"Koupaee, M., Wang, W.Y.: Wikihow: a large scale text summarization dataset. arXiv preprint arXiv:1810.09305 (2018)"},{"key":"1_CR20","doi-asserted-by":"crossref","unstructured":"Krishna, R., et\u00a0al.: Visual genome: connecting language and vision using crowdsourced dense image annotations. IJCV (2017)","DOI":"10.1007\/s11263-016-0981-7"},{"key":"1_CR21","unstructured":"Li, J., Li, D., Savarese, S., Hoi, S.: Blip-2: bootstrapping language-image pre-training with frozen image encoders and large language models. In: ICML (2023)"},{"key":"1_CR22","unstructured":"Li, J., Li, D., Xiong, C., Hoi, S.: Blip: bootstrapping language-image pre-training for unified vision-language understanding and generation. In: ICML (2022)"},{"key":"1_CR23","unstructured":"Li, Z., Chen, Q., Han, T., Zhang, Y., Wang, Y., Xie, W.: A strong baseline for temporal video-text alignment. arXiv preprint arXiv:2312.14055 (2023)"},{"key":"1_CR24","doi-asserted-by":"crossref","unstructured":"Lialin, V., Rawls, S., Chan, D., Ghosh, S., Rumshisky, A., Hamza, W.: Scalable and accurate self-supervised multimodal representation learning without aligned video and text data. In: WACV (2023)","DOI":"10.1109\/WACVW58289.2023.00043"},{"key":"1_CR25","doi-asserted-by":"crossref","unstructured":"Lin, K., et al.: SwinBERT: end-to-end transformers with sparse attention for video captioning. In: CVPR (2022)","DOI":"10.1109\/CVPR52688.2022.01742"},{"key":"1_CR26","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"740","DOI":"10.1007\/978-3-319-10602-1_48","volume-title":"Computer Vision \u2013 ECCV 2014","author":"T-Y Lin","year":"2014","unstructured":"Lin, T.-Y., et al.: Microsoft COCO: common objects in context. In: Fleet, D., Pajdla, T., Schiele, B., Tuytelaars, T. (eds.) ECCV 2014. LNCS, vol. 8693, pp. 740\u2013755. Springer, Cham (2014). https:\/\/doi.org\/10.1007\/978-3-319-10602-1_48"},{"key":"1_CR27","doi-asserted-by":"crossref","unstructured":"Lin, X., Petroni, F., Bertasius, G., Rohrbach, M., Chang, S.F., Torresani, L.: Learning to recognize procedural activities with distant supervision. In: CVPR (2022)","DOI":"10.1109\/CVPR52688.2022.01348"},{"key":"1_CR28","unstructured":"Liu, H., Li, C., Wu, Q., Lee, Y.J.: Visual instruction tuning. In: NeurIPS, vol. 36 (2023)"},{"key":"1_CR29","unstructured":"Lu, J., Batra, D., Parikh, D., Lee, S.: VilBERT: pretraining task-agnostic visiolinguistic representations for vision-and-language tasks. In: NeurIPS (2019)"},{"key":"1_CR30","doi-asserted-by":"crossref","unstructured":"Luo, H., Ji, L., Zhong, M., Chen, Y., Lei, W., Duan, N., Li, T.: CLIP4clip: an empirical study of clip for end to end video clip retrieval and captioning. Neurocomputing (2022)","DOI":"10.1016\/j.neucom.2022.07.028"},{"key":"1_CR31","doi-asserted-by":"crossref","unstructured":"Maaz, M., Rasheed, H., Khan, S., Khan, F.S.: Video-ChatGPT: towards detailed video understanding via large vision and language models. arXiv preprint arXiv:2306.05424 (2023)","DOI":"10.18653\/v1\/2024.acl-long.679"},{"key":"1_CR32","doi-asserted-by":"crossref","unstructured":"Miech, A., Alayrac, J.B., Smaira, L., Laptev, I., Sivic, J., Zisserman, A.: End-to-end learning of visual representations from uncurated instructional videos. In: CVPR (2020)","DOI":"10.1109\/CVPR42600.2020.00990"},{"key":"1_CR33","doi-asserted-by":"crossref","unstructured":"Miech, A., Zhukov, D., Alayrac, J.B., Tapaswi, M., Laptev, I., Sivic, J.: Howto100m: learning a text-video embedding by watching hundred million narrated video clips. In: ICCV (2019)","DOI":"10.1109\/ICCV.2019.00272"},{"key":"1_CR34","series-title":"LNCS","doi-asserted-by":"publisher","first-page":"407","DOI":"10.1007\/978-3-031-19781-9_24","volume-title":"ECCV 2022","author":"A Nagrani","year":"2022","unstructured":"Nagrani, A., et al.: Learning audio-video modalities from image captions. In: Avidan, S., Brostow, G., Ciss\u00e9, M., Farinella, G.M., Hassner, T. (eds.) ECCV 2022. LNCS, vol. 13674, pp. 407\u2013426. Springer, Cham (2022). https:\/\/doi.org\/10.1007\/978-3-031-19781-9_24"},{"key":"1_CR35","unstructured":"Neelakantan, A., et\u00a0al.: Text and code embeddings by contrastive pre-training. arXiv preprint arXiv:2201.10005 (2022)"},{"key":"1_CR36","unstructured":"van den Oord, A., Li, Y., Vinyals, O.: Representation learning with contrastive predictive coding. arXiv preprint arXiv:1807.03748 (2018)"},{"key":"1_CR37","unstructured":"Ordonez, V., Kulkarni, G., Berg, T.: Im2text: describing images using 1 million captioned photographs. In: NeurIPS (2011)"},{"key":"1_CR38","doi-asserted-by":"crossref","unstructured":"Portillo-Quintero, J.A., Ortiz-Bayliss, J.C., Terashima-Mar\u00edn, H.: A straightforward framework for video retrieval using clip. In: Pattern Recognition: 13th Mexican Conference (2021)","DOI":"10.1007\/978-3-030-77004-4_1"},{"key":"1_CR39","unstructured":"Radford, A., et al.: Learning transferable visual models from natural language supervision. In: ICML (2021)"},{"key":"1_CR40","unstructured":"Radford, A., Kim, J.W., Xu, T., Brockman, G., McLeavey, C., Sutskever, I.: Robust speech recognition via large-scale weak supervision. In: ICML (2023)"},{"key":"1_CR41","unstructured":"Radford, A., et\u00a0al.: Language models are unsupervised multitask learners. OpenAI Blog (2019)"},{"key":"1_CR42","unstructured":"Raffel, C., et al.: Exploring the limits of transfer learning with a unified text-to-text transformer. JMLR (2020)"},{"key":"1_CR43","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"209","DOI":"10.1007\/978-3-319-24947-6_17","volume-title":"Pattern Recognition","author":"A Rohrbach","year":"2015","unstructured":"Rohrbach, A., Rohrbach, M., Schiele, B.: The long-short story of movie description. In: Gall, J., Gehler, P., Leibe, B. (eds.) GCPR 2015. LNCS, vol. 9358, pp. 209\u2013221. Springer, Cham (2015). https:\/\/doi.org\/10.1007\/978-3-319-24947-6_17"},{"key":"1_CR44","doi-asserted-by":"crossref","unstructured":"Rouditchenko, A., et al.: AVLnet: learning audio-visual language representations from instructional videos. In: Interspeech (2021)","DOI":"10.21437\/Interspeech.2021-1312"},{"key":"1_CR45","unstructured":"Schuhmann, C., et al.: LAION-400m: open dataset of clip-filtered 400 million image-text pairs. arXiv preprint arXiv:2111.02114 (2021)"},{"key":"1_CR46","doi-asserted-by":"crossref","unstructured":"Seo, P.H., Nagrani, A., Arnab, A., Schmid, C.: End-to-end generative pretraining for multimodal video captioning. In: CVPR (2022)","DOI":"10.1109\/CVPR52688.2022.01743"},{"key":"1_CR47","doi-asserted-by":"crossref","unstructured":"Shvetsova, N., et al.: Everything at once-multi-modal fusion transformer for video retrieval. In: CVPR (2022)","DOI":"10.1109\/CVPR52688.2022.01939"},{"key":"1_CR48","unstructured":"Stroud, J.C., et al.: Learning video representations from textual web supervision. arXiv preprint arXiv:2007.14937 (2020)"},{"key":"1_CR49","unstructured":"Su, W., et al.: VL-BERT: pre-training of generic visual-linguistic representations. In: ICLR (2020)"},{"key":"1_CR50","doi-asserted-by":"crossref","unstructured":"Sun, C., Myers, A., Vondrick, C., Murphy, K., Schmid, C.: VideoBERT: a joint model for video and language representation learning. In: ICCV (2019)","DOI":"10.1109\/ICCV.2019.00756"},{"key":"1_CR51","doi-asserted-by":"crossref","unstructured":"Tan, H., Bansal, M.: LXMERT: learning cross-modality encoder representations from transformers. In: EMNLP (2019)","DOI":"10.18653\/v1\/D19-1514"},{"key":"1_CR52","doi-asserted-by":"crossref","unstructured":"Tang, M., Wang, Z., Liu, Z., Rao, F., Li, D., Li, X.: Clip4caption: clip for video caption. In: ACMMM (2021)","DOI":"10.1145\/3474085.3479207"},{"key":"1_CR53","unstructured":"Taori, R., et al: Alpaca: a strong, replicable instruction-following model. Stanford Center for Research on Foundation Models (2023)"},{"key":"1_CR54","unstructured":"Touvron, H., et\u00a0al.: Llama: open and efficient foundation language models. arXiv preprint arXiv:2302.13971 (2023)"},{"key":"1_CR55","unstructured":"Wang, J., et al.: GIT: a generative image-to-text transformer for vision and language. arXiv preprint arXiv:2205.14100 (2022)"},{"key":"1_CR56","unstructured":"Wang, Y., et\u00a0al.: InternVid: a large-scale video-text dataset for multimodal understanding and generation. arXiv preprint arXiv:2307.06942 (2023)"},{"key":"1_CR57","unstructured":"Xu, H., et\u00a0al.: mPLUG-2: a modularized multi-modal foundation model across text, image and video. arXiv preprint arXiv:2302.00402 (2023)"},{"key":"1_CR58","doi-asserted-by":"crossref","unstructured":"Xu, J., Mei, T., Yao, T., Rui, Y.: MSR-VTT: a large video description dataset for bridging video and language. In: CVPR (2016)","DOI":"10.1109\/CVPR.2016.571"},{"key":"1_CR59","doi-asserted-by":"crossref","unstructured":"Xue, H., et al.: Advancing high-resolution video-language representation with large-scale video transcriptions. In: CVPR (2022)","DOI":"10.1109\/CVPR52688.2022.00498"},{"key":"1_CR60","unstructured":"Yan, S., et al.: Video-text modeling with zero-shot transfer from contrastive captioners. arXiv preprint arXiv:2212.04979 (2022)"},{"key":"1_CR61","unstructured":"Yang, A., Nagrani, A., Laptev, I., Sivic, J., Schmid, C.: Vidchapters-7m: video chapters at scale. In: NeurIPS, vol. 36 (2024)"},{"key":"1_CR62","doi-asserted-by":"crossref","unstructured":"Yang, A., et al.: Vid2seq: large-scale pretraining of a visual language model for dense video captioning. In: CVPR (2023)","DOI":"10.1109\/CVPR52729.2023.01032"},{"key":"1_CR63","doi-asserted-by":"crossref","unstructured":"Ye, Q., et al.: Hitea: hierarchical temporal-aware video-language pre-training. In: ICCV, pp. 15405\u201315416 (2023)","DOI":"10.1109\/ICCV51070.2023.01413"},{"key":"1_CR64","doi-asserted-by":"crossref","unstructured":"Zala, A., et al.: Hierarchical video-moment retrieval and step-captioning. In: CVPR (2023)","DOI":"10.1109\/CVPR52729.2023.02208"},{"key":"1_CR65","unstructured":"Zellers, R., et al.: Merlot: multimodal neural script knowledge models. In: NeurIPS (2021)"},{"key":"1_CR66","doi-asserted-by":"crossref","unstructured":"Zhao, Y., Misra, I., Kr\u00e4henb\u00fchl, P., Girdhar, R.: Learning video representations from large language models. In: CVPR (2023)","DOI":"10.1109\/CVPR52729.2023.00637"},{"key":"1_CR67","doi-asserted-by":"crossref","unstructured":"Zhou, L., Xu, C., Corso, J.: Towards automatic learning of procedures from web instructional videos. In: AAAI (2018)","DOI":"10.1609\/aaai.v32i1.12342"},{"key":"1_CR68","unstructured":"Zhu, D., Chen, J., Shen, X., Li, X., Elhoseiny, M.: MiniGPT-4: enhancing vision-language understanding with advanced large language models. arXiv preprint arXiv:2304.10592 (2023)"}],"container-title":["Lecture Notes in Computer Science","Computer Vision \u2013 ECCV 2024"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-031-72992-8_1","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,10,29]],"date-time":"2024-10-29T08:41:26Z","timestamp":1730191286000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-3-031-72992-8_1"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,10,30]]},"ISBN":["9783031729911","9783031729928"],"references-count":68,"URL":"https:\/\/doi.org\/10.1007\/978-3-031-72992-8_1","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"value":"0302-9743","type":"print"},{"value":"1611-3349","type":"electronic"}],"subject":[],"published":{"date-parts":[[2024,10,30]]},"assertion":[{"value":"30 October 2024","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"ECCV","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"European Conference on Computer Vision","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Milan","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Italy","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2024","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"29 September 2024","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"4 October 2024","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"18","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"eccv2024","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"https:\/\/eccv2024.ecva.net\/","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}}]}}