{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,17]],"date-time":"2026-07-17T11:31:22Z","timestamp":1784287882629,"version":"3.55.0"},"publisher-location":"New York, NY, USA","reference-count":42,"publisher":"ACM","license":[{"start":{"date-parts":[[2024,7,13]],"date-time":"2024-07-13T00:00:00Z","timestamp":1720828800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2024,7,13]]},"DOI":"10.1145\/3641519.3657518","type":"proceedings-article","created":{"date-parts":[[2024,7,12]],"date-time":"2024-07-12T06:39:28Z","timestamp":1720766368000},"page":"1-11","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":168,"title":["MotionCtrl: A Unified and Flexible Motion Controller for Video Generation"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-4677-5760","authenticated-orcid":false,"given":"Zhouxia","family":"Wang","sequence":"first","affiliation":[{"name":"S-Lab, Nanyang Technological University, Singapore"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0008-1302-2352","authenticated-orcid":false,"given":"Ziyang","family":"Yuan","sequence":"additional","affiliation":[{"name":"Tsinghua University, China, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-6585-8604","authenticated-orcid":false,"given":"Xintao","family":"Wang","sequence":"additional","affiliation":[{"name":"ARC Lab, Tencent PCG, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0006-7325-6439","authenticated-orcid":false,"given":"Yaowei","family":"Li","sequence":"additional","affiliation":[{"name":"Peking University, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-2053-5307","authenticated-orcid":false,"given":"Tianshui","family":"Chen","sequence":"additional","affiliation":[{"name":"Guangdong University of Technology, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-9664-4967","authenticated-orcid":false,"given":"Menghan","family":"Xia","sequence":"additional","affiliation":[{"name":"Tencent AI Lab, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-6685-7950","authenticated-orcid":false,"given":"Ping","family":"Luo","sequence":"additional","affiliation":[{"name":"University of Hong Kong, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-7673-8325","authenticated-orcid":false,"given":"Ying","family":"Shan","sequence":"additional","affiliation":[{"name":"ARC Lab, Tencent PCG, China"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2024,7,13]]},"reference":[{"key":"e_1_3_2_2_1_1","unstructured":"[n. d.]. https:\/\/www.pika.art\/."},{"key":"e_1_3_2_2_2_1","doi-asserted-by":"crossref","unstructured":"Max Bain Arsha Nagrani G\u00fcl Varol and Andrew Zisserman. 2021. Frozen in time: A joint video and image encoder for end-to-end retrieval. In ICCV.","DOI":"10.1109\/ICCV48922.2021.00175"},{"key":"e_1_3_2_2_3_1","volume-title":"Stable video diffusion: Scaling latent video diffusion models to large datasets. arXiv preprint arXiv:2311.15127","author":"Blattmann Andreas","year":"2023","unstructured":"Andreas Blattmann, Tim Dockhorn, Sumith Kulal, Daniel Mendelevitch, Maciej Kilian, Dominik Lorenz, Yam Levi, Zion English, Vikram Voleti, Adam Letts, 2023a. Stable video diffusion: Scaling latent video diffusion models to large datasets. arXiv preprint arXiv:2311.15127 (2023)."},{"key":"e_1_3_2_2_4_1","doi-asserted-by":"crossref","unstructured":"Andreas Blattmann Robin Rombach Huan Ling Tim Dockhorn Seung\u00a0Wook Kim Sanja Fidler and Karsten Kreis. 2023b. Align your latents: High-resolution video synthesis with latent diffusion models. In CVPR.","DOI":"10.1109\/CVPR52729.2023.02161"},{"key":"e_1_3_2_2_5_1","volume-title":"Videocrafter1: Open diffusion models for high-quality video generation. arXiv preprint arXiv:2310.19512","author":"Chen Haoxin","year":"2023","unstructured":"Haoxin Chen, Menghan Xia, Yingqing He, Yong Zhang, Xiaodong Cun, Shaoshu Yang, Jinbo Xing, Yaofang Liu, Qifeng Chen, Xintao Wang, 2023b. Videocrafter1: Open diffusion models for high-quality video generation. arXiv preprint arXiv:2310.19512 (2023)."},{"key":"e_1_3_2_2_6_1","volume-title":"Motion-conditioned diffusion model for controllable video synthesis. arXiv preprint arXiv:2304.14404","author":"Chen Tsai-Shien","year":"2023","unstructured":"Tsai-Shien Chen, Chieh\u00a0Hubert Lin, Hung-Yu Tseng, Tsung-Yi Lin, and Ming-Hsuan Yang. 2023a. Motion-conditioned diffusion model for controllable video synthesis. arXiv preprint arXiv:2304.14404 (2023)."},{"key":"e_1_3_2_2_7_1","volume-title":"Cogview: Mastering text-to-image generation via transformers. NeurIPS","author":"Ding Ming","year":"2021","unstructured":"Ming Ding, Zhuoyi Yang, Wenyi Hong, Wendi Zheng, Chang Zhou, Da Yin, Junyang Lin, Xu Zou, Zhou Shao, Hongxia Yang, 2021. Cogview: Mastering text-to-image generation via transformers. NeurIPS (2021)."},{"key":"e_1_3_2_2_8_1","doi-asserted-by":"crossref","unstructured":"Patrick Esser Johnathan Chiu Parmida Atighehchian Jonathan Granskog and Anastasis Germanidis. 2023. Structure and content-guided video synthesis with diffusion models. In ICCV.","DOI":"10.1109\/ICCV51070.2023.00675"},{"key":"e_1_3_2_2_9_1","volume-title":"AnimateDiff: Animate Your Personalized Text-to-Image Diffusion Models without Specific Tuning. arXiv preprint arXiv:2307.04725","author":"Guo Yuwei","year":"2023","unstructured":"Yuwei Guo, Ceyuan Yang, Anyi Rao, Yaohui Wang, Yu Qiao, Dahua Lin, and Bo Dai. 2023. AnimateDiff: Animate Your Personalized Text-to-Image Diffusion Models without Specific Tuning. arXiv preprint arXiv:2307.04725 (2023)."},{"key":"e_1_3_2_2_10_1","volume-title":"Latent video diffusion models for high-fidelity long video generation. arXiv preprint arXiv:2211.13221","author":"He Yingqing","year":"2022","unstructured":"Yingqing He, Tianyu Yang, Yong Zhang, Ying Shan, and Qifeng Chen. 2022. Latent video diffusion models for high-fidelity long video generation. arXiv preprint arXiv:2211.13221 (2022)."},{"key":"e_1_3_2_2_11_1","volume":"202","author":"Ho Jonathan","unstructured":"Jonathan Ho, William Chan, Chitwan Saharia, Jay Whang, Ruiqi Gao, Alexey Gritsenko, Diederik\u00a0P Kingma, Ben Poole, Mohammad Norouzi, David\u00a0J Fleet, 2022. Imagen video: High definition video generation with diffusion models. arXiv preprint arXiv:2210.02303 (2022).","journal-title":"J Fleet"},{"key":"e_1_3_2_2_12_1","volume-title":"Denoising diffusion probabilistic models. NeurIPS","author":"Ho Jonathan","year":"2020","unstructured":"Jonathan Ho, Ajay Jain, and Pieter Abbeel. 2020. Denoising diffusion probabilistic models. NeurIPS (2020)."},{"key":"e_1_3_2_2_13_1","volume-title":"Lora: Low-rank adaptation of large language models. arXiv preprint arXiv:2106.09685","author":"Hu J","year":"2021","unstructured":"Edward\u00a0J Hu, Yelong Shen, Phillip Wallis, Zeyuan Allen-Zhu, Yuanzhi Li, Shean Wang, Lu Wang, and Weizhu Chen. 2021. Lora: Low-rank adaptation of large language models. arXiv preprint arXiv:2106.09685 (2021)."},{"key":"e_1_3_2_2_14_1","volume-title":"Adam: A method for stochastic optimization. arXiv preprint arXiv:1412.6980","author":"Kingma P","year":"2014","unstructured":"Diederik\u00a0P Kingma and Jimmy Ba. 2014. Adam: A method for stochastic optimization. arXiv preprint arXiv:1412.6980 (2014)."},{"key":"e_1_3_2_2_15_1","unstructured":"Junnan Li Dongxu Li Silvio Savarese and Steven Hoi. 2023. Blip-2: Bootstrapping language-image pre-training with frozen image encoders and large language models. In ICML."},{"key":"e_1_3_2_2_16_1","doi-asserted-by":"crossref","unstructured":"Chong Mou Xintao Wang Liangbin Xie Yanze Wu Jian Zhang Zhongang Qi and Ying Shan. 2024. T2i-adapter: Learning adapters to dig out more controllable ability for text-to-image diffusion models. In AAAI.","DOI":"10.1609\/aaai.v38i5.28226"},{"key":"e_1_3_2_2_17_1","unstructured":"Alec Radford Jong\u00a0Wook Kim Chris Hallacy Aditya Ramesh Gabriel Goh Sandhini Agarwal Girish Sastry Amanda Askell Pamela Mishkin Jack Clark 2021. Learning transferable visual models from natural language supervision. In ICML."},{"key":"e_1_3_2_2_18_1","volume-title":"Hierarchical text-conditional image generation with clip latents. arXiv preprint arXiv:2204.06125","author":"Ramesh Aditya","year":"2022","unstructured":"Aditya Ramesh, Prafulla Dhariwal, Alex Nichol, Casey Chu, and Mark Chen. 2022. Hierarchical text-conditional image generation with clip latents. arXiv preprint arXiv:2204.06125 (2022)."},{"key":"e_1_3_2_2_19_1","unstructured":"Aditya Ramesh Mikhail Pavlov Gabriel Goh Scott Gray Chelsea Voss Alec Radford Mark Chen and Ilya Sutskever. 2021. Zero-shot text-to-image generation. In ICML."},{"key":"e_1_3_2_2_20_1","doi-asserted-by":"crossref","unstructured":"Robin Rombach Andreas Blattmann Dominik Lorenz Patrick Esser and Bj\u00f6rn Ommer. 2022. High-resolution image synthesis with latent diffusion models. In CVPR.","DOI":"10.1109\/CVPR52688.2022.01042"},{"key":"e_1_3_2_2_21_1","volume-title":"U-net: Convolutional networks for biomedical image segmentation. In MICCAI.","author":"Ronneberger Olaf","year":"2015","unstructured":"Olaf Ronneberger, Philipp Fischer, and Thomas Brox. 2015. U-net: Convolutional networks for biomedical image segmentation. In MICCAI."},{"key":"e_1_3_2_2_22_1","volume-title":"Photorealistic text-to-image diffusion models with deep language understanding. NeuIPS","author":"Saharia Chitwan","year":"2022","unstructured":"Chitwan Saharia, William Chan, Saurabh Saxena, Lala Li, Jay Whang, Emily\u00a0L Denton, Kamyar Ghasemipour, Raphael Gontijo\u00a0Lopes, Burcu Karagol\u00a0Ayan, Tim Salimans, 2022. Photorealistic text-to-image diffusion models with deep language understanding. NeuIPS (2022)."},{"key":"e_1_3_2_2_23_1","doi-asserted-by":"crossref","unstructured":"Masaki Saito Eiichi Matsumoto and Shunta Saito. 2017. Temporal generative adversarial nets with singular value clipping. In ICCV.","DOI":"10.1109\/ICCV.2017.308"},{"key":"e_1_3_2_2_24_1","unstructured":"Maximilian Seitzer. 2020. pytorch-fid: FID Score for PyTorch. https:\/\/github.com\/mseitzer\/pytorch-fid."},{"key":"e_1_3_2_2_25_1","volume-title":"Make-a-video: Text-to-video generation without text-video data. arXiv preprint arXiv:2209.14792","author":"Singer Uriel","year":"2022","unstructured":"Uriel Singer, Adam Polyak, Thomas Hayes, Xi Yin, Jie An, Songyang Zhang, Qiyuan Hu, Harry Yang, Oron Ashual, Oran Gafni, 2022. Make-a-video: Text-to-video generation without text-video data. arXiv preprint arXiv:2209.14792 (2022)."},{"key":"e_1_3_2_2_26_1","volume-title":"Stylegan-v: A continuous video generator with the price, image quality and perks of stylegan2. In CVPR.","author":"Skorokhodov Ivan","year":"2022","unstructured":"Ivan Skorokhodov, Sergey Tulyakov, and Mohamed Elhoseiny. 2022. Stylegan-v: A continuous video generator with the price, image quality and perks of stylegan2. In CVPR."},{"key":"e_1_3_2_2_27_1","volume-title":"Mocogan: Decomposing motion and content for video generation. In CVPR.","author":"Tulyakov Sergey","year":"2018","unstructured":"Sergey Tulyakov, Ming-Yu Liu, Xiaodong Yang, and Jan Kautz. 2018. Mocogan: Decomposing motion and content for video generation. In CVPR."},{"key":"e_1_3_2_2_28_1","volume-title":"Towards accurate generative models of video: A new metric & challenges. arXiv preprint arXiv:1812.01717","author":"Unterthiner Thomas","year":"2018","unstructured":"Thomas Unterthiner, Sjoerd Van\u00a0Steenkiste, Karol Kurach, Raphael Marinier, Marcin Michalski, and Sylvain Gelly. 2018. Towards accurate generative models of video: A new metric & challenges. arXiv preprint arXiv:1812.01717 (2018)."},{"key":"e_1_3_2_2_29_1","volume-title":"Generating videos with scene dynamics. NeuIPS","author":"Vondrick Carl","year":"2016","unstructured":"Carl Vondrick, Hamed Pirsiavash, and Antonio Torralba. 2016. Generating videos with scene dynamics. NeuIPS (2016)."},{"key":"e_1_3_2_2_30_1","volume-title":"Few-shot video-to-video synthesis. arXiv preprint arXiv:1910.12713","author":"Wang Ting-Chun","year":"2019","unstructured":"Ting-Chun Wang, Ming-Yu Liu, Andrew Tao, Guilin Liu, Jan Kautz, and Bryan Catanzaro. 2019. Few-shot video-to-video synthesis. arXiv preprint arXiv:1910.12713 (2019)."},{"key":"e_1_3_2_2_31_1","volume-title":"VideoComposer: Compositional Video Synthesis with Motion Controllability. arXiv preprint arXiv:2306.02018","author":"Wang Xiang","year":"2023","unstructured":"Xiang Wang, Hangjie Yuan, Shiwei Zhang, Dayou Chen, Jiuniu Wang, Yingya Zhang, Yujun Shen, Deli Zhao, and Jingren Zhou. 2023. VideoComposer: Compositional Video Synthesis with Motion Controllability. arXiv preprint arXiv:2306.02018 (2023)."},{"key":"e_1_3_2_2_32_1","volume-title":"Tune-a-video: One-shot tuning of image diffusion models for text-to-video generation. In ICCV.","author":"Wu Jay\u00a0Zhangjie","year":"2023","unstructured":"Jay\u00a0Zhangjie Wu, Yixiao Ge, Xintao Wang, Stan\u00a0Weixian Lei, Yuchao Gu, Yufei Shi, Wynne Hsu, Ying Shan, Xiaohu Qie, and Mike\u00a0Zheng Shou. 2023b. Tune-a-video: One-shot tuning of image diffusion models for text-to-video generation. In ICCV."},{"key":"e_1_3_2_2_33_1","volume-title":"LAMP: Learn A Motion Pattern for Few-Shot-Based Video Generation. arXiv preprint arXiv:2310.10769","author":"Wu Ruiqi","year":"2023","unstructured":"Ruiqi Wu, Liangyu Chen, Tong Yang, Chunle Guo, Chongyi Li, and Xiangyu Zhang. 2023a. LAMP: Learn A Motion Pattern for Few-Shot-Based Video Generation. arXiv preprint arXiv:2310.10769 (2023)."},{"key":"e_1_3_2_2_34_1","unstructured":"Hongwei Xue Tiankai Hang Yanhong Zeng Yuchong Sun Bei Liu Huan Yang Jianlong Fu and Baining Guo. 2022. Advancing high-resolution video-language representation with large-scale video transcriptions. In CVPR."},{"key":"e_1_3_2_2_35_1","volume-title":"Dragnuwa: Fine-grained control in video generation by integrating text, image, and trajectory. arXiv preprint arXiv:2308.08089","author":"Yin Shengming","year":"2023","unstructured":"Shengming Yin, Chenfei Wu, Jian Liang, Jie Shi, Houqiang Li, Gong Ming, and Nan Duan. 2023a. Dragnuwa: Fine-grained control in video generation by integrating text, image, and trajectory. arXiv preprint arXiv:2308.08089 (2023)."},{"key":"e_1_3_2_2_36_1","volume-title":"Nuwa-xl: Diffusion over diffusion for extremely long video generation. arXiv preprint arXiv:2303.12346","author":"Yin Shengming","year":"2023","unstructured":"Shengming Yin, Chenfei Wu, Huan Yang, Jianfeng Wang, Xiaodong Wang, Minheng Ni, Zhengyuan Yang, Linjie Li, Shuguang Liu, Fan Yang, 2023b. Nuwa-xl: Diffusion over diffusion for extremely long video generation. arXiv preprint arXiv:2303.12346 (2023)."},{"key":"e_1_3_2_2_37_1","doi-asserted-by":"crossref","unstructured":"Lvmin Zhang Anyi Rao and Maneesh Agrawala. 2023. Adding conditional control to text-to-image diffusion models. In ICCV.","DOI":"10.1109\/ICCV51070.2023.00355"},{"key":"e_1_3_2_2_38_1","volume-title":"MotionDirector: Motion Customization of Text-to-Video Diffusion Models. arXiv preprint arXiv:2310.08465","author":"Zhao Rui","year":"2023","unstructured":"Rui Zhao, Yuchao Gu, Jay\u00a0Zhangjie Wu, David\u00a0Junhao Zhang, Jiawei Liu, Weijia Wu, Jussi Keppo, and Mike\u00a0Zheng Shou. 2023. MotionDirector: Motion Customization of Text-to-Video Diffusion Models. arXiv preprint arXiv:2310.08465 (2023)."},{"key":"e_1_3_2_2_39_1","volume-title":"Particlesfm: Exploiting dense point trajectories for localizing moving cameras in the wild. In ECCV.","author":"Zhao Wang","year":"2022","unstructured":"Wang Zhao, Shaohui Liu, Hengkai Guo, Wenping Wang, and Yong-Jin Liu. 2022. Particlesfm: Exploiting dense point trajectories for localizing moving cameras in the wild. In ECCV."},{"key":"e_1_3_2_2_40_1","volume-title":"Magicvideo: Efficient video generation with latent diffusion models. arXiv preprint arXiv:2211.11018","author":"Zhou Daquan","year":"2022","unstructured":"Daquan Zhou, Weimin Wang, Hanshu Yan, Weiwei Lv, Yizhe Zhu, and Jiashi Feng. 2022. Magicvideo: Efficient video generation with latent diffusion models. arXiv preprint arXiv:2211.11018 (2022)."},{"key":"e_1_3_2_2_41_1","volume-title":"Stereo magnification: Learning view synthesis using multiplane images. arXiv preprint arXiv:1805.09817","author":"Zhou Tinghui","year":"2018","unstructured":"Tinghui Zhou, Richard Tucker, John Flynn, Graham Fyffe, and Noah Snavely. 2018. Stereo magnification: Learning view synthesis using multiplane images. arXiv preprint arXiv:1805.09817 (2018)."},{"key":"e_1_3_2_2_42_1","volume-title":"Lafite: Towards language-free training for text-to-image generation. arXiv preprint arXiv:2111.13792","author":"Zhou Yufan","year":"2021","unstructured":"Yufan Zhou, Ruiyi Zhang, Changyou Chen, Chunyuan Li, Chris Tensmeyer, Tong Yu, Jiuxiang Gu, Jinhui Xu, and Tong Sun. 2021. Lafite: Towards language-free training for text-to-image generation. arXiv preprint arXiv:2111.13792 (2021)."}],"event":{"name":"SIGGRAPH '24: Special Interest Group on Computer Graphics and Interactive Techniques Conference","location":"Denver CO USA","acronym":"SIGGRAPH '24","sponsor":["SIGGRAPH ACM Special Interest Group on Computer Graphics and Interactive Techniques"]},"container-title":["Special Interest Group on Computer Graphics and Interactive Techniques Conference Conference Papers 24"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3641519.3657518","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3641519.3657518","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,8,22]],"date-time":"2025-08-22T19:17:15Z","timestamp":1755890235000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3641519.3657518"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,7,13]]},"references-count":42,"alternative-id":["10.1145\/3641519.3657518","10.1145\/3641519"],"URL":"https:\/\/doi.org\/10.1145\/3641519.3657518","relation":{},"subject":[],"published":{"date-parts":[[2024,7,13]]},"assertion":[{"value":"2024-07-13","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}