{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,8,5]],"date-time":"2026-08-05T11:50:30Z","timestamp":1785930630677,"version":"3.56.0"},"reference-count":46,"publisher":"Springer Science and Business Media LLC","issue":"5","license":[{"start":{"date-parts":[[2024,9,18]],"date-time":"2024-09-18T00:00:00Z","timestamp":1726617600000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2024,9,18]],"date-time":"2024-09-18T00:00:00Z","timestamp":1726617600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Vis Comput"],"published-print":{"date-parts":[[2025,3]]},"DOI":"10.1007\/s00371-024-03606-w","type":"journal-article","created":{"date-parts":[[2024,9,19]],"date-time":"2024-09-19T22:25:59Z","timestamp":1726784759000},"page":"3345-3357","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":3,"title":["Video-driven musical composition using large language model with memory-augmented state space"],"prefix":"10.1007","volume":"41","author":[{"given":"Wan-He","family":"Kai","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Kai-Xin","family":"Xing","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2024,9,18]]},"reference":[{"key":"3606_CR1","doi-asserted-by":"crossref","unstructured":"Yin, S., Fu, C., Zhao, S., Li, K., Sun, X., Xu, T., Chen, E.: A survey on multimodal large language models. arXiv preprint arXiv:2306.13549 (2023)","DOI":"10.1093\/nsr\/nwae403"},{"key":"3606_CR2","unstructured":"Hussain, A.S., Liu, S., Sun, C., Shan, Y.: $$\\text{M}^2$$ugen: multi-modal music understanding and generation with the power of large language models. arXiv preprint arXiv:2311.11255 (2023)"},{"key":"3606_CR3","unstructured":"Dai, L., Sheng, B., Chen, T., Wu, Q., Liu, R., Cai, C., Wu, L., Yang, D., Hamzah, H., Liu, Y., et al.: A deep learning system for predicting time to progression of diabetic retinopathy. Nat. Med. 1\u201311 (2024)"},{"key":"3606_CR4","doi-asserted-by":"crossref","unstructured":"Qian, B., Chen, H., Wang, X., Guan, Z., Li, T., Jin, Y., Wu, Y., Wen, Y., Che, H., Kwon, G., et al.: Drac 2022: a public benchmark for diabetic retinopathy analysis on ultra-wide optical coherence tomography angiography images. Patterns (2024)","DOI":"10.1016\/j.patter.2024.100929"},{"issue":"1","key":"3606_CR5","doi-asserted-by":"publisher","first-page":"3242","DOI":"10.1038\/s41467-021-23458-5","volume":"12","author":"L Dai","year":"2021","unstructured":"Dai, L., Wu, L., Li, H., Cai, C., Wu, Q., Kong, H., Liu, R., Wang, X., Hou, X., Liu, Y., et al.: A deep learning system for detecting diabetic retinopathy across the disease spectrum. Nat. Commun. 12(1), 3242 (2021)","journal-title":"Nat. Commun."},{"key":"3606_CR6","doi-asserted-by":"crossref","unstructured":"Qin, Y., Zhao, N., Yang, J., Pan, S., Sheng, B., Lau, R.W.: Urbanevolver: function-aware urban layout regeneration. Int. J. Comput. Vis. 1\u201320 (2024)","DOI":"10.1007\/s11263-024-02030-w"},{"key":"3606_CR7","unstructured":"Agostinelli, A., Denk, T.I., Borsos, Z., Engel, J., Verzetti, M., Caillon, A., Huang, Q., Jansen, A., Roberts, A., Tagliasacchi, M., et al.: Musiclm: generating music from text. arXiv preprint arXiv:2301.11325 (2023)"},{"key":"3606_CR8","unstructured":"Copet, J., Kreuk, F., Gat, I., Remez, T., Kant, D., Synnaeve, G., Adi, Y., D\u00e9fossez, A.: Simple and controllable music generation. Adv. Neural Inf. Process. Syst. 36 (2024)"},{"key":"3606_CR9","doi-asserted-by":"crossref","unstructured":"Liu, H., Tian, Q., Yuan, Y., Liu, X., Mei, X., Kong, Q., Wang, Y., Wang, W., Wang, Y., Plumbley, M.D.: Audioldm 2: learning holistic audio generation with self-supervised pretraining. arXiv preprint arXiv:2308.05734 (2023)","DOI":"10.1109\/TASLP.2024.3399607"},{"key":"3606_CR10","unstructured":"Zhang, R., Zhang, Y., Shao, K., Shan, Y., Xia, G.: Vis2mus: Exploring multimodal representation mapping for controllable music generation. arXiv preprint arXiv:2211.05543 (2022)"},{"key":"3606_CR11","doi-asserted-by":"crossref","unstructured":"Di, S., Jiang, Z., Liu, S., Wang, Z., Zhu, L., He, Z., Liu, H., Yan, S.: Video background music generation with controllable music transformer. In: Proceedings of the 29th ACM International Conference on Multimedia, pp. 2037\u20132045 (2021)","DOI":"10.1145\/3474085.3475195"},{"key":"3606_CR12","doi-asserted-by":"crossref","unstructured":"Zhuo, L., Wang, Z., Wang, B., Liao, Y., Bao, C., Peng, S., Han, S., Zhang, A., Fang, F., Liu, S.: Video background music generation: dataset, method and evaluation. In: Proceedings of the IEEE\/CVF International Conference on Computer Vision, pp. 15637\u201315647 (2023)","DOI":"10.1109\/ICCV51070.2023.01433"},{"key":"3606_CR13","doi-asserted-by":"crossref","unstructured":"He, B., Li, H., Jang, Y.K., Jia, M., Cao, X., Shah, A., Shrivastava, A., Lim, S.-N.: Ma-lmm: Memory-augmented large multimodal model for long-term video understanding. arXiv preprint arXiv:2404.05726 (2024)","DOI":"10.1109\/CVPR52733.2024.01282"},{"key":"3606_CR14","doi-asserted-by":"crossref","unstructured":"Maaz, M., Rasheed, H., Khan, S., Khan, F.S.: Video-chatgpt: Towards detailed video understanding via large vision and language models. arXiv preprint arXiv:2306.05424 (2023)","DOI":"10.18653\/v1\/2024.acl-long.679"},{"key":"3606_CR15","unstructured":"Vaswani, A., Shazeer, N., Parmar, N., Uszkoreit, J., Jones, L., Gomez, A.N., Kaiser, \u0141., Polosukhin, I.: Attention is all you need. Adv. Neural Inf. Process. Syst. 30 (2017)"},{"key":"3606_CR16","unstructured":"Gu, A., Dao, T.: Mamba: Linear-time sequence modeling with selective state spaces. arXiv preprint arXiv:2312.00752 (2023)"},{"key":"3606_CR17","unstructured":"Yizhi, L., Yuan, R., Zhang, G., Ma, Y., Chen, X., Yin, H., Xiao, C., Lin, C., Ragni, A., Benetos, E., et\u00a0al.: Mert: acoustic music understanding model with large-scale self-supervised training. In: The Twelfth International Conference on Learning Representations (2023)"},{"issue":"5","key":"3606_CR18","doi-asserted-by":"publisher","first-page":"2735","DOI":"10.1109\/TCYB.2019.2934823","volume":"51","author":"H Guo","year":"2019","unstructured":"Guo, H., Sheng, B., Li, P., Chen, C.P.: Multiview high dynamic range image synthesis using fuzzy broad learning system. IEEE Trans. Cybern. 51(5), 2735\u20132747 (2019)","journal-title":"IEEE Trans. Cybern."},{"key":"3606_CR19","unstructured":"Touvron, H., Martin, L., Stone, K., Albert, P., Almahairi, A., Babaei, Y., Bashlykov, N., Batra, S., Bhargava, P., Bhosale, S., et al.: Llama 2: open foundation and fine-tuned chat models. arXiv preprint arXiv:2307.09288 (2023)"},{"key":"3606_CR20","doi-asserted-by":"crossref","unstructured":"Liu, Z., Ning, J., Cao, Y., Wei, Y., Zhang, Z., Lin, S., Hu, H.: Video swin transformer. In: Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition, pp. 3202\u20133211 (2022)","DOI":"10.1109\/CVPR52688.2022.00320"},{"key":"3606_CR21","doi-asserted-by":"crossref","unstructured":"Cao, Z., Wu, X., Deng, L.-J., Zhong, Y.: A novel state space model with local enhancement and state sharing for image fusion. arXiv preprint arXiv:2404.09293 (2024)","DOI":"10.1145\/3664647.3680905"},{"issue":"5","key":"3606_CR22","doi-asserted-by":"publisher","first-page":"2152","DOI":"10.1109\/TCYB.2018.2875983","volume":"50","author":"Z Chen","year":"2018","unstructured":"Chen, Z., Gao, T., Sheng, B., Li, P., Chen, C.P.: Outdoor shadow estimating using multiclass geometric decomposition based on bls. IEEE Trans. Cybern. 50(5), 2152\u20132165 (2018)","journal-title":"IEEE Trans. Cybern."},{"key":"3606_CR23","doi-asserted-by":"crossref","unstructured":"Zhang, H., Li, X., Bing, L.: Video-llama: an instruction-tuned audio-visual language model for video understanding. arXiv preprint arXiv:2306.02858 (2023)","DOI":"10.18653\/v1\/2023.emnlp-demo.49"},{"key":"3606_CR24","unstructured":"Yang, L., Yang, H., Cheng, W., Lin, L., Li, C., Chen, Y., Liu, L., Pan, J., Wei, T., Li, B., et al.: Skymath: Technical report. arXiv preprint arXiv:2310.16713 (2023)"},{"key":"3606_CR25","first-page":"1877","volume":"33","author":"T Brown","year":"2020","unstructured":"Brown, T., Mann, B., Ryder, N., Subbiah, M., Kaplan, J.D., Dhariwal, P., Neelakantan, A., Shyam, P., Sastry, G., Askell, A., et al.: Language models are few-shot learners. Adv. Neural. Inf. Process. Syst. 33, 1877\u20131901 (2020)","journal-title":"Adv. Neural. Inf. Process. Syst."},{"key":"3606_CR26","first-page":"6840","volume":"33","author":"J Ho","year":"2020","unstructured":"Ho, J., Jain, A., Abbeel, P.: Denoising diffusion probabilistic models. Adv. Neural. Inf. Process. Syst. 33, 6840\u20136851 (2020)","journal-title":"Adv. Neural. Inf. Process. Syst."},{"key":"3606_CR27","unstructured":"Huang, Q., Park, D.S., Wang, T., Denk, T.I., Ly, A., Chen, N., Zhang, Z., Zhang, Z., Yu, J., Frank, C., et al.: Noise2music: text-conditioned music generation with diffusion models. arXiv preprint arXiv:2302.03917 (2023)"},{"key":"3606_CR28","unstructured":"Radford, A., Kim, J.W., Hallacy, C., Ramesh, A., Goh, G., Agarwal, S., Sastry, G., Askell, A., Mishkin, P., Clark, J., et\u00a0al.: Learning transferable visual models from natural language supervision. In: International Conference on Machine Learning, pp. 8748\u20138763. PMLR (2021)"},{"key":"3606_CR29","unstructured":"Gu, A., Goel, K., R\u00e9, C.: Efficiently modeling long sequences with structured state spaces. arXiv preprint arXiv:2111.00396 (2021)"},{"key":"3606_CR30","first-page":"2846","volume":"35","author":"E Nguyen","year":"2022","unstructured":"Nguyen, E., Goel, K., Gu, A., Downs, G., Shah, P., Dao, T., Baccus, S., R\u00e9, C.: S4nd: modeling images and videos as multidimensional signals with state spaces. Adv. Neural. Inf. Process. Syst. 35, 2846\u20132861 (2022)","journal-title":"Adv. Neural. Inf. Process. Syst."},{"key":"3606_CR31","unstructured":"Liu, Y., Tian, Y., Zhao, Y., Yu, H., Xie, L., Wang, Y., Ye, Q., Liu, Y.: Vmamba: visual state space model. arXiv preprint arXiv:2401.10166 (2024)"},{"key":"3606_CR32","unstructured":"Zhu, L., Liao, B., Zhang, Q., Wang, X., Liu, W., Wang, X.: Vision mamba: efficient visual representation learning with bidirectional state space model. arXiv preprint arXiv:2401.09417 (2024)"},{"issue":"1","key":"3606_CR33","doi-asserted-by":"publisher","first-page":"163","DOI":"10.1109\/TII.2021.3085669","volume":"18","author":"J Li","year":"2021","unstructured":"Li, J., Chen, J., Sheng, B., Li, P., Yang, P., Feng, D.D., Qi, J.: Automatic detection and classification system of domestic waste via multimodel cascaded convolutional neural network. IEEE Trans. Ind. Inf. 18(1), 163\u2013173 (2021)","journal-title":"IEEE Trans. Ind. Inf."},{"issue":"8","key":"3606_CR34","doi-asserted-by":"publisher","first-page":"4499","DOI":"10.1109\/TNNLS.2021.3116209","volume":"34","author":"Z Xie","year":"2021","unstructured":"Xie, Z., Zhang, W., Sheng, B., Li, P., Chen, C.P.: Bagfn: broad attentive graph fusion network for high-order feature interactions. IEEE Trans. Neural Netw. Learn. Syst. 34(8), 4499\u20134513 (2021)","journal-title":"IEEE Trans. Neural Netw. Learn. Syst."},{"key":"3606_CR35","doi-asserted-by":"publisher","first-page":"50","DOI":"10.1109\/TMM.2021.3120873","volume":"25","author":"X Lin","year":"2021","unstructured":"Lin, X., Sun, S., Huang, W., Sheng, B., Li, P., Feng, D.D.: Eapt: efficient attention pyramid transformer for image processing. IEEE Trans. Multimedia 25, 50\u201361 (2021)","journal-title":"IEEE Trans. Multimedia"},{"issue":"7","key":"3606_CR36","doi-asserted-by":"publisher","first-page":"6662","DOI":"10.1109\/TCYB.2021.3079311","volume":"52","author":"B Sheng","year":"2021","unstructured":"Sheng, B., Li, P., Ali, R., Chen, C.P.: Improving video temporal consistency via broad learning system. IEEE Trans. Cybern. 52(7), 6662\u20136675 (2021)","journal-title":"IEEE Trans. Cybern."},{"key":"3606_CR37","doi-asserted-by":"crossref","unstructured":"Cheng, Z., Yang, Q., Sheng, B.: Deep colorization. In: Proceedings of the IEEE International Conference on Computer Vision, pp. 415\u2013423 (2015)","DOI":"10.1109\/ICCV.2015.55"},{"issue":"2","key":"3606_CR38","doi-asserted-by":"publisher","first-page":"1332","DOI":"10.1109\/TVCG.2018.2869326","volume":"26","author":"B Sheng","year":"2018","unstructured":"Sheng, B., Li, P., Jin, Y., Tan, P., Lee, T.-Y.: Intrinsic image decomposition with step and drift shading separation. IEEE Trans. Vis. Comput. Graph. 26(2), 1332\u20131346 (2018)","journal-title":"IEEE Trans. Vis. Comput. Graph."},{"issue":"8","key":"3606_CR39","doi-asserted-by":"publisher","first-page":"2546","DOI":"10.1109\/TVCG.2019.2894627","volume":"26","author":"B Zhang","year":"2019","unstructured":"Zhang, B., Sheng, B., Li, P., Lee, T.-Y.: Depth of field rendering using multilayer-neighborhood optimization. IEEE Trans. Visual Comput. Graph. 26(8), 2546\u20132559 (2019)","journal-title":"IEEE Trans. Visual Comput. Graph."},{"key":"3606_CR40","doi-asserted-by":"publisher","first-page":"7192","DOI":"10.1109\/TIP.2020.2999854","volume":"29","author":"A Nazir","year":"2020","unstructured":"Nazir, A., Cheema, M.N., Sheng, B., Li, H., Li, P., Yang, P., Jung, Y., Qin, J., Kim, J., Feng, D.D.: Off-enet: an optimally fused fully end-to-end network for automatic dense volumetric 3d intracranial blood vessels segmentation. IEEE Trans. Image Process. 29, 7192\u20137202 (2020)","journal-title":"IEEE Trans. Image Process."},{"issue":"4","key":"3606_CR41","doi-asserted-by":"publisher","first-page":"600","DOI":"10.1109\/TIP.2003.819861","volume":"13","author":"Z Wang","year":"2004","unstructured":"Wang, Z., Bovik, A.C., Sheikh, H.R., Simoncelli, E.P.: Image quality assessment: from error visibility to structural similarity. IEEE Trans. Image Process. 13(4), 600\u2013612 (2004)","journal-title":"IEEE Trans. Image Process."},{"key":"3606_CR42","doi-asserted-by":"crossref","unstructured":"Chen, Z., Qiu, G., Li, P., Zhu, L., Yang, X., Sheng, B.: Mngnas: distilling adaptive combination of multiple searched networks for one-shot neural architecture search. IEEE Trans. Pattern Anal. Mach. Intell. (2023)","DOI":"10.1109\/TPAMI.2023.3293885"},{"key":"3606_CR43","doi-asserted-by":"crossref","unstructured":"Jiang, N., Sheng, B., Li, P., Lee, T.-Y.: Photohelper: portrait photographing guidance via deep feature retrieval and fusion. IEEE Trans. Multimed. (2022)","DOI":"10.1109\/TMM.2022.3144890"},{"key":"3606_CR44","unstructured":"Hu, E.J., Shen, Y., Wallis, P., Allen-Zhu, Z., Li, Y., Wang, S., Wang, L., Chen, W.: Lora: low-rank adaptation of large language models. arXiv preprint arXiv:2106.09685 (2021)"},{"key":"3606_CR45","doi-asserted-by":"crossref","unstructured":"Kilgour, K., Zuluaga, M., Roblek, D., Sharifi, M.: Fr\u00e9chet Audio distance: a reference-free metric for evaluating music enhancement algorithms. In: Interspeech (2019)","DOI":"10.21437\/Interspeech.2019-2219"},{"key":"3606_CR46","unstructured":"Tang, Z., Yang, Z., Zhu, C., Zeng, M., Bansal, M.: Any-to-Any Generation via Composable Diffusion. arXiv preprint arXiv:2305.11846 (2023)"}],"container-title":["The Visual Computer"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s00371-024-03606-w.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s00371-024-03606-w\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s00371-024-03606-w.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,3,10]],"date-time":"2025-03-10T09:08:25Z","timestamp":1741597705000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s00371-024-03606-w"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,9,18]]},"references-count":46,"journal-issue":{"issue":"5","published-print":{"date-parts":[[2025,3]]}},"alternative-id":["3606"],"URL":"https:\/\/doi.org\/10.1007\/s00371-024-03606-w","relation":{},"ISSN":["0178-2789","1432-2315"],"issn-type":[{"value":"0178-2789","type":"print"},{"value":"1432-2315","type":"electronic"}],"subject":[],"published":{"date-parts":[[2024,9,18]]},"assertion":[{"value":"8 August 2024","order":1,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"18 September 2024","order":2,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Declarations"}},{"value":"The authors declare no competing interests.","order":2,"name":"Ethics","group":{"name":"EthicsHeading","label":"Competing interests"}}]}}