{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,10]],"date-time":"2026-04-10T05:38:09Z","timestamp":1775799489027,"version":"3.50.1"},"reference-count":57,"publisher":"Springer Science and Business Media LLC","issue":"1","license":[{"start":{"date-parts":[[2024,11,9]],"date-time":"2024-11-09T00:00:00Z","timestamp":1731110400000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by-nc-nd\/4.0"},{"start":{"date-parts":[[2024,11,9]],"date-time":"2024-11-09T00:00:00Z","timestamp":1731110400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by-nc-nd\/4.0"}],"funder":[{"DOI":"10.13039\/501100012166","name":"National Key R&D Program of China","doi-asserted-by":"crossref","award":["No. 2023YFC3805703"],"award-info":[{"award-number":["No. 2023YFC3805703"]}],"id":[{"id":"10.13039\/501100012166","id-type":"DOI","asserted-by":"crossref"}]}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Complex Intell. Syst."],"published-print":{"date-parts":[[2025,1]]},"DOI":"10.1007\/s40747-024-01654-2","type":"journal-article","created":{"date-parts":[[2024,11,9]],"date-time":"2024-11-09T07:34:58Z","timestamp":1731137698000},"update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":2,"title":["Audio-visual event localization with dual temporal-aware scene understanding and image-text knowledge bridging"],"prefix":"10.1007","volume":"11","author":[{"given":"Pufen","family":"Zhang","sequence":"first","affiliation":[]},{"given":"Jiaxiang","family":"Wang","sequence":"additional","affiliation":[]},{"given":"Meng","family":"Wan","sequence":"additional","affiliation":[]},{"given":"Song","family":"Zhang","sequence":"additional","affiliation":[]},{"given":"Jie","family":"Jing","sequence":"additional","affiliation":[]},{"given":"Lianhong","family":"Ding","sequence":"additional","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0002-5349-6383","authenticated-orcid":false,"given":"Peng","family":"Shi","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2024,11,9]]},"reference":[{"key":"1654_CR1","doi-asserted-by":"crossref","unstructured":"Baldrati A, Bertini M, Uricchio T et\u00a0al (2022) Effective conditioned and composed image retrieval combining clip-based features. In: Proceedings of the IEEE\/CVF conference on computer vision and pattern recognition, pp 21466\u201321474","DOI":"10.1109\/CVPR52688.2022.02080"},{"key":"1654_CR2","doi-asserted-by":"crossref","unstructured":"Chen H, Xie W, Vedaldi A et\u00a0al (2020) Vggsound: a large-scale audio-visual dataset. In: ICASSP 2020-2020 IEEE international conference on acoustics, speech and signal processing (ICASSP). IEEE, pp 721\u2013725","DOI":"10.1109\/ICASSP40776.2020.9053174"},{"key":"1654_CR3","doi-asserted-by":"crossref","unstructured":"Chen K, Du X, Zhu B et\u00a0al (2022) Hts-at: a hierarchical token-semantic audio transformer for sound classification and detection. In: ICASSP 2022-2022 IEEE international conference on acoustics, speech and signal processing (ICASSP). IEEE, pp 646\u2013650","DOI":"10.1109\/ICASSP43922.2022.9746312"},{"key":"1654_CR4","doi-asserted-by":"crossref","unstructured":"Chen ZM, Wei XS, Wang P et\u00a0al (2019) Multi-label image recognition with graph convolutional networks. In: Proceedings of the IEEE\/CVF conference on computer vision and pattern recognition, pp 5177\u20135186","DOI":"10.1109\/CVPR.2019.00532"},{"key":"1654_CR5","unstructured":"Devlin J, Chang MW, Lee K et\u00a0al (2019) BERT: pre-training of deep bidirectional transformers for language understanding. In: Proceedings of the 2019 conference of the North American chapter of the Association for Computational Linguistics: Human Language Technologies, Volume 1 (Long and Short Papers), pp 4171\u20134186"},{"key":"1654_CR6","unstructured":"Dosovitskiy A, Beyer L, Kolesnikov A et\u00a0al (2020) An image is worth 16 $$\\times $$ 16 words: transformers for image recognition at scale. arXiv preprint arXiv:2010.11929"},{"key":"1654_CR7","doi-asserted-by":"crossref","unstructured":"Duan B, Tang H, Wang W et\u00a0al (2021) Audio-visual event localization via recursive fusion by joint co-attention. In: Proceedings of the IEEE\/CVF winter conference on applications of computer vision, pp 4013\u20134022","DOI":"10.1109\/WACV48630.2021.00406"},{"key":"1654_CR8","unstructured":"Duan H, Xia Y, Zhou M et\u00a0al (2023) Cross-modal prompts: adapting large pre-trained models for audio-visual downstream tasks, pp 56075\u201356094"},{"key":"1654_CR9","doi-asserted-by":"crossref","unstructured":"Feng G, Hu Z, Zhang L et\u00a0al (2021) Encoder fusion network with co-attention embedding for referring image segmentation. In: Proceedings of the IEEE\/CVF conference on computer vision and pattern recognition, pp 15506\u201315515","DOI":"10.1109\/CVPR46437.2021.01525"},{"key":"1654_CR10","doi-asserted-by":"crossref","unstructured":"Gemmeke JF, Ellis DP, Freedman D et\u00a0al (2017) Audio set: an ontology and human-labeled dataset for audio events. In: 2017 IEEE international conference on acoustics, speech and signal processing (ICASSP), IEEE, pp 776\u2013780","DOI":"10.1109\/ICASSP.2017.7952261"},{"key":"1654_CR11","doi-asserted-by":"crossref","unstructured":"Gong Y, Chung YA, Glass J (2021) AST: audio spectrogram transformer. arXiv preprint arXiv:2104.01778","DOI":"10.21437\/Interspeech.2021-698"},{"key":"1654_CR12","doi-asserted-by":"crossref","unstructured":"Guzhov A, Raue F, Hees J et\u00a0al (2022) Audioclip: extending clip to image, text and audio. In: ICASSP 2022-2022 IEEE international conference on acoustics, speech and signal processing (ICASSP). IEEE, pp 976\u2013980","DOI":"10.1109\/ICASSP43922.2022.9747631"},{"key":"1654_CR13","doi-asserted-by":"crossref","unstructured":"He K, Zhang X, Ren S et\u00a0al (2016) Deep residual learning for image recognition. In: Proceedings of the IEEE conference on computer vision and pattern recognition, pp 770\u2013778","DOI":"10.1109\/CVPR.2016.90"},{"key":"1654_CR14","doi-asserted-by":"crossref","unstructured":"He Y, Xu X, Liu X et\u00a0al (2021) Multimodal transformer networks with latent interaction for audio-visual event localization. In: 2021 IEEE international conference on multimedia and expo (ICME). IEEE, pp 1\u20136","DOI":"10.1109\/ICME51207.2021.9428081"},{"key":"1654_CR15","doi-asserted-by":"crossref","unstructured":"Hershey S, Chaudhuri S, Ellis DP et\u00a0al (2017) CNN architectures for large-scale audio classification. In: 2017 IEEE international conference on acoustics, speech and signal processing (ICASSP). IEEE, pp 131\u2013135","DOI":"10.1109\/ICASSP.2017.7952132"},{"key":"1654_CR16","doi-asserted-by":"publisher","first-page":"4617","DOI":"10.1109\/TMM.2023.3324498","volume":"26","author":"Y Jiang","year":"2024","unstructured":"Jiang Y, Yin J, Dang Y (2024) Leveraging the video-level semantic consistency of event for audio-visual event localization. IEEE Trans Multimed 26:4617\u20134627","journal-title":"IEEE Trans Multimed"},{"issue":"6","key":"1654_CR17","doi-asserted-by":"publisher","first-page":"7213","DOI":"10.1007\/s40747-023-01102-7","volume":"9","author":"Z Jiang","year":"2023","unstructured":"Jiang Z, Dong Y, Pei Y et al (2023) Multiple space based cascaded center point network for object detection. Complex Intell Syst 9(6):7213\u20137225","journal-title":"Complex Intell Syst"},{"key":"1654_CR18","unstructured":"Kingma DP, Ba J (2014) Adam: a method for stochastic optimization. arXiv preprint arXiv:1412.6980"},{"key":"1654_CR19","doi-asserted-by":"publisher","first-page":"107872","DOI":"10.1016\/j.knosys.2021.107872","volume":"238","author":"J Kong","year":"2022","unstructured":"Kong J, Wang J, Zhang X (2022) Hierarchical BERT with an adaptive fine-tuning strategy for document classification. Knowl-Based Syst 238:107872","journal-title":"Knowl-Based Syst"},{"key":"1654_CR20","doi-asserted-by":"publisher","first-page":"2880","DOI":"10.1109\/TASLP.2020.3030497","volume":"28","author":"Q Kong","year":"2020","unstructured":"Kong Q, Cao Y, Iqbal T et al (2020) PANNS: large-scale pretrained audio neural networks for audio pattern recognition. IEEE\/ACM Trans Audio Speech Lang Process 28:2880\u20132894","journal-title":"IEEE\/ACM Trans Audio Speech Lang Process"},{"key":"1654_CR21","doi-asserted-by":"crossref","unstructured":"Li G, Wei Y, Tian Y et\u00a0al (2022) Learning to answer questions in dynamic audio-visual scenarios. In: Proceedings of the IEEE\/CVF conference on computer vision and pattern recognition, pp 19108\u201319118","DOI":"10.1109\/CVPR52688.2022.01852"},{"key":"1654_CR22","doi-asserted-by":"crossref","unstructured":"Li G, Hou W, Hu D (2023) Progressive spatio-temporal perception for audio-visual question answering. In: Proceedings of the 31st ACM international conference on multimedia, pp 7808\u20137816","DOI":"10.1145\/3581783.3612293"},{"key":"1654_CR23","doi-asserted-by":"crossref","unstructured":"Li T, Bai J, Wang Q (2024) Enhancing medical text detection with vision-language pre-training and efficient segmentation. Complex Intell Syst 1\u201313","DOI":"10.1007\/s40747-024-01378-3"},{"key":"1654_CR24","doi-asserted-by":"crossref","unstructured":"Li Y, Mao H, Girshick R et al (2022) Exploring plain vision transformer backbones for object detection. In: European conference on computer vision. Springer, Berlin, pp 280\u2013296","DOI":"10.1007\/978-3-031-20077-9_17"},{"key":"1654_CR25","doi-asserted-by":"crossref","unstructured":"Lin YB, Li YJ, Wang YCF (2019) Dual-modality seq2seq network for audio-visual event localization. In: ICASSP 2019-2019 IEEE international conference on acoustics, speech and signal processing (ICASSP). IEEE, pp 2002\u20132006","DOI":"10.1109\/ICASSP.2019.8683226"},{"key":"1654_CR26","doi-asserted-by":"crossref","unstructured":"Lin YB, Sung YL, Lei J et\u00a0al (2023) Vision transformers are parameter-efficient audio-visual learners. In: Proceedings of the IEEE\/CVF conference on computer vision and pattern recognition, pp 2299\u20132309","DOI":"10.1109\/CVPR52729.2023.00228"},{"key":"1654_CR27","doi-asserted-by":"crossref","unstructured":"Liu S, Quan W, Liu Y et\u00a0al (2022) Bi-directional modality fusion network for audio-visual event localization. In: ICASSP 2022-2022 IEEE International conference on acoustics, speech and signal processing (ICASSP). IEEE, pp 4868\u20134872","DOI":"10.1109\/ICASSP43922.2022.9746280"},{"key":"1654_CR28","doi-asserted-by":"publisher","first-page":"2734","DOI":"10.1109\/TMM.2022.3150469","volume":"25","author":"S Liu","year":"2022","unstructured":"Liu S, Quan W, Wang C et al (2022) Dense modality interaction network for audio-visual event localization. IEEE Trans Multimed 25:2734\u20132748","journal-title":"IEEE Trans Multimed"},{"key":"1654_CR29","doi-asserted-by":"crossref","unstructured":"Liu Z, Lin Y, Cao Y et\u00a0al (2021) Swin transformer: Hierarchical vision transformer using shifted windows. In: Proceedings of the IEEE\/CVF international conference on computer vision, pp 10012\u201310022","DOI":"10.1109\/ICCV48922.2021.00986"},{"key":"1654_CR30","doi-asserted-by":"publisher","first-page":"293","DOI":"10.1016\/j.neucom.2022.07.028","volume":"508","author":"H Luo","year":"2022","unstructured":"Luo H, Ji L, Zhong M et al (2022) Clip4clip: n empirical study of clip for end to end video clip retrieval and captioning. Neurocomputing 508:293\u2013304","journal-title":"Neurocomputing"},{"key":"1654_CR31","doi-asserted-by":"crossref","unstructured":"Ma Y, Xu G, Sun X et\u00a0al (2022) X-clip: end-to-end multi-grained contrastive learning for video-text retrieval. In: Proceedings of the 30th ACM international conference on multimedia, pp 638\u2013647","DOI":"10.1145\/3503161.3547910"},{"key":"1654_CR32","first-page":"2579","volume":"9","author":"L van der Maaten","year":"2008","unstructured":"van der Maaten L, Hinton G (2008) Visualizing data using t-SNE. J Mach Learn Res 9:2579\u20132605","journal-title":"J Mach Learn Res"},{"key":"1654_CR33","doi-asserted-by":"crossref","unstructured":"Mahmud T, Marculescu D (2023) Ave-clip: audioclip-based multi-window temporal transformer for audio visual event localization. In: Proceedings of the IEEE\/CVF Winter conference on applications of computer vision, pp 5158\u20135167","DOI":"10.1109\/WACV56688.2023.00513"},{"issue":"9","key":"1654_CR34","doi-asserted-by":"publisher","first-page":"10602","DOI":"10.1007\/s10489-022-04052-8","volume":"53","author":"K Nassiri","year":"2023","unstructured":"Nassiri K, Akhloufi M (2023) Transformer models used for text-based question answering systems. Appl Intell 53(9):10602\u201310635","journal-title":"Appl Intell"},{"key":"1654_CR35","unstructured":"Radford A, Kim JW, Hallacy C et\u00a0al (2021) Learning transferable visual models from natural language supervision. In: International conference on machine learning. PMLR, pp 8748\u20138763"},{"key":"1654_CR36","doi-asserted-by":"crossref","unstructured":"Rajpurkar P, Zhang J, Lopyrev K et\u00a0al (2016) Squad: 100,000+ questions for machine comprehension of text. arXiv preprint arXiv:1606.05250","DOI":"10.18653\/v1\/D16-1264"},{"key":"1654_CR37","doi-asserted-by":"crossref","unstructured":"Ramaswamy J (2020) What makes the sound?: A dual-modality interacting network for audio-visual event localization. In: ICASSP 2020-2020 IEEE international conference on acoustics, speech and signal processing (ICASSP). IEEE, pp 4372\u20134376","DOI":"10.1109\/ICASSP40776.2020.9053895"},{"key":"1654_CR38","doi-asserted-by":"crossref","unstructured":"Ramaswamy J, Das S (2020) See the sound, hear the pixels. In: Proceedings of the IEEE\/CVF winter conference on applications of computer vision, pp 2970\u20132979","DOI":"10.1109\/WACV45572.2020.9093616"},{"issue":"6","key":"1654_CR39","doi-asserted-by":"publisher","first-page":"1137","DOI":"10.1109\/TPAMI.2016.2577031","volume":"39","author":"S Ren","year":"2016","unstructured":"Ren S, He K, Girshick R et al (2016) Faster R-CNN: towards real-time object detection with region proposal networks. IEEE Trans Pattern Anal Mach Intell 39(6):1137\u20131149","journal-title":"IEEE Trans Pattern Anal Mach Intell"},{"key":"1654_CR40","doi-asserted-by":"publisher","first-page":"128","DOI":"10.1016\/j.patrec.2021.11.010","volume":"155","author":"AC Rouhou","year":"2022","unstructured":"Rouhou AC, Dhiaf M, Kessentini Y et al (2022) Transformer-based approach for joint handwriting and named entity recognition in historical document. Pattern Recogn Lett 155:128\u2013134","journal-title":"Pattern Recogn Lett"},{"issue":"11","key":"1654_CR41","doi-asserted-by":"publisher","first-page":"2673","DOI":"10.1109\/78.650093","volume":"45","author":"M Schuster","year":"1997","unstructured":"Schuster M, Paliwal KK (1997) Bidirectional recurrent neural networks. IEEE Trans Signal Process 45(11):2673\u20132681","journal-title":"IEEE Trans Signal Process"},{"key":"1654_CR42","unstructured":"Simonyan K, Zisserman A (2014) Very deep convolutional networks for large-scale image recognition. arXiv preprint arXiv:1409.1556"},{"key":"1654_CR43","doi-asserted-by":"crossref","unstructured":"Tian Y, Shi J, Li B et\u00a0al (2018) Audio-visual event localization in unconstrained videos. In: Proceedings of the European conference on computer vision (ECCV), pp 247\u2013263","DOI":"10.1007\/978-3-030-01216-8_16"},{"key":"1654_CR44","unstructured":"Vaswani A, Shazeer N, Parmar N et\u00a0al (2017) Attention is all you need. In: Advances in neural information processing systems, vol 30"},{"key":"1654_CR45","doi-asserted-by":"crossref","unstructured":"Wang H, Liao J, Cheng T et\u00a0al (2022) Knowledge mining with scene text for fine-grained recognition. In: Proceedings of the IEEE\/CVF conference on computer vision and pattern recognition, pp 4624\u20134633","DOI":"10.1109\/CVPR52688.2022.00458"},{"issue":"6","key":"1654_CR46","doi-asserted-by":"publisher","first-page":"7711","DOI":"10.1109\/TPAMI.2022.3226328","volume":"45","author":"H Wang","year":"2022","unstructured":"Wang H, Zha ZJ, Li L et al (2022) Semantic and relation modulation for audio-visual event localization. IEEE Trans Pattern Anal Mach Intell 45(6):7711\u20137725","journal-title":"IEEE Trans Pattern Anal Mach Intell"},{"key":"1654_CR47","doi-asserted-by":"crossref","unstructured":"Wu HH, Seetharaman P, Kumar K et\u00a0al (2022) Wav2clip: learning robust audio representations from clip. In: ICASSP 2022-2022 IEEE international conference on acoustics, speech and signal Processing (ICASSP). IEEE, pp 4563\u20134567","DOI":"10.1109\/ICASSP43922.2022.9747669"},{"key":"1654_CR48","doi-asserted-by":"crossref","unstructured":"Wu Y, Zhu L, Yan Y et\u00a0al (2019) Dual attention matching for audio-visual event localization. In: Proceedings of the IEEE\/CVF international conference on computer vision, pp 6292\u20136300","DOI":"10.1109\/ICCV.2019.00639"},{"key":"1654_CR49","doi-asserted-by":"crossref","unstructured":"Xia Y, Zhao Z (2022) Cross-modal background suppression for audio-visual event localization. In: Proceedings of the IEEE\/CVF conference on computer vision and pattern recognition, pp 19989\u201319998","DOI":"10.1109\/CVPR52688.2022.01936"},{"key":"1654_CR50","doi-asserted-by":"crossref","unstructured":"Xu H, Zeng R, Wu Q et\u00a0al (2020) Cross-modal relation-aware networks for audio-visual event localization. In: Proceedings of the 28th ACM international conference on multimedia, pp 3893\u20133901","DOI":"10.1145\/3394171.3413581"},{"key":"1654_CR51","doi-asserted-by":"crossref","unstructured":"Xue C, Zhong X, Cai M et al (2021) Audio-visual event localization by learning spatial and semantic co-attention. IEEE Trans Multimed 25:418\u2013429","DOI":"10.1109\/TMM.2021.3127029"},{"key":"1654_CR52","doi-asserted-by":"crossref","unstructured":"Yu J, Cheng Y, Feng R (2021) MPN: multimodal parallel network for audio-visual event localization. In: 2021 IEEE international conference on multimedia and expo (ICME). IEEE, pp 1\u20136","DOI":"10.1109\/ICME51207.2021.9428373"},{"key":"1654_CR53","doi-asserted-by":"crossref","unstructured":"Zhou J, Zheng L, Zhong Y et\u00a0al (2021) Positive sample propagation along the audio-visual event line. In: Proceedings of the IEEE\/CVF conference on computer vision and pattern recognition, pp 8436\u20138444","DOI":"10.1109\/CVPR46437.2021.00833"},{"issue":"6","key":"1654_CR54","doi-asserted-by":"publisher","first-page":"7239","DOI":"10.1109\/TPAMI.2022.3223688","volume":"45","author":"J Zhou","year":"2022","unstructured":"Zhou J, Guo D, Wang M (2022) Contrastive positive sample propagation along the audio-visual event line. IEEE Trans Pattern Anal Mach Intell 45(6):7239\u20137257","journal-title":"IEEE Trans Pattern Anal Mach Intell"},{"key":"1654_CR55","doi-asserted-by":"crossref","unstructured":"Zhou J, Wang J, Zhang J et al (2022) Audio-visual segmentation. In: European conference on computer vision. Springer, Berlin, pp 386\u2013403","DOI":"10.1007\/978-3-031-19836-6_22"},{"key":"1654_CR56","doi-asserted-by":"crossref","unstructured":"Zhou K, Yang J, Loy CC et\u00a0al (2022) Conditional prompt learning for vision-language models. In: Proceedings of the IEEE\/CVF conference on computer vision and pattern recognition, pp 16816\u201316825","DOI":"10.1109\/CVPR52688.2022.01631"},{"issue":"9","key":"1654_CR57","doi-asserted-by":"publisher","first-page":"2337","DOI":"10.1007\/s11263-022-01653-1","volume":"130","author":"K Zhou","year":"2022","unstructured":"Zhou K, Yang J, Loy CC et al (2022) Learning to prompt for vision-language models. Int J Comput Vis 130(9):2337\u20132348","journal-title":"Int J Comput Vis"}],"container-title":["Complex & Intelligent Systems"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s40747-024-01654-2.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s40747-024-01654-2\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s40747-024-01654-2.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,1,30]],"date-time":"2025-01-30T20:20:59Z","timestamp":1738268459000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s40747-024-01654-2"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,11,9]]},"references-count":57,"journal-issue":{"issue":"1","published-print":{"date-parts":[[2025,1]]}},"alternative-id":["1654"],"URL":"https:\/\/doi.org\/10.1007\/s40747-024-01654-2","relation":{},"ISSN":["2199-4536","2198-6053"],"issn-type":[{"value":"2199-4536","type":"print"},{"value":"2198-6053","type":"electronic"}],"subject":[],"published":{"date-parts":[[2024,11,9]]},"assertion":[{"value":"6 May 2024","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"20 October 2024","order":2,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"9 November 2024","order":3,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Declarations"}},{"value":"The authors declare no Conflict of interest. The authors declare that they have no known competing financial interests or personal relationships that could have appeared to influence the work reported in this paper.","order":2,"name":"Ethics","group":{"name":"EthicsHeading","label":"Conflict of interest"}}],"article-number":"15"}}