{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,3,20]],"date-time":"2026-03-20T13:49:16Z","timestamp":1774014556786,"version":"3.50.1"},"reference-count":50,"publisher":"Springer Science and Business Media LLC","issue":"2","license":[{"start":{"date-parts":[[2025,2,11]],"date-time":"2025-02-11T00:00:00Z","timestamp":1739232000000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2025,2,11]],"date-time":"2025-02-11T00:00:00Z","timestamp":1739232000000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Pattern Anal Applic"],"published-print":{"date-parts":[[2025,6]]},"DOI":"10.1007\/s10044-025-01414-z","type":"journal-article","created":{"date-parts":[[2025,2,11]],"date-time":"2025-02-11T19:05:28Z","timestamp":1739300728000},"update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":4,"title":["Enhancing semantic audio-visual representation learning with supervised multi-scale attention"],"prefix":"10.1007","volume":"28","author":[{"given":"Jiwei","family":"Zhang","sequence":"first","affiliation":[]},{"given":"Yi","family":"Yu","sequence":"additional","affiliation":[]},{"given":"Suhua","family":"Tang","sequence":"additional","affiliation":[]},{"given":"GuoJun","family":"Qi","sequence":"additional","affiliation":[]},{"given":"Haiyuan","family":"Wu","sequence":"additional","affiliation":[]},{"given":"Hirotaka","family":"Hachiya","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2025,2,11]]},"reference":[{"issue":"7553","key":"1414_CR1","doi-asserted-by":"publisher","first-page":"436","DOI":"10.1038\/nature14539","volume":"521","author":"Y LeCun","year":"2015","unstructured":"LeCun Y, Bengio Y, Hinton G (2015) Deep learning. Nature 521(7553):436\u2013444","journal-title":"Nature"},{"key":"1414_CR2","unstructured":"Simonyan K, Zisserman A (2014) Very deep convolutional networks for large-scale image recognition. arXiv preprint arXiv:1409.1556"},{"key":"1414_CR3","unstructured":"Ngiam J, Khosla A, Kim M, Nam J, Lee H, Ng AY (2011) Multimodal deep learning. In: Proceedings of the 28th international conference on machine learning (ICML-11), pp 689\u2013696"},{"issue":"12","key":"1414_CR4","doi-asserted-by":"publisher","first-page":"5812","DOI":"10.1109\/TIP.2015.2490539","volume":"24","author":"C Xu","year":"2015","unstructured":"Xu C, Tao D, Xu C (2015) Multi-view learning with incomplete views. IEEE Trans Image Process 24(12):5812\u20135825","journal-title":"IEEE Trans Image Process"},{"issue":"1","key":"1414_CR5","doi-asserted-by":"publisher","first-page":"188","DOI":"10.1109\/TPAMI.2015.2435740","volume":"38","author":"M Kan","year":"2015","unstructured":"Kan M, Shan S, Zhang H, Lao S, Chen X (2015) Multi-view discriminant analysis. IEEE Trans Pattern Anal Mach Intell 38(1):188\u2013194","journal-title":"IEEE Trans Pattern Anal Mach Intell"},{"key":"1414_CR6","doi-asserted-by":"crossref","unstructured":"Hu P, Zhen L, Peng D, Liu P (2019) Scalable deep multimodal learning for cross-modal retrieval. In: Proceedings of the 42nd International ACM SIGIR conference on research and development in information retrieval, pp 635\u2013644","DOI":"10.1145\/3331184.3331213"},{"key":"1414_CR7","doi-asserted-by":"crossref","unstructured":"Hotelling H (1992) Relations between two sets of variates. In: Breakthroughs in statistics. Springer, pp 162\u2013190","DOI":"10.1007\/978-1-4612-4380-9_14"},{"issue":"05","key":"1414_CR8","doi-asserted-by":"publisher","first-page":"365","DOI":"10.1142\/S012906570000034X","volume":"10","author":"PL Lai","year":"2000","unstructured":"Lai PL, Fyfe C (2000) Kernel and nonlinear canonical correlation analysis. Int J Neural Syst 10(05):365\u2013377","journal-title":"Int J Neural Syst"},{"issue":"3s","key":"1414_CR9","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1145\/3575658","volume":"19","author":"J Zhang","year":"2023","unstructured":"Zhang J, Yu Y, Tang S, Wu J, Li W (2023) Variational autoencoder with cca for audio-visual cross-modal retrieval. ACM Trans Multimed Comput Commun Appl 19(3s):1\u201321","journal-title":"ACM Trans Multimed Comput Commun Appl"},{"key":"1414_CR10","doi-asserted-by":"crossref","unstructured":"Zhen L, Hu P, Wang X, Peng D (2019) Deep supervised cross-modal retrieval. In: Proceedings of the IEEE\/CVF conference on computer vision and pattern recognition, pp 10394\u201310403","DOI":"10.1109\/CVPR.2019.01064"},{"key":"1414_CR11","doi-asserted-by":"crossref","unstructured":"Sharma A, Kumar A, Daume H, Jacobs DW (2012) Generalized multiview analysis: a discriminative latent space. In: 2012 IEEE conference on computer vision and pattern recognition. IEEE, pp 2160\u20132167","DOI":"10.1109\/CVPR.2012.6247923"},{"key":"1414_CR12","doi-asserted-by":"crossref","unstructured":"Jiang Q-Y, Li W-J (2017) Deep cross-modal hashing. In: Proceedings of the IEEE conference on computer vision and pattern recognition, pp 3232\u20133240","DOI":"10.1109\/CVPR.2017.348"},{"key":"1414_CR13","doi-asserted-by":"crossref","unstructured":"Peng Y, Qi J (2019) Cm-gans: cross-modal generative adversarial networks for common representation learning. ACM Trans Multimedia Comput Commun Appl (TOMM) 15(1):1\u201324","DOI":"10.1145\/3284750"},{"key":"1414_CR14","doi-asserted-by":"crossref","unstructured":"Wang B, Yang Y, Xu X, Hanjalic A, Shen HT (2017) Adversarial cross-modal retrieval. In: Proceedings of the 25th ACM international conference on multimedia, pp 154\u2013162","DOI":"10.1145\/3123266.3123326"},{"issue":"11","key":"1414_CR15","doi-asserted-by":"publisher","first-page":"139","DOI":"10.1145\/3422622","volume":"63","author":"I Goodfellow","year":"2020","unstructured":"Goodfellow I, Pouget-Abadie J, Mirza M, Xu B, Warde-Farley D, Ozair S, Courville A, Bengio Y (2020) Generative adversarial networks. Commun ACM 63(11):139\u2013144","journal-title":"Commun ACM"},{"key":"1414_CR16","doi-asserted-by":"crossref","unstructured":"Suris D, Duarte A, Salvador A, Torres J, Giro-i-Nieto X (2018) Cross-modal embeddings for video and audio retrieval. In: Proceedings of the European conference on computer vision (ECCV) workshops","DOI":"10.1007\/978-3-030-11018-5_62"},{"key":"1414_CR17","doi-asserted-by":"crossref","unstructured":"Zeng D, Ikeda K (2023) Triplet loss with curriculum learning for audio-visual retrieval. In: 2023 IEEE international symposium on multimedia (ISM). IEEE, pp 206\u2013207","DOI":"10.1109\/ISM59092.2023.00038"},{"key":"1414_CR18","doi-asserted-by":"crossref","unstructured":"Tsiamas I, Pascual S, Yeh C, Serr\u00e0 J (2024) Sequential contrastive audio-visual learning. arXiv preprint arXiv:2407.05782","DOI":"10.1109\/ICASSP49660.2025.10888656"},{"key":"1414_CR19","unstructured":"Andrew G, Arora R, Bilmes J, Livescu K (2013) Deep canonical correlation analysis. In: International conference on machine learning. PMLR, pp 1247\u20131255"},{"key":"1414_CR20","doi-asserted-by":"crossref","unstructured":"Feng F, Wang X, Li R (2014) Cross-modal retrieval with correspondence autoencoder. In: Proceedings of the 22nd ACM international conference on multimedia, pp 7\u201316","DOI":"10.1145\/2647868.2654902"},{"key":"1414_CR21","unstructured":"Wang W, Arora R, Livescu K, Bilmes J (2015) On deep multi-view representation learning. In: International conference on machine learning. PMLR, pp 1083\u20131092"},{"issue":"3","key":"1414_CR22","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1145\/2168752.2168767","volume":"3","author":"D Zhai","year":"2012","unstructured":"Zhai D, Chang H, Shan S, Chen X, Gao W (2012) Multiview metric learning with global consistency and local smoothness. ACM Trans Intell Syst Technol (TIST) 3(3):1\u201322","journal-title":"ACM Trans Intell Syst Technol (TIST)"},{"key":"1414_CR23","doi-asserted-by":"crossref","unstructured":"Zhai X, Peng Y, Xiao J (2013) Heterogeneous metric learning with joint graph regularization for cross-media retrieval. In: Twenty-seventh AAAI conference on artificial intelligence","DOI":"10.1609\/aaai.v27i1.8464"},{"key":"1414_CR24","doi-asserted-by":"crossref","unstructured":"Wang J, He Y, Kang C, Xiang S, Pan C (2015) Image-text cross-modal retrieval via modality-specific feature learning. In: Proceedings of the 5th ACM on international conference on multimedia retrieval, pp 347\u2013354","DOI":"10.1145\/2671188.2749341"},{"key":"1414_CR25","doi-asserted-by":"crossref","unstructured":"Li J, Li F, Zhu L, Cui H, Li J (2023) Prototype-guided knowledge transfer for federated unsupervised cross-modal hashing. In: Proceedings of the 31st ACM international conference on multimedia, pp 1013\u20131022","DOI":"10.1145\/3581783.3613837"},{"key":"1414_CR26","doi-asserted-by":"crossref","unstructured":"Zheng L, Cheng Y, Yang H, Cao N, He J (2021) Deep co-attention network for multi-view subspace learning. In: Proceedings of the web conference 2021, pp 1528\u20131539","DOI":"10.1145\/3442381.3449801"},{"key":"1414_CR27","doi-asserted-by":"crossref","unstructured":"Cheng Y, Wang R, Pan Z, Feng R, Zhang Y (2020) Look, listen, and attend: co-attention network for self-supervised audio-visual representation learning. In: Proceedings of the 28th ACM international conference on multimedia, pp 3884\u20133892","DOI":"10.1145\/3394171.3413869"},{"key":"1414_CR28","doi-asserted-by":"crossref","unstructured":"Gan C, Huang D, Zhao H, Tenenbaum JB, Torralba A (2020) Music gesture for visual sound separation. In: Proceedings of the IEEE\/CVF conference on computer vision and pattern recognition, pp 10478\u201310487","DOI":"10.1109\/CVPR42600.2020.01049"},{"key":"1414_CR29","doi-asserted-by":"publisher","first-page":"194","DOI":"10.1016\/j.neucom.2022.11.036","volume":"519","author":"W Wang","year":"2023","unstructured":"Wang W, Li Q, Xie J, Hu N, Wang Z, Zhang N (2023) Research on emotional semantic retrieval of attention mechanism oriented to audio-visual synesthesia. Neurocomputing 519:194\u2013204","journal-title":"Neurocomputing"},{"key":"1414_CR30","unstructured":"Dumoulin V, Visin F (2016) A guide to convolution arithmetic for deep learning. arXiv preprint arXiv:1603.07285"},{"key":"1414_CR31","doi-asserted-by":"crossref","unstructured":"Zeng D, Yu Y, Oyama K (2020) Deep triplet neural networks with cluster-cca for audio-visual cross-modal retrieval. ACM Trans Multimedia Comput Commun Appl (TOMM) 16(3):1\u201323","DOI":"10.1145\/3387164"},{"issue":"4","key":"1414_CR32","doi-asserted-by":"publisher","first-page":"1250","DOI":"10.1109\/TNNLS.2018.2856253","volume":"30","author":"Y Yu","year":"2018","unstructured":"Yu Y, Tang S, Aizawa K, Aizawa A (2018) Category-based deep cca for fine-grained venue discovery from multimodal data. IEEE Trans Neural Netw Learn Syst 30(4):1250\u20131258","journal-title":"IEEE Trans Neural Netw Learn Syst"},{"key":"1414_CR33","doi-asserted-by":"crossref","unstructured":"Zeng D, Yu Y, Oyama K (2018) Audio-visual embedding for cross-modal music video retrieval through supervised deep cca. In: 2018 IEEE international symposium on multimedia (ISM). IEEE, pp 143\u2013150","DOI":"10.1109\/ISM.2018.00-21"},{"issue":"27","key":"1414_CR34","doi-asserted-by":"publisher","first-page":"20173","DOI":"10.1007\/s00521-023-08817-1","volume":"35","author":"J Zhang","year":"2023","unstructured":"Zhang J, Yu Y, Tang S, Li W, Wu J (2023) Multi-scale network with shared cross-attention for audio-visual correlation learning. Neural Comput Appl 35(27):20173\u201320187","journal-title":"Neural Comput Appl"},{"key":"1414_CR35","doi-asserted-by":"publisher","first-page":"550","DOI":"10.1109\/TASLP.2022.3226330","volume":"31","author":"X Qian","year":"2022","unstructured":"Qian X, Wang Z, Wang J, Guan G, Li H (2022) Audio-visual cross-attention network for robotic speaker tracking. IEEE\/ACM Trans Audio Speech Lang Process 31:550\u2013562","journal-title":"IEEE\/ACM Trans Audio Speech Lang Process"},{"key":"1414_CR36","doi-asserted-by":"crossref","unstructured":"Wen Y, Zhang K, Li Z, Qiao Y (2016) A discriminative feature learning approach for deep face recognition. In: European conference on computer vision. Springer, pp 499\u2013515","DOI":"10.1007\/978-3-319-46478-7_31"},{"key":"1414_CR37","doi-asserted-by":"crossref","unstructured":"Schroff F, Kalenichenko D, Philbin J (2015) Facenet: a unified embedding for face recognition and clustering. In: Proceedings of the IEEE conference on computer vision and pattern recognition, pp 815\u2013823","DOI":"10.1109\/CVPR.2015.7298682"},{"key":"1414_CR38","doi-asserted-by":"crossref","unstructured":"Bottou L (2012) Stochastic gradient descent tricks. In: Neural networks: tricks of the trade. Springer, pp 421\u2013436","DOI":"10.1007\/978-3-642-35289-8_25"},{"key":"1414_CR39","doi-asserted-by":"crossref","unstructured":"Zhou Y, Wang Z, Fang C, Bui T, Berg TL (2018) Visual to sound: generating natural sound for videos in the wild. In: Proceedings of the IEEE conference on computer vision and pattern recognition, pp 3550\u20133558","DOI":"10.1109\/CVPR.2018.00374"},{"key":"1414_CR40","doi-asserted-by":"crossref","unstructured":"Tian Y, Shi J, Li B, Duan Z, Xu C (2018) Audio-visual event localization in unconstrained videos. In: Proceedings of the European conference on computer vision (ECCV), pp 247\u2013263","DOI":"10.1007\/978-3-030-01216-8_16"},{"key":"1414_CR41","doi-asserted-by":"crossref","unstructured":"Gemmeke JF, Ellis DP, Freedman D, Jansen A, Lawrence W, Moore RC, Plakal M, Ritter M (2017) Audio set: an ontology and human-labeled dataset for audio events. In: 2017 IEEE international conference on acoustics, speech and signal processing (ICASSP). IEEE, pp 776\u2013780","DOI":"10.1109\/ICASSP.2017.7952261"},{"key":"1414_CR42","doi-asserted-by":"crossref","unstructured":"Hershey S, Chaudhuri S, Ellis DP, Gemmeke JF, Jansen A, Moore RC, Plakal M, Platt D, Saurous RA, Seybold B et al (2017) Cnn architectures for large-scale audio classification. In: 2017 Ieee international conference on acoustics, speech and signal processing (icassp). IEEE, pp 131\u2013135","DOI":"10.1109\/ICASSP.2017.7952132"},{"key":"1414_CR43","unstructured":"Abu-El-Haija S, Kothari N, Lee J, Natsev P, Toderici G, Varadarajan B, Vijayanarasimhan S (2016) Youtube-8m: a large-scale video classification benchmark. arXiv preprint arXiv:1609.08675"},{"key":"1414_CR44","unstructured":"Rasiwasia N, Mahajan D, Mahadevan V, Aggarwal G (2014) Cluster canonical correlation analysis. In: Artificial intelligence and statistics. PMLR, pp 823\u2013831"},{"key":"1414_CR45","doi-asserted-by":"crossref","unstructured":"Zhang J, Peng Y, Yuan M (2018) Unsupervised generative adversarial cross-modal hashing. In: Thirty-second AAAI conference on artificial intelligence","DOI":"10.1609\/aaai.v32i1.11263"},{"key":"1414_CR46","doi-asserted-by":"crossref","unstructured":"Gu W, Gu X, Gu J, Li B, Xiong Z, Wang W (2019) Adversary guided asymmetric hashing for cross-modal retrieval. In: Proceedings of the 2019 on international conference on multimedia retrieval, pp 159\u2013167","DOI":"10.1145\/3323873.3325045"},{"key":"1414_CR47","doi-asserted-by":"crossref","unstructured":"He L, Xu X, Lu H, Yang Y, Shen F, Shen HT (2017) Unsupervised cross-modal retrieval through adversarial learning. In: 2017 IEEE international conference on multimedia and expo (ICME). IEEE, pp 1153\u20131158","DOI":"10.1109\/ICME.2017.8019549"},{"key":"1414_CR48","unstructured":"Huang Z, Zeng Z, Liu B, Fu D, Fu J (2020) Pixel-bert: aligning image pixels with text by deep multi-modal transformers. arXiv preprint arXiv:2004.00849"},{"key":"1414_CR49","unstructured":"Zeng D, Wu J, Hattori G, Yu Y, Xu R (2021) Learning explicit and implicit latent common spaces for audio-visual cross-modal retrieval. arXiv preprint arXiv:2110.13556"},{"key":"1414_CR50","unstructured":"Maaten L, Hinton G (2008) Visualizing data using t-sne. J Mach Learn Res 9(11)"}],"container-title":["Pattern Analysis and Applications"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10044-025-01414-z.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s10044-025-01414-z\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10044-025-01414-z.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,7,2]],"date-time":"2025-07-02T16:38:18Z","timestamp":1751474298000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s10044-025-01414-z"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,2,11]]},"references-count":50,"journal-issue":{"issue":"2","published-print":{"date-parts":[[2025,6]]}},"alternative-id":["1414"],"URL":"https:\/\/doi.org\/10.1007\/s10044-025-01414-z","relation":{},"ISSN":["1433-7541","1433-755X"],"issn-type":[{"value":"1433-7541","type":"print"},{"value":"1433-755X","type":"electronic"}],"subject":[],"published":{"date-parts":[[2025,2,11]]},"assertion":[{"value":"11 September 2024","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"9 January 2025","order":2,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"11 February 2025","order":3,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Declarations"}},{"value":"All authors declare that they have no Conflict of interest.","order":2,"name":"Ethics","group":{"name":"EthicsHeading","label":"Conflict of interest"}}],"article-number":"40"}}