{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,7]],"date-time":"2026-07-07T15:57:58Z","timestamp":1783439878938,"version":"3.54.6"},"reference-count":61,"publisher":"Institute of Electrical and Electronics Engineers (IEEE)","issue":"1","license":[{"start":{"date-parts":[[2025,1,1]],"date-time":"2025-01-01T00:00:00Z","timestamp":1735689600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/ieeexplore.ieee.org\/Xplorehelp\/downloads\/license-information\/IEEE.html"},{"start":{"date-parts":[[2025,1,1]],"date-time":"2025-01-01T00:00:00Z","timestamp":1735689600000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2025,1,1]],"date-time":"2025-01-01T00:00:00Z","timestamp":1735689600000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"}],"funder":[{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["62106272"],"award-info":[{"award-number":["62106272"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]},{"name":"Young Elite Scientists Sponsorship Program by CAST","award":["2021QNRC001"],"award-info":[{"award-number":["2021QNRC001"]}]},{"name":"Public Computing Cloud, Renmin University of China"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["IEEE Trans. Pattern Anal. Mach. Intell."],"published-print":{"date-parts":[[2025,1]]},"DOI":"10.1109\/tpami.2024.3468315","type":"journal-article","created":{"date-parts":[[2024,9,25]],"date-time":"2024-09-25T19:23:10Z","timestamp":1727292190000},"page":"469-485","source":"Crossref","is-referenced-by-count":20,"title":["On-the-Fly Modulation for Balanced Multimodal Learning"],"prefix":"10.1109","volume":"47","author":[{"ORCID":"https:\/\/orcid.org\/0009-0008-8779-0637","authenticated-orcid":false,"given":"Yake","family":"Wei","sequence":"first","affiliation":[{"name":"Gaoling School of Artificial Intelligence, Beijing Key Laboratory of Big Data Management and Analysis Methods, Renmin University of China, Beijing, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-7118-6733","authenticated-orcid":false,"given":"Di","family":"Hu","sequence":"additional","affiliation":[{"name":"Gaoling School of Artificial Intelligence, Beijing Key Laboratory of Big Data Management and Analysis Methods, Renmin University of China, Beijing, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Henghui","family":"Du","sequence":"additional","affiliation":[{"name":"Gaoling School of Artificial Intelligence, Beijing Key Laboratory of Big Data Management and Analysis Methods, Renmin University of China, Beijing, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-9777-9676","authenticated-orcid":false,"given":"Ji-Rong","family":"Wen","sequence":"additional","affiliation":[{"name":"Gaoling School of Artificial Intelligence, Beijing Key Laboratory of Big Data Management and Analysis Methods, Renmin University of China, Beijing, China"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"263","reference":[{"issue":"1","key":"ref1","first-page":"87","article-title":"Cognitive neuroscience; The biology of the mind","volume":"4","author":"Herreras","year":"2010","journal-title":"Cuadernos de Neuropsicolog\u00eda\/Panamerican J. Neuropsychol."},{"key":"ref2","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2019.00559"},{"key":"ref3","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR42600.2020.01047"},{"key":"ref4","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-11726-9_14"},{"key":"ref5","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV48922.2021.00700"},{"key":"ref6","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-01216-8_16"},{"key":"ref7","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR42600.2020.01271"},{"key":"ref8","doi-asserted-by":"publisher","DOI":"10.1109\/LSP.2021.3101421"},{"key":"ref9","article-title":"Modality competition: What makes joint training of multi-modal network fail in deep learning?(Provably)","author":"Huang","year":"2022"},{"key":"ref10","first-page":"24043","article-title":"Characterizing and overcoming the greedy nature of learning in multi-modal deep neural networks","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Wu"},{"key":"ref11","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-11024-6_44"},{"key":"ref12","doi-asserted-by":"publisher","DOI":"10.1109\/ICASSP40776.2020.9053174"},{"key":"ref13","doi-asserted-by":"publisher","DOI":"10.1109\/WACV45572.2020.9093438"},{"key":"ref14","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52688.2022.00806"},{"key":"ref15","first-page":"568","article-title":"Two-stream convolutional networks for action recognition in videos","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Simonyan"},{"key":"ref16","article-title":"Audio-visual automatic speech recognition: An overview","volume":"22","author":"Potamianos","year":"2004","journal-title":"Issues Vis. Audio-Vis. Speech Process."},{"key":"ref17","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2016.389"},{"key":"ref18","doi-asserted-by":"publisher","DOI":"10.1109\/ICASSP.2019.8683226"},{"key":"ref19","first-page":"551","article-title":"Multimodal learning and reasoning for visual question answering","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Ilievski"},{"key":"ref20","article-title":"On modality bias in the tvqa dataset","author":"Winterbottom","year":"2020"},{"key":"ref21","article-title":"Improving multi-modal learning with uni-modal teachers","author":"Du","year":"2021"},{"issue":"1","key":"ref22","first-page":"1929","article-title":"Dropout: A simple way to prevent neural networks from overfitting","volume":"15","author":"Srivastava","year":"2014","journal-title":"J. Mach. Learn. Res."},{"key":"ref23","doi-asserted-by":"publisher","DOI":"10.1109\/SLT.2014.7078567"},{"key":"ref24","first-page":"2531","article-title":"Improved dropout for shallow and deep learning","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Li"},{"key":"ref25","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2017.383"},{"key":"ref26","doi-asserted-by":"publisher","DOI":"10.1109\/TPAMI.2015.2461544"},{"key":"ref27","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-55050-3_8"},{"key":"ref28","doi-asserted-by":"publisher","DOI":"10.1145\/3382507.3418840"},{"key":"ref29","doi-asserted-by":"publisher","DOI":"10.1109\/ICIP40778.2020.9190858"},{"key":"ref30","article-title":"Audiovisual slowfast networks for video recognition","author":"Xiao","year":"2020"},{"key":"ref31","first-page":"7654","article-title":"The anisotropic noise in stochastic gradient descent: Its behavior of escaping from sharp minima and regularization effects","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Zhu"},{"key":"ref32","doi-asserted-by":"publisher","DOI":"10.1109\/ITA.2018.8503224"},{"key":"ref33","doi-asserted-by":"publisher","DOI":"10.1162\/neco_a_01403"},{"key":"ref34","first-page":"10367","article-title":"On the noisy gradient descent that generalizes as SGD","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Wu"},{"key":"ref35","first-page":"1143","article-title":"Control batch size and learning rate to generalize well: Theoretical and empirical evidence","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"He"},{"key":"ref36","article-title":"Three factors influencing minima in SGD","author":"Jastrz\u0119bski","year":"2017"},{"key":"ref37","first-page":"1724","article-title":"How to escape saddle points efficiently","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Jin"},{"key":"ref38","article-title":"Adding gradient noise improves learning for very deep networks","volume":"1050","author":"Neelakantan","year":"2015","journal-title":"Stat"},{"key":"ref39","first-page":"7594","article-title":"Toward understanding the importance of noise in training neural networks","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Zhou"},{"key":"ref40","first-page":"10181","article-title":"The implicit and explicit regularization effects of dropout","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Wei"},{"key":"ref41","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v32i1.11671"},{"key":"ref42","doi-asserted-by":"publisher","DOI":"10.1109\/TAFFC.2014.2336244"},{"key":"ref43","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2017.73"},{"key":"ref44","article-title":"The kinetics human action video dataset","author":"Kay","year":"2017"},{"key":"ref45","article-title":"UCF101: A dataset of 101 human actions classes from videos in the wild","author":"Soomro","year":"2012"},{"key":"ref46","article-title":"MOSI: Multimodal corpus of sentiment intensity and subjectivity analysis in online opinion videos","author":"Zadeh","year":"2016"},{"key":"ref47","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52688.2022.01852"},{"key":"ref48","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2016.90"},{"key":"ref49","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-01246-5_35"},{"key":"ref50","first-page":"24206","article-title":"VATT: Transformers for multimodal self-supervised learning from raw video, audio and text","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Akbari"},{"key":"ref51","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR42600.2020.01330"},{"key":"ref52","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR46437.2021.00833"},{"key":"ref53","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2019.01283"},{"key":"ref54","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV48922.2021.00204"},{"key":"ref55","doi-asserted-by":"publisher","DOI":"10.1371\/journal.pone.0220386"},{"key":"ref56","doi-asserted-by":"publisher","DOI":"10.1109\/ICASSP40776.2020.9054317"},{"key":"ref57","doi-asserted-by":"publisher","DOI":"10.1109\/ACII.2019.8925444"},{"key":"ref58","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-46484-8_2"},{"key":"ref59","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2019.00718"},{"key":"ref60","first-page":"5922","article-title":"Neuron shapley: Discovering the responsible neurons","volume-title":"Proc. Int. Conf. Neural Inf. Process. Syst.","author":"Ghorbani"},{"key":"ref61","doi-asserted-by":"publisher","DOI":"10.24963\/ijcai.2022\/425"}],"container-title":["IEEE Transactions on Pattern Analysis and Machine Intelligence"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx8\/34\/10777928\/10694738.pdf?arnumber=10694738","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,12,5]],"date-time":"2024-12-05T06:29:58Z","timestamp":1733380198000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/10694738\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,1]]},"references-count":61,"journal-issue":{"issue":"1"},"URL":"https:\/\/doi.org\/10.1109\/tpami.2024.3468315","relation":{},"ISSN":["0162-8828","2160-9292","1939-3539"],"issn-type":[{"value":"0162-8828","type":"print"},{"value":"2160-9292","type":"electronic"},{"value":"1939-3539","type":"electronic"}],"subject":[],"published":{"date-parts":[[2025,1]]}}}