{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,5]],"date-time":"2026-04-05T09:37:34Z","timestamp":1775381854067,"version":"3.50.1"},"reference-count":62,"publisher":"Springer Science and Business Media LLC","issue":"33-34","license":[{"start":{"date-parts":[[2020,6,28]],"date-time":"2020-06-28T00:00:00Z","timestamp":1593302400000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"},{"start":{"date-parts":[[2020,6,28]],"date-time":"2020-06-28T00:00:00Z","timestamp":1593302400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Multimed Tools Appl"],"published-print":{"date-parts":[[2020,9]]},"DOI":"10.1007\/s11042-020-09227-4","type":"journal-article","created":{"date-parts":[[2020,6,28]],"date-time":"2020-06-28T13:02:38Z","timestamp":1593349358000},"page":"25023-25050","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":12,"title":["A deep multimodal generative and fusion framework for class-imbalanced multimodal data"],"prefix":"10.1007","volume":"79","author":[{"given":"Qing","family":"Li","sequence":"first","affiliation":[]},{"given":"Guanyuan","family":"Yu","sequence":"additional","affiliation":[]},{"given":"Jun","family":"Wang","sequence":"additional","affiliation":[]},{"given":"Yuehao","family":"Liu","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2020,6,28]]},"reference":[{"issue":"1","key":"9227_CR1","doi-asserted-by":"publisher","first-page":"123","DOI":"10.1016\/S0165-1765(03)00032-6","volume":"80","author":"C Ai","year":"2003","unstructured":"Ai C, Norton EC (2003) Interaction terms in logit and probit models. Econ Lett 80(1):123\u2013129","journal-title":"Econ Lett"},{"key":"9227_CR2","unstructured":"Bahdanau D, Cho K, Bengio Y (2015) Neural machine translation by jointly learning to align and translate. In: International conference on learning representations (ICLR)"},{"issue":"2","key":"9227_CR3","doi-asserted-by":"publisher","first-page":"423","DOI":"10.1109\/TPAMI.2018.2798607","volume":"41","author":"T Baltru\u0161aitis","year":"2018","unstructured":"Baltru\u0161aitis T, Ahuja C, Morency L-P (2018) Multimodal machine learning: a survey and taxonomy. IEEE Trans Pattern Anal Mach Intell (TPAMI) 41(2):423\u2013443","journal-title":"IEEE Trans Pattern Anal Mach Intell (TPAMI)"},{"key":"9227_CR4","doi-asserted-by":"crossref","unstructured":"Basu A (1976) Elementary statistical theory in sociology. Brill Archive, 12","DOI":"10.1163\/9789004670211"},{"key":"9227_CR5","doi-asserted-by":"publisher","first-page":"321","DOI":"10.1613\/jair.953","volume":"16","author":"NV Chawla","year":"2002","unstructured":"Chawla NV, Bowyer KW, Hall LO, Kegelmeyer WP (2002) SMOTE: synthetic minority over-sampling technique. J Artif Intell Res (JAIR) 16:321\u2013357","journal-title":"J Artif Intell Res (JAIR)"},{"key":"9227_CR6","doi-asserted-by":"crossref","unstructured":"Chen T, Guestrin C (2016) XGBOOST: a scalable tree boosting system. In: Proceedings of the 22nd ACM international conference on knowledge discovery and data mining (SIGKDD), pp 785\u2013794","DOI":"10.1145\/2939672.2939785"},{"key":"9227_CR7","doi-asserted-by":"crossref","unstructured":"Dai J, Qi H, Xiong Y, Li Y, Zhang G, Hu H, Wei Y (2017) Deformable convolutional networks. In: Proceedings of the IEEE international conference on computer vision (ICCV), pp 764\u2013773","DOI":"10.1109\/ICCV.2017.89"},{"key":"9227_CR8","doi-asserted-by":"crossref","unstructured":"Dal Pozzolo A, Caelen O, Johnson RA, Bontempi G (2015) Calibrating probability with undersampling for unbalanced classification. In: IEEE Computational intelligence, 2015 IEEE symposium series, pp 159\u2013166","DOI":"10.1109\/SSCI.2015.33"},{"key":"9227_CR9","doi-asserted-by":"crossref","unstructured":"Dalal N, Triggs B (2005) Histograms of oriented gradients for human detection. In: Computer Vision and Pattern Recognition (CVPR), IEEE, vol 1, pp 886\u2013893","DOI":"10.1109\/CVPR.2005.177"},{"key":"9227_CR10","doi-asserted-by":"publisher","first-page":"464","DOI":"10.1016\/j.eswa.2017.09.030","volume":"91","author":"G Douzas","year":"2018","unstructured":"Douzas G, Bacao F (2018) Effective data generation for imbalanced learning using conditional generative adversarial networks. Expert Syst Appl (ESWA) 91:464\u2013471","journal-title":"Expert Syst Appl (ESWA)"},{"key":"9227_CR11","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1016\/j.ins.2018.06.056","volume":"465","author":"G Douzas","year":"2018","unstructured":"Douzas G, Bacao F (2018) Improving imbalanced learning through a heuristic oversampling method based on k-means and SMOTE. Inf Sci 465:1\u201320","journal-title":"Inf Sci"},{"key":"9227_CR12","doi-asserted-by":"crossref","unstructured":"Dwibedi D, Aytar Y, Tompson J, Sermanet P, Zisserman A (2019) Temporal cycle-consistency learning. In: 2019 IEEE\/CVF conference on computer vision and pattern recognition (CVPR), pp 1801\u20131810","DOI":"10.1109\/CVPR.2019.00190"},{"key":"9227_CR13","doi-asserted-by":"crossref","unstructured":"Farnadi G, Tang J, De Cock M, Moens M-F (2018) User profiling through deep multimodal fusion. In: Proceedings of the eleventh ACM international conference on web search and data mining, ACM, pp 171\u2013179","DOI":"10.1145\/3159652.3159691"},{"issue":"9","key":"9227_CR14","doi-asserted-by":"publisher","first-page":"2045","DOI":"10.1109\/TMM.2017.2729019","volume":"19","author":"L Gao","year":"2017","unstructured":"Gao L, Guo Z, Zhang H, Xu X, Shen HT (2017) Video captioning with attention-based LSTM and semantic consistency. IEEE Trans Multimed (TMM) 19(9):2045\u20132055","journal-title":"IEEE Trans Multimed (TMM)"},{"key":"9227_CR15","doi-asserted-by":"crossref","unstructured":"Gao L, Li X, Song J, Shen HT (2019) Hierarchical LSTMs with adaptive attention for visual captioning. IEEE Transactions on Pattern Analysis and Machine Intelligence (TPAMI)","DOI":"10.1109\/TPAMI.2019.2894139"},{"key":"9227_CR16","unstructured":"Goodfellow I, Pouge Abadie J, Mirza M, Xu B, Warde Farley D, Ozair S, Courville A, Bengio Y (2014) Generative adversarial nets. In: Advances in Neural Information Processing Systems (NIPS), pp 2672\u20132680"},{"key":"9227_CR17","doi-asserted-by":"publisher","first-page":"63373","DOI":"10.1109\/ACCESS.2019.2916887","volume":"7","author":"W Guo","year":"2019","unstructured":"Guo W, Wang J, Wang S (2019) Deep multimodal representation learning: a survey. IEEE Access 7:63373\u201363394","journal-title":"IEEE Access"},{"key":"9227_CR18","unstructured":"He H, Bai Y, Garcia EA, Li S (2008) ADASYN: adaptive synthetic sampling approach for imbalanced learning. In: 2008 IEEE international joint conference on neural networks (IEEE world congress on computational intelligence), IEEE, pp 1322\u20131328"},{"key":"9227_CR19","doi-asserted-by":"crossref","unstructured":"He H, Garcia EA (2008) Learning from imbalanced data. IEEE Trans Knowl Data Eng (TKDE), (9)1263\u20131284","DOI":"10.1109\/TKDE.2008.239"},{"key":"9227_CR20","unstructured":"He H, Shen X (2007) A ranked subspace learning method for gene expression data classification. In: International conference on artificial intelligence (ICAI), pp 358\u2013364"},{"key":"9227_CR21","unstructured":"Ioffe S, Szegedy C (2015) Batch normalization: accelerating deep network training by reducing internal covariate shift. In: International conference on machine learning (ICML)"},{"key":"9227_CR22","doi-asserted-by":"publisher","first-page":"4","DOI":"10.1016\/j.inffus.2013.12.002","volume":"19","author":"AP James","year":"2014","unstructured":"James AP, Dasarathy BV (2014) Medical image fusion: a survey of the state of the art. Inform Fusion 19:4\u201319","journal-title":"Inform Fusion"},{"issue":"5","key":"9227_CR23","doi-asserted-by":"publisher","first-page":"429","DOI":"10.3233\/IDA-2002-6504","volume":"6","author":"N Japkowicz","year":"2002","unstructured":"Japkowicz N, Stephen S (2002) The class imbalance problem: a systematic study. Intell Data Anal (IDA) 6(5):429\u2013449","journal-title":"Intell Data Anal (IDA)"},{"issue":"1","key":"9227_CR24","doi-asserted-by":"publisher","first-page":"28","DOI":"10.1016\/j.inffus.2011.08.001","volume":"14","author":"B Khaleghi","year":"2013","unstructured":"Khaleghi B, Khamis A, Karray FO, Razavi SN (2013) Multisensor data fusion: a review of the state-of-the-art. Inform Fusion 14(1):28\u201344","journal-title":"Inform Fusion"},{"key":"9227_CR25","unstructured":"Kingma DP, Ba JL (2015) Adam: a method for stochastic optimization. In: International conference on learning representations (ICLR)"},{"issue":"2-3","key":"9227_CR26","doi-asserted-by":"publisher","first-page":"195","DOI":"10.1023\/A:1007452223027","volume":"30","author":"M Kubat","year":"1998","unstructured":"Kubat M, Holte RC, Matwin S (1998) Machine learning for the detection of oil spills in satellite radar images. Mach Learn 30(2-3):195\u2013215","journal-title":"Mach Learn"},{"issue":"2","key":"9227_CR27","doi-asserted-by":"publisher","first-page":"11","DOI":"10.1145\/2838731","volume":"34","author":"Q Li","year":"2016","unstructured":"Li Q, Chen Y, Jiang LL, Li P, Chen H (2016) A tensor-based information framework for predicting the stock market. ACM Trans Inf Syst (TOIS) 34(2):11","journal-title":"ACM Trans Inf Syst (TOIS)"},{"key":"9227_CR28","doi-asserted-by":"publisher","unstructured":"Li Q, Tan J, Wang J, Chen H (2020) A multimodal event-driven LSTM model for stock prediction using online news. IEEE Transactions on Knowledge and Data Engineering (TKDE). https:\/\/doi.org\/10.1109\/TKDE.2020.2968894","DOI":"10.1109\/TKDE.2020.2968894"},{"issue":"10","key":"9227_CR29","doi-asserted-by":"publisher","first-page":"12315","DOI":"10.1007\/s11042-016-3643-4","volume":"76","author":"Q Li","year":"2017","unstructured":"Li Q, Wang J, Wang F, Li P, Liu L, Chen Y (2017) The role of social sentiment in stock markets: a view from joint effects of multiple information sources. Multimed Tools Appl (MTAP) 76(10):12315\u201312345","journal-title":"Multimed Tools Appl (MTAP)"},{"key":"9227_CR30","doi-asserted-by":"publisher","first-page":"93","DOI":"10.1016\/j.dss.2014.01.013","volume":"61","author":"Q Li","year":"2014","unstructured":"Li Q, Wang T, Gong Q, Chen Y, Lin Z, Song S (2014) Media-aware quantitative trading based on public web information. Decision Support Systems (DSS) 61:93\u2013105","journal-title":"Decision Support Systems (DSS)"},{"issue":"9","key":"9227_CR31","doi-asserted-by":"publisher","first-page":"8071","DOI":"10.1016\/j.eswa.2012.01.134","volume":"39","author":"F Louzada","year":"2012","unstructured":"Louzada F, Ferreira Silva PH, Diniz CarlosAR (2012) On the impact of disproportional samples in credit scoring models: an application to a brazilian bank data. Expert Syst Appl (ESWA) 39(9):8071\u20138078","journal-title":"Expert Syst Appl (ESWA)"},{"issue":"2","key":"9227_CR32","doi-asserted-by":"publisher","first-page":"277","DOI":"10.1007\/s11042-009-0344-2","volume":"49","author":"M Mansoorizadeh","year":"2010","unstructured":"Mansoorizadeh M, Charkari NM (2010) Multimodal information fusion application to human emotion recognition from face and speech. Multimed Tools Appl (MTAP) 49(2):277\u2013297","journal-title":"Multimed Tools Appl (MTAP)"},{"key":"9227_CR33","unstructured":"Mathieu MF, Zhao JJ, Zhao J, Ramesh A, Sprechmann P, LeCun Y (2016) Disentangling factors of variation in deep representation using adversarial training. In: Advances in Neural Information Processing Systems (NIPS), pp 5040\u20135048"},{"key":"9227_CR34","doi-asserted-by":"crossref","unstructured":"Metz CE (1978) Basic principles of roc analysis. In: Seminars in Nuclear Medicine, vol 8. Elsevier, Amsterdam, pp 283\u2013298","DOI":"10.1016\/S0001-2998(78)80014-2"},{"key":"9227_CR35","doi-asserted-by":"crossref","unstructured":"Morvant E, Habrard A, Ayache S (2014) Majority vote of diverse classifiers for late fusion. In: Joint IAPR international workshops on statistical techniques in pattern recognition (SPR) and structural and syntactic pattern recognition (SSPR). Springer, Berlin, pp 153\u2013162","DOI":"10.1007\/978-3-662-44415-3_16"},{"key":"9227_CR36","unstructured":"Ngiam J, Khosla A, Kim M, Nam J, Lee H, Ng AY (2011) Multimodal deep learning. In: Proceedings of the 28th international conference on machine learning (ICML), pp 689\u2013696"},{"key":"9227_CR37","unstructured":"Oshri B, Khandwala N (2015) There and back again: autoencoders for textual reconstruction"},{"key":"9227_CR38","unstructured":"Pearson R, Goney G, Shwaber J (2003) Imbalanced clustering for microarray time-series. In: Proceedings of the international conference on machine learning (ICML), vol. 3"},{"key":"9227_CR39","doi-asserted-by":"publisher","first-page":"98","DOI":"10.1016\/j.inffus.2017.02.003","volume":"37","author":"S Poria","year":"2017","unstructured":"Poria S, Cambria E, Bajpai R, Hussain A (2017) A review of affective computing: from unimodal analysis to multimodal fusion. Inform Fusion 37:98\u2013125","journal-title":"Inform Fusion"},{"issue":"9","key":"9227_CR40","doi-asserted-by":"publisher","first-page":"1306","DOI":"10.1109\/JPROC.2003.817150","volume":"91","author":"G Potamianos","year":"2003","unstructured":"Potamianos G, Neti C, Gravier G, Garg A, Senior AW (2003) Recent advances in the automatic recognition of audiovisual speech. Proc IEEE 91(9):1306\u20131326","journal-title":"Proc IEEE"},{"issue":"2","key":"9227_CR41","doi-asserted-by":"publisher","first-page":"12","DOI":"10.1145\/1459352.1459357","volume":"41","author":"X Qi","year":"2009","unstructured":"Qi X, Davison BD (2009) Web page classification: features and algorithms. ACM Comput Surv (CSUR) 41(2):12","journal-title":"ACM Comput Surv (CSUR)"},{"key":"9227_CR42","doi-asserted-by":"crossref","unstructured":"Rendle S (2010) Factorization machines. In: 2010 IEEE international conference on data mining (ICDM), pp 995\u20131000","DOI":"10.1109\/ICDM.2010.127"},{"key":"9227_CR43","unstructured":"Roth K, Lucchi A, Nowozin S, Hofmann T (2017) Stabilizing training of generative adversarial networks through regularization. In: Advances in neural information processing systems (NIPS), pp 2018\u20132028"},{"key":"9227_CR44","doi-asserted-by":"crossref","unstructured":"Shin HC, Tenenholtz NA, Rogers JK, Schwarz CG, Senjem ML, Gunter JL, Andriole KP, Michalski M (2018) Medical image synthesis for data augmentation and anonymization using generative adversarial networks. In: International workshop on simulation and synthesis in medical imaging (SASHIMI). Springer, Berlin, pp 1\u201311","DOI":"10.1007\/978-3-030-00536-8_1"},{"issue":"10","key":"9227_CR45","doi-asserted-by":"publisher","first-page":"3047","DOI":"10.1109\/TNNLS.2018.2851077","volume":"30","author":"J Song","year":"2018","unstructured":"Song J, Guo Y, Gao L, Li X, Hanjalic A, Shen HT (2018) From deterministic to generative: multimodal stochastic rnns for video captioning. IEEE Trans Neural Netw Learn Syst (TNNLS) 30(10):3047\u20133058","journal-title":"IEEE Trans Neural Netw Learn Syst (TNNLS)"},{"key":"9227_CR46","doi-asserted-by":"crossref","unstructured":"Song J, Zhang J, Gao L, Liu X, Shen HT (2018) Dual conditional GANs for face aging and rejuvenation. In: International joint conference on artificial intelligence (IJCAI), pp 899\u2013905","DOI":"10.24963\/ijcai.2018\/125"},{"key":"9227_CR47","unstructured":"Sprent P, Smeeton NC (2000) Applied nonparametric statistical methods. Chapman and Hall\/CRC"},{"key":"9227_CR48","unstructured":"Srivastava N, Mansimov E, Salakhudinov R (2015) Unsupervised learning of video representations using LSTMs. In: International conference on machine learning (ICML), pp 843\u2013852"},{"key":"9227_CR49","unstructured":"Srivastava N, Salakhutdinov RR (2012) Multimodal learning with deep boltzmann machines. In: Advances in neural information processing systems (NIPS), pp 2222\u20132230"},{"key":"9227_CR50","doi-asserted-by":"publisher","first-page":"76","DOI":"10.1016\/j.ins.2017.10.017","volume":"425","author":"J Sun","year":"2018","unstructured":"Sun J, Lang J, Fujita H, Li H (2018) Imbalanced enterprise credit evaluation with DTE-SBD: decision tree ensemble based on SMOTE and bagging with differentiated sampling rates. Inf Sci 425:76\u201391","journal-title":"Inf Sci"},{"key":"9227_CR51","unstructured":"Sutskever I, Vinyals O, Le QV (2014) Sequence to sequence learning with neural networks. In: Advances in neural information processing systems (NIPS), pp 3104\u20133112"},{"key":"9227_CR52","unstructured":"Suzuki M, Nakayama K, Matsuo Y (2017) Joint multimodal learning with deep generative models. In: International conference on learning representations (ICLR) (Workshop)"},{"key":"9227_CR53","doi-asserted-by":"publisher","first-page":"47","DOI":"10.1016\/j.ins.2018.10.029","volume":"477","author":"C Tsai","year":"2019","unstructured":"Tsai C, Lin W, Hu Y, Yao G (2019) Under-sampling class imbalanced datasets by combining clustering analysis and instance selection. Inf Sci 477:47\u201354","journal-title":"Inf Sci"},{"key":"9227_CR54","doi-asserted-by":"crossref","unstructured":"Tsai Y-HH, Bai S, Liang PP, Kolter JZ, Morency L-P, Salakhutdinov R (2019) Multimodal transformer for unaligned multimodal language sequences. In: The 57th annual meeting of the association for computational linguistics (ACL 2019), pp 6558\u20136569","DOI":"10.18653\/v1\/P19-1656"},{"issue":"3","key":"9227_CR55","doi-asserted-by":"publisher","first-page":"420","DOI":"10.1214\/aoms\/1177728488","volume":"26","author":"MN Vartak","year":"1955","unstructured":"Vartak MN, et al. (1955) On an application of kronecker product of matrices to statistical designs. Ann Math Stat 26(3):420\u2013438","journal-title":"Ann Math Stat"},{"key":"9227_CR56","unstructured":"Wu M, Goodman N (2018) Multimodal generative models for scalable weakly-supervised learning. In: Advances in neural information processing systems (NIPS), pp 5575\u20135585"},{"key":"9227_CR57","doi-asserted-by":"crossref","unstructured":"Xu T, Zhang P, Huang Q, Zhang H, Gan Z, Huang X, He X (2018) AttnGAN: fine-grained text to image generation with attentional generative adversarial networks. In: Proceedings of the IEEE conference on computer vision and pattern recognition (CVPR), pp 1316\u20131324","DOI":"10.1109\/CVPR.2018.00143"},{"key":"9227_CR58","doi-asserted-by":"crossref","unstructured":"Yih Wt, He X, Meek C (2014) Semantic parsing for single-relation question answering. In: Proceedings of the 52nd annual meeting of the association for computational linguistics (NAACL) (Short Papers), vol 2, pp 643\u2013648","DOI":"10.3115\/v1\/P14-2105"},{"key":"9227_CR59","unstructured":"Yingzhen L, Mandt S (2018) Disentangled sequential autoencoder. In: International conference on machine learning (ICML), pp 5670\u20135679"},{"key":"9227_CR60","doi-asserted-by":"publisher","first-page":"205","DOI":"10.1016\/j.ins.2020.03.005","volume":"525","author":"G Yu","year":"2020","unstructured":"Yu G, Li Q, Wang J, Zhang D, Liu Y (2020) A multimodal generative and fusion framework for recognizing faculty homepages. Inf Sci 525:205\u2013220","journal-title":"Inf Sci"},{"key":"9227_CR61","doi-asserted-by":"crossref","unstructured":"Zhang C, Yang Z, He X, Deng L (2020) Multimodal intelligence: representation learning, information fusion, and applications. IEEE Journal of Selected Topics in Signal Processing, 1\u20131","DOI":"10.1109\/JSTSP.2020.2987728"},{"key":"9227_CR62","doi-asserted-by":"crossref","unstructured":"Zhu X, Hu H, Lin S, Dai J (2019) Deformable convnets v2: more deformable, better results. In: Proceedings of the IEEE conference on computer vision and pattern recognition (CVPR), pp 9308\u20139316","DOI":"10.1109\/CVPR.2019.00953"}],"container-title":["Multimedia Tools and Applications"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s11042-020-09227-4.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s11042-020-09227-4\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s11042-020-09227-4.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2023,10,3]],"date-time":"2023-10-03T01:48:19Z","timestamp":1696297699000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s11042-020-09227-4"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2020,6,28]]},"references-count":62,"journal-issue":{"issue":"33-34","published-print":{"date-parts":[[2020,9]]}},"alternative-id":["9227"],"URL":"https:\/\/doi.org\/10.1007\/s11042-020-09227-4","relation":{},"ISSN":["1380-7501","1573-7721"],"issn-type":[{"value":"1380-7501","type":"print"},{"value":"1573-7721","type":"electronic"}],"subject":[],"published":{"date-parts":[[2020,6,28]]},"assertion":[{"value":"27 May 2019","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"12 June 2020","order":2,"name":"revised","label":"Revised","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"15 June 2020","order":3,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"28 June 2020","order":4,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}}]}}