{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,14]],"date-time":"2026-07-14T08:39:26Z","timestamp":1784018366373,"version":"3.55.0"},"reference-count":261,"publisher":"Institute of Electrical and Electronics Engineers (IEEE)","issue":"2","license":[{"start":{"date-parts":[[2019,2,1]],"date-time":"2019-02-01T00:00:00Z","timestamp":1548979200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/ieeexplore.ieee.org\/Xplorehelp\/downloads\/license-information\/IEEE.html"},{"start":{"date-parts":[[2019,2,1]],"date-time":"2019-02-01T00:00:00Z","timestamp":1548979200000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2019,2,1]],"date-time":"2019-02-01T00:00:00Z","timestamp":1548979200000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["IEEE Trans. Pattern Anal. Mach. Intell."],"published-print":{"date-parts":[[2019,2,1]]},"DOI":"10.1109\/tpami.2018.2798607","type":"journal-article","created":{"date-parts":[[2018,1,25]],"date-time":"2018-01-25T19:19:51Z","timestamp":1516907991000},"page":"423-443","source":"Crossref","is-referenced-by-count":3608,"title":["Multimodal Machine Learning: A Survey and Taxonomy"],"prefix":"10.1109","volume":"41","author":[{"ORCID":"https:\/\/orcid.org\/0000-0001-7923-8780","authenticated-orcid":false,"given":"Tadas","family":"Baltrusaitis","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-4396-2050","authenticated-orcid":false,"given":"Chaitanya","family":"Ahuja","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-6376-7696","authenticated-orcid":false,"given":"Louis-Philippe","family":"Morency","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"263","reference":[{"key":"ref170","doi-asserted-by":"publisher","DOI":"10.1109\/JPROC.2003.817150"},{"key":"ref172","doi-asserted-by":"publisher","DOI":"10.1109\/TPAMI.2007.1124"},{"key":"ref171","first-page":"1281","article-title":"Global ranking using continuous conditional random fields","author":"qin","year":"2008","journal-title":"Proc 28th Int Conf Neural Inf Process Syst"},{"key":"ref174","first-page":"171","article-title":"Bridge correlational neural networks for multilingual multimodal representation learning","author":"rajendran","year":"2015","journal-title":"Proc Conf North Amer Chapter Assoc Comput Linguistics Human Language Technol"},{"key":"ref173","first-page":"338","article-title":"Extending long short-term memory for multi-view structured learning","author":"rajagopalan","year":"2016","journal-title":"Proc Eur Conf Comput Vis"},{"key":"ref176","doi-asserted-by":"publisher","DOI":"10.1145\/1873951.1873987"},{"key":"ref175","doi-asserted-by":"crossref","first-page":"396","DOI":"10.1007\/978-3-642-24571-8_51","article-title":"Modeling latent discriminative dynamic of multi-dimensional affective signals","author":"ramirez","year":"2011","journal-title":"Proc Int Conf Affective Comput Intell Interaction"},{"key":"ref178","first-page":"1060","article-title":"Generative adversarial text to image synthesis","author":"reed","year":"2016","journal-title":"Proc 29th Int Conf Mach Learn"},{"key":"ref177","first-page":"194","article-title":"Trainable methods for surface natural language generation","author":"ratnaparkhi","year":"2000","journal-title":"Proc Conf North Amer Chapter Assoc Comput Linguistics Human Language Technol"},{"key":"ref168","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2015.303"},{"key":"ref169","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D15-1303"},{"key":"ref39","doi-asserted-by":"publisher","DOI":"10.1145\/2808196.2811638"},{"key":"ref38","doi-asserted-by":"publisher","DOI":"10.1145\/2663204.2666277"},{"key":"ref33","doi-asserted-by":"publisher","DOI":"10.1145\/2939672.2939812"},{"key":"ref32","doi-asserted-by":"crossref","first-page":"1354","DOI":"10.1109\/TPAMI.2013.212","article-title":"Multiple kernel learning for visual object recognition: A review","volume":"36","author":"bucak","year":"2014","journal-title":"IEEE Trans Pattern Anal Mach Intell"},{"key":"ref31","doi-asserted-by":"publisher","DOI":"10.1613\/jair.4135"},{"key":"ref30","first-page":"136","article-title":"Distributional semantics in technicolor","author":"bruni","year":"2012","journal-title":"Proc Annual Meeting of the Assoc Computational Linguistics"},{"key":"ref37","doi-asserted-by":"publisher","DOI":"10.3115\/v1\/P15-1006"},{"key":"ref36","doi-asserted-by":"publisher","DOI":"10.1109\/ICASSP.2016.7472621"},{"key":"ref35","doi-asserted-by":"crossref","DOI":"10.1007\/978-3-540-85099-1_8","article-title":"Emotion recognition through multiple modalities:Face, body gesture, speech","author":"castellano","year":"2008","journal-title":"Affect and Emotion in Human-Computer Interaction"},{"key":"ref34","first-page":"28","article-title":"The AMI meeting corpus: A pre-announcement","author":"carletta","year":"2005","journal-title":"Proc Int Conf Methods Tech Behav Res"},{"key":"ref181","doi-asserted-by":"crossref","first-page":"209","DOI":"10.1007\/978-3-319-24947-6_17","article-title":"The long-short story of movie description","author":"rohrbach","year":"2015","journal-title":"Proc German Conf Pattern Recognit"},{"key":"ref180","doi-asserted-by":"publisher","DOI":"10.1109\/ICME.2007.4284731"},{"key":"ref185","doi-asserted-by":"publisher","DOI":"10.3115\/1073336.1073359"},{"key":"ref184","doi-asserted-by":"publisher","DOI":"10.1109\/TMM.2007.906583"},{"key":"ref183","first-page":"448","article-title":"Deep Boltzmann machines","author":"salakhutdinov","year":"2009","journal-title":"Proc Conf Artif Intell Statist"},{"key":"ref182","doi-asserted-by":"publisher","DOI":"10.1007\/s11263-016-0987-1"},{"key":"ref189","doi-asserted-by":"publisher","DOI":"10.1145\/2522848.2531741"},{"key":"ref188","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/N16-1020"},{"key":"ref187","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2011.6126545"},{"key":"ref186","doi-asserted-by":"crossref","first-page":"415","DOI":"10.1007\/978-3-642-24571-8_53","article-title":"AVEC 2011 the first international audio\/visual emotion challenge","author":"schuller","year":"2011","journal-title":"Proc Int Conf Affective Comput Intell Interaction"},{"key":"ref28","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2010.5539928"},{"key":"ref27","doi-asserted-by":"publisher","DOI":"10.1145\/258734.258880"},{"key":"ref179","doi-asserted-by":"publisher","DOI":"10.1162\/tacl_a_00207"},{"key":"ref29","first-page":"263","article-title":"The mathematics of statistical machine translation: Parameter estimation","author":"brown","year":"1993","journal-title":"Comput Linguistics"},{"key":"ref20","doi-asserted-by":"publisher","DOI":"10.1613\/jair.4900"},{"key":"ref22","doi-asserted-by":"publisher","DOI":"10.1145\/279943.279962"},{"key":"ref21","doi-asserted-by":"publisher","DOI":"10.1145\/1866029.1866080"},{"key":"ref24","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2015.507"},{"key":"ref23","first-page":"628","article-title":"Weakly supervised action labeling in videos under ordering constraints","author":"bojanowski","year":"2014","journal-title":"Proc Eur Conf Comput Vis"},{"key":"ref26","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.1997.609450"},{"key":"ref25","first-page":"426","article-title":"A new ASR approach based on independent processing and recombination of partial frequency bands","author":"bourlard","year":"1996","journal-title":"Proc Int Conf Spoken Language"},{"key":"ref50","doi-asserted-by":"publisher","DOI":"10.3115\/v1\/W14-3348"},{"key":"ref51","doi-asserted-by":"publisher","DOI":"10.3115\/v1\/P15-2017"},{"key":"ref154","doi-asserted-by":"publisher","DOI":"10.1613\/jair.3540"},{"key":"ref153","first-page":"1558","article-title":"Unsupervised alignment of natural language instructions with video segments","author":"naim","year":"2014","journal-title":"Proc 26th AAAI Conf Artif Intell"},{"key":"ref156","doi-asserted-by":"publisher","DOI":"10.1109\/TPAMI.2015.2461544"},{"key":"ref155","first-page":"ii-2013","article-title":"A coupled HMM for audio-visual speech recognition","volume":"2","author":"nefian","year":"2002","journal-title":"InterSpeech"},{"key":"ref150","doi-asserted-by":"publisher","DOI":"10.1109\/ICASSP.2015.7178347"},{"key":"ref152","doi-asserted-by":"publisher","DOI":"10.3115\/v1\/N15-1017"},{"key":"ref151","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-540-74048-3_4"},{"key":"ref146","first-page":"3111","article-title":"Distributed representations of words and phrases and their compositionality","author":"mikolov","year":"2013","journal-title":"Proc 28th Int Conf Neural Inf Process Syst"},{"key":"ref147","first-page":"747","article-title":"Midge: Generating image descriptions from computer vision detections","author":"mitchell","year":"2012","journal-title":"Proc 14th Conf Eur Chapter Assoc Comput Linguistics"},{"key":"ref148","article-title":"Multimodal transfer deep learning for audio-visual recognition","author":"moon","year":"2015","journal-title":"Proc 28th Int Conf Neural Inf Process Syst Workshops"},{"key":"ref149","doi-asserted-by":"crossref","DOI":"10.1007\/978-3-662-44415-3_16","article-title":"Majority vote of diverse classifiers for latefusion","author":"morvant","year":"2014","journal-title":"Lecture Notes in Computer Science Description"},{"key":"ref59","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2016.213"},{"key":"ref58","doi-asserted-by":"crossref","DOI":"10.1007\/978-3-642-15561-1_2","article-title":"Every picture tells a story: Generating sentences from images","author":"farhadi","year":"2010","journal-title":"Lecture Notes in Computer Science"},{"key":"ref57","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2009.5206772"},{"key":"ref56","first-page":"1964","article-title":"TTS synthesis with bidirectional LSTM based Recurrent Neural Networks","author":"fan","year":"2014","journal-title":"Proc Annu Conf Int Speech Commun Assoc"},{"key":"ref55","doi-asserted-by":"publisher","DOI":"10.1109\/TMM.2013.2267205"},{"key":"ref54","doi-asserted-by":"publisher","DOI":"10.3115\/v1\/P14-2074"},{"key":"ref53","first-page":"1292","article-title":"Image description using visual dependency representations","author":"elliott","year":"2013","journal-title":"Proc Conf Empirical Methods Natural Language Process"},{"key":"ref52","doi-asserted-by":"publisher","DOI":"10.1145\/2682899"},{"key":"ref40","article-title":"Microsoft COCO captions: Data collection and evaluation server","author":"chen","year":"2015"},{"key":"ref167","first-page":"311","article-title":"BLEU: A method for automatic evaluation of machine translation","author":"papineni","year":"2002","journal-title":"Proc Annual Meeting of the Assoc Computational Linguistics"},{"key":"ref166","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2016.497"},{"key":"ref165","first-page":"1410","article-title":"Zero-shot learning with semantic output codes","author":"palatucci","year":"2009","journal-title":"Proc 28th Int Conf Neural Inf Process Syst"},{"key":"ref164","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2016.264"},{"key":"ref163","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2014.299"},{"key":"ref162","first-page":"1143","article-title":"Im2text: Describing images using 1 million captioned photographs","author":"ordonez","year":"2011","journal-title":"Proc 28th Int Conf Neural Inf Process Syst"},{"key":"ref161","article-title":"WaveNet: A generative model for raw audio","author":"oord","year":"2016"},{"key":"ref160","doi-asserted-by":"publisher","DOI":"10.1109\/TPAMI.2011.47"},{"key":"ref4","doi-asserted-by":"publisher","DOI":"10.1007\/s10462-012-9368-5"},{"key":"ref3","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D16-1203"},{"key":"ref6","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2016.12"},{"key":"ref5","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2013.434"},{"key":"ref8","first-page":"1405","article-title":"Audio-to-text alignment for speech recognition with very limited resources","author":"anguera","year":"2014","journal-title":"Proc Annu Conf Int Speech Commun Assoc"},{"key":"ref159","doi-asserted-by":"publisher","DOI":"10.1145\/2993148.2993176"},{"key":"ref7","article-title":"Deep canonical correlation analysis","author":"andrew","year":"2013","journal-title":"Proc 29th Int Conf Mach Learn"},{"key":"ref49","doi-asserted-by":"crossref","first-page":"89","DOI":"10.1007\/978-3-642-10331-5_9","article-title":"Speech-driven facial animation using a shared gaussian process latent variable model","author":"deena","year":"2009","journal-title":"Proc Int Symp Adv Vis Comput"},{"key":"ref157","first-page":"689","article-title":"Multimodal deep learning","author":"ngiam","year":"2011","journal-title":"Proc 29th Int Conf Mach Learn"},{"key":"ref9","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2015.279"},{"key":"ref158","doi-asserted-by":"publisher","DOI":"10.1109\/T-AFFC.2011.9"},{"key":"ref46","first-page":"158","article-title":"Movie\/script: Alignment and parsing of video and text transcription","author":"cour","year":"2008","journal-title":"Proc Eur Conf Comput Vis"},{"key":"ref45","doi-asserted-by":"publisher","DOI":"10.1109\/ICASSP.1994.389596"},{"key":"ref48","doi-asserted-by":"publisher","DOI":"10.1007\/978-0-85729-997-0_19"},{"key":"ref47","doi-asserted-by":"publisher","DOI":"10.1145\/383259.383316"},{"key":"ref42","doi-asserted-by":"publisher","DOI":"10.1145\/1180995.1181013"},{"key":"ref41","first-page":"577","article-title":"Attention-based models for speech recognition","author":"chorowski","year":"2015","journal-title":"Proc 28th Int Conf Neural Inf Process Syst"},{"key":"ref44","article-title":"Wav2letter: An end-to-end convnet-based speech recognition system","author":"collobert","year":"2016"},{"key":"ref43","first-page":"88","article-title":"Multi-view learning in the presence of view disagreement","author":"christoudias","year":"2008","journal-title":"Proc 24th Conf Uncertainty Artif Intell"},{"key":"ref73","first-page":"2211","article-title":"Multiple kernel learning algorithms","volume":"12","author":"g\u00f6nen","year":"2011","journal-title":"J Mach Learn Res"},{"key":"ref72","first-page":"249","article-title":"Understanding the difficulty of training deep feedforward neural networks","author":"glorot","year":"2010","journal-title":"Proc Conf Artif Intell Statist"},{"key":"ref71","first-page":"359","article-title":"Multiple classifier systems for the classification of audio-visual emotional states","author":"glodek","year":"2011","journal-title":"Proc Int Conf Affect Emotion Human-Comput Interaction"},{"key":"ref70","doi-asserted-by":"publisher","DOI":"10.21236\/ADA307097"},{"key":"ref76","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2013.337"},{"key":"ref77","first-page":"606","article-title":"Choosing linguistics over vision to describe images","author":"gupta","year":"2012","journal-title":"Proc 26th AAAI Conf Artif Intell"},{"key":"ref74","first-page":"2672","article-title":"Generative adversarial nets","author":"goodfellow","year":"2014","journal-title":"Proc 28th Int Conf Neural Inf Process Syst"},{"key":"ref75","doi-asserted-by":"publisher","DOI":"10.1109\/ICASSP.2013.6638947"},{"key":"ref78","doi-asserted-by":"publisher","DOI":"10.1145\/1452392.1452442"},{"key":"ref79","doi-asserted-by":"publisher","DOI":"10.1162\/0899766042321814"},{"key":"ref60","doi-asserted-by":"publisher","DOI":"10.1016\/j.neucom.2014.12.020"},{"key":"ref62","first-page":"91","article-title":"Visual information in semantic representation","author":"feng","year":"2010","journal-title":"Proc Conf North Amer Chapter Assoc Comput Linguistics Human Language Technol"},{"key":"ref61","doi-asserted-by":"publisher","DOI":"10.1145\/2647868.2654902"},{"key":"ref63","first-page":"1995","article-title":"A sentence is worth a thousand pixels holistic CRF model","author":"fidler","year":"2013","journal-title":"Proc IEEE Conf Comput Vis Pattern Recognit"},{"key":"ref64","first-page":"2121","article-title":"DeViSE: A deep visual-semantic embedding model","author":"frome","year":"2013","journal-title":"Proc 28th Int Conf Neural Inf Process Syst"},{"key":"ref65","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D16-1044"},{"key":"ref66","first-page":"2296","article-title":"Are you talking to a machine? dataset and methods for multilingual image question answering","author":"gao","year":"2015","journal-title":"Proc 28th Int Conf Neural Inf Process Syst"},{"key":"ref67","doi-asserted-by":"publisher","DOI":"10.1109\/JPROC.2003.817119"},{"key":"ref68","doi-asserted-by":"publisher","DOI":"10.1109\/TPAMI.2017.2648793"},{"key":"ref69","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2009.5459169"},{"key":"ref197","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2010.5540112"},{"key":"ref198","first-page":"935","article-title":"Zero-shot learning through cross-modal transfer","author":"socher","year":"2013","journal-title":"Proc 28th Int Conf Neural Inf Process Syst"},{"key":"ref199","doi-asserted-by":"publisher","DOI":"10.1162\/tacl_a_00177"},{"key":"ref193","article-title":"Very deep convolutional networks for large-scale image recognition","author":"simonyan","year":"2015","journal-title":"Proc Int Conf Learn Representations"},{"key":"ref194","first-page":"93","article-title":"An HMM-based system for automatic segmentation and alignment of speech","author":"sj\u00f6lander","year":"2003","journal-title":"Proc Fonetik"},{"key":"ref195","article-title":"FaceSync: A linear operator for measuring synchronization of video facial images and audio tracks","author":"slaney","year":"2000","journal-title":"Proc 28th Int Conf Neural Inf Process Syst"},{"key":"ref196","doi-asserted-by":"publisher","DOI":"10.1023\/B:MTAP.0000046380.27575.a5"},{"key":"ref95","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2015.277"},{"key":"ref94","first-page":"1","article-title":"Multi-task, multi-kernel learning for estimating individual wellbeing","author":"jaques","year":"2015","journal-title":"Proc Multimodal Mach Learn Workshop Conjunction NIPS"},{"key":"ref190","first-page":"1423","article-title":"Grounded models of semantic representation","author":"silberer","year":"2012","journal-title":"Proc Conf Empirical Methods Natural Language Process"},{"key":"ref93","doi-asserted-by":"publisher","DOI":"10.1016\/j.inffus.2013.12.002"},{"key":"ref191","doi-asserted-by":"publisher","DOI":"10.3115\/v1\/P14-1068"},{"key":"ref92","first-page":"448","article-title":"Batch normalization: Accelerating deep network training by reducing internal covariate shift","author":"ioffe","year":"2015","journal-title":"Proc Int Conf Mach Learning"},{"key":"ref192","first-page":"568","article-title":"Two-stream convolutional networks for action recognition in videos","author":"simonyan","year":"2014","journal-title":"Proc 28th Int Conf Neural Inf Process Syst"},{"key":"ref91","doi-asserted-by":"publisher","DOI":"10.1109\/ICASSP.1996.541110"},{"key":"ref90","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/N16-1147"},{"key":"ref98","doi-asserted-by":"publisher","DOI":"10.1145\/2911996.2912043"},{"key":"ref99","doi-asserted-by":"publisher","DOI":"10.1080\/00401706.1991.10484833"},{"key":"ref96","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2017.348"},{"key":"ref97","doi-asserted-by":"publisher","DOI":"10.1016\/j.patrec.2014.08.005"},{"key":"ref82","doi-asserted-by":"publisher","DOI":"10.1109\/MSP.2012.2205597"},{"key":"ref81","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2016.8"},{"key":"ref84","doi-asserted-by":"publisher","DOI":"10.1162\/neco.2006.18.7.1527"},{"key":"ref83","first-page":"3","article-title":"Autoencoders, minimum description length and Helmoltz free energy","author":"hinton","year":"1993","journal-title":"Proc 28th Int Conf Neural Inf Process Syst"},{"key":"ref80","doi-asserted-by":"publisher","DOI":"10.1109\/ICME.2007.4284627"},{"key":"ref89","doi-asserted-by":"publisher","DOI":"10.1109\/ICASSP.2013.6639140"},{"key":"ref85","doi-asserted-by":"publisher","DOI":"10.1162\/neco.1997.9.8.1735"},{"key":"ref86","doi-asserted-by":"publisher","DOI":"10.1613\/jair.3994"},{"key":"ref87","doi-asserted-by":"publisher","DOI":"10.1093\/biomet\/28.3-4.321"},{"key":"ref88","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2016.493"},{"key":"ref200","doi-asserted-by":"publisher","DOI":"10.1109\/T-AFFC.2011.37"},{"key":"ref101","first-page":"1700","article-title":"Recurrent continuous translation models","author":"kalchbrenner","year":"2013","journal-title":"Proc Conf Empirical Methods Natural Language Process"},{"key":"ref100","doi-asserted-by":"publisher","DOI":"10.1007\/s12193-015-0195-2"},{"key":"ref209","article-title":"Introduction to conditional random fields for relationallearning","author":"sutton","year":"2006","journal-title":"Introduction to Statistical Relational Learning"},{"key":"ref203","first-page":"2025","article-title":"Unsupervised alignment of actions in video with text descriptions","author":"song","year":"2016","journal-title":"Proc 7th Int Joint Conf Artif Intell"},{"key":"ref204","first-page":"1929","article-title":"Dropout : A simple way to prevent neural networks from overfitting","volume":"15","author":"srivastava","year":"2014","journal-title":"J Mach Learn Res"},{"key":"ref201","first-page":"2120","article-title":"Multi-view latent variable discriminative models for action recognition","author":"song","year":"2012","journal-title":"Proc IEEE Conf Comput Vis Pattern Recognit"},{"key":"ref202","doi-asserted-by":"publisher","DOI":"10.1145\/2388676.2388684"},{"key":"ref207","doi-asserted-by":"publisher","DOI":"10.1016\/j.neuroimage.2014.06.077"},{"key":"ref208","first-page":"3104","article-title":"Sequence to sequence learning with neural networks","author":"sutskever","year":"2014","journal-title":"Proc 28th Int Conf Neural Inf Process Syst"},{"key":"ref205","article-title":"Learning representations for multimodal data with deep belief nets","author":"srivastava","year":"2012","journal-title":"Proc 29th Int Conf Mach Learn"},{"key":"ref206","first-page":"2949","article-title":"Multimodal learning with deep Boltzmann machines","author":"srivastava","year":"2012","journal-title":"Proc 28th Int Conf Neural Inf Process Syst"},{"key":"ref211","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2015.7298792"},{"key":"ref210","doi-asserted-by":"publisher","DOI":"10.1007\/s13735-014-0065-9"},{"key":"ref212","first-page":"275","article-title":"Dynamic units of visual speech","author":"taylor","year":"2012","journal-title":"Proc 28th Annu Conf Comput Graph Interactive Techn"},{"key":"ref213","first-page":"1218","article-title":"Integrating language and vision to generate natural language descriptions of videos in the wild","author":"thomason","year":"2014","journal-title":"Proc 23rd Int Conf Comput Linguistics Posters"},{"key":"ref214","article-title":"Using descriptive video services to create a large data source for video annotation research","author":"torabi","year":"2015"},{"key":"ref215","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2016.552"},{"key":"ref216","doi-asserted-by":"publisher","DOI":"10.1109\/ICASSP.2016.7472669"},{"key":"ref217","doi-asserted-by":"publisher","DOI":"10.1145\/2512530.2512533"},{"key":"ref218","first-page":"1747","article-title":"Pixel recurrent neural networks","author":"van den oord","year":"2016","journal-title":"Proc 29th Int Conf Mach Learn"},{"key":"ref219","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2015.7299087"},{"key":"ref220","article-title":"Order-embeddings of images and language","author":"vendrov","year":"2016","journal-title":"Proc Int Conf Learn Representations"},{"key":"ref222","doi-asserted-by":"publisher","DOI":"10.3115\/v1\/N15-1173"},{"key":"ref221","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D16-1204"},{"key":"ref229","doi-asserted-by":"crossref","first-page":"21","DOI":"10.1007\/s10994-010-5198-3","article-title":"Web scale image annotation: Learning to rank with joint word-image embeddings image annotation","author":"weston","year":"2010","journal-title":"Proc Eur Conf Mach Learn"},{"key":"ref228","first-page":"1083","article-title":"On deep multi-view representation learning","author":"wang","year":"2015","journal-title":"Proc 29th Int Conf Mach Learn"},{"key":"ref227","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2016.541"},{"key":"ref226","article-title":"Hashing for similarity search: A survey","author":"wang","year":"2014"},{"key":"ref225","first-page":"2291","article-title":"Deep multimodal hashing with orthogonal regularization","author":"wang","year":"2015","journal-title":"Proc 7th Int Joint Conf Artif Intell"},{"key":"ref224","doi-asserted-by":"publisher","DOI":"10.3115\/993268.993313"},{"key":"ref223","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2015.7298935"},{"key":"ref127","doi-asserted-by":"publisher","DOI":"10.1016\/j.neucom.2014.08.003"},{"key":"ref126","first-page":"220","article-title":"Composing simple image descriptions using web-scale n-grams","author":"li","year":"2011","journal-title":"Proc 15th Conf Computational Natural Language Learning"},{"key":"ref125","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2003.1238406"},{"key":"ref124","first-page":"2085","article-title":"Phrase-based image captioning","author":"lebret","year":"2015","journal-title":"Proc 29th Int Conf Mach Learn"},{"key":"ref129","doi-asserted-by":"publisher","DOI":"10.3115\/1073445.1073465"},{"key":"ref128","doi-asserted-by":"crossref","first-page":"290","DOI":"10.1117\/12.333848","article-title":"Comparison of automatic shot boundary detection algorithms","author":"lienhart","year":"1998","journal-title":"Proc SPIE"},{"key":"ref130","doi-asserted-by":"publisher","DOI":"10.1109\/JBHI.2013.2285378"},{"key":"ref133","first-page":"1","article-title":"Hierarchical co-attention for visual question answering","author":"lu","year":"2016","journal-title":"Proc 28th Int Conf Neural Inf Process Syst"},{"key":"ref134","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2016.333"},{"key":"ref131","doi-asserted-by":"publisher","DOI":"10.1111\/j.1756-8765.2010.01106.x"},{"key":"ref132","doi-asserted-by":"publisher","DOI":"10.1023\/B:VISI.0000029664.99615.94"},{"key":"ref232","first-page":"2362","article-title":"Context-sensitive multimodal emotion recognition from speech and facial expression using bidirectional LSTM modeling","author":"w\u00f6llmer","year":"2010","journal-title":"Proc Annu Conf Int Speech Commun Assoc"},{"key":"ref233","doi-asserted-by":"publisher","DOI":"10.1145\/2647868.2654969"},{"key":"ref230","first-page":"2764","article-title":"WSABIE: Scaling up to large vocabulary image annotation","author":"weston","year":"2011","journal-title":"Proc 7th Int Joint Conf Artif Intell"},{"key":"ref231","doi-asserted-by":"publisher","DOI":"10.1016\/j.imavis.2012.03.001"},{"key":"ref239","first-page":"2346","article-title":"Jointly modeling deep video and compositional text to bridge vision and language in a unified framework","author":"xu","year":"2015","journal-title":"Proc 26th AAAI Conf Artif Intell"},{"key":"ref238","first-page":"2048","article-title":"Show, attend and tell: Neural image caption generation with visual attention","author":"xu","year":"2015","journal-title":"Proc 29th Int Conf Mach Learn"},{"key":"ref235","doi-asserted-by":"publisher","DOI":"10.1145\/2647868.2654931"},{"key":"ref234","doi-asserted-by":"crossref","first-page":"493","DOI":"10.1007\/11608288_66","article-title":"Multi-level fusion of audio and visual features for speaker identification","author":"wu","year":"2005","journal-title":"Proc Int Conf Adv Biometrics"},{"key":"ref237","first-page":"451","article-title":"Ask, attend and answer: Exploring question-guided spatial attention for visual question answering","author":"xu","year":"2016","journal-title":"Proc Eur Conf Comput Vis"},{"key":"ref236","first-page":"2397","article-title":"Dynamic memory networks for visual and textual question answering","author":"xiong","year":"2016","journal-title":"Proc 29th Int Conf Mach Learn"},{"key":"ref136","article-title":"What’s cookin’? interpreting cooking videos using text, speech and vision","author":"malmaud","year":"2015","journal-title":"Proc Conf North Amer Chapter Assoc Comput Linguistics Human Language Technol"},{"key":"ref135","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2015.9"},{"key":"ref138","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2016.9"},{"key":"ref137","article-title":"Generating images from captions with attention","author":"mansimov","year":"2016","journal-title":"Proc Int Conf Learn Representations"},{"key":"ref139","article-title":"Deep captioning with multimodal recurrent neural networks (m-RNN)","author":"mao","year":"2015","journal-title":"Proc Int Conf Learn Representations"},{"key":"ref140","doi-asserted-by":"publisher","DOI":"10.3115\/v1\/P14-2097"},{"key":"ref141","doi-asserted-by":"publisher","DOI":"10.1109\/ICASSP.1998.679698"},{"key":"ref142","first-page":"491","article-title":"Learning multi-modal similarity","volume":"12","author":"mcfee","year":"2011","journal-title":"J Mach Learn Res"},{"key":"ref143","doi-asserted-by":"crossref","first-page":"746","DOI":"10.1038\/264746a0","article-title":"Hearing lips and seeing voices","volume":"264","author":"mcgurk","year":"1976","journal-title":"Nature"},{"key":"ref2","year":"2016"},{"key":"ref144","doi-asserted-by":"publisher","DOI":"10.1109\/ICME.2010.5583006"},{"key":"ref1","year":"2017"},{"key":"ref145","first-page":"2772","article-title":"Listen, attend, and walk: Neural mapping of navigational instructions to action sequences","author":"mei","year":"2016","journal-title":"Proc 26th AAAI Conf Artif Intell"},{"key":"ref241","first-page":"444","article-title":"Corpus-guided sentence generation of natural images","author":"yang","year":"2011","journal-title":"Proc Conf Empirical Methods Natural Language Process"},{"key":"ref242","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2016.10"},{"key":"ref243","doi-asserted-by":"publisher","DOI":"10.1109\/JPROC.2010.2050411"},{"key":"ref244","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2015.512"},{"key":"ref240","doi-asserted-by":"publisher","DOI":"10.3115\/v1\/P15-2018"},{"key":"ref248","first-page":"53","article-title":"Grounded language learning from video described with sentences","author":"yu","year":"2013","journal-title":"Proc Annual Meeting of the Assoc Computational Linguistics"},{"key":"ref247","first-page":"488","article-title":"On the integration of grounding language and learning objects","author":"yu","year":"2004","journal-title":"Proc 26th AAAI Conf Artif Intell"},{"key":"ref246","doi-asserted-by":"publisher","DOI":"10.1162\/tacl_a_00166"},{"key":"ref245","doi-asserted-by":"publisher","DOI":"10.1109\/TMM.2012.2188783"},{"key":"ref249","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2016.496"},{"key":"ref109","article-title":"Adam: A method for stochastic optimization","author":"kingma","year":"2015","journal-title":"Proc Int Conf Learn Representations"},{"key":"ref108","doi-asserted-by":"publisher","DOI":"10.1109\/ICASSP.2013.6638346"},{"key":"ref107","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D15-1293"},{"key":"ref106","doi-asserted-by":"publisher","DOI":"10.3115\/v1\/P15-2038"},{"key":"ref105","doi-asserted-by":"publisher","DOI":"10.3115\/v1\/D14-1005"},{"key":"ref104","first-page":"420","article-title":"Everybody loves a rich cousin: An empirical study of transliteration through bridge languages","author":"khapra","year":"2010","journal-title":"Proc Conf North Amer Chapter Assoc Comput Linguistics Human Language Technol"},{"key":"ref103","first-page":"1889","article-title":"Deep fragment embeddings for bidirectional image sentence mapping","author":"karpathy","year":"2014","journal-title":"Proc 28th Int Conf Neural Inf Process Syst"},{"key":"ref102","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2015.7298932"},{"key":"ref111","first-page":"4437","article-title":"Fisher vectors derived from hybrid gaussian-laplacian mixture models for image annotation","author":"klein","year":"2015","journal-title":"Proc IEEE Conf Comput Vis Pattern Recognit"},{"key":"ref112","doi-asserted-by":"publisher","DOI":"10.1023\/A:1020346032608"},{"key":"ref110","first-page":"1","article-title":"Unifying visual-semantic embeddings with multimodal neural language models","author":"kiros","year":"2015","journal-title":"Transactions of the Association for Computational Linguistics"},{"key":"ref250","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-46475-6_5"},{"key":"ref251","doi-asserted-by":"publisher","DOI":"10.1109\/35.41402"},{"key":"ref254","first-page":"4334","article-title":"Leveraging video descriptions to learn video question answering","author":"zeng","year":"2017","journal-title":"Proc 26th AAAI Conf Artif Intell"},{"key":"ref255","doi-asserted-by":"publisher","DOI":"10.1109\/TPAMI.2008.52"},{"key":"ref252","doi-asserted-by":"publisher","DOI":"10.1109\/TASL.2012.2187195"},{"key":"ref253","doi-asserted-by":"publisher","DOI":"10.1016\/j.specom.2009.04.004"},{"key":"ref257","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/P16-1169"},{"key":"ref256","first-page":"2177","article-title":"Large-scale supervised multimodal hashing with semantic correlation maximization","author":"zhang","year":"2014","journal-title":"Proc 26th AAAI Conf Artif Intell"},{"key":"ref259","first-page":"2286","article-title":"Canonical time warping for alignment of human behavior","author":"zhou","year":"2009","journal-title":"Proc 28th Int Conf Neural Inf Process Syst"},{"key":"ref10","doi-asserted-by":"publisher","DOI":"10.1109\/ICASSP.2013.6639047"},{"key":"ref258","first-page":"1282","article-title":"Generalized time warping for multi-modal alignment of human motion","author":"zhou","year":"2012","journal-title":"Proc IEEE Conf Comput Vis Pattern Recognit"},{"key":"ref11","doi-asserted-by":"publisher","DOI":"10.1007\/s00530-010-0182-0"},{"key":"ref12","article-title":"SoundNet: Learning sound representations from unlabeled video","author":"aytar","year":"2016","journal-title":"Proc 28th Int Conf Neural Inf Process Syst"},{"key":"ref13","article-title":"Neural machine translation by jointly learning to align and translate","author":"bahdanau","year":"2014","journal-title":"Proc Int Conf Learn Representations"},{"key":"ref14","first-page":"1","article-title":"Dimensional Affect recognition using continuous conditional random fields","author":"baltru\u0161aitis","year":"2013","journal-title":"Proc 10th IEEE Int Conf Workshops Autom Face Gesture Recognit"},{"key":"ref15","first-page":"102","article-title":"Video in sentences out","author":"barbu","year":"2012","journal-title":"Proc Conf Uncertainty Artif Intell"},{"key":"ref16","first-page":"1107","article-title":"Matching words and pictures","volume":"3","author":"barnard","year":"2003","journal-title":"J Mach Learn Res"},{"key":"ref118","first-page":"1360","article-title":"Learning hash functions for cross-view similarity search","author":"kumar","year":"2011","journal-title":"Proc 7th Int Joint Conf Artif Intell"},{"key":"ref17","doi-asserted-by":"publisher","DOI":"10.1111\/lnc3.12170"},{"key":"ref117","doi-asserted-by":"publisher","DOI":"10.1109\/TPAMI.2012.162"},{"key":"ref18","doi-asserted-by":"publisher","DOI":"10.1146\/annurev.psych.59.103006.093639"},{"key":"ref19","doi-asserted-by":"publisher","DOI":"10.1109\/TPAMI.2013.50"},{"key":"ref119","first-page":"359","article-title":"Collective generation of natural image descriptions","author":"kuznetsova","year":"2012","journal-title":"Proc Annual Meeting of the Assoc Computational Linguistics"},{"key":"ref114","first-page":"1097","article-title":"ImageNet classification with deep convolutional neural networks","author":"krizhevsky","year":"2012","journal-title":"Proc 28th Int Conf Neural Inf Process Syst"},{"key":"ref113","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2014.455"},{"key":"ref116","doi-asserted-by":"publisher","DOI":"10.1137\/1025045"},{"key":"ref115","doi-asserted-by":"publisher","DOI":"10.1023\/B:MACH.0000035472.73496.0c"},{"key":"ref120","first-page":"282","article-title":"Conditional random fields : Probabilistic models for segmenting and labeling sequence data","author":"lafferty","year":"2001","journal-title":"Proc 29th Int Conf Mach Learn"},{"key":"ref121","doi-asserted-by":"publisher","DOI":"10.1142\/S012906570000034X"},{"key":"ref122","doi-asserted-by":"publisher","DOI":"10.1007\/s11042-013-1391-2"},{"key":"ref123","doi-asserted-by":"publisher","DOI":"10.3115\/v1\/P14-1132"},{"key":"ref260","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2015.11"},{"key":"ref261","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2013.387"}],"container-title":["IEEE Transactions on Pattern Analysis and Machine Intelligence"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx7\/34\/8605394\/08269806.pdf?arnumber=8269806","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2022,7,13]],"date-time":"2022-07-13T21:08:25Z","timestamp":1657746505000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/8269806\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2019,2,1]]},"references-count":261,"journal-issue":{"issue":"2"},"URL":"https:\/\/doi.org\/10.1109\/tpami.2018.2798607","relation":{},"ISSN":["0162-8828","2160-9292","1939-3539"],"issn-type":[{"value":"0162-8828","type":"print"},{"value":"2160-9292","type":"electronic"},{"value":"1939-3539","type":"electronic"}],"subject":[],"published":{"date-parts":[[2019,2,1]]}}}