{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,2,18]],"date-time":"2026-02-18T23:46:49Z","timestamp":1771458409651,"version":"3.50.1"},"reference-count":66,"publisher":"Springer Science and Business Media LLC","issue":"1","license":[{"start":{"date-parts":[[2019,1,12]],"date-time":"2019-01-12T00:00:00Z","timestamp":1547251200000},"content-version":"tdm","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"},{"start":{"date-parts":[[2019,1,12]],"date-time":"2019-01-12T00:00:00Z","timestamp":1547251200000},"content-version":"vor","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"}],"funder":[{"DOI":"10.13039\/100000001","name":"National Science Foundation","doi-asserted-by":"publisher","award":["IIS-1746031"],"award-info":[{"award-number":["IIS-1746031"]}],"id":[{"id":"10.13039\/100000001","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000001","name":"National Science Foundation","doi-asserted-by":"publisher","award":["CNS-1544969"],"award-info":[{"award-number":["CNS-1544969"]}],"id":[{"id":"10.13039\/100000001","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Int J Multimed Info Retr"],"published-print":{"date-parts":[[2019,3]]},"DOI":"10.1007\/s13735-018-00166-3","type":"journal-article","created":{"date-parts":[[2019,1,12]],"date-time":"2019-01-12T15:34:23Z","timestamp":1547307263000},"page":"3-18","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":23,"title":["Joint embeddings with multimodal cues for video-text retrieval"],"prefix":"10.1007","volume":"8","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-3611-4141","authenticated-orcid":false,"given":"Niluthpol C.","family":"Mithun","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Juncheng","family":"Li","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Florian","family":"Metze","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Amit K.","family":"Roy-Chowdhury","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2019,1,12]]},"reference":[{"key":"166_CR1","unstructured":"Andrew G, Arora R, Bilmes J, Livescu K (2013) Deep canonical correlation analysis. In: International conference on machine learning, pp 1247\u20131255"},{"key":"166_CR2","unstructured":"Awad G, Butt A, Fiscus J, Joy D, Delgado A, Michel M, Smeaton AF, Graham Y, Kraaij W, Qu\u00e9not G et\u00a0al (2017) Trecvid 2017: evaluating ad-hoc and instance video search, events detection, video captioning and hyperlinking. In: Proceedings of TRECVID"},{"key":"166_CR3","unstructured":"Aytar Y, Vondrick C, Torralba A (2016) Soundnet: learning sound representations from unlabeled video. In: Advances in neural information processing systems, pp 892\u2013900"},{"key":"166_CR4","unstructured":"Aytar Y, Vondrick C, Torralba A (2017) See, hear, and read: deep aligned representations. arXiv preprint arXiv:1706.00932"},{"key":"166_CR5","doi-asserted-by":"crossref","unstructured":"Bois R, Vukoti\u0107 V, Simon AR, Sicre R, Raymond C, S\u00e9billot P, Gravier G (2017) Exploiting multimodality in video hyperlinking to improve target diversity. In: International conference on multimedia modeling, Springer, pp 185\u2013197","DOI":"10.1007\/978-3-319-51814-5_16"},{"key":"166_CR6","doi-asserted-by":"crossref","unstructured":"Budnik M, Demirdelen M, Gravier G (2018) A study on multimodal video hyperlinking with visual aggregation. In: 2018 IEEE international conference on multimedia and expo, IEEE, pp 1\u20136","DOI":"10.1109\/ICME.2018.8486549"},{"key":"166_CR7","doi-asserted-by":"crossref","unstructured":"Carreira J, Zisserman A (2017) Quo vadis, action recognition? A new model and the kinetics dataset. In: IEEE conference on computer vision and pattern recognition, IEEE, pp 4724\u20134733","DOI":"10.1109\/CVPR.2017.502"},{"key":"166_CR8","unstructured":"Cha M, Gwon Y, Kung H (2015) Multimodal sparse representation learning and applications. arXiv preprint arXiv:1511.06238"},{"key":"166_CR9","unstructured":"Chen DL, Dolan WB (2011) Collecting highly parallel data for paraphrase evaluation. In: Annual meeting of the association for computational linguistics: human language technologies, vol 1, ACL, pp 190\u2013200"},{"key":"166_CR10","doi-asserted-by":"crossref","unstructured":"Chi J, Peng Y (2018) Dual adversarial networks for zero-shot cross-media retrieval. In: International joint conferences on artificial intelligence, pp 663\u2013669","DOI":"10.24963\/ijcai.2018\/92"},{"key":"166_CR11","unstructured":"Chung J, Gulcehre C, Cho K, Bengio Y (2014) Empirical evaluation of gated recurrent neural networks on sequence modeling. arXiv preprint arXiv:1412.3555"},{"key":"166_CR12","unstructured":"Dong J, Li X, Snoek CG (2016) Word2visualvec: Image and video to sentence matching by visual feature prediction. arXiv preprint arXiv:1604.06838"},{"key":"166_CR13","unstructured":"Faghri F, Fleet DJ, Kiros JR, Fidler S (2018) Vse++: improved visual-semantic embeddings. In: British machine vision conference (BMVC)"},{"key":"166_CR14","doi-asserted-by":"crossref","unstructured":"Farhadi A, Hejrati M, Sadeghi MA, Young P, Rashtchian C, Hockenmaier J, Forsyth D (2010) Every picture tells a story: generating sentences from images. In: European conference on computer vision, Springer, pp 15\u201329","DOI":"10.1007\/978-3-642-15561-1_2"},{"key":"166_CR15","doi-asserted-by":"crossref","unstructured":"Feng F, Wang X, Li R (2014) Cross-modal retrieval with correspondence autoencoder. In: ACM multimedia conference, ACM, pp 7\u201316","DOI":"10.1145\/2647868.2654902"},{"issue":"9","key":"166_CR16","doi-asserted-by":"publisher","first-page":"2538","DOI":"10.1109\/TBME.2012.2205687","volume":"59","author":"MM Fraz","year":"2012","unstructured":"Fraz MM, Remagnino P, Hoppe A, Uyyanonvara B, Rudnicka AR, Owen CG, Barman SA (2012) An ensemble classification-based approach applied to retinal blood vessel segmentation. IEEE Trans Biomed Eng 59(9):2538\u20132548","journal-title":"IEEE Trans Biomed Eng"},{"key":"166_CR17","unstructured":"Frome A, Corrado GS, Shlens J, Bengio S, Dean J, Mikolov T et\u00a0al (2013) Devise: a deep visual-semantic embedding model. In: Advances in neural information processing systems, pp 2121\u20132129"},{"issue":"2","key":"166_CR18","doi-asserted-by":"publisher","first-page":"210","DOI":"10.1007\/s11263-013-0658-4","volume":"106","author":"Y Gong","year":"2014","unstructured":"Gong Y, Ke Q, Isard M, Lazebnik S (2014) A multi-view embedding space for modeling internet images, tags, and their semantics. Int J Comput Vis 106(2):210\u2013233","journal-title":"Int J Comput Vis"},{"issue":"12","key":"166_CR19","doi-asserted-by":"publisher","first-page":"2639","DOI":"10.1162\/0899766042321814","volume":"16","author":"DR Hardoon","year":"2004","unstructured":"Hardoon DR, Szedmak S, Shawe-Taylor J (2004) Canonical correlation analysis: an overview with application to learning methods. Neural Comput 16(12):2639\u20132664","journal-title":"Neural Comput"},{"key":"166_CR20","doi-asserted-by":"crossref","unstructured":"He K, Zhang X, Ren S, Sun J (2016) Deep residual learning for image recognition. In: IEEE conference on computer vision and pattern recognition, IEEE, pp 770\u2013778","DOI":"10.1109\/CVPR.2016.90"},{"key":"166_CR21","unstructured":"Henning CA, Ewerth R (2017) Estimating the information gap between textual and visual representations. In: International conference on multimedia retrieval, ACM, pp 14\u201322"},{"key":"166_CR22","doi-asserted-by":"publisher","first-page":"853","DOI":"10.1613\/jair.3994","volume":"47","author":"M Hodosh","year":"2013","unstructured":"Hodosh M, Young P, Hockenmaier J (2013) Framing image description as a ranking task: data, models and evaluation metrics. J Artif Intell Res 47:853\u2013899","journal-title":"J Artif Intell Res"},{"key":"166_CR23","doi-asserted-by":"crossref","unstructured":"Huang G, Liu Z, van\u00a0der Maaten L, Weinberger KQ (2017) Densely connected convolutional networks. In: IEEE conference on computer vision and pattern recognition, IEEE, pp 2261\u20132269","DOI":"10.1109\/CVPR.2017.243"},{"key":"166_CR24","doi-asserted-by":"crossref","unstructured":"Huang Y, Wang W, Wang L (2017) Instance-aware image and sentence matching with selective multimodal LSTM. In: IEEE conference on computer vision and pattern recognition, IEEE, pp 2310\u20132318","DOI":"10.1109\/CVPR.2017.767"},{"key":"166_CR25","doi-asserted-by":"crossref","unstructured":"Karpathy A, Fei-Fei L (2015) Deep visual-semantic alignments for generating image descriptions. In: IEEE conference on computer vision and pattern recognition, IEEE, pp 3128\u20133137","DOI":"10.1109\/CVPR.2015.7298932"},{"key":"166_CR26","unstructured":"Karpathy A, Joulin A, Li FFF (2014) Deep fragment embeddings for bidirectional image sentence mapping. In: Advances in neural information processing systems, pp 1889\u20131897"},{"key":"166_CR27","unstructured":"Kingma D, Ba J (2014) Adam: a method for stochastic optimization. arXiv preprint arXiv:1412.6980"},{"key":"166_CR28","unstructured":"Kiros R, Salakhutdinov R, Zemel RS (2014) Unifying visual-semantic embeddings with multimodal neural language models. arXiv preprint arXiv:1411.2539"},{"key":"166_CR29","unstructured":"Kiros R, Zhu Y, Salakhutdinov RR, Zemel R, Urtasun R, Torralba A, Fidler S (2015) Skip-thought vectors. In: Advances in neural information processing systems, pp 3294\u20133302"},{"key":"166_CR30","doi-asserted-by":"crossref","unstructured":"Klein B, Lev G, Sadeh G, Wolf L (2015) Associating neural word embeddings with deep image representations using fisher vectors. In: IEEE conference on computer vision and pattern recognition, IEEE, pp 4437\u20134446","DOI":"10.1109\/CVPR.2015.7299073"},{"key":"166_CR31","doi-asserted-by":"crossref","unstructured":"Lee JH (1997) Analyses of multiple evidence combination. In: ACM SIGIR forum, vol\u00a031, ACM, pp 267\u2013276","DOI":"10.1145\/278459.258587"},{"key":"166_CR32","unstructured":"Ma Z, Lu Y, Foster D (2015) Finding linear structure in large datasets with scalable canonical correlation analysis. In: International conference on machine learning, pp 169\u2013178"},{"key":"166_CR33","unstructured":"Manmatha R, Wu CY, Smola AJ, Krahenbuhl P (2017) Sampling matters in deep embedding learning. In: IEEE international conference on computer vision, IEEE, pp 2859\u20132867"},{"key":"166_CR34","unstructured":"Mikolov T, Sutskever I, Chen K, Corrado GS, Dean J (2013) Distributed representations of words and phrases and their compositionality. In: Advances in neural information processing systems, pp 3111\u20133119"},{"key":"166_CR35","doi-asserted-by":"crossref","unstructured":"Mithun NC, Li J, Metze F, Roy-Chowdhury AK (2018) Learning joint embedding with multimodal cues for cross-modal video-text retrieval. In: ACM international conference on multimedia retrieval","DOI":"10.1145\/3206025.3206064"},{"key":"166_CR36","doi-asserted-by":"crossref","unstructured":"Mithun NC, Munir S, Guo K, Shelton C (2018) Odds: real-time object detection using depth sensors on embedded gpus. In: ACM\/IEEE international conference on information processing in sensor networks, IEEE Press, pp 230\u2013241","DOI":"10.1109\/IPSN.2018.00051"},{"key":"166_CR37","doi-asserted-by":"crossref","unstructured":"Mithun NC, Panda R, Roy-Chowdhury AK (2016) Generating diverse image datasets with limited labeling. In: ACM multimedia conference, ACM, pp 566\u2013570","DOI":"10.1145\/2964284.2967285"},{"key":"166_CR38","doi-asserted-by":"crossref","unstructured":"Mithun NC, Rameswar P, Papalexakis E, Roy-Chowdhury A (2018) Webly supervised joint embedding for cross-modal image-text retrieval. In: ACM international conference on multimedia","DOI":"10.1145\/3240508.3240712"},{"key":"166_CR39","doi-asserted-by":"crossref","unstructured":"Nam H, Ha JW, Kim J (2017) Dual attention networks for multimodal reasoning and matching. In: IEEE conference on computer vision and pattern recognition, IEEE, pp 299\u2013307","DOI":"10.1109\/CVPR.2017.232"},{"key":"166_CR40","doi-asserted-by":"crossref","unstructured":"Otani M, Nakashima Y, Rahtu E, Heikkil\u00e4 J, Yokoya N (2016) Learning joint representations of videos and sentences with web image search. In: European conference on computer vision, Springer, pp 651\u2013667","DOI":"10.1007\/978-3-319-46604-0_46"},{"key":"166_CR41","doi-asserted-by":"crossref","unstructured":"Pan Y, Mei T, Yao T, Li H, Rui Y (2016) Jointly modeling embedding and translation to bridge video and language. In: IEEE conference on computer vision and pattern recognition, IEEE, pp 4594\u20134602","DOI":"10.1109\/CVPR.2016.497"},{"issue":"3","key":"166_CR42","doi-asserted-by":"publisher","first-page":"21","DOI":"10.1109\/MCAS.2006.1688199","volume":"6","author":"R Polikar","year":"2006","unstructured":"Polikar R (2006) Ensemble based systems in decision making. IEEE Circuits Syst Mag 6(3):21\u201345","journal-title":"IEEE Circuits Syst Mag"},{"issue":"4","key":"166_CR43","doi-asserted-by":"publisher","first-page":"59","DOI":"10.1109\/MSP.2007.4286565","volume":"24","author":"R Polikar","year":"2007","unstructured":"Polikar R (2007) Bootstrap inspired techniques in computational intelligence: ensemble of classifiers, incremental learning, data fusion and missing features. IEEE Signal Process Mag 24(4):59\u201372","journal-title":"IEEE Signal Process Mag"},{"key":"166_CR44","doi-asserted-by":"crossref","unstructured":"TQN Tran, Le\u00a0Borgne H, Crucianu M (2016) Aggregating image and text quantized correlated components. In: Proceedings of the IEEE conference on computer vision and pattern recognition, pp 2046\u20132054","DOI":"10.1109\/CVPR.2016.225"},{"key":"166_CR45","doi-asserted-by":"crossref","unstructured":"Ramanishka V, Das A, Park DH, Venugopalan S, Hendricks LA, Rohrbach M, Saenko K (2016) Multimodal video description. In: ACM multimedia conference, ACM, pp 1092\u20131096","DOI":"10.1145\/2964284.2984066"},{"key":"166_CR46","doi-asserted-by":"crossref","unstructured":"Redmon J, Divvala S, Girshick R, Farhadi A (2016) You only look once: Unified, real-time object detection. In: IEEE conference on computer vision and pattern recognition, IEEE, pp 779\u2013788","DOI":"10.1109\/CVPR.2016.91"},{"key":"166_CR47","unstructured":"Ren S, He K, Girshick R, Sun J (2015) Faster r-cnn: towards real-time object detection with region proposal networks. In: Advances in neural information processing systems, pp 91\u201399"},{"key":"166_CR48","doi-asserted-by":"crossref","unstructured":"Schroff F, Kalenichenko D, Philbin J (2015) Facenet: a unified embedding for face recognition and clustering. In: IEEE conference on computer vision and pattern recognition, IEEE, pp 815\u2013823","DOI":"10.1109\/CVPR.2015.7298682"},{"key":"166_CR49","doi-asserted-by":"crossref","unstructured":"Socher R, Fei-Fei L (2010) Connecting modalities: Semi-supervised segmentation and annotation of images using unaligned text corpora. In: IEEE conference on computer vision and pattern recognition, IEEE, pp 966\u2013973","DOI":"10.1109\/CVPR.2010.5540112"},{"key":"166_CR50","unstructured":"Torabi A, Tandon N, Sigal L (2016) Learning language-visual embedding for movie understanding with natural-language. arXiv preprint arXiv:1609.08124"},{"key":"166_CR51","doi-asserted-by":"crossref","unstructured":"Usunier N, Buffoni D, Gallinari P (2009) Ranking with ordered weighted pairwise classification. In: International conference on machine learning, ACM, pp 1057\u20131064","DOI":"10.1145\/1553374.1553509"},{"key":"166_CR52","unstructured":"Vendrov I, Kiros R, Fidler S, Urtasun R (2016) Order-embeddings of images and language. In: International conference on learning representations"},{"key":"166_CR53","doi-asserted-by":"crossref","unstructured":"Venugopalan S, Rohrbach M, Donahue J, Mooney R, Darrell T, Saenko K (2015) Sequence to sequence-video to text. In: IEEE International conference on computer vision, IEEE, pp 4534\u20134542","DOI":"10.1109\/ICCV.2015.515"},{"key":"166_CR54","doi-asserted-by":"crossref","unstructured":"Vukoti\u0107 V, Raymond C, Gravier G (2016) Bidirectional joint representation learning with symmetrical deep neural networks for multimodal and crossmodal applications. In: ACM international conference on multimedia retrieval, ACM, pp 343\u2013346","DOI":"10.1145\/2911996.2912064"},{"key":"166_CR55","doi-asserted-by":"crossref","unstructured":"Vukoti\u0107 V, Raymond C, Gravier G (2017) Generative adversarial networks for multimodal representation learning in video hyperlinking. In: ACM international conference on multimedia retrieval, ACM, pp 416\u2013419","DOI":"10.1145\/3078971.3079038"},{"issue":"2","key":"166_CR56","doi-asserted-by":"publisher","first-page":"11","DOI":"10.1109\/MMUL.2018.023121161","volume":"25","author":"V Vukoti\u0107","year":"2018","unstructured":"Vukoti\u0107 V, Raymond C, Gravier G (2018) A crossmodal approach to multimodal fusion in video hyperlinking. IEEE Multimed 25(2):11\u201323","journal-title":"IEEE Multimed"},{"key":"166_CR57","doi-asserted-by":"crossref","unstructured":"Wang B, Yang Y, Xu X, Hanjalic A, Shen HT (2017) Adversarial cross-modal retrieval. In: ACM multimedia conference, ACM, pp 154\u2013162","DOI":"10.1145\/3123266.3123326"},{"key":"166_CR58","unstructured":"Wang L, Li Y, Huang J, Lazebnik S (2018) Learning two-branch neural networks for image-text matching tasks. IEEE Trans Pattern Anal Mach Intell 41(2):394\u2013407"},{"key":"166_CR59","doi-asserted-by":"crossref","unstructured":"Wang L, Li Y, Lazebnik S (2016) Learning deep structure-preserving image-text embeddings. In: IEEE conference on computer vision and pattern recognition, IEEE, pp 5005\u20135013","DOI":"10.1109\/CVPR.2016.541"},{"key":"166_CR60","doi-asserted-by":"crossref","unstructured":"Xu J, Mei T, Yao T, Rui Y (2016) Msr-vtt: A large video description dataset for bridging video and language. In: IEEE conference on computer vision and pattern recognition, pp 5288\u20135296","DOI":"10.1109\/CVPR.2016.571"},{"key":"166_CR61","doi-asserted-by":"crossref","unstructured":"Xu R, Xiong C, Chen W, Corso JJ (2015) Jointly modeling deep video and compositional text to bridge vision and language in a unified framework. In: AAAI, vol\u00a05, p\u00a06","DOI":"10.1609\/aaai.v29i1.9512"},{"key":"166_CR62","doi-asserted-by":"crossref","unstructured":"Yan F, Mikolajczyk K (2015) Deep correlation for matching images and text. In: IEEE conference on computer vision and pattern recognition, IEEE, pp 3441\u20133450","DOI":"10.1109\/CVPR.2015.7298966"},{"key":"166_CR63","doi-asserted-by":"crossref","unstructured":"Yan R, Yang J, Hauptmann AG (2004) Learning query-class dependent weights in automatic video retrieval. In: ACM multimedia conference, ACM, pp 548\u2013555","DOI":"10.1145\/1027527.1027661"},{"key":"166_CR64","doi-asserted-by":"crossref","unstructured":"Zhang L, Ma B, Li G, Huang Q, Tian Q (2017) Multi-networks joint learning for large-scale cross-modal retrieval. In: ACM multimedia conference, ACM, pp 907\u2013915","DOI":"10.1145\/3123266.3123317"},{"key":"166_CR65","doi-asserted-by":"crossref","unstructured":"Zhang X, Gao K, Zhang Y, Zhang D, Li J, Tian Q (2017) Task-driven dynamic fusion: reducing ambiguity in video description. In: IEEE conference on computer vision and pattern recognition, IEEE, pp 3713\u20133721","DOI":"10.1109\/CVPR.2017.662"},{"key":"166_CR66","doi-asserted-by":"publisher","first-page":"1452","DOI":"10.1109\/TPAMI.2017.2723009","volume":"40","author":"B Zhou","year":"2017","unstructured":"Zhou B, Lapedriza A, Khosla A, Oliva A, Torralba A (2017) Places: a 10 million image database for scene recognition. IEEE Trans Pattern Anal Mach Intell 40:1452\u20131464","journal-title":"IEEE Trans Pattern Anal Mach Intell"}],"container-title":["International Journal of Multimedia Information Retrieval"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/link.springer.com\/article\/10.1007\/s13735-018-00166-3\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s13735-018-00166-3.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s13735-018-00166-3.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2022,9,10]],"date-time":"2022-09-10T05:03:12Z","timestamp":1662786192000},"score":1,"resource":{"primary":{"URL":"http:\/\/link.springer.com\/10.1007\/s13735-018-00166-3"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2019,1,12]]},"references-count":66,"journal-issue":{"issue":"1","published-print":{"date-parts":[[2019,3]]}},"alternative-id":["166"],"URL":"https:\/\/doi.org\/10.1007\/s13735-018-00166-3","relation":{},"ISSN":["2192-6611","2192-662X"],"issn-type":[{"value":"2192-6611","type":"print"},{"value":"2192-662X","type":"electronic"}],"subject":[],"published":{"date-parts":[[2019,1,12]]},"assertion":[{"value":"16 September 2018","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"6 December 2018","order":2,"name":"revised","label":"Revised","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"25 December 2018","order":3,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"12 January 2019","order":4,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}}]}}