{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,6,18]],"date-time":"2026-06-18T10:14:56Z","timestamp":1781777696740,"version":"3.54.5"},"publisher-location":"New York, NY, USA","reference-count":75,"publisher":"ACM","license":[{"start":{"date-parts":[[2023,10,9]],"date-time":"2023-10-09T00:00:00Z","timestamp":1696809600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2023,10,9]]},"DOI":"10.1145\/3610661.3617602","type":"proceedings-article","created":{"date-parts":[[2023,10,9]],"date-time":"2023-10-09T16:51:22Z","timestamp":1696870282000},"page":"101-104","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":46,"title":["Tutorial on Multimodal Machine Learning: Principles, Challenges, and Open Questions"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0001-7768-3610","authenticated-orcid":false,"given":"Paul Pu","family":"Liang","sequence":"first","affiliation":[{"name":"Machine Learning Department, Carnegie Mellon University, United States of America"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-6376-7696","authenticated-orcid":false,"given":"Louis-Philippe","family":"Morency","sequence":"additional","affiliation":[{"name":"Language Technologies Institute, Carnegie Mellon University, United States of America"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2023,10,9]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"publisher","DOI":"10.1109\/JSTSP.2018.2875385"},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"crossref","unstructured":"Estelle Aflalo Meng Du Shao-Yen Tseng Yongfei Liu Chenfei Wu Nan Duan and Vasudev Lal. 2022. VL-InterpreT: An Interactive Visualization Tool for Interpreting Vision-Language Transformers. In CVPR. 21406\u201321415.","DOI":"10.1109\/CVPR52688.2022.02072"},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1080\/0163853X.2020.1768500"},{"key":"e_1_3_2_1_4_1","unstructured":"Saeed Amizadeh Hamid Palangi Alex Polozov Yichen Huang and Kazuhito Koishida. 2020. Neuro-Symbolic Visual Reasoning: Disentangling Visual from Reasoning. In ICML. PMLR 279\u2013290."},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"crossref","unstructured":"Jacob Andreas Marcus Rohrbach Trevor Darrell and Dan Klein. 2016. Neural module networks. In CVPR. 39\u201348.","DOI":"10.1109\/CVPR.2016.12"},{"key":"e_1_3_2_1_6_1","volume-title":"On the Benefits of Early Fusion in Multimodal Representation Learning. In NeurIPS 2020 Workshop SVRHM.","author":"Barnum George","year":"2020","unstructured":"George Barnum, Sabera\u00a0J Talukder, and Yisong Yue. 2020. On the Benefits of Early Fusion in Multimodal Representation Learning. In NeurIPS 2020 Workshop SVRHM."},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"crossref","unstructured":"Roland Barthes. 1977. Image-music-text. Macmillan.","DOI":"10.1007\/978-1-349-03518-2"},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"crossref","unstructured":"Avrim Blum and Tom Mitchell. 1998. Combining labeled and unlabeled data with co-training. In COLT. 92\u2013100.","DOI":"10.1145\/279943.279962"},{"key":"e_1_3_2_1_9_1","volume-title":"Geometric deep learning: Grids, groups, graphs, geodesics, and gauges. arXiv preprint arXiv:2104.13478","author":"Bronstein M","year":"2021","unstructured":"Michael\u00a0M Bronstein, Joan Bruna, Taco Cohen, and Petar Veli\u010dkovi\u0107. 2021. Geometric deep learning: Grids, groups, graphs, geodesics, and gauges. arXiv preprint arXiv:2104.13478 (2021)."},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2021.findings-acl.375"},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"crossref","unstructured":"Brian Chen Andrew Rouditchenko Kevin Duarte Hilde Kuehne Samuel Thomas Angie Boggust 2021. Multimodal clustering networks for self-supervised learning from unlabeled videos. In ICCV. 8012\u20138021.","DOI":"10.1109\/ICCV48922.2021.00791"},{"key":"e_1_3_2_1_12_1","volume-title":"Extractive Text-Image Summarization Using Multi-Modal RNN. In 2018 14th International Conference on Semantics, Knowledge and Grids (SKG). IEEE, 245\u2013248","author":"Chen Jingqiang","year":"2018","unstructured":"Jingqiang Chen and Hai Zhuge. 2018. Extractive Text-Image Summarization Using Multi-Modal RNN. In 2018 14th International Conference on Semantics, Knowledge and Grids (SKG). IEEE, 245\u2013248."},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"crossref","unstructured":"Volkan Cirik Louis-Philippe Morency and Taylor Berg-Kirkpatrick. 2018. Visual Referring Expression Recognition: What Do Systems Actually Learn?. In NAACL. 781\u2013787.","DOI":"10.18653\/v1\/N18-2123"},{"key":"e_1_3_2_1_14_1","unstructured":"Emily Denton and Rob Fergus. 2018. Stochastic video generation with a learned prior. In ICML. PMLR 1174\u20131183."},{"key":"e_1_3_2_1_15_1","volume-title":"Cross-modal data programming enables rapid medical machine learning. Patterns","author":"Dunnmon A","year":"2020","unstructured":"Jared\u00a0A Dunnmon, Alexander\u00a0J Ratner, Khaled Saab, Nishith Khandwala, Matthew Markert, Hersh Sagreiya, Roger Goldman, 2020. Cross-modal data programming enables rapid medical machine learning. Patterns (2020)."},{"key":"e_1_3_2_1_16_1","volume-title":"Devise: A deep visual-semantic embedding model. In Advances in neural information processing systems. 2121\u20132129.","author":"Frome Andrea","year":"2013","unstructured":"Andrea Frome, Greg\u00a0S Corrado, Jon Shlens, Samy Bengio, Jeff Dean, Marc\u2019Aurelio Ranzato, and Tomas Mikolov. 2013. Devise: A deep visual-semantic embedding model. In Advances in neural information processing systems. 2121\u20132129."},{"key":"e_1_3_2_1_17_1","volume-title":"Perceptual Score: What Data Modalities Does Your Model Perceive?NeurIPS 34","author":"Gat Itai","year":"2021","unstructured":"Itai Gat, Idan Schwartz, and Alex Schwing. 2021. Perceptual Score: What Data Modalities Does Your Model Perceive?NeurIPS 34 (2021)."},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-58589-1_23"},{"key":"e_1_3_2_1_19_1","volume-title":"KAT: A Knowledge Augmented Transformer for Vision-and-Language. arXiv preprint arXiv:2112.08614","author":"Gui Liangke","year":"2021","unstructured":"Liangke Gui, Borui Wang, Qiuyuan Huang, Alex Hauptmann, Yonatan Bisk, and Jianfeng Gao. 2021. KAT: A Knowledge Augmented Transformer for Vision-and-Language. arXiv preprint arXiv:2112.08614 (2021)."},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"crossref","unstructured":"Sanjay Haresh Sateesh Kumar Huseyin Coskun Shahram\u00a0N Syed Andrey Konin Zeeshan Zia and Quoc-Huy Tran. 2021. Learning by aligning videos in time. In CVPR. 5548\u20135558.","DOI":"10.1109\/CVPR46437.2021.00550"},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-01219-9_47"},{"key":"e_1_3_2_1_22_1","volume-title":"Decoupling the role of data, attention, and losses in multimodal transformers. Transactions of the Association for Computational Linguistics","author":"Hendricks Lisa\u00a0Anne","year":"2021","unstructured":"Lisa\u00a0Anne Hendricks, John Mellor, Rosalia Schneider, Jean-Baptiste Alayrac, and Aida Nematzadeh. 2021. Decoupling the role of data, attention, and losses in multimodal transformers. Transactions of the Association for Computational Linguistics (2021)."},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"crossref","unstructured":"Jack Hessel and Lillian Lee. 2020. Does my multimodal model learn cross-modal interactions? It\u2019s harder to tell than you might think!. In EMNLP.","DOI":"10.18653\/v1\/2020.emnlp-main.62"},{"key":"e_1_3_2_1_24_1","volume-title":"Unsupervised multimodal representation learning across medical images and reports. arXiv preprint arXiv:1811.08615","author":"Hsu Ming\u00a0Harry","year":"2018","unstructured":"Tzu-Ming\u00a0Harry Hsu, Wei-Hung Weng, Willie Boag, Matthew McDermott, and Peter Szolovits. 2018. Unsupervised multimodal representation learning across medical images and reports. arXiv preprint arXiv:1811.08615 (2018)."},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.knosys.2019.05.017"},{"key":"e_1_3_2_1_26_1","volume-title":"Learning by abstraction: The neural state machine. NeurIPS","author":"Hudson Drew","year":"2019","unstructured":"Drew Hudson and Christopher\u00a0D Manning. 2019. Learning by abstraction: The neural state machine. NeurIPS (2019)."},{"key":"e_1_3_2_1_27_1","volume-title":"International Conference on Learning Representations. https:\/\/openreview.net\/forum?id=rylnK6VtDH","author":"Jayakumar M.","year":"2020","unstructured":"Siddhant\u00a0M. Jayakumar, Wojciech\u00a0M. Czarnecki, Jacob Menick, Jonathan Schwarz, Jack Rae, Simon Osindero, Yee\u00a0Whye Teh, Tim Harley, and Razvan Pascanu. 2020. Multiplicative Interactions and Where to Find Them. In International Conference on Learning Representations. https:\/\/openreview.net\/forum?id=rylnK6VtDH"},{"key":"e_1_3_2_1_28_1","unstructured":"Chao Jia Yinfei Yang Ye Xia Yi-Ting Chen Zarana Parekh Hieu Pham 2021. Scaling up visual and vision-language representation learning with noisy text supervision. In ICML. PMLR 4904\u20134916."},{"key":"e_1_3_2_1_29_1","volume-title":"Visualbert: A simple and performant baseline for vision and language. arXiv preprint arXiv:1908.03557","author":"Li Liunian\u00a0Harold","year":"2019","unstructured":"Liunian\u00a0Harold Li, Mark Yatskar, Da Yin, Cho-Jui Hsieh, and Kai-Wei Chang. 2019. Visualbert: A simple and performant baseline for vision and language. arXiv preprint arXiv:1908.03557 (2019)."},{"key":"e_1_3_2_1_30_1","volume-title":"Clip-event: Connecting text and images with event structures. In CVPR. 16420\u201316429.","author":"Li Manling","year":"2022","unstructured":"Manling Li, Ruochen Xu, Shuohang Wang, Luowei Zhou, Xudong Lin, Chenguang Zhu, Michael Zeng, Heng Ji, and Shih-Fu Chang. 2022. Clip-event: Connecting text and images with event structures. In CVPR. 16420\u201316429."},{"key":"e_1_3_2_1_31_1","volume-title":"Keep meeting summaries on topic: Abstractive multi-modal meeting summarization. In ACL. 2190\u20132196","author":"Li Manling","year":"2019","unstructured":"Manling Li, Lingyu Zhang, Heng Ji, and Richard\u00a0J Radke. 2019. Keep meeting summaries on topic: Abstractive multi-modal meeting summarization. In ACL. 2190\u20132196."},{"key":"e_1_3_2_1_32_1","volume-title":"An information decomposition framework. arXiv preprint arXiv:2302.12247","author":"Liang Paul\u00a0Pu","year":"2023","unstructured":"Paul\u00a0Pu Liang, Yun Cheng, Xiang Fan, Chun\u00a0Kai Ling, Suzanne Nie, Richard Chen, Zihao Deng, Faisal Mahmood, Ruslan Salakhutdinov, and Louis-Philippe Morency. 2023. Quantifying & modeling feature interactions: An information decomposition framework. arXiv preprint arXiv:2302.12247 (2023)."},{"key":"e_1_3_2_1_33_1","volume-title":"MultiViz: Towards Visualizing and Understanding Multimodal Models. International Conference on Learning Representations","author":"Liang Paul\u00a0Pu","year":"2023","unstructured":"Paul\u00a0Pu Liang, Yiwei Lyu, Gunjan Chhablani, Nihal Jain, Zihao Deng, Xingbo Wang, Louis-Philippe Morency, and Ruslan Salakhutdinov. 2023. MultiViz: Towards Visualizing and Understanding Multimodal Models. International Conference on Learning Representations (2023)."},{"key":"e_1_3_2_1_34_1","volume-title":"High-Modality Multimodal Transformer: Quantifying Modality & Interaction Heterogeneity for High-Modality Representation Learning. Transactions on Machine Learning Research","author":"Liang Paul\u00a0Pu","year":"2023","unstructured":"Paul\u00a0Pu Liang, Yiwei Lyu, Xiang Fan, Jeffrey Tsaw, Yudong Liu, Shentong Mo, Dani Yogatama, Louis-Philippe Morency, and Russ Salakhutdinov. 2023. High-Modality Multimodal Transformer: Quantifying Modality & Interaction Heterogeneity for High-Modality Representation Learning. Transactions on Machine Learning Research (2023)."},{"key":"e_1_3_2_1_35_1","unstructured":"Paul\u00a0Pu Liang Yiwei Lyu Xiang Fan Zetian Wu Yun Cheng Jason Wu Leslie\u00a0Yufan Chen Peter Wu Michelle\u00a0A Lee Yuke Zhu Ruslan Salakhutdinov and Louis-Philippe Morency. 2021. MultiBench: Multiscale Benchmarks for Multimodal Representation Learning. In NeurIPS Datasets and Benchmarks Track."},{"key":"e_1_3_2_1_36_1","volume-title":"Foundations and recent trends in multimodal machine learning: Principles, challenges, and open questions. arXiv preprint arXiv:2209.03430","author":"Liang Paul\u00a0Pu","year":"2022","unstructured":"Paul\u00a0Pu Liang, Amir Zadeh, and Louis-Philippe Morency. 2022. Foundations and recent trends in multimodal machine learning: Principles, challenges, and open questions. arXiv preprint arXiv:2209.03430 (2022)."},{"key":"e_1_3_2_1_37_1","volume-title":"Vilbert: Pretraining task-agnostic visiolinguistic representations for vision-and-language tasks. In Advances in Neural Information Processing Systems. 13\u201323.","author":"Lu Jiasen","year":"2019","unstructured":"Jiasen Lu, Dhruv Batra, Devi Parikh, and Stefan Lee. 2019. Vilbert: Pretraining task-agnostic visiolinguistic representations for vision-and-language tasks. In Advances in Neural Information Processing Systems. 13\u201323."},{"key":"e_1_3_2_1_38_1","doi-asserted-by":"crossref","unstructured":"Jelena Luketina Nantas Nardelli Gregory Farquhar Jakob\u00a0N Foerster Jacob Andreas Edward Grefenstette Shimon Whiteson and Tim Rockt\u00e4schel. 2019. A Survey of Reinforcement Learning Informed by Natural Language. In IJCAI.","DOI":"10.24963\/ijcai.2019\/880"},{"key":"e_1_3_2_1_39_1","volume-title":"Smil: Multimodal learning with severely missing modality. arXiv preprint arXiv:2103.05677","author":"Ma Mengmeng","year":"2021","unstructured":"Mengmeng Ma, Jian Ren, Long Zhao, Sergey Tulyakov, Cathy Wu, and Xi Peng. 2021. Smil: Multimodal learning with severely missing modality. arXiv preprint arXiv:2103.05677 (2021)."},{"key":"e_1_3_2_1_40_1","volume-title":"A taxonomy of relationships between images and text. Journal of documentation","author":"Marsh E","year":"2003","unstructured":"Emily\u00a0E Marsh and Marilyn Domas\u00a0White. 2003. A taxonomy of relationships between images and text. Journal of documentation (2003)."},{"key":"e_1_3_2_1_41_1","doi-asserted-by":"publisher","DOI":"10.1109\/JPROC.2015.2483592"},{"key":"e_1_3_2_1_42_1","doi-asserted-by":"crossref","unstructured":"Yulei Niu Kaihua Tang Hanwang Zhang Zhiwu Lu Xian-Sheng Hua and Ji-Rong Wen. 2021. Counterfactual vqa: A cause-effect look at language bias. In CVPR. 12700\u201312710.","DOI":"10.1109\/CVPR46437.2021.01251"},{"key":"e_1_3_2_1_43_1","unstructured":"Aaron Oord Yazhe Li Igor Babuschkin Karen Simonyan Oriol Vinyals 2018. Parallel wavenet: Fast high-fidelity speech synthesis. In ICML. PMLR 3918\u20133926."},{"key":"e_1_3_2_1_44_1","doi-asserted-by":"publisher","DOI":"10.1007\/s13735-019-00187-6"},{"key":"e_1_3_2_1_45_1","volume-title":"Multimodal abstractive summarization for how2 videos. arXiv preprint arXiv:1906.07901","author":"Palaskar Shruti","year":"2019","unstructured":"Shruti Palaskar, Jindrich Libovick\u1ef3, Spandana Gella, and Florian Metze. 2019. Multimodal abstractive summarization for how2 videos. arXiv preprint arXiv:1906.07901 (2019)."},{"key":"e_1_3_2_1_46_1","doi-asserted-by":"publisher","DOI":"10.1126\/science.283.5406.1272"},{"key":"e_1_3_2_1_47_1","doi-asserted-by":"crossref","unstructured":"Alejandro Pe\u00f1a Ignacio Serna Aythami Morales and Julian Fierrez. 2020. FairCVtest Demo: Understanding Bias in Multimodal Learning with a Testbed in Fair Automatic Recruitment. In ICMI. 760\u2013761.","DOI":"10.1145\/3382507.3421165"},{"key":"e_1_3_2_1_48_1","volume-title":"Mfas: Multimodal fusion architecture search. In CVPR. 6966\u20136975.","author":"P\u00e9rez-R\u00faa Juan-Manuel","year":"2019","unstructured":"Juan-Manuel P\u00e9rez-R\u00faa, Valentin Vielzeuf, St\u00e9phane Pateux, Moez Baccouche, and Fr\u00e9d\u00e9ric Jurie. 2019. Mfas: Multimodal fusion architecture search. In CVPR. 6966\u20136975."},{"key":"e_1_3_2_1_49_1","doi-asserted-by":"crossref","unstructured":"Bryan\u00a0A Plummer Liwei Wang Chris\u00a0M Cervantes Juan\u00a0C Caicedo Julia Hockenmaier and Svetlana Lazebnik. 2015. Flickr30k entities: Collecting region-to-phrase correspondences for richer image-to-sentence models. In ICCV.","DOI":"10.1109\/ICCV.2015.303"},{"key":"e_1_3_2_1_50_1","unstructured":"Alec Radford Jong\u00a0Wook Kim Chris Hallacy Aditya Ramesh Gabriel Goh Sandhini Agarwal 2021. Learning transferable visual models from natural language supervision. In ICML. PMLR 8748\u20138763."},{"key":"e_1_3_2_1_51_1","volume-title":"International Conference on Machine Learning. PMLR, 8821\u20138831","author":"Ramesh Aditya","year":"2021","unstructured":"Aditya Ramesh, Mikhail Pavlov, Gabriel Goh, Scott Gray, Chelsea Voss, Alec Radford, Mark Chen, and Ilya Sutskever. 2021. Zero-shot text-to-image generation. In International Conference on Machine Learning. PMLR, 8821\u20138831."},{"key":"e_1_3_2_1_52_1","doi-asserted-by":"crossref","unstructured":"Robin Rombach Andreas Blattmann Dominik Lorenz Patrick Esser and Bj\u00f6rn Ommer. 2022. High-resolution image synthesis with latent diffusion models. In CVPR. 10684\u201310695.","DOI":"10.1109\/CVPR52688.2022.01042"},{"key":"e_1_3_2_1_53_1","volume-title":"Make-a-video: Text-to-video generation without text-video data. arXiv preprint arXiv:2209.14792","author":"Singer Uriel","year":"2022","unstructured":"Uriel Singer, Adam Polyak, Thomas Hayes, Xi Yin, Jie An, Songyang Zhang, Qiyuan Hu, 2022. Make-a-video: Text-to-video generation without text-video data. arXiv preprint arXiv:2209.14792 (2022)."},{"key":"e_1_3_2_1_54_1","volume-title":"FLAVA: A Foundational Language And Vision Alignment Model. arXiv preprint arXiv:2112.04482","author":"Singh Amanpreet","year":"2021","unstructured":"Amanpreet Singh, Ronghang Hu, Vedanuj Goswami, Guillaume Couairon, 2021. FLAVA: A Foundational Language And Vision Alignment Model. arXiv preprint arXiv:2112.04482 (2021)."},{"key":"e_1_3_2_1_55_1","unstructured":"Karthik Sridharan and Sham\u00a0M Kakade. 2008. An information theoretic framework for multi-view learning. (2008)."},{"key":"e_1_3_2_1_56_1","volume-title":"Carl Vondrick, Kevin Murphy, and Cordelia Schmid.","author":"Sun Chen","year":"2019","unstructured":"Chen Sun, Austin Myers, Carl Vondrick, Kevin Murphy, and Cordelia Schmid. 2019. Videobert: A joint model for video and language representation learning. In ICCV. 7464\u20137473."},{"key":"e_1_3_2_1_57_1","volume-title":"Visually-Grounded Supervision. In Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing (EMNLP). 2066\u20132080","author":"Tan Hao","year":"2020","unstructured":"Hao Tan and Mohit Bansal. 2020. Vokenization: Improving Language Understanding via Contextualized, Visually-Grounded Supervision. In Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing (EMNLP). 2066\u20132080."},{"key":"e_1_3_2_1_58_1","volume-title":"Winoground: Probing Vision and Language Models for Visio-Linguistic Compositionality. In CVPR. 5238\u20135248.","author":"Thrush Tristan","year":"2022","unstructured":"Tristan Thrush, Ryan Jiang, Max Bartolo, Amanpreet Singh, Adina Williams, Douwe Kiela, and Candace Ross. 2022. Winoground: Probing Vision and Language Models for Visio-Linguistic Compositionality. In CVPR. 5238\u20135248."},{"key":"e_1_3_2_1_59_1","volume-title":"What makes for good views for contrastive learning?NeurIPS 33","author":"Tian Yonglong","year":"2020","unstructured":"Yonglong Tian, Chen Sun, Ben Poole, Dilip Krishnan, Cordelia Schmid, and Phillip Isola. 2020. What makes for good views for contrastive learning?NeurIPS 33 (2020), 6827\u20136839."},{"key":"e_1_3_2_1_60_1","volume-title":"Learning factorized multimodal representations. ICLR","author":"Tsai Hung\u00a0Hubert","year":"2019","unstructured":"Yao-Hung\u00a0Hubert Tsai, Paul\u00a0Pu Liang, Amir Zadeh, Louis-Philippe Morency, and Ruslan Salakhutdinov. 2019. Learning factorized multimodal representations. ICLR (2019)."},{"key":"e_1_3_2_1_61_1","volume-title":"Multimodal few-shot learning with frozen language models. NeurIPS 34","author":"Tsimpoukelli Maria","year":"2021","unstructured":"Maria Tsimpoukelli, Jacob Menick, Serkan Cabi, SM Eslami, Oriol Vinyals, and Felix Hill. 2021. Multimodal few-shot learning with frozen language models. NeurIPS 34 (2021)."},{"key":"e_1_3_2_1_62_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10994-005-0913-1"},{"key":"e_1_3_2_1_63_1","volume-title":"Multimodality and reading: The construction of meaning through image-text interaction","author":"Unsworth Len","unstructured":"Len Unsworth and Chris Cl\u00e9irigh. 2014. Multimodality and reading: The construction of meaning through image-text interaction. Routledge."},{"key":"e_1_3_2_1_64_1","doi-asserted-by":"crossref","unstructured":"Weiyao Wang Du Tran and Matt Feiszli. 2020. What Makes Training Multi-Modal Classification Networks Hard?. In CVPR. 12695\u201312705.","DOI":"10.1109\/CVPR42600.2020.01271"},{"key":"e_1_3_2_1_65_1","doi-asserted-by":"publisher","DOI":"10.1109\/TVCG.2021.3114794"},{"key":"e_1_3_2_1_66_1","volume-title":"Nonnegative decomposition of multivariate information. arXiv preprint arXiv:1004.2515","author":"Williams L","year":"2010","unstructured":"Paul\u00a0L Williams and Randall\u00a0D Beer. 2010. Nonnegative decomposition of multivariate information. arXiv preprint arXiv:1004.2515 (2010)."},{"key":"e_1_3_2_1_67_1","volume-title":"Characterizing and overcoming the greedy nature of learning in multi-modal deep neural networks. arXiv preprint arXiv:2202.05306","author":"Wu Nan","year":"2022","unstructured":"Nan Wu, Stanis\u0142aw Jastrz\u0119bski, Kyunghyun Cho, and Krzysztof\u00a0J Geras. 2022. Characterizing and overcoming the greedy nature of learning in multi-modal deep neural networks. arXiv preprint arXiv:2202.05306 (2022)."},{"key":"e_1_3_2_1_68_1","unstructured":"Caiming Xiong Stephen Merity and Richard Socher. 2016. Dynamic memory networks for visual and textual question answering. In ICML."},{"key":"e_1_3_2_1_69_1","volume-title":"MTAG: Modal-Temporal Attention Graph for Unaligned Human Multimodal Language Sequences. In NAACL-HLT.","author":"Yang Jianing","year":"2021","unstructured":"Jianing Yang, Yongxin Wang, Ruitao Yi, Yuying Zhu, Azaan Rehman, Amir Zadeh, 2021. MTAG: Modal-Temporal Attention Graph for Unaligned Human Multimodal Language Sequences. In NAACL-HLT."},{"key":"e_1_3_2_1_70_1","volume-title":"Tensor fusion network for multimodal sentiment analysis. arXiv preprint arXiv:1707.07250","author":"Zadeh Amir","year":"2017","unstructured":"Amir Zadeh, Minghai Chen, Soujanya Poria, Erik Cambria, and Louis-Philippe Morency. 2017. Tensor fusion network for multimodal sentiment analysis. arXiv preprint arXiv:1707.07250 (2017)."},{"key":"e_1_3_2_1_71_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.inffus.2020.06.001"},{"key":"e_1_3_2_1_72_1","volume-title":"Taskonomy: Disentangling task transfer learning. In CVPR. 3712\u20133722.","author":"Zamir R","year":"2018","unstructured":"Amir\u00a0R Zamir, Alexander Sax, William Shen, Leonidas\u00a0J Guibas, Jitendra Malik, and Silvio Savarese. 2018. Taskonomy: Disentangling task transfer learning. In CVPR. 3712\u20133722."},{"key":"e_1_3_2_1_73_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2019.00688"},{"key":"e_1_3_2_1_74_1","volume-title":"Socratic models: Composing zero-shot multimodal reasoning with language. arXiv preprint arXiv:2204.00598","author":"Zeng Andy","year":"2022","unstructured":"Andy Zeng, Adrian Wong, Stefan Welker, Krzysztof Choromanski, Federico Tombari, 2022. Socratic models: Composing zero-shot multimodal reasoning with language. arXiv preprint arXiv:2204.00598 (2022)."},{"key":"e_1_3_2_1_75_1","doi-asserted-by":"crossref","unstructured":"Hao Zhu Huaibo Huang Yi Li Aihua Zheng and Ran He. 2021. Arbitrary talking face generation via attentional audio-visual coherence learning. In IJCAI. 2362\u20132368.","DOI":"10.24963\/ijcai.2020\/327"}],"event":{"name":"ICMI '23: INTERNATIONAL CONFERENCE ON MULTIMODAL INTERACTION","location":"Paris France","acronym":"ICMI '23","sponsor":["SIGCHI ACM Special Interest Group on Computer-Human Interaction"]},"container-title":["International Cconference on Multimodal Interaction"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3610661.3617602","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3610661.3617602","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,8,22]],"date-time":"2025-08-22T19:33:10Z","timestamp":1755891190000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3610661.3617602"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023,10,9]]},"references-count":75,"alternative-id":["10.1145\/3610661.3617602","10.1145\/3610661"],"URL":"https:\/\/doi.org\/10.1145\/3610661.3617602","relation":{},"subject":[],"published":{"date-parts":[[2023,10,9]]},"assertion":[{"value":"2023-10-09","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}