{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,9,1]],"date-time":"2026-09-01T00:01:13Z","timestamp":1788220873967,"version":"build-2803163510"},"reference-count":42,"publisher":"Springer Science and Business Media LLC","issue":"4","license":[{"start":{"date-parts":[[2024,7,3]],"date-time":"2024-07-03T00:00:00Z","timestamp":1719964800000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2024,7,3]],"date-time":"2024-07-03T00:00:00Z","timestamp":1719964800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Vis Comput"],"published-print":{"date-parts":[[2025,3]]},"DOI":"10.1007\/s00371-024-03546-5","type":"journal-article","created":{"date-parts":[[2024,7,3]],"date-time":"2024-07-03T14:03:03Z","timestamp":1720015383000},"page":"2461-2476","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":7,"title":["Deep attentive multimodal learning for food information enhancement via early-stage heterogeneous fusion"],"prefix":"10.1007","volume":"41","author":[{"given":"Avantika","family":"Saklani","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-7209-0437","authenticated-orcid":false,"given":"Shailendra","family":"Tiwari","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"H. S.","family":"Pannu","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2024,7,3]]},"reference":[{"key":"3546_CR1","unstructured":"News, A.: YouTube mistakenly flags Notre Dame Cathedral fire videos as 9\/11 conspiracy abcnews.go.com. https:\/\/abcnews.go.com\/Business\/youtube-mistakenly-flags-notre-dame-fire\/-videos-911\/ story?id=62419884"},{"key":"3546_CR2","doi-asserted-by":"publisher","first-page":"26","DOI":"10.1016\/j.knosys.2019.01.019","volume":"167","author":"F Huang","year":"2019","unstructured":"Huang, F., Zhang, X., Zhao, Z., Xu, J., Li, Z.: Image-text sentiment analysis via deep multimodal attentive fusion. Knowl.-Based Syst. 167, 26\u201337 (2019)","journal-title":"Knowl.-Based Syst."},{"key":"3546_CR3","doi-asserted-by":"publisher","first-page":"4014","DOI":"10.1109\/TMM.2020.3035277","volume":"23","author":"X Yang","year":"2020","unstructured":"Yang, X., Feng, S., Wang, D., Zhang, Y.: Image-text multimodal emotion classification via multi-view attentional network. IEEE Trans. Multimed. 23, 4014\u20134026 (2020)","journal-title":"IEEE Trans. Multimed."},{"key":"3546_CR4","doi-asserted-by":"publisher","first-page":"1653696","DOI":"10.1155\/2022\/1653696","volume":"2022","author":"SK Bharti","year":"2022","unstructured":"Bharti, S.K., Gupta, R.K., Shukla, P.K., Hatamleh, W.A., Tarazi, H., Nuagah, S.J.: Multimodal sarcasm detection: a deep learning approach. Wirel. Commun. Mob. Comput. 2022, 1653696 (2022)","journal-title":"Wirel. Commun. Mob. Comput."},{"key":"3546_CR5","doi-asserted-by":"publisher","DOI":"10.1016\/j.eswa.2021.115412","volume":"184","author":"R Kumari","year":"2021","unstructured":"Kumari, R., Ekbal, A.: Amfb: attention based multimodal factorized bilinear pooling for multimodal fake news detection. Expert Syst. Appl. 184, 115412 (2021)","journal-title":"Expert Syst. Appl."},{"key":"3546_CR6","doi-asserted-by":"publisher","DOI":"10.3390\/info13060284","author":"I Segura-Bedmar","year":"2022","unstructured":"Segura-Bedmar, I., Alonso-Bartolome, S.: Multimodal fake news detection. Information (2022). https:\/\/doi.org\/10.3390\/info13060284","journal-title":"Information"},{"issue":"4","key":"3546_CR7","doi-asserted-by":"publisher","first-page":"28","DOI":"10.1109\/MMUL.2020.3012675","volume":"27","author":"W Gao","year":"2020","unstructured":"Gao, W., Li, L., Zhu, X., Wang, Y.: Detecting disaster-related tweets via multimodal adversarial neural network. IEEE MultiMed. 27(4), 28\u201337 (2020). https:\/\/doi.org\/10.1109\/MMUL.2020.3012675","journal-title":"IEEE MultiMed."},{"key":"3546_CR8","doi-asserted-by":"publisher","DOI":"10.1016\/j.ijdrr.2020.101760","volume":"51","author":"H Hao","year":"2020","unstructured":"Hao, H., Wang, Y.: Leveraging multimodal social media data for rapid disaster damage assessment. Int. J. Disaster Risk Reduct. 51, 101760 (2020)","journal-title":"Int. J. Disaster Risk Reduct."},{"issue":"8","key":"3546_CR9","doi-asserted-by":"publisher","first-page":"8172","DOI":"10.1007\/s11227-021-03622-w","volume":"77","author":"RD Kumar","year":"2021","unstructured":"Kumar, R.D., Julie, E.G., Robinson, Y.H., Vimal, S., Seo, S.: Recognition of food type and calorie estimation using neural network. J. Supercomput. 77(8), 8172\u20138193 (2021)","journal-title":"J. Supercomput."},{"issue":"1","key":"3546_CR10","doi-asserted-by":"publisher","first-page":"e21926","DOI":"10.2196\/21926","volume":"9","author":"N Bahador","year":"2021","unstructured":"Bahador, N., Ferreira, D., Tamminen, S., Kortelainen, J., et al.: Deep learning-based multimodal data fusion: case study in food intake episodes detection using wearable sensors. JMIR mHealth uHealth 9(1), e21926 (2021)","journal-title":"JMIR mHealth uHealth"},{"key":"3546_CR11","doi-asserted-by":"publisher","first-page":"35370","DOI":"10.1109\/ACCESS.2019.2904519","volume":"7","author":"MA Subhi","year":"2019","unstructured":"Subhi, M.A., Ali, S.H., Mohammed, M.A.: Vision-based approaches for automatic food recognition and dietary assessment: a survey. IEEE Access 7, 35370\u201335381 (2019)","journal-title":"IEEE Access"},{"key":"3546_CR12","unstructured":"Yu, Q., Mao, D., Wang, J.: Deep learning based food recognition. Technical report, Stanford University (2016)"},{"key":"3546_CR13","doi-asserted-by":"crossref","unstructured":"Chen, J., Ngo, C.W.: Deep-based ingredient recognition for cooking recipe retrieval. In: Proceedings of the 24th ACM International Conference on Multimedia, pp. 32\u201341 (2016)","DOI":"10.1145\/2964284.2964315"},{"key":"3546_CR14","doi-asserted-by":"publisher","first-page":"238","DOI":"10.1109\/TAI.2021.3108126","volume":"3","author":"B Arslan","year":"2021","unstructured":"Arslan, B., Memis, S., Battinisonmez, E., Batur, O.Z.: Fine-grained food classification methods on the UEC food-100 database. IEEE Trans. Artif. Intell. 3, 238 (2021)","journal-title":"IEEE Trans. Artif. Intell."},{"key":"3546_CR15","doi-asserted-by":"publisher","first-page":"225","DOI":"10.1016\/j.gltp.2022.03.027","volume":"3","author":"G VijayaKumari","year":"2022","unstructured":"VijayaKumari, G., Vutkur, P., Vishwanath, P.: Food classification using transfer learning technique. Global Transit. Proc. 3, 225 (2022)","journal-title":"Global Transit. Proc."},{"issue":"5","key":"3546_CR16","doi-asserted-by":"publisher","first-page":"1100","DOI":"10.1109\/TMM.2016.2639382","volume":"19","author":"W Min","year":"2016","unstructured":"Min, W., Jiang, S., Sang, J., Wang, H., Liu, X., Herranz, L.: Being a supercook: joint food attributes and multimodal content modeling for recipe retrieval and exploration. IEEE Trans. Multimed. 19(5), 1100\u20131113 (2016)","journal-title":"IEEE Trans. Multimed."},{"issue":"6","key":"3546_CR17","doi-asserted-by":"publisher","first-page":"2480","DOI":"10.1109\/TCSVT.2020.3020079","volume":"31","author":"C Liu","year":"2020","unstructured":"Liu, C., Liang, Y., Xue, Y., Qian, X., Fu, J.: Food and ingredient joint learning for fine-grained recognition. IEEE Trans. Circuits Syst. Video Technol. 31(6), 2480\u20132493 (2020)","journal-title":"IEEE Trans. Circuits Syst. Video Technol."},{"key":"3546_CR18","unstructured":"Wang, X., Kumar, D., Thome, N., Cord, M., Precioso, F.: Recipe recognition with large multimodal food dataset. In: 2015 IEEE International Conference on Multimedia & Expo Workshops (ICMEW), pp. 1\u20136. IEEE (2015)"},{"key":"3546_CR19","doi-asserted-by":"crossref","unstructured":"Wiegand, M., Roth, B., Klakow, D.: Automatic food categorization from large unlabeled corpora and its impact on relation extraction. In: Proceedings of the 14th Conference of the European Chapter of the Association for Computational Linguistics, April 26\u201330, 2014, Gothenburg, Sweden, pp. 673\u2013682. Association for Computational Linguistics (2019)","DOI":"10.3115\/v1\/E14-1071"},{"key":"3546_CR20","doi-asserted-by":"crossref","unstructured":"Yang, S., Chen, M., Pomerleau, D., Sukthankar, R.: Food recognition using statistics of pairwise local features. In: 2010 IEEE Computer Society Conference on Computer Vision and Pattern Recognition, pp. 2249\u20132256. IEEE (2010)","DOI":"10.1109\/CVPR.2010.5539907"},{"key":"3546_CR21","doi-asserted-by":"crossref","unstructured":"Lasne, K.S., Nandrekar, S.S., Khan, A.A., Ghorpade, T.: Food reviews classification using multi-label convolutional neural network text classifier. In: ITM Web of Conferences, p. 01009. EDP Sciences (2021)","DOI":"10.1051\/itmconf\/20214001009"},{"key":"3546_CR22","doi-asserted-by":"crossref","unstructured":"Malekzadeh, M., Hajibabaee, P., Heidari, M., Zad, S., Uzuner, O., Jones, J.H., Review of graph neural network in text classification. In: IEEE 12th Annual Ubiquitous Computing, Electronics & Mobile Communication Conference (UEMCON), vol. 2021, pp. 0084\u20130091. IEEE (2021)","DOI":"10.1109\/UEMCON53757.2021.9666633"},{"issue":"21","key":"3546_CR23","doi-asserted-by":"publisher","first-page":"33011","DOI":"10.1007\/s11042-021-11329-6","volume":"80","author":"A Fakhrou","year":"2021","unstructured":"Fakhrou, A., Kunhoth, J., Al, Maadeed S.: Smartphone-based food recognition system using multiple deep CNN models. Multimed. Tools Appl. 80(21), 33011\u201333032 (2021)","journal-title":"Multimed. Tools Appl."},{"key":"3546_CR24","doi-asserted-by":"crossref","unstructured":"Gallo, I., Calefati, A., Nawaz, S., Image, Janjua, M.K.: encoded text fusion for multi-modal classification. In: Digital image computing: techniques and applications (DICTA), vol. 2018, pp. 1\u20137. IEEE (2018)","DOI":"10.1109\/DICTA.2018.8615789"},{"issue":"1","key":"3546_CR25","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1109\/LSENS.2018.2880790","volume":"3","author":"S Nawaz","year":"2018","unstructured":"Nawaz, S., Calefati, A., Janjua, M.K., Anwaar, M.U., Gallo, I.: Learning fused representations for large-scale multimodal classification. IEEE Sens. Lett. 3(1), 1\u20134 (2018)","journal-title":"IEEE Sens. Lett."},{"key":"3546_CR26","doi-asserted-by":"crossref","unstructured":"Kiela, D., Grave, E., Joulin, A., Mikolov, T.: Efficient large-scale multi-modal classification. In Proceedings of the AAAI Conference on Artificial Intelligence, vol.\u00a032 (2018)","DOI":"10.1609\/aaai.v32i1.11945"},{"key":"3546_CR27","unstructured":"Narayana, P., Pednekar, A., Krishnamoorthy, A., Sone, K., Basu, S., Huse: Hierarchical universal semantic embeddings (2019). arXiv preprint arXiv:1911.05978"},{"key":"3546_CR28","doi-asserted-by":"crossref","unstructured":"Abavisani, M., Patel, V.M.: Deep multimodal sparse representation-based classification. In: 2020 IEEE International Conference on Image Processing (ICIP), pp. 773\u2013777. IEEE (2020)","DOI":"10.1109\/ICIP40778.2020.9191317"},{"key":"3546_CR29","unstructured":"De\u00a0la Comble, A., Dutt, A., Montalvo, P., Salah, A.: Multi-modal attribute extraction for e-commerce (2022). arXiv preprint arXiv:2203.03441"},{"key":"3546_CR30","doi-asserted-by":"crossref","unstructured":"Liang, T., Lin, G., Wan, M., Li, T., Ma, G., Lv, F.: Expanding large pre-trained unimodal models with multimodal information injection for image-text multimodal classification. In: Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition, pp. 15492\u201315501 (2022)","DOI":"10.1109\/CVPR52688.2022.01505"},{"key":"3546_CR31","doi-asserted-by":"crossref","unstructured":"Ma, M., Ren, J., Zhao, L., Testuggine, D., Peng, X.: Are multimodal transformers robust to missing modality? In: Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition, pp. 18177\u201318186 (2022)","DOI":"10.1109\/CVPR52688.2022.01764"},{"key":"3546_CR32","unstructured":"Kiela, D., Bhooshan, S., Firooz, H., Perez, E., Testuggine, D.: Supervised multimodal bitransformers for classifying images and text (2019). arXiv preprint arXiv:1909.02950"},{"key":"3546_CR33","unstructured":"Liu, Y., Ott, M., Goyal, N., Du, J., Joshi, M., Chen, D., et\u00a0al.: Roberta: a robustly optimized bert pretraining approach (2019). arXiv preprint arXiv:1907.11692"},{"key":"3546_CR34","doi-asserted-by":"publisher","first-page":"154704","DOI":"10.1109\/ACCESS.2021.3128742","volume":"9","author":"P Rajapaksha","year":"2021","unstructured":"Rajapaksha, P., Farahbakhsh, R., Crespi, N.: BERT, XLNet or RoBERTa: the best transfer learning model to detect Clickbaits. IEEE Access 9, 154704\u2013154716 (2021)","journal-title":"IEEE Access"},{"key":"3546_CR35","unstructured":"Simonyan, K., Zisserman, A.: Very deep convolutional networks for large-scale image recognition (2014). arXiv preprint arXiv:1409.1556"},{"key":"3546_CR36","doi-asserted-by":"crossref","unstructured":"Thakkar, V., Tewary, S., Batch, Chakraborty C.: Normalization in convolutional neural networks-a comparative study with CIFAR-10 data. In: Fifth International Conference on Emerging Applications of Information Technology (EAIT), vol. 2018, pp. 1\u20135. IEEE (2018)","DOI":"10.1109\/EAIT.2018.8470438"},{"key":"3546_CR37","doi-asserted-by":"crossref","unstructured":"Josephine, V.H., Nirmala, A., Alluri, V.L.: Impact of hidden dense layers in convolutional neural network to enhance performance of classification model. In: IOP Conference Series: Materials Science and Engineering, vol. 1131, p. 012007. IOP Publishing (2021)","DOI":"10.1088\/1757-899X\/1131\/1\/012007"},{"issue":"2","key":"3546_CR38","doi-asserted-by":"publisher","first-page":"423","DOI":"10.1109\/TPAMI.2018.2798607","volume":"41","author":"T Baltru\u0161aitis","year":"2018","unstructured":"Baltru\u0161aitis, T., Ahuja, C., Morency, L.P.: Multimodal machine learning: a survey and taxonomy. IEEE Trans. Pattern Anal. Mach. Intell. 41(2), 423\u2013443 (2018)","journal-title":"IEEE Trans. Pattern Anal. Mach. Intell."},{"key":"3546_CR39","unstructured":"Bruni, E., Tran, G.B., Baroni, M.: Distributional semantics from text and images. In: Proceedings of the GEMS 2011 Workshop on Geometrical Models of Natural Language Semantics, pp. 22\u201332 (2011)"},{"issue":"2","key":"3546_CR40","doi-asserted-by":"publisher","first-page":"241","DOI":"10.1016\/S0893-6080(05)80023-1","volume":"5","author":"DH Wolpert","year":"1992","unstructured":"Wolpert, D.H.: Stacked generalization. Neural Netw. 5(2), 241\u2013259 (1992). https:\/\/doi.org\/10.1016\/S0893-6080(05)80023-1","journal-title":"Neural Netw."},{"issue":"1","key":"3546_CR41","doi-asserted-by":"publisher","first-page":"196","DOI":"10.1038\/srep00196","volume":"1","author":"YY Ahn","year":"2011","unstructured":"Ahn, Y.Y., Ahnert, S.E., Bagrow, J.P., Barab\u00e1si, A.L.: Flavor network and the principles of food pairing. Sci. Rep. 1(1), 196 (2011)","journal-title":"Sci. Rep."},{"key":"3546_CR42","doi-asserted-by":"crossref","unstructured":"Sajadmanesh, S., Jafarzadeh, S., Ossia, S.A., Rabiee, H.R., Haddadi, H., Mejova, Y., et\u00a0al.: Kissing cuisines: exploring worldwide culinary habits on the web. In: Proceedings of the 26th International Conference on World Wide Web Companion, pp. 1013\u20131021 (2017)","DOI":"10.1145\/3041021.3055137"}],"container-title":["The Visual Computer"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s00371-024-03546-5.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s00371-024-03546-5\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s00371-024-03546-5.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,3,3]],"date-time":"2025-03-03T11:30:25Z","timestamp":1741001425000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s00371-024-03546-5"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,7,3]]},"references-count":42,"journal-issue":{"issue":"4","published-print":{"date-parts":[[2025,3]]}},"alternative-id":["3546"],"URL":"https:\/\/doi.org\/10.1007\/s00371-024-03546-5","relation":{},"ISSN":["0178-2789","1432-2315"],"issn-type":[{"value":"0178-2789","type":"print"},{"value":"1432-2315","type":"electronic"}],"subject":[],"published":{"date-parts":[[2024,7,3]]},"assertion":[{"value":"26 May 2024","order":1,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"3 July 2024","order":2,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Declarations"}},{"value":"The authors have no relevant financial or non-financial interests to disclose. The authors have no conflict of interest to declare that are relevant to the content of this article. All authors certify that they have no affiliations with or involvement in any organization or entity with any financial interest or non-financial interest in the subject matter or materials discussed in this manuscript. The authors have no financial or proprietary interests in any material discussed in this article. We confirm that the manuscript has been read and approved by all named authors and that there are no other persons who satisfied the criteria for authorship but are not listed. We further confirm that the order of authors listed in the manuscript has been approved by all of us.","order":2,"name":"Ethics","group":{"name":"EthicsHeading","label":"Conflict of interest"}},{"value":"This article does not contain any studies with human or animal subjects. And I confirm that this work is original and has not been published nor under consideration elsewhere.","order":3,"name":"Ethics","group":{"name":"EthicsHeading","label":"Ethical approval"}},{"value":"Not applicable.","order":4,"name":"Ethics","group":{"name":"EthicsHeading","label":"Consent to participate"}},{"value":"Not applicable. (No third-party images or data were used in this paper. All the data or images used in the article were from the study conducted at the laboratory. There is no copyright infringement in figures or in any part of manuscript.)","order":5,"name":"Ethics","group":{"name":"EthicsHeading","label":"Consent for publication"}}]}}