{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,13]],"date-time":"2026-07-13T19:57:07Z","timestamp":1783972627329,"version":"3.55.0"},"reference-count":43,"publisher":"Springer Science and Business Media LLC","issue":"7","license":[{"start":{"date-parts":[[2022,5,5]],"date-time":"2022-05-05T00:00:00Z","timestamp":1651708800000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2022,5,5]],"date-time":"2022-05-05T00:00:00Z","timestamp":1651708800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"funder":[{"DOI":"10.13039\/501100003399","name":"Science and Technology Commission of Shanghai Municipality","doi-asserted-by":"publisher","award":["20DZ2254400"],"award-info":[{"award-number":["20DZ2254400"]}],"id":[{"id":"10.13039\/501100003399","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100003399","name":"Science and Technology Commission of Shanghai Municipality","doi-asserted-by":"publisher","award":["21DZ2200600"],"award-info":[{"award-number":["21DZ2200600"]}],"id":[{"id":"10.13039\/501100003399","id-type":"DOI","asserted-by":"publisher"}]},{"name":"National Scientific Foundation of China","award":["82170110"],"award-info":[{"award-number":["82170110"]}]},{"name":"Shanghai Pujiang Program","award":["20PJ1402400"],"award-info":[{"award-number":["20PJ1402400"]}]}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Vis Comput"],"published-print":{"date-parts":[[2023,7]]},"DOI":"10.1007\/s00371-022-02492-4","type":"journal-article","created":{"date-parts":[[2022,5,5]],"date-time":"2022-05-05T19:28:23Z","timestamp":1651778903000},"page":"2781-2793","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":144,"title":["A multimodal transformer to fuse images and metadata for skin disease classification"],"prefix":"10.1007","volume":"39","author":[{"given":"Gan","family":"Cai","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-1535-6520","authenticated-orcid":false,"given":"Yu","family":"Zhu","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Yue","family":"Wu","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Xiaoben","family":"Jiang","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Jiongyao","family":"Ye","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Dawei","family":"Yang","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2022,5,5]]},"reference":[{"key":"2492_CR1","doi-asserted-by":"crossref","unstructured":"He, K., Zhang, X., Ren, S., Sun, J.: Deep residual learning for image recognition. In: Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition, pp. 770\u2013778 (2016)","DOI":"10.1109\/CVPR.2016.90"},{"key":"2492_CR2","doi-asserted-by":"crossref","unstructured":"Huang, G., Liu, Z., Van Der Maaten, L., Weinberger, K.Q.: Densely connected convolutional networks. In: Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition, pp. 4700\u20134708 (2017)","DOI":"10.1109\/CVPR.2017.243"},{"key":"2492_CR3","doi-asserted-by":"crossref","unstructured":"Xiao, H., Ran, Z., Mabu, S., Li, Y., Li, L.: SAUNet++: an automatic segmentation model of COVID-19 lesion from CT slices. Vis. Comput. pp. 1\u201314 (2022)","DOI":"10.21203\/rs.3.rs-812932\/v1"},{"key":"2492_CR4","doi-asserted-by":"crossref","unstructured":"Mohamed, E.H., El-Behaidy, W.H.: Enhanced skin lesions classification using deep convolutional networks. In: 2019 Ninth International Conference on Intelligent Computing and Information Systems (ICICIS), IEEE, pp. 180\u2013188 (2019)","DOI":"10.1109\/ICICIS46948.2019.9014823"},{"key":"2492_CR5","doi-asserted-by":"crossref","unstructured":"Zhang, Y., Wang, C.: SIIM-ISIC melanoma classification with DenseNet. In: 2021 IEEE 2nd international conference on big data, artificial intelligence and internet of things engineering (ICBAIE), IEEE, pp. 14\u201317 (2021)","DOI":"10.1109\/ICBAIE52039.2021.9389983"},{"issue":"7","key":"2492_CR6","doi-asserted-by":"publisher","first-page":"1837","DOI":"10.1007\/s00371-020-01941-2","volume":"37","author":"K Karthik","year":"2021","unstructured":"Karthik, K., Kamath, S.S.: A deep neural network model for content-based medical image retrieval with multi-view classification. Vis. Comput. 37(7), 1837\u20131850 (2021)","journal-title":"Vis. Comput."},{"key":"2492_CR7","doi-asserted-by":"crossref","unstructured":"Wang, W., et al.: Pyramid vision transformer: a versatile backbone for dense prediction without convolutions. arXiv preprint arXiv:2102.12122 (2021)","DOI":"10.1109\/ICCV48922.2021.00061"},{"key":"2492_CR8","unstructured":"Yang, J., et al.: Focal self-attention for local-global interactions in vision transformers. arXiv preprint arXiv:2107.00641 (2021)"},{"key":"2492_CR9","doi-asserted-by":"crossref","unstructured":"Liu, Z., et al.: Swin transformer: Hierarchical vision transformer using shifted windows. arXiv preprint arXiv:2103.14030 (2021)","DOI":"10.1109\/ICCV48922.2021.00986"},{"key":"2492_CR10","unstructured":"Zhang, Z., Zhang, H., Zhao, L., Chen, T., Pfister, T.: Aggregating nested transformers. arXiv preprint arXiv:2105.12723 (2021)"},{"key":"2492_CR11","doi-asserted-by":"crossref","unstructured":"Chen, C-F., Fan, Q., Panda, R.: Crossvit: cross-attention multi-scale vision transformer for image classification. arXiv preprint arXiv:2103.14899 (2021)","DOI":"10.1109\/ICCV48922.2021.00041"},{"key":"2492_CR12","unstructured":"Dosovitskiy, A., et al.: An image is worth 16x16 words: transformers for image recognition at scale, arXiv preprint arXiv:2010.11929 (2020)"},{"key":"2492_CR13","unstructured":"Simonyan, K., Zisserman, A.: Very deep convolutional networks for large-scale image recognition. arXiv preprint arXiv:1409.1556 (2014)"},{"key":"2492_CR14","unstructured":"Tan, M., Le, Q.: Efficientnet: rethinking model scaling for convolutional neural networks. In: International Conference on Machine Learning, PMLR, pp. 6105\u20136114 (2019)"},{"key":"2492_CR15","unstructured":"Vaswani, A., et al.: Attention is all you need. In: Advances in neural information processing systems, pp. 5998\u20136008 (2017)"},{"issue":"1","key":"2492_CR16","doi-asserted-by":"publisher","first-page":"148","DOI":"10.1109\/TPAMI.2005.17","volume":"27","author":"SL Phung","year":"2005","unstructured":"Phung, S.L., Bouzerdoum, A., Chai, D.: Skin segmentation using color pixel classification: analysis and comparison. IEEE Trans. Pattern Anal. Mach. Intell. 27(1), 148\u2013154 (2005)","journal-title":"IEEE Trans. Pattern Anal. Mach. Intell."},{"key":"2492_CR17","doi-asserted-by":"publisher","first-page":"10","DOI":"10.1016\/j.media.2019.02.010","volume":"54","author":"J Zhang","year":"2019","unstructured":"Zhang, J., Xie, Y., Wu, Q., Xia, Y.: Medical image classification using synergic deep learning. Med. Image Anal. 54, 10\u201319 (2019)","journal-title":"Med. Image Anal."},{"key":"2492_CR18","doi-asserted-by":"crossref","unstructured":"Gao, X., Zhang, Y., Wang, H., Sun, Y., Zhao, F., Zhang, X.: A modified fuzzy clustering algorithm based on dynamic relatedness model for image segmentation. Vis. Comput. pp. 1\u201314 (2022)","DOI":"10.1007\/s00371-022-02430-4"},{"key":"2492_CR19","doi-asserted-by":"publisher","DOI":"10.1016\/j.compbiomed.2019.103423","volume":"113","author":"S Serte","year":"2019","unstructured":"Serte, S., Demirel, H.: Gabor wavelet-based deep learning for skin lesion classification. Comput. Biol. Med. 113, 103423 (2019)","journal-title":"Comput. Biol. Med."},{"key":"2492_CR20","doi-asserted-by":"crossref","unstructured":"Javed, R., Saba, T., Shafry, M., Rahim, M.: An intelligent saliency segmentation technique and classification of low contrast skin lesion dermoscopic images based on histogram decision. In: 2019 12th International Conference on Developments in eSystems Engineering (DeSE), IEEE, pp. 164\u2013169 (2019)","DOI":"10.1109\/DeSE.2019.00039"},{"key":"2492_CR21","first-page":"1","volume":"38","author":"KB Salah","year":"2021","unstructured":"Salah, K.B., Othmani, M., Kherallah, M.: A novel approach for human skin detection using convolutional neural network. Vis. Comput. 38, 1\u201311 (2021)","journal-title":"Vis. Comput."},{"key":"2492_CR22","doi-asserted-by":"crossref","unstructured":"Hao, Y., et al.: An end-to-end model for question answering over knowledge base with cross-attention combining global knowledge. In: Proceedings of the 55th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers), pp. 221\u2013231 (2017)","DOI":"10.18653\/v1\/P17-1021"},{"issue":"2","key":"2492_CR23","doi-asserted-by":"publisher","first-page":"547","DOI":"10.1109\/JBHI.2018.2806962","volume":"23","author":"I Gonzalez-Diaz","year":"2018","unstructured":"Gonzalez-Diaz, I.: Dermaknet: Incorporating the knowledge of dermatologists to convolutional neural networks for skin lesion diagnosis. IEEE J. Biomed. Health Inform. 23(2), 547\u2013559 (2018)","journal-title":"IEEE J. Biomed. Health Inform."},{"issue":"10","key":"2492_CR24","doi-asserted-by":"publisher","first-page":"2912","DOI":"10.1109\/JBHI.2020.2973614","volume":"24","author":"L Song","year":"2020","unstructured":"Song, L., Lin, J., Wang, Z.J., Wang, H.: An end-to-end multi-task deep learning framework for skin lesion analysis. IEEE J. Biomed. Health Inform. 24(10), 2912\u20132921 (2020)","journal-title":"IEEE J. Biomed. Health Inform."},{"issue":"10","key":"2492_CR25","doi-asserted-by":"publisher","first-page":"2870","DOI":"10.1109\/JBHI.2020.2977013","volume":"24","author":"P Tang","year":"2020","unstructured":"Tang, P., Liang, Q., Yan, X., Xiang, S., Zhang, D.: Gp-cnn-dtel: Global-part cnn model with data-transformed ensemble learning for skin lesion classification. IEEE J. Biomed. Health Inform. 24(10), 2870\u20132882 (2020)","journal-title":"IEEE J. Biomed. Health Inform."},{"issue":"2","key":"2492_CR26","doi-asserted-by":"publisher","first-page":"538","DOI":"10.1109\/JBHI.2018.2824327","volume":"23","author":"J Kawahara","year":"2018","unstructured":"Kawahara, J., Daneshvar, S., Argenziano, G., Hamarneh, G.: Seven-point checklist and skin lesion classification using multitask multimodal neural nets. IEEE J. Biomed. Health Inform. 23(2), 538\u2013546 (2018)","journal-title":"IEEE J. Biomed. Health Inform."},{"key":"2492_CR27","doi-asserted-by":"crossref","unstructured":"Pacheco, A.G.C., Krohling, R.: An attention-based mechanism to combine images and metadata in deep learning models applied to skin cancer classification. IEEE J. Biomed. Health Inform. (2021)","DOI":"10.1109\/JBHI.2021.3062002"},{"key":"2492_CR28","doi-asserted-by":"crossref","unstructured":"Zhou, L., Luo, Y.: Deep features fusion with mutual attention transformer for skin lesion diagnosis. In: Presented at the 2021 IEEE International Conference on Image Processing (ICIP) (2021)","DOI":"10.1109\/ICIP42928.2021.9506211"},{"key":"2492_CR29","doi-asserted-by":"publisher","DOI":"10.1016\/j.mex.2020.100864","volume":"7","author":"N Gessert","year":"2020","unstructured":"Gessert, N., Nielsen, M., Shaikh, M., Werner, R., Schlaefer, A.: Skin lesion classification using ensembles of multi-resolution EfficientNets with meta data. MethodsX 7, 100864 (2020)","journal-title":"MethodsX"},{"issue":"7","key":"2492_CR30","doi-asserted-by":"publisher","DOI":"10.2196\/20708","volume":"23","author":"J H\u00f6hn","year":"2021","unstructured":"H\u00f6hn, J., et al.: Integrating patient data into skin cancer classification using convolutional neural networks: systematic review. J. Med. Internet Res. 23(7), e20708 (2021)","journal-title":"J. Med. Internet Res."},{"key":"2492_CR31","doi-asserted-by":"publisher","first-page":"877","DOI":"10.2147\/JMDH.S306284","volume":"14","author":"DNA Ningrum","year":"2021","unstructured":"Ningrum, D.N.A., et al.: Deep learning classifier with patient\u2019s metadata of dermoscopic images in malignant melanoma detection. J. Multidiscip. Healthc. 14, 877 (2021)","journal-title":"J. Multidiscip. Healthc."},{"issue":"9","key":"2492_CR32","doi-asserted-by":"publisher","first-page":"3554","DOI":"10.1109\/JBHI.2021.3062002","volume":"25","author":"AG Pacheco","year":"2021","unstructured":"Pacheco, A.G., Krohling, R.A.: An attention-based mechanism to combine images and metadata in deep learning models applied to skin cancer classification. IEEE J. Biomed. Health Inform. 25(9), 3554\u20133563 (2021)","journal-title":"IEEE J. Biomed. Health Inform."},{"key":"2492_CR33","unstructured":"Kim, J.-H., On, K.-W., Lim, W., Kim, J., Ha, J.-W., Zhang, B.-T.: Hadamard product for low-rank bilinear pooling. arXiv preprint arXiv:1610.04325 (2016)"},{"key":"2492_CR34","unstructured":"Kim, J.-H., Jun, J., Zhang, B.-T.: Bilinear attention networks. arXiv preprint arXiv:1805.07932 (2018)"},{"key":"2492_CR35","unstructured":"Xiong, C., Merity, S., Socher, R.: Dynamic memory networks for visual and textual question answering. In: International Conference on Machine Learning, PMLR, pp. 2397\u20132406 (2016)"},{"key":"2492_CR36","doi-asserted-by":"crossref","unstructured":"Bose, R., Pande, S., Banerjee, B.: Two headed dragons: multimodal fusion and cross modal transactions. In: 2021 IEEE International Conference on Image Processing (ICIP), IEEE, pp. 2893\u20132897 (2021)","DOI":"10.1109\/ICIP42928.2021.9506341"},{"issue":"1","key":"2492_CR37","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1038\/sdata.2018.161","volume":"5","author":"P Tschandl","year":"2018","unstructured":"Tschandl, P., Rosendahl, C., Kittler, H.: The HAM10000 dataset, a large collection of multi-source dermatoscopic images of common pigmented skin lesions. Sci. Data 5(1), 1\u20139 (2018)","journal-title":"Sci. Data"},{"key":"2492_CR38","unstructured":"Codella, N., et al.: Skin lesion analysis toward melanoma detection 2018: a challenge hosted by the international skin imaging collaboration (isic). arXiv preprint arXiv:1902.03368 (2019)"},{"key":"2492_CR39","doi-asserted-by":"crossref","unstructured":"Yu, Z., Yu, J., Fan, J., Tao, D.: Multi-modal factorized bilinear pooling with co-attention learning for visual question answering. In: Proceedings of the IEEE international conference on computer vision, pp. 1821\u20131830 (2017)","DOI":"10.1109\/ICCV.2017.202"},{"key":"2492_CR40","doi-asserted-by":"crossref","unstructured":"Khan, M.A., Javed, M.Y., Sharif, M., Saba, T., Rehman, A.: Multi-model deep neural network based features extraction and optimal selection approach for skin lesion classification. In: 2019 international conference on computer and information sciences (ICCIS), IEEE, pp. 1\u20137 (2019)","DOI":"10.1109\/ICCISci.2019.8716400"},{"issue":"3","key":"2492_CR41","doi-asserted-by":"publisher","first-page":"310","DOI":"10.1111\/1346-8138.15683","volume":"48","author":"HW Huang","year":"2021","unstructured":"Huang, H.W., Hsu, B.W.Y., Lee, C.H., Tseng, V.S.: Development of a light-weight deep learning model for cloud applications and remote diagnosis of skin cancers. J. Dermatol. 48(3), 310\u2013316 (2021)","journal-title":"J. Dermatol."},{"issue":"11","key":"2492_CR42","doi-asserted-by":"publisher","first-page":"3429","DOI":"10.1109\/TMI.2020.2995518","volume":"39","author":"Q Liu","year":"2020","unstructured":"Liu, Q., Yu, L., Luo, L., Dou, Q., Heng, P.A.: Semi-supervised medical image classification with relation-driven self-ensembling model. IEEE Trans. Med. Imaging 39(11), 3429\u20133440 (2020)","journal-title":"IEEE Trans. Med. Imaging"},{"issue":"5","key":"2492_CR43","doi-asserted-by":"publisher","first-page":"1379","DOI":"10.1109\/JBHI.2019.2942429","volume":"24","author":"Y Gu","year":"2019","unstructured":"Gu, Y., Ge, Z., Bonnington, C.P., Zhou, J.: Progressive transfer learning and adversarial domain adaptation for cross-domain skin disease classification. IEEE J. Biomed. Health Inform. 24(5), 1379\u20131393 (2019)","journal-title":"IEEE J. Biomed. Health Inform."}],"container-title":["The Visual Computer"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s00371-022-02492-4.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s00371-022-02492-4\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s00371-022-02492-4.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,4,8]],"date-time":"2025-04-08T16:28:26Z","timestamp":1744129706000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s00371-022-02492-4"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2022,5,5]]},"references-count":43,"journal-issue":{"issue":"7","published-print":{"date-parts":[[2023,7]]}},"alternative-id":["2492"],"URL":"https:\/\/doi.org\/10.1007\/s00371-022-02492-4","relation":{},"ISSN":["0178-2789","1432-2315"],"issn-type":[{"value":"0178-2789","type":"print"},{"value":"1432-2315","type":"electronic"}],"subject":[],"published":{"date-parts":[[2022,5,5]]},"assertion":[{"value":"4 April 2022","order":1,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"5 May 2022","order":2,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}}]}}