{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,1,31]],"date-time":"2026-01-31T02:15:24Z","timestamp":1769825724308,"version":"3.49.0"},"reference-count":30,"publisher":"Springer Science and Business Media LLC","issue":"30","license":[{"start":{"date-parts":[[2024,2,13]],"date-time":"2024-02-13T00:00:00Z","timestamp":1707782400000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2024,2,13]],"date-time":"2024-02-13T00:00:00Z","timestamp":1707782400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Multimed Tools Appl"],"DOI":"10.1007\/s11042-024-18362-1","type":"journal-article","created":{"date-parts":[[2024,2,13]],"date-time":"2024-02-13T08:01:55Z","timestamp":1707811315000},"page":"73911-73921","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":4,"title":["Enhancing masked facial expression recognition with multimodal deep learning"],"prefix":"10.1007","volume":"83","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-2452-6571","authenticated-orcid":false,"given":"H.M","family":"Shahzad","sequence":"first","affiliation":[]},{"given":"Sohail\u00a0Masood","family":"Bhatti","sequence":"additional","affiliation":[]},{"given":"Arfan","family":"Jaffar","sequence":"additional","affiliation":[]},{"given":"Sheeraz","family":"Akram","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2024,2,13]]},"reference":[{"key":"18362_CR1","doi-asserted-by":"publisher","first-page":"57","DOI":"10.1016\/j.ijcce.2021.02.002","volume":"2","author":"B Li","year":"2021","unstructured":"Li B, Lima D (2021) Facial expression recognition via resnet-50. International Journal of Cognitive Computing in Engineering 2:57\u201364. https:\/\/doi.org\/10.1016\/j.ijcce.2021.02.002","journal-title":"International Journal of Cognitive Computing in Engineering"},{"key":"18362_CR2","doi-asserted-by":"publisher","unstructured":"Yildirim E, Akbulut FP, Catal C (2023) Analysis of facial emotion expression in eating occasions using deep learning. Multimedia Tools and Applications 1\u201313. https:\/\/doi.org\/10.1007\/s11042-023-15008-6","DOI":"10.1007\/s11042-023-15008-6"},{"key":"18362_CR3","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1038\/s41598-021-84806-5","volume":"11","author":"M Marini","year":"2021","unstructured":"Marini M, Ansani A, Paglieri F, Caruana F, Viola M (2021) The impact of facemasks on emotion recognition, trust attribution and re-identification. Sci Rep 11:1\u201314. https:\/\/doi.org\/10.1038\/s41598-021-84806-5","journal-title":"Sci Rep"},{"key":"18362_CR4","doi-asserted-by":"publisher","unstructured":"Kong Y, Ren Z, Zhang K, Zhang S, Ni Q, Han J (2021) Lightweight facial expression recognition method based on attention mechanism and key region fusion. J Electron Imaging 30:063002\u2013063002. https:\/\/doi.org\/10.1117\/1.JEI.30.6.063002","DOI":"10.1117\/1.JEI.30.6.063002"},{"issue":"4","key":"18362_CR5","doi-asserted-by":"publisher","first-page":"0249792","DOI":"10.1371\/journal.pone.0249792","volume":"16","author":"F Grundmann","year":"2021","unstructured":"Grundmann F, Epstude K, Scheibe S (2021) Face masks reduce emotion recognition accuracy and perceived closeness. PLoS ONE 16(4):0249792. https:\/\/doi.org\/10.1371\/journal.pone.0249792","journal-title":"PLoS ONE"},{"key":"18362_CR6","doi-asserted-by":"publisher","first-page":"0257740","DOI":"10.1371\/journal.pone.0257740","volume":"16","author":"F Pazhoohi","year":"2021","unstructured":"Pazhoohi F, Forby L, Kingstone A (2021) Facial masks affect emotion recognition in the general population and individuals with autistic traits. PLoS ONE 16:0257740. https:\/\/doi.org\/10.1371\/journal.pone.0257740","journal-title":"PLoS ONE"},{"key":"18362_CR7","doi-asserted-by":"publisher","unstructured":"Puri T, Soni M, Dhiman G, Ibrahim Khalaf O, Raza Khan I et al (2022) Detection of emotion of speech for ravdess audio using hybrid convolution neural network. J Healthcare Eng 2022. https:\/\/doi.org\/10.1155\/2022\/8472947","DOI":"10.1155\/2022\/8472947"},{"key":"18362_CR8","doi-asserted-by":"publisher","unstructured":"Tawhid MNA, Siuly S, Wang H, Whittaker F, Wang K, Zhang Y (2021) A spectrogram image based intelligent technique for automatic detection of autism spectrum disorder from eeg. PLoS ONE 16:0253094. https:\/\/doi.org\/10.1371\/journal.pone.0253094","DOI":"10.1371\/journal.pone.0253094"},{"key":"18362_CR9","doi-asserted-by":"publisher","unstructured":"Franzoni V, Biondi G, Milani A (2020) Emotional sounds of crowds: spectrogram-based analysis using deep learning. Multimedia tools and applications 79:36063\u201336075. https:\/\/doi.org\/10.1007\/s11042-020-09428-x","DOI":"10.1007\/s11042-020-09428-x"},{"key":"18362_CR10","doi-asserted-by":"publisher","first-page":"0262840","DOI":"10.1371\/journal.pone.0262840","volume":"17","author":"M Grahlow","year":"2022","unstructured":"Grahlow M, Rupp CI, Derntl B (2022) The impact of face masks on emotion recognition performance and perception of threat. PLoS ONE 17:0262840. https:\/\/doi.org\/10.1371\/journal.pone.0262840","journal-title":"PLoS ONE"},{"key":"18362_CR11","doi-asserted-by":"publisher","first-page":"0249792","DOI":"10.1371\/journal.pone.0249792","volume":"16","author":"F Grundmann","year":"2021","unstructured":"Grundmann F, Epstude K, Scheibe S (2021) Face masks reduce emotion recognition accuracy and perceived closeness. PLoS ONE 16:0249792. https:\/\/doi.org\/10.1371\/journal.pone.0249792","journal-title":"PLoS ONE"},{"key":"18362_CR12","doi-asserted-by":"publisher","first-page":"16839","DOI":"10.1109\/JSEN.2021.3077029","volume":"21","author":"S Vachmanus","year":"2021","unstructured":"Vachmanus S, Ravankar AA, Emaru T, Kobayashi Y (2021) Multi-modal sensor fusion-based semantic segmentation for snow driving scenarios. IEEE Sens J 21:16839\u201316851. https:\/\/doi.org\/10.1109\/JSEN.2021.3077029","journal-title":"IEEE Sens J"},{"key":"18362_CR13","doi-asserted-by":"publisher","first-page":"39","DOI":"10.1007\/s10462-018-9633-3","volume":"52","author":"Q Abbas","year":"2019","unstructured":"Abbas Q, Ibrahim ME, Jaffar MA (2019) A comprehensive review of recent advances on deep vision systems. Artif Intell Rev 52:39\u201376. https:\/\/doi.org\/10.1007\/s10462-018-9633-3","journal-title":"Artif Intell Rev"},{"key":"18362_CR14","doi-asserted-by":"publisher","unstructured":"Sun W, Chen X, Zhang X, Dai G, Chang P, He X (2021) A multi-feature learning model with enhanced local attention for vehicle re-identification. CMC-Computers Materials & Continua 69(3):3549\u20133561. https:\/\/doi.org\/10.32604\/cmc.2021.021627","DOI":"10.32604\/cmc.2021.021627"},{"key":"18362_CR15","doi-asserted-by":"publisher","first-page":"35","DOI":"10.1007\/s00138-017-0870-2","volume":"29","author":"AS Al-Waisy","year":"2018","unstructured":"Al-Waisy AS, Qahwaji R, Ipson S, Al-Fahdawi S (2018) A multimodal deep learning framework using local feature representations for face recognition. Mach Vis Appl 29:35\u201354. https:\/\/doi.org\/10.1007\/s00138-017-0870-2","journal-title":"Mach Vis Appl"},{"key":"18362_CR16","doi-asserted-by":"publisher","first-page":"17","DOI":"10.1007\/s12193-019-00308-9","volume":"14","author":"W Wei","year":"2020","unstructured":"Wei W, Jia Q, Feng Y, Chen G, Chu M (2020) Multi-modal facial expression feature based on deep-neural networks. Journal on Multimodal User Interfaces 14:17\u201323. https:\/\/doi.org\/10.1007\/s12193-019-00308-9","journal-title":"Journal on Multimodal User Interfaces"},{"key":"18362_CR17","doi-asserted-by":"publisher","unstructured":"Hamester D, Barros P, Wermter S (2015) Face expression recognition with a 2-channel convolutional neural network. In: 2015 International Joint Conference on Neural Networks (IJCNN), pp 1\u20138. https:\/\/doi.org\/10.1109\/IJCNN.2015.7280539","DOI":"10.1109\/IJCNN.2015.7280539"},{"key":"18362_CR18","doi-asserted-by":"publisher","DOI":"10.1016\/j.eswa.2022.118523","volume":"211","author":"N Jaafar","year":"2023","unstructured":"Jaafar N, Lachiri Z (2023) Multimodal fusion methods with deep neural networks and meta-information for aggression detection in surveillance. Expert Syst Appl 211:118523. https:\/\/doi.org\/10.1016\/j.eswa.2022.118523","journal-title":"Expert Syst Appl"},{"key":"18362_CR19","doi-asserted-by":"publisher","DOI":"10.1016\/j.eswa.2022.118523","volume":"196","author":"K Wang","year":"2022","unstructured":"Wang K, Song Y, Huang Z, Sun Y, Xu J, Zhang S (2022) Additive manufacturing energy consumption measurement and prediction in fabricating lattice structure based on recallable multimodal fusion network. Measurement 196:111215. https:\/\/doi.org\/10.1016\/j.eswa.2022.118523","journal-title":"Measurement"},{"key":"18362_CR20","doi-asserted-by":"publisher","first-page":"51258","DOI":"10.1109\/ACCESS.2021.3069770","volume":"9","author":"SA Kashinath","year":"2021","unstructured":"Kashinath SA, Mostafa SA, Mustapha A, Mahdin H, Lim D, Mahmoud MA, Mohammed MA, Al-Rimy BAS, Fudzee MFM, Yang TJ (2021) Review of data fusion methods for real-time and multi-sensor traffic flow analysis. IEEE Access 9:51258\u201351276. https:\/\/doi.org\/10.1109\/ACCESS.2021.3069770","journal-title":"IEEE Access"},{"key":"18362_CR21","doi-asserted-by":"publisher","DOI":"10.1016\/j.bdr.2021.100287","volume":"27","author":"A Gumaei","year":"2022","unstructured":"Gumaei A, Ismail WN, Hassan MR, Hassan MM, Mohamed E, Alelaiwi A, Fortino G (2022) A decision-level fusion method for covid-19 patient health prediction. Big Data Research 27:100287. https:\/\/doi.org\/10.1016\/j.bdr.2021.100287","journal-title":"Big Data Research"},{"key":"18362_CR22","doi-asserted-by":"crossref","unstructured":"Yang B, Wu J, Hattori G (2020) Facial expression recognition with the advent of human beings all behind face masks. (2020). Paper presented at the proceedings of the 19th international conference on mobile and ubiquitous multimedia, November, Essen, Germany","DOI":"10.1145\/3428361.3432075"},{"key":"18362_CR23","doi-asserted-by":"publisher","unstructured":"Cao H, Cooper DG, Keutmann MK, Gur RC, Nenkova A, Verma R (2014) Crema-d: crowd-sourced emotional multimodal actors dataset. IEEE Trans Affect Comput 5:377\u2013390. https:\/\/doi.org\/10.1109\/TAFFC.2014.2336244","DOI":"10.1109\/TAFFC.2014.2336244"},{"key":"18362_CR24","doi-asserted-by":"crossref","unstructured":"Pappagari R, Wang T, Villalba J, Chen N, Dehak N (2020) x-vectors meet emotions: a study on dependencies between emotion and speaker recognition (2020) Paper presented at the ICASSP 2020-2020 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)","DOI":"10.1109\/ICASSP40776.2020.9054317"},{"key":"18362_CR25","doi-asserted-by":"publisher","first-page":"26061","DOI":"10.1007\/s11042-022-12803-5","volume":"81","author":"SA Gebereselassie","year":"2022","unstructured":"Gebereselassie SA, Roy BK (2022) Secure speech communication based on the combination of chaotic oscillator and logistic map. Multimedia Tools and Applications 81:26061\u201326079. https:\/\/doi.org\/10.1007\/s11042-022-12803-5","journal-title":"Multimedia Tools and Applications"},{"key":"18362_CR26","doi-asserted-by":"crossref","unstructured":"Zheng Y, Sarigul E, Panicker G, Stott D (2022) Vineyard lai and canopy coverage estimation with convolutional neural network models and drone pictures. Paper presented at the Sensing for Agriculture and Food Quality and Safety XIV","DOI":"10.1117\/12.2620100"},{"issue":"5","key":"18362_CR27","doi-asserted-by":"publisher","first-page":"3099","DOI":"10.3390\/su14053099","volume":"14","author":"F Liu","year":"2022","unstructured":"Liu F, Xu H, Qi M, Liu D, Wang J, Kong J (2022) Depth-wise separable convolution attention module for garbage image classification. Sustainability 14(5):3099. https:\/\/doi.org\/10.3390\/su14053099","journal-title":"Sustainability"},{"key":"18362_CR28","doi-asserted-by":"publisher","first-page":"62830","DOI":"10.1109\/ACCESS.2020.2983774","volume":"8","author":"L Qian","year":"2020","unstructured":"Qian L, Hu L, Zhao L, Wang T, Jiang R (2020) Sequence-dropout block for reducing overfitting problem in image classification. IEEE Access 8:62830\u201362840. https:\/\/doi.org\/10.1109\/ACCESS.2020.2983774","journal-title":"IEEE Access"},{"key":"18362_CR29","doi-asserted-by":"publisher","unstructured":"Chen L, Li M, Lai X, Hirota K, Pedrycz W (2020) Cnn-based broad learning with efficient incremental reconstruction model for facial emotion recognition. IFAC-PapersOnLine 53(2):10236\u201310241. https:\/\/doi.org\/10.1016\/j.ifacol.2020.12.2754","DOI":"10.1016\/j.ifacol.2020.12.2754"},{"key":"18362_CR30","doi-asserted-by":"publisher","unstructured":"Shahzad H, Bhatti SM, Jaffar A, Rashid M (2023) A multi-modal deep learning approach for emotion recognition. Intelligent Automation & Soft Computing 36. https:\/\/doi.org\/10.32604\/iasc.2023.032525","DOI":"10.32604\/iasc.2023.032525"}],"container-title":["Multimedia Tools and Applications"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s11042-024-18362-1.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s11042-024-18362-1\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s11042-024-18362-1.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,9,3]],"date-time":"2024-09-03T02:09:42Z","timestamp":1725329382000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s11042-024-18362-1"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,2,13]]},"references-count":30,"journal-issue":{"issue":"30","published-online":{"date-parts":[[2024,9]]}},"alternative-id":["18362"],"URL":"https:\/\/doi.org\/10.1007\/s11042-024-18362-1","relation":{},"ISSN":["1573-7721"],"issn-type":[{"value":"1573-7721","type":"electronic"}],"subject":[],"published":{"date-parts":[[2024,2,13]]},"assertion":[{"value":"20 May 2023","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"28 September 2023","order":2,"name":"revised","label":"Revised","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"19 January 2024","order":3,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"13 February 2024","order":4,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Declarations"}},{"value":"The authors declare that they have no conflicts of interest to report regarding the present study.","order":2,"name":"Ethics","group":{"name":"EthicsHeading","label":"Conflict of interest\/Competing"}}]}}