{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,26]],"date-time":"2026-10-26T12:01:13Z","timestamp":1793016073975,"version":"4.5.0"},"reference-count":56,"publisher":"Elsevier BV","license":[{"start":{"date-parts":[[2025,4,1]],"date-time":"2025-04-01T00:00:00Z","timestamp":1743465600000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.elsevier.com\/tdm\/userlicense\/1.0\/"},{"start":{"date-parts":[[2025,4,1]],"date-time":"2025-04-01T00:00:00Z","timestamp":1743465600000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.elsevier.com\/legal\/tdmrep-license"},{"start":{"date-parts":[[2024,12,5]],"date-time":"2024-12-05T00:00:00Z","timestamp":1733356800000},"content-version":"vor","delay-in-days":0,"URL":"http:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"funder":[{"DOI":"10.13039\/100000138","name":"U.S. Department of Education","doi-asserted-by":"publisher","award":["H327S210005"],"award-info":[{"award-number":["H327S210005"]}],"id":[{"id":"10.13039\/100000138","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000138","name":"U.S. Department of Education","doi-asserted-by":"publisher","award":["H327S200009"],"award-info":[{"award-number":["H327S200009"]}],"id":[{"id":"10.13039\/100000138","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100007900","name":"University of Central Florida","doi-asserted-by":"publisher","id":[{"id":"10.13039\/100007900","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000001","name":"National Science Foundation","doi-asserted-by":"publisher","award":["2114808"],"award-info":[{"award-number":["2114808"]}],"id":[{"id":"10.13039\/100000001","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["elsevier.com","sciencedirect.com"],"crossmark-restriction":true},"short-container-title":["Biomedical Signal Processing and Control"],"published-print":{"date-parts":[[2025,4]]},"DOI":"10.1016\/j.bspc.2024.107232","type":"journal-article","created":{"date-parts":[[2024,12,5]],"date-time":"2024-12-05T08:48:12Z","timestamp":1733388492000},"page":"107232","update-policy":"https:\/\/doi.org\/10.1016\/elsevier_cm_policy","source":"Crossref","is-referenced-by-count":11,"special_numbering":"C","title":["A Unified Biosensor\u2013Vision Multi-Modal Transformer network for emotion recognition"],"prefix":"10.1016","volume":"102","author":[{"ORCID":"https:\/\/orcid.org\/0009-0006-2312-0138","authenticated-orcid":false,"given":"Kamran","family":"Ali","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Charles E.","family":"Hughes","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"78","reference":[{"issue":"4","key":"10.1016\/j.bspc.2024.107232_b1","article-title":"Quadcopter control in three-dimensional space using a noninvasive motor imagery-based brain\u2013computer interface","volume":"10","author":"Karl","year":"2013","journal-title":"J. Neural Eng."},{"issue":"6","key":"10.1016\/j.bspc.2024.107232_b2","doi-asserted-by":"crossref","first-page":"1026","DOI":"10.1109\/TBME.2004.827086","article-title":"Noninvasive brain-actuated control of a mobile robot by human EEG","volume":"51","author":"Jd","year":"2004","journal-title":"IEEE Trans. Biomed. Eng."},{"issue":"4","key":"10.1016\/j.bspc.2024.107232_b3","doi-asserted-by":"crossref","first-page":"715","DOI":"10.1016\/j.intcom.2004.06.009","article-title":"Affective computing in the era of contemporary neurophysiology and health informatics","volume":"16","author":"Bamidis","year":"2004","journal-title":"Interact. Comput."},{"key":"10.1016\/j.bspc.2024.107232_b4","doi-asserted-by":"crossref","unstructured":"S. Coyle, Y. Wu, K.-T. Lau, S. Brady, G. Wallace, D. Diamond, Bio-sensing textiles-wearable chemical biosensors for health monitoring, in: In 4th International Workshop on Wearable and Implantable Body Sensor Networks, 2007, pp. 35\u201339.","DOI":"10.1007\/978-3-540-70994-7_6"},{"issue":"1","key":"10.1016\/j.bspc.2024.107232_b5","doi-asserted-by":"crossref","first-page":"45","DOI":"10.1089\/cpb.2006.9993","article-title":"Affective interactions using virtual reality: the link between presence and emotions","volume":"10","author":"Riva","year":"2007","journal-title":"Cyberpsychology Behav."},{"key":"10.1016\/j.bspc.2024.107232_b6","doi-asserted-by":"crossref","unstructured":"A. Kamran, C.E. Hughes., Facial expression recognition by using a disentangled identity-invariant expression representation, in: In 2020 25th International Conference on Pattern Recognition, ICPR, 2021, pp. 9460\u20139467.","DOI":"10.1109\/ICPR48806.2021.9412172"},{"issue":"1","key":"10.1016\/j.bspc.2024.107232_b7","first-page":"96","article-title":"Utilizing deep learning towards multi-modal bio-sensing and vision-based affective computing","volume":"13","author":"Tzyy-Ping","year":"2019","journal-title":"IEEE Trans. Affect. Comput."},{"issue":"1","key":"10.1016\/j.bspc.2024.107232_b8","first-page":"26756","article-title":"Ad-corre: Adaptive correlation-based loss for facial expression recognition in the wild.","volume":"10","author":"Pourramezan","year":"2022","journal-title":"IEEE Access"},{"issue":"3","key":"10.1016\/j.bspc.2024.107232_b9","first-page":"595","article-title":"Exploiting multi-cnn features in cnn-rnn based dimensional emotion recognition on the omg in-the-wild dataset.","volume":"12","author":"Dimitrios","year":"2020","journal-title":"IEEE Trans. Affect. Comput."},{"issue":"1","key":"10.1016\/j.bspc.2024.107232_b10","first-page":"18","article-title":"Affectnet: A database for facial expression, valence, and arousal computing in the wild.","volume":"10","author":"Ali","year":"2017","journal-title":"IEEE Trans. Affect. Comput."},{"issue":"2","key":"10.1016\/j.bspc.2024.107232_b11","doi-asserted-by":"crossref","first-page":"66","DOI":"10.1080\/2326263X.2014.912881","article-title":"A survey of affective brain computer interfaces: principles, state-of-the-art, and challenges","volume":"1","author":"M\u00fchl","year":"2014","journal-title":"Brain-Comput. Interfaces"},{"issue":"3","key":"10.1016\/j.bspc.2024.107232_b12","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1145\/2682899","article-title":"A review and meta-analysis of multimodal affect detection systems","volume":"47","author":"D\u2019mello","year":"2015","journal-title":"ACM Comput. Surv. (CSUR)"},{"issue":"5","key":"10.1016\/j.bspc.2024.107232_b13","first-page":"505","article-title":"Modeling physiological data with deep belief networks","volume":"3","author":"Dan","year":"2013","journal-title":"Int. J. Inf. Educ. Technol."},{"key":"10.1016\/j.bspc.2024.107232_b14","series-title":"AMIGOS: A dataset for Mood, personality and affect research on Individuals and GrOupS","author":"Correa","year":"2017"},{"issue":"2","key":"10.1016\/j.bspc.2024.107232_b15","doi-asserted-by":"crossref","first-page":"164","DOI":"10.1016\/j.imavis.2012.10.002","article-title":"Fusion of facial expressions and EEG for implicit affective tagging","volume":"31","author":"Sander","year":"2013","journal-title":"Image Vis. Comput."},{"key":"10.1016\/j.bspc.2024.107232_b16","series-title":"2016 IEEE Conference on Computational Intelligence in Bioinformatics and Computational Biology","first-page":"1","article-title":"Comparing features from ECG pattern and HRV analysis for emotion recognition system","author":"Ferdinando","year":"2016"},{"issue":"14","key":"10.1016\/j.bspc.2024.107232_b17","first-page":"50","article-title":"Emotion recognition by heart rate variability","volume":"8","author":"Hany","year":"2014","journal-title":"Aust. J. Basic Appl. Sci."},{"key":"10.1016\/j.bspc.2024.107232_b18","series-title":"Advances in Neural Information Processing Systems","article-title":"Attention is all you need","author":"Ashish","year":"2017"},{"key":"10.1016\/j.bspc.2024.107232_b19","series-title":"Advances in Neural Information Processing Systems","first-page":"19400","article-title":"Multi-task temporal shift attention networks for on-device contactless vitals measurement","author":"Xin","year":"2020"},{"key":"10.1016\/j.bspc.2024.107232_b20","series-title":"Annual Meeting of the Association for Computational Linguistics","first-page":"2236","article-title":"Multimodal language analysis in the wild: Cmu-mosei dataset and interpretable dynamic fusion graph","author":"Bagher","year":"2018"},{"key":"10.1016\/j.bspc.2024.107232_b21","series-title":"Advances in Neural Information Processing Systems","first-page":"9617","article-title":"TVLT: Textless vision-language transformer","author":"Zineng","year":"2022"},{"issue":"3","key":"10.1016\/j.bspc.2024.107232_b22","doi-asserted-by":"crossref","first-page":"54","DOI":"10.1109\/MCG.2023.3260599","article-title":"WAVE: a web-based platform for delivering knowledge-driven virtual experiences","volume":"43","author":"Shah","year":"2023","journal-title":"IEEE Comput. Graph. Appl."},{"issue":"1","key":"10.1016\/j.bspc.2024.107232_b23","first-page":"42","article-title":"A multimodal database for affect recognition and implicit tagging","volume":"3","author":"Mohammad","year":"2011","journal-title":"IEEE Trans. Affect. Comput."},{"issue":"1","key":"10.1016\/j.bspc.2024.107232_b24","first-page":"18","article-title":"Deap: A database for emotion analysis; using physiological signals","volume":"3","author":"Sander","year":"2011","journal-title":"IEEE Trans. Affect. Comput."},{"issue":"6","key":"10.1016\/j.bspc.2024.107232_b25","doi-asserted-by":"crossref","first-page":"3539","DOI":"10.1016\/j.jksuci.2022.04.012","article-title":"A comparison of emotion recognition system using electrocardiogram (ECG) and photoplethysmogram (PPG)","volume":"34","author":"Ismail","year":"2022","journal-title":"J. King Saud Univ.-Comput. Inform. Sci."},{"issue":"16","key":"10.1016\/j.bspc.2024.107232_b26","doi-asserted-by":"crossref","first-page":"3355","DOI":"10.3390\/app9163355","article-title":"Fast emotion recognition based on single pulse PPG signal with convolutional neural network","volume":"9","author":"Lee","year":"2019","journal-title":"Appl. Sci."},{"issue":"11","key":"10.1016\/j.bspc.2024.107232_b27","doi-asserted-by":"crossref","first-page":"4945","DOI":"10.3390\/app11114945","article-title":"Emotion recognition from ECG signals using wavelet scattering and machine learning","volume":"11","author":"Sep\u00falveda","year":"2021","journal-title":"Appl. Sci."},{"issue":"3","key":"10.1016\/j.bspc.2024.107232_b28","first-page":"1541","article-title":"Self-supervised ECG representation learning for emotion recognition","volume":"13","author":"P.","year":"2020","journal-title":"IEEE Trans. Affect. Comput."},{"key":"10.1016\/j.bspc.2024.107232_b29","doi-asserted-by":"crossref","DOI":"10.1016\/j.bspc.2023.104907","article-title":"CNN-LSTM for automatic emotion recognition using contactless photoplythesmographic signals","volume":"85","author":"Mellouk","year":"2023","journal-title":"Biomed. Signal Process. Control"},{"key":"10.1016\/j.bspc.2024.107232_b30","series-title":"2020 25th International Conference on Pattern Recognition","first-page":"9460","article-title":"Facial expression recognition by using a disentangled identity-invariant expression representation","author":"Ali","year":"2021"},{"key":"10.1016\/j.bspc.2024.107232_b31","doi-asserted-by":"crossref","unstructured":"H. Li, N. Wang, X. Yang, X. Wang, X. Gao, Towards semi-supervised deep facial expression recognition with an adaptive confidence margin, in: Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition, 2022, pp. 4166\u20134175.","DOI":"10.1109\/CVPR52688.2022.00413"},{"key":"10.1016\/j.bspc.2024.107232_b32","doi-asserted-by":"crossref","unstructured":"H. Wang, B. Li, S. Wu, S. Shen, F. Liu, S. Ding, A. Zhou, Rethinking the Learning Paradigm for Dynamic Facial Expression Recognition, in: Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition, 2023, pp. 17958\u201317968.","DOI":"10.1109\/CVPR52729.2023.01722"},{"key":"10.1016\/j.bspc.2024.107232_b33","article-title":"Modeling uncertainty for low-resolution facial expression recognition","author":"Lo","year":"2023","journal-title":"IEEE Trans. Affect. Comput."},{"key":"10.1016\/j.bspc.2024.107232_b34","series-title":"Advances in Visual Computing: 15th International Symposium, ISVC 2020, San Diego, CA, USA, October 5\u20137, 2020, Proceedings, Part I 15","first-page":"501","article-title":"Face reenactment based facial expression recognition","author":"Ali","year":"2020"},{"key":"10.1016\/j.bspc.2024.107232_b35","series-title":"A multi-component CNN-RNN approach for dimensional emotion recognition in-the-wild","author":"Kollias","year":"2018"},{"key":"10.1016\/j.bspc.2024.107232_b36","series-title":"2020 15th IEEE International Conference on Automatic Face and Gesture Recognition (FG 2020)","first-page":"592","article-title":"Multitask emotion recognition with incomplete labels","author":"Deng","year":"2020"},{"key":"10.1016\/j.bspc.2024.107232_b37","series-title":"2020 15th IEEE International Conference on Automatic Face and Gesture Recognition (FG 2020)","first-page":"600","article-title":"Two-stream aural-visual affect analysis in the wild","author":"Kuhnke","year":"2020"},{"key":"10.1016\/j.bspc.2024.107232_b38","series-title":"2020 15th IEEE International Conference on Automatic Face and Gesture Recognition (FG 2020)","first-page":"632","article-title":"M 3 f: Multi-modal continuous valence-arousal estimation in the wild","author":"Zhang","year":"2020"},{"key":"10.1016\/j.bspc.2024.107232_b39","series-title":"Emotion recognition based on multimodal information","first-page":"241","author":"Zeng","year":"2009"},{"key":"10.1016\/j.bspc.2024.107232_b40","doi-asserted-by":"crossref","first-page":"93","DOI":"10.1016\/j.cmpb.2016.12.005","article-title":"Recognition of emotions using multimodal physiological signals and an ensemble deep learning model","volume":"140","author":"Yin","year":"2017","journal-title":"Comput. Methods Programs Biomed."},{"issue":"1","key":"10.1016\/j.bspc.2024.107232_b41","doi-asserted-by":"crossref","first-page":"98","DOI":"10.1109\/JBHI.2017.2688239","article-title":"DREAMER: A database for emotion recognition through EEG and ECG signals from wireless low-cost off-the-shelf devices","volume":"22","author":"Katsigiannis","year":"2017","journal-title":"IEEE J. Biomed. Health Inform."},{"key":"10.1016\/j.bspc.2024.107232_b42","doi-asserted-by":"crossref","first-page":"57","DOI":"10.1109\/ACCESS.2018.2883213","article-title":"Using deep convolutional neural network for emotion detection on a physiological signals dataset (AMIGOS)","volume":"7","author":"Santamaria-Granados","year":"2018","journal-title":"IEEE Access"},{"key":"10.1016\/j.bspc.2024.107232_b43","series-title":"2021 9th International Conference on Affective Computing and Intelligent Interaction","first-page":"1","article-title":"Multi-modal emotion recognition using recurrence plots and transfer learning on physiological signals","author":"Elalamy","year":"2021"},{"issue":"1","key":"10.1016\/j.bspc.2024.107232_b44","doi-asserted-by":"crossref","first-page":"17","DOI":"10.1109\/TAFFC.2015.2436926","article-title":"Analysis of EEG signals and facial expressions for continuous emotion detection","volume":"7","author":"Soleymani","year":"2015","journal-title":"IEEE Trans. Affect. Comput."},{"key":"10.1016\/j.bspc.2024.107232_b45","doi-asserted-by":"crossref","first-page":"33061","DOI":"10.1109\/ACCESS.2023.3263670","article-title":"Multimodal emotion recognition from EEG signals and facial expressions","volume":"11","author":"Wang","year":"2023","journal-title":"IEEE Access"},{"key":"10.1016\/j.bspc.2024.107232_b46","doi-asserted-by":"crossref","unstructured":"K. He, X. Chen, S. Xie, Y. Li, P. Doll\u00e1r, R. Girshick, Masked autoencoders are scalable vision learners, in: Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition, 2022, pp. 16000\u201316009.","DOI":"10.1109\/CVPR52688.2022.01553"},{"key":"10.1016\/j.bspc.2024.107232_b47","series-title":"Lxmert: Learning cross-modality encoder representations from transformers","author":"Hao","year":"2019"},{"key":"10.1016\/j.bspc.2024.107232_b48","series-title":"Advances in Neural Information Processing Systems","first-page":"32","article-title":"Vilbert: Pretraining task-agnostic visiolinguistic representations for vision-and-language tasks","author":"Jiasen","year":"2019"},{"key":"10.1016\/j.bspc.2024.107232_b49","unstructured":"K. Wonjae, B. Son, I. Kim, Vilt: Vision-and-language transformer without convolution or region supervision, in: International Conference on Machine Learning, 2021, pp. 5583\u20135594."},{"key":"10.1016\/j.bspc.2024.107232_b50","series-title":"Uniter: Learning universal image-text representations","author":"Yen-Chun","year":"2019"},{"key":"10.1016\/j.bspc.2024.107232_b51","doi-asserted-by":"crossref","unstructured":"Y. Zitong, W. Peng, X. Li, X. Hong, G. Zhao, Remote heart rate measurement from highly compressed facial videos: an end-to-end deep learning solution with video enhancement, in: IEEE\/CVF International Conference on Computer Vision, 2019, pp. 151\u2013160.","DOI":"10.1109\/ICCV.2019.00024"},{"key":"10.1016\/j.bspc.2024.107232_b52","doi-asserted-by":"crossref","first-page":"31","DOI":"10.3389\/frobt.2017.00031","article-title":"Evaluating the usability of a wearable social skills training technology for children with autism spectrum disorder","volume":"4","author":"Kinsella","year":"2017","journal-title":"Front. Robot. AI"},{"issue":"10","key":"10.1016\/j.bspc.2024.107232_b53","doi-asserted-by":"crossref","first-page":"1499","DOI":"10.1109\/LSP.2016.2603342","article-title":"Joint face detection and alignment using multitask cascaded convolutional networks","volume":"23","author":"Kaipeng","year":"2016","journal-title":"IEEE Signal Process. Lett."},{"key":"10.1016\/j.bspc.2024.107232_b54","series-title":"Adam: A method for stochastic optimization","author":"Kingma","year":"2014"},{"key":"10.1016\/j.bspc.2024.107232_b55","unstructured":"I. Loshchilov, F. Hutter, Decoupled weight decay regularization, in: The International Conference on Learning Representations, 2019, pp. 234\u2013778."},{"key":"10.1016\/j.bspc.2024.107232_b56","doi-asserted-by":"crossref","unstructured":"G. Francesca, A. Grossi, S. Bandini, A deep learning approach to recognize cognitive load using ppg signals, in: PErvasive Technologies Related To Assistive Environments Conference, 2021, pp. 489\u2013495.","DOI":"10.1145\/3453892.3461625"}],"container-title":["Biomedical Signal Processing and Control"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/api.elsevier.com\/content\/article\/PII:S1746809424012904?httpAccept=text\/xml","content-type":"text\/xml","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/api.elsevier.com\/content\/article\/PII:S1746809424012904?httpAccept=text\/plain","content-type":"text\/plain","content-version":"vor","intended-application":"text-mining"}],"deposited":{"date-parts":[[2026,5,15]],"date-time":"2026-05-15T06:05:30Z","timestamp":1778825130000},"score":1,"resource":{"primary":{"URL":"https:\/\/linkinghub.elsevier.com\/retrieve\/pii\/S1746809424012904"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,4]]},"references-count":56,"alternative-id":["S1746809424012904"],"URL":"https:\/\/doi.org\/10.1016\/j.bspc.2024.107232","relation":{},"ISSN":["1746-8094"],"issn-type":[{"value":"1746-8094","type":"print"}],"subject":[],"published":{"date-parts":[[2025,4]]},"assertion":[{"value":"Elsevier","name":"publisher","label":"This article is maintained by"},{"value":"A Unified Biosensor\u2013Vision Multi-Modal Transformer network for emotion recognition","name":"articletitle","label":"Article Title"},{"value":"Biomedical Signal Processing and Control","name":"journaltitle","label":"Journal Title"},{"value":"https:\/\/doi.org\/10.1016\/j.bspc.2024.107232","name":"articlelink","label":"CrossRef DOI link to publisher maintained version"},{"value":"article","name":"content_type","label":"Content Type"},{"value":"\u00a9 2024 The Authors. Published by Elsevier Ltd.","name":"copyright","label":"Copyright"}],"article-number":"107232"}}