{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,7]],"date-time":"2026-07-07T15:15:31Z","timestamp":1783437331531,"version":"3.54.6"},"reference-count":43,"publisher":"Springer Science and Business Media LLC","issue":"16","license":[{"start":{"date-parts":[[2023,2,2]],"date-time":"2023-02-02T00:00:00Z","timestamp":1675296000000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2023,2,2]],"date-time":"2023-02-02T00:00:00Z","timestamp":1675296000000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Multimed Tools Appl"],"published-print":{"date-parts":[[2023,7]]},"DOI":"10.1007\/s11042-023-14381-6","type":"journal-article","created":{"date-parts":[[2023,2,2]],"date-time":"2023-02-02T18:04:27Z","timestamp":1675361067000},"page":"25427-25448","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":26,"title":["Real-time multimodal interaction in virtual reality - a case study with a large virtual interface"],"prefix":"10.1007","volume":"82","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-0888-8289","authenticated-orcid":false,"given":"Lizhou","family":"Cao","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Huadong","family":"Zhang","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Chao","family":"Peng","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Jeffrey T.","family":"Hansberger","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2023,2,2]]},"reference":[{"issue":"2","key":"14381_CR1","doi-asserted-by":"publisher","first-page":"423","DOI":"10.1109\/TPAMI.2018.2798607","volume":"41","author":"T Baltru\u0161aitis","year":"2018","unstructured":"Baltru\u0161aitis T, Ahuja C, Morency LP (2018) Multimodal machine learning: a survey and taxonomy. IEEE Trans Pattern Anal Mach Intell 41(2):423\u2013443","journal-title":"IEEE Trans Pattern Anal Mach Intell"},{"issue":"1","key":"14381_CR2","doi-asserted-by":"publisher","first-page":"96","DOI":"10.1162\/105474601750182342","volume":"10","author":"DA Bowman","year":"2001","unstructured":"Bowman DA, Kruijff E, LaViola JJ, Poupyrev I (2001) An introduction to 3-D user interface design. Presence 10(1):96\u2013108","journal-title":"Presence"},{"key":"14381_CR3","doi-asserted-by":"crossref","unstructured":"Cao L, Peng C, Dong Y (2020) Ellic\u2019s exercise class: promoting physical activities during exergaming with immersive virtual reality. Virtual Reality:1\u201316","DOI":"10.1007\/s10055-020-00477-z"},{"key":"14381_CR4","doi-asserted-by":"publisher","unstructured":"Cao L, Peng C, Hansberger JT (2019) A large curved display system in virtual reality for immersive data interaction. In: 2019 IEEE games, entertainment, media conference (GEM). IEEE, pp 1\u20134. https:\/\/doi.org\/10.1109\/GEM.2019.8811550","DOI":"10.1109\/GEM.2019.8811550"},{"key":"14381_CR5","doi-asserted-by":"crossref","unstructured":"Cao L, Peng C, Hansberger JT (2019) Usability and engagement study for a serious virtual reality game of lunar exploration missions. In: Informatics, vol 6, p 44. Multidisciplinary digital publishing institute","DOI":"10.3390\/informatics6040044"},{"key":"14381_CR6","doi-asserted-by":"crossref","unstructured":"Chatterjee I, Xiao R, Harrison C (2015) Gaze+ gesture: expressive, precise and targeted free-space interactions. In: Proceedings of the 2015 ACM on international conference on multimodal interaction, pp 131\u2013138","DOI":"10.1145\/2818346.2820752"},{"key":"14381_CR7","doi-asserted-by":"crossref","unstructured":"Chun LM, Arshad H, Piumsomboon T, Billinghurst M (2015) A combination of static and stroke gesture with speech for multimodal interaction in a virtual environment. In: 2015 International conference on electrical engineering and informatics (ICEEI). IEEE, pp 59\u201364","DOI":"10.1109\/ICEEI.2015.7352470"},{"key":"14381_CR8","doi-asserted-by":"crossref","unstructured":"Creed C, Frutos-Pascual M, Williams I (2020) Multimodal gaze interaction for creative design. In: Proceedings of the 2020 CHI conference on human factors in computing systems, pp 1\u201313","DOI":"10.1145\/3313831.3376196"},{"key":"14381_CR9","doi-asserted-by":"publisher","unstructured":"Ferracani A, Pezzatini D, Del Bimbo A (2014) A natural and immersive virtual interface for the surgical safety checklist training. In: Proceedings of the 2014 ACM international workshop on serious games, pp 27\u201332. https:\/\/doi.org\/10.1145\/2656719.2656725","DOI":"10.1145\/2656719.2656725"},{"issue":"5","key":"14381_CR10","doi-asserted-by":"publisher","first-page":"829","DOI":"10.1162\/neco_a_01273","volume":"32","author":"J Gao","year":"2020","unstructured":"Gao J, Li P, Chen Z, Zhang J (2020) A survey on deep learning for multimodal data fusion. Neural Comput 32(5):829\u2013864","journal-title":"Neural Comput"},{"key":"14381_CR11","first-page":"2211","volume":"12","author":"M G\u00f6nen","year":"2011","unstructured":"G\u00f6nen M, Alpayd\u0131n E (2011) Multiple kernel learning algorithms. J Mach Learn Res 12:2211\u20132268","journal-title":"J Mach Learn Res"},{"key":"14381_CR12","doi-asserted-by":"publisher","first-page":"63373","DOI":"10.1109\/ACCESS.2019.2916887","volume":"7","author":"W Guo","year":"2019","unstructured":"Guo W, Wang J, Wang S (2019) Deep multimodal representation learning: a survey. IEEE Access 7:63373\u201363394","journal-title":"IEEE Access"},{"key":"14381_CR13","doi-asserted-by":"crossref","unstructured":"Hansberger JT, Peng C, Blakely V, Meacham S, Cao L, Diliberti N (2019) A multimodal interface for virtual information environments. In: Chen JY, Fragomeni G (eds) Virtual, augmented and mixed reality. Multimodal interaction. Springer international publishing, Cham, pp 59\u201370","DOI":"10.1007\/978-3-030-21607-8_5"},{"key":"14381_CR14","doi-asserted-by":"crossref","unstructured":"Hansberger JT, Peng C, Mathis SL, Shanthakumar VA, Meacham SC, Cao L, Blakely VR (2017) Dispelling the gorilla arm syndrome: the viability of prolonged gesture interactions. In: International conference on virtual, augmented and mixed reality. Springer, pp 505\u2013520","DOI":"10.1007\/978-3-319-57987-0_41"},{"issue":"1-2","key":"14381_CR15","doi-asserted-by":"publisher","first-page":"116","DOI":"10.1016\/j.cviu.2006.10.019","volume":"108","author":"A Jaimes","year":"2007","unstructured":"Jaimes A, Sebe N (2007) Multimodal human\u2013computer interaction: a survey. Comput Vis Image Understand 108(1-2):116\u2013134","journal-title":"Comput Vis Image Understand"},{"key":"14381_CR16","doi-asserted-by":"publisher","unstructured":"Jin T, Huang S, Li Y, Zhang Z (2020) Dual low-rank multimodal fusion. In: Findings of the association for computational linguistics: EMNLP 2020, pp 377\u2013387. Association for computational linguistics, online. https:\/\/doi.org\/10.18653\/v1\/2020.findings-emnlp.35. https:\/\/aclanthology.org\/2020.findings-emnlp.35","DOI":"10.18653\/v1\/2020.findings-emnlp.35"},{"key":"14381_CR17","unstructured":"Joze HRV, Shaban A, Iuzzolino ML, Koishida K (2020) MMTM: multimodal transfer module for CNN fusion. In: Proceedings of the IEEE\/CVF conference on computer vision and pattern recognition, pp 13289\u201313299"},{"key":"14381_CR18","doi-asserted-by":"crossref","unstructured":"Kang R, Guo A, Laput G, Li Y, Chen X (2019) Minuet: multimodal interaction with an internet of things. In: Symposium on spatial user interaction, pp 1\u201310","DOI":"10.1145\/3357251.3357581"},{"issue":"3","key":"14381_CR19","doi-asserted-by":"publisher","first-page":"882","DOI":"10.1016\/j.cmpb.2013.12.024","volume":"113","author":"K Kim","year":"2014","unstructured":"Kim K, Rosenthal MZ, Zielinski DJ, Brady R (2014) Effects of virtual environment platforms on emotional responses. Comput Methods Programs Biomed 113(3):882\u2013893. https:\/\/doi.org\/10.1016\/j.cmpb.2013.12.024","journal-title":"Comput Methods Programs Biomed"},{"key":"14381_CR20","doi-asserted-by":"publisher","unstructured":"Kimani S (2009) WIMP interfaces. Springer US, Boston, MA, pp 3529\u20133533. https:\/\/doi.org\/10.1007\/978-0-387-39940-9_467","DOI":"10.1007\/978-0-387-39940-9_467"},{"issue":"9","key":"14381_CR21","doi-asserted-by":"publisher","first-page":"1449","DOI":"10.1109\/JPROC.2015.2460697","volume":"103","author":"D Lahat","year":"2015","unstructured":"Lahat D, Adali T, Jutten C (2015) Multimodal data fusion: an overview of methods, challenges, and prospects. Proc IEEE 103(9):1449\u20131477. https:\/\/doi.org\/10.1109\/JPROC.2015.2460697","journal-title":"Proc IEEE"},{"key":"14381_CR22","doi-asserted-by":"publisher","unstructured":"Liu Z, Shen Y, Lakshminarasimhan VB, Liang PP, Bagher Zadeh A, Morency LP (2018) Efficient low-rank multimodal fusion with modality-specific factors. In: Proceedings of the 56th annual meeting of the association for computational linguistics (vol 1: long papers). Association for computational linguistics, Melbourne, Australia, pp 2247\u20132256. https:\/\/doi.org\/10.18653\/v1\/P18-1209. https:\/\/aclanthology.org\/P18-1209","DOI":"10.18653\/v1\/P18-1209"},{"issue":"6","key":"14381_CR23","doi-asserted-by":"publisher","first-page":"1009","DOI":"10.1093\/bioinformatics\/btx682","volume":"34","author":"J Mariette","year":"2018","unstructured":"Mariette J, Villa-Vialaneix N (2018) Unsupervised multiple kernel learning for heterogeneous data integration. Bioinformatics 34(6):1009\u20131015","journal-title":"Bioinformatics"},{"issue":"10s","key":"14381_CR24","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1145\/3508361","volume":"54","author":"D Martin","year":"2022","unstructured":"Martin D, Malpica S, Gutierrez D, Masia B, Serrano A (2022) Multimodality in VR: a survey. ACM Comput Surveys (CSUR) 54(10s):1\u201336","journal-title":"ACM Comput Surveys (CSUR)"},{"key":"14381_CR25","unstructured":"Meta: use quest link with meta quest headsets (2022). https:\/\/www.meta.com\/help\/quest\/articles\/headsets-and-accessories\/oculus-link\/connect-link-with-quest-2\/. Accessed 24 Sept 2022"},{"key":"14381_CR26","doi-asserted-by":"crossref","unstructured":"Mohan P, Boon Goh W, Fu CW, Yeung SK (2019) Head-fingers-arms: physically-coupled and decoupled multimodal interaction designs in mobile VR. In: The 17th international conference on virtual-reality continuum and its applications in industry, pp 1\u20139","DOI":"10.1145\/3359997.3365697"},{"key":"14381_CR27","unstructured":"Ngiam J, Khosla A, Kim M, Nam J, Lee H, Ng AY (2011) Multimodal deep learning. In: ICML"},{"key":"14381_CR28","doi-asserted-by":"publisher","unstructured":"Ord\u00f3\u00f1ez FJ, Roggen D (2016) Deep convolutional and LSTM recurrent neural networks for multimodal wearable activity recognition. Sensors, vol 16(1). https:\/\/doi.org\/10.3390\/s16010115. https:\/\/www.mdpi.com\/1424-8220\/16\/1\/115","DOI":"10.3390\/s16010115"},{"key":"14381_CR29","doi-asserted-by":"publisher","unstructured":"Peng C, Hansberger JT, Cao L, Shanthakumar VA (2017) Hand gesture controls for image categorization in immersive virtual environments. In: 2017 IEEE virtual reality (VR), pp 331\u2013332. https:\/\/doi.org\/10.1109\/VR.2017.7892311","DOI":"10.1109\/VR.2017.7892311"},{"key":"14381_CR30","doi-asserted-by":"crossref","unstructured":"Pfeuffer K, Mayer B, Mardanbegi D, Gellersen H (2017) Gaze+ pinch interaction in virtual reality. In: Proceedings of the 5th symposium on spatial user interaction, pp 99\u2013108","DOI":"10.1145\/3131277.3132180"},{"key":"14381_CR31","unstructured":"Reuten A, Bos J, Smeets JB (2020) The metrics for measuring motion sickness. In: Driving simul conf Europe, vol 2020, pp 1\u20134"},{"key":"14381_CR32","doi-asserted-by":"publisher","unstructured":"Schneider A, Cernea D, Ebert A (2016) HMD-enabled virtual screens as alternatives to large physical displays. In: 2016 20th International conference information visualisation (IV), pp 390\u2013394. https:\/\/doi.org\/10.1109\/IV.2016.48","DOI":"10.1109\/IV.2016.48"},{"issue":"6","key":"14381_CR33","doi-asserted-by":"publisher","first-page":"103","DOI":"10.9781\/ijimai.2017.09.001","volume":"4","author":"M Schrepp","year":"2017","unstructured":"Schrepp M, Hinderks A, Thomaschewski J (2017) Design and evaluation of a short version of the user experience questionnaire (UEQ-s). IJIMAI 4 (6):103\u2013108","journal-title":"IJIMAI"},{"issue":"25","key":"14381_CR34","doi-asserted-by":"publisher","first-page":"17707","DOI":"10.1007\/s11042-019-08520-1","volume":"79","author":"VA Shanthakumar","year":"2020","unstructured":"Shanthakumar VA, Peng C, Hansberger J, Cao L, Meacham S, Blakely V (2020) Design and evaluation of a hand gesture recognition approach for real-time interactions. Multimed Tools Appl 79(25):17707\u201317730. https:\/\/doi.org\/10.1007\/s11042-019-08520-1","journal-title":"Multimed Tools Appl"},{"key":"14381_CR35","doi-asserted-by":"publisher","unstructured":"Tsandilas T (2018) Fallacies of agreement: a critical review of consensus assessment methods for gesture elicitation. ACM Trans Comput-Hum Interact, vol 25(3). https:\/\/doi.org\/10.1145\/3182168","DOI":"10.1145\/3182168"},{"key":"14381_CR36","doi-asserted-by":"publisher","first-page":"189","DOI":"10.1016\/j.patrec.2013.07.003","volume":"36","author":"M Turk","year":"2014","unstructured":"Turk M (2014) Multimodal interaction: a review. Pattern Recogn Lett 36:189\u2013195","journal-title":"Pattern Recogn Lett"},{"key":"14381_CR37","unstructured":"Vanoni D (2013) Human-centered interfaces for large, high-resolution visualization systems. Res Exam"},{"key":"14381_CR38","doi-asserted-by":"publisher","unstructured":"Villarreal-Narvaez S, Vanderdonckt J, Vatavu RD, Wobbrock JO (2020) A systematic review of gesture elicitation studies: what can we learn from 216 studies?. Association for computing machinery, New York, NY, USA, pp 855\u2013872. https:\/\/doi.org\/10.1145\/3357236.3395511","DOI":"10.1145\/3357236.3395511"},{"key":"14381_CR39","unstructured":"Vogiazou Y (2016) Beyond touch: designing effective gestural interactions. https:\/\/www.invisionapp.com\/inside-design\/effective-gestural-interaction-design\/"},{"issue":"5","key":"14381_CR40","doi-asserted-by":"publisher","first-page":"524","DOI":"10.1109\/THMS.2021.3097973","volume":"51","author":"Z Wang","year":"2021","unstructured":"Wang Z, Wang H, Yu H, Lu F (2021) Interaction with gaze, gesture, and speech in a flexibly configurable augmented reality system. IEEE Trans Human-Mach Syst 51(5):524\u2013534","journal-title":"IEEE Trans Human-Mach Syst"},{"issue":"12","key":"14381_CR41","doi-asserted-by":"publisher","first-page":"3479","DOI":"10.1109\/TVCG.2020.3023566","volume":"26","author":"AS Williams","year":"2020","unstructured":"Williams AS, Garcia J, Ortega F (2020) Understanding multimodal user gesture and speech behavior for object manipulation in augmented reality using elicitation. IEEE Trans Vis Comput Graph 26(12):3479\u20133489","journal-title":"IEEE Trans Vis Comput Graph"},{"issue":"ISS","key":"14381_CR42","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1145\/3427330","volume":"4","author":"AS Williams","year":"2020","unstructured":"Williams AS, Ortega FR (2020) Understanding gesture and speech multimodal interactions for manipulation tasks in augmented reality using unconstrained elicitation. Proc ACM on Human-Comput Interaction 4(ISS):1\u201321","journal-title":"Proc ACM on Human-Comput Interaction"},{"key":"14381_CR43","doi-asserted-by":"crossref","unstructured":"Zimmerer C, Wolf E, Wolf S, Fischbach M, Lugrin JL, Latoschik ME (2020) Finally on par?! multimodal and unimodal interaction for open creative design tasks in virtual reality. In: Proceedings of the 2020 international conference on multimodal interaction, pp 222\u2013231","DOI":"10.1145\/3382507.3418850"}],"container-title":["Multimedia Tools and Applications"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s11042-023-14381-6.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s11042-023-14381-6\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s11042-023-14381-6.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2023,6,23]],"date-time":"2023-06-23T18:45:47Z","timestamp":1687545947000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s11042-023-14381-6"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023,2,2]]},"references-count":43,"journal-issue":{"issue":"16","published-print":{"date-parts":[[2023,7]]}},"alternative-id":["14381"],"URL":"https:\/\/doi.org\/10.1007\/s11042-023-14381-6","relation":{},"ISSN":["1380-7501","1573-7721"],"issn-type":[{"value":"1380-7501","type":"print"},{"value":"1573-7721","type":"electronic"}],"subject":[],"published":{"date-parts":[[2023,2,2]]},"assertion":[{"value":"21 June 2022","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"26 December 2022","order":2,"name":"revised","label":"Revised","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"6 January 2023","order":3,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"2 February 2023","order":4,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Declarations"}},{"value":"The authors have no conflicts of interest\/competing interests to declare that are relevant to the content of this article.","order":2,"name":"Ethics","group":{"name":"EthicsHeading","label":"<!--Emphasis Type='Bold' removed-->Conflict of Interests"}}]}}