{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,8,2]],"date-time":"2025-08-02T05:00:45Z","timestamp":1754110845226,"version":"3.40.3"},"publisher-location":"Cham","reference-count":35,"publisher":"Springer International Publishing","isbn-type":[{"type":"print","value":"9783030870157"},{"type":"electronic","value":"9783030870164"}],"license":[{"start":{"date-parts":[[2021,1,1]],"date-time":"2021-01-01T00:00:00Z","timestamp":1609459200000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"},{"start":{"date-parts":[[2021,1,1]],"date-time":"2021-01-01T00:00:00Z","timestamp":1609459200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2021]]},"DOI":"10.1007\/978-3-030-87016-4_46","type":"book-chapter","created":{"date-parts":[[2021,9,11]],"date-time":"2021-09-11T06:02:42Z","timestamp":1631340162000},"page":"641-657","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":5,"title":["Evaluation Methods Applied to Virtual Reality Educational Applications: A\u00a0Systematic Review"],"prefix":"10.1007","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-2182-794X","authenticated-orcid":false,"given":"Daniel Bueno","family":"Domingueti","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-9619-2171","authenticated-orcid":false,"given":"Diego Roberto Colombo","family":"Dias","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-5026-1581","authenticated-orcid":false,"given":"Marcelo de Paiva","family":"Guimar\u00e3es","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-3844-0178","authenticated-orcid":false,"given":"D\u00e1rlinton Barbosa Feres","family":"Carvalho","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2021,9,12]]},"reference":[{"key":"46_CR1","unstructured":"American Educational Research Association, American Psychological Association, N.C.o.M.i.E.: Standards for Educational and Psychological Testing. Amer Educational Research Assn (2014)"},{"key":"46_CR2","unstructured":"Bach, C., Scapin, D.L.: Obstacles and perspectives for evaluating mixed reality systems usability. In: Acte du Workshop MIXER, IUI-CADUI, vol. 4. Citeseer (2004)"},{"issue":"5","key":"46_CR3","doi-asserted-by":"publisher","first-page":"821","DOI":"10.1007\/s00779-016-0953-5","volume":"20","author":"Y Bian","year":"2016","unstructured":"Bian, Y., Yang, C., Gao, F., Li, H., Zhou, S., Li, H., Sun, X., Meng, X.: A framework for physiological indicators of flow in vr games: construction and preliminary evaluation. Pers. Ubiq. Comput. 20(5), 821\u2013832 (2016)","journal-title":"Pers. Ubiq. Comput."},{"issue":"194","key":"46_CR4","first-page":"4","volume":"189","author":"J Brooke","year":"1996","unstructured":"Brooke, J., et al.: Sus-a quick and dirty usability scale. Usab. Eval. Ind. 189(194), 4\u20137 (1996)","journal-title":"Usab. Eval. Ind."},{"key":"46_CR5","doi-asserted-by":"publisher","first-page":"35","DOI":"10.1007\/978-94-017-9088-8_3","volume-title":"Flow and the Foundations of Positive Psychology","author":"M Csikszentmihalyi","year":"2014","unstructured":"Csikszentmihalyi, M., Larson, R.: Validity and reliability of the experience-sampling method. In: Flow and the Foundations of Positive Psychology, pp. 35\u201354. Springer, Dordrecht (2014). https:\/\/doi.org\/10.1007\/978-94-017-9088-8_3"},{"key":"46_CR6","doi-asserted-by":"crossref","unstructured":"Davis, F.D.: Perceived usefulness, perceived ease of use, and user acceptance of information technology. MIS Q., 319\u2013340 (1989)","DOI":"10.2307\/249008"},{"key":"46_CR7","doi-asserted-by":"crossref","unstructured":"D\u00fcnser, A., Grasset, R., Billinghurst, M.: A survey of evaluation techniques used in augmented reality studies. Human Interface Technology Laboratory New Zealand (2008)","DOI":"10.1145\/1508044.1508049"},{"issue":"2","key":"46_CR8","doi-asserted-by":"publisher","first-page":"19","DOI":"10.1177\/2150137810386108","volume":"1","author":"PB Elmore","year":"2010","unstructured":"Elmore, P.B.: Reporting standards for research publications. Couns. Outcome Res. Eval. 1(2), 19\u201329 (2010)","journal-title":"Couns. Outcome Res. Eval."},{"key":"46_CR9","doi-asserted-by":"publisher","first-page":"158","DOI":"10.1007\/s11031-008-9102-4","volume":"32","author":"S Engeser","year":"2008","unstructured":"Engeser, S., Rheinberg, F.: Flow, performance and moderators of challenge-skill balance. Motiv. Emot. 32, 158\u2013172 (2008). https:\/\/doi.org\/10.1007\/s11031-008-9102-4","journal-title":"Motiv. Emot."},{"issue":"3","key":"46_CR10","doi-asserted-by":"publisher","first-page":"255","DOI":"10.3102\/01623737011003255","volume":"11","author":"JC Greene","year":"1989","unstructured":"Greene, J.C., Caracelli, V.J., Graham, W.F.: Toward a conceptual framework for mixed-method evaluation designs. Educ. Eval. Policy Anal. 11(3), 255\u2013274 (1989)","journal-title":"Educ. Eval. Policy Anal."},{"key":"46_CR11","doi-asserted-by":"crossref","unstructured":"Hart, S.G.: Nasa-task load index (nasa-tlx); 20 years later. In: Proceedings of the Human Factors and Ergonomics Society Annual Meeting, vol. 50, pp. 904\u2013908. Sage Publications Sage CA, Los Angeles (2006)","DOI":"10.1177\/154193120605000909"},{"key":"46_CR12","doi-asserted-by":"publisher","first-page":"75","DOI":"10.2307\/25148625","volume":"28","author":"AR Hevner","year":"2004","unstructured":"Hevner, A.R., March, S.T., Park, J., Ram, S.: Design science in information systems research. Manag. Inf. Syst. Q. 28, 75\u2013105 (2004)","journal-title":"Manag. Inf. Syst. Q."},{"issue":"1","key":"46_CR13","doi-asserted-by":"publisher","first-page":"129","DOI":"10.1007\/s12369-016-0380-9","volume":"9","author":"CC Ho","year":"2017","unstructured":"Ho, C.C., MacDorman, K.F.: Measuring the uncanny valley effect. Int. J. Soc. Rob. 9(1), 129\u2013139 (2017)","journal-title":"Int. J. Soc. Rob."},{"issue":"3","key":"46_CR14","doi-asserted-by":"publisher","first-page":"203","DOI":"10.1207\/s15327108ijap0303_3","volume":"3","author":"RS Kennedy","year":"1993","unstructured":"Kennedy, R.S., Lane, N.E., Berbaum, K.S., Lilienthal, M.G.: Simulator sickness questionnaire: an enhanced method for quantifying simulator sickness. Int. J. Aviat. Psychol. 3(3), 203\u2013220 (1993)","journal-title":"Int. J. Aviat. Psychol."},{"issue":"2004","key":"46_CR15","first-page":"1","volume":"33","author":"B Kitchenham","year":"2004","unstructured":"Kitchenham, B.: Procedures for performing systematic reviews. Keele, UK, Keele Univ. 33(2004), 1\u201326 (2004)","journal-title":"Keele, UK, Keele Univ."},{"key":"46_CR16","doi-asserted-by":"publisher","DOI":"10.1093\/0195096533.001.0001","volume-title":"The Advancement of Science: Science without Legend, Objectivity without Illusions","author":"P Kitcher","year":"1995","unstructured":"Kitcher, P.: The Advancement of Science: Science without Legend, Objectivity without Illusions. Oxford University Press, Oxford (1995)"},{"issue":"1","key":"46_CR17","doi-asserted-by":"publisher","first-page":"113","DOI":"10.1207\/S15327647JCD0101N_11","volume":"1","author":"D Kuhn","year":"2000","unstructured":"Kuhn, D., Pearsall, S.: Developmental origins of scientific thinking. J. Cogn. Dev. 1(1), 113\u2013129 (2000)","journal-title":"J. Cogn. Dev."},{"key":"46_CR18","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"63","DOI":"10.1007\/978-3-540-89350-9_6","volume-title":"HCI and Usability for Education and Work","author":"B Laugwitz","year":"2008","unstructured":"Laugwitz, B., Held, T., Schrepp, M.: Construction and evaluation of a user experience questionnaire. In: Holzinger, A. (ed.) USAB 2008. LNCS, vol. 5298, pp. 63\u201376. Springer, Heidelberg (2008). https:\/\/doi.org\/10.1007\/978-3-540-89350-9_6"},{"issue":"3","key":"46_CR19","doi-asserted-by":"publisher","first-page":"282","DOI":"10.1162\/105474601300343612","volume":"10","author":"J Lessiter","year":"2001","unstructured":"Lessiter, J., Freeman, J., Keogh, E., Davidoff, J.: A cross-media presence questionnaire: the itc-sense of presence inventory. Pres. Teleoper. Virtual Environ. 10(3), 282\u2013297 (2001)","journal-title":"Pres. Teleoper. Virtual Environ."},{"issue":"2","key":"46_CR20","first-page":"3","volume":"8","author":"AM Lund","year":"2001","unstructured":"Lund, A.M.: Measuring usability with the use questionnaire12. Usab. Interface 8(2), 3\u20136 (2001)","journal-title":"Usab. Interface"},{"key":"46_CR21","doi-asserted-by":"publisher","unstructured":"Makransky, G., Lilleholt, L., Aaby, A.: Development and validation of the multimodal presence scale for virtual reality environments: a confirmatory factor analysis and item response theory approach. Comput. Human Behav. 72 (2017). https:\/\/doi.org\/10.1016\/j.chb.2017.02.066","DOI":"10.1016\/j.chb.2017.02.066"},{"key":"46_CR22","unstructured":"Nielsen, J.: Thinking aloud: The# 1 usability tool. Nielsen Norman Group 16 (2012)"},{"key":"46_CR23","doi-asserted-by":"crossref","unstructured":"Rauschenberger, M., Schrepp, M., P\u00e9rez Cota, M., Olschner, S., Thomaschewski, J.: Efficient measurement of the user experience of interactive products. How to use the user experience questionnaire (ueq). example: Spanish language version. Int. J. Interact. Multimedia Artif. Intell. (2013)","DOI":"10.9781\/ijimai.2013.215"},{"issue":"2","key":"46_CR24","doi-asserted-by":"publisher","first-page":"616","DOI":"10.1007\/s00167-016-4114-1","volume":"25","author":"PG Roberts","year":"2017","unstructured":"Roberts, P.G., Guyver, P., Baldwin, M., Akhtar, K., Alvand, A., Price, A.J., Rees, J.L.: Validation of the updated arthros simulator: face and construct validity of a passive haptic virtual reality simulator with novel performance metrics. Knee Surg. Sports Traumatol. Arthrosc. 25(2), 616\u2013625 (2017)","journal-title":"Knee Surg. Sports Traumatol. Arthrosc."},{"issue":"1","key":"46_CR25","doi-asserted-by":"publisher","first-page":"16","DOI":"10.1057\/jos.2014.14","volume":"9","author":"GS Ruthenbeck","year":"2015","unstructured":"Ruthenbeck, G.S., Reynolds, K.J.: Virtual reality for medical training: the state-of-the-art. J. Simul. 9(1), 16\u201326 (2015)","journal-title":"J. Simul."},{"issue":"1","key":"46_CR26","doi-asserted-by":"publisher","first-page":"136","DOI":"10.1016\/j.compedu.2011.07.017","volume":"58","author":"N Rutten","year":"2012","unstructured":"Rutten, N., Van Joolingen, W.R., Van Der Veen, J.T.: The learning effects of computer simulations in science education. Comput. Educ. 58(1), 136\u2013153 (2012)","journal-title":"Comput. Educ."},{"issue":"3","key":"46_CR27","doi-asserted-by":"publisher","first-page":"266","DOI":"10.1162\/105474601300343603","volume":"10","author":"T Schubert","year":"2001","unstructured":"Schubert, T., Friedmann, F., Regenbrecht, H.: The experience of presence: factor analytic insights. Pres. Teleoper. Virt. Environ. 10(3), 266\u2013281 (2001)","journal-title":"Pres. Teleoper. Virt. Environ."},{"key":"46_CR28","unstructured":"Shadish, W.R., Cook, T.D., Campbell, D.T., et al.: Experimental and quasi-experimental designs for generalized causal inference\/William R. Shedish, Thomas D. Cook, Donald T. Campbell. Houghton Mifflin, Boston (2002)"},{"key":"46_CR29","unstructured":"Sharp, H., Rogers, Y.J.P.: Design de Intera\u00e7\u00e3o: Al\u00e9m da intera\u00e7\u00e3o humano computador. Bookman (2005)"},{"issue":"2","key":"46_CR30","doi-asserted-by":"publisher","first-page":"130","DOI":"10.1162\/pres.1994.3.2.130","volume":"3","author":"M Slater","year":"1994","unstructured":"Slater, M., Usoh, M., Steed, A.: Depth of presence in virtual environments. Pres. Teleoper. Virt. Environ. 3(2), 130\u2013144 (1994)","journal-title":"Pres. Teleoper. Virt. Environ."},{"key":"46_CR31","unstructured":"Sommerville, I.: Engenharia de software. PEARSON BRASIL (2011). https:\/\/books.google.com.br\/books?id=H4u5ygAACAAJ"},{"key":"46_CR32","doi-asserted-by":"crossref","unstructured":"Venkatesh, V., Morris, M.G., Davis, G.B., Davis, F.D.: User acceptance of information technology: toward a unified view. MIS Q., 425\u2013478 (2003)","DOI":"10.2307\/30036540"},{"key":"46_CR33","doi-asserted-by":"crossref","unstructured":"Villagrasa, S., Fonseca, D., Dur\u00e1n, J.: Teaching case: applying gamification techniques and virtual reality for learning building engineering 3d arts. In: Proceedings of the Second International Conference on Technological Ecosystems for Enhancing Multiculturality, pp. 171\u2013177 (2014)","DOI":"10.1145\/2669711.2669896"},{"issue":"3","key":"46_CR34","doi-asserted-by":"publisher","first-page":"225","DOI":"10.1162\/105474698565686","volume":"7","author":"BG Witmer","year":"1998","unstructured":"Witmer, B.G., Singer, M.J.: Measuring presence in virtual environments: a presence questionnaire. Presence 7(3), 225\u2013240 (1998)","journal-title":"Presence"},{"key":"46_CR35","doi-asserted-by":"publisher","unstructured":"Zeri, F., Livi, S.: Visual discomfort while watching stereoscopic three-dimensional movies at the cinema. Ophthal. Physiol. Opt. 35 (2015). https:\/\/doi.org\/10.1111\/opo.12194","DOI":"10.1111\/opo.12194"}],"container-title":["Lecture Notes in Computer Science","Computational Science and Its Applications \u2013 ICCSA 2021"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-030-87016-4_46","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2021,9,11]],"date-time":"2021-09-11T06:12:59Z","timestamp":1631340779000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-3-030-87016-4_46"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2021]]},"ISBN":["9783030870157","9783030870164"],"references-count":35,"URL":"https:\/\/doi.org\/10.1007\/978-3-030-87016-4_46","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"type":"print","value":"0302-9743"},{"type":"electronic","value":"1611-3349"}],"subject":[],"published":{"date-parts":[[2021]]},"assertion":[{"value":"12 September 2021","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"ICCSA","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"International Conference on Computational Science and Its Applications","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Cagliari","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Italy","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2021","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"13 September 2021","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"16 September 2021","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"21","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"iccsa2021","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"https:\/\/iccsa.org\/","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Single-blind","order":1,"name":"type","label":"Type","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"Customed version of CyberChair 4","order":2,"name":"conference_management_system","label":"Conference Management System","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"1588","order":3,"name":"number_of_submissions_sent_for_review","label":"Number of Submissions Sent for Review","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"466","order":4,"name":"number_of_full_papers_accepted","label":"Number of Full Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"18","order":5,"name":"number_of_short_papers_accepted","label":"Number of Short Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"29% - The value is computed by the equation \"Number of Full Papers Accepted \/ Number of Submissions Sent for Review * 100\" and then rounded to a whole number.","order":6,"name":"acceptance_rate_of_full_papers","label":"Acceptance Rate of Full Papers","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"2,5","order":7,"name":"average_number_of_reviews_per_paper","label":"Average Number of Reviews per Paper","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"8","order":8,"name":"average_number_of_papers_per_reviewer","label":"Average Number of Papers per Reviewer","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"Yes","order":9,"name":"external_reviewers_involved","label":"External Reviewers Involved","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}}]}}