{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,27]],"date-time":"2026-07-27T11:18:41Z","timestamp":1785151121842,"version":"3.55.0"},"reference-count":117,"publisher":"Association for Computing Machinery (ACM)","issue":"7","license":[{"start":{"date-parts":[[2022,12,15]],"date-time":"2022-12-15T00:00:00Z","timestamp":1671062400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"name":"Brazilian Federal Agency for Post-Graduation Education","award":["887.388306\/2019-00"],"award-info":[{"award-number":["887.388306\/2019-00"]}]},{"name":"Brazilian National Council of Scientific and Technological Development","award":["315180\/2018-8 and 309030\/2019-6"],"award-info":[{"award-number":["315180\/2018-8 and 309030\/2019-6"]}]},{"DOI":"10.13039\/501100001807","name":"S\u00e3o Paulo Research Foundation","doi-asserted-by":"crossref","award":["2016.26290-3"],"award-info":[{"award-number":["2016.26290-3"]}],"id":[{"id":"10.13039\/501100001807","id-type":"DOI","asserted-by":"crossref"}]},{"name":"National Institute of Science and Technology - Medicine Assisted by Scientific Computing","award":["2014\/50889-7"],"award-info":[{"award-number":["2014\/50889-7"]}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":["ACM Comput. Surv."],"published-print":{"date-parts":[[2023,7,31]]},"abstract":"<jats:p>This study presents a literature systematic review of automatic performance assessment in three-dimensional interactive medical and dental simulators with haptic feedback, resulting in 63 included articles. The main contributions regard analysis and discussion of investigated procedures, extracted metrics, experiment types, and assessment techniques. Studies have mostly focused on assessing performance by analyzing metrics using statistical techniques, while machine learning algorithms appear to be underexplored. Metrics related to time were observed in 84% of the studies, and aspects related to force, error and precision were investigated to a lesser degree. Difficulties reported in the articles are discussed, and research opportunities are presented.<\/jats:p>","DOI":"10.1145\/3539222","type":"journal-article","created":{"date-parts":[[2022,6,8]],"date-time":"2022-06-08T09:56:09Z","timestamp":1654682169000},"page":"1-35","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":5,"title":["Automatic Performance Assessment in Three-dimensional Interactive Haptic Medical Simulators: A Systematic Review"],"prefix":"10.1145","volume":"55","author":[{"ORCID":"https:\/\/orcid.org\/0000-0001-8893-5983","authenticated-orcid":false,"given":"Lucas H.","family":"Sallaberry","sequence":"first","affiliation":[{"name":"Escola Polit\u00e9cnica, University of S\u00e3o Paulo, S\u00e3o Paulo, Brazil"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-9381-9565","authenticated-orcid":false,"given":"Romero","family":"Tori","sequence":"additional","affiliation":[{"name":"Escola Polit\u00e9cnica, University of S\u00e3o Paulo, S\u00e3o Paulo, Brazil"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-0040-0752","authenticated-orcid":false,"given":"F\u00e1tima L. S.","family":"Nunes","sequence":"additional","affiliation":[{"name":"School of Arts, Sciences and Humanities, University of S\u00e3o Paulo, S\u00e3o Paulo, Brazil"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2022,12,15]]},"reference":[{"key":"e_1_3_1_2_2","unstructured":"IEEE. 2021. IEEE Xplore. Retrieved 12 January 2021 from https:\/\/ieeexplore.ieee.org\/Xplore\/home.jsp."},{"key":"e_1_3_1_3_2","unstructured":"National Library of Medicine. 2021. PubMed. Retrieved 12 January 2021 from https:\/\/pubmed.ncbi.nlm.nih.gov\/."},{"key":"e_1_3_1_4_2","unstructured":"Elsevier. 2021. Scopus. Retrieved 30 December 2020 from https:\/\/www.elsevier.com\/solutions\/scopus."},{"key":"e_1_3_1_5_2","unstructured":"Association for Computing Machinery. 2021. ACM Digital Library. Retrieved 12 January 2021 from https:\/\/dl.acm.org\/."},{"key":"e_1_3_1_6_2","doi-asserted-by":"publisher","DOI":"10.3109\/17453674.2015.1041083"},{"key":"e_1_3_1_7_2","doi-asserted-by":"publisher","DOI":"10.1038\/nrurol.2016.147"},{"key":"e_1_3_1_8_2","doi-asserted-by":"publisher","DOI":"10.3171\/2016.7.JNS16322"},{"key":"e_1_3_1_9_2","doi-asserted-by":"publisher","DOI":"10.3171\/JNS-07\/09\/0515"},{"key":"e_1_3_1_10_2","doi-asserted-by":"publisher","DOI":"10.1016\/j.gie.2009.08.033"},{"key":"e_1_3_1_11_2","doi-asserted-by":"publisher","DOI":"10.1007\/s00464-008-0240-2"},{"key":"e_1_3_1_12_2","doi-asserted-by":"publisher","DOI":"10.1007\/s00167-019-05526-y"},{"key":"e_1_3_1_13_2","doi-asserted-by":"publisher","DOI":"10.1111\/j.1464-410X.2009.09193.x"},{"key":"e_1_3_1_14_2","doi-asserted-by":"publisher","DOI":"10.1016\/j.jsurg.2018.06.012"},{"key":"e_1_3_1_15_2","doi-asserted-by":"publisher","DOI":"10.1007\/s00464-014-3764-7"},{"key":"e_1_3_1_16_2","doi-asserted-by":"publisher","DOI":"10.1097\/SIH.0000000000000352"},{"key":"e_1_3_1_17_2","doi-asserted-by":"publisher","DOI":"10.1109\/TOH.2010.19"},{"key":"e_1_3_1_18_2","doi-asserted-by":"publisher","DOI":"10.1002\/jdd.12503"},{"key":"e_1_3_1_19_2","doi-asserted-by":"publisher","DOI":"10.1097\/ACM.0b013e31828ffdcf"},{"key":"e_1_3_1_20_2","doi-asserted-by":"publisher","DOI":"10.1590\/1678-7757-2016-0386"},{"key":"e_1_3_1_21_2","doi-asserted-by":"publisher","DOI":"10.1016\/j.medengphy.2018.11.002."},{"key":"e_1_3_1_22_2","doi-asserted-by":"publisher","DOI":"10.1162\/PRES_a_00054"},{"key":"e_1_3_1_23_2","doi-asserted-by":"publisher","DOI":"10.1109\/CBMS.2016.74"},{"key":"e_1_3_1_24_2","doi-asserted-by":"publisher","DOI":"10.1109\/HAPTIC.2010.5444607"},{"key":"e_1_3_1_25_2","doi-asserted-by":"publisher","DOI":"10.1097\/ACM.0000000000002414"},{"key":"e_1_3_1_26_2","doi-asserted-by":"publisher","DOI":"10.1136\/gutjnl-2014-307173"},{"key":"e_1_3_1_27_2","doi-asserted-by":"publisher","DOI":"10.1186\/1471-2288-14-91"},{"key":"e_1_3_1_28_2","doi-asserted-by":"publisher","DOI":"10.3390\/app10175752"},{"key":"e_1_3_1_29_2","doi-asserted-by":"publisher","DOI":"10.1007\/s10916-016-0459-8"},{"key":"e_1_3_1_30_2","doi-asserted-by":"publisher","DOI":"10.1109\/ROMAN.2007.4415242"},{"key":"e_1_3_1_31_2","doi-asserted-by":"publisher","DOI":"10.1002\/j.0022-0337.2014.78.4.tb05715.x"},{"key":"e_1_3_1_32_2","doi-asserted-by":"publisher","DOI":"10.1016\/j.cmpb.2013.11.005"},{"key":"e_1_3_1_33_2","doi-asserted-by":"publisher","DOI":"10.1016\/j.juro.2008.06.042"},{"key":"e_1_3_1_34_2","doi-asserted-by":"publisher","DOI":"10.1089\/end.2013.0454"},{"key":"e_1_3_1_35_2","doi-asserted-by":"publisher","DOI":"10.1007\/s00167-014-2888-6"},{"key":"e_1_3_1_36_2","doi-asserted-by":"publisher","DOI":"10.1007\/s11548-013-0905-8"},{"key":"e_1_3_1_37_2","doi-asserted-by":"publisher","DOI":"10.2147\/JMDH.S92695"},{"key":"e_1_3_1_38_2","doi-asserted-by":"publisher","DOI":"10.1002\/bjs.8819"},{"key":"e_1_3_1_39_2","doi-asserted-by":"publisher","DOI":"10.1007\/s00500-019-04341-4"},{"key":"e_1_3_1_40_2","doi-asserted-by":"publisher","DOI":"10.1080\/13645706.2018.1539012"},{"key":"e_1_3_1_41_2","doi-asserted-by":"publisher","DOI":"10.1055\/s-0029-1215193"},{"key":"e_1_3_1_42_2","doi-asserted-by":"publisher","DOI":"10.1097\/MAO.0000000000001867"},{"key":"e_1_3_1_43_2","volume-title":"The VR Book: Human-centered Design for Virtual Reality (1st ed.)","author":"Jerald Jason","year":"2016","unstructured":"Jason Jerald. 2016. The VR Book: Human-centered Design for Virtual Reality (1st ed.).Association for Computing Machinery, New York."},{"key":"e_1_3_1_44_2","doi-asserted-by":"publisher","DOI":"10.1109\/TMI.2008.917246"},{"key":"e_1_3_1_45_2","doi-asserted-by":"publisher","DOI":"10.1109\/HAPTIC.2012.6183795"},{"key":"e_1_3_1_46_2","first-page":"192","article-title":"Evaluation of haptic teaching approaches for laparoscopic surgery training","volume":"196","author":"Kato Toma","year":"2014","unstructured":"Toma Kato, Kazuyoshi Tagawa, Takafumi Marutani, Hiromi Tanaka, Masaru Komori, Yoshimasa Kurumi, and Shigehiro Morikawa. 2014. Evaluation of haptic teaching approaches for laparoscopic surgery training. Stud. Health Technol. Inform. 196 (2014), 192\u2013196.","journal-title":"Stud. Health Technol. Inform."},{"key":"e_1_3_1_47_2","doi-asserted-by":"publisher","DOI":"10.1177\/1553350616672971"},{"key":"e_1_3_1_48_2","doi-asserted-by":"publisher","DOI":"10.1023\/B:JOMS.0000021522.30587.ff"},{"key":"e_1_3_1_49_2","doi-asserted-by":"publisher","DOI":"10.2106\/JBJS.K.01284"},{"key":"e_1_3_1_50_2","doi-asserted-by":"publisher","DOI":"10.1016\/j.jsurg.2019.08.008"},{"key":"e_1_3_1_51_2","doi-asserted-by":"publisher","DOI":"10.1109\/ROBIO.2007.4522233"},{"key":"e_1_3_1_52_2","doi-asserted-by":"publisher","DOI":"10.1016\/j.jbi.2013.12.010"},{"key":"e_1_3_1_53_2","doi-asserted-by":"publisher","DOI":"10.1243\/095441104323118860"},{"key":"e_1_3_1_54_2","doi-asserted-by":"publisher","DOI":"10.1097\/01.blo.0000194678.10130.ff"},{"key":"e_1_3_1_55_2","doi-asserted-by":"publisher","DOI":"10.1089\/end.2007.9985"},{"key":"e_1_3_1_56_2","doi-asserted-by":"publisher","DOI":"10.1111\/eje.12245"},{"key":"e_1_3_1_57_2","doi-asserted-by":"publisher","DOI":"10.1109\/IROS40897.2019.8967712"},{"key":"e_1_3_1_58_2","volume-title":"Medical Simulation in Medical Education: Results of an AAMC Survey","author":"Passiment Morgan","year":"2011","unstructured":"Morgan Passiment, Heather Sacks, and Grace Huang. 2011. Medical Simulation in Medical Education: Results of an AAMC Survey. Association of American Medical Colleges."},{"key":"e_1_3_1_59_2","doi-asserted-by":"publisher","DOI":"10.32614\/RJ-2016-058"},{"key":"e_1_3_1_60_2","doi-asserted-by":"publisher","DOI":"10.1109\/MCG.2006.140"},{"key":"e_1_3_1_61_2","doi-asserted-by":"publisher","DOI":"10.1109\/TBME.2020.3048516"},{"key":"e_1_3_1_62_2","doi-asserted-by":"publisher","DOI":"10.1016\/S1072-7515(99)00076-9"},{"key":"e_1_3_1_63_2","doi-asserted-by":"publisher","DOI":"10.1109\/ICALT.2018.00022"},{"key":"e_1_3_1_64_2","doi-asserted-by":"publisher","DOI":"10.1016\/j.ejogrb.2019.01.016"},{"key":"e_1_3_1_65_2","doi-asserted-by":"publisher","DOI":"10.1109\/HAPTICS.2008.4479944"},{"key":"e_1_3_1_66_2","doi-asserted-by":"publisher","DOI":"10.1016\/j.jsurg.2016.04.009"},{"key":"e_1_3_1_67_2","doi-asserted-by":"publisher","DOI":"10.1177\/1553350618773666"},{"key":"e_1_3_1_68_2","doi-asserted-by":"publisher","DOI":"10.1016\/j.jsurg.2020.10.004"},{"key":"e_1_3_1_69_2","doi-asserted-by":"publisher","DOI":"10.1007\/s10439-017-1791-y"},{"key":"e_1_3_1_70_2","doi-asserted-by":"publisher","DOI":"10.1007\/s00167-016-4022-4"},{"key":"e_1_3_1_71_2","doi-asserted-by":"publisher","DOI":"10.1016\/j.jsurg.2019.09.006"},{"key":"e_1_3_1_72_2","doi-asserted-by":"publisher","DOI":"10.1007\/s00464-010-1190-z"},{"key":"e_1_3_1_73_2","doi-asserted-by":"publisher","DOI":"10.1002\/rcs.386"},{"key":"e_1_3_1_74_2","doi-asserted-by":"publisher","DOI":"10.3414\/ME9310"},{"key":"e_1_3_1_75_2","doi-asserted-by":"publisher","DOI":"10.1016\/j.artmed.2011.04.003"},{"key":"e_1_3_1_76_2","doi-asserted-by":"publisher","DOI":"10.21815\/JDE.018.028"},{"key":"e_1_3_1_77_2","doi-asserted-by":"publisher","DOI":"10.1016\/j.jsurg.2015.04.030"},{"key":"e_1_3_1_78_2","doi-asserted-by":"publisher","DOI":"10.1109\/TBME.2005.869771"},{"key":"e_1_3_1_79_2","doi-asserted-by":"publisher","DOI":"10.1109\/10.918597"},{"key":"e_1_3_1_80_2","first-page":"279","article-title":"Hidden Markov models of minimally invasive surgery","volume":"70","author":"Rosen Jacob","year":"2000","unstructured":"Jacob Rosen, Christina Richards, Blake Hannaford, and Mika Sinanan. 2000. Hidden Markov models of minimally invasive surgery. Stud. Health Technol. Inform.-Med. Meets Virt. Real. 70 (2000), 279\u2013285.","journal-title":"Stud. Health Technol. Inform.-Med. Meets Virt. Real."},{"key":"e_1_3_1_81_2","doi-asserted-by":"publisher","DOI":"10.1002\/rcs.2039"},{"key":"e_1_3_1_82_2","doi-asserted-by":"publisher","DOI":"10.3109\/13645706.2013.827122"},{"key":"e_1_3_1_83_2","doi-asserted-by":"publisher","DOI":"10.1007\/s00464-010-1306-5"},{"key":"e_1_3_1_84_2","doi-asserted-by":"publisher","DOI":"10.1016\/j.jsurg.2016.04.006"},{"key":"e_1_3_1_85_2","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-20148-7_21"},{"key":"e_1_3_1_86_2","volume-title":"Motor Learning and Performance: From Principles to Application (5th ed.)","author":"Schmidt Richard A.","year":"2014","unstructured":"Richard A. Schmidt and Timothy Donald Lee. 2014. Motor Learning and Performance: From Principles to Application (5th ed.). Human Kinetics, Champaign, IL. BF295.S248 2014"},{"key":"e_1_3_1_87_2","doi-asserted-by":"publisher","DOI":"10.1097\/ACM.0000000000000056"},{"key":"e_1_3_1_88_2","doi-asserted-by":"publisher","DOI":"10.1227\/NEU.0000000000000853"},{"key":"e_1_3_1_89_2","doi-asserted-by":"publisher","DOI":"10.2147\/OPTH.S69970"},{"key":"e_1_3_1_90_2","doi-asserted-by":"publisher","DOI":"10.1016\/j.jsurg.2019.11.011"},{"key":"e_1_3_1_91_2","doi-asserted-by":"publisher","DOI":"10.1109\/CarpathianCC.2019.8765930"},{"key":"e_1_3_1_92_2","doi-asserted-by":"publisher","DOI":"10.1007\/s00464-002-9078-1"},{"key":"e_1_3_1_93_2","doi-asserted-by":"publisher","DOI":"10.1145\/3025171.3025217"},{"key":"e_1_3_1_94_2","doi-asserted-by":"publisher","DOI":"10.1002\/j.0022-0337.2010.74.12.tb05011.x"},{"key":"e_1_3_1_95_2","doi-asserted-by":"publisher","DOI":"10.1016\/j.jdent.2009.04.008"},{"key":"e_1_3_1_96_2","doi-asserted-by":"publisher","DOI":"10.1109\/SMC.2017.8123149"},{"key":"e_1_3_1_97_2","doi-asserted-by":"publisher","DOI":"10.1177\/1550147719840173"},{"key":"e_1_3_1_98_2","doi-asserted-by":"publisher","DOI":"10.1109\/SMC.2018.00511"},{"key":"e_1_3_1_99_2","doi-asserted-by":"publisher","DOI":"10.1016\/j.amjsurg.2009.04.015"},{"key":"e_1_3_1_100_2","first-page":"181","article-title":"A review of the role of simulation in developing and assessing orthopaedic surgical skills","volume":"34","author":"Thomas Geb W.","year":"2014","unstructured":"Geb W. Thomas, Brian D. Johns, J. Lawrence Marsh, and Donald D. Anderson. 2014. A review of the role of simulation in developing and assessing orthopaedic surgical skills. Iowa Orthopaed. J. 34 (2014), 181\u2013189.","journal-title":"Iowa Orthopaed. J."},{"key":"e_1_3_1_101_2","doi-asserted-by":"publisher","DOI":"10.1111\/aos.13505"},{"key":"e_1_3_1_102_2","doi-asserted-by":"publisher","DOI":"10.1016\/j.ophtha.2015.02.028"},{"key":"e_1_3_1_103_2","doi-asserted-by":"publisher","DOI":"10.1162\/pres.17.2.212"},{"key":"e_1_3_1_104_2","doi-asserted-by":"publisher","DOI":"10.1007\/s00464-008-0298-x"},{"key":"e_1_3_1_105_2","doi-asserted-by":"publisher","DOI":"10.1002\/bjs.7115"},{"key":"e_1_3_1_106_2","doi-asserted-by":"publisher","DOI":"10.1007\/s00464-012-2621-9"},{"key":"e_1_3_1_107_2","doi-asserted-by":"publisher","DOI":"10.1007\/s00464-012-2745-y"},{"key":"e_1_3_1_108_2","doi-asserted-by":"publisher","DOI":"10.1115\/DETC2016-59286"},{"key":"e_1_3_1_109_2","doi-asserted-by":"publisher","DOI":"10.1016\/j.engappai.2020.103760"},{"key":"e_1_3_1_110_2","doi-asserted-by":"publisher","DOI":"10.1109\/TOH.2011.59"},{"key":"e_1_3_1_111_2","doi-asserted-by":"publisher","DOI":"10.3233\/BME-151366"},{"key":"e_1_3_1_112_2","doi-asserted-by":"publisher","DOI":"10.1109\/ACCESS.2020.2982219"},{"key":"e_1_3_1_113_2","doi-asserted-by":"publisher","DOI":"10.2106\/JBJS.19.00505"},{"key":"e_1_3_1_114_2","doi-asserted-by":"publisher","DOI":"10.1109\/TITB.2012.2204406"},{"key":"e_1_3_1_115_2","doi-asserted-by":"publisher","DOI":"10.1016\/j.jsurg.2019.05.015"},{"key":"e_1_3_1_116_2","doi-asserted-by":"publisher","DOI":"10.1109\/VR46266.2020.00077"},{"key":"e_1_3_1_117_2","doi-asserted-by":"publisher","DOI":"10.1007\/s10846-019-01082-2"},{"key":"e_1_3_1_118_2","doi-asserted-by":"publisher","DOI":"10.1007\/s00464-011-2011-8"}],"container-title":["ACM Computing Surveys"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3539222","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3539222","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T17:51:29Z","timestamp":1750182689000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3539222"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2022,12,15]]},"references-count":117,"journal-issue":{"issue":"7","published-print":{"date-parts":[[2023,7,31]]}},"alternative-id":["10.1145\/3539222"],"URL":"https:\/\/doi.org\/10.1145\/3539222","relation":{},"ISSN":["0360-0300","1557-7341"],"issn-type":[{"value":"0360-0300","type":"print"},{"value":"1557-7341","type":"electronic"}],"subject":[],"published":{"date-parts":[[2022,12,15]]},"assertion":[{"value":"2021-05-27","order":0,"name":"received","label":"Received","group":{"name":"publication_history","label":"Publication History"}},{"value":"2022-05-06","order":1,"name":"accepted","label":"Accepted","group":{"name":"publication_history","label":"Publication History"}},{"value":"2022-12-15","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}