{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,11,7]],"date-time":"2025-11-07T09:22:57Z","timestamp":1762507377375,"version":"3.37.3"},"reference-count":33,"publisher":"Springer Science and Business Media LLC","issue":"7","license":[{"start":{"date-parts":[[2018,3,24]],"date-time":"2018-03-24T00:00:00Z","timestamp":1521849600000},"content-version":"tdm","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"}],"funder":[{"DOI":"10.13039\/100010477","name":"Intuitive Surgical","doi-asserted-by":"publisher","id":[{"id":"10.13039\/100010477","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Int J CARS"],"published-print":{"date-parts":[[2018,7]]},"DOI":"10.1007\/s11548-018-1738-2","type":"journal-article","created":{"date-parts":[[2018,3,24]],"date-time":"2018-03-24T02:42:48Z","timestamp":1521859368000},"page":"1037-1048","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":23,"title":["Meaningful Assessment of Robotic Surgical Style using the Wisdom of Crowds"],"prefix":"10.1007","volume":"13","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-0579-1409","authenticated-orcid":false,"given":"M.","family":"Ershad","sequence":"first","affiliation":[]},{"given":"R.","family":"Rege","sequence":"additional","affiliation":[]},{"given":"A. Majewicz","family":"Fey","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2018,3,24]]},"reference":[{"issue":"3","key":"1738_CR1","first-page":"72","volume":"7","author":"B Morris","year":"2005","unstructured":"Morris B (2005) Robotic surgery: applications, limitations, and impact on surgical education. Medscape Gen Med 7(3):72","journal-title":"Medscape Gen Med"},{"issue":"1","key":"1738_CR2","doi-asserted-by":"publisher","first-page":"21","DOI":"10.1016\/S0039-6060(03)00156-9","volume":"135","author":"JH Peters","year":"2004","unstructured":"Peters JH, Fried GM, Swanstrom LL, Soper NJ, Sillin LF, Schirmer B, Hoffman K, Committee SF (2004) Development and validation of a comprehensive program of education and assessment of the basic fundamentals of laparoscopic surgery. Surgery 135(1):21\u201327","journal-title":"Surgery"},{"issue":"3","key":"1738_CR3","doi-asserted-by":"publisher","first-page":"535","DOI":"10.1016\/j.suc.2010.02.012","volume":"90","author":"MC Vassiliou","year":"2010","unstructured":"Vassiliou MC, Dunkin BJ, Marks JM, Fried GM (2010) FLS and FES: comprehensive models of training and assessment. Surg Clin N Am 90(3):535\u2013558","journal-title":"Surg Clin N Am"},{"issue":"2","key":"1738_CR4","doi-asserted-by":"publisher","first-page":"273","DOI":"10.1002\/bjs.1800840237","volume":"84","author":"JA Martin","year":"1997","unstructured":"Martin JA, Regehr G, Reznick R, Macrae H, Murnaghan J, Hutchison C, Brown M (1997) Objective structured assessment of technical skill (OSATS) for surgical residents. Br J Surg 84(2):273\u2013278. \n                    https:\/\/doi.org\/10.1002\/bjs.1800840237","journal-title":"Br J Surg"},{"issue":"1","key":"1738_CR5","doi-asserted-by":"publisher","first-page":"107","DOI":"10.1016\/j.amjsurg.2005.04.004","volume":"190","author":"MC Vassiliou","year":"2005","unstructured":"Vassiliou MC, Feldman LS, Andrew CG, Bergman S, Leffondr\u00e9 K, Stanbridge D, Fried GM (2005) A global assessment tool for evaluation of intraoperative laparoscopic skills. Am J Surg 190(1):107\u2013113. \n                    https:\/\/doi.org\/10.1016\/j.amjsurg.2005.04.004","journal-title":"Am J Surg"},{"issue":"1","key":"1738_CR6","doi-asserted-by":"publisher","first-page":"247","DOI":"10.1016\/j.juro.2011.09.032","volume":"187","author":"AC Goh","year":"2012","unstructured":"Goh AC, Goldfarb DW, Sander JC, Miles BJ, Dunkin BJ (2012) Global evaluative assessment of robotic skills: validation of a clinical assessment tool to measure robotic surgical skills. J Urol 187(1):247\u2013252. \n                    https:\/\/doi.org\/10.1016\/j.juro.2011.09.032","journal-title":"J Urol"},{"key":"1738_CR7","doi-asserted-by":"crossref","unstructured":"Jain S, Barsness KA, Argall B (2015) Automated and objective assessment of surgical training: detection of procedural steps on videotaped performances. In: International conference on digital image computing: techniques and applications (DICTA), IEEE, pp 1\u20136","DOI":"10.1109\/DICTA.2015.7371233"},{"issue":"2","key":"1738_CR8","doi-asserted-by":"publisher","first-page":"364","DOI":"10.1097\/01.sla.0000151982.85062.80","volume":"241","author":"AG Gallagher","year":"2005","unstructured":"Gallagher AG, Ritter EM, Champion H, Higgins G, Fried MP, Moses G, Smith CD, Satava RM (2005) Virtual reality simulation for the operating room: proficiency-based training as a paradigm shift in surgical skills training. Ann Surg 241(2):364\u2013372","journal-title":"Ann Surg"},{"issue":"3","key":"1738_CR9","doi-asserted-by":"publisher","first-page":"335","DOI":"10.1016\/j.arthro.2007.08.033","volume":"24","author":"NR Howells","year":"2008","unstructured":"Howells NR, Brinsden MD, Gill RS, Carr AJ, Rees JL (2008) Motion analysis: a validated method for showing skill levels in arthroscopy. Arthroscopy 24(3):335\u2013342. \n                    https:\/\/doi.org\/10.1016\/j.arthro.2007.08.033","journal-title":"Arthroscopy"},{"issue":"3","key":"1738_CR10","doi-asserted-by":"publisher","first-page":"295","DOI":"10.1007\/978-3-642-15711-0_37","volume":"6363","author":"N Ahmidi","year":"2010","unstructured":"Ahmidi N, Hager GD, Ishii L, Fichtinger G, Gallia GL, Ishii M (2010) Surgical task and skill classification from eye tracking and tool motion in minimally invasive surgery. Lect Notes Comput Sci 6363(3):295\u2013302","journal-title":"Lect Notes Comput Sci"},{"issue":"1","key":"1738_CR11","doi-asserted-by":"publisher","first-page":"65","DOI":"10.1016\/j.jss.2013.09.024","volume":"187","author":"C Chen","year":"2014","unstructured":"Chen C, White L, Kowalewski T, Aggarwal R, Lintott C, Comstock B, Kuksenok K, Aragon C, Holst D, Lendvay T (2014) Crowd-sourced assessment of technical skills: a novel method to evaluate surgical performance. J Surg Res 187(1):65\u201371","journal-title":"J Surg Res"},{"issue":"1","key":"1738_CR12","doi-asserted-by":"publisher","first-page":"e81","DOI":"10.1016\/j.jss.2011.06.034","volume":"171","author":"I Oropesa","year":"2011","unstructured":"Oropesa I, S\u00e1nchez-Gonz\u00e1lez P, Lamata P, Chmarra MK, Pagador JB, S\u00e1nchez-Margallo JA, S\u00e1nchez-Margallo FM, G\u00f3mez EJ (2011) Methods and tools for objective assessment of psychomotor skills in laparoscopic surgery. J Surg Res 171(1):e81\u2013e95","journal-title":"J Surg Res"},{"issue":"1","key":"1738_CR13","doi-asserted-by":"publisher","first-page":"81","DOI":"10.1016\/j.amjsurg.2009.07.033","volume":"199","author":"ED Grober","year":"2010","unstructured":"Grober ED, Roberts M, Shin EJ, Mahdi M, Bacal V (2010) Intraoperative assessment of technical skills on live patients using economy of hand motion: establishing learning curves of surgical competence. Am J Surg 199(1):81\u201385. \n                    https:\/\/doi.org\/10.1016\/j.amjsurg.2009.07.033","journal-title":"Am J Surg"},{"key":"1738_CR14","doi-asserted-by":"crossref","unstructured":"Lea C, Hager GD, Vidal R (2015) An improved model for segmentation and recognition of fine-grained activities with application to surgical training tasks. In: 2015 IEEE winter conference on applications of computer vision (WACV), pp 1123\u20131129","DOI":"10.1109\/WACV.2015.154"},{"key":"1738_CR15","doi-asserted-by":"crossref","unstructured":"DiPietro R, Lea C, Malpani A, Ahmidi N, Vedula SS, Lee GI, Lee MR, Hager GD (2016) Recognizing surgical activities with recurrent neural networks. In: International conference on medical image computing and computer-assisted intervention. Springer, New York, pp 551\u2013558","DOI":"10.1007\/978-3-319-46720-7_64"},{"issue":"11","key":"1738_CR16","doi-asserted-by":"publisher","first-page":"1295","DOI":"10.1089\/end.2015.0191","volume":"29","author":"LW White","year":"2015","unstructured":"White LW, Kowalewski TM, Dockter RL, Comstock B, Hannaford B, Lendvay TS (2015) Crowd-sourced assessment of technical skill: a valid method for discriminating basic robotic surgery skills. J Endourol 29(11):1295\u20131301","journal-title":"J Endourol"},{"issue":"6","key":"1738_CR17","doi-asserted-by":"publisher","first-page":"1859","DOI":"10.1016\/j.juro.2016.01.005","volume":"195","author":"TM Kowalewski","year":"2016","unstructured":"Kowalewski TM, Comstock B, Sweet R, Schaffhausen C, Menhadji A, Averch T, Box G, Brand T, Ferrandino M, Kaouk J (2016) Crowd-sourced assessment of technical skills for validation of basic laparoscopic urologic skills tasks. J Urol 195(6):1859\u20131865","journal-title":"J Urol"},{"issue":"9","key":"1738_CR18","doi-asserted-by":"publisher","first-page":"1435","DOI":"10.1007\/s11548-015-1238-6","volume":"10","author":"A Malpani","year":"2015","unstructured":"Malpani A, Vedula SS, Chen CCG, Hager GD (2015) A study of crowdsourced segment-level surgical skill assessment using pairwise rankings. Int J Comput Assist Radiol Surg 10(9):1435\u20131447. \n                    https:\/\/doi.org\/10.1007\/s11548-015-1238-6","journal-title":"Int J Comput Assist Radiol Surg"},{"issue":"1","key":"1738_CR19","doi-asserted-by":"publisher","first-page":"90","DOI":"10.1001\/jamasurg.2015.3121","volume":"151","author":"SP Chen","year":"2016","unstructured":"Chen SP, Kirsch S, Zlatev DV, Chang T, Comstock B, Lendvay TS, Liao JC (2016) Optical biopsy of bladder cancer using crowd-sourced assessment. JAMA Surg 151(1):90\u201393","journal-title":"JAMA Surg"},{"issue":"1","key":"1738_CR20","doi-asserted-by":"publisher","first-page":"105","DOI":"10.1016\/j.amjsurg.2006.03.016","volume":"193","author":"MC Porte","year":"2007","unstructured":"Porte MC, Xeroulis G, Reznick RK, Dubrowski A (2007) Verbal feedback from an expert is more effective than self-accessed feedback about motion efficiency in learning new surgical skills. Am J Surg 193(1):105\u2013110. \n                    https:\/\/doi.org\/10.1016\/j.amjsurg.2006.03.016","journal-title":"Am J Surg"},{"key":"1738_CR21","doi-asserted-by":"crossref","unstructured":"Ershad M, Koesters Z, Rege R, Majewicz A (2016) Meaningful assessment of surgical expertise: semantic labeling with data and crowds. In: International conference on medical image computing and computer-assisted intervention (MICCAI), Springer, New York, pp 508\u2013515","DOI":"10.1007\/978-3-319-46720-7_59"},{"key":"1738_CR22","unstructured":"Quigley M, Conley K, Gerkey BP, Faust J, Foote T, Leibs J, Wheeler R, Ng AY (2009) ROS: an open-source robot operating system. In: ICRA workshop on open source software"},{"key":"1738_CR23","doi-asserted-by":"publisher","unstructured":"Moorthy K, Munz Y (2003) Objective assessment of technical skills in surgery. Br Med J 327(7422):1032\u20131037. \n                    https:\/\/doi.org\/10.1136\/bmj.327.7422.1032","DOI":"10.1136\/bmj.327.7422.1032"},{"issue":"5","key":"1738_CR24","doi-asserted-by":"publisher","first-page":"479","DOI":"10.1016\/S1072-7515(01)01041-9","volume":"193","author":"V Datta","year":"2001","unstructured":"Datta V, Mackay S, Mandalia M, Darzi A (2001) The use of electromagnetic motion tracking analysis to objectively measure open surgical skill in the laboratory-based model. J Am Coll Surg 193(5):479\u2013485","journal-title":"J Am Coll Surg"},{"key":"1738_CR25","doi-asserted-by":"publisher","unstructured":"Nisky I, Hsieh MH, Okamura AM (2013) A framework for analysis of surgeon arm posture variability in robot-assisted surgery. In: IEEE international conference on robotics and automation. pp 245\u2013251. \n                    https:\/\/doi.org\/10.1109\/ICRA.2013.6630583","DOI":"10.1109\/ICRA.2013.6630583"},{"issue":"1","key":"1738_CR26","doi-asserted-by":"publisher","first-page":"112","DOI":"10.1186\/s12984-015-0090-9","volume":"12","author":"S Balasubramanian","year":"2015","unstructured":"Balasubramanian S, Melendez-Calderon A, Roby-Brami A, Burdet E (2015) On the analysis of movement smoothness. J NeuroEng Rehabil 12(1):112. \n                    https:\/\/doi.org\/10.1186\/s12984-015-0090-9","journal-title":"J NeuroEng Rehabil"},{"issue":"6","key":"1738_CR27","doi-asserted-by":"publisher","first-page":"269","DOI":"10.3109\/10929088.2012.727641","volume":"17","author":"F Anderson","year":"2012","unstructured":"Anderson F, Birch DW, Boulanger P, Bischof WF (2012) Sensor fusion for laparoscopic surgery skill acquisition. J Int Soc Comput Aided Surg 17(6):269\u201383. \n                    https:\/\/doi.org\/10.3109\/10929088.2012.727641","journal-title":"J Int Soc Comput Aided Surg"},{"issue":"12","key":"1738_CR28","doi-asserted-by":"publisher","first-page":"3192","DOI":"10.1016\/j.jbiomech.2015.07.002","volume":"48","author":"R Postacchini","year":"2015","unstructured":"Postacchini R, Paoloni M, Carbone S, Fini M, Santilli V, Postacchini F, Mangone M (2015) Kinematic analysis of reaching movements of the upper limb after total or reverse shoulder arthroplasty. J Biomech 48(12):3192\u20133198. \n                    https:\/\/doi.org\/10.1016\/j.jbiomech.2015.07.002","journal-title":"J Biomech"},{"issue":"12","key":"1738_CR29","doi-asserted-by":"publisher","first-page":"2869","DOI":"10.1109\/TBME.2014.2332359","volume":"61","author":"I Nisky","year":"2014","unstructured":"Nisky I, Hsieh MH, Okamura AM (2014) Uncontrolled manifold analysis of arm joint angle variability during robotic teleoperation and freehand movement of surgeons and novices. IEEE Trans Biomed Eng 61(12):2869\u20132881","journal-title":"IEEE Trans Biomed Eng"},{"key":"1738_CR30","doi-asserted-by":"publisher","DOI":"10.1002\/0471656372","volume-title":"Introduction to digital signal processing and filter design","author":"BA Shenoi","year":"2005","unstructured":"Shenoi BA (2005) Introduction to digital signal processing and filter design. Wiley, New York"},{"key":"1738_CR31","doi-asserted-by":"crossref","unstructured":"Halaki M, Ginn KA (2012) Normalization of EMG signals: to normalize or not to normalize and what to normalize to? In: Computational intelligence in electromyography analysis\u2014a perspective on current applications and future challenges, pp 175\u2013194, 40113","DOI":"10.5772\/49957"},{"issue":"8","key":"1738_CR32","doi-asserted-by":"publisher","first-page":"1573","DOI":"10.1016\/j.jbiomech.2010.01.027","volume":"43","author":"CJ Luca De","year":"2010","unstructured":"De Luca CJ, Gilmore LD, Kuznetsov M, Roy SH (2010) Filtering the surface EMG signal: movement artifact and baseline noise contamination. J Biomech 43(8):1573\u20131579","journal-title":"J Biomech"},{"issue":"1","key":"1738_CR33","doi-asserted-by":"publisher","first-page":"80","DOI":"10.1016\/j.jneumeth.2010.04.028","volume":"190","author":"M Benedek","year":"2010","unstructured":"Benedek M, Kaernbach C (2010) A continuous measure of phasic electrodermal activity. J Neurosci Methods 190(1):80\u201391. \n                    https:\/\/doi.org\/10.1016\/j.jneumeth.2010.04.028","journal-title":"J Neurosci Methods"}],"container-title":["International Journal of Computer Assisted Radiology and Surgery"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/link.springer.com\/article\/10.1007\/s11548-018-1738-2\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s11548-018-1738-2.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s11548-018-1738-2.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2019,3,23]],"date-time":"2019-03-23T20:07:04Z","timestamp":1553371624000},"score":1,"resource":{"primary":{"URL":"http:\/\/link.springer.com\/10.1007\/s11548-018-1738-2"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2018,3,24]]},"references-count":33,"journal-issue":{"issue":"7","published-print":{"date-parts":[[2018,7]]}},"alternative-id":["1738"],"URL":"https:\/\/doi.org\/10.1007\/s11548-018-1738-2","relation":{},"ISSN":["1861-6410","1861-6429"],"issn-type":[{"type":"print","value":"1861-6410"},{"type":"electronic","value":"1861-6429"}],"subject":[],"published":{"date-parts":[[2018,3,24]]},"assertion":[{"value":"7 November 2017","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"15 March 2018","order":2,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"24 March 2018","order":3,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Compliance with ethical standards"}},{"value":"The authors declared that they have no conflict of interest.","order":2,"name":"Ethics","group":{"name":"EthicsHeading","label":"Conflict of interest"}},{"value":"All procedures performed in studies involving human participants were in accordance with the ethical standards of the institutional and\/or national research committee and with the 1964 Declaration of Helsinki and its later amendments or comparable ethical standards. The study protocol was approved by both UTD and UTSW IRB offices (UTD #14-57, UTSW #STU 032015-053).","order":3,"name":"Ethics","group":{"name":"EthicsHeading","label":"Ethical approval"}},{"value":"Informed consent was obtained from all individual participants included in the study.","order":4,"name":"Ethics","group":{"name":"EthicsHeading","label":"Informed consent"}}]}}