{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,5,6]],"date-time":"2026-05-06T15:12:05Z","timestamp":1778080325155,"version":"3.51.4"},"reference-count":22,"publisher":"Springer Science and Business Media LLC","issue":"6","license":[{"start":{"date-parts":[[2024,3,25]],"date-time":"2024-03-25T00:00:00Z","timestamp":1711324800000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0"},{"start":{"date-parts":[[2024,3,25]],"date-time":"2024-03-25T00:00:00Z","timestamp":1711324800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0"}],"funder":[{"name":"Stiftung Innovation in der Hochschullehre","award":["FBM2020-EA-1560-00371"],"award-info":[{"award-number":["FBM2020-EA-1560-00371"]}]},{"DOI":"10.13039\/501100009398","name":"Medizinische Fakult\u00e4t, RWTH Aachen University","doi-asserted-by":"publisher","id":[{"id":"10.13039\/501100009398","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100001659","name":"Deutsche Forschungsgemeinschaft","doi-asserted-by":"publisher","award":["390696704"],"award-info":[{"award-number":["390696704"]}],"id":[{"id":"10.13039\/501100001659","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100002347","name":"Bundesministerium f\u00fcr Bildung und Forschung","doi-asserted-by":"publisher","award":["57616814"],"award-info":[{"award-number":["57616814"]}],"id":[{"id":"10.13039\/501100002347","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100002347","name":"Bundesministerium f\u00fcr Bildung und Forschung","doi-asserted-by":"publisher","award":["16KISK001K"],"award-info":[{"award-number":["16KISK001K"]}],"id":[{"id":"10.13039\/501100002347","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Int J CARS"],"abstract":"<jats:title>Abstract<\/jats:title><jats:sec>\n                <jats:title>Purpose<\/jats:title>\n                <jats:p>Efficient and precise surgical skills are essential in ensuring positive patient outcomes. By continuously providing real-time, data driven, and objective evaluation of surgical performance, automated skill assessment has the potential to greatly improve surgical skill training. Whereas machine learning-based surgical skill assessment is gaining traction for minimally invasive techniques, this cannot be said for open surgery skills. Open surgery generally has more degrees of freedom when compared to minimally invasive surgery, making it more difficult to interpret. In this paper, we present novel approaches for skill assessment for open surgery skills.\n<\/jats:p>\n              <\/jats:sec><jats:sec>\n                <jats:title>Methods<\/jats:title>\n                <jats:p>We analyzed a novel video dataset for open suturing training. We provide a detailed analysis of the dataset and define evaluation guidelines, using state of the art deep learning models. Furthermore, we present novel benchmarking results for surgical skill assessment in open suturing. The models are trained to classify a video into three skill levels based on the global rating score. To obtain initial results for video-based surgical skill classification, we benchmarked a temporal segment network with both an I3D and a Video Swin backbone on this dataset.<\/jats:p>\n              <\/jats:sec><jats:sec>\n                <jats:title>Results<\/jats:title>\n                <jats:p>The dataset is composed of 314 videos of approximately five minutes each. Model benchmarking results are an accuracy and F1 score of up to 75 and 72%, respectively. This is similar to the performance achieved by the individual raters, regarding inter-rater agreement and rater variability. We present the first end-to-end trained approach for skill assessment for open surgery training.<\/jats:p>\n              <\/jats:sec><jats:sec>\n                <jats:title>Conclusion<\/jats:title>\n                <jats:p>We provide a thorough analysis of a new dataset as well as novel benchmarking results for surgical skill assessment. This opens the doors to new advances in skill assessment by enabling video-based skill assessment for classic surgical techniques with the potential to improve the surgical outcome of patients.<\/jats:p>\n              <\/jats:sec>","DOI":"10.1007\/s11548-024-03093-3","type":"journal-article","created":{"date-parts":[[2024,3,25]],"date-time":"2024-03-25T13:02:11Z","timestamp":1711371731000},"page":"1045-1052","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":16,"title":["AIxSuture: vision-based assessment of open suturing skills"],"prefix":"10.1007","volume":"19","author":[{"ORCID":"https:\/\/orcid.org\/0009-0008-4917-7911","authenticated-orcid":false,"given":"Hanna","family":"Hoffmann","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Isabel","family":"Funke","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Philipp","family":"Peters","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Danush Kumar","family":"Venkatesh","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Jan","family":"Egger","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Dominik","family":"Rivoir","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Rainer","family":"R\u00f6hrig","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Frank","family":"H\u00f6lzle","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Sebastian","family":"Bodenstedt","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Marie-Christin","family":"Willemer","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Stefanie","family":"Speidel","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Behrus","family":"Puladi","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2024,3,25]]},"reference":[{"key":"3093_CR1","doi-asserted-by":"publisher","first-page":"1434","DOI":"10.1056\/NEJMSA1300625","volume":"369","author":"JD Birkmeyer","year":"2013","unstructured":"Birkmeyer JD, Finks JF, O\u2019Reilly A, Oerline M et al (2013) Surgical skill and complication rates after bariatric surgery. N Engl J Med 369:1434\u20131442. https:\/\/doi.org\/10.1056\/NEJMSA1300625\/SUPPL_FILE\/NEJMSA1300625_DISCLOSURES.PDF","journal-title":"N Engl J Med"},{"key":"3093_CR2","doi-asserted-by":"publisher","first-page":"469","DOI":"10.1016\/J.AMJSURG.2010.10.020","volume":"202","author":"K Ahmed","year":"2011","unstructured":"Ahmed K, Miskovic D, Darzi A, Athanasiou T, Hanna GB (2011) Observational tools for assessment of procedural skills: a systematic review. Am J Surg 202:469\u20134806. https:\/\/doi.org\/10.1016\/J.AMJSURG.2010.10.020","journal-title":"Am J Surg"},{"key":"3093_CR3","doi-asserted-by":"publisher","first-page":"437","DOI":"10.1007\/S11548-022-02559-6\/FIGURES\/5","volume":"17","author":"A Goldbraikh","year":"2022","unstructured":"Goldbraikh A, D\u2019Angelo AL, Pugh CM, Laufer S (2022) Video-based fully automatic assessment of open surgery suturing skills. Int J Comput Assist Radiol Surg 17:437\u2013448. https:\/\/doi.org\/10.1007\/S11548-022-02559-6\/FIGURES\/5","journal-title":"Int J Comput Assist Radiol Surg"},{"key":"3093_CR4","doi-asserted-by":"publisher","first-page":"247","DOI":"10.1016\/J.JURO.2011.09.032","volume":"187","author":"AC Goh","year":"2012","unstructured":"Goh AC, Goldfarb DW, Sander JC, Miles BJ, Dunkin BJ (2012) Global evaluative assessment of robotic skills: validation of a clinical assessment tool to measure robotic surgical skills. J Urol 187:247\u2013252. https:\/\/doi.org\/10.1016\/J.JURO.2011.09.032","journal-title":"J Urol"},{"issue":"1","key":"3093_CR5","doi-asserted-by":"publisher","first-page":"301","DOI":"10.1146\/annurev-bioeng-071516-044435","volume":"19","author":"SS Vedula","year":"2017","unstructured":"Vedula SS, Ishii M, Hager GD (2017) Objective assessment of surgical technical skill and competency in the operating room. Annu Rev Biomed Eng 19(1):301\u2013325. https:\/\/doi.org\/10.1146\/annurev-bioeng-071516-044435. (PMID: 28375649)","journal-title":"Annu Rev Biomed Eng"},{"key":"3093_CR6","doi-asserted-by":"publisher","DOI":"10.1007\/S00464-023-10335-Z","author":"R Pedrett","year":"2023","unstructured":"Pedrett R, Mascagni P, Beldi G, Padoy N, Lavanchy JL (2023) Technical skill assessment in minimally invasive surgery using artificial intelligence: a systematic review. Surg Endosc. https:\/\/doi.org\/10.1007\/S00464-023-10335-Z","journal-title":"Surg Endosc"},{"key":"3093_CR7","doi-asserted-by":"publisher","DOI":"10.1002\/RCS.1850","author":"MJ Fard","year":"2018","unstructured":"Fard MJ, Ameri S, Ellis RD, Chinnam RB, Pandya AK, Klein MD (2018) Automated robot-assisted surgical skill evaluation: predictive analytics approach. Int J Med Robot Comput Assist Surg. https:\/\/doi.org\/10.1002\/RCS.1850","journal-title":"Int J Med Robot Comput Assist Surg"},{"key":"3093_CR8","doi-asserted-by":"publisher","unstructured":"Fathabadi FR, Grantner JL, Shebrain SA, Abdel-Qader I (2021) Surgical skill assessment system using fuzzy logic in a multi-class detection of laparoscopic box-trainer instruments. In: Conference proceedings\u2014IEEE international conference on systems, man and cybernetics, 1248\u20131253 https:\/\/doi.org\/10.1109\/SMC52423.2021.9658766","DOI":"10.1109\/SMC52423.2021.9658766"},{"key":"3093_CR9","doi-asserted-by":"publisher","first-page":"731","DOI":"10.1007\/S11548-018-1735-5\/FIGURES\/5","volume":"13","author":"A Zia","year":"2018","unstructured":"Zia A, Essa I (2018) Automated surgical skill assessment in RMIS training. Int J Comput Assist Radiol Surg 13:731\u2013739. https:\/\/doi.org\/10.1007\/S11548-018-1735-5\/FIGURES\/5","journal-title":"Int J Comput Assist Radiol Surg"},{"key":"3093_CR10","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1038\/s41598-022-15053-5","volume":"12","author":"RJ Kuo","year":"2022","unstructured":"Kuo RJ, Chen HJ, Kuo YH (2022) The development of an eye movement-based deep learning system for laparoscopic surgical skills assessment. Sci Rep 12:1\u201312. https:\/\/doi.org\/10.1038\/s41598-022-15053-5","journal-title":"Sci Rep"},{"key":"3093_CR11","doi-asserted-by":"publisher","first-page":"1217","DOI":"10.1007\/S11548-019-01995-1\/FIGURES\/4","volume":"14","author":"I Funke","year":"2019","unstructured":"Funke I, Mees ST, Weitz J, Speidel S (2019) Video-based surgical skill assessment using 3d convolutional neural networks. Int J Comput Assist Radiol Surg 14:1217\u20131225. https:\/\/doi.org\/10.1007\/S11548-019-01995-1\/FIGURES\/4","journal-title":"Int J Comput Assist Radiol Surg"},{"issue":"11","key":"3093_CR12","doi-asserted-by":"publisher","first-page":"2740","DOI":"10.1109\/TPAMI.2018.2868668","volume":"41","author":"L Wang","year":"2019","unstructured":"Wang L, Xiong Y, Wang Z, Qiao Y, Lin D, Tang X, Van Gool L (2019) Temporal segment networks for action recognition in videos. IEEE Trans Pattern Anal Mach Intell 41(11):2740\u20132755. https:\/\/doi.org\/10.1109\/TPAMI.2018.2868668","journal-title":"IEEE Trans Pattern Anal Mach Intell"},{"key":"3093_CR13","doi-asserted-by":"publisher","first-page":"1755","DOI":"10.1109\/LRA.2023.3242466","volume":"8","author":"D Anastasiou","year":"2023","unstructured":"Anastasiou D, Jin Y, Stoyanov D, Mazomenos E (2023) Keep your eye on the best: contrastive regression transformer for skill assessment in robotic surgery. IEEE Robot Autom Lett 8:1755\u20131762. https:\/\/doi.org\/10.1109\/LRA.2023.3242466","journal-title":"IEEE Robot Autom Lett"},{"key":"3093_CR14","doi-asserted-by":"publisher","first-page":"780","DOI":"10.1038\/s41551-023-01010-8","volume":"7","author":"D Kiyasseh","year":"2023","unstructured":"Kiyasseh D, Ma R, Haque TF, Miles BJ, Wagner C, Donoho DA, Anandkumar A, Hung AJ (2023) A vision transformer for decoding surgeon activity from surgical videos. Nat Biomed Eng 7:780\u2013796. https:\/\/doi.org\/10.1038\/s41551-023-01010-8","journal-title":"Nat Biomed Eng"},{"key":"3093_CR15","doi-asserted-by":"publisher","unstructured":"Liu Z, Ning J, Cao Y, Wei Y, Zhang Z, Lin S, Hu H (2021) Video swin transformer. In: Proceedings of the IEEE computer society conference on computer vision and pattern recognition 2022-June, 3192\u20133201 https:\/\/doi.org\/10.1109\/CVPR52688.2022.00320","DOI":"10.1109\/CVPR52688.2022.00320"},{"key":"3093_CR16","doi-asserted-by":"publisher","first-page":"290","DOI":"10.1055\/S-0042-102966\/ID\/R1180-23\/BIB","volume":"141","author":"S Axt","year":"2016","unstructured":"Axt S, Johannink J, Storz P, Mees ST, R\u00f6th AA, Kirschniak A (2016) Chirurgische weiterbildung an deutschen kliniken - wunsch und realit\u00e4t. Zentralblatt fur Chirurgie - Zeitschrift fur Allgemeine, Viszeral- und Gefasschirurgie 141:290\u2013296. https:\/\/doi.org\/10.1055\/S-0042-102966\/ID\/R1180-23\/BIB","journal-title":"Zentralblatt fur Chirurgie - Zeitschrift fur Allgemeine, Viszeral- und Gefasschirurgie"},{"key":"3093_CR17","doi-asserted-by":"publisher","DOI":"10.3389\/FMED.2022.897219","author":"I Kil","year":"2022","unstructured":"Kil I, Eidt JF, Groff RE, Singapogu RB (2022) Assessment of open surgery suturing skill: simulator platform, force-based, and motion-based metrics. Front Med. https:\/\/doi.org\/10.3389\/FMED.2022.897219","journal-title":"Front Med"},{"key":"3093_CR18","doi-asserted-by":"publisher","first-page":"910","DOI":"10.1016\/J.JSURG.2015.04.023","volume":"72","author":"G Saggio","year":"2015","unstructured":"Saggio G, Lazzaro A, Sbernini L, Carrano FM, Passi D, Corona A, Panetta V, Gaspari AL, Lorenzo ND (2015) Objective surgical skill assessment: an initial experience by means of a sensory glove paving the way to open surgery simulation? J Surg Educ 72:910\u2013917. https:\/\/doi.org\/10.1016\/J.JSURG.2015.04.023","journal-title":"J Surg Educ"},{"key":"3093_CR19","doi-asserted-by":"publisher","DOI":"10.5281\/ZENODO.7940583","author":"P Peters","year":"2023","unstructured":"Peters P, Lemos M, B\u00f6nsch A, Ooms M, Ulbrich M, Rashad A, Krause F, Lipprandt M, Kuhlen TW, R\u00f6hrig R, H\u00f6lzle F, Puladi B (2023) Dataset from: effect of head-mounted displays on students\u2019 acquisition of surgical suturing techniques compared to an e-learning and tutor-led course: a randomized controlled trial. Int J Surg. https:\/\/doi.org\/10.5281\/ZENODO.7940583","journal-title":"Int J Surg"},{"key":"3093_CR20","doi-asserted-by":"publisher","DOI":"10.1097\/JS9.0000000000000464","author":"P Peters","year":"2023","unstructured":"Peters P, Lemos M, B\u00f6nsch A, Ooms M, Ulbrich M, Rashad A, Krause F, Lipprandt M, Kuhlen TW, R\u00f6hrig R, H\u00f6lzle F, Puladi B (2023) Behrus Puladi: effect of head-mounted displays on students\u2019 acquisition of surgical suturing techniques compared to an e-learning and tutor-led course: A randomized controlled trial. Int J Surg. https:\/\/doi.org\/10.1097\/JS9.0000000000000464","journal-title":"Int J Surg"},{"key":"3093_CR21","doi-asserted-by":"publisher","first-page":"273","DOI":"10.1046\/J.1365-2168.1997.02502.X","volume":"84","author":"JA Martin","year":"2005","unstructured":"Martin JA, Regehr G, Reznick R, Macrae H, Murnaghan J, Hutchison C, Brown M (2005) Objective structured assessment of technical skill (osats) for surgical residents. Br J Surg 84:273\u2013278. https:\/\/doi.org\/10.1046\/J.1365-2168.1997.02502.X","journal-title":"Br J Surg"},{"key":"3093_CR22","doi-asserted-by":"crossref","unstructured":"Carreira J, Zisserman A (2017) Quo vadis, action recognition? a new model and the kinetics dataset. In: Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition, pp. 6299\u20136308","DOI":"10.1109\/CVPR.2017.502"}],"container-title":["International Journal of Computer Assisted Radiology and Surgery"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s11548-024-03093-3.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s11548-024-03093-3\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s11548-024-03093-3.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,6,14]],"date-time":"2024-06-14T11:23:11Z","timestamp":1718364191000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s11548-024-03093-3"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,3,25]]},"references-count":22,"journal-issue":{"issue":"6","published-online":{"date-parts":[[2024,6]]}},"alternative-id":["3093"],"URL":"https:\/\/doi.org\/10.1007\/s11548-024-03093-3","relation":{},"ISSN":["1861-6429"],"issn-type":[{"value":"1861-6429","type":"electronic"}],"subject":[],"published":{"date-parts":[[2024,3,25]]},"assertion":[{"value":"19 January 2024","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"28 February 2024","order":2,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"25 March 2024","order":3,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Declarations"}},{"value":"The authors have no conflicts of interest.","order":2,"name":"Ethics","group":{"name":"EthicsHeading","label":"Conflict of interest"}},{"value":"The data used was authorized by the local ethics committee of the University Hospital RWTH Aachen (approval code EK 352\/21 and EK 22-329) and was registered, including the study protocol, in the German Clinical Trials Register (DRKS00029307).","order":3,"name":"Ethics","group":{"name":"EthicsHeading","label":"Ethical approval"}}]}}