{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,5,5]],"date-time":"2026-05-05T06:12:49Z","timestamp":1777961569510,"version":"3.51.4"},"publisher-location":"Cham","reference-count":42,"publisher":"Springer Nature Switzerland","isbn-type":[{"value":"9783032097835","type":"print"},{"value":"9783032097842","type":"electronic"}],"license":[{"start":{"date-parts":[[2026,1,1]],"date-time":"2026-01-01T00:00:00Z","timestamp":1767225600000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2026,1,1]],"date-time":"2026-01-01T00:00:00Z","timestamp":1767225600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2026]]},"DOI":"10.1007\/978-3-032-09784-2_3","type":"book-chapter","created":{"date-parts":[[2026,1,1]],"date-time":"2026-01-01T23:31:26Z","timestamp":1767310286000},"page":"21-31","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":2,"title":["Towards Robust Surgical Automation via\u00a0Digital Twin Representations from\u00a0Foundation Models"],"prefix":"10.1007","author":[{"given":"Hao","family":"Ding","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Lalithkumar","family":"Seenivasan","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Hongchao","family":"Shu","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Grayson","family":"Byrd","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Han","family":"Zhang","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Pu","family":"Xiao","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Juan Antonio","family":"Barrag","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Russell H.","family":"Taylor","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Peter","family":"Kazanzides","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Mathias","family":"Unberath","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2026,1,2]]},"reference":[{"key":"3_CR1","unstructured":"Brohan, A., et\u00a0al.: Rt-1: robotics transformer for real-world control at scale. arXiv preprint arXiv:2212.06817 (2022)"},{"key":"3_CR2","unstructured":"Brohan, A., et\u00a0al.: Rt-2: vision-language-action models transfer web knowledge to robotic control. arXiv preprint arXiv:2307.15818 (2023)"},{"key":"3_CR3","unstructured":"Chen, K., et\u00a0al.: Hybrid task cascade for instance segmentation. In: Proceedings of CVPR, pp. 4974\u20134983 (2019)"},{"key":"3_CR4","doi-asserted-by":"crossref","unstructured":"Cheng, B., et\u00a0al.: Masked-attention mask transformer for universal image segmentation. In: Proceedings of CVPR, pp. 1290\u20131299 (2022)","DOI":"10.1109\/CVPR52688.2022.00135"},{"key":"3_CR5","doi-asserted-by":"crossref","unstructured":"Dharmarajan, K., et\u00a0al.: Automating vascular shunt insertion with the dvrk surgical robot. In: Proceedings of ICRA, pp. 6781\u20136788. IEEE (2023)","DOI":"10.1109\/ICRA48891.2023.10160966"},{"key":"3_CR6","unstructured":"Ding, H., et al.: SegSTRONG-C: segmenting surgical tools robustly on non-adversarial generated corruptions \u2013 an endovis\u201924 challenge (2024)"},{"key":"3_CR7","doi-asserted-by":"crossref","unstructured":"Ding, H., Qiao, S., Yuille, A., Shen, W.: Deeply shape-guided cascade for instance segmentation. In: Proceedings of CVPR, pp. 8278\u20138288 (2021)","DOI":"10.1109\/CVPR46437.2021.00818"},{"key":"3_CR8","doi-asserted-by":"crossref","unstructured":"Ding, H., Seenivasan, L., Killeen, B.D., Cho, S.M., Unberath, M.: Digital twins as a unifying framework for surgical data science: the enabling role of geometric scene understanding. ais 4(3), 109\u2013138 (2024)","DOI":"10.20517\/ais.2024.16"},{"key":"3_CR9","doi-asserted-by":"crossref","unstructured":"Ding, H., Wu, J.Y., Li, Z., Unberath, M.: Rethinking causality-driven robot tool segmentation with temporal constraints. Int. J. CARS 1009 \u2013 1016 (2022)","DOI":"10.1007\/s11548-023-02872-8"},{"key":"3_CR10","doi-asserted-by":"crossref","unstructured":"Ding, H., Zhang, J., Kazanzides, P., Wu, J.Y., Unberath, M.: CaRTS: causality-driven robot tool segmentation from vision and kinematics data. In: Proceedings of MICCAI, pp. 387\u2013398. Springer (2022)","DOI":"10.1007\/978-3-031-16449-1_37"},{"key":"3_CR11","doi-asserted-by":"crossref","unstructured":"Fu, J., Long, Y., Chen, K., Wei, W., Dou, Q.: Multi-objective cross-task learning via goal-conditioned GPT-based decision transformers for surgical robot task automation. arXiv preprint arXiv:2405.18757 (2024)","DOI":"10.1109\/ICRA57147.2024.10611051"},{"key":"3_CR12","doi-asserted-by":"crossref","unstructured":"Hari, K., et\u00a0al.: STITCH: augmented dexterity for suture throws including thread coordination and handoffs. arXiv preprint arXiv:2404.05151 (2024)","DOI":"10.1109\/ISMR63436.2024.10585751"},{"key":"3_CR13","doi-asserted-by":"crossref","unstructured":"He, K., Gkioxari, G., Doll\u00e1r, P., Girshick, R.: Mask r-cnn. In: Proceedings of ICCV, pp. 2961\u20132969 (2017)","DOI":"10.1109\/ICCV.2017.322"},{"issue":"1","key":"3_CR14","doi-asserted-by":"publisher","first-page":"228","DOI":"10.3390\/app11010228","volume":"11","author":"Z He","year":"2020","unstructured":"He, Z., Feng, W., Zhao, X., Lv, Y.: 6d pose estimation of objects: recent technologies and challenges. Appl. Sci. 11(1), 228 (2020)","journal-title":"Appl. Sci."},{"issue":"5","key":"3_CR15","doi-asserted-by":"publisher","first-page":"799","DOI":"10.1007\/s11548-021-02369-2","volume":"16","author":"J Hein","year":"2021","unstructured":"Hein, J., et al.: Towards markerless surgical tool and hand pose estimation. Int. J. Comput. Assist. Radiol. Surg. 16(5), 799\u2013808 (2021). https:\/\/doi.org\/10.1007\/s11548-021-02369-2","journal-title":"Int. J. Comput. Assist. Radiol. Surg."},{"key":"3_CR16","doi-asserted-by":"crossref","unstructured":"Hein, J., et\u00a0al.: Creating a digital twin of spinal surgery: a proof of concept. In: Proceedings of CVPR, pp. 2355\u20132364 (2024)","DOI":"10.1109\/CVPRW63382.2024.00241"},{"issue":"2","key":"3_CR17","doi-asserted-by":"publisher","first-page":"909","DOI":"10.1109\/TASE.2022.3171795","volume":"20","author":"M Hwang","year":"2022","unstructured":"Hwang, M., et al.: Automating surgical peg transfer: calibration with deep learning can exceed speed, accuracy, and consistency of humans. IEEE Trans. Autom. Sci. Eng. 20(2), 909\u2013922 (2022)","journal-title":"IEEE Trans. Autom. Sci. Eng."},{"issue":"4","key":"3_CR18","first-page":"5937","volume":"5","author":"M Hwang","year":"2020","unstructured":"Hwang, M., et al.: Efficiently calibrating cable-driven surgical robots with RGBD fiducial sensing and recurrent neural networks. IEEE RAL 5(4), 5937\u20135944 (2020)","journal-title":"IEEE RAL"},{"issue":"7","key":"3_CR19","first-page":"3916","volume":"8","author":"M Kam","year":"2023","unstructured":"Kam, M., et al.: Autonomous system for vaginal cuff closure via model-based planning and markerless tracking techniques. IEEE RAL 8(7), 3916\u20133923 (2023)","journal-title":"IEEE RAL"},{"key":"3_CR20","doi-asserted-by":"crossref","unstructured":"Kazanzides, P., et\u00a0al.: An open-source research kit for the da Vinci\u00ae surgical system. In: Proceedings of ICRA, pp. 6434\u20136439. IEEE (2014)","DOI":"10.1109\/ICRA.2014.6907809"},{"key":"3_CR21","unstructured":"Killeen, B.D., et al.: Stand in surgeon\u2019s shoes: virtual reality cross-training to enhance teamwork in surgery. Int. J. CARS 1\u201310 (2024)"},{"key":"3_CR22","unstructured":"Kim, J.W., et al.: Surgical robot transformer (srt): imitation learning for surgical tasks. arXiv preprint arXiv:2407.12998 (2024)"},{"key":"3_CR23","unstructured":"Kirillov, A., et\u00a0al.: Segment anything. In: Proceedings of ICCV, pp. 4015\u20134026 (2023)"},{"issue":"7","key":"3_CR24","doi-asserted-by":"publisher","first-page":"1301","DOI":"10.1007\/s11548-024-03143-w","volume":"19","author":"C Kleinbeck","year":"2024","unstructured":"Kleinbeck, C., Zhang, H., Killeen, B.D., Roth, D., Unberath, M.: Neural digital twins: reconstructing complex medical environments for spatial planning in virtual reality. Int. J. CARS 19(7), 1301\u20131312 (2024)","journal-title":"Int. J. CARS"},{"issue":"7","key":"3_CR25","doi-asserted-by":"publisher","first-page":"1303","DOI":"10.1007\/s11548-023-02959-2","volume":"18","author":"Z Li","year":"2023","unstructured":"Li, Z., et al.: Tatoo: vision-based joint tracking of anatomy and tool for skull-base surgery. Int. J. CARS 18(7), 1303\u20131310 (2023)","journal-title":"Int. J. CARS"},{"issue":"16","key":"3_CR26","doi-asserted-by":"publisher","first-page":"24605","DOI":"10.1007\/s11042-022-14213-z","volume":"82","author":"G Marullo","year":"2023","unstructured":"Marullo, G., et al.: 6d object position estimation from 2d images: a literature review. Multimedia Tools Appl. 82(16), 24605\u201324643 (2023)","journal-title":"Multimedia Tools Appl."},{"key":"3_CR27","doi-asserted-by":"crossref","unstructured":"Moghani, M., et\u00a0al.: SuFIA: language-guided augmented dexterity for robotic surgical assistants. arXiv preprint arXiv:2405.05226 (2024)","DOI":"10.1109\/IROS58592.2024.10802053"},{"key":"3_CR28","doi-asserted-by":"crossref","unstructured":"Oguine, K.J., Mukul, R.D.S., Drenkow, N., Unberath, M.: From generalization to precision: exploring SAM for tool segmentation in surgical environments. In: Medical Imaging 2024: Image Processing, vol. 12926, pp. 7\u201312. SPIE (2024)","DOI":"10.1117\/12.3006981"},{"key":"3_CR29","doi-asserted-by":"crossref","unstructured":"Peng, S., Liu, Y., Huang, Q., Zhou, X., Bao, H.: Pvnet: pixel-wise voting network for 6dof pose estimation. In: Proceedings of CVPR, pp. 4561\u20134570 (2019)","DOI":"10.1109\/CVPR.2019.00469"},{"key":"3_CR30","unstructured":"Qin, Y., et\u00a0al.: ToolLLM: facilitating large language models to master 16000+ real-world APIs (2023)"},{"key":"3_CR31","doi-asserted-by":"crossref","unstructured":"Raiciu, C., Rosenblum, D.S.: Enabling confidentiality in content-based publish\/subscribe infrastructures. In: Securecomm and Workshops, pp. 1\u201311 (2006)","DOI":"10.1109\/SECCOMW.2006.359552"},{"key":"3_CR32","unstructured":"Ravi, N., et\u00a0al.: Sam 2: segment anything in images and videos. arXiv preprint arXiv:2408.00714 (2024)"},{"key":"3_CR33","doi-asserted-by":"crossref","unstructured":"Saeidi, H., et al.: Autonomous robotic laparoscopic surgery for intestinal anastomosis. Sci. Robot. 7(62), eabj2908 (2022)","DOI":"10.1126\/scirobotics.abj2908"},{"key":"3_CR34","unstructured":"Schick, T., et al.: Toolformer: language models can teach themselves to use tools (2023)"},{"key":"3_CR35","doi-asserted-by":"crossref","unstructured":"Shen, Y., Ding, H., Shao, X., Unberath, M.: Performance and non-adversarial robustness of the segment anything model 2 in surgical video segmentation. arXiv preprint arXiv:2408.04098 (2024)","DOI":"10.1117\/12.3047383"},{"issue":"6","key":"3_CR36","doi-asserted-by":"publisher","first-page":"1077","DOI":"10.1007\/s11548-023-02863-9","volume":"18","author":"H Shu","year":"2023","unstructured":"Shu, H., et al.: Twin-S: a digital twin for skull base surgery. Int. J. CARS 18(6), 1077\u20131084 (2023)","journal-title":"Int. J. CARS"},{"key":"3_CR37","doi-asserted-by":"crossref","unstructured":"Teufel, T., et\u00a0al.: OneSLAM to map them all: a generalized approach to SLAM for monocular endoscopic imaging based on tracking any point. Int. J. CARS 1\u20138 (2024)","DOI":"10.1007\/s11548-024-03171-6"},{"key":"3_CR38","doi-asserted-by":"crossref","unstructured":"Varghese, R., Sambath, M.: YOLOv8: a novel object detection algorithm with enhanced performance and robustness. In: Proceedings of ADICS, pp.\u00a01\u20136. IEEE (2024)","DOI":"10.1109\/ADICS58448.2024.10533619"},{"key":"3_CR39","doi-asserted-by":"crossref","unstructured":"Wen, B., Yang, W., Kautz, J., Birchfield, S.: Foundationpose: unified 6d pose estimation and tracking of novel objects. In: Proceedings of CVPR, pp. 17868\u201317879 (2024)","DOI":"10.1109\/CVPR52733.2024.01692"},{"key":"3_CR40","doi-asserted-by":"crossref","unstructured":"Wilcox, A., et\u00a0al.: Learning to localize, grasp, and hand over unmodified surgical needles. In: Proceedings of ICRA, pp. 9637\u20139643. IEEE (2022)","DOI":"10.1109\/ICRA46639.2022.9812393"},{"key":"3_CR41","doi-asserted-by":"crossref","unstructured":"Xiao, Y., et al.: Spatialtracker: tracking any 2d pixels in 3d space. In: Proceedings of CVPR, pp. 20406\u201320417 (2024)","DOI":"10.1109\/CVPR52733.2024.01929"},{"key":"3_CR42","doi-asserted-by":"crossref","unstructured":"Yang, L., et\u00a0al.: Depth anything: Unleashing the power of large-scale unlabeled data. In: Proceedings of CVPR, pp. 10371\u201310381 (2024)","DOI":"10.1109\/CVPR52733.2024.00987"}],"container-title":["Lecture Notes in Computer Science","Collaborative Intelligence and Autonomy in Image-Guided Surgery"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-032-09784-2_3","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,1,1]],"date-time":"2026-01-01T23:31:29Z","timestamp":1767310289000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-3-032-09784-2_3"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026]]},"ISBN":["9783032097835","9783032097842"],"references-count":42,"URL":"https:\/\/doi.org\/10.1007\/978-3-032-09784-2_3","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"value":"0302-9743","type":"print"},{"value":"1611-3349","type":"electronic"}],"subject":[],"published":{"date-parts":[[2026]]},"assertion":[{"value":"2 January 2026","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"The authors have no competing interests to declare that are relevant to the content of this article.","order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Disclosure of Interests"}},{"value":"COLAS","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"International Workshop on Collaborative Intelligence and Autonomy in Image-Guided Surgery","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Daejeon","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Korea (Republic of)","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2025","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"23 September 2025","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"23 September 2025","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"1","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"colas2025","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"http:\/\/sites.google.com\/view\/miccai-2025-colas\/home","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}}]}}