{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,9,11]],"date-time":"2025-09-11T16:38:06Z","timestamp":1757608686026,"version":"3.44.0"},"reference-count":56,"publisher":"IEEE","license":[{"start":{"date-parts":[[2025,5,19]],"date-time":"2025-05-19T00:00:00Z","timestamp":1747612800000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2025,5,19]],"date-time":"2025-05-19T00:00:00Z","timestamp":1747612800000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2025,5,19]]},"DOI":"10.1109\/icra55743.2025.11128038","type":"proceedings-article","created":{"date-parts":[[2025,9,2]],"date-time":"2025-09-02T17:28:56Z","timestamp":1756834136000},"page":"10561-10569","source":"Crossref","is-referenced-by-count":0,"title":["KUDA: Keypoints to Unify Dynamics Learning and Visual Prompting for Open-Vocabulary Robotic Manipulation"],"prefix":"10.1109","author":[{"given":"Zixian","family":"Liu","sequence":"first","affiliation":[{"name":"Tsinghua University"}]},{"given":"Mingtong","family":"Zhang","sequence":"additional","affiliation":[{"name":"University of Illinois Urbana-Champaign"}]},{"given":"Yunzhu","family":"Li","sequence":"additional","affiliation":[{"name":"Columbia University"}]}],"member":"263","reference":[{"key":"ref1","article-title":"Bert: Pre-training of deep bidirectional transformers for language understanding","author":"Devlin","year":"2018","journal-title":"arXiv preprint"},{"journal-title":"Improving language understanding by generative pre-training","year":"2018","author":"Radford","key":"ref2"},{"issue":"8","key":"ref3","first-page":"9","article-title":"Language models are unsupervised multitask learners","volume":"1","author":"Radford","year":"2019","journal-title":"OpenAI blog"},{"key":"ref4","article-title":"Language models are few-shot learners","author":"Brown","year":"2020","journal-title":"arXiv preprint"},{"issue":"240","key":"ref5","first-page":"1","article-title":"Palm: Scaling language modeling with pathways","volume":"24","author":"Chowdhery","year":"2023","journal-title":"Journal of Machine Learning Research"},{"key":"ref6","article-title":"Gpt-4 technical report","author":"Achiam","year":"2023","journal-title":"arXiv preprint"},{"key":"ref7","first-page":"8748","article-title":"Learning transferable visual models from natural language supervision","volume-title":"International conference on machine learning. PMLR","author":"Radford","year":"2021"},{"key":"ref8","first-page":"1288812900","article-title":"Blip: Bootstrapping language-image pre-training for unified vision-language understanding and generation","volume-title":"International conference on machine learning. PMLR","author":"Li","year":"2022"},{"key":"ref9","first-page":"19730","article-title":"Blip-2: Bootstrapping language-image pre-training with frozen image encoders and large language models","volume-title":"International conference on machine learning. PMLR","author":"Li","year":"2023"},{"key":"ref10","article-title":"Voxposer: Composable 3d value maps for robotic manipulation with language models","author":"Huang","year":"2023","journal-title":"arXiv preprint"},{"key":"ref11","article-title":"Moka: Open-vocabulary robotic manipulation through mark-based visual prompting","author":"Liu","year":"2024","journal-title":"arXiv preprint"},{"key":"ref12","article-title":"Rekep: Spatio-temporal reasoning of relational keypoint constraints for robotic manipulation","author":"Huang","year":"2024","journal-title":"arXiv preprint"},{"key":"ref13","doi-asserted-by":"publisher","DOI":"10.1109\/ICRA.2017.7989324"},{"key":"ref14","first-page":"1101","article-title":"Deep dynamics models for learning dexterous manipulation","volume-title":"Conference on Robot Learning. PMLR","author":"Nagabandi","year":"2020"},{"key":"ref15","article-title":"Interaction networks for learning about objects, relations and physics","volume":"29","author":"Battaglia","year":"2016","journal-title":"Advances in neural information processing systems"},{"key":"ref16","article-title":"Learning particle dynamics for manipulating rigid bodies, deformable objects, and fluids","author":"Li","year":"2019","journal-title":"ICLR"},{"key":"ref17","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.intexsempar-1.4"},{"key":"ref18","article-title":"Policy adaptation via language optimization: Decomposing tasks for few-shot imitation","author":"Myers","year":"2024","journal-title":"arXiv preprint"},{"key":"ref19","first-page":"1531","article-title":"Leveraging language for accelerated learning of tool manipulation","volume-title":"Conference on Robot Learning. PMLR","author":"Ren","year":"2023"},{"key":"ref20","doi-asserted-by":"publisher","DOI":"10.1109\/HRI.2010.5453186"},{"key":"ref21","doi-asserted-by":"publisher","DOI":"10.15607\/rss.2014.x.005"},{"key":"ref22","article-title":"Do as i can and not as i say: Grounding language in robotic affordances","author":"Ahn","year":"2022","journal-title":"arXiv preprint"},{"key":"ref23","article-title":"Learning to interpret natural language commands through human-robot dialog","volume-title":"Twenty-Fourth International Joint Conference on Artificial Intelligence","author":"Thomason","year":"2015"},{"key":"ref24","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v25i1.7979"},{"key":"ref25","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-28572-1_3"},{"key":"ref26","article-title":"Roboexp: Action-conditioned scene graph via interactive exploration for robotic manipulation","author":"Jiang","year":"2024","journal-title":"arXiv preprint"},{"key":"ref27","article-title":"Look before you leap: Unveiling the power of gpt-4v in robotic vision-language planning","author":"Hu","year":"2023","journal-title":"arXiv preprint"},{"key":"ref28","article-title":"Code as policies: Language model programs for embodied control","author":"Liang","year":"2022","journal-title":"arXiv preprint"},{"key":"ref29","article-title":"Cliport: What and where pathways for robotic manipulation","volume-title":"Proceedings of the 5th Conference on Robot Learning (CoRL)","author":"Shridhar","year":"2021"},{"key":"ref30","article-title":"Perceiver-actor: A multi-task transformer for robotic manipulation","volume-title":"Proceedings of the 6th Conference on Robot Learning (CoRL)","author":"Shridhar","year":"2022"},{"key":"ref31","first-page":"9911002","article-title":"Bc-z: Zero-shot task generalization with robotic imitation learning","volume-title":"Conference on Robot Learning. PMLR","author":"Jang","year":"2022"},{"key":"ref32","doi-asserted-by":"publisher","DOI":"10.15607\/RSS.2022.XVIII.065"},{"key":"ref33","article-title":"Language to rewards for robotic skill synthesis","author":"Yu","year":"2023","journal-title":"arXiv preprint"},{"key":"ref34","article-title":"Pivot: Iterative visual prompting elicits actionable knowledge for vlms","author":"Nasiriany","year":"2024","journal-title":"arXiv preprint"},{"key":"ref35","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2023.emnlp-main.985"},{"key":"ref36","article-title":"Autotamp: Autoregressive task and motion planning with 1 lms as translators and checkers","author":"Chen","year":"2023","journal-title":"arXiv preprint"},{"key":"ref37","doi-asserted-by":"publisher","DOI":"10.1109\/ICRA48891.2023.10161317"},{"key":"ref38","doi-asserted-by":"publisher","DOI":"10.1109\/ACCESS.2024.3387941"},{"key":"ref39","article-title":"Inner monologue: Embodied reasoning through planning with language models","author":"Huang","year":"2022","journal-title":"arXiv preprint"},{"key":"ref40","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-20080-9_42"},{"key":"ref41","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-72970-6_3"},{"key":"ref42","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV51070.2023.00371"},{"key":"ref43","doi-asserted-by":"publisher","DOI":"10.1109\/IROS58592.2024.10801352"},{"key":"ref44","doi-asserted-by":"publisher","DOI":"10.15607\/rss.2023.xix.025"},{"key":"ref45","article-title":"Rt-2: Vision-language-action models transfer web knowledge to robotic control","author":"Brohan","year":"2023","journal-title":"arXiv preprint"},{"key":"ref46","article-title":"Open x-embodiment: Robotic learning datasets and rt-x models","author":"Padalkar","year":"2023","journal-title":"arXiv preprint"},{"key":"ref47","doi-asserted-by":"publisher","DOI":"10.15607\/RSS.2024.XX.120"},{"key":"ref48","article-title":"Set-of-mark prompting unleashes extraordinary visual grounding in gpt-4v","author":"Yang","year":"2023","journal-title":"arXiv preprint"},{"key":"ref49","doi-asserted-by":"publisher","DOI":"10.1109\/cvpr52733.2024.01227"},{"key":"ref50","first-page":"23716","article-title":"Flamingo: a visual language model for few-shot learning","volume":"35","author":"Alayrac","year":"2022","journal-title":"Advances in neural information processing systems"},{"key":"ref51","first-page":"25005","article-title":"Visual prompting via image inpainting","volume":"35","author":"Bar","year":"2022","journal-title":"Advances in Neural Information Processing Systems"},{"key":"ref52","article-title":"Imagebrush: Learning visual incontext instructions for exemplar-based image manipulation","volume":"36","author":"Yang","year":"2024","journal-title":"Advances in Neural Information Processing Systems"},{"key":"ref53","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52733.2024.01222"},{"key":"ref54","first-page":"17773","article-title":"What makes good examples for visual in-context learning?","volume":"36","author":"Zhang","year":"2023","journal-title":"Advances in Neural Information Processing Systems"},{"key":"ref55","doi-asserted-by":"publisher","DOI":"10.1109\/ICRA57147.2024.10610455"},{"key":"ref56","article-title":"Model predictive path integral control using covariance variable importance sampling","author":"Williams","year":"2015","journal-title":"arXiv preprint"}],"event":{"name":"2025 IEEE International Conference on Robotics and Automation (ICRA)","start":{"date-parts":[[2025,5,19]]},"location":"Atlanta, GA, USA","end":{"date-parts":[[2025,5,23]]}},"container-title":["2025 IEEE International Conference on Robotics and Automation (ICRA)"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx8\/11127273\/11127223\/11128038.pdf?arnumber=11128038","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,9,3]],"date-time":"2025-09-03T06:12:17Z","timestamp":1756879937000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/11128038\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,5,19]]},"references-count":56,"URL":"https:\/\/doi.org\/10.1109\/icra55743.2025.11128038","relation":{},"subject":[],"published":{"date-parts":[[2025,5,19]]}}}