{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,5,29]],"date-time":"2026-05-29T18:31:48Z","timestamp":1780079508122,"version":"3.54.0"},"reference-count":30,"publisher":"IEEE","license":[{"start":{"date-parts":[[2023,10,1]],"date-time":"2023-10-01T00:00:00Z","timestamp":1696118400000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2023,10,1]],"date-time":"2023-10-01T00:00:00Z","timestamp":1696118400000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"}],"funder":[{"DOI":"10.13039\/501100001659","name":"German Research Foundation (DFG) in the project Crossmodal Learning","doi-asserted-by":"publisher","award":["TRR-169"],"award-info":[{"award-number":["TRR-169"]}],"id":[{"id":"10.13039\/501100001659","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100004543","name":"China Scholarship Council (CSC)","doi-asserted-by":"publisher","id":[{"id":"10.13039\/501100004543","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2023,10,1]]},"DOI":"10.1109\/iros55552.2023.10342363","type":"proceedings-article","created":{"date-parts":[[2023,12,13]],"date-time":"2023-12-13T19:17:55Z","timestamp":1702495075000},"page":"3590-3596","source":"Crossref","is-referenced-by-count":50,"title":["Chat with the Environment: Interactive Multimodal Perception Using Large Language Models"],"prefix":"10.1109","author":[{"given":"Xufeng","family":"Zhao","sequence":"first","affiliation":[{"name":"Universität Hamburg,Knowledge Technology Group,Department of Informatics,Hamburg,Germany,22527"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Mengdi","family":"Li","sequence":"additional","affiliation":[{"name":"Universität Hamburg,Knowledge Technology Group,Department of Informatics,Hamburg,Germany,22527"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Cornelius","family":"Weber","sequence":"additional","affiliation":[{"name":"Universität Hamburg,Knowledge Technology Group,Department of Informatics,Hamburg,Germany,22527"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Muhammad Burhan","family":"Hafez","sequence":"additional","affiliation":[{"name":"Universität Hamburg,Knowledge Technology Group,Department of Informatics,Hamburg,Germany,22527"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Stefan","family":"Wermter","sequence":"additional","affiliation":[{"name":"Universität Hamburg,Knowledge Technology Group,Department of Informatics,Hamburg,Germany,22527"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"263","reference":[{"key":"ref1","article-title":"Do as i can, not as i say: Grounding language in robotic affordances","author":"Ahn","year":"2022","journal-title":"arXiv preprint"},{"key":"ref2","article-title":"Multimodal deep learning","author":"Akkus","year":"2023","journal-title":"arXiv preprint"},{"key":"ref3","article-title":"Active Information Acquisition with Mobile Robots","author":"Atanasov","year":"2015","journal-title":"University of Pennsylvania"},{"key":"ref4","doi-asserted-by":"publisher","DOI":"10.1007\/11957959_13"},{"key":"ref5","article-title":"RT-2: Vision-language-action models transfer web knowledge to robotic control","author":"Brohan","year":"2023","journal-title":"arXiv Preprint"},{"key":"ref6","doi-asserted-by":"publisher","DOI":"10.1007\/s00521-022-08118-z"},{"key":"ref7","article-title":"No, to the Right\u201d-Online language corrections for robotic manipulation via shared autonomy","author":"Cui","year":"2023","journal-title":"arXiv preprint"},{"key":"ref8","doi-asserted-by":"publisher","DOI":"10.1109\/IROS47612.2022.9981578"},{"key":"ref9","article-title":"Open-vocabulary object detection via vision and language knowledge distillation","volume-title":"International Conference on Learning Representations (ICLR)","author":"Gu","year":"2022"},{"key":"ref10","article-title":"Language models as zero-shot planners: Extracting actionable knowledge for embodied agents","author":"Huang","year":"2022","journal-title":"arXiv preprint"},{"key":"ref11","doi-asserted-by":"publisher","DOI":"10.1109\/ACCESS.2023.3329370"},{"key":"ref12","first-page":"30:1","article-title":"A review of robot learning for manipulation: Challenges, representations, and algorithms","volume":"22","author":"Kroemer","year":"2021","journal-title":"J. Mach. Learn. Res."},{"key":"ref13","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52688.2022.00337"},{"key":"ref14","article-title":"Cornelius Weber and Stefan Wermter. Internally Rewarded Reinforcement Learning","volume-title":"International Conference on Machine Learning (ICML)","author":"Li"},{"key":"ref15","article-title":"Interactive language: Talking to robots in real time","author":"Lynch","year":"2022","journal-title":"arXiv preprint"},{"key":"ref16","article-title":"Augmented language models: A survey","author":"Mialon","year":"2023","journal-title":"arXiv preprint"},{"key":"ref17","volume-title":"Training language models to follow instructions with human feedback","author":"Ouyang","year":"2022"},{"key":"ref18","doi-asserted-by":"publisher","DOI":"10.1109\/ICRA.2019.8793667"},{"key":"ref19","doi-asserted-by":"publisher","DOI":"10.1109\/icra48506.2021.9561317"},{"key":"ref20","first-page":"894","article-title":"CLIPort: What and where pathways for robotic manipulation","volume-title":"Conference on Robot Learning (CoRL)","volume":"164","author":"Shridhar","year":"2022"},{"key":"ref21","article-title":"Perceiver-actor: A multi-task transformer for robotic manipulation","volume-title":"Conference on Robot Learning (CoRL)","author":"Shridhar","year":"2022"},{"key":"ref22","article-title":"COG: Connecting New Skills to Past Experience with Offline Reinforcement Learning","volume-title":"Conference on Robot Learning (CoRL)","author":"Singh"},{"key":"ref23","volume-title":"VideoMAE: Masked Autoencoders are Data-Efficient Learners for Self-Supervised Video Pre-Training","author":"Tong","year":"2022"},{"key":"ref24","doi-asserted-by":"publisher","DOI":"10.1109\/ICRA48506.2021.9561479"},{"key":"ref25","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-19836-6_12"},{"key":"ref26","article-title":"Learning in audio-visual context: A review, analysis, and new perspective","author":"Wei","year":"2022","journal-title":"arXiv preprint"},{"key":"ref27","article-title":"Socratic models: Composing zero-shot multimodal reasoning with language","volume-title":"International Conference on Learning Representations (ICLR)","author":"Zeng","year":"2023"},{"key":"ref28","doi-asserted-by":"publisher","DOI":"10.1109\/IROS47612.2022.9981510"},{"key":"ref29","volume-title":"MiniGPT-4: Enhancing Vision-Language Understanding with Advanced Large Language Models","author":"Zhu","year":"2023"},{"key":"ref30","doi-asserted-by":"publisher","DOI":"10.1007\/s11633-021-1293-0"}],"event":{"name":"2023 IEEE\/RSJ International Conference on Intelligent Robots and Systems (IROS)","location":"Detroit, MI, USA","start":{"date-parts":[[2023,10,1]]},"end":{"date-parts":[[2023,10,5]]}},"container-title":["2023 IEEE\/RSJ International Conference on Intelligent Robots and Systems (IROS)"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx7\/10341341\/10341342\/10342363.pdf?arnumber=10342363","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2023,12,20]],"date-time":"2023-12-20T00:16:16Z","timestamp":1703031376000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/10342363\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023,10,1]]},"references-count":30,"URL":"https:\/\/doi.org\/10.1109\/iros55552.2023.10342363","relation":{},"subject":[],"published":{"date-parts":[[2023,10,1]]}}}