{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,9,17]],"date-time":"2025-09-17T16:41:48Z","timestamp":1758127308808,"version":"3.40.3"},"publisher-location":"Cham","reference-count":58,"publisher":"Springer Nature Switzerland","isbn-type":[{"type":"print","value":"9783031729669"},{"type":"electronic","value":"9783031729676"}],"license":[{"start":{"date-parts":[[2024,11,3]],"date-time":"2024-11-03T00:00:00Z","timestamp":1730592000000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2024,11,3]],"date-time":"2024-11-03T00:00:00Z","timestamp":1730592000000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2025]]},"DOI":"10.1007\/978-3-031-72967-6_23","type":"book-chapter","created":{"date-parts":[[2024,11,2]],"date-time":"2024-11-02T19:10:34Z","timestamp":1730574634000},"page":"416-434","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":5,"title":["Dense Multimodal Alignment for\u00a0Open-Vocabulary 3D Scene Understanding"],"prefix":"10.1007","author":[{"given":"Ruihuang","family":"Li","sequence":"first","affiliation":[]},{"given":"Zhengqiang","family":"Zhang","sequence":"additional","affiliation":[]},{"given":"Chenhang","family":"He","sequence":"additional","affiliation":[]},{"given":"Zhiyuan","family":"Ma","sequence":"additional","affiliation":[]},{"given":"Vishal M.","family":"Patel","sequence":"additional","affiliation":[]},{"given":"Lei","family":"Zhang","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2024,11,3]]},"reference":[{"key":"23_CR1","first-page":"23716","volume":"35","author":"JB Alayrac","year":"2022","unstructured":"Alayrac, J.B., et al.: Flamingo: a visual language model for few-shot learning. Adv. Neural. Inf. Process. Syst. 35, 23716\u201323736 (2022)","journal-title":"Adv. Neural. Inf. Process. Syst."},{"key":"23_CR2","doi-asserted-by":"crossref","unstructured":"Armeni, I., et al.: 3d semantic parsing of large-scale indoor spaces. In: Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition, pp. 1534\u20131543 (2016)","DOI":"10.1109\/CVPR.2016.170"},{"key":"23_CR3","unstructured":"Cadena, C., Dick, A.R., Reid, I.D.: Multi-modal auto-encoders as joint estimators for robotics scene understanding. In: Robotics: Science and Systems, vol.\u00a05 (2016)"},{"key":"23_CR4","doi-asserted-by":"crossref","unstructured":"Caesar, H., et al.: nuscenes: a multimodal dataset for autonomous driving. In: Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition, pp. 11621\u201311631 (2020)","DOI":"10.1109\/CVPR42600.2020.01164"},{"key":"23_CR5","unstructured":"Cao, Y., Yihan, Z., Xu, H., Xu, D.: Coda: collaborative novel box discovery and cross-modal alignment for open-vocabulary 3d object detection. Adv. Neural Inf. Process. Syst. 36 (2024)"},{"key":"23_CR6","doi-asserted-by":"crossref","unstructured":"Chang, A., et al.: Matterport3d: learning from rgb-d data in indoor environments. In: 2017 International Conference on 3D Vision (3DV), pp. 667\u2013676. IEEE Computer Society (2017)","DOI":"10.1109\/3DV.2017.00081"},{"key":"23_CR7","unstructured":"Chang, A.X., et\u00a0al.: Shapenet: an information-rich 3d model repository. arXiv preprint arXiv:1512.03012 (2015)"},{"key":"23_CR8","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"202","DOI":"10.1007\/978-3-030-58565-5_13","volume-title":"Computer Vision \u2013 ECCV 2020","author":"DZ Chen","year":"2020","unstructured":"Chen, D.Z., Chang, A.X., Nie\u00dfner, M.: ScanRefer: 3D object localization in RGB-D scans using natural language. In: Vedaldi, A., Bischof, H., Brox, T., Frahm, J.-M. (eds.) ECCV 2020. LNCS, vol. 12365, pp. 202\u2013221. Springer, Cham (2020). https:\/\/doi.org\/10.1007\/978-3-030-58565-5_13"},{"key":"23_CR9","doi-asserted-by":"crossref","unstructured":"Chen, L., Lei, C., Li, R., Li, S., Zhang, Z., Zhang, L.: Fpr: false positive rectification for weakly supervised semantic segmentation. In: Proceedings of the IEEE\/CVF International Conference on Computer Vision, pp. 1108\u20131118 (2023)","DOI":"10.1109\/ICCV51070.2023.00108"},{"key":"23_CR10","doi-asserted-by":"crossref","unstructured":"Choy, C., Gwak, J., Savarese, S.: 4d spatio-temporal convnets: minkowski convolutional neural networks. In: Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition, pp. 3075\u20133084 (2019)","DOI":"10.1109\/CVPR.2019.00319"},{"key":"23_CR11","doi-asserted-by":"crossref","unstructured":"Dai, A., et al.: Scannet: richly-annotated 3d reconstructions of indoor scenes. In: Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition, pp. 5828\u20135839 (2017)","DOI":"10.1109\/CVPR.2017.261"},{"key":"23_CR12","doi-asserted-by":"crossref","unstructured":"Dai, A., Ritchie, D., Bokeloh, M., Reed, S., Sturm, J., Nie\u00dfner, M.: Scancomplete: large-scale scene completion and semantic segmentation for 3d scans. In: Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition, pp. 4578\u20134587 (2018)","DOI":"10.1109\/CVPR.2018.00481"},{"key":"23_CR13","doi-asserted-by":"crossref","unstructured":"Ding, R., Yang, J., Xue, C., Zhang, W., Bai, S., Qi, X.: Pla: Language-driven open-vocabulary 3d scene understanding. In: Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition, pp. 7010\u20137019 (2023)","DOI":"10.1109\/CVPR52729.2023.00677"},{"key":"23_CR14","doi-asserted-by":"publisher","unstructured":"Ghiasi, G., Gu, X., Cui, Y., Lin, T.Y.: Scaling open-vocabulary image segmentation with image-level labels. In: European Conference on Computer Vision, pp. 540\u2013557. Springer, Heidelberg (2022). https:\/\/doi.org\/10.1007\/978-3-031-20059-5_31","DOI":"10.1007\/978-3-031-20059-5_31"},{"key":"23_CR15","doi-asserted-by":"crossref","unstructured":"Han, L., Zheng, T., Xu, L., Fang, L.: Occuseg: occupancy-aware 3d instance segmentation. In: Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition, pp. 2940\u20132949 (2020)","DOI":"10.1109\/CVPR42600.2020.00301"},{"key":"23_CR16","doi-asserted-by":"crossref","unstructured":"Hou, J., Graham, B., Nie\u00dfner, M., Xie, S.: Exploring data-efficient 3d scene understanding with contrastive scene contexts. In: Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition, pp. 15587\u201315597 (2021)","DOI":"10.1109\/CVPR46437.2021.01533"},{"key":"23_CR17","unstructured":"Houlsby, N., et al.: Parameter-efficient transfer learning for nlp. In: International Conference on Machine Learning, pp. 2790\u20132799. PMLR (2019)"},{"key":"23_CR18","unstructured":"Hu, E.J., et al.: Lora: low-rank adaptation of large language models. arXiv preprint arXiv:2106.09685 (2021)"},{"key":"23_CR19","doi-asserted-by":"crossref","unstructured":"Hu, W., Zhao, H., Jiang, L., Jia, J., Wong, T.T.: Bidirectional projection network for cross dimension scene understanding. In: Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition, pp. 14373\u201314382 (2021)","DOI":"10.1109\/CVPR46437.2021.01414"},{"key":"23_CR20","doi-asserted-by":"crossref","unstructured":"Hu, Z., et al.: Vmnet: voxel-mesh network for geodesic-aware 3d semantic segmentation. In: Proceedings of the IEEE\/CVF International Conference on Computer Vision, pp. 15488\u201315498 (2021)","DOI":"10.1109\/ICCV48922.2021.01520"},{"key":"23_CR21","doi-asserted-by":"crossref","unstructured":"Huang, J., Zhang, H., Yi, L., Funkhouser, T., Nie\u00dfner, M., Guibas, L.J.: Texturenet: consistent local parametrizations for learning from high-resolution signals on meshes. In: Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition, pp. 4440\u20134449 (2019)","DOI":"10.1109\/CVPR.2019.00457"},{"key":"23_CR22","unstructured":"Huang, X., et al.: Open-set image tagging with multi-grained text supervision. arXiv e-prints pp. arXiv\u20132310 (2023)"},{"key":"23_CR23","unstructured":"Huang, X., et al.: Tag2text: guiding vision-language model via image tagging. arXiv preprint arXiv:2303.05657 (2023)"},{"key":"23_CR24","unstructured":"Jia, C., et al.: Scaling up visual and vision-language representation learning with noisy text supervision. In: International Conference on Machine Learning, pp. 4904\u20134916. PMLR (2021)"},{"key":"23_CR25","unstructured":"Kingma, D.P., Ba, J.: Adam: a method for stochastic optimization. arXiv preprint arXiv:1412.6980 (2014)"},{"key":"23_CR26","doi-asserted-by":"crossref","unstructured":"Kirillov, A., et al.: Segment anything. In: Proceedings of the IEEE\/CVF International Conference on Computer Vision (ICCV), pp. 4015\u20134026 (2023)","DOI":"10.1109\/ICCV51070.2023.00371"},{"key":"23_CR27","unstructured":"Li, B., Weinberger, K.Q., Belongie, S., Koltun, V., Ranftl, R.: Language-driven semantic segmentation. In: International Conference on Learning Representations (2022). https:\/\/openreview.net\/forum?id=RriDjddCLN"},{"key":"23_CR28","unstructured":"Li, J., Li, D., Savarese, S., Hoi, S.: Blip-2: bootstrapping language-image pre-training with frozen image encoders and large language models. arXiv preprint arXiv:2301.12597 (2023)"},{"key":"23_CR29","doi-asserted-by":"crossref","unstructured":"Li, R., He, C., Li, S., Zhang, Y., Zhang, L.: Dynamask: dynamic mask selection for instance segmentation. In: Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition, pp. 11279\u201311288 (2023)","DOI":"10.1109\/CVPR52729.2023.01085"},{"key":"23_CR30","doi-asserted-by":"crossref","unstructured":"Li, R., He, C., Zhang, Y., Li, S., Chen, L., Zhang, L.: Sim: Semantic-aware instance mask generation for box-supervised instance segmentation. In: Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition, pp. 7193\u20137203 (2023)","DOI":"10.1109\/CVPR52729.2023.00695"},{"key":"23_CR31","doi-asserted-by":"crossref","unstructured":"Li, R., Li, S., He, C., Zhang, Y., Jia, X., Zhang, L.: Class-balanced pixel-level self-labeling for domain adaptive semantic segmentation. In: Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition, pp. 11593\u201311603 (2022)","DOI":"10.1109\/CVPR52688.2022.01130"},{"key":"23_CR32","doi-asserted-by":"crossref","unstructured":"Li, Y., et\u00a0al.: Deepfusion: lidar-camera deep fusion for multi-modal 3d object detection. In: Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition, pp. 17182\u201317191 (2022)","DOI":"10.1109\/CVPR52688.2022.01667"},{"key":"23_CR33","doi-asserted-by":"crossref","unstructured":"Liang, F., et al.: Open-vocabulary semantic segmentation with mask-adapted clip. In: Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition, pp. 7061\u20137070 (2023)","DOI":"10.1109\/CVPR52729.2023.00682"},{"key":"23_CR34","unstructured":"Liong, V.E., Nguyen, T.N.T., Widjaja, S., Sharma, D., Chong, Z.J.: Amvnet: assertion-based multi-view fusion network for lidar semantic segmentation. arXiv preprint arXiv:2012.04934 (2020)"},{"key":"23_CR35","unstructured":"Liu, H., Li, C., Wu, Q., Lee, Y.J.: Visual instruction tuning (2023)"},{"key":"23_CR36","unstructured":"Liu, K., et al.: Weakly supervised 3d open-vocabulary segmentation. arXiv preprint arXiv:2305.14093 (2023)"},{"key":"23_CR37","doi-asserted-by":"publisher","unstructured":"Liu, M., Zhou, Y., Qi, C.R., Gong, B., Su, H., Anguelov, D.: Less: label-efficient semantic segmentation for lidar point clouds. In: European Conference on Computer Vision, pp. 70\u201389. Springer, Heidelberg (2022). https:\/\/doi.org\/10.1007\/978-3-031-19842-7_5","DOI":"10.1007\/978-3-031-19842-7_5"},{"key":"23_CR38","doi-asserted-by":"crossref","unstructured":"Lu, Y., et al.: Open-vocabulary point-cloud object detection without 3d annotation. In: Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition, pp. 1190\u20131199 (2023)","DOI":"10.1109\/CVPR52729.2023.00121"},{"key":"23_CR39","doi-asserted-by":"crossref","unstructured":"Milioto, A., Vizzo, I., Behley, J., Stachniss, C.: Rangenet++: fast and accurate lidar semantic segmentation. In: 2019 IEEE\/RSJ International Conference on Intelligent Robots and Systems (IROS), pp. 4213\u20134220. IEEE (2019)","DOI":"10.1109\/IROS40897.2019.8967762"},{"key":"23_CR40","doi-asserted-by":"crossref","unstructured":"Misra, I., Girdhar, R., Joulin, A.: An end-to-end transformer model for 3d object detection. In: Proceedings of the IEEE\/CVF International Conference on Computer Vision, pp. 2906\u20132917 (2021)","DOI":"10.1109\/ICCV48922.2021.00290"},{"key":"23_CR41","doi-asserted-by":"crossref","unstructured":"Nekrasov, A., Schult, J., Litany, O., Leibe, B., Engelmann, F.: Mix3d: out-of-context data augmentation for 3d scenes. In: 2021 International Conference on 3D Vision (3DV), pp. 116\u2013125. IEEE (2021)","DOI":"10.1109\/3DV53792.2021.00022"},{"key":"23_CR42","doi-asserted-by":"crossref","unstructured":"Peng, S., et\u00a0al.: Openscene: 3d scene understanding with open vocabularies. In: Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition, pp. 815\u2013824 (2023)","DOI":"10.1109\/CVPR52729.2023.00085"},{"key":"23_CR43","unstructured":"Radford, A., et\u00a0al.: Learning transferable visual models from natural language supervision. In: International Conference on Machine Learning, pp. 8748\u20138763. PMLR (2021)"},{"key":"23_CR44","doi-asserted-by":"crossref","unstructured":"Rombach, R., Blattmann, A., Lorenz, D., Esser, P., Ommer, B.: High-resolution image synthesis with latent diffusion models. In: Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition, pp. 10684\u201310695 (2022)","DOI":"10.1109\/CVPR52688.2022.01042"},{"key":"23_CR45","doi-asserted-by":"crossref","unstructured":"Rozenberszki, D., Litany, O., Dai, A.: Language-grounded indoor 3d semantic segmentation in the wild. In: Proceedings of the European Conference on Computer Vision (ECCV) (2022)","DOI":"10.1007\/978-3-031-19827-4_8"},{"key":"23_CR46","doi-asserted-by":"crossref","unstructured":"Schult, J., Engelmann, F., Kontogianni, T., Leibe, B.: Dualconvmesh-net: Joint geodesic and euclidean convolutions on 3d meshes. In: Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition, pp. 8612\u20138622 (2020)","DOI":"10.1109\/CVPR42600.2020.00864"},{"key":"23_CR47","unstructured":"Takmaz, A., Fedele, E., Sumner, R.W., Pollefeys, M., Tombari, F., Engelmann, F.: Openmask3d: open-vocabulary 3d instance segmentation. arXiv preprint arXiv:2306.13631 (2023)"},{"key":"23_CR48","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"685","DOI":"10.1007\/978-3-030-58604-1_41","volume-title":"Computer Vision \u2013 ECCV 2020","author":"H Tang","year":"2020","unstructured":"Tang, H., et al.: Searching efficient 3D architectures with sparse point-voxel convolution. In: Vedaldi, A., Bischof, H., Brox, T., Frahm, J.-M. (eds.) ECCV 2020. LNCS, vol. 12373, pp. 685\u2013702. Springer, Cham (2020). https:\/\/doi.org\/10.1007\/978-3-030-58604-1_41"},{"key":"23_CR49","doi-asserted-by":"crossref","unstructured":"Tatarchenko, M., Park, J., Koltun, V., Zhou, Q.Y.: Tangent convolutions for dense prediction in 3d. In: Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition, pp. 3887\u20133896 (2018)","DOI":"10.1109\/CVPR.2018.00409"},{"key":"23_CR50","doi-asserted-by":"crossref","unstructured":"Vu, T., Kim, K., Luu, T.M., Nguyen, T., Yoo, C.D.: Softgroup for 3d instance segmentation on point clouds. In: Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition, pp. 2708\u20132717 (2022)","DOI":"10.1109\/CVPR52688.2022.00273"},{"key":"23_CR51","unstructured":"Wu, Z., et al.: 3d shapenets: a deep representation for volumetric shapes. In: Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition, pp. 1912\u20131920 (2015)"},{"key":"23_CR52","doi-asserted-by":"crossref","unstructured":"Xu, J., Liu, S., Vahdat, A., Byeon, W., Wang, X., De\u00a0Mello, S.: Open-vocabulary panoptic segmentation with text-to-image diffusion models. In: Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition, pp. 2955\u20132966 (2023)","DOI":"10.1109\/CVPR52729.2023.00289"},{"key":"23_CR53","doi-asserted-by":"crossref","unstructured":"Xue, L., et al.: Ulip: learning a unified representation of language, images, and point clouds for 3d understanding. In: Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition, pp. 1179\u20131189 (2023)","DOI":"10.1109\/CVPR52729.2023.00120"},{"key":"23_CR54","doi-asserted-by":"crossref","unstructured":"Xue, L., et al.: Ulip-2: towards scalable multimodal pre-training for 3d understanding. arXiv preprint arXiv:2305.08275 (2023)","DOI":"10.1109\/CVPR52733.2024.02558"},{"key":"23_CR55","doi-asserted-by":"crossref","unstructured":"Yang, J., Ding, R., Wang, Z., Qi, X.: Regionplc: regional point-language contrastive learning for open-world 3d scene understanding. arXiv preprint arXiv:2304.00962 (2023)","DOI":"10.1109\/CVPR52733.2024.01874"},{"key":"23_CR56","unstructured":"Yu, Q., He, J., Deng, X., Shen, X., Chen, L.C.: Convolutions die hard: open-vocabulary segmentation with single frozen convolutional clip. In: NeurIPS (2023)"},{"key":"23_CR57","unstructured":"Zhang, Y., et\u00a0al.: Recognize anything: a strong image tagging model. arXiv preprint arXiv:2306.03514 (2023)"},{"key":"23_CR58","doi-asserted-by":"crossref","unstructured":"Zhu, X., et al.: Cylindrical and asymmetrical 3d convolution networks for lidar segmentation. In: Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition, pp. 9939\u20139948 (2021)","DOI":"10.1109\/CVPR46437.2021.00981"}],"container-title":["Lecture Notes in Computer Science","Computer Vision \u2013 ECCV 2024"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-031-72967-6_23","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,11,2]],"date-time":"2024-11-02T19:16:36Z","timestamp":1730574996000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-3-031-72967-6_23"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,11,3]]},"ISBN":["9783031729669","9783031729676"],"references-count":58,"URL":"https:\/\/doi.org\/10.1007\/978-3-031-72967-6_23","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"type":"print","value":"0302-9743"},{"type":"electronic","value":"1611-3349"}],"subject":[],"published":{"date-parts":[[2024,11,3]]},"assertion":[{"value":"3 November 2024","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"ECCV","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"European Conference on Computer Vision","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Milan","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Italy","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2024","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"29 September 2024","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"4 October 2024","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"18","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"eccv2024","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"https:\/\/eccv2024.ecva.net\/","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}}]}}