{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,3,26]],"date-time":"2026-03-26T15:28:57Z","timestamp":1774538937902,"version":"3.50.1"},"reference-count":34,"publisher":"MDPI AG","issue":"22","license":[{"start":{"date-parts":[[2024,11,14]],"date-time":"2024-11-14T00:00:00Z","timestamp":1731542400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"funder":[{"DOI":"10.13039\/501100001809","name":"The National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["62272426"],"award-info":[{"award-number":["62272426"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100001809","name":"The National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["202201150401021"],"award-info":[{"award-number":["202201150401021"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100001809","name":"The National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["202203021212138"],"award-info":[{"award-number":["202203021212138"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100001809","name":"The National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["202303021212206"],"award-info":[{"award-number":["202303021212206"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100001809","name":"The National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["202303021211153"],"award-info":[{"award-number":["202303021211153"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100001809","name":"The National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["202303021212372"],"award-info":[{"award-number":["202303021212372"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100001809","name":"The National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["202203021222027"],"award-info":[{"award-number":["202203021222027"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]},{"name":"The Shanxi Province Science and Technology Major Special Plan \u201cUnveiling and Leading\u201d Project","award":["62272426"],"award-info":[{"award-number":["62272426"]}]},{"name":"The Shanxi Province Science and Technology Major Special Plan \u201cUnveiling and Leading\u201d Project","award":["202201150401021"],"award-info":[{"award-number":["202201150401021"]}]},{"name":"The Shanxi Province Science and Technology Major Special Plan \u201cUnveiling and Leading\u201d Project","award":["202203021212138"],"award-info":[{"award-number":["202203021212138"]}]},{"name":"The Shanxi Province Science and Technology Major Special Plan \u201cUnveiling and Leading\u201d Project","award":["202303021212206"],"award-info":[{"award-number":["202303021212206"]}]},{"name":"The Shanxi Province Science and Technology Major Special Plan \u201cUnveiling and Leading\u201d Project","award":["202303021211153"],"award-info":[{"award-number":["202303021211153"]}]},{"name":"The Shanxi Province Science and Technology Major Special Plan \u201cUnveiling and Leading\u201d Project","award":["202303021212372"],"award-info":[{"award-number":["202303021212372"]}]},{"name":"The Shanxi Province Science and Technology Major Special Plan \u201cUnveiling and Leading\u201d Project","award":["202203021222027"],"award-info":[{"award-number":["202203021222027"]}]},{"DOI":"10.13039\/501100004480","name":"The Shanxi Provincial Natural Science Foundation","doi-asserted-by":"publisher","award":["62272426"],"award-info":[{"award-number":["62272426"]}],"id":[{"id":"10.13039\/501100004480","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100004480","name":"The Shanxi Provincial Natural Science Foundation","doi-asserted-by":"publisher","award":["202201150401021"],"award-info":[{"award-number":["202201150401021"]}],"id":[{"id":"10.13039\/501100004480","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100004480","name":"The Shanxi Provincial Natural Science Foundation","doi-asserted-by":"publisher","award":["202203021212138"],"award-info":[{"award-number":["202203021212138"]}],"id":[{"id":"10.13039\/501100004480","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100004480","name":"The Shanxi Provincial Natural Science Foundation","doi-asserted-by":"publisher","award":["202303021212206"],"award-info":[{"award-number":["202303021212206"]}],"id":[{"id":"10.13039\/501100004480","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100004480","name":"The Shanxi Provincial Natural Science Foundation","doi-asserted-by":"publisher","award":["202303021211153"],"award-info":[{"award-number":["202303021211153"]}],"id":[{"id":"10.13039\/501100004480","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100004480","name":"The Shanxi Provincial Natural Science Foundation","doi-asserted-by":"publisher","award":["202303021212372"],"award-info":[{"award-number":["202303021212372"]}],"id":[{"id":"10.13039\/501100004480","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100004480","name":"The Shanxi Provincial Natural Science Foundation","doi-asserted-by":"publisher","award":["202203021222027"],"award-info":[{"award-number":["202203021222027"]}],"id":[{"id":"10.13039\/501100004480","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["Remote Sensing"],"abstract":"<jats:p>Sensor data fusion is increasingly crucial in the field of autonomous driving. In sensor fusion research, LiDAR and camera have become prevalent topics. However, accurate data calibration from different modalities is essential for effective fusion. Current calibration methods often depend on specific targets or manual intervention, which are time-consuming and have limited generalization capabilities. To address these issues, we introduce MSANet: LiDAR-Camera Online Calibration with Multi-Scale Fusion and Attention Mechanisms, an end-to-end deep learn-based online calibration network for inferring 6-degree of freedom (DOF) rigid body transformations between 2D images and 3D point clouds. By fusing multi-scale features, we obtain feature representations that contain a lot of detail and rich semantic information. The attention module is used to carry out feature correlation among different modes to complete feature matching. Rather than acquiring the precise parameters directly, MSANet online corrects deviations, aligning the initial calibration with the ground truth. We conducted extensive experiments on the KITTI datasets, demonstrating that our method performs well across various scenarios, the average error of translation prediction especially improves the accuracy by 2.03 cm compared with the best results in the comparison method.<\/jats:p>","DOI":"10.3390\/rs16224233","type":"journal-article","created":{"date-parts":[[2024,11,14]],"date-time":"2024-11-14T04:15:49Z","timestamp":1731557749000},"page":"4233","update-policy":"https:\/\/doi.org\/10.3390\/mdpi_crossmark_policy","source":"Crossref","is-referenced-by-count":3,"title":["MSANet: LiDAR-Camera Online Calibration with Multi-Scale Fusion and Attention Mechanisms"],"prefix":"10.3390","volume":"16","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-3596-6457","authenticated-orcid":false,"given":"Fengguang","family":"Xiong","sequence":"first","affiliation":[{"name":"School of Data Science and Technology, North University of China, Taiyuan 030051, China"},{"name":"Shanxi Provinces Vision Information Processing and Intelligent Robot Engineering Research Center, Taiyuan 030051, China"},{"name":"Shanxi Key Laboratory of Machine Vision and Virtual Reality, Taiyuan 030051, China"}]},{"given":"Zhiqiang","family":"Zhang","sequence":"additional","affiliation":[{"name":"School of Data Science and Technology, North University of China, Taiyuan 030051, China"}]},{"given":"Yu","family":"Kong","sequence":"additional","affiliation":[{"name":"School of Data Science and Technology, North University of China, Taiyuan 030051, China"}]},{"given":"Chaofan","family":"Shen","sequence":"additional","affiliation":[{"name":"School of Data Science and Technology, North University of China, Taiyuan 030051, China"}]},{"given":"Mingyue","family":"Hu","sequence":"additional","affiliation":[{"name":"School of Data Science and Technology, North University of China, Taiyuan 030051, China"}]},{"given":"Liqun","family":"Kuang","sequence":"additional","affiliation":[{"name":"School of Data Science and Technology, North University of China, Taiyuan 030051, China"},{"name":"Shanxi Provinces Vision Information Processing and Intelligent Robot Engineering Research Center, Taiyuan 030051, China"},{"name":"Shanxi Key Laboratory of Machine Vision and Virtual Reality, Taiyuan 030051, China"}]},{"given":"Xie","family":"Han","sequence":"additional","affiliation":[{"name":"School of Data Science and Technology, North University of China, Taiyuan 030051, China"},{"name":"Shanxi Provinces Vision Information Processing and Intelligent Robot Engineering Research Center, Taiyuan 030051, China"},{"name":"Shanxi Key Laboratory of Machine Vision and Virtual Reality, Taiyuan 030051, China"}]}],"member":"1968","published-online":{"date-parts":[[2024,11,14]]},"reference":[{"key":"ref_1","doi-asserted-by":"crossref","unstructured":"Li, Y., Yu, A.W., Meng, T., Caine, B., Ngiam, J., Peng, D., Shen, J., Lu, Y., Zhou, D., and Le, Q.V. (2022, January 18\u201324). Deepfusion: Lidar-camera deep fusion for multi-modal 3d object detection. Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition, New Orleans, LA, USA.","DOI":"10.1109\/CVPR52688.2022.01667"},{"key":"ref_2","doi-asserted-by":"crossref","unstructured":"Zhuang, Z., Li, R., Jia, K., Wang, Q., Li, Y., and Tan, M. (2021, January 11\u201317). Perception-aware multi-sensor fusion for 3d lidar semantic segmentation. Proceedings of the IEEE\/CVF International Conference on Computer Vision, Montreal, BC, Canada.","DOI":"10.1109\/ICCV48922.2021.01597"},{"key":"ref_3","doi-asserted-by":"crossref","unstructured":"Krispel, G., Opitz, M., Waltner, G., Possegger, H., and Bischof, H. (2020, January 2\u20135). Fuseseg: Lidar point cloud segmentation fusing multi-modal data. Proceedings of the IEEE\/CVF Winter Conference on Applications of Computer Vision, Pitkin County, CO, USA.","DOI":"10.1109\/WACV45572.2020.9093584"},{"key":"ref_4","unstructured":"Zhang, Q., and Pless, R. (October, January 28). Extrinsic calibration of a camera and laser range finder (improves camera calibration). Proceedings of the 2004 IEEE\/RSJ International Conference on Intelligent Robots and Systems (IROS) (IEEE Cat. No. 04CH37566), Sendai, Japan."},{"key":"ref_5","doi-asserted-by":"crossref","unstructured":"Kummerle, J., Kuhner, T., and Lauer, M. (2018, January 1\u20135). Automatic calibration of multiple cameras and depth sensors with a spherical target. Proceedings of the 2018 IEEE\/RSJ International Conference on Intelligent Robots and Systems (IROS), Madrid, Spain.","DOI":"10.1109\/IROS.2018.8593955"},{"key":"ref_6","doi-asserted-by":"crossref","unstructured":"Geiger, A., Moosmann, F., Car, \u00d6., and Schuster, B. (2012, January 14\u201318). Automatic camera and range sensor calibration using a single shot. Proceedings of the 2012 IEEE International Conference on Robotics and Automation, Saint Paul, MN, USA.","DOI":"10.1109\/ICRA.2012.6224570"},{"key":"ref_7","unstructured":"Unnikrishnan, R., and Hebert, M. (2005). Fast Extrinsic Calibration of a Laser Rangefinder to a Camera, Robotics Institute. Tech. Rep. CMU-RI-TR-05-09."},{"key":"ref_8","doi-asserted-by":"crossref","first-page":"10080","DOI":"10.3390\/s91210080","article-title":"Sensors for 3d imaging: Metric evaluation and calibration of a ccd\/cmos time-of-flight camera","volume":"9","author":"Chiabrando","year":"2009","journal-title":"Sensors"},{"key":"ref_9","unstructured":"Fremont, V., and Bonnifait, P. (2008, January 20\u201322). Extrinsic calibration between a multi-layer lidar and a camera. Proceedings of the 2008 IEEE International Conference on Multisensor Fusion and Integration for Intelligent Systems, Seoul, Republic of Korea."},{"key":"ref_10","doi-asserted-by":"crossref","unstructured":"Levinson, J., and Thrun, S. (2013, January 24\u201328). Automatic Online Calibration of Cameras and Lasers. Proceedings of the Robotics: Science and Systems, Berlin, Germany.","DOI":"10.15607\/RSS.2013.IX.029"},{"key":"ref_11","doi-asserted-by":"crossref","unstructured":"Pandey, G., McBride, J., Savarese, S., and Eustice, R. (2012, January 22\u201326). Automatic targetless extrinsic calibration of a 3d lidar and camera by maximizing mutual information. Proceedings of the AAAI Conference on Artificial Intelligence, Toronto, ON, Canada.","DOI":"10.1609\/aaai.v26i1.8379"},{"key":"ref_12","doi-asserted-by":"crossref","unstructured":"Taylor, Z., and Nieto, J. (2015, January 26\u201330). Motion-based calibration of multimodal sensor arrays. Proceedings of the 2015 IEEE International Conference on Robotics and Automation (ICRA), Seattle, WA, USA.","DOI":"10.1109\/ICRA.2015.7139872"},{"key":"ref_13","doi-asserted-by":"crossref","unstructured":"Wang, G., Qiu, J., Guo, Y., and Wang, H. (2022, January 23\u201327). Fusionnet: Coarse-to-fine extrinsic calibration network of lidar and camera with hierarchical point-pixel fusion. Proceedings of the 2022 International Conference on Robotics and Automation (ICRA), Philadelphia, PA, USA.","DOI":"10.1109\/ICRA46639.2022.9811945"},{"key":"ref_14","doi-asserted-by":"crossref","unstructured":"Lv, X., Wang, B., Dou, Z., Ye, D., and Wang, S. (2021, January 20\u201325). LCCNet: LiDAR and camera self-calibration using cost volume network. Proceedings of the IEEE\/CVF Conference on Computer Vision and Pattern Recognition, Nashville, TN, USA.","DOI":"10.1109\/CVPRW53098.2021.00324"},{"key":"ref_15","doi-asserted-by":"crossref","unstructured":"Schneider, N., Piewak, F., Stiller, C., and Franke, U. (2017, January 11\u201314). RegNet: Multimodal sensor registration using deep neural networks. Proceedings of the 2017 IEEE intelligent vehicles symposium (IV), Los Angeles, CA, USA.","DOI":"10.1109\/IVS.2017.7995968"},{"key":"ref_16","doi-asserted-by":"crossref","unstructured":"Zhao, G., Hu, J., You, S., and Kuo, C.C.J. (2021). CalibDNN: Multimodal sensor calibration for perception using deep neural networks. Signal Processing, Sensor\/Information Fusion, and Target Recognition XXX, SPIE.","DOI":"10.1117\/12.2587994"},{"key":"ref_17","doi-asserted-by":"crossref","unstructured":"Iyer, G., Ram, R.K., Murthy, J.K., and Krishna, K.M. (2018, January 1\u20135). CalibNet: Geometrically supervised extrinsic calibration using 3D spatial transformer networks. Proceedings of the 2018 IEEE\/RSJ International Conference on Intelligent Robots and Systems (IROS), Madrid, Spain.","DOI":"10.1109\/IROS.2018.8593693"},{"key":"ref_18","doi-asserted-by":"crossref","unstructured":"Shi, J., Zhu, Z., Zhang, J., Liu, R., Wang, Z., Chen, S., and Liu, H. (2020, January 25\u201329). Calibrcnn: Calibrating camera and lidar by recurrent convolutional neural network and geometric constraints. Proceedings of the 2020 IEEE\/RSJ International Conference on Intelligent Robots and Systems (IROS), Las Vegas, NV, USA.","DOI":"10.1109\/IROS45743.2020.9341147"},{"key":"ref_19","doi-asserted-by":"crossref","unstructured":"Lv, X., Wang, S., and Ye, D. (2021). CFNet: LiDAR-camera registration using calibration flow network. Sensors, 21.","DOI":"10.3390\/s21238112"},{"key":"ref_20","doi-asserted-by":"crossref","unstructured":"Tamas, L., and Kato, Z. (2013, January 2\u20138). Targetless calibration of a lidar-perspective camera pair. Proceedings of the IEEE International Conference on Computer Vision Workshops, Sydney, Australia.","DOI":"10.1109\/ICCVW.2013.92"},{"key":"ref_21","unstructured":"Liu, L., and Stamos, I. (2005, January 20\u201326). Automatic 3D to 2D registration for the photorealistic rendering of urban scenes. Proceedings of the 2005 IEEE Computer Society Conference on Computer Vision and Pattern Recognition (CVPR\u201905), San Diego, CA, USA."},{"key":"ref_22","doi-asserted-by":"crossref","first-page":"7517","DOI":"10.1109\/LRA.2021.3098923","article-title":"Pixel-level extrinsic self calibration of high resolution lidar and camera in targetless environments","volume":"6","author":"Yuan","year":"2021","journal-title":"IEEE Robot. Autom. Lett."},{"key":"ref_23","doi-asserted-by":"crossref","unstructured":"Nagy, B., Kov\u00e1cs, L., and Benedek, C. (2019, January 27\u201331). Online targetless end-to-end camera-LiDAR self-calibration. Proceedings of the 2019 16th International Conference on Machine Vision Applications (MVA), Tokyo, Japan.","DOI":"10.23919\/MVA.2019.8757887"},{"key":"ref_24","doi-asserted-by":"crossref","first-page":"2036","DOI":"10.1109\/LRA.2021.3061387","article-title":"Extrinsic calibration of multiple lidars of small fov in targetless environments","volume":"6","author":"Liu","year":"2021","journal-title":"IEEE Robot. Autom. Lett."},{"key":"ref_25","doi-asserted-by":"crossref","unstructured":"Wu, S., Hadachi, A., Vivet, D., and Prabhakar, Y. (2021, January 10\u201315). NetCalib: A Novel Approach for LiDAR-Camera Auto-Calibration Based on Deep Learning. Proceedings of the 2020 25th International Conference on Pattern Recognition (ICPR), Milan, Italy.","DOI":"10.1109\/ICPR48806.2021.9412653"},{"key":"ref_26","doi-asserted-by":"crossref","unstructured":"Zhu, Y., Li, C., and Zhang, Y. (August, January 31). Online camera-lidar calibration with sensor semantic information. Proceedings of the 2020 IEEE International Conference on Robotics and Automation (ICRA), Paris, France.","DOI":"10.1109\/ICRA40945.2020.9196627"},{"key":"ref_27","unstructured":"Wang, W., Nobuhara, S., Nakamura, R., and Sakurada, K. (2020). Soic: Semantic online initialization and calibration for lidar and camera. arXiv."},{"key":"ref_28","doi-asserted-by":"crossref","first-page":"155","DOI":"10.1007\/s11263-008-0152-6","article-title":"EPnP: An accurate O(n) solution to the PnP problem","volume":"81","author":"Lepetit","year":"2009","journal-title":"Int. J. Comput. Vis."},{"key":"ref_29","doi-asserted-by":"crossref","unstructured":"He, K., Zhang, X., Ren, S., and Sun, J. (2016, January 27\u201330). Deep residual learning for image recognition. Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition, Las Vegas, NV, USA.","DOI":"10.1109\/CVPR.2016.90"},{"key":"ref_30","first-page":"6789","article-title":"Non-deep networks","volume":"35","author":"Goyal","year":"2022","journal-title":"Adv. Neural Inf. Process. Syst."},{"key":"ref_31","doi-asserted-by":"crossref","first-page":"70553","DOI":"10.1109\/ACCESS.2022.3186974","article-title":"PSNet: LiDAR and camera registration using parallel Subnetworks","volume":"10","author":"Wu","year":"2022","journal-title":"IEEE Access"},{"key":"ref_32","doi-asserted-by":"crossref","first-page":"1231","DOI":"10.1177\/0278364913491297","article-title":"Vision meets robotics: The kitti dataset","volume":"32","author":"Geiger","year":"2013","journal-title":"Int. J. Robot. Res."},{"key":"ref_33","doi-asserted-by":"crossref","unstructured":"Geiger, A., Lenz, P., and Urtasun, R. (2012, January 16\u201321). Are we ready for autonomous driving? The kitti vision benchmark suite. Proceedings of the 2012 IEEE Conference on Computer Vision and Pattern Recognition, Providence, RI, USA.","DOI":"10.1109\/CVPR.2012.6248074"},{"key":"ref_34","unstructured":"Kingma, D.P., and Ba, J.L. (2014). Adam: A method for stochastic optimization. arXiv."}],"container-title":["Remote Sensing"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/www.mdpi.com\/2072-4292\/16\/22\/4233\/pdf","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,10,10]],"date-time":"2025-10-10T16:31:54Z","timestamp":1760113914000},"score":1,"resource":{"primary":{"URL":"https:\/\/www.mdpi.com\/2072-4292\/16\/22\/4233"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,11,14]]},"references-count":34,"journal-issue":{"issue":"22","published-online":{"date-parts":[[2024,11]]}},"alternative-id":["rs16224233"],"URL":"https:\/\/doi.org\/10.3390\/rs16224233","relation":{},"ISSN":["2072-4292"],"issn-type":[{"value":"2072-4292","type":"electronic"}],"subject":[],"published":{"date-parts":[[2024,11,14]]}}}