从零到端到端自动驾驶实战-第 4 章-BEV 感知—让车看懂整个世界

在第3章中，我们用一个前视摄像头+ ResNet实现了最基础的模仿学习。但那个模型有一个致命缺陷：它只能看到前方，看不到左右和后方。真实的自动驾驶车辆装有6个摄像头（前左、前中、前右、后左、后中、后右），它们合在一起才能看到周围360°的环境。本章的核心问题是：怎么把6个摄像头的透视图像“拼接+压平”成一张俯视图？这就是BEV感知要
解决的问题，而BEVFormer是目前最经典的解决方案。

nuScenes 数据集深度实操

理解了 BEVFormer 的理论，现在让我们动手操作 nuScenes 数据集，看看真实数据长什么样。

from nuscenes.nuscenes import NuScenes
from nuscenes.utils.data_classes import LidarPointCloud
from PIL import Image
import matplotlib.pyplot as plt
import numpy as np

NUSCENES_ROOT = "/home/lionsking/data/nuscenes"  # 你自己的 NuScenes 数据集路径
nusc = NuScenes(version='v1.0-mini',
                dataroot=NUSCENES_ROOT,
                verbose=True)

print(f"nusc: {nusc}")

# 查看第一个场景的第一个样本
sample = nusc.sample[0]

print(f"sample: {sample}")

# 获取 6 个摄像头的图像
cam_names = ['CAM_FRONT_LEFT', 'CAM_FRONT', 'CAM_FRONT_RIGHT',
             'CAM_BACK_LEFT', 'CAM_BACK', 'CAM_BACK_RIGHT']

fig, axes = plt.subplots(2, 3, figsize=(18, 8))

print(f"fig 类型: {type(fig)}")
print(f"fig 大小: {fig.get_size_inches()}")
print(f"fig DPI: {fig.get_dpi()}")

print(f"axes 类型: {type(axes)}")
print(f"axes 形状: {axes.shape}")
print(f"axes 总数量: {axes.size}")

for i, cam in enumerate(cam_names):
    cam_data = nusc.get('sample_data', sample['data'][cam])
    img = Image.open(f"{NUSCENES_ROOT}/{cam_data['filename']}")
    axes[i//3, i%3].imshow(img)
    axes[i//3, i%3].set_title(cam, fontsize=12)
    axes[i//3, i%3].axis('off')
plt.tight_layout()
plt.savefig('6_cameras.png', dpi=150)
print('✅ 6 个摄像头图像已保存')

#查看相机内外参数（BEVFormer用它来做3D→2D投影）
cam_token=sample['data']['CAM_FRONT']
cam_data =nusc.get('sample_data',cam_token)

print(f"cam_data: {cam_data}")
calibrated=nusc.get('calibrated_sensor',cam_data['calibrated_sensor_token'])
print(f"相机内参(3x3):")
print(np.array(calibrated['camera_intrinsic']))
print(f"\n相机外参(平移):{calibrated['translation']}")
print(f"相机外参(旋转):{calibrated['rotation']}")
#这些参数就是BEVFormer做空间交叉注意力时
#把3D参考点投影到2D图像上的关键输入

#查看3D框标注
for ann_token in sample['anns'][:5]:
    ann=nusc.get('sample_annotation', ann_token)
    print(f"类别:{ann['category_name']:<30}"
    f"位置:({ann['translation'][0]:.1f},"
    f"{ann['translation'][1]:.1f},"
    f"{ann['translation'][2]:.1f})"
    f"尺寸:{ann['size']}")

打印下边的结果：

(carla) lionsking@ai-dev:~/Code/auto_self/carla_ch04$ python nuscenes_test.py 
======
Loading NuScenes tables for version v1.0-mini...
23 category,
8 attribute,
4 visibility,
911 instance,
12 sensor,
120 calibrated_sensor,
31206 ego_pose,
8 log,
10 scene,
404 sample,
31206 sample_data,
18538 sample_annotation,
4 map,
Done loading in 0.180 seconds.
======
Reverse indexing ...
Done reverse indexing in 0.0 seconds.
======
nusc: <nuscenes.nuscenes.NuScenes object at 0x7a4ac6f9fd30>
sample: {'token': 'ca9a282c9e77460f8360f564131a8af5', 'timestamp': 1532402927647951, 'prev': '', 'next': '39586f9d59004284a7114a68825e8eec', 'scene_token': 'cc8c0bf57f984915a77078b10eb33198', 'data': {'RADAR_FRONT': '37091c75b9704e0daa829ba56dfa0906', 'RADAR_FRONT_LEFT': '11946c1461d14016a322916157da3c7d', 'RADAR_FRONT_RIGHT': '491209956ee3435a9ec173dad3aaf58b', 'RADAR_BACK_LEFT': '312aa38d0e3e4f01b3124c523e6f9776', 'RADAR_BACK_RIGHT': '07b30d5eb6104e79be58eadf94382bc1', 'LIDAR_TOP': '9d9bf11fb0e144c8b446d54a8a00184f', 'CAM_FRONT': 'e3d495d4ac534d54b321f50006683844', 'CAM_FRONT_RIGHT': 'aac7867ebf4f446395d29fbd60b63b3b', 'CAM_BACK_RIGHT': '79dbb4460a6b40f49f9c150cb118247e', 'CAM_BACK': '03bea5763f0f4722933508d5999c5fd8', 'CAM_BACK_LEFT': '43893a033f9c46d4a51b5e08a67a1eb7', 'CAM_FRONT_LEFT': 'fe5422747a7d4268a4b07fc396707b23'}, 'anns': ['ef63a697930c4b20a6b9791f423351da', '6b89da9bf1f84fd6a5fbe1c3b236f809', '924ee6ac1fed440a9d9e3720aac635a0', '91e3608f55174a319246f361690906ba', 'cd051723ed9c40f692b9266359f547af', '36d52dfedd764b27863375543c965376', '70af124fceeb433ea73a79537e4bea9e', '63b89fe17f3e41ecbe28337e0e35db8e', 'e4a3582721c34f528e3367f0bda9485d', 'fcb2332977ed4203aa4b7e04a538e309', 'a0cac1c12246451684116067ae2611f6', '02248ff567e3497c957c369dc9a1bd5c', '9db977e264964c2887db1e37113cddaa', 'ca9c5dd6cf374aa980fdd81022f016fd', '179b8b54ee74425893387ebc09ee133d', '5b990ac640bf498ca7fd55eaf85d3e12', '16140fbf143d4e26a4a7613cbd3aa0e8', '54939f11a73d4398b14aeef500bf0c23', '83d881a6b3d94ef3a3bc3b585cc514f8', '74986f1604f047b6925d409915265bf7', 'e86330c5538c4858b8d3ffe874556cc5', 'a7bd5bb89e27455bbb3dba89a576b6a1', 'fbd9d8c939b24f0eb6496243a41e8c41', '198023a1fb5343a5b6fad033ab8b7057', 'ffeafb90ecd5429cba23d0be9a5b54ee', 'cc636a58e27e446cbdd030c14f3718fd', '076a7e3ec6244d3b84e7df5ebcbac637', '0603fbaef1234c6c86424b163d2e3141', 'd76bd5dcc62f4c57b9cece1c7bcfabc5', '5acb6c71bcd64aa188804411b28c4c8f', '49b74a5f193c4759b203123b58ca176d', '77519174b48f4853a895f58bb8f98661', 'c5e9455e98bb42c0af7d1990db1df0c9', 'fcc5b4b5c4724179ab24962a39ca6d65', '791d1ca7e228433fa50b01778c32449a', '316d20eb238c43ef9ee195642dd6e3fe', 'cda0a9085607438c9b1ea87f4360dd64', 'e865152aaa194f22b97ad0078c012b21', '7962506dbc24423aa540a5e4c7083dad', '29cca6a580924b72a90b9dd6e7710d3e', 'a6f7d4bb60374f868144c5ba4431bf4c', 'f1ae3f713ba946069fa084a6b8626fbf', 'd7af8ede316546f68d4ab4f3dbf03f88', '91cb8f15ed4444e99470d43515e50c1d', 'bc638d33e89848f58c0b3ccf3900c8bb', '26fb370c13f844de9d1830f6176ebab6', '7e66fdf908d84237943c833e6c1b317a', '67c5dbb3ddcc4aff8ec5140930723c37', 'eaf2532c820740ae905bb7ed78fb1037', '3e2d17fa9aa5484d9cabc1dfca532193', 'de6bd5ffbed24aa59c8891f8d9c32c44', '9d51d699f635478fbbcd82a70396dd62', 'b7cbc6d0e80e4dfda7164871ece6cb71', '563a3f547bd64a2f9969278c5ef447fd', 'df8917888b81424f8c0670939e61d885', 'bb3ef5ced8854640910132b11b597348', 'a522ce1d7f6545d7955779f25d01783b', '1fafb2468af5481ca9967407af219c32', '05de82bdb8484623906bb9d97ae87542', 'bfedb0d85e164b7697d1e72dd971fb72', 'ca0f85b4f0d44beb9b7ff87b1ab37ff5', 'bca4bbfdef3d4de980842f28be80b3ca', 'a834fb0389a8453c810c3330e3503e16', '6c804cb7d78943b195045082c5c2d7fa', 'adf1594def9e4722b952fea33b307937', '49f76277d07541c5a584aa14c9d28754', '15a3b4d60b514db5a3468e2aef72a90c', '18cc2837f2b9457c80af0761a0b83ccc', '2bfcc693ae9946daba1d9f2724478fd4']}
fig 类型: <class 'matplotlib.figure.Figure'>
fig 大小: [18.  8.]
fig DPI: 100.0
axes 类型: <class 'numpy.ndarray'>
axes 形状: (2, 3)
axes 总数量: 6
✅ 6 个摄像头图像已保存
cam_data: {'token': 'e3d495d4ac534d54b321f50006683844', 'sample_token': 'ca9a282c9e77460f8360f564131a8af5', 'ego_pose_token': 'e3d495d4ac534d54b321f50006683844', 'calibrated_sensor_token': '1d31c729b073425e8e0202c5c6e66ee1', 'timestamp': 1532402927612460, 'fileformat': 'jpg', 'is_key_frame': True, 'height': 900, 'width': 1600, 'filename': 'samples/CAM_FRONT/n015-2018-07-24-11-22-45+0800__CAM_FRONT__1532402927612460.jpg', 'prev': '', 'next': '68e8e98cf7b0487baa139df808641db7', 'sensor_modality': 'camera', 'channel': 'CAM_FRONT'}
相机内参(3x3):
[[1.26641720e+03 0.00000000e+00 8.16267020e+02]
 [0.00000000e+00 1.26641720e+03 4.91507066e+02]
 [0.00000000e+00 0.00000000e+00 1.00000000e+00]]

相机外参(平移):[1.70079118954, 0.0159456324149, 1.51095763913]
相机外参(旋转):[0.4998015430569128, -0.5030316162024876, 0.4997798114386805, -0.49737083824542755]
类别:human.pedestrian.adult        位置:(373.3,1130.4,0.8)尺寸:[0.621, 0.669, 1.642]
类别:human.pedestrian.adult        位置:(378.9,1153.3,0.9)尺寸:[0.775, 0.769, 1.711]
类别:vehicle.car                   位置:(353.8,1132.4,0.6)尺寸:[2.011, 4.633, 1.573]
类别:human.pedestrian.adult        位置:(376.1,1158.5,0.9)尺寸:[0.752, 0.819, 1.637]
类别:movable_object.trafficcone    位置:(410.1,1196.8,0.7)尺寸:[0.427, 0.359, 0.794]
(carla) lionsking@ai-dev:~/Code/auto_self/carla_ch04$ 

跑通 BEVFormer 推理 + 可视化

为者常成，行者常至