系列文章目录
CARLA pygame window界面大小调节两种方法-Ubuntu18.04
如何在carla中加入车辆群[基于traffic manager]
前言
博客上一些常见的语义分割相机demo在ubuntu18.4上运行时往往会出现摄像头画面只有一帧的情况，而在windows上似乎是可以正常运行的，对于这些代码，我没能找出问题的具体原因，只是怀疑是由于是系统差异造成的，知道成因和解决方法的朋友欢迎在评论区留言讨论。
下面展示一个能在ubuntu18.4上跑通的demo,具体可以实现的效果有：
1.实时连续显示语义分割图像画面
2.可搭载在车上指定位置
3.视角可随意在三轴坐标系转换，实现水平/鸟瞰等任意视角的显示
4.定义对应的场视角，组合多个模块的摄像头，可达到环视效果
5.存储采集的数据画面
一、demo模块说明
1.定义对应的场视角，组合多个模块的摄像头，达到环视效果需要“360/场视角”个摄像头，这里展示三个摄像头的效果
#-------------------------- 添加语义分割相机--------------------------#
sensor_queue = Queue()
sem_bp = blueprint_library.find('sensor.camera.semantic_segmentation')
sem_bp.set_attribute("image_size_x",f"{IM_WIDTH}")
sem_bp.set_attribute("image_size_y",f"{IM_HEIGHT}")
sem_bp.set_attribute("fov",str(60))
#场视角，需要组成环视需要“360/场视角”个摄像头
2.视角可随意在三轴坐标系转换，实现水平/鸟瞰等任意视角的显示：
sem01 = world.spawn_actor(sem_bp,carla.Transform(carla.Location(0,0,1.8),carla.Rotation(yaw=60 )),attach_to=ego_vehicle)
sem01.listen(lambda data: sensor_callback(data, sensor_queue, "sem_01"))
sensor_list.append(sem01)
sem02 = world.spawn_actor(sem_bp,carla.Transform(carla.Location(0,0,1.8),carla.Rotation(yaw=0)),attach_to=ego_vehicle)
carla.Transform(carla.Location(0,0,1.8),carla.Rotation(yaw=60 ))
location是相对车辆底盘中心的平移位置，调整可实现车上任意位置摄像头的安装
Rotation是相对车辆底盘中心的旋转位置
Yaw（偏航）：欧拉角向量的y轴
Pitch（俯仰）：欧拉角向量的x轴
Roll（翻滚）： 欧拉角向量的z轴
这里默认Pitch=0 Roll=0,对其进行调整就可以实现任意视角的显示
关于Yaw-Pitch-Roll的说明可以看这篇博客：python旋转矩阵与欧拉角互转
3.继续添加相机模块
sem02.listen(lambda data: sensor_callback(data, sensor_queue, "sem_02"))
sensor_list.append(sem02)
sem03 = world.spawn_actor(sem_bp,carla.Transform(carla.Location(0,0,1.8),carla.Rotation(yaw=-60)),attach_to=ego_vehicle)
sem03.listen(lambda data: sensor_callback(data, sensor_queue, "sem_03"))
sensor_list.append(sem03)
#-------------------------- 语义分割相机设置完毕 --------------------------#
4.相机画面组合及可视化展示
w_frame = world.get_snapshot().frame
print("\nWorld's frame: %d" % w_frame)
try:
sems = []
for i in range (0, len(sensor_list)):
s_frame, s_name, s_data = sensor_queue.get(True, 1.0)
print(" Frame: %d Sensor: %s" % (s_frame, s_name))
sensor_type = s_name.split('_')[0]
if sensor_type == 'sem':
sems.append(process_semantic(s_data))
# 仅用来可视化 可注释
rgb=np.concatenate(sems, axis=1)[...,:3] # 合并图像
cv2.imshow('vizs', visualize_data(rgb))
cv2.waitKey(100)
if rgb is None or args.save_path is not None:
# 检查是否有各自传感器的文件夹
mkdir_folder(args.save_path)
filename = args.save_path +'rgb/'+str(w_frame)+'.png'
cv2.imwrite(filename, np.array(rgb[...,::-1]))
except Empty:
print(" Some of the sensor information is missed")
5.存储数据
if rgb is None or args.save_path is not None:
# 检查是否有各自传感器的文件夹
mkdir_folder(args.save_path)
filename = args.save_path +'rgb/'+str(w_frame)+'.png'
cv2.imwrite(filename, np.array(rgb[...,::-1]))二、整体代码
import glob
import os
import sys
import time
try:
sys.path.append(glob.glob('../carla/dist/carla-*%d.%d-%s.egg' % (
sys.version_info.major,
sys.version_info.minor,
'win-amd64' if os.name == 'nt' else 'linux-x86_64'))[0])
except IndexError:
pass
import carla
import random
import numpy as np
import cv2
from queue import Queue, Empty
import random
random.seed(50)
# args
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('--host', metavar='H', default='127.0.0.1', help='IP of the host server (default: 127.0.0.1)')
parser.add_argument('--port', '-p', default=2000, type=int, help='TCP port to listen to (default: 2000)')
parser.add_argument('--tm_port', default=8000, type=int, help='Traffic Manager Port (default: 8000)')
parser.add_argument('--ego-spawn', type=list, default=None, help='[x,y] in world coordinate')
parser.add_argument('--save-path', default='存储路径', help='Synchronous mode execution')
args = parser.parse_args()
# 图片大小可自行修改
IM_WIDTH = 500
IM_HEIGHT= 500
actor_list, sensor_list = [], []
sensor_type = ['sem']
def main(args):
# We start creating the client
client = carla.Client(args.host, args.port)
client.set_timeout(5.0)
world = client.get_world()
#world = client.load_world('Town04')
blueprint_library = world.get_blueprint_library()
try:
original_settings = world.get_settings()
settings = world.get_settings()
# We set CARLA syncronous mode
settings.fixed_delta_seconds = 0.05
settings.synchronous_mode = True
world.apply_settings(settings)
spectator = world.get_spectator()
# 手动规定
# transform_vehicle = carla.Transform(carla.Location(0, 10, 0), carla.Rotation(0, 0, 0))
# 自动选择
transform_vehicle = random.choice(world.get_map().get_spawn_points())
ego_vehicle = world.spawn_actor(random.choice(blueprint_library.filter("model3")), transform_vehicle)
actor_list.append(ego_vehicle)
#-------------------------- 添加语义分割相机--------------------------#
sensor_queue = Queue()
sem_bp = blueprint_library.find('sensor.camera.semantic_segmentation')
sem_bp.set_attribute("image_size_x",f"{IM_WIDTH}")
sem_bp.set_attribute("image_size_y",f"{IM_HEIGHT}")
sem_bp.set_attribute("fov",str(60))
#场视角，需要组成环视需要“360/场视角”个摄像头，这里展示两个摄像头的效果
# Yaw（偏航）：欧拉角向量的y轴
# Pitch（俯仰）：欧拉角向量的x轴
# Roll（翻滚）： 欧拉角向量的z轴
# sem_transform = carla.Transform(carla.Location(0,0,10),carla.Rotation(0,0,0))
sem01 = world.spawn_actor(sem_bp,carla.Transform(carla.Location(0,0,1.8),carla.Rotation(yaw=60 )),attach_to=ego_vehicle)
sem01.listen(lambda data: sensor_callback(data, sensor_queue, "sem_01"))
sensor_list.append(sem01)
sem02 = world.spawn_actor(sem_bp,carla.Transform(carla.Location(0,0,1.8),carla.Rotation(yaw=0)),attach_to=ego_vehicle)
sem02.listen(lambda data: sensor_callback(data, sensor_queue, "sem_02"))
sensor_list.append(sem02)
sem03 = world.spawn_actor(sem_bp,carla.Transform(carla.Location(0,0,1.8),carla.Rotation(yaw=-60)),attach_to=ego_vehicle)
sem03.listen(lambda data: sensor_callback(data, sensor_queue, "sem_03"))
sensor_list.append(sem03)
#-------------------------- 设置完毕 --------------------------#
# 设置traffic manager
tm = client.get_trafficmanager(args.tm_port)
tm.set_synchronous_mode(True)
# 是否忽略红绿灯
# tm.ignore_lights_percentage(ego_vehicle, 100)
# 如果限速30km/h -> 30*(1-10%)=27km/h
tm.global_percentage_speed_difference(10.0)
ego_vehicle.set_autopilot(True, tm.get_port())
while True:
# Tick the server
world.tick()
# 将CARLA界面摄像头跟随车动
loc = ego_vehicle.get_transform().location
spectator.set_transform(carla.Transform(carla.Location(x=loc.x,y=loc.y,z=35),carla.Rotation(yaw=0,pitch=-90,roll=0)))
w_frame = world.get_snapshot().frame
print("\nWorld's frame: %d" % w_frame)
try:
sems = []
for i in range (0, len(sensor_list)):
s_frame, s_name, s_data = sensor_queue.get(True, 1.0)
print(" Frame: %d Sensor: %s" % (s_frame, s_name))
sensor_type = s_name.split('_')[0]
if sensor_type == 'sem':
sems.append(process_semantic(s_data))
# 仅用来可视化 可注释
rgb=np.concatenate(sems, axis=1)[...,:3] # 合并图像
cv2.imshow('vizs', visualize_data(rgb))
cv2.waitKey(100)
if rgb is None or args.save_path is not None:
# 检查是否有各自传感器的文件夹
mkdir_folder(args.save_path)
filename = args.save_path +'rgb/'+str(w_frame)+'.png'
cv2.imwrite(filename, np.array(rgb[...,::-1]))
except Empty:
print(" Some of the sensor information is missed")
finally:
world.apply_settings(original_settings)
tm.set_synchronous_mode(False)
for sensor in sensor_list:
sensor.destroy()
for actor in actor_list:
actor.destroy()
print("All cleaned up!")
def mkdir_folder(path):
for s_type in sensor_type:
if not os.path.isdir(os.path.join(path, s_type)):
os.makedirs(os.path.join(path, s_type))
return True
def sensor_callback(sensor_data, sensor_queue, sensor_name):
# Do stuff with the sensor_data data like save it to disk
# Then you just need to add to the queue
sensor_queue.put((sensor_data.frame, sensor_name, sensor_data))
# modify from world on rail code
def visualize_data(rgb, text_args=(cv2.FONT_HERSHEY_SIMPLEX, 0.3, (255,255,255), 1)):
canvas = np.array(rgb[...,::-1])
return canvas
# modify from manual control
def process_semantic(image):
image.convert(carla.ColorConverter.CityScapesPalette)
array = np.frombuffer(image.raw_data, dtype=np.dtype("uint8"))
array = np.reshape(array, (image.height, image.width, 4))
array = array[:, :, :3]
return array
if __name__ == "__main__":
try:
main(args)
except KeyboardInterrupt:
print(' - Exited by user.')
三、效果展示
carla segmentation语义分割相机运行录制
四、拓展：什么是语义分割
图像语义分割（semantic segmentation），从字面意思上理解就是让计算机根据图像的语义来进行分割，例如让计算机在输入下面左图的情况下，能够输出右图。语义在语音识别中指的是语音的意思，在图像领域，语义指的是图像的内容，对图片意思的理解，比如左图的语义就是三个人骑着三辆自行车；分割的意思是从像素的角度分割出图片中的不同对象，对原图中的每个像素都进行标注，比如右图中粉红色代表人，绿色代表自行车。
语义分割当前应用
目前语义分割的应用领域主要有：
- 地理信息系统
- 无人车驾驶
- 医疗影像分析
- 机器人等领域
地理信息系统：可以通过训练神经网络让机器输入卫星遥感影像，自动识别道路，河流，庄稼，建筑物等，并且对图像中每个像素进行标注。（下图左边为卫星遥感影像，中间为真实的标签，右边为神经网络预测的标签结果，可以看到，随着训练加深，预测准确率不断提升。使用ResNet FCN网络进行训练）
**无人车驾驶：**语义分割也是无人车驾驶的核心算法技术，车载摄像头，或者激光雷达探查到图像后输入到神经网络中，后台计算机可以自动将图像分割归类，以避让行人和车辆等障碍。
**医疗影像分析:**随着人工智能的崛起，将神经网络与医疗诊断结合也成为研究热点，智能医疗研究逐渐成熟。在智能医疗领域，语义分割主要应用有肿瘤图像分割，龋齿诊断等。(下图分别是龋齿诊断，头部CT扫描紧急护理诊断辅助和肺癌诊断辅助)
拓展知识链接：计算机视觉之语义分割
文章出处登录后可见！
已经登录？立即刷新
