diff --git a/src/tracking_car/README.md b/src/tracking_car/README.md new file mode 100644 index 000000000..7b4ccc615 --- /dev/null +++ b/src/tracking_car/README.md @@ -0,0 +1,182 @@ +```markdown +# 基于CARLA的智能代理系统 + +## 项目概述 + +本项目旨在利用神经网络技术，实现CARLA模拟器中车辆和行人的全栈智能代理，涵盖感知、规划与控制三大核心模块。通过深度学习算法赋予虚拟智能体环境理解、决策规划和动态控制能力，同时包含具身人仿真、机械臂控制等扩展功能，构建多智能体协同的仿真系统。 + +## 环境配置 + +* **支持平台**：Windows 10/11，Ubuntu 20.04/22.04 +* **核心软件**： + * Python 3.7-3.12（需兼容3.7版本） + * PyTorch（优先采用，不依赖TensorFlow） + * CARLA 0.9.11+（推荐0.9.13/0.9.15版本） +* **依赖安装**： + ```bash + # 基础依赖 + pip install -r requirements.txt -i http://mirrors.aliyun.com/pypi/simple --trusted-host mirrors.aliyun.com + + # CARLA客户端安装（需替换为对应版本） + pip install carla==0.9.15 + + # 文档生成工具 + pip install mkdocs + ``` + +## 文档生成 + +1. 安装文档工具链： + ```bash + pip install mkdocs -i http://mirrors.aliyun.com/pypi/simple --trusted-host mirrors.aliyun.com + pip install -r requirements.txt + ``` + +2. 构建并预览文档： + ```bash + # 进入项目根目录 + cd nn + + # 构建静态文档 + mkdocs build + + # 启动本地文档服务 + mkdocs serve + ``` + +3. 浏览器访问 [http://127.0.0.1:8000](http://127.0.0.1:8000) 查看文档。 + +## 核心功能模块 + +1. **车辆智能代理** + * **感知系统**：基于CNN的目标检测（车辆、行人、交通信号灯）、车道线识别 + * **规划系统**：全局路径规划（A*、RRT*算法）、局部避障 + * **控制系统**：强化学习车辆控制（油门、刹车、转向）、PID参数自适应调优 + * **手动控制**：支持键盘操作的车辆交互模式（WASD控制方向，空格/左Shift控制升降） + +2. **具身人仿真** + * **感知模块**：william开发的具身人环境感知系统（`humanoid_perception`） + * **运动模拟**：基于Mujoco的物理引擎实现具身人运动控制（`Mujoco_manrun`） + +3. **神经网络模型** + * **CNN模型**：图像识别与目标检测（`chap05_CNN`），包含完整训练流程（Adam优化器、交叉熵损失） + * **RNN模型**：序列数据处理与生成（`chap06_RNN`），基于LSTM实现唐诗生成等文本任务 + * **FNN模型**：基础神经网络结构（`chap04_simple_neural_network`），包含测试评估函数 + +4. **辅助系统** + * **车道辅助**：Active-Lane-Keeping-Assistant实现车道偏离预警与辅助 + * **机械臂测试**：humantest模块提供机械臂力控仿真与交互功能 + +## 快速启动 + +1. 启动CARLA服务器： + ```bash + # Linux + ./CarlaUE4.sh + + # Windows + CarlaUE4.exe + ``` + +2. 运行示例场景： + ```bash + # 自动驾驶车辆（强化学习） + python src/driverless_car/main.py + + # 车辆手动控制 + python src/manual_control/main.py + + # CNN模型训练 + python src/chap05_CNN/CNN_pytorch.py + + # RNN文本生成 + python src/chap06_RNN/tangshi_for_pytorch/rnn.py + ``` + +## 关键代码说明 + +### 神经网络训练框架 +```python +# CNN训练示例（chap05_CNN/CNN_pytorch.py） +def train(cnn): + optimizer = torch.optim.Adam(cnn.parameters(), lr=learning_rate) + loss_func = nn.CrossEntropyLoss() + + for epoch in range(max_epoch): + for step, (x_, y_) in enumerate(train_loader): + x, y = Variable(x_), Variable(y_) + output = cnn(x) + loss = loss_func(output, y) + optimizer.zero_grad(set_to_none=True) + loss.backward() + optimizer.step() + + if step != 0 and step % 20 == 0: + print(f"测试准确率: {test(cnn)}") +``` + +### 强化学习智能体 +```python +# 车辆强化学习（driverless_car/main.py） +def main(): + env = DroneEnv() + agent = Agent() + episodes = 1000 + epsilon = 1.0 # 探索率 + + for episode in range(episodes): + state = env.reset() + total_reward = 0 + + while True: + action = agent.get_action(state, epsilon) # 根据状态和探索率获取动作 + next_state, reward, done = env.step(action) + agent.remember(state, action, reward, next_state, done) # 存储经验 + agent.train(batch_size=32) # 训练模型 + + if done: + epsilon = max(0.01, epsilon * 0.995) # 衰减探索率 + break +``` + +### RNN模型结构 +```python +# 唐诗生成RNN（chap06_RNN/tangshi_for_pytorch/rnn.py） +class RNN_model(nn.Module): + def __init__(self, batch_sz, vocab_len, word_embedding, embedding_dim, lstm_hidden_dim): + super(RNN_model, self).__init__() + self.word_embedding_lookup = word_embedding + self.rnn_lstm = nn.LSTM(input_size=embedding_dim, + hidden_size=lstm_hidden_dim, + num_layers=2, + batch_first=False) + self.fc = nn.Linear(lstm_hidden_dim, vocab_len) + self.softmax = nn.LogSoftmax(dim=1) + + def forward(self, sentence, is_test=False): + batch_input = self.word_embedding_lookup(sentence).view(1, -1, self.word_embedding_dim) + h0 = torch.zeros(2, batch_input.size(1), self.lstm_dim) + c0 = torch.zeros(2, batch_input.size(1), self.lstm_dim) + + output, (hn, cn) = self.rnn_lstm(batch_input, (h0, c0)) + out = self.fc(output.contiguous().view(-1, self.lstm_dim)) + return self.softmax(out) +``` + +## 贡献指南 + +请在提交代码前阅读 [贡献指南](https://github.com/OpenHUTB/.github/blob/master/CONTRIBUTING.md)，代码优化方向包括： + +* 遵循 [PEP 8 代码风格](https://peps.pythonlang.cn/pep-0008/) 并完善注释 +* 实现神经网络在CARLA场景中的端到端应用 +* 撰写模块功能说明与API文档 +* 添加自动化测试（模型性能、场景稳定性、数据一致性） +* 优化感知-规划-控制链路的实时性 + +## 参考资源 + +* [CARLA官方文档](https://carla.readthedocs.io/) +* [PyTorch神经网络教程](https://pytorch.org/tutorials/) +* [项目文档中心](https://openhutb.github.io/nn/) +* [代理模拟器文档](https://openhutb.github.io/carla_doc/) +* [神经网络原理](https://github.com/OpenHUTB/neuro) \ No newline at end of file diff --git a/src/tracking_car/main.py b/src/tracking_car/main.py new file mode 100644 index 000000000..9aa5f82b5 --- /dev/null +++ b/src/tracking_car/main.py @@ -0,0 +1,258 @@ +import argparse +import carla +import queue +import random +import cv2 +import numpy as np +from ultralytics import YOLO + +# 内置简易SORT跟踪器（无第三方跟踪库依赖） +class KalmanFilter: + def __init__(self): + self.dt = 0.05 # 与CARLA同步步长一致 + self.x = np.zeros(8) # [x1, y1, x2, y2, vx, vy, vw, vh] + self.F = np.array([ + [1, 0, 0, 0, self.dt, 0, 0, 0], + [0, 1, 0, 0, 0, self.dt, 0, 0], + [0, 0, 1, 0, 0, 0, self.dt, 0], + [0, 0, 0, 1, 0, 0, 0, self.dt], + [0, 0, 0, 0, 1, 0, 0, 0], + [0, 0, 0, 0, 0, 1, 0, 0], + [0, 0, 0, 0, 0, 0, 1, 0], + [0, 0, 0, 0, 0, 0, 0, 1] + ]) + self.H = np.array([[1,0,0,0,0,0,0,0], [0,1,0,0,0,0,0,0], [0,0,1,0,0,0,0,0], [0,0,0,1,0,0,0,0]]) + self.Q = np.diag([1,1,1,1,10,10,10,10]) + self.R = np.diag([10,10,10,10]) + self.P = np.eye(8) * 100 + + def predict(self): + self.x = self.F @ self.x + self.P = self.F @ self.P @ self.F.T + self.Q + return self.x[:4] + + def update(self, z): + y = z - self.H @ self.x + S = self.H @ self.P @ self.H.T + self.R + K = self.P @ self.H.T @ np.linalg.inv(S) + self.x = self.x + K @ y + self.P = (np.eye(8) - K @ self.H) @ self.P + return self.x[:4] + +class Track: + def __init__(self, track_id, bbox): + self.track_id = track_id + self.kf = KalmanFilter() + self.bbox = bbox + self.hits = 1 + self.age = 0 + self.time_since_update = 0 + + def predict(self): + self.bbox = self.kf.predict() + self.age += 1 + self.time_since_update += 1 + return self.bbox + + def update(self, bbox): + self.bbox = self.kf.update(bbox) + self.hits += 1 + self.time_since_update = 0 + +class SimpleSORT: + def __init__(self, max_age=1, min_hits=3, iou_threshold=0.3): + self.max_age = max_age + self.min_hits = min_hits + self.iou_threshold = iou_threshold + self.tracks = [] + self.next_id = 1 + + def update(self, detections): + for track in self.tracks: + track.predict() + + track_boxes = np.array([t.bbox for t in self.tracks]) + det_boxes = np.array([d[:4] for d in detections]) + iou_matrix = self._iou_batch(det_boxes, track_boxes) + + matches = [] + used_dets = set() + used_tracks = set() + + for t_idx in range(len(self.tracks)): + if t_idx in used_tracks: + continue + d_idx = np.argmax(iou_matrix[:, t_idx]) + if iou_matrix[d_idx, t_idx] > self.iou_threshold and d_idx not in used_dets: + matches.append((t_idx, d_idx)) + used_dets.add(d_idx) + used_tracks.add(t_idx) + + for t_idx, d_idx in matches: + self.tracks[t_idx].update(detections[d_idx][:4]) + + for d_idx in range(len(detections)): + if d_idx not in used_dets: + self.tracks.append(Track(self.next_id, detections[d_idx][:4])) + self.next_id += 1 + + self.tracks = [t for t in self.tracks if t.time_since_update <= self.max_age] + + confirmed = [(t.bbox, t.track_id) for t in self.tracks if t.hits >= self.min_hits] + return np.array([b for b, _ in confirmed]), np.array([i for _, i in confirmed]) + + def _iou_batch(self, boxes1, boxes2): + ious = np.zeros((len(boxes1), len(boxes2))) + for i, b1 in enumerate(boxes1): + for j, b2 in enumerate(boxes2): + ious[i, j] = self._iou(b1, b2) + return ious + + def _iou(self, b1, b2): + x1, y1, x2, y2 = b1 + a1, b1, a2, b2 = b2 + inter = max(0, min(x2, a2) - max(x1, a1)) * max(0, min(y2, b2) - max(y1, b1)) + area1 = (x2 - x1) * (y2 - y1) + area2 = (a2 - a1) * (b2 - b1) + return inter / (area1 + area2 - inter) if (area1 + area2 - inter) > 0 else 0 + +# 工具函数（无需外部utils库） +def draw_bounding_boxes(image, boxes, ids): + """在图像上绘制跟踪框和ID""" + for i, box in enumerate(boxes): + x1, y1, x2, y2 = box.astype(int) + # 绘制边界框 + cv2.rectangle(image, (x1, y1), (x2, y2), (0, 255, 0), 2) + # 绘制跟踪ID + cv2.putText(image, f"ID: {ids[i]}", (x1, y1 - 10), + cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 0), 2) + return image + +def clear_actors(world): + """清理CARLA中的所有车辆和传感器""" + for actor in world.get_actors(): + if actor.type_id.startswith('vehicle.') or actor.type_id.startswith('sensor.'): + actor.destroy() + +def camera_callback(image, queue): + """相机数据回调函数""" + img = np.reshape(np.copy(image.raw_data), (image.height, image.width, 4))[:, :, :3] # 去除Alpha通道 + queue.put(img) + +def main(): + parser = argparse.ArgumentParser(description="CARLA目标跟踪（YOLOv5 + 内置SORT）") + parser.add_argument("--host", default="localhost", help="CARLA服务器地址") + parser.add_argument("--port", type=int, default=2000, help="CARLA服务器端口") + parser.add_argument("--num_npcs", type=int, default=30, help="NPC车辆数量") + args = parser.parse_args() + + # 初始化CARLA客户端 + client = carla.Client(args.host, args.port) + client.set_timeout(10.0) + world = client.get_world() + + # 开启同步模式 + settings = world.get_settings() + settings.synchronous_mode = True + settings.fixed_delta_seconds = 0.05 + world.apply_settings(settings) + + # 清理现有Actor + clear_actors(world) + + # 生成主车辆 + bp_lib = world.get_blueprint_library() + ego_bp = bp_lib.find('vehicle.lincoln.mkz_2020') + spawn_points = world.get_map().get_spawn_points() + ego_vehicle = world.try_spawn_actor(ego_bp, random.choice(spawn_points)) + if not ego_vehicle: + print("无法生成主车辆，退出程序") + return + ego_vehicle.set_autopilot(True) + + # 生成相机（挂载到主车辆） + camera_bp = bp_lib.find('sensor.camera.rgb') + camera_bp.set_attribute('image_size_x', '640') + camera_bp.set_attribute('image_size_y', '480') + camera_bp.set_attribute('fov', '90') + camera_transform = carla.Transform(carla.Location(x=1.5, z=2.0)) # 相机位置 + camera = world.spawn_actor(camera_bp, camera_transform, attach_to=ego_vehicle) + + # 图像队列 + image_queue = queue.Queue() + camera.listen(lambda img: camera_callback(img, image_queue)) + + # 生成NPC车辆 + for _ in range(args.num_npcs): + npc_bp = random.choice([bp for bp in bp_lib.filter('vehicle') + if int(bp.get_attribute('number_of_wheels')) == 4]) + npc = world.try_spawn_actor(npc_bp, random.choice(spawn_points)) + if npc: + npc.set_autopilot(True) + + # 初始化YOLOv5检测器和SORT跟踪器 + detector = YOLO("yolov5s.pt") # 自动下载轻量版模型 + tracker = SimpleSORT(max_age=2, min_hits=2, iou_threshold=0.3) + + # 主循环 + try: + while True: + world.tick() # 推进仿真 + + # 调整 spectator 视角到主车辆上方 + ego_transform = ego_vehicle.get_transform() + world.get_spectator().set_transform(carla.Transform( + ego_transform.location + carla.Location(x=-8, z=12), + carla.Rotation(pitch=-45, yaw=ego_transform.rotation.yaw) + )) + + # 处理相机图像 + if not image_queue.empty(): + image = image_queue.get() + height, width, _ = image.shape + + # YOLOv5目标检测（仅保留车辆类） + results = detector.predict(image, conf=0.4) # 置信度阈值0.4 + boxes, scores = [], [] + for result in results: + for box, cls, conf in zip(result.boxes.xyxy, result.boxes.cls, result.boxes.conf): + cls_id = int(cls) + # COCO数据集车辆类：2=car, 5=bus, 7=truck + if cls_id in [2, 5, 7]: + boxes.append(box.numpy()) # [x1, y1, x2, y2] + scores.append(conf.numpy()) + + # 转换为numpy数组 + boxes = np.array(boxes) if boxes else np.array([]) + scores = np.array(scores) if scores else np.array([]) + + # 目标跟踪 + tracked_boxes, track_ids = [], [] + if len(boxes) > 0: + # 跟踪器输入格式：[x1, y1, x2, y2, score] + detections = np.hstack((boxes, scores.reshape(-1, 1))) + tracked_boxes, track_ids = tracker.update(detections) + + # 绘制跟踪结果 + if len(tracked_boxes) > 0: + display_img = image.copy() + display_img = draw_bounding_boxes(display_img, tracked_boxes, track_ids) + cv2.imshow("CARLA Vehicle Tracking (YOLOv5 + SORT)", display_img) + + # 按'q'退出 + if cv2.waitKey(1) & 0xFF == ord('q'): + break + + except KeyboardInterrupt: + print("用户中断程序") + finally: + # 清理资源 + camera.stop() + clear_actors(world) + # 关闭同步模式 + settings.synchronous_mode = False + world.apply_settings(settings) + cv2.destroyAllWindows() + +if __name__ == "__main__": + main() \ No newline at end of file