init

.github/workflows/documentation.yaml

@@ -0,0 +1,49 @@
+# This is a basic workflow to help you get started with Actions
+
+name: Build-sphinx-docs
+
+# Controls when the workflow will run
+on:
+  # Triggers the workflow on push or pull request events but only for the main branch
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+# A workflow run is made up of one or more jobs that can run sequentially or in parallel
+jobs:
+  # This workflow contains a single job called "build"
+  build:
+    # The type of runner that the job will run on
+    runs-on: ubuntu-latest
+
+    # Steps represent a sequence of tasks that will be executed as part of the job
+    steps:
+      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
+      - uses: actions/checkout@v4
+
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v4
+        with:
+           python-version: "3.10"
+
+      - name: Install dependencies
+        run: | 
+          python -m pip install --upgrade pip
+          pip install sphinx sphinx_rtd_theme recommonmark ghp-import
+#          pip install torch numpy
+
+      # Building html requires all necessary dependencies be installed.
+      # For cosense3d, the installation requires a lot of time,
+      # therefore it is temporally built locally and copy the built html files to the gh-pages branch.
+
+#      - name: Build HTML
+#        run: |
+#          cd docs/
+#          make html
+      - name: Run ghp-import
+        run: | 
+          ghp-import -n -p -f docs/_build/html

Dockerfile

@@ -0,0 +1,43 @@
+FROM pytorch/pytorch:2.2.1-cuda11.8-cudnn8-devel
+LABEL hostname="cosense3d-docker"
+
+##############################################
+# You should modify this to match your GPU compute capability
+# ENV TORCH_CUDA_ARCH_LIST="6.0 6.1 7.0+PTX"
+ENV TORCH_CUDA_ARCH_LIST="6.0 6.1 6.2 7.0 7.2 7.5 8.0 8.6"
+##############################################
+
+ENV TORCH_NVCC_FLAGS="-Xfatbin -compress-all"
+ENV OMP_NUM_THREADS 16
+
+WORKDIR /project
+COPY requirements.txt /project/requirements.txt
+COPY ./cosense3d/ops/ /project/ops/
+
+# Install dependencies
+RUN apt-get update
+RUN apt-get install -y git ninja-build cmake build-essential libopenblas-dev \
+    xterm xauth openssh-server tmux wget mate-desktop-environment-core
+
+RUN apt-get clean
+RUN rm -rf /var/lib/apt/lists/*
+
+# For faster build, use more jobs.
+ENV MAX_JOBS=4
+RUN git clone --recursive "https://github.com/NVIDIA/MinkowskiEngine"
+RUN cd MinkowskiEngine; python setup.py install --force_cuda --blas=openblas
+RUN cd ..
+
+RUN conda update conda -y
+RUN apt-get update
+RUN apt install python3-dev  -y
+RUN apt install libgl1-mesa-glx libglib2.0-0 -y
+
+RUN cd ops && pip install . && cd ..
+
+RUN pip install -r requirements.txt
+
+#RUN conda install pybind11 -y
+#RUN conda install -c conda-forge libstdcxx-ng -y
+
+WORKDIR /workspace

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Yunshuang Yuan
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

cosense3d/__init__.py

@@ -0,0 +1 @@
+

cosense3d/agents/cav_prototype/RLseg.py

@@ -0,0 +1,98 @@
+
+
+import torch
+import torch_scatter
+from cosense3d.agents.utils.transform import DataOnlineProcessor as DOP
+from cosense3d.agents.cav_prototype.base_cav import BaseCAV
+
+
+class RLsegCAV(BaseCAV):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.dataset = kwargs.get('dataset', None)
+        self.lidar_range = torch.nn.Parameter(self.lidar_range)
+        self.prepare_data_keys = ['points', 'annos_local', 'roadline_tgts']
+        self.aug_transform = None
+        self.T_aug2g = None
+        self.T_g2aug = None
+        self.T_e2g = None
+        self.use_aug = True
+
+    def apply_transform(self):
+        if self.use_aug:
+            if self.is_ego:
+                T_e2g = self.lidar_pose
+                T_g2e = self.lidar_pose.inverse()
+                T_c2e = torch.eye(4).to(self.lidar_pose.device)
+            else:
+                # cav to ego
+                T_e2g = self.data['received_request']['lidar_pose']
+                T_g2e = self.data['received_request']['lidar_pose'].inverse()
+                T_c2e = T_g2e @ self.lidar_pose
+
+            if self.aug_transform is None:
+                self.aug_transform = DOP.update_transform_with_aug(
+                    torch.eye(4).to(self.lidar_pose.device), self.data['augment_params'])
+                T_e2aug = self.aug_transform
+            else:
+                # adapt aug params to the current ego frame
+                T_e2aug = self.T_g2aug @ T_e2g
+
+            T_c2aug = T_e2aug @ T_c2e
+            T_g2aug = T_e2aug @ T_g2e
+
+            DOP.apply_transform(self.data, T_c2aug, apply_to=self.prepare_data_keys)
+
+            self.T_e2g = T_e2g
+            self.T_g2aug = T_g2aug
+            self.T_aug2g = T_g2aug.inverse() # ego aug to global
+
+        else:
+            if self.is_ego:
+                transform = torch.eye(4).to(self.lidar_pose.device)
+            else:
+                # cav to ego
+                request = self.data['received_request']
+                transform = request['lidar_pose'].inverse() @ self.lidar_pose
+
+            T_c2aug = DOP.update_transform_with_aug(transform, self.data['augment_params'])
+            DOP.apply_transform(self.data, T_c2aug, apply_to=self.prepare_data_keys)
+            self.T_aug2g = T_c2aug
+
+    def prepare_data(self):
+        DOP.adaptive_free_space_augmentation(self.data, res=0.5, min_h=0)
+        DOP.generate_sparse_target_roadline_points(self.data)
+
+    def get_request_cpm(self):
+        return {'lidar_pose': self.lidar_pose}
+
+    def get_response_cpm(self):
+        cpm = {}
+        for k in ['bev_feat']:
+            if k in self.data:
+                cpm[k] = self.data[k]
+        return cpm
+
+    def forward_local(self, tasks, training_mode, **kwargs):
+        if (self.is_ego or self.require_grad) and training_mode:
+            grad_mode = 'with_grad'
+        else:
+            grad_mode = 'no_grad'
+        tasks[grad_mode].append((self.id, '11:pts_backbone', {}))
+        tasks[grad_mode].append((self.id, '12:backbone_neck', {}))
+
+    def forward_fusion(self, tasks, training_mode, **kwargs):
+        return tasks
+
+    def forward_head(self, tasks, training_mode, **kwargs):
+        if self.is_ego:
+            tasks['with_grad'].append((self.id, '21:rlseg_head', {}))
+        return tasks
+
+    def loss(self, tasks, **kwargs):
+        if self.is_ego:
+            tasks['loss'].append((self.id, '31:rlseg_head', {}))
+        return tasks
+
+
+

cosense3d/agents/cav_prototype/__init__.py

@@ -0,0 +1,15 @@
+# This module provides prototypes for CAVs/agents.
+# The prototype has the following features:
+# 1. Data processing logics for each prototyped agent/CAV.
+# 2. All intermediate data processed are stored locally at prototype class.
+# 3. Specify the requesting and responding CPMs
+
+import importlib
+
+
+def get_prototype(module_full_path: str):
+    module_name, cls_name = module_full_path.rsplit('.', 1)
+    module = importlib.import_module(f'cosense3d.agents.cav_prototype.{module_name}')
+    cls_obj = getattr(module, cls_name, None)
+    assert cls_obj is not None, f'Class \'{module_name}\' not found.'
+    return cls_obj